<commit_before>module Docks
  class Parser

    def self.current_file
      @@current_file
    end

    # Public: Processes each file in a group of files. Each file is individually
    # run through `.parse_file`, and the results are grouped based on the file
    # type of each (markup, script, or style parse results). The result also
    # includes a list of all files in the group and the name of the file group.
    #
    # file_group - An Array of file names as Strings.
    #
    # Returns the parsed file group as a Hash.

    def self.parse_group(file_group)
      parsed_file_group = {
        files: file_group,
        name: Docks::Group.group_identifier(file_group.first)
      }

      Docks::Cache::PARSE_RESULT_TYPES.each do |parse_result_type|
        parsed_file_group[parse_result_type] = []
      end

      file_group.each do |file|
        # Do not process a file that is either a) does not have a supported extension
        # or b) does not exist
        next unless File.exists?(file)
        next unless Docks::Language.is_supported_file_type?(file)
        next unless is_parseable_file?(file)
        parsed_file_group[Docks::Language.file_type(file)].concat(parse_file(file))
      end

      fix_pattern_block(parsed_file_group)
      parsed_file_group
    end



    @@parsers = []

    def self.register(parser, options)
      return if !options[:extensions] && !options[:matches]
      options[:matches] = /\.(#{[options[:extensions]].flatten.join("|")})$/ if options[:extensions]
      @@parsers << { parser: parser, matcher: options[:matches] }
    end

    def self.is_parseable_file?(file)
      !parser_for(file).nil?
    end

    def self.parse_comment_block(comment_block, language, post_process = false)
      pseudo_file = "foo.#{language.to_s}"
      @@current_file = pseudo_file
      setup_current_details

      return {} if Docks.current_parser.nil?
      result = Docks.current_parser.parse_comment_block(comment_block)
      result = Docks::Tag.join_synonymous_tags(result)
      result = Docks::Process.process(result)
      result = Docks::Process.post_process([results]).first if post_process

      @@current_file = nil
      teardown_current_details

      result
    end

    private

    def self.fix_pattern_block(parsed_file)
      pattern_result = Hash.new
      Docks::Cache::PARSE_RESULT_TYPES.each do |parse_result_type|
        new_pattern_result, parsed_file[parse_result_type] = parsed_file[parse_result_type].partition do |parse_result|
          parse_result[:symbol_type] == Docks::Types::Symbol::PAGE
        end

        pattern_result = new_pattern_result.first || pattern_result
      end

      parsed_file[:pattern] = pattern_result
    end

    def self.parser_for(file)
      @@parsers.reverse_each do |parser_details|
        return parser_details[:parser].instance if parser_details[:matcher] =~ file
      end

      nil
    end



    # Private: Parses and processes a single file. First, the file is read
    # and line endings normalized. The file is then parsed by the parser for
    # that file extension, then has all synonymous tags joined together,
    # then has all of the tag-specific processors run on it. Finally,
    # the required post processors are run on the full parse result and
    # it is returned.
    #
    # file - The file name as a String.
    #
    # Returns the parsed file group as an Array of Hashes.

    def self.parse_file(file)
      @@current_file = file
      setup_current_details

      return [] if Docks.current_parser.nil?

      puts "Parsing #{file} with #{Docks.current_parser}"
      parse_results = Docks.current_parser.parse(File.read(file).gsub(/\r\n?/, "\n"))

      parse_results.map! do |parse_result|
        parse_result = Docks::Tag.join_synonymous_tags(parse_result)
        Docks::Process.process(parse_result)<commit_msg>Killed this stupid logging line.<commit_after>module Docks
  class Parser

    def self.current_file
      @@current_file
    end

    # Public: Processes each file in a group of files. Each file is individually
    # run through `.parse_file`, and the results are grouped based on the file
    # type of each (markup, script, or style parse results). The result also
    # includes a list of all files in the group and the name of the file group.
    #
    # file_group - An Array of file names as Strings.
    #
    # Returns the parsed file group as a Hash.

    def self.parse_group(file_group)
      parsed_file_group = {
        files: file_group,
        name: Docks::Group.group_identifier(file_group.first)
      }

      Docks::Cache::PARSE_RESULT_TYPES.each do |parse_result_type|
        parsed_file_group[parse_result_type] = []
      end

      file_group.each do |file|
        # Do not process a file that is either a) does not have a supported extension
        # or b) does not exist
        next unless File.exists?(file)
        next unless Docks::Language.is_supported_file_type?(file)
        next unless is_parseable_file?(file)
        parsed_file_group[Docks::Language.file_type(file)].concat(parse_file(file))
      end

      fix_pattern_block(parsed_file_group)
      parsed_file_group
    end



    @@parsers = []

    def self.register(parser, options)
      return if !options[:extensions] && !options[:matches]
      options[:matches] = /\.(#{[options[:extensions]].flatten.join("|")})$/ if options[:extensions]
      @@parsers << { parser: parser, matcher: options[:matches] }
    end

    def self.is_parseable_file?(file)
      !parser_for(file).nil?
    end

    def self.parse_comment_block(comment_block, language, post_process = false)
      pseudo_file = "foo.#{language.to_s}"
      @@current_file = pseudo_file
      setup_current_details

      return {} if Docks.current_parser.nil?
      result = Docks.current_parser.parse_comment_block(comment_block)
      result = Docks::Tag.join_synonymous_tags(result)
      result = Docks::Process.process(result)
      result = Docks::Process.post_process([results]).first if post_process

      @@current_file = nil
      teardown_current_details

      result
    end

    private

    def self.fix_pattern_block(parsed_file)
      pattern_result = Hash.new
      Docks::Cache::PARSE_RESULT_TYPES.each do |parse_result_type|
        new_pattern_result, parsed_file[parse_result_type] = parsed_file[parse_result_type].partition do |parse_result|
          parse_result[:symbol_type] == Docks::Types::Symbol::PAGE
        end

        pattern_result = new_pattern_result.first || pattern_result
      end

      parsed_file[:pattern] = pattern_result
    end

    def self.parser_for(file)
      @@parsers.reverse_each do |parser_details|
        return parser_details[:parser].instance if parser_details[:matcher] =~ file
      end

      nil
    end



    # Private: Parses and processes a single file. First, the file is read
    # and line endings normalized. The file is then parsed by the parser for
    # that file extension, then has all synonymous tags joined together,
    # then has all of the tag-specific processors run on it. Finally,
    # the required post processors are run on the full parse result and
    # it is returned.
    #
    # file - The file name as a String.
    #
    # Returns the parsed file group as an Array of Hashes.

    def self.parse_file(file)
      @@current_file = file
      setup_current_details

      return [] if Docks.current_parser.nil?
      parse_results = Docks.current_parser.parse(File.read(file).gsub(/\r\n?/, "\n"))

      parse_results.map! do |parse_result|
        parse_result = Docks::Tag.join_synonymous_tags(parse_result)
        Docks::Process.process(parse_result)
{'fraction_non_alphanumeric': 0.08884835852006254, 'fraction_numerical': 0.0, 'mean_word_length': 3.02992125984252, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7515212', 'n_tokens_mistral': 2307, 'n_tokens_neox': 2253, 'n_words': 622}