<commit_before>require 'nokogiri'
module DocParser
  # The Document class loads and parses the files.
  # @see Parser
  # @see Output
  class Document
    attr_reader :filename, :doc, :encoding, :results

    # @return [String] the source of the document
    attr_reader :html

    def initialize(filename: nil, encoding: 'utf-8', parser: nil)
      @logger = Log4r::Logger.new('docparser::document')
      @logger.debug { "Parsing #{filename}" }
      @encoding = encoding
      @parser = parser
      @filename = filename
      @results = Array.new(@parser.outputs ? @parser.outputs.length : 0) { [] }
      read_file
    end

    # Adds a row to an output
    def add_row(*row, output: 0)
      output = @parser.outputs.index(output) if output.is_a? Output
      @logger.debug { "#{filename}: Adding row #{row.flatten.to_s}" }
      results[output] << row.flatten
    end

    # Extracts the document title
    # @return [String] the title of the document
    def title
      @title ||= xpath_content('//head/title')<commit_msg>Updated documentation<commit_after>require 'nokogiri'
module DocParser
  # The Document class loads and parses the files.
  # @see Parser
  # @see Output
  class Document
    # @return [String] the filename of the current document
    attr_reader :filename

    # @return [Nokogiri::HTML::Document] a reference to the Nokogiri document
    attr_reader :doc

    # @return [String] the encoding of the document
    attr_reader :encoding

    # @return [Array] the results from this document
    attr_reader :results

    # @return [String] the source of the document
    attr_reader :html

    def initialize(filename: nil, encoding: 'utf-8', parser: nil)
      @logger = Log4r::Logger.new('docparser::document')
      @logger.debug { "Parsing #{filename}" }
      @encoding = encoding
      @parser = parser
      @filename = filename
      @results = Array.new(@parser.outputs ? @parser.outputs.length : 0) { [] }
      read_file
    end

    # Adds a row to an output
    def add_row(*row, output: 0)
      output = @parser.outputs.index(output) if output.is_a? Output
      @logger.debug { "#{filename}: Adding row #{row.flatten.to_s}" }
      results[output] << row.flatten
    end

    # Extracts the document title
    # @return [String] the title of the document
    def title
      @title ||= xpath_content('//head/title')
{'fraction_non_alphanumeric': 0.11308767471410419, 'fraction_numerical': 0.0033883947479881405, 'mean_word_length': 3.0445205479452055, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6057300', 'n_tokens_mistral': 719, 'n_tokens_neox': 699, 'n_words': 221}