<commit_before>require_relative '../../test_helper'
describe DocParser::Document do
  before do
    Log4r::Logger['docparser'].level = Log4r::INFO
    $output = DocParser::NilOutput.new
    @parser = Class.new do
      define_method(:outputs) { [$output] }
    end.new
    @test_doc_path = File.join($SUPPORT_DIR, 'test_html.html')
    @test_doc = DocParser::Document.new(filename: @test_doc_path,
                                        parser: @parser)
  end

  it 'should read HTML contents' do
    file = File.join($SUPPORT_DIR, 'test_html.html')
    doc = DocParser::Document.new(filename: file, parser: @parser)
    doc.doc.must_be_instance_of Nokogiri::HTML::Document
    doc.html.must_equal(open(file).read)
  end

  it 'should read XML contents' do
    file = File.join($SUPPORT_DIR, 'test_xml.xml')
    doc = DocParser::Document.new(filename: file, parser: @parser)
    doc.doc.must_be_instance_of Nokogiri::XML::Document
    doc.html.must_equal(open(file).read)
    doc.xpath_content('xmltest > title').must_equal('Test XML')
    doc.xpath_content('xmltest > test').must_equal('Character Data')
  end

  it 'should read remote contents' do
    url = 'https://gist.github.com/jurriaan/3f2750aa546e3e6719cf/raw'
    doc = DocParser::Document.new(filename: url, parser: @parser)
    doc.html.must_equal(open(url).read)
  end

  it 'should use the correct encoding' do
    file = File.join($SUPPORT_DIR, 'test_encoding.html')
    file2 = File.join($SUPPORT_DIR, 'test_encoding2.html')
    doc = DocParser::Document.new(filename: file, parser: @parser)
    doc2 = DocParser::Document.new(filename: file2,
                                   parser: @parser,
                                   encoding: 'iso-8859-1')
    doc.html.must_equal(doc2.html)
    doc.css_content('#encoding').must_equal(doc2.css_content('#encoding'))
  end

  it 'should specify filename and encoding in #inspect' do
    @test_doc.inspect.must_include(@test_doc.filename)
    @test_doc.inspect.must_include(@test_doc.encoding)
  end

  it 'should get the title of a document' do
    @test_doc.title.must_equal('Test HTML')
  end

  it 'should store the path to the document' do
    @test_doc.filename.must_equal(@test_doc_path)
  end

  it 'should be possible to use css queries' do
    css = 'article > h1 + p'
    css_content = @test_doc.css_content(css)
    css_element = @test_doc.css(css)
    css_content.must_equal('Great article it is')
    css_content.must_equal(css_element.first.content)
  end

  it 'should be possible to use xpath queries' do
    xpath = '//li/ancestor::article/h1'
    xpath_content = @test_doc.xpath_content(xpath)
    xpath_element = @test_doc.xpath(xpath)
    xpath_content.must_equal('This is an article')
    xpath_content.must_equal(xpath_element.first.content)
  end

  it 'should be possible to use regular expressions' do
    regex = @test_doc.regexp(/\<h1\>([^\<])*/)
    regex.must_equal(@test_doc.html.match(/\<h1\>([^\<])*/))
  end

  it 'should be possible to use blocks on query methods' do
    array = []
    @test_doc.css('p') do |element|
      array << element.content
    end
    array.last.must_equal('This is the last paragraph')
    array2 = []
    @test_doc.xpath('//p') do |element|
      array2 << element.content
    end
    array2.must_equal(array)
  end

  it 'should warn when providing an empty file' do
    file = Tempfile.new('empty')
    file.write('')
    file.close

    open(file.path).read.empty?.must_equal true
    _, err = capture_io do
      # Switch to hijacked IO
      Log4r::Outputter['docparser'].instance_variable_set(:@out, $stderr)
      DocParser::Document.new(filename: file.path, parser: @parser)
    end
    # Restore IO
    Log4r::Outputter['docparser'].instance_variable_set(:@out, $stderr)
    err.must_include "#{file.path} is empty"
  end

  it 'should add the row to the results' do
    @test_doc.add_row ['test']
    @test_doc.add_row 'test', 'test2'
    @test_doc.results.must_equal [[['test'], ['test', 'test2']]]
  end

  it 'should be possible to specify outputs directly' do
    @test_doc.add_row ['test!'], output: $output
    @test_doc.results.must_equal [[['test!']]]
  end

  it 'should be possible to use multiple outputs' do<commit_msg>Added a document test case<commit_after>require_relative '../../test_helper'
describe DocParser::Document do
  before do
    Log4r::Logger['docparser'].level = Log4r::INFO
    $output = DocParser::NilOutput.new
    @parser = Class.new do
      define_method(:outputs) { [$output] }
    end.new
    @test_doc_path = File.join($SUPPORT_DIR, 'test_html.html')
    @test_doc = DocParser::Document.new(filename: @test_doc_path,
                                        parser: @parser)
  end

  it 'should read HTML contents' do
    file = File.join($SUPPORT_DIR, 'test_html.html')
    doc = DocParser::Document.new(filename: file, parser: @parser)
    doc.doc.must_be_instance_of Nokogiri::HTML::Document
    doc.html.must_equal(open(file).read)
  end

  it 'should read XML contents' do
    file = File.join($SUPPORT_DIR, 'test_xml.xml')
    doc = DocParser::Document.new(filename: file, parser: @parser)
    doc.doc.must_be_instance_of Nokogiri::XML::Document
    doc.html.must_equal(open(file).read)
    doc.xpath_content('xmltest > title').must_equal('Test XML')
    doc.xpath_content('xmltest > test').must_equal('Character Data')
  end

  it 'should read remote contents' do
    url = 'https://gist.github.com/jurriaan/3f2750aa546e3e6719cf/raw'
    doc = DocParser::Document.new(filename: url, parser: @parser)
    doc.html.must_equal(open(url).read)
  end

  it 'should use the correct encoding' do
    file = File.join($SUPPORT_DIR, 'test_encoding.html')
    file2 = File.join($SUPPORT_DIR, 'test_encoding2.html')
    doc = DocParser::Document.new(filename: file, parser: @parser)
    doc2 = DocParser::Document.new(filename: file2,
                                   parser: @parser,
                                   encoding: 'iso-8859-1')
    doc.html.must_equal(doc2.html)
    doc.css_content('#encoding').must_equal(doc2.css_content('#encoding'))
  end

  it 'should specify filename and encoding in #inspect' do
    @test_doc.inspect.must_include(@test_doc.filename)
    @test_doc.inspect.must_include(@test_doc.encoding)
  end

  it 'should get the title of a document' do
    @test_doc.title.must_equal('Test HTML')
  end

  it 'should store the path to the document' do
    @test_doc.filename.must_equal(@test_doc_path)
  end

  it 'should be possible to use css queries' do
    css = 'article > h1 + p'
    css_content = @test_doc.css_content(css)
    css_element = @test_doc.css(css)
    css_content.must_equal('Great article it is')
    css_content.must_equal(css_element.first.content)
  end

  it 'should be possible to use xpath queries' do
    xpath = '//li/ancestor::article/h1'
    xpath_content = @test_doc.xpath_content(xpath)
    xpath_element = @test_doc.xpath(xpath)
    xpath_content.must_equal('This is an article')
    xpath_content.must_equal(xpath_element.first.content)
  end

  it 'should be possible to use regular expressions' do
    regex = @test_doc.regexp(/\<h1\>([^\<])*/)
    regex.must_equal(@test_doc.html.match(/\<h1\>([^\<])*/))
  end

  it 'should be possible to use blocks on query methods' do
    array = []
    @test_doc.css('p') do |element|
      array << element.content
    end
    array.last.must_equal('This is the last paragraph')
    array2 = []
    @test_doc.xpath('//p') do |element|
      array2 << element.content
    end
    array2.must_equal(array)
  end

  it 'should warn when providing an empty file' do
    file = Tempfile.new('empty')
    file.write('')
    file.close

    open(file.path).read.empty?.must_equal true
    _, err = capture_io do
      # Switch to hijacked IO
      Log4r::Outputter['docparser'].instance_variable_set(:@out, $stderr)
      DocParser::Document.new(filename: file.path, parser: @parser)
    end
    # Restore IO
    Log4r::Outputter['docparser'].instance_variable_set(:@out, $stderr)
    err.must_include "#{file.path} is empty"
  end

  it 'should add the row to the results' do
    @test_doc.add_row ['test']
    @test_doc.add_row 'test', 'test2'
    @test_doc.results.must_equal [[['test'], ['test', 'test2']]]
  end

  it 'should be possible to not use outputs' do
    parser = Class.new do
      define_method(:outputs) { [] }
    end.new
    test_doc = DocParser::Document.new(filename: @test_doc_path,
                                        parser: parser)
    test_doc.html.must_include('Test HTML')
  end

  it 'should be possible to specify outputs directly' do
    @test_doc.add_row ['test!'], output: $output
    @test_doc.results.must_equal [[['test!']]]
  end

  it 'should be possible to use multiple outputs' do
{'fraction_non_alphanumeric': 0.12943059684427166, 'fraction_numerical': 0.00846101074777041, 'mean_word_length': 3.3301980198019803, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1768545', 'n_tokens_mistral': 2938, 'n_tokens_neox': 2876, 'n_words': 659}