<commit_before># -*- coding: utf-8 -*-

require 'spec_helper'
require 'stringio'
require 'tempfile'
require 'zlib'

describe FilterIO do
  def matches_reference_io_behaviour(input)
    expected_io = StringIO.new(input)
    actual_io = FilterIO.new(StringIO.new(input))

    results = [expected_io, actual_io].map do |io|
      results = []
      errors = []
      positions = []

      # call the block repeatedly until we get to EOF
      # and once more at the end to check what happens at EOF
      one_more_time = [true]
      while !io.eof? || one_more_time.pop
        begin
          results << yield(io)
          errors << nil
        rescue Exception => e
          results << nil
          errors << [e.class, e.message]
        end
        positions << io.pos
        raise 'Too many iterations' if results.size > 100
      end

      [results, errors, positions]
    end

    # compare the filtered output against the reference
    results[0].zip(results[1]).each do |expected, actual|
      expect(actual).to eq expected
    end
  end

  it 'works with an empty source' do
    io = FilterIO.new(StringIO.new(''))
    expect(io.bof?).to be_true
    io = FilterIO.new(StringIO.new(''))
    expect(io.eof?).to be_true
    io = FilterIO.new(StringIO.new(''))
    expect {
      io.readchar
    }.to raise_error EOFError
  end

  it 'supports `eof?`' do
    io = FilterIO.new(StringIO.new('x'))
    expect(io.eof?).to be_false
    expect(io.readchar.chr).to eq 'x'
    expect(io.eof?).to be_true
    expect(io.read).to eq ''
    expect(io.read(8)).to eq nil<commit_msg>Ensure output encoding matches reference behaviour<commit_after># -*- coding: utf-8 -*-

require 'spec_helper'
require 'stringio'
require 'tempfile'
require 'zlib'

describe FilterIO do
  def matches_reference_io_behaviour(input)
    expected_io = StringIO.new(input)
    actual_io = FilterIO.new(StringIO.new(input))

    results = [expected_io, actual_io].map do |io|
      results = []
      errors = []
      positions = []

      # call the block repeatedly until we get to EOF
      # and once more at the end to check what happens at EOF
      one_more_time = [true]
      while !io.eof? || one_more_time.pop
        begin
          results << yield(io)
          errors << nil
        rescue Exception => e
          results << nil
          errors << [e.class, e.message]
        end
        positions << io.pos
        raise 'Too many iterations' if results.size > 100
      end

      [results, errors, positions]
    end

    # compare the filtered output against the reference
    results[0].zip(results[1]).each do |expected, actual|
      expect(actual).to eq expected
      if actual.respond_to? :encoding
        expect(actual.encoding).to eq expected.encoding
      end
    end
  end

  it 'works with an empty source' do
    io = FilterIO.new(StringIO.new(''))
    expect(io.bof?).to be_true
    io = FilterIO.new(StringIO.new(''))
    expect(io.eof?).to be_true
    io = FilterIO.new(StringIO.new(''))
    expect {
      io.readchar
    }.to raise_error EOFError
  end

  it 'supports `eof?`' do
    io = FilterIO.new(StringIO.new('x'))
    expect(io.eof?).to be_false
    expect(io.readchar.chr).to eq 'x'
    expect(io.eof?).to be_true
    expect(io.read).to eq ''
    expect(io.read(8)).to eq nil
{'fraction_non_alphanumeric': 0.11707612981498332, 'fraction_numerical': 0.004246284501061571, 'mean_word_length': 2.48994708994709, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7112879', 'n_tokens_mistral': 1080, 'n_tokens_neox': 1048, 'n_words': 289}