<commit_before>require 'spec_helper'

RSpec.describe CSV2Avro do
  describe '#convert' do
    let(:options) { { schema: './spec/support/schema.avsc' } }

    before do
      ARGV.replace ['./spec/support/data.csv']
    end
    subject(:converter) { CSV2Avro.new(options) }

    it 'should write errors to STDERR' do
      expect { converter.convert }.to output("line 4: Missing value at name\nline 5: Unable to parse\n").to_stderr
    end

    it 'should have a bad row' do
      File.open('./spec/support/data.bad.csv', 'r') do |file|
        expect(file.read).to eq("id,name,description\n3,,Bras\n")
      end
    end

    it 'should contain the avro data' do
      File.open('./spec/support/data.avro', 'r') do |file|
        expect(AvroReader.new(file).read).to eq(
          [
            { 'id'=>1, 'name'=>'dresses',     'description'=>'Dresses' },
            { 'id'=>2, 'name'=>'female-tops', 'description'=>nil }
          ]
        )
      end
    end
  end
end
<commit_msg>Update tests to new test data<commit_after>require 'spec_helper'

RSpec.describe CSV2Avro do
  describe '#convert' do
    let(:options) { { schema: './spec/support/schema.avsc' } }

    before do
      ARGV.replace ['./spec/support/data.csv']
    end
    subject(:converter) { CSV2Avro.new(options) }

    it 'should write errors to STDERR' do
      expect { converter.convert }.to output("line 4: Missing value at name\nline 7: Unable to parse\n").to_stderr
    end

    it 'should have a bad row' do
      File.open('./spec/support/data.bad.csv', 'r') do |file|
        expect(file.read).to eq("id,name,description\n3,,Bras\n")
      end
    end

    it 'should contain the avro data' do
      File.open('./spec/support/data.avro', 'r') do |file|
        expect(AvroReader.new(file).read).to eq(
          [
            { 'id'=>1, 'name'=>'dresses',     'description'=>'Dresses' },
            { 'id'=>2, 'name'=>'female-tops', 'description'=>nil },
            { 'id'=>4, 'name'=>'male-tops',   'description'=>"Male Tops\nand Male Shirts"},
            { 'id'=>6, 'name'=>'male-shoes', 'description'=>'Male Shoes'}
          ]
        )
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.1667440780306549, 'fraction_numerical': 0.0074314909428704135, 'mean_word_length': 2.468599033816425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '197581', 'n_tokens_mistral': 744, 'n_tokens_neox': 721, 'n_words': 169}