<commit_before># encoding: UTF-8
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper.rb')
require 'stringio'
describe "Chunked parser" do
  before(:all) do
    @final = [{"abc" => 123}, {"def" => 456}]
  end
  
  before(:each) do
    @callback = lambda { |hash| 
      puts hash.inspect
      hash.should == @final
    }
    Yajl::Chunked.on_parse_complete = @callback
  end
  
  it "should parse a single chunk" do
    @callback.should_receive(:call)
    Yajl::Chunked << '[{"abc": 123},{"def": 456}]'
  end
  
  it "should parse a single chunk, 3 times" do
    @callback.should_receive(:call).exactly(3).times
    Yajl::Chunked << '[{"abc": 123},{"def": 456}]'
    Yajl::Chunked << '[{"abc": 123},{"def": 456}]'
    Yajl::Chunked << '[{"abc": 123},{"def": 456}]'
  end
  
  it "should parse in two chunks" do
    @callback.should_receive(:call)
    Yajl::Chunked << '[{"abc": 123},'
    Yajl::Chunked << '{"def": 456}]'
  end
  
  it "should parse in 2 chunks, twice" do
    @callback.should_receive(:call).exactly(2).times
    Yajl::Chunked << '[{"abc": 123},'
    Yajl::Chunked << '{"def": 456}]'
    Yajl::Chunked << '[{"abc": 123},'
    Yajl::Chunked << '{"def": 456}]'
  end
  
  it "should parse 2 JSON strings, in 3 chunks" do
    @callback.should_receive(:call).exactly(2).times
    Yajl::Chunked << '[{"abc": 123},'
    Yajl::Chunked << '{"def": 456}][{"abc": 123},{"def":'
    Yajl::Chunked << ' 456}]'
  end
  
  it "should parse 2 JSON strings in 1 chunk" do
    pending
    @callback.should_receive(:call).exactly(2).times
    Yajl::Chunked << '[{"abc": 123},{"def": 456}][{"abc": 123},{"def": 456}]'
  end
  
  it "should parse an io" do
    @callback.should_receive(:call)
    Yajl::Stream.parse(StringIO.new('[{"abc": 123},{"def": 456}]'))
  end
end<commit_msg>adding another pending test for multiple JSON strings in the same pass/from an IO<commit_after># encoding: UTF-8
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper.rb')
require 'stringio'
describe "Chunked parser" do
  before(:all) do
    @final = [{"abc" => 123}, {"def" => 456}]
  end
  
  before(:each) do
    @callback = lambda { |hash| 
      puts hash.inspect
      hash.should == @final
    }
    Yajl::Chunked.on_parse_complete = @callback
  end
  
  it "should parse a single chunk" do
    @callback.should_receive(:call)
    Yajl::Chunked << '[{"abc": 123},{"def": 456}]'
  end
  
  it "should parse a single chunk, 3 times" do
    @callback.should_receive(:call).exactly(3).times
    Yajl::Chunked << '[{"abc": 123},{"def": 456}]'
    Yajl::Chunked << '[{"abc": 123},{"def": 456}]'
    Yajl::Chunked << '[{"abc": 123},{"def": 456}]'
  end
  
  it "should parse in two chunks" do
    @callback.should_receive(:call)
    Yajl::Chunked << '[{"abc": 123},'
    Yajl::Chunked << '{"def": 456}]'
  end
  
  it "should parse in 2 chunks, twice" do
    @callback.should_receive(:call).exactly(2).times
    Yajl::Chunked << '[{"abc": 123},'
    Yajl::Chunked << '{"def": 456}]'
    Yajl::Chunked << '[{"abc": 123},'
    Yajl::Chunked << '{"def": 456}]'
  end
  
  it "should parse 2 JSON strings, in 3 chunks" do
    @callback.should_receive(:call).exactly(2).times
    Yajl::Chunked << '[{"abc": 123},'
    Yajl::Chunked << '{"def": 456}][{"abc": 123},{"def":'
    Yajl::Chunked << ' 456}]'
  end
  
  it "should parse 2 JSON strings in 1 chunk" do
    pending
    @callback.should_receive(:call).exactly(2).times
    Yajl::Chunked << '[{"abc": 123},{"def": 456}][{"abc": 123},{"def": 456}]'
  end
  
  it "should parse 2 JSON strings from an IO" do
    pending
    @callback.should_receive(:call).exactly(2).times
    Yajl::Stream.parse(StringIO.new('[{"abc": 123},{"def": 456}][{"abc": 123},{"def": 456}]'))
  end
  
  it "should parse an IO" do
    @callback.should_receive(:call)
    Yajl::Stream.parse(StringIO.new('[{"abc": 123},{"def": 456}]'))
  end
end
{'fraction_non_alphanumeric': 0.20908621579762518, 'fraction_numerical': 0.0495611770779556, 'mean_word_length': 3.6407185628742513, 'pattern_counts': {'":': 52, '<': 59, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6512413', 'n_tokens_mistral': 1611, 'n_tokens_neox': 1410, 'n_words': 344}