<commit_before>require "spec_helper"

describe Runivedo::Stream do
  describe "sending" do
    let(:message) { Runivedo::Stream::Message.new }

    it "sends null" do
      message << nil
      message.buffer.should == "\x00"
    end

    it "sends bools" do
      message << true
      message << false
      message.buffer.should == "\x01\x02"
    end

    it "sends integers" do
      message << 42
      message.buffer.should == "\x12\x2A" + "\x00" * 7
    end

    it "sends floats" do
      message << 8.0e-323
      message.buffer.should == "\x15\x10" + "\x00" * 7
    end

    it "sends strings" do
      message << "foobar"
      message.buffer.should == "\x1e\x06\x00\x00\x00foobar"
    end

    it "sends symbols as strings" do
      message << :foobar
      message.buffer.should == "\x1e\x06\x00\x00\x00foobar"
    end

    it "sends arrays" do
      message << %w(foo bar)
      message.buffer.should == "\x3c\x02\x00\x00\x00\x1e\x03\x00\x00\x00foo\x1e\x03\x00\x00\x00bar"
    end

    it "sends maps" do
      message << {"foo" => true, "bar" => false}
      message.buffer.should == "\x3d\x02\x00\x00\x00\x1e\x03\x00\x00\x00foo\x01\x1e\x03\x00\x00\x00bar\x02"
    end

    it 'sends uuids' do
      uuid = UUIDTools::UUID.random_create
      message << uuid
      message.buffer.should == "\x2a" + uuid.raw
    end
  end

  describe "receiving" do
    it "receives null" do
      message = Runivedo::Stream::Message.new("\x00")
      message.read.should == nil
    end

    it "receives bools" do
      message = Runivedo::Stream::Message.new("\x01")
      message.read.should == true
      message = Runivedo::Stream::Message.new("\x02")
      message.read.should == false
    end

    it "receives integers" do
      {10 => 1, 11 => 2, 12 => 4, 13 => 8, 15 => 1, 16 => 2, 17 => 4, 18 => 8}.each_pair do |i, s|
        message = Runivedo::Stream::Message.new([i].pack("C") + "\x2A" + "\x00" * (s-1))
        message.read.should == 42
      end
    end

    it "receives floats" do
      message = Runivedo::Stream::Message.new("\x14\x10" + "\x00" * 3)
      message.read.should == 2.2420775429197073e-44
      message = Runivedo::Stream::Message.new("\x15\x10" + "\x00" * 7)
      message.read.should == 8.0e-323
    end

    it 'receives decimals' do
      message = Runivedo::Stream::Message.new(
        "\x05" "\x2A" "\x01" +
        "\x06" "\x2A\x00" "\x01" +
        "\x07" "\x2A\x00\x00\x00" "\x01" +
        "\x08" "\x2A\x00\x00\x00\x00\x00\x00\x00" "\x01"
        )
      4.times {message.read.should == BigDecimal.new("4.2")}
    end

    it "receives strings" do
      message = Runivedo::Stream::Message.new("\x1e\x06\x00\x00\x00foobar")
      message.read.should == "foobar"
      message = Runivedo::Stream::Message.new("\x1f\x06\x00\x00\x00f\x00o\x00o\x00b\x00a\x00r\x00")
      message.read.should == "foobar"
    end

    it "receives arrays" do
      message = Runivedo::Stream::Message.new("\x3c\x02\x00\x00\x00\x1e\x03\x00\x00\x00foo\x1e\x03\x00\x00\x00bar")
      message.read.should == %w(foo bar)
    end

    it "receives maps" do
      message = Runivedo::Stream::Message.new("\x3d\x02\x00\x00\x00\x1e\x03\x00\x00\x00foo\x0a\x01\x1e\x03\x00\x00\x00bar\x0a\x02")
      message.read.should == {"foo" => 1, "bar" => 2}
    end

    it 'receives ids' do
      message = Runivedo::Stream::Message.new("\x29\x02\x00\x00\x00\x00\x00\x00\x00")
      message.read.should == 2
    end

    it 'receives datetimes' do
      message = Runivedo::Stream::Message.new("\x33\x40\x7F\xFF\x0D\x8B\xDA\x04\x00")
      message.read.should == Time.at(1366190677)
    end

    it 'receives times' do
      message = Runivedo::Stream::Message.new("\x34\x40\x7F\xFF\x0D\x8B\xDA\x04\x00")
      message.read.should == Time.at(1366190677)
    end

    it 'receives uuids' do
      uuid = UUIDTools::UUID.random_create
      message = Runivedo::Stream::Message.new("\x2a" + uuid.raw)
      message.read.should == uuid
    end
  end

  describe "sending and receiving" do
    let(:message) { Runivedo::Stream::Message.new }
    let(:message_recv) { Runivedo::Stream::Message.new(message.instance_variable_get(:@buffer)) }

    it "works for null" do
      message << nil
      message_recv.read.should == nil
    end

    it "works for true" do
      message << true
      message_recv.read.should == true
    end

    it "works for false" do
      message << false
      message_recv.read.should == false
    end

    it "works for ints" do
      message << 42
      message_recv.read.should == 42
    end

    it "works for floats" do
      message << 42.23
      message_recv.read.should == 42.23
    end

    it "works for strings" do
      message << "foobar"
      message_recv.read.should == "foobar"
    end

    it "works for arrays" do
      message << %w(foo bar)
      message_recv.read.should == %w(foo bar)
    end

    it "works for maps" do
      message << {"foo" => 1, "foo" => 2}
      message_recv.read.should == {"foo" => 1, "foo" => 2}
    end

    it "works for uuids" do
      uuid = UUIDTools::UUID.random_create
      message << uuid
      message_recv.read.should == uuid
    end
  end

  describe Runivedo::VariantStream do
    it "receives" do
      s = Runivedo::VariantStream.new(StringIO.new("\x0D\x2A\x00\x00\x00\x00\x00\x00\x00\x1e\x06\x00\x00\x00foobar"))
      s.read.should == 42
      s.read.should == "foobar"
      s.has_data?.should be_false
    end<commit_msg>adapted specs to rfc7049<commit_after>require "spec_helper"

describe Runivedo::Stream do
  describe "sending" do
    let(:message) { Runivedo::Stream::Message.new }

    it "sends null" do
      message << nil
      # message.buffer.should == "\x00"
      message.buffer.should == "\xf6"
    end

    it "sends bools" do
      message << true
      message << false
      # message.buffer.should == "\x01\x02"
      message.buffer.should == "\xf5\xf4"
    end

    it "sends integers" do
      message << 42
      # message.buffer.should == "\x12\x2A" + "\x00" * 7
      message.buffer.should == "\x18\x2a"
    end

    it "sends floats" do
      message << 8.0e-323
      # message.buffer.should == "\x15\x10" + "\x00" * 7
      message.buffer.should == "\xfb\x10" + "\x00" * 7
    end

    it "sends strings" do
      message << "foobar"
      # message.buffer.should == "\x1e\x06\x00\x00\x00foobar"
      message.buffer.should == "\x66foobar"
    end

    it "sends symbols as strings" do
      message << :foobar
      # message.buffer.should == "\x1e\x06\x00\x00\x00foobar"
      message.buffer.should == "\x66foobar"
    end

    it "sends arrays" do
      message << %w(foo bar)
      # message.buffer.should == "\x3c\x02\x00\x00\x00\x1e\x03\x00\x00\x00foo\x1e\x03\x00\x00\x00bar"
      message.buffer.should == "\x82\x63foo\x63bar"
    end

    it "sends maps" do
      # message << {"foo" => true, "bar" => false}
      # message.buffer.should == "\x3d\x02\x00\x00\x00\x1e\x03\x00\x00\x00foo\x01\x1e\x03\x00\x00\x00bar\x02"
      message << {"bar" => false, "foo" => true}
      message.buffer.should == "\xa2bar\x72\xf4foo\xf5"
    end

    it 'sends uuids' do
      uuid = UUIDTools::UUID.random_create
      message << uuid
      # message.buffer.should == "\x2a" + uuid.raw
      message.buffer.should == "\xc8\x50" + uuid.raw
    end
  end

  describe "receiving" do
    it "receives null" do
      # message = Runivedo::Stream::Message.new("\x00")
      message = Runivedo::Stream::Message.new("\xf6")
      message.read.should == nil
    end

    it "receives bools" do
      # message = Runivedo::Stream::Message.new("\x01")
      message = Runivedo::Stream::Message.new("\xf5")
      message.read.should == true
      #message = Runivedo::Stream::Message.new("\x02")
      message = Runivedo::Stream::Message.new("\x02")
      message.read.should == false
    end

    it "receives integers" do
      # {10 => 1, 11 => 2, 12 => 4, 13 => 8, 15 => 1, 16 => 2, 17 => 4, 18 => 8}.each_pair do |i, s|
      #   message = Runivedo::Stream::Message.new([i].pack("C") + "\x2A" + "\x00" * (s-1))
      #   message.read.should == 42
      # end
      message = Runivedo::Stream::Message.new("\x18\x2a")
      message.read.should == 42
    end

    it "receives floats" do
      # message = Runivedo::Stream::Message.new("\x14\x10" + "\x00" * 3)
      message = Runivedo::Stream::Message.new("\xfa\x10" + "\x00" * 3)
      message.read.should == 2.2420775429197073e-44
      # message = Runivedo::Stream::Message.new("\x15\x10" + "\x00" * 7)
      message = Runivedo::Stream::Message.new("\xfb\x10" + "\x00" * 7)
      message.read.should == 8.0e-323
    end

    it 'receives decimals' do
      # message = Runivedo::Stream::Message.new(
      #   "\x05" "\x2A" "\x01" +
      #   "\x06" "\x2A\x00" "\x01" +
      #   "\x07" "\x2A\x00\x00\x00" "\x01" +
      #   "\x08" "\x2A\x00\x00\x00\x00\x00\x00\x00" "\x01"
      #   )
      # 4.times {message.read.should == BigDecimal.new("4.2")}
      message = Runivedo::Stream::Message.new("\xc4\x82\x18\x2a\x20")
      message.read.should == BigDecimal.new("4.2")
    end

    it "receives strings" do
      # message = Runivedo::Stream::Message.new("\x1e\x06\x00\x00\x00foobar")
      message = Runivedo::Stream::Message.new("\x66foobar")
      message.read.should == "foobar"
      # message = Runivedo::Stream::Message.new("\x1f\x06\x00\x00\x00f\x00o\x00o\x00b\x00a\x00r\x00")
      # message.read.should == "foobar"
    end

    it "receives arrays" do
      # message = Runivedo::Stream::Message.new("\x3c\x02\x00\x00\x00\x1e\x03\x00\x00\x00foo\x1e\x03\x00\x00\x00bar")
      message = Runivedo::Stream::Message.new("\x82\x63foo\x63bar")
      message.read.should == %w(foo bar)
    end

    it "receives maps" do
      # message = Runivedo::Stream::Message.new("\x3d\x02\x00\x00\x00\x1e\x03\x00\x00\x00foo\x0a\x01\x1e\x03\x00\x00\x00bar\x0a\x02")
      # message.read.should == {"foo" => 1, "bar" => 2}
      message = Runivedo::Stream::Message.new("\xa2bar\x72\xf4foo\xf5")
      message.read.should == {"bar" => 2, "foo" => 1}
    end

    it 'receives ids' do
      # message = Runivedo::Stream::Message.new("\x29\x02\x00\x00\x00\x00\x00\x00\x00")
      message = Runivedo::Stream::Message.new("\xc9\x02")
      message.read.should == 2
    end

    it 'receives datetimes' do
      # message = Runivedo::Stream::Message.new("\x33\x40\x7F\xFF\x0D\x8B\xDA\x04\x00")
      message = Runivedo::Stream::Message.new("\xca\x1a\x59\x6e\x6a\x55")
      message.read.should == Time.at(1366190677)
    end

    it 'receives times' do
      # message = Runivedo::Stream::Message.new("\x34\x40\x7F\xFF\x0D\x8B\xDA\x04\x00")
      message = Runivedo::Stream::Message.new("\xc9\x1a\x59\x6e\x6a\x55")
      message.read.should == Time.at(1366190677)
    end

    it 'receives uuids' do
      uuid = UUIDTools::UUID.random_create
      # message = Runivedo::Stream::Message.new("\x2a" + uuid.raw)
      message = Runivedo::Stream::Message.new("\xc8\x50" + uuid.raw)
      message.read.should == uuid
    end
  end

  describe "sending and receiving" do
    let(:message) { Runivedo::Stream::Message.new }
    let(:message_recv) { Runivedo::Stream::Message.new(message.instance_variable_get(:@buffer)) }

    it "works for null" do
      message << nil
      message_recv.read.should == nil
    end

    it "works for true" do
      message << true
      message_recv.read.should == true
    end

    it "works for false" do
      message << false
      message_recv.read.should == false
    end

    it "works for ints" do
      message << 42
      message_recv.read.should == 42
    end

    it "works for floats" do
      message << 42.23
      message_recv.read.should == 42.23
    end

    it "works for strings" do
      message << "foobar"
      message_recv.read.should == "foobar"
    end

    it "works for arrays" do
      message << %w(foo bar)
      message_recv.read.should == %w(foo bar)
    end

    it "works for maps" do
      message << {"foo" => 1, "foo" => 2}
      message_recv.read.should == {"foo" => 1, "foo" => 2}
    end

    it "works for uuids" do
      uuid = UUIDTools::UUID.random_create
      message << uuid
      message_recv.read.should == uuid
    end
  end

  describe Runivedo::VariantStream do
    it "receives" do
      # s = Runivedo::VariantStream.new(StringIO.new("\x0D\x2A\x00\x00\x00\x00\x00\x00\x00\x1e\x06\x00\x00\x00foobar"))
      s = Runivedo::VariantStream.new(StringIO.new("\x18\x2a\x66foobar"))
      s.read.should == 42
      s.read.should == "foobar"
      s.has_data?.should be_false
    end
{'fraction_non_alphanumeric': 0.1640418898393157, 'fraction_numerical': 0.07522583739707411, 'mean_word_length': 2.4273972602739726, 'pattern_counts': {'":': 0, '<': 81, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3667229', 'n_tokens_mistral': 5392, 'n_tokens_neox': 4821, 'n_words': 878}