<commit_before>require 'spec_helper'

describe Datapipes::Sink do
  describe 'composable' do
    context 'with 3 sinks' do
      subject { sink_a + sink_b + sink_c }

      it 'composes' do
        subject.run_all(5)
        expect(list).to eq [8, 20, 6]
      end
    end

    context 'with 4 sinks' do
      subject { sink_a + sink_b + sink_c + sink_d }

      it 'composes' do
        subject.run_all(5)
        expect(list).to eq [8, 20, 6, 50]
      end
    end

    describe 'associative law' do
      let(:a) { (sink_a + sink_b) + sink_c }
      let(:b) { sink_a + (sink_b + sink_c) }

      it 'keeps' do
        expect(a.run_all(3)).to eq b.run_all(3)
      end
    end

    let(:list) { [] }

    let(:sink_a) do
      list_ = list
      Class.new(Datapipes::Sink) do
        define_method(:run) do |data|
          list_ << (data + 3)
        end
      end.new
    end<commit_msg>Fix sink spec because sinks are executed concurrently<commit_after>require 'spec_helper'

describe Datapipes::Sink do
  describe 'composable' do
    context 'with 3 sinks' do
      subject { sink_a + sink_b + sink_c }

      it 'composes' do
        subject.run_all(5)
        expect(list).to have(3).items
      end
    end

    context 'with 4 sinks' do
      subject { sink_a + sink_b + sink_c + sink_d }

      it 'composes' do
        subject.run_all(5)
        expect(list).to have(4).items
      end
    end

    describe 'associative law' do
      let(:a) { (sink_a + sink_b) + sink_c }
      let(:b) { sink_a + (sink_b + sink_c) }

      it 'keeps' do
        a.run_all(3)
        size_a = list.size
        list.clear

        b.run_all(3)
        size_b = list.size

        expect(size_a).to eq size_b
      end
    end

    let(:list) { Queue.new }

    let(:sink_a) do
      list_ = list
      Class.new(Datapipes::Sink) do
        define_method(:run) do |data|
          list_ << (data + 3)
        end
      end.new
    end
{'fraction_non_alphanumeric': 0.10932642487046632, 'fraction_numerical': 0.013471502590673576, 'mean_word_length': 1.8735119047619047, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3990842', 'n_tokens_mistral': 728, 'n_tokens_neox': 690, 'n_words': 179}