<commit_before>package stream

import (
	"strings"
	"testing"
	"time"

	"github.com/vladimirvivien/automi/api/tuple"
)

func TestStream_GroupByInt(t *testing.T) {
	src := NewSliceSource("Hello World", "Hello Milkyway", "Hello Universe")
	snk := NewDrain()
	strm := New().From(src).FlatMap(func(line string) []string {
		return strings.Split(line, " ")
	}).Map(func(data string) tuple.KV {
		return tuple.KV{data, 1}
	}).GroupBy(0).To(snk)

	wait := make(chan struct{})
	go func() {
		defer close(wait)
		result := <-snk.GetOutput()
		m := result.(map[interface{}][]interface{})
		if len(m["Hello"]) != 3 {
			t.Fatal("Items not grouped properly")
		}

	}()

	select {
	case err := <-strm.Open():
		if err != nil {
			t.Fatal(err)
		}
		select {
		case <-wait:
		case <-time.After(10 * time.Millisecond):
			t.Fatal("Stream took too long to complete")
		}
	case <-time.After(10 * time.Millisecond):
		t.Fatal("Took too long")
	}

}
<commit_msg>New GroupByInt test for different scenarios<commit_after>package stream

import (
	"strings"
	"testing"
	"time"

	"github.com/vladimirvivien/automi/api/tuple"
)

// TestStream_GroupByInt_KV groups incoming tuple.KV data items.
func TestStream_GroupByInt_KV(t *testing.T) {
	src := NewSliceSource("Hello World", "Hello Milkyway", "Hello Universe")
	snk := NewDrain()
	strm := New().From(src).FlatMap(func(line string) []string {
		return strings.Split(line, " ")
	}).Map(func(data string) tuple.KV {
		return tuple.KV{data, 1}
	}).GroupBy(0).To(snk)

	wait := make(chan struct{})
	go func() {
		defer close(wait)
		result := <-snk.GetOutput()
		m := result.(map[interface{}][]interface{})
		if len(m["Hello"]) != 3 {
			t.Fatal("Items not grouped properly")
		}

	}()

	select {
	case err := <-strm.Open():
		if err != nil {
			t.Fatal(err)
		}
		select {
		case <-wait:
		case <-time.After(10 * time.Millisecond):
			t.Fatal("Stream took too long to complete")
		}
	case <-time.After(10 * time.Millisecond):
		t.Fatal("Took too long")
	}

}

// TestStream_GroupByInt_Slice tests  grouping data items received as slice.
func TestStream_GroupByInt_Slice(t *testing.T) {
	src := NewSliceSource("Hello World Wide", "Hello Milkyway Away", "Our World Now")
	snk := NewDrain()
	strm := New().From(src).Map(func(item string) []string {
		return strings.Split(item, " ")
	}).GroupBy(1).To(snk)

	wait := make(chan struct{})
	go func() {
		defer close(wait)
		result := <-snk.GetOutput()
		m := result.(map[interface{}][]interface{})
		t.Log(m)
		if len(m["World"]) != 4 {
			t.Fatal("Items not grouped properly")
		}

	}()

	select {
	case err := <-strm.Open():
		if err != nil {
			t.Fatal(err)
		}
		select {
		case <-wait:
		case <-time.After(10 * time.Millisecond):
			t.Fatal("Stream took too long to complete")
		}
	case <-time.After(10 * time.Millisecond):
		t.Fatal("Took too long")
	}

}

{'fraction_non_alphanumeric': 0.18050158954433063, 'fraction_numerical': 0.007064641469445425, 'mean_word_length': 3.808149405772496, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '746614', 'n_tokens_mistral': 1165, 'n_tokens_neox': 1047, 'n_words': 238}