<commit_before>package library

import (
	"container/heap"
	"errors"
	"github.com/nytlabs/gojee"
	"github.com/nytlabs/streamtools/st/blocks" // blocks
	"github.com/nytlabs/streamtools/st/util"
	"time"
)

// specify those channels we're going to use to communicate with streamtools
type Pack struct {
	blocks.Block
	queryrule chan blocks.MsgChan
	inrule    blocks.MsgChan
	in        blocks.MsgChan
	out       blocks.MsgChan
	quit      blocks.MsgChan
}

// we need to build a simple factory so that streamtools can make new blocks of this kind
func NewPack() blocks.BlockInterface {
	return &Pack{}
}

// Setup is called once before running the block. We build up the channels and specify what kind of block this is.
func (b *Pack) Setup() {
	b.Kind = "Pack"
	b.Desc = "groups messages together based on a common value, similar to 'group-by' in other languages"
	b.in = b.InRoute("in")
	b.inrule = b.InRoute("rule")
	b.queryrule = b.QueryRoute("rule")
	b.quit = b.Quit()
	b.out = b.Broadcast()
}

// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Pack) Run() {
	var tree *jee.TokenTree
	var emitAfter, path string
	var err error

	afterDuration := time.Duration(0)
	waitTimer := time.NewTimer(100 * time.Millisecond)
	bunches := make(map[string][]interface{})
	pq := &PriorityQueue{}
	heap.Init(pq)

	for {
		select {
		case ruleI := <-b.inrule:
			// set a parameter of the block
			rule, ok := ruleI.(map[string]interface{})
			if !ok {
				b.Error(errors.New("coudln't assert rule to map"))
				continue
			}<commit_msg>quick fix to pack<commit_after>package library

import (
	"container/heap"
	"errors"
	"github.com/nytlabs/gojee"
	"github.com/nytlabs/streamtools/st/blocks" // blocks
	"github.com/nytlabs/streamtools/st/util"
	"time"
)

// specify those channels we're going to use to communicate with streamtools
type Pack struct {
	blocks.Block
	queryrule chan blocks.MsgChan
	inrule    blocks.MsgChan
	in        blocks.MsgChan
	out       blocks.MsgChan
	quit      blocks.MsgChan
}

// we need to build a simple factory so that streamtools can make new blocks of this kind
func NewPack() blocks.BlockInterface {
	return &Pack{}
}

// Setup is called once before running the block. We build up the channels and specify what kind of block this is.
func (b *Pack) Setup() {
	b.Kind = "Pack"
	b.Desc = "groups messages together based on a common value, similar to 'group-by' in other languages"
	b.in = b.InRoute("in")
	b.inrule = b.InRoute("rule")
	b.queryrule = b.QueryRoute("rule")
	b.quit = b.Quit()
	b.out = b.Broadcast()
}

// Run is the block's main loop. Here we listen on the different channels we set up.
func (b *Pack) Run() {
	var tree *jee.TokenTree
	var emitAfter, path string
	var err error

	afterDuration := time.Duration(0)
	waitTimer := time.NewTimer(100 * time.Millisecond)
	bunches := make(map[string][]interface{})
	pq := &PriorityQueue{}
	heap.Init(pq)

	for {
		select {
		case <-waitTimer.C:
		case ruleI := <-b.inrule:
			// set a parameter of the block
			rule, ok := ruleI.(map[string]interface{})
			if !ok {
				b.Error(errors.New("coudln't assert rule to map"))
				continue
			}
{'fraction_non_alphanumeric': 0.11787072243346007, 'fraction_numerical': 0.0025348542458808617, 'mean_word_length': 3.51, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1633900', 'n_tokens_mistral': 1123, 'n_tokens_neox': 1055, 'n_words': 349}