<commit_before>package storm

//TupleMessage - A tuple an it's processing metadata
type TupleMessage struct {

	//Id - To identifiy this tuple, for messaging guarantees
	Id string `json:"id",omitempty`

	//Stream - Id of the stream this tuple is emmited to. Blank==default stream
	Stream string `json:"stream",omitempty`

	//Task - For direct emit
	Task *int32 `json:"task",omitempty`

	//Tuple - A 'row' of data
	Tuple []interface{} `json:"tuple"`
}

//Spout Input & Output
type SpoutMessage struct {

	//TupleMessage - the tuple and it's metadata
	TupleMessage

	// Command - next, ack, fail, emit, log, sync
	Command string `json:"command"`

	//Message - for log only
	Message string `json:"message",omitempty`
}

// Transformer - Enables an implementation of a Bolt Transform
type TupleProcessor interface {
	Process(tuple *TupleMessage) (error, *TupleMessage)
}

type Spouter interface {
	Emit() *TupleMessage
	Ack(id string)
	Fail(id string)
	AssociateTasks(id string, taskIds []int)
}

type ComponentDef struct {
	ShellCommand string //to execute
	OutputFields []string
	Direct       bool
	Parallelism  int32
}

type DistributeHow string

const (
	DistributeByShuffle      = "shuffle"
	DistributeByField        = "field"<commit_msg>tweak<commit_after>package storm

//TupleMessage - A tuple an it's processing metadata
type TupleMessage struct {

	//Id - To identifiy this tuple, for messaging guarantees
	Id string `json:"id",omitempty`

	//Stream - Id of the stream this tuple is emmited to. Blank==default stream
	Stream string `json:"stream",omitempty`

	//Task - For direct emit
	Task *int32 `json:"task",omitempty`

	//Tuple - A 'row' of data
	Tuple []interface{} `json:"tuple"`
}

//Spout Input & Output
type SpoutMessage struct {

	//TupleMessage - the tuple and it's metadata
	TupleMessage

	// Command - next, ack, fail, emit, log, sync
	Command string `json:"command"`

	//Message - for log only
	Message string `json:"message",omitempty`
}

// Transformer - Enables an implementation of a Bolt Transform
type TupleProcessor interface {
	Process(tuple *TupleMessage) (*TupleMessage, error)
}

type Spouter interface {
	Emit() *TupleMessage
	Ack(id string)
	Fail(id string)
	AssociateTasks(id string, taskIds []int)
}

type ComponentDef struct {
	ShellCommand string //to execute
	OutputFields []string
	Direct       bool
	Parallelism  int32
}

type DistributeHow string

const (
	DistributeByShuffle      = "shuffle"
	DistributeByField        = "field"
{'fraction_non_alphanumeric': 0.10121457489878542, 'fraction_numerical': 0.0032388663967611335, 'mean_word_length': 4.012170385395538, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3030423', 'n_tokens_mistral': 763, 'n_tokens_neox': 736, 'n_words': 263}