<commit_before>package meritop

import "log"

// These two are useful for task to inform the framework their status change.
// metaData has to be really small, since it might be stored in etcd.
type Framework interface {
	// Flags and Sends the metaData to partent of the current task.
	FlagReadyForParent(metaData []byte)
	// Flags and Sends the metaData to chlidren of the current task.
	FlagReadyForChildren(metaData []byte)

	// These allow application developer to set the task configuration so framework
	// implementation knows which task to invoke at each node.
	SetTaskBuilder(taskBuilder TaskBuilder)

	// This allow the application
	SetTopology(topology Topology)

	// After all the configure is done, driver need to call start so that all
	// nodes will get into the event loop to run the application.
	Start()

	// Some task can inform all participating tasks to exit.
	Exit()

	// Some task can inform all participating tasks to new epoch
	SetEpoch(epochID uint64)

	GetLogger() log.Logger

	// Request data from parent or children.
	DataRequest(toID uint64, meta TGMeta)

	// This allow task implementation to node corresponding to taskID so that
	// it can carry out application dependent communication.
	GetNode(taskID uint64) Node

	// Return true if this node has children
	HasChildren() bool
	HasParents() bool
}
<commit_msg>Add payload rpc interface<commit_after>package meritop

import "log"

// These two are useful for task to inform the framework their status change.
// metaData has to be really small, since it might be stored in etcd.
type Framework interface {
	// Flags and Sends the metaData to partent of the current task.
	FlagParentMetaReady(meta TGMeta)
	FlagChildMetaReady(meta TGMeta)

	// These allow application developer to set the task configuration so framework
	// implementation knows which task to invoke at each node.
	SetTaskBuilder(taskBuilder TaskBuilder)

	// This allow the application
	SetTopology(topology Topology)

	// After all the configure is done, driver need to call start so that all
	// nodes will get into the event loop to run the application.
	Start()

	// Some task can inform all participating tasks to exit.
	Exit()

	// Some task can inform all participating tasks to new epoch
	SetEpoch(epochID uint64)

	GetLogger() log.Logger

	// Request data from parent or children.
	DataRequest(toID uint64, meta TGMeta)

	// This allow task implementation to node corresponding to taskID so that
	// it can carry out application dependent communication.
	GetNode(taskID uint64) Node

	// Return true if this node has children
	HasChildren() bool
	HasParents() bool
}

{'fraction_non_alphanumeric': 0.05671869052150742, 'fraction_numerical': 0.004567948229920061, 'mean_word_length': 3.9398496240601504, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6044752', 'n_tokens_mistral': 716, 'n_tokens_neox': 689, 'n_words': 350}