<commit_before>package chevalier

import (
	"code.google.com/p/goprotobuf/proto"
	"github.com/anchor/dataframe"
)

func UnmarshalSource(packet []byte) (*DataSource, error) {
	source := new(DataSource)
	err := proto.Unmarshal(packet, source)
	return source, err
}

func UnmarshalSourceBurst(packet []byte) (*DataSourceBurst, error) {
	burst := new(DataSourceBurst)
	err := proto.Unmarshal(packet, burst)
	return burst, err
}

func BuildSourceBurst(sources []*DataSource) *DataSourceBurst {
	burst := new(DataSourceBurst)
	burst.Sources = sources
	return burst
}

func MarshalSourceBurst(burst *DataSourceBurst) ([]byte, error) {
	marshalled, err := proto.Marshal(burst)
	return marshalled, err
}

// DataFrameSource takes a Vaultaire DataFrame and returns just its
// source component as a DataSource.
func DataFrameSource(frame *dataframe.DataFrame) *DataSource {
	source := new(DataSource)
	source.Source = make([]*DataSource_Tag, len(frame.Source))
	for i, tag := range frame.Source {
		source.Source[i] = new(DataSource_Tag)
		source.Source[i].Field = tag.Field
		source.Source[i].Value = tag.Value
	}
	return source
}

func NewDataSourceTag(field, value string) *DataSource_Tag {<commit_msg>Don't need that DataFrame cruft anymore either<commit_after>package chevalier

import (
	"code.google.com/p/goprotobuf/proto"
)

func UnmarshalSource(packet []byte) (*DataSource, error) {
	source := new(DataSource)
	err := proto.Unmarshal(packet, source)
	return source, err
}

func UnmarshalSourceBurst(packet []byte) (*DataSourceBurst, error) {
	burst := new(DataSourceBurst)
	err := proto.Unmarshal(packet, burst)
	return burst, err
}

func BuildSourceBurst(sources []*DataSource) *DataSourceBurst {
	burst := new(DataSourceBurst)
	burst.Sources = sources
	return burst
}

func MarshalSourceBurst(burst *DataSourceBurst) ([]byte, error) {
	marshalled, err := proto.Marshal(burst)
	return marshalled, err
}

func NewDataSourceTag(field, value string) *DataSource_Tag {
{'fraction_non_alphanumeric': 0.12061068702290076, 'fraction_numerical': 0.0, 'mean_word_length': 5.826388888888889, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5746760', 'n_tokens_mistral': 648, 'n_tokens_neox': 621, 'n_words': 155}