<commit_before>package expr

type calcFN func(left float64, right float64) float64

// calc creates an Expr that obtains its value by applying the given calcFN
func calc(left interface{}, right interface{}, calc calcFN) Expr {
	return &calculator{binaryExpr{exprFor(left), exprFor(right)}, calc}
}

type calcAccumulator struct {
	binaryAccumulator
	calc calcFN
}

func (a *calcAccumulator) Get() float64 {
	log.Debugf("%v : %v", a.left.Get(), a.right.Get())
	return a.calc(a.left.Get(), a.right.Get())
}

type calculator struct {
	binaryExpr
	calc calcFN
}

func (e *calculator) Accumulator() Accumulator {
	return &calcAccumulator{binaryAccumulator{e.left.Accumulator(), e.right.Accumulator()}, e.calc}
}<commit_msg>Parsing numbers in expressions<commit_after>package expr

type calcFN func(left float64, right float64) float64

// calc creates an Expr that obtains its value by applying the given calcFN
func calc(left interface{}, right interface{}, calc calcFN) Expr {
	return &calculator{binaryExpr{exprFor(left), exprFor(right)}, calc}
}

type calcAccumulator struct {
	binaryAccumulator
	calc calcFN
}

func (a *calcAccumulator) Get() float64 {
	return a.calc(a.left.Get(), a.right.Get())
}

type calculator struct {
	binaryExpr
	calc calcFN
}

func (e *calculator) Accumulator() Accumulator {
	return &calcAccumulator{binaryAccumulator{e.left.Accumulator(), e.right.Accumulator()}, e.calc}
}
{'fraction_non_alphanumeric': 0.12008577555396711, 'fraction_numerical': 0.01143674052894925, 'mean_word_length': 5.6350710900473935, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6737159', 'n_tokens_mistral': 477, 'n_tokens_neox': 459, 'n_words': 133}