<commit_before>package taskgraph

// For more information read: http://ewencp.org/blog/golang-iterators/
// We need an reasonably efficient way to enumerate all indexes.
//
// For example, we can do:
// sum := 0.0
// for it := param.GetIndexIterator(); it.Next(); {
//    sum += param.Get(it.Index());
//}
type IndexIterator interface {
	Index() uint64
	Next() bool
}

// We need some interface to define function and how we optimize them.
type Parameter interface {
	Get(index uint64) float32
	Set(index uint64, value float32)

	// This allow us to generate a same wide parameter for manipulation.
	Clone() Parameter

	// This allow one to enumerate through all parameters
	IndexIterator() IndexIterator
}

// There are many different ways one can optimize a function, but the
// most effective ones need gradient.
type Function interface {
	Evaluate(param Parameter) (value float32, gradient Parameter)
}

// High level interface for minimization. This assume that we start
// with one point in the parameter space, and end with an optimal
// point. Return true if we find optimal point.
type Minimizer interface {
	Minimize(func Function, param Parameter) bool
}<commit_msg>add regularization<commit_after>package taskgraph.op

// For more information read: http://ewencp.org/blog/golang-iterators/
// We need an reasonably efficient way to enumerate all indexes.
//
// For example, we can do:
// sum := 0.0
// for it := param.GetIndexIterator(); it.Next(); {
//    sum += param.Get(it.Index());
//}
type IndexIterator interface {
	Index() uint64
	Next() bool
	Rewind()
}

// We need some interface to define function and how we optimize them.
type Parameter interface {
	Get(index uint64) float32
	Set(index uint64, value float32)
	Add(index uint64, value float32)

	// This allow us to generate a same wide parameter for manipulation.
	Clone() Parameter

	// This allow one to enumerate through all parameters
	IndexIterator() IndexIterator
}

// There are many different ways one can optimize a function, but the
// most effective ones need gradient.
// The interface is designed so that one can compose function easily.
// The semantics is simply add gradient from this function to corresponding
// dimensions on the output paramenter gradient.
type Function interface {
	Evaluate(param Parameter, gradient *Parameter) float32
}


// This defines an additive function. One can compose the function this way easily.
type SumFunction struct {
	func1,func2 *Function
}

func (rf *SumFunction) Evaluate(param Parameter, gradient *Parameter) float32 {
	sum := rf.func1.Evaluate(param, gradient)
	sum += rf.func2.Evaluate(param, gradient)
	return sum
}

// This implements l1 l2 regularization.
type Regularization struct {
	iter IndexIterator
	l1reg, l2reg float32
}

func (r *Regularization) Evaluate(param Parameter, gradient *Parameter) float32 {
	r.iter.Rewind()
	sum := float64(0)
	for r.iter.Next() {
		index := r.iter.Index()
		value := param.Get(index)
		sum += l1reg*value
		sum += 0.5*l2reg*value*value;
		gradient.Add(index, l1reg)
		gradient.Add(index, l2reg*value)
	}
	return float32(sum)
}


// High level interface for minimization. This assume that we start
// with one point in the parameter space, and end with an optimal
// point. Return true if we find optimal point.
type Minimizer interface {
	Minimize(func Function, param Parameter) bool
}

{'fraction_non_alphanumeric': 0.09536541889483066, 'fraction_numerical': 0.01693404634581105, 'mean_word_length': 3.966076696165192, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5355077', 'n_tokens_mistral': 1005, 'n_tokens_neox': 935, 'n_words': 399}