<commit_before>//  Copyright (c) 2013 Couchbase, Inc.

package common

import "fmt"
import "strconv"

var _ = fmt.Sprintf("dummy")

// EvalForms will evaluate the non-terminal form `name`
// by randomly picking one of the []*Form defined by its
// rules.
func EvalForms(name string, scope Scope, forms []*Form) interface{} {
	if len(forms) == 0 {
		return nil
	}
	rnd := scope.GetRandom()
	lookup, failed := make([]bool, len(forms)), 0
	maxWeigh := maxWeighsOfForms(name, scope, forms)
	for failed < len(forms) {
		f := float64(rnd.Int31n(maxWeigh)) / float64(0x7FFFFFFF)
		for i, form := range forms {
			weight := currWeight(name, i, scope, form)
			if weight <= 0.0 {
				failed++

			} else if lookup[i] == false && f <= weight {
				lookup[i] = true
				scope = decWeight(name, i, scope, form) // weight restrainer
				val := form.Eval(scope.Clone())
				if val == nil {
					failed++
					continue
				}
				return val
			}
		}
	}
	return nil
}

func currWeight(name string, i int, scope Scope, form *Form) float64 {
	nm := name + strconv.Itoa(i)
	if w, ok := scope.GetWeight(nm); ok {
		return w
	}
	return form.Weight
}

func decWeight(name string, i int, scope Scope, form *Form) Scope {
	nm := name + strconv.Itoa(i)
	weight := form.Weight
	if w, ok := scope.GetWeight(nm); ok {
		weight = w
	}
	scope.SetWeight(nm, weight-form.Restrain)
	return scope
}

func maxWeighsOfForms(name string, scope Scope, forms []*Form) int32 {
	max := int32(0)
	for i, form := range forms {
		weight := currWeight(name, i, scope, form)
		x := int32(weight * float64(0x7FFFFFFF))
		if x > max {
			max = x
		}
	}
	return max
}<commit_msg>use float64 for weight calculations.<commit_after>//  Copyright (c) 2013 Couchbase, Inc.

package common

import "fmt"
import "strconv"
import "math"

var _ = fmt.Sprintf("dummy")

// EvalForms will evaluate the non-terminal form `name`
// by randomly picking one of the []*Form defined by its
// rules.
func EvalForms(name string, scope Scope, forms []*Form) interface{} {
	if len(forms) == 0 {
		return nil
	}
	rnd := scope.GetRandom()
	lookup, failed := make([]bool, len(forms)), 0
	maxWeigh := maxWeighsOfForms(name, scope, forms)
	//fmt.Println()
	for failed < len(forms) {
		f := rnd.Float64() * maxWeigh
		for i, form := range forms {
			weight := currWeight(name, i, scope, form)
			//fmt.Println(name, i, maxWeigh, f, weight, f <= weight, weight <= 0.0)
			if weight <= 0.0 {
				failed++

			} else if lookup[i] == false && f <= weight {
				lookup[i] = true
				scope = decWeight(name, i, scope, form) // weight restrainer
				val := form.Eval(scope.Clone())
				if val == nil {
					failed++
					continue
				}
				return val
			}
		}
	}
	return nil
}

func currWeight(name string, i int, scope Scope, form *Form) float64 {
	nm := name + strconv.Itoa(i)
	if w, ok := scope.GetWeight(nm); ok {
		return w
	}
	return form.Weight
}

func decWeight(name string, i int, scope Scope, form *Form) Scope {
	nm := name + strconv.Itoa(i)
	weight := form.Weight
	if w, ok := scope.GetWeight(nm); ok {
		weight = w
	}
	if weight -= form.Restrain; weight <= 0 {
		weight = 0
	}
	scope.SetWeight(nm, weight)
	return scope
}

func maxWeighsOfForms(name string, scope Scope, forms []*Form) float64 {
	max := math.Inf(-1)
	for i, form := range forms {
		weight := currWeight(name, i, scope, form)
		//fmt.Println(i, weight, x)
		if weight > max {
			max = weight
		}
	}
	//fmt.Println(max)
	return max
}
{'fraction_non_alphanumeric': 0.14585764294049008, 'fraction_numerical': 0.014585764294049008, 'mean_word_length': 3.2333333333333334, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '566669', 'n_tokens_mistral': 1377, 'n_tokens_neox': 1241, 'n_words': 384}