<commit_before>package gortex

import "math"
import (
	"encoding/json"
	"fmt"

	"github.com/vseledkin/gortex/assembler"

	"math/rand"
	"os"
	"time"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func Abs(f float32) float32 {
	if f < 0 {
		return -f
	}
	return f
}
func Max(x, y float32) float32 {
	if x < y {
		return y
	}
	return x
}
func MaxInt(x, y int) int {
	if x < y {
		return y
	}
	return x
}
func MinInt(x, y int) int {
	if x > y {
		return y
	}
	return x
}

func Pow(x, y float32) float32 {
	return float32(math.Pow(float64(x), float64(y)))
}

func Zeros(n int) []float32 {
	return make([]float32, n)
}

func ScaleGradient(model map[string]*Matrix, v float32) {
	for _, m := range model {
		assembler.Sscale(v, m.DW)
	}
}

func PrintGradient(model map[string]*Matrix) {
	for k, m := range model {
		fmt.Printf("Grad: %s %f %f\n", k, m.Norm(), m.NormGradient())
	}
}

func PrintZeroGradient(model map[string]*Matrix) {
	for k, m := range model {
		wnorm := m.Norm()
		gnorm := m.NormGradient()
		if wnorm == 0 {
			fmt.Printf("\033[93mWarning!!! Weights of %s = %f\033[0m\n", k, wnorm)
		}
		if gnorm == 0 {
			fmt.Printf("\033[93mWarning!!! Gradients of %s = %f\033[0m\n", k, gnorm)
		}
	}
}

func InitWeights(model map[string]*Matrix, dev float32) {
	for _, m := range model {
		for i := range m.W {
			m.W[i] = dev * float32(rand.NormFloat64()) // standard normal distribution (mean = 0, stddev = 1)
		}
	}
}

func ResetGradients(model map[string]*Matrix) {
	for _, m := range model {
		assembler.Sclean(m.DW)
	}
}

//Softmax probability distribution interpretation of any vector/matrix
func Softmax(m *Matrix) *Matrix {
	out := Mat(m.Rows, m.Columns) // probability volume
	maxval := m.W[assembler.Ismax(m.W)]
	for i := range m.W {
		out.W[i] = float32(math.Exp(float64(m.W[i] - maxval)))
	}
	sum := assembler.Sum(out.W)
	assembler.Sscale(1/sum, out.W)

	// no backward pass here needed
	// since we will use the computed probabilities outside
	// to set gradients directly on m<commit_msg>add function to normalize vector to unit length<commit_after>package gortex

import "math"
import (
	"encoding/json"
	"fmt"

	"github.com/vseledkin/gortex/assembler"

	"math/rand"
	"os"
	"time"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func Abs(f float32) float32 {
	if f < 0 {
		return -f
	}
	return f
}
func Max(x, y float32) float32 {
	if x < y {
		return y
	}
	return x
}
func MaxInt(x, y int) int {
	if x < y {
		return y
	}
	return x
}
func MinInt(x, y int) int {
	if x > y {
		return y
	}
	return x
}

func Pow(x, y float32) float32 {
	return float32(math.Pow(float64(x), float64(y)))
}

func Zeros(n int) []float32 {
	return make([]float32, n)
}

func ScaleGradient(model map[string]*Matrix, v float32) {
	for _, m := range model {
		assembler.Sscale(v, m.DW)
	}
}

func PrintGradient(model map[string]*Matrix) {
	for k, m := range model {
		fmt.Printf("Grad: %s %f %f\n", k, m.Norm(), m.NormGradient())
	}
}

func PrintZeroGradient(model map[string]*Matrix) {
	for k, m := range model {
		wnorm := m.Norm()
		gnorm := m.NormGradient()
		if wnorm == 0 {
			fmt.Printf("\033[93mWarning!!! Weights of %s = %f\033[0m\n", k, wnorm)
		}
		if gnorm == 0 {
			fmt.Printf("\033[93mWarning!!! Gradients of %s = %f\033[0m\n", k, gnorm)
		}
	}
}

func NormalizeToUnitLength(m *Matrix) {
	assembler.Sscale(1/assembler.L2(m.W), m.W)
}

func InitWeights(model map[string]*Matrix, dev float32) {
	for _, m := range model {
		for i := range m.W {
			m.W[i] = dev * float32(rand.NormFloat64()) // standard normal distribution (mean = 0, stddev = 1)
		}
	}
}

func ResetGradients(model map[string]*Matrix) {
	for _, m := range model {
		assembler.Sclean(m.DW)
	}
}

//Softmax probability distribution interpretation of any vector/matrix
func Softmax(m *Matrix) *Matrix {
	out := Mat(m.Rows, m.Columns) // probability volume
	maxval := m.W[assembler.Ismax(m.W)]
	for i := range m.W {
		out.W[i] = float32(math.Exp(float64(m.W[i] - maxval)))
	}
	sum := assembler.Sum(out.W)
	assembler.Sscale(1/sum, out.W)

	// no backward pass here needed
	// since we will use the computed probabilities outside
	// to set gradients directly on m
{'fraction_non_alphanumeric': 0.16107220478145376, 'fraction_numerical': 0.028495532480077277, 'mean_word_length': 3.453763440860215, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2784754', 'n_tokens_mistral': 1801, 'n_tokens_neox': 1675, 'n_words': 437}