<commit_before>package gopark

import (
    "encoding/gob"
    "fmt"
    "math"
)

type Vector []float64

func init() {
    gob.Register(new(Vector))
}

func (v Vector) Plus(o Vector) Vector {
    return biVectorsOp(v, o, func(x, y float64) float64 {
        return x + y
    })
}

func (v Vector) Minus(o Vector) Vector {
    return biVectorsOp(v, o, func(x, y float64) float64 {
        return x - y
    })
}

func (v Vector) Multiply(m float64) Vector {
    result := make(Vector, len(v))
    for i := range v {
        result[i] = v[i] * m
    }
    return result
}

func (v Vector) Divide(d float64) Vector {
    if d == 0 {
        panic(fmt.Errorf("Vector divided by zero."))
    }
    result := make(Vector, len(v))
    for i := range v {
        result[i] = v[i] / d
    }
    return v
}

func (v Vector) Sign() Vector {
    return v.Multiply(-1)
}

func (v Vector) Sum() float64 {
    sum := 0.0
    for i := range v {
        sum += v[i]
    }
    return sum
}

func (v Vector) Dot(o Vector) float64 {
    if len(v) != len(o) {
        panic(fmt.Errorf("Two vectors of different length"))
    }
    sum := 0.0
    for i := range v {
        sum += v[i] * o[i]
    }
    return sum
}

func (v Vector) EulaDistance(o Vector) float64 {
    if len(v) != len(o) {
        panic(fmt.Errorf("Two vectors of different length"))
    }
    dist := 0.0
    for i := range v {
        dist += (v[i] - o[i]) * (v[i] - o[i])
    }
    return math.Sqrt(dist)
}

type operand func(x, y float64) float64

func biVectorsOp(v1, v2 Vector, fn operand) Vector {
    if len(v1) != len(v2) {
        panic(fmt.Errorf("Two vectors of different length"))
    }
    result := make(Vector, len(v1))
    for i := range v1 {
        result[i] = fn(v1[i], v2[i])
    }
    return result
}<commit_msg>Add some tools for vector<commit_after>package gopark

import (
    "encoding/gob"
    "fmt"
    "math"
    "math/rand"
    "time"
)

type Vector []float64

func init() {
    gob.Register(new(Vector))
}

func NewZeroVector(size int) Vector {
    v := make(Vector, size)
    return v
}

func NewRandomVector(size int) Vector {
    r := rand.New(rand.NewSource(time.Now().Unix()))
    v := make(Vector, size)
    for i := range v {
        v[i] = r.Float64()
    }
    return v
}

func (v Vector) Plus(o Vector) Vector {
    return biVectorsOp(v, o, func(x, y float64) float64 {
        return x + y
    })
}

func (v Vector) Minus(o Vector) Vector {
    return biVectorsOp(v, o, func(x, y float64) float64 {
        return x - y
    })
}

func (v Vector) Multiply(m float64) Vector {
    result := make(Vector, len(v))
    for i := range v {
        result[i] = v[i] * m
    }
    return result
}

func (v Vector) Divide(d float64) Vector {
    if d == 0 {
        panic(fmt.Errorf("Vector divided by zero."))
    }
    result := make(Vector, len(v))
    for i := range v {
        result[i] = v[i] / d
    }
    return v
}

func (v Vector) Sign() Vector {
    return v.Multiply(-1)
}

func (v Vector) Sum() float64 {
    sum := 0.0
    for i := range v {
        sum += v[i]
    }
    return sum
}

func (v Vector) Dot(o Vector) float64 {
    if len(v) != len(o) {
        panic(fmt.Errorf("Two vectors of different length"))
    }
    sum := 0.0
    for i := range v {
        sum += v[i] * o[i]
    }
    return sum
}

func (v Vector) Magnitude() float64 {
    return math.Sqrt(v.Dot(v))
}

func (v Vector) EulaDistance(o Vector) float64 {
    if len(v) != len(o) {
        panic(fmt.Errorf("Two vectors of different length"))
    }
    dist := 0.0
    for i := range v {
        dist += (v[i] - o[i]) * (v[i] - o[i])
    }
    return math.Sqrt(dist)
}

type operand func(x, y float64) float64

func biVectorsOp(v1, v2 Vector, fn operand) Vector {
    if len(v1) != len(v2) {
        panic(fmt.Errorf("Two vectors of different length"))
    }
    result := make(Vector, len(v1))
    for i := range v1 {
        result[i] = fn(v1[i], v2[i])
    }
    return result
}
{'fraction_non_alphanumeric': 0.15070921985815602, 'fraction_numerical': 0.02127659574468085, 'mean_word_length': 2.210569105691057, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1001507', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1483, 'n_words': 427}