<commit_before>// bytecoder provides functions for converting standard arrays into byte arrays.
// Based on the Bytes function from golang.org\x\mobile\exp\f32\f32.go
package bytecoder

import (
	"encoding/binary"
	"fmt"
	"math"

	"github.com/go-gl/mathgl/mgl32"
)

// getByteOrder returns false for LittleEndian and true for BigEndian. It panics if the provided parameter isn't one of the two.
func getByteOrder(byteOrder binary.ByteOrder) bool {
	le := false
	switch byteOrder {
	case binary.BigEndian:
	case binary.LittleEndian:
		le = true
	default:
		panic(fmt.Sprintf("invalid byte order %v", byteOrder))
	}
	return le
}

// Bytes returns the byte representation of float32 values in the given byte
// order. byteOrder must be either binary.BigEndian or binary.LittleEndian.
func Float32(byteOrder binary.ByteOrder, values ...float32) []byte {
	le := getByteOrder(byteOrder)
	width := 4
	b := make([]byte, 4*len(values))
	for i, v := range values {
		u := math.Float32bits(v)
		if le {
			b[width*i+0] = byte(u >> 0)
			b[width*i+1] = byte(u >> 8)
			b[width*i+2] = byte(u >> 16)
			b[width*i+3] = byte(u >> 24)
		} else {
			b[width*i+0] = byte(u >> 24)
			b[width*i+1] = byte(u >> 16)
			b[width*i+2] = byte(u >> 8)
			b[width*i+3] = byte(u >> 0)
		}
	}
	return b
}

func Vec2(byteOrder binary.ByteOrder, values ...mgl32.Vec2) []byte {
	le := getByteOrder(byteOrder)
	width := 8
	b := make([]byte, 8*len(values))
	for i, v := range values {
		x := math.Float32bits(v[0])
		y := math.Float32bits(v[1])
		if le {
			b[width*i+0] = byte(x >> 0)
			b[width*i+1] = byte(x >> 8)
			b[width*i+2] = byte(x >> 16)
			b[width*i+3] = byte(x >> 24)
			b[width*i+4] = byte(y >> 0)
			b[width*i+5] = byte(y >> 8)
			b[width*i+6] = byte(y >> 16)
			b[width*i+7] = byte(y >> 24)
		} else {
			b[width*i+0] = byte(x >> 24)
			b[width*i+1] = byte(x >> 16)
			b[width*i+2] = byte(x >> 8)
			b[width*i+3] = byte(x >> 0)
			b[width*i+4] = byte(y >> 24)
			b[width*i+5] = byte(y >> 16)
			b[width*i+6] = byte(y >> 8)
			b[width*i+7] = byte(y >> 0)<commit_msg>Add Vec3 encoding<commit_after>// bytecoder provides functions for converting standard arrays into byte arrays.
// Based on the Bytes function from golang.org\x\mobile\exp\f32\f32.go
package bytecoder

import (
	"encoding/binary"
	"fmt"
	"math"

	"github.com/go-gl/mathgl/mgl32"
)

// getByteOrder returns false for LittleEndian and true for BigEndian. It panics if the provided parameter isn't one of the two.
func getByteOrder(byteOrder binary.ByteOrder) bool {
	le := false
	switch byteOrder {
	case binary.BigEndian:
	case binary.LittleEndian:
		le = true
	default:
		panic(fmt.Sprintf("invalid byte order %v", byteOrder))
	}
	return le
}

// Bytes returns the byte representation of float32 values in the given byte
// order. byteOrder must be either binary.BigEndian or binary.LittleEndian.
func Float32(byteOrder binary.ByteOrder, values ...float32) []byte {
	le := getByteOrder(byteOrder)
	width := 4
	b := make([]byte, width*len(values))
	for i, v := range values {
		u := math.Float32bits(v)
		if le {
			b[width*i+0] = byte(u >> 0)
			b[width*i+1] = byte(u >> 8)
			b[width*i+2] = byte(u >> 16)
			b[width*i+3] = byte(u >> 24)
		} else {
			b[width*i+0] = byte(u >> 24)
			b[width*i+1] = byte(u >> 16)
			b[width*i+2] = byte(u >> 8)
			b[width*i+3] = byte(u >> 0)
		}
	}
	return b
}

func Vec2(byteOrder binary.ByteOrder, values ...mgl32.Vec2) []byte {
	le := getByteOrder(byteOrder)
	width := 8
	b := make([]byte, width*len(values))
	for i, v := range values {
		x := math.Float32bits(v[0])
		y := math.Float32bits(v[1])
		if le {
			b[width*i+0] = byte(x >> 0)
			b[width*i+1] = byte(x >> 8)
			b[width*i+2] = byte(x >> 16)
			b[width*i+3] = byte(x >> 24)
			b[width*i+4] = byte(y >> 0)
			b[width*i+5] = byte(y >> 8)
			b[width*i+6] = byte(y >> 16)
			b[width*i+7] = byte(y >> 24)
		} else {
			b[width*i+0] = byte(x >> 24)
			b[width*i+1] = byte(x >> 16)
			b[width*i+2] = byte(x >> 8)
			b[width*i+3] = byte(x >> 0)
			b[width*i+4] = byte(y >> 24)
			b[width*i+5] = byte(y >> 16)
			b[width*i+6] = byte(y >> 8)
			b[width*i+7] = byte(y >> 0)
		}
	}
	return b
}

func Vec3(byteOrder binary.ByteOrder, values ...mgl32.Vec3) []byte {
	le := getByteOrder(byteOrder)
	width := 12
	b := make([]byte, width*len(values))
	for i, v := range values {
		x := math.Float32bits(v[0])
		y := math.Float32bits(v[1])
		z := math.Float32bits(v[2])
		if le {
			b[width*i+0] = byte(x >> 0)
			b[width*i+1] = byte(x >> 8)
			b[width*i+2] = byte(x >> 16)
			b[width*i+3] = byte(x >> 24)
			b[width*i+4] = byte(y >> 0)
			b[width*i+5] = byte(y >> 8)
			b[width*i+6] = byte(y >> 16)
			b[width*i+7] = byte(y >> 24)
			b[width*i+8] = byte(z >> 0)
			b[width*i+9] = byte(z >> 8)
			b[width*i+10] = byte(z >> 16)
			b[width*i+11] = byte(z >> 24)
		} else {
			b[width*i+0] = byte(x >> 24)
			b[width*i+1] = byte(x >> 16)
			b[width*i+2] = byte(x >> 8)
			b[width*i+3] = byte(x >> 0)
			b[width*i+4] = byte(y >> 24)
			b[width*i+5] = byte(y >> 16)
			b[width*i+6] = byte(y >> 8)
			b[width*i+7] = byte(y >> 0)
			b[width*i+8] = byte(z >> 24)
			b[width*i+9] = byte(z >> 16)
			b[width*i+10] = byte(z >> 8)
			b[width*i+11] = byte(z >> 0)
{'fraction_non_alphanumeric': 0.19721577726218098, 'fraction_numerical': 0.04911059551430781, 'mean_word_length': 3.391341256366723, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 147, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3716123', 'n_tokens_mistral': 2565, 'n_tokens_neox': 2239, 'n_words': 520}