<commit_before>package struc

import (
	"encoding/binary"
	"io"
	"reflect"
	"strings"
)

type Fields []*Field

func (f Fields) SetByteOrder(order binary.ByteOrder) {
	for _, field := range f {
		field.Order = order
	}
}

func (f Fields) String() string {
	fields := make([]string, len(f))
	for i, field := range f {
		fields[i] = field.String()
	}
	return "{" + strings.Join(fields, ", ") + "}"
}

func (f Fields) Sizeof(data interface{}) int {
	val := reflect.ValueOf(data).Elem()
	size := 0
	for _, field := range f {
		switch field.Type {
		case String:
			size += val.Field(field.Index).Len()
		default:
			size += field.Size()
		}
	}
	return size
}

func (f Fields) Pack(w io.Writer, val reflect.Value) error {
	for i, field := range f {
		if !field.CanSet {
			continue
		}
		v := val.Field(i)
		length := field.Len
		if field.Slice && field.CanSet && field.Type != Pad {
			length = v.Len()
		} else if field.Sizefrom != nil {
			length = int(val.FieldByIndex(field.Sizefrom).Int())
		}
		if field.Sizeof != nil {
			length := val.FieldByIndex(field.Sizeof).Len()<commit_msg>fix broken Sizeof signature<commit_after>package struc

import (
	"encoding/binary"
	"io"
	"reflect"
	"strings"
)

type Fields []*Field

func (f Fields) SetByteOrder(order binary.ByteOrder) {
	for _, field := range f {
		field.Order = order
	}
}

func (f Fields) String() string {
	fields := make([]string, len(f))
	for i, field := range f {
		fields[i] = field.String()
	}
	return "{" + strings.Join(fields, ", ") + "}"
}

func (f Fields) Sizeof(val reflect.Value) int {
	size := 0
	for _, field := range f {
		switch field.Type {
		case String:
			size += val.Field(field.Index).Len()
		default:
			size += field.Size()
		}
	}
	return size
}

func (f Fields) Pack(w io.Writer, val reflect.Value) error {
	for i, field := range f {
		if !field.CanSet {
			continue
		}
		v := val.Field(i)
		length := field.Len
		if field.Slice && field.CanSet && field.Type != Pad {
			length = v.Len()
		} else if field.Sizefrom != nil {
			length = int(val.FieldByIndex(field.Sizefrom).Int())
		}
		if field.Sizeof != nil {
			length := val.FieldByIndex(field.Sizeof).Len()
{'fraction_non_alphanumeric': 0.16479925303454715, 'fraction_numerical': 0.0009337068160597573, 'mean_word_length': 3.277445109780439, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5024392', 'n_tokens_mistral': 844, 'n_tokens_neox': 787, 'n_words': 202}