<commit_before>package block

import (
	"bytes"
	"encoding/binary"
	"reflect"
)

const MAX_BLOCK_DATA_SIZE = 1024

type Pieces struct {
	pieces []interface{}
}

type PieceInternal interface {
}

type Piece interface {
	buildInternal() PieceInternal
}

func getBlocksFromInternals(query uint16, internals []PieceInternal) []Block {
	var blocks []Block
	var buffer []byte

	for _, internal := range internals {
		data := GetBytes(internal)
		if len(data) <= MAX_BLOCK_DATA_SIZE &&
			len(data)+len(buffer) > MAX_BLOCK_DATA_SIZE {
			blocks = append(
				blocks,
				NewBlock(query, GenericBody{buffer}),
			)
			buffer = nil
		}
		buffer = append(buffer, data...)
	}
	blocks = append(
		blocks,
		NewBlock(query, GenericBody{buffer}),
	)
	return blocks
}

func GetBytes(b PieceInternal) []byte {
	buf := new(bytes.Buffer)
	binary.Write(buf, binary.BigEndian, b)
	return buf.Bytes()
}

func GetBlocks(query uint16, pieces []Piece) []Block {
	var internals []PieceInternal
	for _, piece := range pieces {
		internals = append(internals, piece.buildInternal())
	}
	return getBlocksFromInternals(query, internals)
}

func GetPieces(slice reflect.Value) []Piece {
	if slice.Kind() != reflect.Slice {
		return []Piece{slice.Interface().(Piece)}
	}<commit_msg>Can't maintain logic while splitting blocks, just byte count<commit_after>package block

import (
	"bytes"
	"encoding/binary"
	"reflect"
)

const MAX_BLOCK_DATA_SIZE = 1024

type Pieces struct {
	pieces []interface{}
}

type PieceInternal interface {
}

type Piece interface {
	buildInternal() PieceInternal
}

func getBytesFromInternals(internals []PieceInternal) []byte {
	var data []byte
	for _, internal := range internals {
		data = append(data, GetBytes(internal)...)
	}

	return data
}

func getBlocksFromInternals(query uint16, internals []PieceInternal) []Block {
	var blocks []Block

	data := getBytesFromInternals(internals)

	for len(data) > MAX_BLOCK_DATA_SIZE {
		chunk := data[:MAX_BLOCK_DATA_SIZE]
		data = data[MAX_BLOCK_DATA_SIZE:]
		blocks = append(blocks, NewBlock(query, GenericBody{chunk}))
	}

	if len(data) > 0 {
		blocks = append(blocks, NewBlock(query, GenericBody{data}))
	}
	return blocks
}

func GetBytes(b PieceInternal) []byte {
	buf := new(bytes.Buffer)
	binary.Write(buf, binary.BigEndian, b)
	return buf.Bytes()
}

func GetBlocks(query uint16, pieces []Piece) []Block {
	var internals []PieceInternal
	for _, piece := range pieces {
		internals = append(internals, piece.buildInternal())
	}
	return getBlocksFromInternals(query, internals)
}

func GetPieces(slice reflect.Value) []Piece {
	if slice.Kind() != reflect.Slice {
		return []Piece{slice.Interface().(Piece)}
	}
{'fraction_non_alphanumeric': 0.125, 'fraction_numerical': 0.006400602409638554, 'mean_word_length': 4.930803571428571, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7514874', 'n_tokens_mistral': 938, 'n_tokens_neox': 883, 'n_words': 217}