<commit_before>package erasure

// #cgo LDFLAGS: longhair/bin/liblonghair.a
// #include "bridge.c"
import "C"

import (
	"common"
	"fmt"
	"unsafe"
)

func EncodeRing(originalSlices []byte, m int, sliceSize int) (redundantSlices []string, err error) {
	// check that 'm' is legal
	k := common.SWARMSIZE - m
	if k <= 0 || k >= common.SWARMSIZE {
		err = fmt.Errorf("m must be greater than 0 and smaller than %v", common.SWARMSIZE)
		return
	}

	// check that sliceSize is not to big or small
	if sliceSize < common.MINSLICESIZE || sliceSize > common.MAXSLICESIZE {
		err = fmt.Errorf("sliceSize must be greater than %v and smaller than %v", common.MINSLICESIZE, common.MAXSLICESIZE)
		return
	}

	// check that sliceSize is divisible by 8
	if sliceSize%8 != 0 {
		err = fmt.Errorf("sliceSize must be divisible by 8")
		return
	}

	// check that originalSlices is the correct size
	if len(originalSlices) != sliceSize*k {
		err = fmt.Errorf("originalSlices incorrectly padded, must be of size 'sliceSize' * %v - 'm'", common.SWARMSIZE)
		return
	}

	// call c library to encode data
	redundantChunk := C.encodeRedundancy(C.int(k), C.int(m), C.int(sliceSize), (*C.char)(unsafe.Pointer(&originalSlices[0])))
	redundantString := C.GoStringN(redundantChunk, C.int(m*sliceSize))

	// redundantChunk into redundantSlices
	redundantSlices = make([]string, m)
	for i := 0; i < m; i++ {
		redundantSlices[i] = redundantString[i*sliceSize : (i*sliceSize)+sliceSize]
	}

	// free the memory allocated by the C file
	// I'm not sure at this point where everything is pointing,
	// this might also kill the redundant slices
	C.free(unsafe.Pointer(redundantChunk))

	return
}<commit_msg>spelling in comments, memory is managed correctly<commit_after>package erasure

// #cgo LDFLAGS: longhair/bin/liblonghair.a
// #include "bridge.c"
import "C"

import (
	"common"
	"fmt"
	"unsafe"
)

func EncodeRing(originalSlices []byte, m int, sliceSize int) (redundantSlices []string, err error) {
	// check that 'm' is legal
	k := common.SWARMSIZE - m
	if k <= 0 || k >= common.SWARMSIZE {
		err = fmt.Errorf("m must be greater than 0 and smaller than %v", common.SWARMSIZE)
		return
	}

	// check that sliceSize is not too big or small
	if sliceSize < common.MINSLICESIZE || sliceSize > common.MAXSLICESIZE {
		err = fmt.Errorf("sliceSize must be greater than %v and smaller than %v", common.MINSLICESIZE, common.MAXSLICESIZE)
		return
	}

	// check that sliceSize is divisible by 8
	if sliceSize%8 != 0 {
		err = fmt.Errorf("sliceSize must be divisible by 8")
		return
	}

	// check that originalSlices is the correct size
	if len(originalSlices) != sliceSize*k {
		err = fmt.Errorf("originalSlices incorrectly padded, must be of size 'sliceSize' * %v - 'm'", common.SWARMSIZE)
		return
	}

	// call c library to encode data
	redundantChunk := C.encodeRedundancy(C.int(k), C.int(m), C.int(sliceSize), (*C.char)(unsafe.Pointer(&originalSlices[0])))
	redundantString := C.GoStringN(redundantChunk, C.int(m*sliceSize))

	// redundantChunk into redundantSlices
	redundantSlices = make([]string, m)
	for i := 0; i < m; i++ {
		redundantSlices[i] = redundantString[i*sliceSize : (i*sliceSize)+sliceSize]
	}

	// free the memory allocated by the C file
	C.free(unsafe.Pointer(redundantChunk))

	return
}
{'fraction_non_alphanumeric': 0.12568807339449542, 'fraction_numerical': 0.004892966360856269, 'mean_word_length': 4.1269592476489025, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5190991', 'n_tokens_mistral': 1228, 'n_tokens_neox': 1144, 'n_words': 339}