<commit_before>package sodium

import "fmt"
import "unsafe"

// #include <stdio.h>
// #include <sodium.h>
import "C"

func MemZero(buff1 []byte) {
	if len(buff1) > 0 {
		C.sodium_memzero(unsafe.Pointer(&buff1[0]), C.size_t(len(buff1)))
	}
}

func MemCmp(buff1, buff2 []byte, length int) int {
	if length >= len(buff1) || length >= len(buff2) {
		panic(fmt.Sprintf("Attempt to compare more bytes (%d) than provided " + 
			"(%d, %d)", length, len(buff1), len(buff2)))
	}
	return int(C.sodium_memcmp(unsafe.Pointer(&buff1[0]),
		unsafe.Pointer(&buff2[0]),
		C.size_t(length)))
}

func Bin2hex(bin []byte) string {
	maxlen := len(bin) * 2
	binPtr := (*C.uchar)(unsafe.Pointer(&bin[0]))
	buf := (*C.char)(C.malloc(C.size_t(maxlen)))
	defer C.free(unsafe.Pointer(buf))
	
	C.sodium_bin2hex(buf, C.size_t(maxlen), binPtr, C.size_t(len(bin)))

	return C.GoString(buf)
}<commit_msg>Fix Bin2hex: the length has to include the trailing \0<commit_after>package sodium

import "fmt"
import "unsafe"

// #include <stdio.h>
// #include <sodium.h>
import "C"

func MemZero(buff1 []byte) {
	if len(buff1) > 0 {
		C.sodium_memzero(unsafe.Pointer(&buff1[0]), C.size_t(len(buff1)))
	}
}

func MemCmp(buff1, buff2 []byte, length int) int {
	if length >= len(buff1) || length >= len(buff2) {
		panic(fmt.Sprintf("Attempt to compare more bytes (%d) than provided " + 
			"(%d, %d)", length, len(buff1), len(buff2)))
	}
	return int(C.sodium_memcmp(unsafe.Pointer(&buff1[0]),
		unsafe.Pointer(&buff2[0]),
		C.size_t(length)))
}

func Bin2hex(bin []byte) string {
	maxlen := len(bin) * 2 + 1
	binPtr := (*C.uchar)(unsafe.Pointer(&bin[0]))
	buf := (*C.char)(C.malloc(C.size_t(maxlen)))
	defer C.free(unsafe.Pointer(buf))
	
	C.sodium_bin2hex(buf, C.size_t(maxlen), binPtr, C.size_t(len(bin)))

	return C.GoString(buf)
}
{'fraction_non_alphanumeric': 0.19821328866554996, 'fraction_numerical': 0.02400893355667225, 'mean_word_length': 4.914191419141914, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '604694', 'n_tokens_mistral': 779, 'n_tokens_neox': 728, 'n_words': 146}