<commit_before>package main

import (
	"bytes"
	"crypto/md5"
	"fmt"
	"github.com/peterbourgon/diskv"
	"io"
)

const (
	transformBlockSize = 2 // grouping of chars per directory depth
)

func BlockTransform(s string) []string {
	sliceSize := len(s) / transformBlockSize
	pathSlice := make([]string, sliceSize)
	for i := 0; i < sliceSize; i++ {
		from, to := i*transformBlockSize, (i*transformBlockSize)+transformBlockSize
		pathSlice[i] = s[from:to]
	}
	return pathSlice
}

func main() {
	d := diskv.New(diskv.Options{
		BasePath:     "data",
		Transform:    BlockTransform,
		CacheSizeMax: 1024 * 1024, // 1MB
	})

	data := []string{
		"I am the very model of a modern Major-General",
		"I've information vegetable, animal, and mineral",
		"I know the kings of England, and I quote the fights historical",
		"From Marathon to Waterloo, in order categorical",
		"I'm very well acquainted, too, with matters mathematical",
		"I understand equations, both the simple and quadratical",
		"About binomial theorem I'm teeming with a lot o' news",
		"With many cheerful facts about the square of the hypotenuse",
	}
	for _, valueStr := range data {
		key, value := md5sum(valueStr), bytes.NewBufferString(valueStr).Bytes()
		d.Write(key, value)
	}

	keyChan, keyCount := d.Keys(), 0
	for key, ok := <-keyChan; ok; key, ok = <-keyChan {
		value, err := d.Read(key)
		if err != nil {
			panic(fmt.Sprintf("key %s had no value", key))
		}
		fmt.Printf("%s: %s\n", key, value)
		keyCount++
	}
	fmt.Printf("%d total keys\n", keyCount)

	// d.EraseAll() // leave it commented out to see how data is kept on disk
}

func md5sum(s string) string {
	h := md5.New()
	io.WriteString(h, s)
	return fmt.Sprintf("%x", h.Sum(nil))
}<commit_msg>Drop needless bytes.Stuff<commit_after>package main

import (
	"crypto/md5"
	"fmt"
	"github.com/peterbourgon/diskv"
	"io"
)

const (
	transformBlockSize = 2 // grouping of chars per directory depth
)

func BlockTransform(s string) []string {
	sliceSize := len(s) / transformBlockSize
	pathSlice := make([]string, sliceSize)
	for i := 0; i < sliceSize; i++ {
		from, to := i*transformBlockSize, (i*transformBlockSize)+transformBlockSize
		pathSlice[i] = s[from:to]
	}
	return pathSlice
}

func main() {
	d := diskv.New(diskv.Options{
		BasePath:     "data",
		Transform:    BlockTransform,
		CacheSizeMax: 1024 * 1024, // 1MB
	})

	data := []string{
		"I am the very model of a modern Major-General",
		"I've information vegetable, animal, and mineral",
		"I know the kings of England, and I quote the fights historical",
		"From Marathon to Waterloo, in order categorical",
		"I'm very well acquainted, too, with matters mathematical",
		"I understand equations, both the simple and quadratical",
		"About binomial theorem I'm teeming with a lot o' news",
		"With many cheerful facts about the square of the hypotenuse",
	}
	for _, valueStr := range data {
		key, val := md5sum(valueStr), []byte(valueStr)
		d.Write(key, val)
	}

	keyChan, keyCount := d.Keys(), 0
	for key, ok := <-keyChan; ok; key, ok = <-keyChan {
		val, err := d.Read(key)
		if err != nil {
			panic(fmt.Sprintf("key %s had no value", key))
		}
		fmt.Printf("%s: %s\n", key, val)
		keyCount++
	}
	fmt.Printf("%d total keys\n", keyCount)

	// d.EraseAll() // leave it commented out to see how data is kept on disk
}

func md5sum(s string) string {
	h := md5.New()
	io.WriteString(h, s)
	return fmt.Sprintf("%x", h.Sum(nil))
}
{'fraction_non_alphanumeric': 0.140807962529274, 'fraction_numerical': 0.00936768149882904, 'mean_word_length': 3.8675213675213675, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '514669', 'n_tokens_mistral': 1289, 'n_tokens_neox': 1179, 'n_words': 385}