<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"math/rand"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/influxdb/influxdb/datastore/storage"
)

func main() {
	rand.Seed(time.Now().UnixNano())

	points := flag.Int("points", 200000000, "Number of points")
	batchSize := flag.Int("batch", 1000, "Batch size")
	series := flag.Int("series", 1, "Number of series")
	path := flag.String("path", "/tmp", "Path to DB files")
	threads := flag.Int("threads", 1, "Number of threads that write data")
	flag.Parse()

	if *threads < 1 {
		fmt.Fprintf(os.Stderr, "Threads cannot be less than one. Run %s --help for more info\n", os.Args[0])
		os.Exit(2)
	}

	os.RemoveAll("/tmp/test-leveldb")
	os.RemoveAll("/tmp/test-lmdb")
	os.RemoveAll("/tmp/test-rocksdb")
	os.RemoveAll("/tmp/test-hyperleveldb")

	// benchmark("lmdb", Config{*points, *batchSize, *series, 0, 0, time.Now(), *path, *threads})
	// benchmark("leveldb", Config{*points, *batchSize, *series, 0, 0, time.Now(), *path, *threads})
	benchmark("rocksdb", Config{*points, *batchSize, *series, 0, 0, time.Now(), *path, *threads})
	// benchmark("hyperleveldb", Config{*points, *batchSize, *series, 0, 0, time.Now(), *path, *threads})
}

func benchmark(name string, c Config) {
	init, err := storage.GetInitializer(name)
	if err != nil {
		panic(err)
	}
<commit_msg>make the directory removal optional<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"math/rand"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/influxdb/influxdb/datastore/storage"
)

func main() {
	rand.Seed(time.Now().UnixNano())

	points := flag.Int("points", 200000000, "Number of points")
	batchSize := flag.Int("batch", 1000, "Batch size")
	series := flag.Int("series", 1, "Number of series")
	path := flag.String("path", "/tmp", "Path to DB files")
	threads := flag.Int("threads", 1, "Number of threads that write data")
	rmdir := flag.Bool("remove-dir", false, "Remove directory before running the benchmark")
	flag.Parse()

	if *threads < 1 {
		fmt.Fprintf(os.Stderr, "Threads cannot be less than one. Run %s --help for more info\n", os.Args[0])
		os.Exit(2)
	}

	if *rmdir {
		os.RemoveAll("/tmp/test-leveldb")
		os.RemoveAll("/tmp/test-lmdb")
		os.RemoveAll("/tmp/test-rocksdb")
		os.RemoveAll("/tmp/test-hyperleveldb")
	}

	benchmark("lmdb", Config{*points, *batchSize, *series, 0, 0, time.Now(), *path, *threads})
	benchmark("leveldb", Config{*points, *batchSize, *series, 0, 0, time.Now(), *path, *threads})
	benchmark("rocksdb", Config{*points, *batchSize, *series, 0, 0, time.Now(), *path, *threads})
	benchmark("hyperleveldb", Config{*points, *batchSize, *series, 0, 0, time.Now(), *path, *threads})
}

func benchmark(name string, c Config) {
	init, err := storage.GetInitializer(name)
	if err != nil {
		panic(err)
	}

{'fraction_non_alphanumeric': 0.1966113660430639, 'fraction_numerical': 0.018355100600070597, 'mean_word_length': 4.702213279678069, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5085964', 'n_tokens_mistral': 1134, 'n_tokens_neox': 1057, 'n_words': 259}