<commit_before>package main

// Given a file with filenames, this times how fast we can stat them
// in parallel.  This is useful for benchmarking purposes.

import (
	"os"
	"flag"
	"time"
	"fmt"
	"encoding/line"
)

func main() {
	flag.Parse()

	filename := flag.Args()[0]
	f, err := os.Open(filename, os.O_RDONLY, 0)
	if err != nil {
		panic("err" + err.String())
	}

	linelen := 1000
	reader := line.NewReader(f, linelen)

	files := make([]string, 0)
	for {
		l, _, err := reader.ReadLine()
		if err != nil {
			break
		}
		files = append(files, string(l))
	}

	runs := 10
	tot := 0.0
	sleeptime := 4.0
	for j := 0; j < runs; j++ {
		tot += BulkStat(10, files)
		fmt.Printf("Sleeping %.2f seconds\n", sleeptime)
		time.Sleep(int64(sleeptime * 1e9))
	}

	fmt.Printf("Average of %d runs: %f ms\n", runs, tot/float64(runs))
}

func BulkStat(parallelism int, files []string) float64 {
	parallel := 10
	todo := make(chan string, len(files))
	dts := make(chan int64, parallel)

	allStart := time.Nanoseconds()

	fmt.Printf("Statting %d files with %d threads\n", len(files), parallel)
	for i := 0; i < parallel; i++ {
		go func() {
			for {
				fn := <-todo
				if fn == "" {
					break
				}

				t := time.Nanoseconds()
				os.Lstat(fn)
				dts <- time.Nanoseconds() - t
			}
		}()
	}

	for _, v := range files {
		todo <- v
	}

	total := 0.0
	for i := 0; i < len(files); i++ {
		total += float64(<-dts) * 1e-6
	}

	allEnd := time.Nanoseconds()
	avg := total / float64(len(files))

	fmt.Printf("Elapsed: %f sec. Average stat %f ms\n",<commit_msg>Make bulkstat tunable over the command line.<commit_after>package main

// Given a file with filenames, this times how fast we can stat them
// in parallel.  This is useful for benchmarking purposes.

import (
	"os"
	"flag"
	"time"
	"fmt"
	"encoding/line"
)

func main() {
	threads := flag.Int("threads", 12, "number of parallel threads in a run.")
	sleepTime := flag.Float64("sleep", 4.0, "amount of sleep between runs.")
	runs := flag.Int("runs", 10, "number of runs.")

	flag.Parse()

	filename := flag.Args()[0]
	f, err := os.Open(filename, os.O_RDONLY, 0)
	if err != nil {
		panic("err" + err.String())
	}

	linelen := 1000
	reader := line.NewReader(f, linelen)

	files := make([]string, 0)
	for {
		l, _, err := reader.ReadLine()
		if err != nil {
			break
		}
		files = append(files, string(l))
	}

	tot := 0.0
	for j := *runs; j > 0; j-- {
		tot += BulkStat(*threads, files)
		if j > 1 {
			fmt.Printf("Sleeping %.2f seconds\n", *sleepTime)
			time.Sleep(int64(*sleepTime * 1e9))
		}
	}

	fmt.Printf("Average of %d runs: %f ms\n", *runs, tot/float64(*runs))
}

func BulkStat(parallelism int, files []string) float64 {
	todo := make(chan string, len(files))
	dts := make(chan int64, parallelism)

	allStart := time.Nanoseconds()

	fmt.Printf("Statting %d files with %d threads\n", len(files), parallelism)
	for i := 0; i < parallelism; i++ {
		go func() {
			for {
				fn := <-todo
				if fn == "" {
					break
				}

				t := time.Nanoseconds()
				os.Lstat(fn)
				dts <- time.Nanoseconds() - t
			}
		}()
	}

	for _, v := range files {
		todo <- v
	}

	total := 0.0
	for i := 0; i < len(files); i++ {
		total += float64(<-dts) * 1e-6
	}

	allEnd := time.Nanoseconds()
	avg := total / float64(len(files))

	fmt.Printf("Elapsed: %f sec. Average stat %f ms\n",
{'fraction_non_alphanumeric': 0.17643504531722054, 'fraction_numerical': 0.023867069486404834, 'mean_word_length': 3.003627569528416, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6859672', 'n_tokens_mistral': 1477, 'n_tokens_neox': 1291, 'n_words': 360}