<commit_before>package main

import (
	"errors"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"text/template"
)

var flags = struct {
	Package string
	Var     string
	Abs     bool
}{}

var stdout = log.New(os.Stdout, "", 0)
var stderr = log.New(os.Stderr, "", 0)

const helpString = `fmap generates a go source file containing a map[string][]byte
for the specified directory trees. The keys are the paths
of files and the values are the contents of the file at that path.

The generated go file is printed to stdout. Empty directories are
ignored, and symlinks are not followed.

usage:
  fmap [flags] path/to/dir path/to/dir2 ...

flags:
  -package  package name to use in generated file (default: "main")
  -var      variable name of the map (default: "files")
  -abs      use absolute paths for map keys

example:
  fmap static/css static/js | gofmt > static_files.go`

const fileTmpl = `package << .Package >>

var << .Var >> = map[string][]byte{
	<<- range $k, $v := .M >>
	"<< $k >>": << $v | printf "%#v" | trimPrefix >>,
	<<- end >>
}`

func main() {
	flag.StringVar(&flags.Package, "package", "main", "package name")
	flag.StringVar(&flags.Var, "var", "files", "variable name of map")
	flag.BoolVar(&flags.Abs, "abs", false, "use absolute path for keys")
	flag.Usage = func() {
		stderr.Println(helpString)
		os.Exit(2)
	}
	flag.Parse()

	roots := flag.Args()
	if len(roots) == 0 {
		stderr.Println(errors.New(`fmap: error: require path argument`))
		stderr.Println(helpString)
		os.Exit(2)
	}
	for _, dirRoot := range roots {
		info, err := os.Stat(dirRoot)
		if err != nil {
			stderr.Println(err)
			os.Exit(1)
		}
		if !info.IsDir() {
			stderr.Println(errors.New(`fmap: error: path argument must be a directory`))
			os.Exit(1)
		}
	}

	tmpl, err := template.New("file").Funcs(template.FuncMap{
		"trimPrefix": func(s string) string { return strings.TrimPrefix(s, "[]byte") },
	}).Delims("<<", ">>").Parse(fileTmpl)
	if err != nil {
		stderr.Println(err)
		os.Exit(1)
	}

	wg := sync.WaitGroup{}
	m := make(map[string][]byte)
	merged := make(chan File)

	for _, dirRoot := range roots {
		dirRoot := dirRoot
		wg.Add(1)
		go func() {
			defer wg.Done()
			out := fileContents(dirRoot)
			for f := range out {
				merged <- f
			}
		}()
	}

	go func() {
		for f := range merged {
			if f.Err != nil {
				stderr.Println(f.Err)
				os.Exit(1)
			}

			k := f.Path
			if flags.Abs {
				absP, err := filepath.Abs(f.Path)
				if err != nil {
					stderr.Println(err)
					os.Exit(1)
				}
				k = absP
			}
			m[filepath.ToSlash(k)] = f.Content
		}
	}()

	wg.Wait()
	close(merged)

	if err := tmpl.Execute(os.Stdout, struct {
		Package string
		Var     string
		M       map[string][]byte
	}{
		flags.Package,
		flags.Var,
		m,<commit_msg>Fix sublte race<commit_after>package main

import (
	"errors"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"text/template"
)

var flags = struct {
	Package string
	Var     string
	Abs     bool
}{}

var stdout = log.New(os.Stdout, "", 0)
var stderr = log.New(os.Stderr, "", 0)

const helpString = `fmap generates a go source file containing a map[string][]byte
for the specified directory trees. The keys are the paths
of files and the values are the contents of the file at that path.

The generated go file is printed to stdout. Empty directories are
ignored, and symlinks are not followed.

usage:
  fmap [flags] path/to/dir path/to/dir2 ...

flags:
  -package  package name to use in generated file (default: "main")
  -var      variable name of the map (default: "files")
  -abs      use absolute paths for map keys

example:
  fmap static/css static/js | gofmt > static_files.go`

const fileTmpl = `package << .Package >>

var << .Var >> = map[string][]byte{
	<<- range $k, $v := .M >>
	"<< $k >>": << $v | printf "%#v" | trimPrefix >>,
	<<- end >>
}`

func main() {
	flag.StringVar(&flags.Package, "package", "main", "package name")
	flag.StringVar(&flags.Var, "var", "files", "variable name of map")
	flag.BoolVar(&flags.Abs, "abs", false, "use absolute path for keys")
	flag.Usage = func() {
		stderr.Println(helpString)
		os.Exit(2)
	}
	flag.Parse()

	roots := flag.Args()
	if len(roots) == 0 {
		stderr.Println(errors.New(`fmap: error: require path argument`))
		stderr.Println(helpString)
		os.Exit(2)
	}
	for _, dirRoot := range roots {
		info, err := os.Stat(dirRoot)
		if err != nil {
			stderr.Println(err)
			os.Exit(1)
		}
		if !info.IsDir() {
			stderr.Println(errors.New(`fmap: error: path argument must be a directory`))
			os.Exit(1)
		}
	}

	tmpl, err := template.New("file").Funcs(template.FuncMap{
		"trimPrefix": func(s string) string { return strings.TrimPrefix(s, "[]byte") },
	}).Delims("<<", ">>").Parse(fileTmpl)
	if err != nil {
		stderr.Println(err)
		os.Exit(1)
	}

	wg := sync.WaitGroup{}
	m := make(map[string][]byte)
	merged := make(chan File)

	for _, dirRoot := range roots {
		dirRoot := dirRoot
		wg.Add(1)
		go func() {
			defer wg.Done()
			out := fileContents(dirRoot)
			for f := range out {
				merged <- f
			}
		}()
	}

	go func() {
		wg.Wait()
		close(merged)
	}()

	for f := range merged {
		if f.Err != nil {
			stderr.Println(f.Err)
			os.Exit(1)
		}

		k := f.Path
		if flags.Abs {
			absP, err := filepath.Abs(f.Path)
			if err != nil {
				stderr.Println(err)
				os.Exit(1)
			}
			k = absP
		}
		m[filepath.ToSlash(k)] = f.Content
	}

	if err := tmpl.Execute(os.Stdout, struct {
		Package string
		Var     string
		M       map[string][]byte
	}{
		flags.Package,
		flags.Var,
		m,
{'fraction_non_alphanumeric': 0.16226960607155763, 'fraction_numerical': 0.0043368268883267076, 'mean_word_length': 3.1030392883617495, 'pattern_counts': {'":': 4, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6750536', 'n_tokens_mistral': 2276, 'n_tokens_neox': 2079, 'n_words': 565}