<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"text/template"
)

var input = flag.String("f", "-", "Input source")

func main() {
	flag.Parse()
	in, err := getInput(*input)
	if err != nil {
		fmt.Fprintln(os.Stderr, "error opening input:", err)
		os.Exit(1)
	}
	if err := tmpl(in, os.Stdout, envMap()); err != nil {
		fmt.Fprintln(os.Stderr, "error opening input:", err)
		os.Exit(1)
	}
}

func getInput(path string) (io.Reader, error) {
	if path == "-" {
		return os.Stdin, nil
	}
	return os.Open(path)
}

func envMap() map[string]string {
	result := map[string]string{}
	for _, envvar := range os.Environ() {
		parts := strings.SplitN(envvar, "=", 2)
		result[parts[0]] = parts[1]
	}
	return result
}

func tmpl(in io.Reader, out io.Writer, ctx interface{}) error {
	i, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}
	tmpl, err := template.New("format string").Parse(string(i))
	if err != nil {
		return err
	}
	return tmpl.Execute(out, ctx)
}
<commit_msg>add sprig templates<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"html/template"

	"github.com/Masterminds/sprig"
)

var input = flag.String("f", "-", "Input source")

func main() {
	flag.Parse()
	in, err := getInput(*input)
	if err != nil {
		fmt.Fprintln(os.Stderr, "error opening input:", err)
		os.Exit(1)
	}
	if err := tmpl(in, os.Stdout, envMap()); err != nil {
		fmt.Fprintln(os.Stderr, "error opening input:", err)
		os.Exit(1)
	}
}

func getInput(path string) (io.Reader, error) {
	if path == "-" {
		return os.Stdin, nil
	}
	return os.Open(path)
}

func envMap() map[string]string {
	result := map[string]string{}
	for _, envvar := range os.Environ() {
		parts := strings.SplitN(envvar, "=", 2)
		result[parts[0]] = parts[1]
	}
	return result
}

func tmpl(in io.Reader, out io.Writer, ctx interface{}) error {
	i, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}
	tmpl, err := template.New("format string").Funcs(sprig.FuncMap()).Parse(string(i))
	if err != nil {
		return err
	}
	return tmpl.Execute(out, ctx)
}

{'fraction_non_alphanumeric': 0.18403441682600383, 'fraction_numerical': 0.004780114722753346, 'mean_word_length': 3.746031746031746, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2284538', 'n_tokens_mistral': 870, 'n_tokens_neox': 846, 'n_words': 201}