<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/alanctgardner/gogen-avro/container"
	"github.com/alanctgardner/gogen-avro/generator"
	"github.com/alanctgardner/gogen-avro/types"
)

func main() {
	generateContainer := flag.Bool("container", false, "Whether to emit container file writer code")
	packageName := flag.String("package", "avro", "Name of generated package")
	flag.Parse()
	if flag.NArg() < 2 {
		fmt.Fprintf(os.Stderr, "Usage: gogen-avro [--container] [--package=<package name>] <target directory> <schema files>\n")
		os.Exit(1)
	}
	targetDir := flag.Arg(0)
	files := flag.Args()[1:]

	var err error
	pkg := generator.NewPackage(*packageName)

	if *generateContainer {
		err = addRecordDefinition([]byte(container.AVRO_BLOCK_SCHEMA), pkg, false)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error generating Avro container block schema - %v\n", err)
			os.Exit(2)
		}

		err = addRecordDefinition([]byte(container.AVRO_HEADER_SCHEMA), pkg, false)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error generating Avro container header schema - %v\n", err)
			os.Exit(2)
		}
	}

	for _, fileName := range files {
		schema, err := ioutil.ReadFile(fileName)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error reading file %q - %v\n", fileName, err)
			os.Exit(2)
		}

		err = addRecordDefinition(schema, pkg, *generateContainer)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error decoding schema for file %q - %v\n", fileName, err)
			os.Exit(3)
		}
	}

	// Add header comment to all generated files.
	// for _, f := range pkg.Files() {
	// 	pkg.AddHeader(f, codegenComment(files))
	// }
<commit_msg>Uncomment code to write headers to files.<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/alanctgardner/gogen-avro/container"
	"github.com/alanctgardner/gogen-avro/generator"
	"github.com/alanctgardner/gogen-avro/types"
)

func main() {
	generateContainer := flag.Bool("container", false, "Whether to emit container file writer code")
	packageName := flag.String("package", "avro", "Name of generated package")
	flag.Parse()
	if flag.NArg() < 2 {
		fmt.Fprintf(os.Stderr, "Usage: gogen-avro [--container] [--package=<package name>] <target directory> <schema files>\n")
		os.Exit(1)
	}
	targetDir := flag.Arg(0)
	files := flag.Args()[1:]

	var err error
	pkg := generator.NewPackage(*packageName)

	if *generateContainer {
		err = addRecordDefinition([]byte(container.AVRO_BLOCK_SCHEMA), pkg, false)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error generating Avro container block schema - %v\n", err)
			os.Exit(2)
		}

		err = addRecordDefinition([]byte(container.AVRO_HEADER_SCHEMA), pkg, false)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error generating Avro container header schema - %v\n", err)
			os.Exit(2)
		}
	}

	for _, fileName := range files {
		schema, err := ioutil.ReadFile(fileName)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error reading file %q - %v\n", fileName, err)
			os.Exit(2)
		}

		err = addRecordDefinition(schema, pkg, *generateContainer)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error decoding schema for file %q - %v\n", fileName, err)
			os.Exit(3)
		}
	}

	// Add header comment to all generated files.
	for _, f := range pkg.Files() {
		pkg.AddHeader(f, codegenComment(files))
	}

{'fraction_non_alphanumeric': 0.15419296663660956, 'fraction_numerical': 0.004809137360985873, 'mean_word_length': 3.9303703703703703, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '838227', 'n_tokens_mistral': 1304, 'n_tokens_neox': 1206, 'n_words': 286}