<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	"sync"
)

var (
	optimized   = flag.Bool("opt", true, "add some optimization passes")
	printTokens = flag.Bool("tok", false, "print tokens")
	printAst    = flag.Bool("ast", false, "print abstract syntax tree")
	printLLVMIR = flag.Bool("llvm", false, "print LLVM generated code")
)

func main() {
	flag.Parse()
	if *optimized {
		optimize()
	}

	l, tokens := NewLex(*printTokens)
	nodes := NewTree(tokens)
	wg := sync.WaitGroup{}
	wg.Add(1)
	go func() {
		Exec(nodes, *printAst, *printLLVMIR)
		wg.Done()
	}()
	// handle files
	for _, fn := range flag.Args() {
		f, err := os.Open(fn)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(-1)
		}
		l.AddFile(f)
	}

	// handle stdin
	l.AddFile(os.Stdin)
	l.Stop()<commit_msg>Added non-interactive mode<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	"sync"
)

var (
	batch       = flag.Bool("b", false, "batch (non-interactive) mode")
	optimized   = flag.Bool("opt", true, "add some optimization passes")
	printTokens = flag.Bool("tok", false, "print tokens")
	printAst    = flag.Bool("ast", false, "print abstract syntax tree")
	printLLVMIR = flag.Bool("llvm", false, "print LLVM generated code")
)

func main() {
	flag.Parse()
	if *optimized {
		optimize()
	}

	l, tokens := NewLex(*printTokens)
	nodes := NewTree(tokens)
	wg := sync.WaitGroup{}
	wg.Add(1)
	go func() {
		Exec(nodes, *printAst, *printLLVMIR)
		wg.Done()
	}()

	// handle files
	for _, fn := range flag.Args() {
		f, err := os.Open(fn)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(-1)
		}
		l.AddFile(f)
	}

	// handle stdin
	if !*batch {
		l.AddFile(os.Stdin)
	}

	l.Stop()
{'fraction_non_alphanumeric': 0.1767497034400949, 'fraction_numerical': 0.002372479240806643, 'mean_word_length': 3.451187335092348, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1702282', 'n_tokens_mistral': 712, 'n_tokens_neox': 672, 'n_words': 157}