<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"os/exec"
	"strings"
)

func main() {
	args := append([]string{"test"}, os.Args[1:]...)

	cmd := exec.Command("go", args...)
	p, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error getting StdoutPipe from command: %v", err)
		os.Exit(1)
	}
	if err := cmd.Start(); err != nil {
		fmt.Fprintf(os.Stderr, "Error running command: %v", err)
		os.Exit(1)
	}
	scanner := bufio.NewScanner(p)
	for scanner.Scan() {
		if !strings.HasPrefix(string(scanner.Bytes()), "[LOG]") {
			fmt.Println(string(scanner.Bytes()))
		}
	}
	if scanner.Err() != nil {
		fmt.Fprintf(os.Stderr, "Error reading output of go test: %v", scanner.Err())
		os.Exit(1)
	}
}<commit_msg>add -f flag to write to file, and make sure we print out stderr as well<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"sync"
)

func main() {
	args := []string{}
	var f *os.File
	if len(os.Args) > 1 && os.Args[1] == "-f" {
		var err error
		f, err = os.OpenFile("tests.out", os.O_TRUNC|os.O_CREATE|os.O_RDWR, 0644)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error opening file tests.out: %v", err)
			os.Exit(1)
		}
		defer f.Close()
		args = append([]string{"test"}, os.Args[2:]...)
	} else {
		args = append([]string{"test"}, os.Args[1:]...)
	}

	cmd := exec.Command("go", args...)
	out, err := cmd.StdoutPipe()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error getting StdoutPipe from command: %v", err)
		os.Exit(1)
	}
	stderr, err := cmd.StderrPipe()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error getting StderrPipe from command: %v", err)
		os.Exit(1)
	}
	if err := cmd.Start(); err != nil {
		fmt.Fprintf(os.Stderr, "Error running command: %v", err)
		os.Exit(1)
	}
	wg := &sync.WaitGroup{}
	wg.Add(2)
	go printOut(f, out, wg)
	go printErr(f, stderr, wg)
	wg.Wait()
}

func printOut(f *os.File, r io.Reader, wg *sync.WaitGroup) {
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		s := string(scanner.Bytes())
		if f != nil {
			fmt.Fprintln(f, s)
		}
		if !strings.HasPrefix(s, "[LOG]") {
			fmt.Println(s)
		}
	}
	if scanner.Err() != nil {
		fmt.Fprintf(os.Stderr, "Error reading output of go test: %v", scanner.Err())
		os.Exit(1)
	}
	wg.Done()
}

func printErr(f *os.File, r io.Reader, wg *sync.WaitGroup) {
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		s := string(scanner.Bytes())
		if f != nil {
			fmt.Fprintln(f, s)
		}
		fmt.Fprintln(os.Stderr, s)
	}
	if scanner.Err() != nil {
		fmt.Fprintf(os.Stderr, "Error reading output of go test: %v", scanner.Err())
		os.Exit(1)
	}
	wg.Done()
}
{'fraction_non_alphanumeric': 0.19930609097918273, 'fraction_numerical': 0.007324595219737857, 'mean_word_length': 3.4207836456558773, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3323582', 'n_tokens_mistral': 1170, 'n_tokens_neox': 1095, 'n_words': 256}