<commit_before>// Reading and writing files are basic tasks needed for
// many Go programs. First we'll look at some examples of
// reading files.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
)

// Reading files requires checking most calls for errors.
// This helper will streamline our error checks below.
func check(e error) {
	if e != nil {
		panic(e)
	}
}

func main() {

	inFile := os.Args[1]
	outFile := flag.String("outFile", inFile+".out", "output file for list")

	fmt.Printf("prog<%s> in<%s> out<%s>\n", os.Args[0], os.Args[1], *outFile)
	os.Exit(0)

	// Perhaps the most basic file reading task is
	// slurping a file's entire contents into memory.<commit_msg>format<commit_after>// Reading and writing files are basic tasks needed for
// many Go programs. First we'll look at some examples of
// reading files.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
)

// Reading files requires checking most calls for errors.
// This helper will streamline our error checks below.
func check(e error) {
	if e != nil {
		panic(e)
	}
}

func main() {

	inFile := os.Args[1]
	outFile := flag.String("outFile", inFile+".out", "output file for list")

	fmt.Printf("prog<%s>\n", os.Args[0])
	fmt.Printf("    in(%s)    out(%s)\n", os.Args[1], *outFile)
	os.Exit(0)

	// Perhaps the most basic file reading task is
	// slurping a file's entire contents into memory.
{'fraction_non_alphanumeric': 0.14537755822159493, 'fraction_numerical': 0.0056457304163726185, 'mean_word_length': 3.473186119873817, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7003942', 'n_tokens_mistral': 512, 'n_tokens_neox': 496, 'n_words': 174}