<commit_before>// csvsplit: Split a .csv into multiple files.
package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
)

var (
	flagRecords = flag.Int("records", 0, "The number of records per file")
	flagInput   = flag.String("input", "", "Filename of the input file to split (if blank, uses stdin)")
	flagOutput  = flag.String("output", "", "filename / path of the file output (optional)")
	flagHeaders = flag.Int("headers", 0, "Number of header lines in the input file (will be repeated in each output file")
)

func main() {
	flag.Parse()

	// Sanity check flags
	if *flagRecords < 1 || *flagHeaders < 0 || *flagHeaders >= *flagRecords {
		flag.Usage()
		os.Exit(1)
	}

	// Get input from a given file or stdin
	var r *csv.Reader
	if *flagInput != "" {
		f, err := os.Open(*flagInput)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		r = csv.NewReader(f)
	} else {
		r = csv.NewReader(os.Stdin)
	}

	recs := make([][]string, 0)
	count := 1
	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		recs = append(recs, record)
		if len(recs) == *flagRecords {
			saveCSVFile(recs, count)
			// Reset records to include just the header lines (if any)
			recs = recs[:*flagHeaders]
			count += 1
		}
	}
	if len(recs) > 0 {
		saveCSVFile(recs, count)
	}
}

func saveCSVFile(r [][]string, c int) {
	name := fmt.Sprintf("%v%03d%v", *flagOutput, c, ".csv")

	// Make sure we don't overwrite existing files
	if _, err := os.Stat(name); err == nil {
		log.Fatal("File exists: ", name)
	}

	f, err := os.Create(name)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	w := csv.NewWriter(f)
	w.WriteAll(r)
}<commit_msg>Pass by reference to minimize memory footprint<commit_after>// csvsplit: Split a .csv into multiple files.
package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
)

var (
	flagRecords = flag.Int("records", 0, "The number of records per file")
	flagInput   = flag.String("input", "", "Filename of the input file to split (if blank, uses stdin)")
	flagOutput  = flag.String("output", "", "filename / path of the file output (optional)")
	flagHeaders = flag.Int("headers", 0, "Number of header lines in the input file (will be repeated in each output file")
)

func main() {
	flag.Parse()

	// Sanity check flags
	if *flagRecords < 1 || *flagHeaders < 0 || *flagHeaders >= *flagRecords {
		flag.Usage()
		os.Exit(1)
	}

	// Get input from a given file or stdin
	var r *csv.Reader
	if *flagInput != "" {
		f, err := os.Open(*flagInput)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		r = csv.NewReader(f)
	} else {
		r = csv.NewReader(os.Stdin)
	}

	recs := make([][]string, 0)
	count := 1
	for {
		record, err := r.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			log.Fatal(err)
		}

		recs = append(recs, record)
		if len(recs) == *flagRecords {
			saveCSVFile(&recs, count)
			// Reset records to include just the header lines (if any)
			recs = recs[:*flagHeaders]
			count += 1
		}
	}
	if len(recs) > 0 {
		saveCSVFile(&recs, count)
	}
}

func saveCSVFile(recs *[][]string, c int) {
	name := fmt.Sprintf("%v%03d%v", *flagOutput, c, ".csv")

	// Make sure we don't overwrite existing files
	if _, err := os.Stat(name); err == nil {
		log.Fatal("File exists: ", name)
	}

	f, err := os.Create(name)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	w := csv.NewWriter(f)
	w.WriteAll(*recs)
}
{'fraction_non_alphanumeric': 0.16332461493751815, 'fraction_numerical': 0.006393490264458007, 'mean_word_length': 3.202686202686203, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '384399', 'n_tokens_mistral': 1411, 'n_tokens_neox': 1290, 'n_words': 388}