<commit_before>package main

import (
	"encoding/csv"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"strconv"
	"sync"
	"time"
)

var numberStarted int
var numberSaved int
var numberFailed int
var numberDropped int

var blocks []Block
var tracts []Tract
var counties []County

var Config configuration
var BlocksToTracts map[string][]int
var TractsToCounties map[string][]int

func main() {
	var configAddress = flag.String("configAddress", "./config.json", "Configuration File")

	Config = importConfig(*configAddress)

	var err error
	counties, tracts, blocks, err = getCensusData()
	if err != nil {
		panic(err)
	}
	fmt.Printf("Found %v couties\nFound %v tracts\nFound %v blocks", len(counties), len(tracts), len(blocks))

	BlocksToTracts = mapBlocksToTracts()
	fmt.Printf("\nMapped %v tracts\n", len(BlocksToTracts))

	TractsToCounties = mapTractsToCounties()
	fmt.Printf("\nMapped %v counties\n", len(TractsToCounties))

	fmt.Printf("Beginning Labelling ")
	fmt.Printf("Using %v max processors.\n", runtime.NumCPU())
	fmt.Printf("Using %v labeling routines.\n", Config.LabelingRoutines)

	channelSize := 250
	//make our channels
	//Channel for unlabeled data to be channeled
	preChannel := make(chan []string, channelSize)
	//Channel for labeled data to be saved
	postChannel := make(chan []string, channelSize)
	//channel to save dropped records for later
	droppedChannel := make(chan []string, 25)
	//channel to save failed records for later
	failedChannel := make(chan []string, 25)

	var wg sync.WaitGroup
	wg.Add(1)

	go readIntoChannel(preChannel, postChannel)
	go finishAndSave(postChannel, wg)
	go saveFailed(failedChannel)
	go saveDropped(droppedChannel)

	for i := 0; i < 100; i++ {
		go labeler(preChannel, postChannel, failedChannel, droppedChannel)
	}

	oldStarted := 0
	oldSaved := 0
	oldDropped := 0
	oldFailed := 0

	for {
		time.Sleep(time.Second * 5)
		log.Printf("%v rows started.", numberStarted)<commit_msg>set to use config file for number of labeling routines<commit_after>package main

import (
	"encoding/csv"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"strconv"
	"sync"
	"time"
)

var numberStarted int
var numberSaved int
var numberFailed int
var numberDropped int

var blocks []Block
var tracts []Tract
var counties []County

var Config configuration
var BlocksToTracts map[string][]int
var TractsToCounties map[string][]int

func main() {
	var configAddress = flag.String("configAddress", "./config.json", "Configuration File")

	Config = importConfig(*configAddress)

	var err error
	counties, tracts, blocks, err = getCensusData()
	if err != nil {
		panic(err)
	}
	fmt.Printf("Found %v couties\nFound %v tracts\nFound %v blocks", len(counties), len(tracts), len(blocks))

	BlocksToTracts = mapBlocksToTracts()
	fmt.Printf("\nMapped %v tracts\n", len(BlocksToTracts))

	TractsToCounties = mapTractsToCounties()
	fmt.Printf("\nMapped %v counties\n", len(TractsToCounties))

	fmt.Printf("Beginning Labelling ")
	fmt.Printf("Using %v max processors.\n", runtime.NumCPU())
	fmt.Printf("Using %v labeling routines.\n", Config.LabelingRoutines)

	channelSize := 250
	//make our channels
	//Channel for unlabeled data to be channeled
	preChannel := make(chan []string, channelSize)
	//Channel for labeled data to be saved
	postChannel := make(chan []string, channelSize)
	//channel to save dropped records for later
	droppedChannel := make(chan []string, 25)
	//channel to save failed records for later
	failedChannel := make(chan []string, 25)

	var wg sync.WaitGroup
	wg.Add(1)

	go readIntoChannel(preChannel, postChannel)
	go finishAndSave(postChannel, wg)
	go saveFailed(failedChannel)
	go saveDropped(droppedChannel)

	for i := 0; i < Config.LabelingRoutines; i++ {
		go labeler(preChannel, postChannel, failedChannel, droppedChannel)
	}

	oldStarted := 0
	oldSaved := 0
	oldDropped := 0
	oldFailed := 0

	for {
		time.Sleep(time.Second * 5)
		log.Printf("%v rows started.", numberStarted)
{'fraction_non_alphanumeric': 0.12192648922686945, 'fraction_numerical': 0.007858048162230671, 'mean_word_length': 4.951734539969834, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2707464', 'n_tokens_mistral': 1421, 'n_tokens_neox': 1339, 'n_words': 386}