<commit_before>package main

import (
	"fmt"

	"time"
)

// WORKERS AND CORES
const numberWorkers = 20
const numberCores = 3

// OTHER
var channelBufferSize = 1 // How many channel buffers

// struct to pass in channel
type workerStats struct {
	id       int
	timeTook time.Duration
}

func doWork(msgCh chan *workerStats, id int) {

	timeStart := time.Now().UTC()
	time.Sleep(3 * time.Second)
	timeEnd := time.Now().UTC()
	timeTook := timeEnd.Sub(timeStart)

	msgCh <- &workerStats{
		id:       id,
		timeTook: timeTook,
	}

}

func getStats(msgCh chan *workerStats) {
	// Print out stats from each worker
	for r := range msgCh {
		fmt.Printf("From Worker %v, Took %v\n", r.id, r.timeTook)
	}
}

func main() {
	// START
	fmt.Println("Main start time", time.Since(start))

	// Make Channel Buffer
	msgCh := make(chan *workerStats, channelBufferSize)

	// Create Workers on Core
	for id := 1; id < numberWorkers+1; id++ {
		fmt.Println("Starting worker", id)
		go doWork(msgCh, id)

	}

	// Gather Stats from workers
	go getStats(msgCh)

	// END
	fmt.Println("Main end time", time.Since(start))

}<commit_msg>updating goroutine multi-core<commit_after>package main

import (
	"fmt"
	"runtime"
	"sync"

	"time"
)

// WORKERS AND CORES
const useGoroutine = true
const numberWorkers = 10
const timeWork = 1
const numberCores = 2

// OTHER
var channelBufferSize = numberWorkers + 1 // How many channel buffers

// struct to pass in channel
type workerStats struct {
	id       int
	timeTook time.Duration
}

func doWork(msgCh chan *workerStats, wg *sync.WaitGroup, id int) {

	timeStart := time.Now().UTC()
	time.Sleep(timeWork * time.Second)
	timeEnd := time.Now().UTC()
	timeTook := timeEnd.Sub(timeStart)

	msgCh <- &workerStats{
		id:       id,
		timeTook: timeTook,
	}
	wg.Done()

}

func getStats(msgCh chan *workerStats, wg *sync.WaitGroup, numberWorkers int) {

	// WAIT FOR ALL WORKERS TO FINISH
	fmt.Println("Waiting for all the workers to finish")
	fmt.Println("There are currently this many goroutines", runtime.NumGoroutine())
	wg.Wait()

	// Print out stats from each worker
	for i := 0; i < numberWorkers; i++ {
		r := <-msgCh
		fmt.Printf("From Worker %v, Took %v\n", r.id, r.timeTook)
	}

}

func main() {

	// START
	start := time.Now()

	// Create wait group
	var wg sync.WaitGroup

	// SET CORES
	runtime.GOMAXPROCS(numberCores)

	fmt.Println("Main start time", time.Since(start))

	// How many core amd i using
	fmt.Println("I have this many cores to use: ", runtime.NumCPU())

	// Make Channel Buffer
	msgCh := make(chan *workerStats, channelBufferSize)

	// Create Workers on Core
	for id := 1; id < numberWorkers+1; id++ {
		wg.Add(1)
		fmt.Println("Starting worker", id)
		if useGoroutine {
			go doWork(msgCh, &wg, id)
		} else {
			doWork(msgCh, &wg, id)
		}
	}

	// Gather Stats from workers
	getStats(msgCh, &wg, numberWorkers)

	// END
	fmt.Println("Main end time", time.Since(start))

}
{'fraction_non_alphanumeric': 0.13122638268636208, 'fraction_numerical': 0.005496392992098935, 'mean_word_length': 3.432267884322679, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1368875', 'n_tokens_mistral': 1130, 'n_tokens_neox': 1053, 'n_words': 309}