<commit_before>package main

import(
	"fmt"
	"math/rand"
)

type job struct {
	message int
}

// a single <-f is to trigger a recieve, no code after
//this will execute before this is done.


func worker(id int, input chan job, output chan job) {
	job := <-input
	fmt.Println("worker executing : ", job.message )
	output <- job
}

func main() {
	jobChannel := make(chan job)
	finishedJobsChanel := make(chan job)
	numJobs := 10
	jobs := createJobs(numJobs)

	for i := 0; i < numJobs; i++ {
		go worker(i, jobChannel, finishedJobsChanel)
	}
	for i := 0; i < numJobs; i++ {
		jobChannel <- jobs[i]
	}
	close(jobChannel)

}

func createJobs(numJobs int) []job {
	jobs := []job{}
	for i := 0; i < numJobs; i++ {
		jobs = append(jobs, job{message: rand.Intn(500)})
	}
	return jobs
}<commit_msg>make the worker actually execute functions as jobs<commit_after>package main

import(
	"fmt"
)


type job struct {
	fn func(string, string) string
	from, to string
}

// a single <-f is to trigger a recieve, no code after
//this will execute before this is done.

//worker to perform the task its given
func worker(id int, incoming chan job, outGoing chan job) {
	//assign the incoming job to the job variable
	job := <-incoming
	//execute the job
	job.fn(job.from, job.to)
	//push the result of job to the outGoing channel
	outGoing <- job
}

func main() {
	//channel to push new jobs onto
	jobChannel := make(chan job)
	//channel to push done jobs restults onto
	finishedJobsChanel := make(chan job)
	//amount of jobs to be done
	numJobs := 2000
	//create the jobs
	jobs := createJobs(numJobs)

	//instantiate the workers
	//because they run on channels, they
	//are blocking until a job comes over 
	//the channel
	for i := 0; i < numJobs; i++ {
		go worker(i, jobChannel, finishedJobsChanel)
	}
	//push all jobs onto the job channel
	for i := 0; i < numJobs; i++ {
		jobChannel <- jobs[i]
	}
	//close the job channel when done
	close(jobChannel)

}

//helper function to generate n amount of jobs
func createJobs(numJobs int) []job {
	jobs := []job{}
	for i := 0; i < numJobs; i++ {
		jobs = append(jobs, job{doThing, "a", "b"})
	}
	return jobs
}

//job function to be performed
func doThing(f string, t string) string{
	fmt.Println("moving file from ", f, "to", t)
	return t
}
{'fraction_non_alphanumeric': 0.12466960352422908, 'fraction_numerical': 0.006607929515418502, 'mean_word_length': 3.426900584795322, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1898422', 'n_tokens_mistral': 849, 'n_tokens_neox': 793, 'n_words': 304}