<commit_before>//A workerpool that can get expanded & shrink dynamically.
package workerpool

//License: See the LICENSE File.

import (
	"runtime"
	"time"
)

//Sample usage:
//	func main() {
//		jobChannel := make(chan workerpool.Job)
//
//		pool := workerpool.New(0, jobChannel)
//		pool.Run()
//
//		done := make(chan bool)
//		jobChannel <- func() {
//			log.Info("Job done!")
//			done <- true
//		}
//		<-done
//	}
type WorkerPool struct {
	workerPool chan chan Job
	JobChannel chan Job
	minWorkers int
}

//Starts the worker pool. Initial workers never timeout (well; they do; after a year of unemployment!) and never quit.
func (d *WorkerPool) Run() {
	var defaultConf WorkerConfig
	defaultConf.Timeout = time.Hour * 24 * 365

	for i := 0; i < d.minWorkers; i++ {
		worker := newWorker(d.workerPool, defaultConf)
		worker.Start()
	}

	go d.dispatch()
}

//Putting more 'Worker's into work. If there is'nt any job to do, and a timeout is set,
//they will simply get timed-out and worker pool will shrink to it's minimum size.<commit_msg>timeout for initial workers removed<commit_after>//A workerpool that can get expanded & shrink dynamically.
package workerpool

//License: See the LICENSE File.

import (
	"runtime"
	"time"
)

//Sample usage:
//	func main() {
//		jobChannel := make(chan workerpool.Job)
//
//		pool := workerpool.New(0, jobChannel)
//		pool.Run()
//
//		done := make(chan bool)
//		jobChannel <- func() {
//			log.Info("Job done!")
//			done <- true
//		}
//		<-done
//	}
type WorkerPool struct {
	workerPool chan chan Job
	JobChannel chan Job
	minWorkers int
}

//Starts the worker pool. Initial workers never timeout and never quit.
func (d *WorkerPool) Run() {
	var defaultConf WorkerConfig

	for i := 0; i < d.minWorkers; i++ {
		worker := newWorker(d.workerPool, defaultConf)
		worker.Start()
	}

	go d.dispatch()
}

//Putting more 'Worker's into work. If there is'nt any job to do, and a timeout is set,
//they will simply get timed-out and worker pool will shrink to it's minimum size.
{'fraction_non_alphanumeric': 0.1357107478949975, 'fraction_numerical': 0.004457652303120356, 'mean_word_length': 3.1478439425051334, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3044573', 'n_tokens_mistral': 722, 'n_tokens_neox': 678, 'n_words': 239}