<commit_before>package template

import (
	"bytes"
	"context"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/ironman-project/ironman/template/engine"
	"github.com/ironman-project/ironman/template/engine/goengine"
	"github.com/ironman-project/ironman/template/model"
	"github.com/ironman-project/ironman/template/values"
	"github.com/pkg/errors"
)

//GeneratorData represents the data to be passed to each generator file template
type GeneratorData struct {
	Template  *model.Template
	Generator *model.Generator
	Values    values.Values
}

var _ Generator = (*generator)(nil)

//Generator defines a template generator
type Generator interface {
	Generate(context context.Context) error
}

type generator struct {
	path           string
	generationPath string
	ignore         []string
	data           GeneratorData
	engine         engine.Factory
	logger         *log.Logger
	force          bool
}

//NewGenerator returns a new instance of a generator
func NewGenerator(path string, generationPath string, data GeneratorData, options ...GeneratorOption) Generator {

	g := &generator{
		path:           path,
		generationPath: generationPath,
		data:           data,
		ignore:         []string{".ironman.yaml"},
		engine: func() engine.Engine {
			return goengine.New("ironman")
		},
		logger: log.New(os.Stdout, "", 0),
		force:  false,
	}

	for _, option := range options {
		option(g)
	}

	return g
}

type processResult struct {
	bytes      []byte
	pathResult pathResult
	err        error
}

type writeResult struct {
	pathFrom string
	pathTo   string
	err      error
}

type pathResult struct {
	path  string
	isDir bool
}

func (g *generator) Generate(ctx context.Context) error {

	childCtx, cancelFunc := context.WithCancel(ctx)
	defer cancelFunc()

	paths, errc := g.walkFiles(childCtx)

	presults := make(chan processResult)

	workersExecute(1, func(w int, wg *sync.WaitGroup) {
		g.processor(childCtx, paths, presults)
		wg.Done()
	}, func() {
		close(presults)

	})

	wresults := make(chan writeResult)
	workersExecute(1, func(w int, wg *sync.WaitGroup) {
		g.write(childCtx, presults, wresults)
		wg.Done()
	},
		func() {
			close(wresults)

		},
	)

	for wresult := range wresults {
		if wresult.err != nil {
			cancelFunc()
			return wresult.err
		}
	}

	err := <-errc

	if err != nil {
		return errors.Wrapf(err, "Failed to process generator path templates: %s", g.path)
	}

	return nil
}
<commit_msg>make number of  generate workers a constant<commit_after>package template

import (
	"bytes"
	"context"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/ironman-project/ironman/template/engine"
	"github.com/ironman-project/ironman/template/engine/goengine"
	"github.com/ironman-project/ironman/template/model"
	"github.com/ironman-project/ironman/template/values"
	"github.com/pkg/errors"
)

//arbitrary number
const noGeneratorWorkers = 20

//GeneratorData represents the data to be passed to each generator file template
type GeneratorData struct {
	Template  *model.Template
	Generator *model.Generator
	Values    values.Values
}

var _ Generator = (*generator)(nil)

//Generator defines a template generator
type Generator interface {
	Generate(context context.Context) error
}

type generator struct {
	path           string
	generationPath string
	ignore         []string
	data           GeneratorData
	engine         engine.Factory
	logger         *log.Logger
	force          bool
}

//NewGenerator returns a new instance of a generator
func NewGenerator(path string, generationPath string, data GeneratorData, options ...GeneratorOption) Generator {

	g := &generator{
		path:           path,
		generationPath: generationPath,
		data:           data,
		ignore:         []string{".ironman.yaml"},
		engine: func() engine.Engine {
			return goengine.New("ironman")
		},
		logger: log.New(os.Stdout, "", 0),
		force:  false,
	}

	for _, option := range options {
		option(g)
	}

	return g
}

type processResult struct {
	bytes      []byte
	pathResult pathResult
	err        error
}

type writeResult struct {
	pathFrom string
	pathTo   string
	err      error
}

type pathResult struct {
	path  string
	isDir bool
}

func (g *generator) Generate(ctx context.Context) error {

	childCtx, cancelFunc := context.WithCancel(ctx)
	defer cancelFunc()

	paths, errc := g.walkFiles(childCtx)

	presults := make(chan processResult)

	workersExecute(noGeneratorWorkers, func(w int, wg *sync.WaitGroup) {
		g.processor(childCtx, paths, presults)
		wg.Done()
	}, func() {
		close(presults)

	})

	wresults := make(chan writeResult)
	workersExecute(noGeneratorWorkers, func(w int, wg *sync.WaitGroup) {
		g.write(childCtx, presults, wresults)
		wg.Done()
	},
		func() {
			close(wresults)

		},
	)

	for wresult := range wresults {
		if wresult.err != nil {
			cancelFunc()
			return wresult.err
		}
	}

	err := <-errc

	if err != nil {
		return errors.Wrapf(err, "Failed to process generator path templates: %s", g.path)
	}

	return nil
}

{'fraction_non_alphanumeric': 0.10975609756097561, 'fraction_numerical': 0.0011995201919232307, 'mean_word_length': 3.4079295154185023, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1267799', 'n_tokens_mistral': 1721, 'n_tokens_neox': 1623, 'n_words': 425}