<commit_before>package static

import (
	"net/http"
	"os"
	"strings"
	"sync"
)

func Build(o Options, h http.Handler, paths []string, eh EventHandler) {
	var wg sync.WaitGroup

	pathsChan := make(chan string)

	for i := 0; i < o.Concurrency; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			buildPaths(o, h, pathsChan, eh)
		}()
	}

	for _, path := range paths {
		pathsChan <- path
	}

	close(pathsChan)

	wg.Wait()
}

func buildPaths(o Options, h http.Handler, paths <-chan string, eh EventHandler) {
	for path := range paths {
		err := BuildSingle(o, h, path)
		if eh != nil {
			eh(Event{Action: "build", Path: path, Error: err})
		}
	}
}

func BuildSingle(o Options, h http.Handler, path string) error {
	_, err := os.Stat(o.OutputDir)
	if os.IsNotExist(err) {
		err = os.MkdirAll(o.OutputDir, 0755)
	}
	if err != nil {
		return err
	}

	fp := o.OutputDir + path
	if strings.HasSuffix(fp, "/") {
		fp += o.DirFilename
	}

	f, err := os.Create(fp)
	if err != nil {
		return err
	}
	defer f.Close()

	r, err := http.NewRequest("GET", path, nil)
	if err != nil {
		return err
	}
	rw := newResponseWriter(f)<commit_msg>Added inline documentation.<commit_after>package static

import (
	"net/http"
	"os"
	"strings"
	"sync"
)

// Build the paths. Uses the http.Handler to get the response for each path, and writes that response to a file with it's respective path in the OutputDir specified in the Options. Does so concurrently as defined in the Options, and calls the EventHandler for every path with an Event that states that the path was built and if an error occurred.
func Build(o Options, h http.Handler, paths []string, eh EventHandler) {
	var wg sync.WaitGroup

	pathsChan := make(chan string)

	for i := 0; i < o.Concurrency; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			buildWorker(o, h, pathsChan, eh)
		}()
	}

	for _, path := range paths {
		pathsChan <- path
	}

	close(pathsChan)

	wg.Wait()
}

func buildWorker(o Options, h http.Handler, paths <-chan string, eh EventHandler) {
	for path := range paths {
		err := BuildSingle(o, h, path)
		if eh != nil {
			eh(Event{Action: "build", Path: path, Error: err})
		}
	}
}

// Build a single path. Uses the http.Handler to get the response for each path, and writes that response to a file with it's respective path in the OutputDir specified in the Options. Returns an error if one occurs.
func BuildSingle(o Options, h http.Handler, path string) error {
	_, err := os.Stat(o.OutputDir)
	if os.IsNotExist(err) {
		err = os.MkdirAll(o.OutputDir, 0755)
	}
	if err != nil {
		return err
	}

	fp := o.OutputDir + path
	if strings.HasSuffix(fp, "/") {
		fp += o.DirFilename
	}

	f, err := os.Create(fp)
	if err != nil {
		return err
	}
	defer f.Close()

	r, err := http.NewRequest("GET", path, nil)
	if err != nil {
		return err
	}
	rw := newResponseWriter(f)
{'fraction_non_alphanumeric': 0.1321554770318021, 'fraction_numerical': 0.004240282685512367, 'mean_word_length': 3.348694316436252, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2687117', 'n_tokens_mistral': 1098, 'n_tokens_neox': 1008, 'n_words': 342}