<commit_before><commit_msg>adding waitgroups<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"sync"

	"github.com/brnstz/routine/wikimg"
)

var (
	// print a blank line with the given 256 ANSI color
	fmtSpec = "\x1b[30;48;5;%dm%-80s\x1b[0m\n"
)

func main() {
	var max, workers, buffer int

	flag.IntVar(&max, "max", 10, "maximum number of images to retrieve")
	flag.IntVar(&workers, "workers", 5, "number of background workers")
	flag.IntVar(&buffer, "buffer", 0, "buffer size of image channel")
	flag.Parse()

	// Create a new image puller with our max
	p := wikimg.NewPuller(max)

	// Create a buffered channel for communicating between image
	// puller loop and workers
	imgURLs := make(chan string, buffer)

	wg := sync.WaitGroup{}

	for i := 0; i < workers; i++ {
		wg.Add(1)
		go func() {
			for imgURL := range imgURLs {
				// Get the top color in this image
				color, err := wikimg.OneColor(imgURL)
				if err != nil {
					log.Println(err)
					continue
				}

				// Print color to the terminal
				fmt.Printf(fmtSpec, color.XTermCode, "")
			}
			wg.Done()
		}()
	}

	// Loop to retrieve more images
	for {
		imgURL, err := p.Next()

		if err == wikimg.EndOfResults {
			// Break from loop when end of results is reached
			break

		} else if err != nil {
			// Log error and continue getting URLs
			log.Println(err)
			continue
		}

		imgURLs <- imgURL
	}
	close(imgURLs)
	wg.Wait()

}

{'fraction_non_alphanumeric': 0.1316348195329087, 'fraction_numerical': 0.013446567586694975, 'mean_word_length': 2.9607843137254903, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3323084', 'n_tokens_mistral': 590, 'n_tokens_neox': 506, 'n_words': 156}