<commit_before>package main

import (
	"runtime"
	"sync"

	"github.com/ikawaha/kagome/tokenizer"
)

// NewMorphizer provides new Morphizer.
func NewMorphizer() (<-chan Morphs, chan<- string, func()) {
	n := runtime.GOMAXPROCS(0)
	wg := new(sync.WaitGroup)

	out := make(chan Morphs)
	in := make(chan string)

	sign := make(chan struct{})
	stop := func() {
		close(sign)
		for range out {
		}
	}

	// kagome is NOT goroutine safe
	for i := 0; i < n; i++ {
		wg.Add(1)
		go func() {
			kagome := tokenizer.New()
			for {
				select {
				// interrupt
				case <-sign:
					wg.Done()
					return

				// parse
				case str, ok := <-in:
					if !ok {
						wg.Done()
						return
					}
					tokens := kagome.Tokenize(str)
					out <- NewMorphs(tokens)
				}
			}
		}()
	}

	// tell finishing
	go func() {
		wg.Wait()
		close(out)
	}()

	return out, in, stop
}
<commit_msg>deffering wg.Done() in a func.<commit_after>package main

import (
	"runtime"
	"sync"

	"github.com/ikawaha/kagome/tokenizer"
)

// NewMorphizer provides new Morphizer.
func NewMorphizer() (<-chan Morphs, chan<- string, func()) {
	n := runtime.GOMAXPROCS(0)
	wg := new(sync.WaitGroup)

	out := make(chan Morphs)
	in := make(chan string)

	sign := make(chan struct{})
	stop := func() {
		close(sign)
		for range out {
		}
	}

	// kagome is NOT goroutine safe
	for i := 0; i < n; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			kagome := tokenizer.New()
			for {
				select {
				// interrupt
				case <-sign:
					return

				// parse
				case str, ok := <-in:
					if !ok {
						return
					}
					tokens := kagome.Tokenize(str)
					out <- NewMorphs(tokens)
				}
			}
		}()
	}

	// tell finishing
	go func() {
		wg.Wait()
		close(out)
	}()

	return out, in, stop
}

{'fraction_non_alphanumeric': 0.16227064220183487, 'fraction_numerical': 0.0034403669724770644, 'mean_word_length': 2.3111954459203035, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3975883', 'n_tokens_mistral': 851, 'n_tokens_neox': 684, 'n_words': 168}