<commit_before>package packer

import (
	"io"
	"sync"

	"github.com/cheggaaa/pb"
)

// ProgressBar allows to graphically display
// a self refreshing progress bar.
type ProgressBar interface {
	Start(total int64)
	Add(current int64)
	NewProxyReader(r io.Reader) (proxy io.Reader)
	Finish()
}

// StackableProgressBar is a progress bar pool that
// allows to track multiple advencments at once,
// by displaying multiple bars.
type StackableProgressBar struct {
	mtx sync.Mutex // locks in Start & Finish

	pool *pb.Pool
	bars []*BasicProgressBar

	wg sync.WaitGroup
}

func (spb *StackableProgressBar) cleanup() {
	spb.wg.Wait()

	spb.mtx.Lock()
	defer spb.mtx.Unlock()

	spb.pool.Stop()
	spb.pool = nil
	spb.bars = nil
}

func (spb *StackableProgressBar) New(identifier string) ProgressBar {
	spb.mtx.Lock()
	spb.wg.Add(1)
	defer spb.mtx.Unlock()

	if spb.pool == nil {
		spb.pool = pb.NewPool()
		go spb.cleanup()
	}

	bar := NewProgressBar(identifier)
	bar.Prefix(identifier)
	bar.finishCb = spb.wg.Done
	spb.bars = append(spb.bars, bar)

	spb.pool.Add(bar.ProgressBar)
	spb.pool.Start()
	return bar
}

// BasicProgressBar is packer's basic progress bar.
// Current implementation will always try to keep
// itself at the bottom of a terminal.
type BasicProgressBar struct {
	*pb.ProgressBar
	finishCb func()
}
<commit_msg>start the pool only once<commit_after>package packer

import (
	"io"
	"sync"

	"github.com/cheggaaa/pb"
)

// ProgressBar allows to graphically display
// a self refreshing progress bar.
type ProgressBar interface {
	Start(total int64)
	Add(current int64)
	NewProxyReader(r io.Reader) (proxy io.Reader)
	Finish()
}

// StackableProgressBar is a progress bar pool that
// allows to track multiple advencments at once,
// by displaying multiple bars.
type StackableProgressBar struct {
	mtx sync.Mutex // locks in Start & Finish

	pool *pb.Pool
	bars []*BasicProgressBar

	wg sync.WaitGroup
}

func (spb *StackableProgressBar) cleanup() {
	spb.wg.Wait()

	spb.mtx.Lock()
	defer spb.mtx.Unlock()

	spb.pool.Stop()
	spb.pool = nil
	spb.bars = nil
}

func (spb *StackableProgressBar) New(identifier string) ProgressBar {
	spb.mtx.Lock()
	spb.wg.Add(1)
	defer spb.mtx.Unlock()

	start := false
	if spb.pool == nil {
		spb.pool = pb.NewPool()
		go spb.cleanup()
		start = true
	}

	bar := NewProgressBar(identifier)
	bar.Prefix(identifier)
	bar.finishCb = spb.wg.Done
	spb.bars = append(spb.bars, bar)

	spb.pool.Add(bar.ProgressBar)
	if start {
		spb.pool.Start()
	}
	return bar
}

// BasicProgressBar is packer's basic progress bar.
// Current implementation will always try to keep
// itself at the bottom of a terminal.
type BasicProgressBar struct {
	*pb.ProgressBar
	finishCb func()
}

{'fraction_non_alphanumeric': 0.11476014760147601, 'fraction_numerical': 0.0036900369003690036, 'mean_word_length': 3.8152753108348136, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1318784', 'n_tokens_mistral': 1011, 'n_tokens_neox': 972, 'n_words': 268}