<commit_before>package getter

import (
	"io"

	pb "gopkg.in/cheggaaa/pb.v1"
)

// WithCheggaaaProgressBarV1 will have the downloads tracked
// by a github.com/cheggaaa/pb v2 multi progress bar.
func WithCheggaaaProgressBarV1() func(*Client) error {
	return func(c *Client) error {
		c.ProgressListener = cheggaaaProgressBar
		return nil
	}
}

// CheggaaaProgressBar just wraps
// a pb.Pool to display a progress
type CheggaaaProgressBar struct {
	pool *pb.Pool

	pbs int
}

var cheggaaaProgressBar ProgressTracker = &CheggaaaProgressBar{}

func defaultCheggaaaProgressBarConfigFN(bar *pb.ProgressBar, format string) {
	bar.SetUnits(pb.U_BYTES)
	bar.Format(format)
}

// TrackProgress instantiates a new progress bar that will
// display the progress of stream until closed.
// total can be 0.
func (cpb *CheggaaaProgressBar) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) io.ReadCloser {
	newPb := pb.New64(totalSize)
	newPb.Set64(currentSize)
	defaultCheggaaaProgressBarConfigFN(newPb, src)
	if cpb.pool == nil {
		cpb.pool = pb.NewPool()
		cpb.pool.Start()
	}
	cpb.pool.Add(newPb)
	reader := newPb.NewProxyReader(stream)

	cpb.pbs++
	return &cheggaaaReadCloser{
		Reader: reader,
		close: func() error {
			cpb.pbs--
			if cpb.pbs <= 0 {
				cpb.pool.Stop()
			}
			newPb.Finish()
			return nil
		},
	}
}

type cheggaaaReadCloser struct {
	io.Reader
	close func() error
}
<commit_msg>progress bar: display filename using filepath.Base(src)<commit_after>package getter

import (
	"io"
	"path/filepath"

	pb "gopkg.in/cheggaaa/pb.v1"
)

// WithCheggaaaProgressBarV1 will have the downloads tracked
// by a github.com/cheggaaa/pb v2 multi progress bar.
func WithCheggaaaProgressBarV1() func(*Client) error {
	return func(c *Client) error {
		c.ProgressListener = cheggaaaProgressBar
		return nil
	}
}

// CheggaaaProgressBar just wraps
// a pb.Pool to display a progress
type CheggaaaProgressBar struct {
	pool *pb.Pool

	pbs int
}

var cheggaaaProgressBar ProgressTracker = &CheggaaaProgressBar{}

func defaultCheggaaaProgressBarConfigFN(bar *pb.ProgressBar, prefix string) {
	bar.SetUnits(pb.U_BYTES)
	bar.Prefix(prefix)
}

// TrackProgress instantiates a new progress bar that will
// display the progress of stream until closed.
// total can be 0.
func (cpb *CheggaaaProgressBar) TrackProgress(src string, currentSize, totalSize int64, stream io.ReadCloser) io.ReadCloser {
	newPb := pb.New64(totalSize)
	newPb.Set64(currentSize)
	defaultCheggaaaProgressBarConfigFN(newPb, filepath.Base(src))
	if cpb.pool == nil {
		cpb.pool = pb.NewPool()
		cpb.pool.Start()
	}
	cpb.pool.Add(newPb)
	reader := newPb.NewProxyReader(stream)

	cpb.pbs++
	return &cheggaaaReadCloser{
		Reader: reader,
		close: func() error {
			cpb.pbs--
			if cpb.pbs <= 0 {
				cpb.pool.Stop()
			}
			newPb.Finish()
			return nil
		},
	}
}

type cheggaaaReadCloser struct {
	io.Reader
	close func() error
}

{'fraction_non_alphanumeric': 0.10652920962199312, 'fraction_numerical': 0.008247422680412371, 'mean_word_length': 4.188948306595365, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7130199', 'n_tokens_mistral': 1152, 'n_tokens_neox': 1050, 'n_words': 256}