<commit_before>package main

import (
	"bufio"
	"crypto"
	"crypto/hmac"
	"golang.org/x/crypto/blake2b"
	"hash"
	"io"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

import flag "github.com/spf13/pflag"

// Wrapper for the Blake2 New() methods that need an optional key for MAC
func blake2_(f func([]byte) (hash.Hash, error), key []byte) hash.Hash {
	h, err := f(key)
	if err != nil {
		logger.Fatal(err)
	}
	return h
}

func initHash(h *Checksum) {
	switch h.hash {
	case crypto.BLAKE2b_256:
		h.Hash = blake2_(blake2b.New256, macKey)
	case crypto.BLAKE2b_384:
		h.Hash = blake2_(blake2b.New384, macKey)
	case crypto.BLAKE2b_512:
		h.Hash = blake2_(blake2b.New512, macKey)
	default:
		if macKey != nil {
			h.Hash = hmac.New(h.hash.New, macKey)
		} else {
			h.Hash = h.hash.New()
		}
	}
}

func getChosen() []*Checksum {
	checksums := make([]*Checksum, len(chosen))
	for i := range chosen {
		checksums[i] = &Checksum{hash: chosen[i].hash}
		initHash(checksums[i])
	}
	return checksums
}

func hashF(f *os.File) (checksums []*Checksum) {
	var wg sync.WaitGroup
	var writers []io.Writer
	var pipeWriters []*io.PipeWriter

	checksums = getChosen()

	wg.Add(len(checksums))
	for _, h := range checksums {
		pr, pw := io.Pipe()
		writers = append(writers, pw)
		pipeWriters = append(pipeWriters, pw)
		go func(h *Checksum) {
			defer wg.Done()
			if _, err := io.Copy(h, pr); err != nil {
				logger.Print(err)
			}
			h.sum = h.Sum(nil)
		}(h)
	}

	wg.Add(1)
	go func() {
		defer wg.Done()
		defer func() {
			for _, pw := range pipeWriters {
				pw.Close()
			}
		}()
		// build the multiwriter for all the pipes
		mw := io.MultiWriter(writers...)
		// copy the data into the multiwriter
		if _, err := io.Copy(mw, f); err != nil {
			logger.Print(err)
		}
	}()
	wg.Wait()

	return
}

func hashFile(input *Input) *Results {
	file := input.file
	f, err := os.Open(file)
	if err != nil {
		stats.unreadable++
		if !opts.ignore {
			logger.Print(err)
		}
		return nil
	}
	defer f.Close()
	if !opts.recursive {
		info, err := f.Stat()
		if err != nil {
			logger.Print(err)
			return nil
		}
		if info.IsDir() {
			logger.Printf("%s is a directory and the -r option was not specified", file)
			return nil
		}
	}
	if opts.check != "\x00" {
		checksum := &Checksum{hash: input.hash}
		initHash(checksum)
		if _, err := io.Copy(checksum, f); err != nil {
			logger.Print(err)
			return nil
		}
		checksum.sum = checksum.Sum(nil)
		return &Results{
			file:      file,
			checksums: []*Checksum{checksum},
			check:     input.sum,
		}
	}
	return &Results{
		file:      file,
		checksums: hashF(f),
	}
}

func hashStdin() *Results {
	return &Results{
		file:      "",
		checksums: hashF(os.Stdin),
	}
}<commit_msg>Do not create goroutines when hashing only one algorithm<commit_after>package main

import (
	"bufio"
	"crypto"
	"crypto/hmac"
	"golang.org/x/crypto/blake2b"
	"hash"
	"io"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

import flag "github.com/spf13/pflag"

// Wrapper for the Blake2 New() methods that need an optional key for MAC
func blake2_(f func([]byte) (hash.Hash, error), key []byte) hash.Hash {
	h, err := f(key)
	if err != nil {
		logger.Fatal(err)
	}
	return h
}

func initHash(h *Checksum) {
	switch h.hash {
	case crypto.BLAKE2b_256:
		h.Hash = blake2_(blake2b.New256, macKey)
	case crypto.BLAKE2b_384:
		h.Hash = blake2_(blake2b.New384, macKey)
	case crypto.BLAKE2b_512:
		h.Hash = blake2_(blake2b.New512, macKey)
	default:
		if macKey != nil {
			h.Hash = hmac.New(h.hash.New, macKey)
		} else {
			h.Hash = h.hash.New()
		}
	}
}

func getChosen() []*Checksum {
	checksums := make([]*Checksum, len(chosen))
	for i := range chosen {
		checksums[i] = &Checksum{hash: chosen[i].hash}
		initHash(checksums[i])
	}
	return checksums
}

// Hash file with just one algorithm
func hashF1(f *os.File, hash crypto.Hash) (checksums []*Checksum) {
	checksum := &Checksum{hash: hash}
	initHash(checksum)
	if _, err := io.Copy(checksum, f); err != nil {
		logger.Print(err)
		return nil
	}
	checksum.sum = checksum.Sum(nil)
	return []*Checksum{checksum}
}

func hashF(f *os.File) (checksums []*Checksum) {
	if len(chosen) == 1 {
		return hashF1(f, chosen[0].hash)
	}

	var wg sync.WaitGroup
	var writers []io.Writer
	var pipeWriters []*io.PipeWriter
	checksums = getChosen()

	wg.Add(len(checksums))
	for _, h := range checksums {
		pr, pw := io.Pipe()
		writers = append(writers, pw)
		pipeWriters = append(pipeWriters, pw)
		go func(h *Checksum) {
			defer wg.Done()
			if _, err := io.Copy(h, pr); err != nil {
				logger.Print(err)
			}
			h.sum = h.Sum(nil)
		}(h)
	}

	wg.Add(1)
	go func() {
		defer wg.Done()
		defer func() {
			for _, pw := range pipeWriters {
				pw.Close()
			}
		}()
		// build the multiwriter for all the pipes
		mw := io.MultiWriter(writers...)
		// copy the data into the multiwriter
		if _, err := io.Copy(mw, f); err != nil {
			logger.Print(err)
		}
	}()
	wg.Wait()

	return
}

func hashFile(input *Input) *Results {
	file := input.file
	f, err := os.Open(file)
	if err != nil {
		stats.unreadable++
		if !opts.ignore {
			logger.Print(err)
		}
		return nil
	}
	defer f.Close()
	if !opts.recursive {
		info, err := f.Stat()
		if err != nil {
			logger.Print(err)
			return nil
		}
		if info.IsDir() {
			logger.Printf("%s is a directory and the -r option was not specified", file)
			return nil
		}
	}
	if opts.check != "\x00" {
		return &Results{
			file:      file,
			checksums: hashF1(f, input.hash),
			check:     input.sum,
		}
	}
	return &Results{
		file:      file,
		checksums: hashF(f),
	}
}

func hashStdin() *Results {
	return &Results{
		file:      "",
		checksums: hashF(os.Stdin),
	}
}
{'fraction_non_alphanumeric': 0.15784816846575828, 'fraction_numerical': 0.013271987258892231, 'mean_word_length': 3.12253829321663, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5334076', 'n_tokens_mistral': 2560, 'n_tokens_neox': 2322, 'n_words': 552}