<commit_before>package bagutil

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"errors"
	"fmt"
	"hash"
	"io/ioutil"
	"os"
)

// Takes a filepath as a string and produces a checksum.
func FileChecksum(filepath string, algo string) string {
	hsh, err := newHash(algo)
	if err != nil {
		panic(err)
	}
	file, err := os.Open(filepath)
	if err != nil {
		panic(err)
	}

	fileBytes, err := ioutil.ReadAll(file)
	if err != nil {
		panic(err)
	}
	hsh.Write(fileBytes)
	byteSum := hsh.Sum(nil)
	return fmt.Sprintf("%x", byteSum) // Convert to base16 on formatting.
}

func newHash(algo string) (hash.Hash, error) {
	switch algo {
	case "md5":
		return md5.New(), nil
	case "sha1":
		return sha1.New(), nil
	case "sha256":
		return sha256.New(), nil
	case "sha512":
		return sha512.New(), nil
	}
	return nil, errors.New("Unsupported hash value.")
}<commit_msg>Update to stream bytes to checksum algo.<commit_after>package bagutil

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"errors"
	"fmt"
	"hash"
	"io"
	"os"
)

// Takes a filepath as a string and produces a checksum.
func FileChecksum(filepath string, algo string) string {
	hsh, err := newHash(algo)
	if err != nil {
		panic(err)
	}
	file, err := os.Open(filepath)
	if err != nil {
		panic(err)
	}

	_, err = io.Copy(hsh, file)
	if err != nil {
		panic(err)
	}
	byteSum := hsh.Sum(nil)
	return fmt.Sprintf("%x", byteSum) // Convert to base16 on formatting.
}

func newHash(algo string) (hash.Hash, error) {
	switch algo {
	case "md5":
		return md5.New(), nil
	case "sha1":
		return sha1.New(), nil
	case "sha256":
		return sha256.New(), nil
	case "sha512":
		return sha512.New(), nil
	}
	return nil, errors.New("Unsupported hash value.")
}
{'fraction_non_alphanumeric': 0.15238642898217367, 'fraction_numerical': 0.029902242668200116, 'mean_word_length': 3.6900269541778976, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6670243', 'n_tokens_mistral': 736, 'n_tokens_neox': 694, 'n_words': 189}