<commit_before>package pfs

import (
	"hash/adler32"
	"path"
	"path/filepath"
	"strings"

	"github.com/pachyderm/pachyderm/src/client/pfs"
)

type Hasher struct {
	FileModulus  uint64
	BlockModulus uint64
}

func NewHasher(fileModulus uint64, blockModulus uint64) *Hasher {
	return &Hasher{
		FileModulus:  fileModulus,
		BlockModulus: blockModulus,
	}
}

// HashFile hashes the first part of the file path.  That is, for a path like
// foo/bar/buzz, HashFile only considers foo
func (s *Hasher) HashFile(file *pfs.File) uint64 {
	parts := strings.Split(path.Clean(file.Path), string(filepath.Separator))<commit_msg>Use a pachyderm specific separator<commit_after>package pfs

import (
	"hash/adler32"
	"path"
	"strings"

	"github.com/pachyderm/pachyderm/src/client/pfs"
)

type Hasher struct {
	FileModulus  uint64
	BlockModulus uint64
}

func NewHasher(fileModulus uint64, blockModulus uint64) *Hasher {
	return &Hasher{
		FileModulus:  fileModulus,
		BlockModulus: blockModulus,
	}
}

// HashFile hashes the first part of the file path.  That is, for a path like
// foo/bar/buzz, HashFile only considers foo
func (s *Hasher) HashFile(file *pfs.File) uint64 {
	parts := strings.Split(path.Clean(file.Path), "/")
{'fraction_non_alphanumeric': 0.10964550700741962, 'fraction_numerical': 0.01978565539983512, 'mean_word_length': 4.568807339449541, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '200687', 'n_tokens_mistral': 477, 'n_tokens_neox': 453, 'n_words': 113}