<commit_before>// Copyright 2014 <NAME> (github.com/mohae) All rights reserved.
// Use of this source code is governed by a BSD-style license that
// can be found in the LICENSE file.
//
package synchronicity

import (
	"bufio"
	"crypto/sha256"
	"fmt"
	"hash"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
)

// This allows for up to 128k of read data. If the file is larger than that,
// a different approach should be done, i.e. don't precompute the hash and use
// other rules for determining difference.
//
var MaxChunks = 16       // Modify directly to change buffered hashes
var chunkSize = int64(8 * 1024) // use 8k chunks as default

// SetChunkSize sets the chunkSize as 1k * i, i.e. 8 == 8k chunkSize
// If the multiplier, i, is <= 0, the default is used, 8.
func SetChunkSize(i int) {
	if i <= 0 {
		i = 8
	}
	chunkSize = int64(1024 * i)
}

const (
	invalid hashType = iota
	SHA256
)

type hashType int // Not really needed atm, but it'll be handy for adding other types.
var useHashType hashType

// SHA256 sized for hashed blocks.
type Hash256 [32]byte

func (h hashType) String() string {
	switch h {
	case SHA256:
		return "sha256"
	case invalid:
		return "invalid"
	}
	return "unknown"
}

func init() {
	useHashType = SHA256
}

type FileData struct {
	Processed bool
	Hashes    []Hash256
	HashType  hashType
	ChunkSize int64 // The chunksize that this was created with.
	MaxChunks int
	CurByte   int64  // for when the while file hasn't been hashed and
	Root      string // the relative root of this file: allows for synch support
	Dir       string // relative path to parent directory of Fi
	Fi        os.FileInfo
}

// Returns a FileData struct for the passed file using the defaults.
// Set any overrides before performing an operation.
func NewFileData(root, dir string, fi os.FileInfo) FileData {
	if dir == "." {
		dir = ""
	}
	fd := FileData{HashType: useHashType, ChunkSize: chunkSize, MaxChunks: MaxChunks, Root: root, Dir: dir, Fi: fi}
	return fd
}

// String is an alias to RelPath
func (fd *FileData) String() string {
	return fd.RelPath()
}

// SetHash computes the hash of the FileData. The path of the file is passed
// because FileData only knows it's name, not its location.
func (fd *FileData) SetHash() error {
	f, hasher, err := fd.getFileHasher()
	if err != nil {
		return err
	}
	if fd.ChunkSize == 0 {
		return fd.hashFile(f, hasher)
	}
	return fd.chunkedHashFile(f, hasher)
}

// getHasher returns a file and the hasher to use it on. If error, return that.
func (fd *FileData) getFileHasher() (f *os.File, hasher hash.Hash, err error) {
	f, err = os.Open(fd.RootPath())
	if err != nil {
		return 
	}
	hasher, err = fd.getHasher()
	return
}

// getHasher returns a hasher or an error
func (fd *FileData) getHasher() (hasher hash.Hash, err error) {
	switch fd.HashType {
	case SHA256:
		hasher = sha256.New() //
	default:
		err = fmt.Errorf("%s hash type", fd.HashType.String())
		return
	}
	return
}

// hashFile hashes the entire file.
func (fd *FileData) hashFile(f *os.File, hasher hash.Hash) error {
	_, err := io.Copy(hasher, f)
	if err != nil {
		log.Printf("%s/n", err)
		return err
	}
	h := Hash256{}
	copy(h[:], hasher.Sum(nil))
	fd.Hashes = append(fd.Hashes, Hash256(h))
	return nil
}

// chunkedHashFile reads up to max chunks, or the entire file, whichever comes
// first. 
func (fd *FileData) chunkedHashFile(f *os.File, hasher hash.Hash) (err error) {
	reader := bufio.NewReaderSize(f, int(fd.ChunkSize))
	var cnt int
	var bytes int64
	h := Hash256{}
	for cnt = 0; cnt < MaxChunks; cnt++ { // read until EOF || MaxChunks
		n, err := io.CopyN(hasher, reader, int64(fd.ChunkSize))
		if err != nil && err != io.EOF {
			return err
		}
		bytes += n
		copy(h[:], hasher.Sum(nil))
		fd.Hashes = append(fd.Hashes, Hash256(h))		
	}
	if err != nil {
		log.Printf("%s\n", err)
		return err
	}
	_ = cnt
	return nil
}

// isEqual compares the current file with the passed file and returns 
// whether or not they are equal. If the file length is greater than our
// checksum buffer, the rest of the file is read in chunks, until EOF or
// a difference is found, whichever comes first.
//
// If they are of different lengths, we assume they are different
func (fd *FileData) isEqual(dstFd FileData) (bool, error) {
	if fd.Fi.Size() != dstFd.Fi.Size() {
		return false, nil
	}
	// otherwise, examine the file contents
	f, hasher, err := fd.getFileHasher()
	if err != nil {
		return false, err
	}
	defer f.Close()
	// TODO support adaptive
	chunks := int(fd.Fi.Size()/int64(fd.ChunkSize) + 1)
	if chunks > len(fd.Hashes) {
		return fd.isEqualMixed(chunks, f, hasher, dstFd)
	}

	return fd.isEqualCached(chunks, f, hasher, dstFd)
}

func (fd *FileData) isEqualMixed(chunks int, f *os.File, hasher hash.Hash, dstFd FileData) (bool, error) {
	equal, err := fd.isEqualCached(chunks, f, hasher, dstFd)
	if err != nil {
		return equal, err
	}
	if !equal {
		return equal, nil
	}
	// Otherwise check the file from the current point
	dstF, err := os.Open(dstFd.RootPath())
	
	// Go to the last read byte
	pos, err := dstF.Seek(dstFd.CurByte, 0)
	if err != nil {
		return false, err
	}	
	_ = pos
	dstHasher, err := fd.getHasher()
	if err != nil {
		return false, err
	}
	dH := Hash256{}
	sH := Hash256{}
	// Check until EOF or a difference is found
	for {
		s, err := io.CopyN(hasher, f, fd.ChunkSize)
		if err != nil {<commit_msg>get chunked comparison and handling of larger file comparison working<commit_after>// Copyright 2014 <NAME> (github.com/mohae) All rights reserved.
// Use of this source code is governed by a BSD-style license that
// can be found in the LICENSE file.
//
package synchronicity

import (
	"bufio"
	"crypto/sha256"
	"fmt"
	"hash"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
)

// This allows for up to 128k of read data. If the file is larger than that,
// a different approach should be done, i.e. don't precompute the hash and use
// other rules for determining difference.
//
var MaxChunks = 16       // Modify directly to change buffered hashes
var chunkSize = int64(8 * 1024) // use 8k chunks as default

// SetChunkSize sets the chunkSize as 1k * i, i.e. 8 == 8k chunkSize
// If the multiplier, i, is <= 0, the default is used, 8.
func SetChunkSize(i int) {
	if i <= 0 {
		i = 8
	}
	chunkSize = int64(1024 * i)
}

const (
	invalid hashType = iota
	SHA256
)

type hashType int // Not really needed atm, but it'll be handy for adding other types.
var useHashType hashType

// SHA256 sized for hashed blocks.
type Hash256 [32]byte

func (h hashType) String() string {
	switch h {
	case SHA256:
		return "sha256"
	case invalid:
		return "invalid"
	}
	return "unknown"
}

func init() {
	useHashType = SHA256
}

type FileData struct {
	Processed bool
	Hashes    []Hash256
	HashType  hashType
	ChunkSize int64 // The chunksize that this was created with.
	MaxChunks int
	CurByte   int64  // for when the while file hasn't been hashed and
	Root      string // the relative root of this file: allows for synch support
	Dir       string // relative path to parent directory of Fi
	Fi        os.FileInfo
}

// Returns a FileData struct for the passed file using the defaults.
// Set any overrides before performing an operation.
func NewFileData(root, dir string, fi os.FileInfo) FileData {
	if dir == "." {
		dir = ""
	}
	fd := FileData{HashType: useHashType, ChunkSize: chunkSize, MaxChunks: MaxChunks, Root: root, Dir: dir, Fi: fi}
	return fd
}

// String is an alias to RelPath
func (fd *FileData) String() string {
	return fd.RelPath()
}

// SetHash computes the hash of the FileData. The path of the file is passed
// because FileData only knows it's name, not its location.
func (fd *FileData) SetHash() error {
	f, hasher, err := fd.getFileHasher()
	if err != nil {
		return err
	}
	if fd.ChunkSize == 0 {
		return fd.hashFile(f, hasher)
	}
	return fd.chunkedHashFile(f, hasher)
}

// getHasher returns a file and the hasher to use it on. If error, return that.
func (fd *FileData) getFileHasher() (f *os.File, hasher hash.Hash, err error) {
	f, err = os.Open(fd.RootPath())
	if err != nil {
		return 
	}
	hasher, err = fd.getHasher()
	return
}

// getHasher returns a hasher or an error
func (fd *FileData) getHasher() (hasher hash.Hash, err error) {
	switch fd.HashType {
	case SHA256:
		hasher = sha256.New() //
	default:
		err = fmt.Errorf("%s hash type", fd.HashType.String())
		return
	}
	return
}

// hashFile hashes the entire file.
func (fd *FileData) hashFile(f *os.File, hasher hash.Hash) error {
	_, err := io.Copy(hasher, f)
	if err != nil {
		log.Printf("%s/n", err)
		return err
	}
	h := Hash256{}
	copy(h[:], hasher.Sum(nil))
	fd.Hashes = append(fd.Hashes, Hash256(h))
	return nil
}

// chunkedHashFile reads up to max chunks, or the entire file, whichever comes
// first. 
func (fd *FileData) chunkedHashFile(f *os.File, hasher hash.Hash) (err error) {
	reader := bufio.NewReaderSize(f, int(fd.ChunkSize))
	var cnt int
	var bytes int64
	h := Hash256{}
	for cnt = 0; cnt < MaxChunks; cnt++ { // read until EOF || MaxChunks
		n, err := io.CopyN(hasher, reader, int64(fd.ChunkSize))
		if err != nil && err != io.EOF {
			return err
		}
		bytes += n
		copy(h[:], hasher.Sum(nil))
		fd.Hashes = append(fd.Hashes, Hash256(h))		
	}
	if err != nil {
		log.Printf("%s\n", err)
		return err
	}
	_ = cnt
	return nil
}

// isEqual compares the current file with the passed file and returns 
// whether or not they are equal. If the file length is greater than our
// checksum buffer, the rest of the file is read in chunks, until EOF or
// a difference is found, whichever comes first.
//
// If they are of different lengths, we assume they are different
func (fd *FileData) isEqual(dstFd FileData) (bool, error) {
	if fd.Fi.Size() != dstFd.Fi.Size() {
		return false, nil
	}
	// otherwise, examine the file contents
	f, hasher, err := fd.getFileHasher()
	if err != nil {
		return false, err
	}
	defer f.Close()
	// TODO support adaptive
	chunks := int(fd.Fi.Size()/int64(fd.ChunkSize) + 1)
	if chunks > len(fd.Hashes) {
		return fd.isEqualMixed(chunks, f, hasher, dstFd)
	}

	return fd.isEqualCached(chunks, f, hasher, dstFd)
}

func (fd *FileData) isEqualMixed(chunks int, f *os.File, hasher hash.Hash, dstFd FileData) (bool, error) {
	equal, err := fd.isEqualCached(dstFd.MaxChunks, f, hasher, dstFd)
	if err != nil {
		return equal, err
	}
	if !equal {
		return equal, nil
	}
	// Otherwise check the file from the current point
	dstF, err := os.Open(dstFd.RootPath())
	
	// Go to the last read byte
	pos, err := dstF.Seek(dstFd.CurByte, 0)
	if err != nil {
		return false, err
	}	
	_ = pos
	dstHasher, err := fd.getHasher()
	if err != nil {
		return false, err
	}
	dH := Hash256{}
	sH := Hash256{}
	// Check until EOF or a difference is found
	for {
		s, err := io.CopyN(hasher, f, fd.ChunkSize)
		if err != nil {
{'fraction_non_alphanumeric': 0.11949568925558543, 'fraction_numerical': 0.01742838602020951, 'mean_word_length': 3.289463220675944, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4684536', 'n_tokens_mistral': 4048, 'n_tokens_neox': 3825, 'n_words': 1399}