<commit_before>package main

import (
	"errors"
	"fmt"
	"log"
	"os"
	"os/signal"
	"path/filepath"

	"github.com/fsnotify/fsnotify"

	"github.com/NebulousLabs/Sia/api"
)

// SiaFolder is a folder that is synchronized to a Sia node.
type SiaFolder struct {
	path    string
	client  *api.Client
	watcher *fsnotify.Watcher
}

// NewSiafolder creates a new SiaFolder using the provided path and api
// address.
func NewSiafolder(path string, apiaddr string) (*SiaFolder, error) {
	sf := &SiaFolder{}
	sf.path = path

	sf.client = api.NewClient(apiaddr, "")
	var contracts api.RenterContracts
	err := sf.client.Get("/renter/contracts", &contracts)
	if err != nil {
		return nil, err
	}
	if len(contracts.Contracts) == 0 {
		return nil, errors.New("you must have formed contracts to upload to Sia")
	}

	// walk the provided path
	var files []string
	err = filepath.Walk(path, func(walkpath string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if walkpath == path {
			return nil
		}

		relpath, err := filepath.Rel(path, walkpath)
		if err != nil {
			return err
		}
		files = append(files, relpath)
		return nil
	})
	if err != nil {
		return nil, err
	}

	// upload any files not in the sia node already
	err = sf.uploadNonExisting(files)
	if err != nil {
		return nil, err
	}

	// watch for file changes
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}
	go func() {
		for {
			select {
			case event := <-watcher.Events:
				if event.Op&fsnotify.Remove == fsnotify.Remove {
					sf.handleRemove(event.Name)
				}

				if event.Op&fsnotify.Create == fsnotify.Create {
					sf.handleCreate(event.Name)
				}

			case err := <-watcher.Errors:
				log.Println("fsevents error:", err)
			}
		}
	}()
	err = watcher.Add(path)
	if err != nil {
		return nil, err
	}

	sf.watcher = watcher
	return sf, nil
}

// Close releases any resources allocated by a SiaFolder.
func (sf *SiaFolder) Close() error {
	return sf.watcher.Close()
}

// handleCreate handles a file creation event. `file` is a relative path to the
// file on disk.
func (sf *SiaFolder) handleCreate(file string) {
	log.Println("sync updated detected, uploading", file)
	abspath, err := filepath.Abs(file)
	if err != nil {
		log.Println("error getting absolute path to upload:", err)
		return
	}
	relpath, err := filepath.Rel(sf.path, file)
	if err != nil {
		log.Println("error getting relative path to upload:", err)
		return
	}
	err = sf.client.Post(fmt.Sprintf("/renter/upload/%v", relpath), fmt.Sprintf("source=%v", abspath), nil)
	if err != nil {
		log.Printf("error uploading %v: %v\n", file, err)
	}
}

// handleRemove handles a file removal event.
func (sf *SiaFolder) handleRemove(file string) {
	log.Println("sync updated detected, removing", file)
	relpath, err := filepath.Rel(sf.path, file)<commit_msg>track file changes using a hash map<commit_after>package main

import (
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/signal"
	"path/filepath"

	"github.com/fsnotify/fsnotify"

	"github.com/NebulousLabs/Sia/api"
)

// SiaFolder is a folder that is synchronized to a Sia node.
type SiaFolder struct {
	path    string
	client  *api.Client
	watcher *fsnotify.Watcher

	files map[string]string // files is a map of file paths to SHA256 checksums, used to reconcile file changes
}

// NewSiafolder creates a new SiaFolder using the provided path and api
// address.
func NewSiafolder(path string, apiaddr string) (*SiaFolder, error) {
	sf := &SiaFolder{}
	sf.path = path
	sf.files = make(map[string]string)

	sf.client = api.NewClient(apiaddr, "")
	var contracts api.RenterContracts
	err := sf.client.Get("/renter/contracts", &contracts)
	if err != nil {
		return nil, err
	}
	if len(contracts.Contracts) == 0 {
		return nil, errors.New("you must have formed contracts to upload to Sia")
	}

	// watch for file changes
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}
	err = watcher.Add(path)
	if err != nil {
		return nil, err
	}

	sf.watcher = watcher

	// walk the provided path, accumulating a slice of files to potentially
	// upload and adding any subdirectories to the watcher.
	var files []string
	err = filepath.Walk(path, func(walkpath string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if walkpath == path {
			return nil
		}

		if f.IsDir() {
			// subdirectories must be added to the watcher.
			watcher.Add(walkpath)
		} else {
			relpath, err := filepath.Rel(path, walkpath)
			if err != nil {
				return err
			}
			files = append(files, relpath)
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	// upload any files not in the sia node already
	err = sf.uploadNonExisting(files)
	if err != nil {
		return nil, err
	}

	// perform initial hashing to track file changes
	for _, file := range files {
		checksum, err := checksumFile(file)
		if err != nil {
			return nil, err
		}
		sf.files[file] = checksum
	}

	go sf.eventWatcher()

	return sf, nil
}

// checksumFile returns a sha256 checksum of a given file on disk.
func checksumFile(path string) (string, error) {
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", err
	}

	return string(h.Sum(nil)), nil
}

// eventWatcher continuously listens on the SiaFolder's watcher channels and
// performs the necessary upload/delete operations.
func (sf *SiaFolder) eventWatcher() {
	for {
		select {
		case event := <-sf.watcher.Events:
			f, err := os.Stat(event.Name)
			if err != nil {
				log.Println("error opening updated file:", err)
				continue
			}
			if f.IsDir() {
				sf.watcher.Add(event.Name)
				continue
			}

			// WRITE event, checksum the file and re-upload it if it has changed
			if event.Op&fsnotify.Write == fsnotify.Write {
				checksum, err := checksumFile(event.Name)
				if err != nil {
					log.Println("error getting checksum:", err)
					continue
				}
				oldChecksum, exists := sf.files[event.Name]
				if exists && oldChecksum != checksum {
					log.Printf("change in %v detected, re-uploading\n", event.Name)
					sf.handleRemove(event.Name)
					sf.handleCreate(event.Name)
				}
				sf.files[event.Name] = checksum
			}

			// REMOVE event
			if event.Op&fsnotify.Remove == fsnotify.Remove {
				log.Println("sync updated detected, removing", event.Name)
				sf.handleRemove(event.Name)
			}

			// CREATE event
			if event.Op&fsnotify.Create == fsnotify.Create {
				log.Println("sync updated detected, uploading", event.Name)
				sf.handleCreate(event.Name)
			}

		case err := <-sf.watcher.Errors:
			log.Println("fsevents error:", err)
		}
	}
}

// Close releases any resources allocated by a SiaFolder.
func (sf *SiaFolder) Close() error {
	return sf.watcher.Close()
}

// handleCreate handles a file creation event. `file` is a relative path to the
// file on disk.
func (sf *SiaFolder) handleCreate(file string) {
	abspath, err := filepath.Abs(file)
	if err != nil {
		log.Println("error getting absolute path to upload:", err)
		return
	}
	relpath, err := filepath.Rel(sf.path, file)
	if err != nil {
		log.Println("error getting relative path to upload:", err)
		return
	}
	err = sf.client.Post(fmt.Sprintf("/renter/upload/%v", relpath), fmt.Sprintf("source=%v", abspath), nil)
	if err != nil {
		log.Printf("error uploading %v: %v\n", file, err)
	}
}

// handleRemove handles a file removal event.
func (sf *SiaFolder) handleRemove(file string) {
	relpath, err := filepath.Rel(sf.path, file)
{'fraction_non_alphanumeric': 0.12731635781895748, 'fraction_numerical': 0.0018664178109585388, 'mean_word_length': 3.2795208214489446, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '148205', 'n_tokens_mistral': 2840, 'n_tokens_neox': 2586, 'n_words': 788}