<commit_before>package git

import (
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	git2go "github.com/libgit2/git2go"
	"github.com/lucas-clemente/treewatch"
)

// GitRepo is a git repository implementing the Repo interface for goldfish.
type GitRepo struct {
	path string
	repo *git2go.Repository
	tw   treewatch.TreeWatcher
	fo   *fanout
}

// NewGitRepo opens or makes a git repo at the given path
func NewGitRepo(repoPath string) (*GitRepo, error) {
	repo, err := git2go.OpenRepository(repoPath)
	if err != nil {
		repo, err = git2go.InitRepository(repoPath, false)
		if err != nil {
			return nil, err
		}

		// Make empty tree
		index, err := repo.Index()
		if err != nil {
			return nil, err
		}
		defer index.Free()

		treeID, err := index.WriteTree()
		if err != nil {
			return nil, err
		}

		tree, err := repo.LookupTree(treeID)
		if err != nil {
			return nil, err
		}

		defer tree.Free()
		sig := &git2go.Signature{Name: "system", Email: "<EMAIL>", When: time.Now()}
		_, err = repo.CreateCommit("refs/heads/master", sig, sig, "initial commit", tree)
		if err != nil {
			return nil, err
		}
	}

	tw, err := treewatch.NewTreeWatcher(repoPath)
	if err != nil {
		return nil, err
	}

	foChan := make(chan string)
	r := &GitRepo{path: repoPath, repo: repo, tw: tw, fo: newFanout(foChan)}

	go func() {
		for file := range tw.Changes() {
			if !strings.HasPrefix(file, r.path) {
				continue
			}
			file = strings.TrimPrefix(file, r.path)
			if strings.HasPrefix(file, "/.git") {
				continue
			}
			// Don't block commits on network
			go func() {
				foChan <- file
			}()
			log.Printf("file %s changed\n", file)
			err := r.addAllAndCommit("changed " + file)
			if err != nil {
				log.Println(err)
			}
		}
		close(foChan)
	}()

	return r, nil
}

// StopWatching stops watching for changes in the repo
func (r *GitRepo) StopWatching() {
	r.tw.Stop()
}

// ReadFile reads a file from the repo
func (r *GitRepo) ReadFile(path string) (io.ReadCloser, error) {
	return os.Open(r.absolutePath(path))
}

// StoreFile writes a file to the repo and commits it
func (r *GitRepo) StoreFile(p string, data io.Reader) error {
	if err := os.MkdirAll(path.Dir(r.absolutePath(p)), 0755); err != nil {
		return err
	}

	file, err := os.OpenFile(r.absolutePath(p), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = io.Copy(file, data)
	return err
}

// ListFiles lists the files in a given directory
func (r *GitRepo) ListFiles(prefix string) ([]string, error) {
	if prefix[len(prefix)-1] != '/' {
		prefix += "/"
	}

	fileInfos, err := ioutil.ReadDir(r.absolutePath(prefix))
	if err != nil {
		return nil, err
	}

	files := make([]string, 0, len(fileInfos))

	for _, f := range fileInfos {
		name := f.Name()
		if name[0] == '.' {
			continue
		}
		name = prefix + name
		if f.IsDir() {
			name += "/"
		}
		files = append(files, name)
	}

	return files, nil
}

// SearchFiles looks for markdown files containing `term` and returns the paths.
func (r *GitRepo) SearchFiles(term string) ([]string, error) {
	// Walk through all files
	matches := []string{}
	err := filepath.Walk(r.path, func(path string, f os.FileInfo, err error) error {
		if strings.Contains(path, "/.git/") || !strings.HasSuffix(path, ".md") {
			return nil
		}

		b, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}

		if strings.Contains(string(b), term) {
			matches = append(matches, strings.TrimPrefix(path, r.path))
		}

		return nil
	})
	if err != nil {
		return nil, err
	}
	return matches, nil
}

// Observer sends file paths on changes
func (r *GitRepo) Observer() <-chan string {
	return r.fo.Output()
}

// CloseObserver closes an observer obtained from Observer()
func (r *GitRepo) CloseObserver(c <-chan string) {
	r.fo.Close(c)
}

func (r *GitRepo) addAllAndCommit(message string) error {
	index, err := r.repo.Index()
	if err != nil {
		return err
	}
	defer index.Free()<commit_msg>make search case-insensitive<commit_after>package git

import (
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	git2go "github.com/libgit2/git2go"
	"github.com/lucas-clemente/treewatch"
)

// GitRepo is a git repository implementing the Repo interface for goldfish.
type GitRepo struct {
	path string
	repo *git2go.Repository
	tw   treewatch.TreeWatcher
	fo   *fanout
}

// NewGitRepo opens or makes a git repo at the given path
func NewGitRepo(repoPath string) (*GitRepo, error) {
	repo, err := git2go.OpenRepository(repoPath)
	if err != nil {
		repo, err = git2go.InitRepository(repoPath, false)
		if err != nil {
			return nil, err
		}

		// Make empty tree
		index, err := repo.Index()
		if err != nil {
			return nil, err
		}
		defer index.Free()

		treeID, err := index.WriteTree()
		if err != nil {
			return nil, err
		}

		tree, err := repo.LookupTree(treeID)
		if err != nil {
			return nil, err
		}

		defer tree.Free()
		sig := &git2go.Signature{Name: "system", Email: "<EMAIL>", When: time.Now()}
		_, err = repo.CreateCommit("refs/heads/master", sig, sig, "initial commit", tree)
		if err != nil {
			return nil, err
		}
	}

	tw, err := treewatch.NewTreeWatcher(repoPath)
	if err != nil {
		return nil, err
	}

	foChan := make(chan string)
	r := &GitRepo{path: repoPath, repo: repo, tw: tw, fo: newFanout(foChan)}

	go func() {
		for file := range tw.Changes() {
			if !strings.HasPrefix(file, r.path) {
				continue
			}
			file = strings.TrimPrefix(file, r.path)
			if strings.HasPrefix(file, "/.git") {
				continue
			}
			// Don't block commits on network
			go func() {
				foChan <- file
			}()
			log.Printf("file %s changed\n", file)
			err := r.addAllAndCommit("changed " + file)
			if err != nil {
				log.Println(err)
			}
		}
		close(foChan)
	}()

	return r, nil
}

// StopWatching stops watching for changes in the repo
func (r *GitRepo) StopWatching() {
	r.tw.Stop()
}

// ReadFile reads a file from the repo
func (r *GitRepo) ReadFile(path string) (io.ReadCloser, error) {
	return os.Open(r.absolutePath(path))
}

// StoreFile writes a file to the repo and commits it
func (r *GitRepo) StoreFile(p string, data io.Reader) error {
	if err := os.MkdirAll(path.Dir(r.absolutePath(p)), 0755); err != nil {
		return err
	}

	file, err := os.OpenFile(r.absolutePath(p), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = io.Copy(file, data)
	return err
}

// ListFiles lists the files in a given directory
func (r *GitRepo) ListFiles(prefix string) ([]string, error) {
	if prefix[len(prefix)-1] != '/' {
		prefix += "/"
	}

	fileInfos, err := ioutil.ReadDir(r.absolutePath(prefix))
	if err != nil {
		return nil, err
	}

	files := make([]string, 0, len(fileInfos))

	for _, f := range fileInfos {
		name := f.Name()
		if name[0] == '.' {
			continue
		}
		name = prefix + name
		if f.IsDir() {
			name += "/"
		}
		files = append(files, name)
	}

	return files, nil
}

// SearchFiles looks for markdown files containing `term` and returns the paths.
func (r *GitRepo) SearchFiles(term string) ([]string, error) {
	term = strings.ToLower(term)

	// Walk through all files
	matches := []string{}
	err := filepath.Walk(r.path, func(path string, f os.FileInfo, err error) error {
		if strings.Contains(path, "/.git/") || !strings.HasSuffix(path, ".md") {
			return nil
		}

		b, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}

		if strings.Contains(strings.ToLower(string(b)), term) {
			matches = append(matches, strings.TrimPrefix(path, r.path))
		}

		return nil
	})
	if err != nil {
		return nil, err
	}
	return matches, nil
}

// Observer sends file paths on changes
func (r *GitRepo) Observer() <-chan string {
	return r.fo.Output()
}

// CloseObserver closes an observer obtained from Observer()
func (r *GitRepo) CloseObserver(c <-chan string) {
	r.fo.Close(c)
}

func (r *GitRepo) addAllAndCommit(message string) error {
	index, err := r.repo.Index()
	if err != nil {
		return err
	}
	defer index.Free()
{'fraction_non_alphanumeric': 0.14662170600724367, 'fraction_numerical': 0.004496065942300487, 'mean_word_length': 3.284644194756554, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4352858', 'n_tokens_mistral': 3217, 'n_tokens_neox': 2939, 'n_words': 839}