<commit_before>package lfs

import (
	"context"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/client"
	"github.com/docker/go-connections/nat"
	git "github.com/libgit2/git2go"
	"github.com/pkg/errors"
)

// git lfs track
// Since the git-lfs devs discourage using git-lfs in go projects we're just
// calling the git-lfs CLI.
func Track(filename, repositoryLocation string) (string, error) {
	cmd := exec.Command("git-lfs", "track", filename)
	cmd.Dir = repositoryLocation
	out, err := cmd.Output()

	// wait to ensure .gitattributes file is up to date.
	// a monument to all my sins.
	time.Sleep(2 * time.Second)

	output := string(out)
	output = strings.TrimRight(output, "\n")

	return output, err
}

// Adds and commits data found in inputPath
func AddAndCommitData(path, msg string) (string, error) {

	repositoryLocation, err := Add(path)
	if err != nil {
		return "", err
	}

	commitId, err := commit(repositoryLocation, msg)
	if err != nil {
		return "", err
	}

	return commitId, nil
}

// Add the given path to the
func Add(path string) (string, error) {

	wd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	defer os.Chdir(wd)

	repo, repositoryLocation, err := openRepository(path)
	if err != nil {
		return "", err
	}

	os.Chdir(repositoryLocation)

	dataPath, err := filepath.Rel(repositoryLocation, path)
	if err != nil {
		return "", err
	}

	// ensure git-lfs tracks all files recursively by adding ** pattern, see
	// git PATTERN FORMAT description for more details.
	dataPattern := "" + dataPath + "/**"

	output, err := Track(dataPattern, repositoryLocation)
	if err != nil {
		return "", errors.Wrap(err, "Could not track files using git-lfs: "+output)
	}

	gitAttr := ".gitattributes"
	changed, err := fileChanged(repo, gitAttr)
	if err != nil {
		return "", err
	}

	if changed {
		err := addToIndex(repo, gitAttr)
		if err != nil {
			return "", err
		}
	}

	output, err = add(dataPath, repositoryLocation)
	if err != nil {
		return "", errors.Wrap(err, "Could not add files "+output)
	}

	return repositoryLocation, nil
}

func addToIndex(repo *git.Repository, path string) error {

	index, err := repo.Index()
	if err != nil {
		return err
	}

	err = index.AddByPath(path)
	if err != nil {<commit_msg>prevent identical entries in .gitattributes<commit_after>package lfs

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/network"
	"github.com/docker/docker/client"
	"github.com/docker/go-connections/nat"
	git "github.com/libgit2/git2go"
	"github.com/pkg/errors"
)

// git lfs track
// Since the git-lfs devs discourage using git-lfs in go projects we're just
// calling the git-lfs CLI.
func Track(filename, repositoryLocation string) (string, error) {

	cmd := exec.Command("git-lfs", "track", filename)
	cmd.Dir = repositoryLocation
	out, err := cmd.Output()

	// wait to ensure .gitattributes file is up to date.
	// a monument to all my sins.
	time.Sleep(2 * time.Second)

	output := string(out)
	output = strings.TrimRight(output, "\n")

	return output, err
}

// Adds and commits data found in inputPath
func AddAndCommitData(path, msg string) (string, error) {

	repositoryLocation, err := Add(path)
	if err != nil {
		return "", err
	}

	commitId, err := commit(repositoryLocation, msg)
	if err != nil {
		return "", err
	}

	return commitId, nil
}

// Add the given path to the
func Add(path string) (string, error) {

	wd, err := os.Getwd()
	if err != nil {
		return "", err
	}
	defer os.Chdir(wd)

	repo, repositoryLocation, err := openRepository(path)
	if err != nil {
		return "", err
	}

	os.Chdir(repositoryLocation)

	dataPath, err := filepath.Rel(repositoryLocation, path)
	if err != nil {
		return "", err
	}

	// ensure git-lfs tracks all files recursively by adding ** pattern, see
	// git PATTERN FORMAT description for more details.
	dataPattern := "" + dataPath + "/**"

	gitAttr := ".gitattributes"

	// if pattern already exists don't rerun the track command
	b, err := ioutil.ReadFile(gitAttr)
	if err != nil {
		pe := err.(*os.PathError)
		if pe.Err.Error() != "no such file or directory" {
			return "", err
		}
	}

	if !strings.Contains(string(b), dataPattern) {
		output, err := Track(dataPattern, repositoryLocation)
		if err != nil {
			return "", errors.Wrap(err, "Could not track files using git-lfs: "+output)
		}
	}

	changed, err := fileChanged(repo, gitAttr)
	if err != nil {
		return "", err
	}

	if changed {
		err := addToIndex(repo, gitAttr)
		if err != nil {
			return "", err
		}
	}

	output, err := add(dataPath, repositoryLocation)
	if err != nil {
		return "", errors.Wrap(err, "Could not add files "+output)
	}

	return repositoryLocation, nil
}

func addToIndex(repo *git.Repository, path string) error {

	index, err := repo.Index()
	if err != nil {
		return err
	}

	err = index.AddByPath(path)
	if err != nil {
{'fraction_non_alphanumeric': 0.1411329569787814, 'fraction_numerical': 0.0011679968853416391, 'mean_word_length': 3.5875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3332562', 'n_tokens_mistral': 1871, 'n_tokens_neox': 1756, 'n_words': 518}