<commit_before>package gitdb

import (
	"fmt"
	"os"
	"path/filepath"

	log "github.com/Sirupsen/logrus"

	snap "github.com/scootdev/scoot/snapshot"
	"github.com/scootdev/scoot/snapshot/git/repo"
)

func (db *DB) readFileAll(id snap.ID, path string) (string, error) {
	v, err := db.parseID(id)
	if err != nil {
		return "", err
	}

	if err := v.Download(db); err != nil {
		return "", err
	}

	if v.Kind() != kindFSSnapshot {
		return "", fmt.Errorf("can only ReadFileAll from an FSSnapshot, but %v is a %v", id, v.Kind())
	}

	return db.dataRepo.Run("cat-file", "-p", fmt.Sprintf("%s:%s", v.SHA(), path))
}

// checkout creates a checkout of id.
func (db *DB) checkout(id snap.ID) (path string, err error) {
	defer func() {
		// If we're returning our repo dir, we need to keep the work tree locked, otherwise, we can unlock it.
		// Note: we defer this to capture the various places 'path' is returned.
		if path != db.dataRepo.Dir() {
			db.workTreeLock.Unlock()
		}
	}()

	v, err := db.parseID(id)
	if err != nil {
		return "", err
	}

	if err := v.Download(db); err != nil {
		return "", err
	}

	switch v.Kind() {
	case kindFSSnapshot:
		// For FSSnapshots, we make a "bare checkout".
		return db.checkoutFSSnapshot(v.SHA())
	case kindGitCommitSnapshot:
		// For GitCommitSnapshot's, we use dataRepo's work tree.
		if id == db.currentSnapID {
			log.Infof("Using cached checkout for id=%s", id)
			return db.dataRepo.Dir(), nil
		}
		path, err := db.checkoutGitCommitSnapshot(v.SHA())
		if err != nil {
			db.currentSnapID = ""
		} else {
			db.currentSnapID = id
		}
		return path, err
	default:
		return "", fmt.Errorf("cannot checkout value kind %v; id %v", v.Kind(), v.ID())
	}
}

// checkoutFSSnapshot creates a new dir with a new index and checks out exactly that tree.
func (db *DB) checkoutFSSnapshot(sha string) (path string, err error) {
	// we don't need the work tree
	indexDir, err := db.tmp.TempDir("git-index")
	if err != nil {
		return "", err
	}

	indexFilename := filepath.Join(indexDir.Dir, "index")
	defer os.RemoveAll(indexDir.Dir)

	coDir, err := db.tmp.TempDir("checkout")
	if err != nil {
		return "", err
	}

	extraEnv := []string{"GIT_INDEX_FILE=" + indexFilename, "GIT_WORK_TREE=" + coDir.Dir}

	cmd := db.dataRepo.Command("read-tree", sha)
	cmd.Env = append(cmd.Env, extraEnv...)
	_, err = db.dataRepo.RunCmd(cmd)
	if err != nil {
		return "", err
	}

	cmd = db.dataRepo.Command("checkout-index", "-a")
	cmd.Env = append(cmd.Env, extraEnv...)
	_, err = db.dataRepo.RunCmd(cmd)
	if err != nil {
		return "", err
	}

	db.checkouts[coDir.Dir] = true

	return coDir.Dir, nil
}

// checkoutGitCommitSnapshot checks out a commit into our work tree.
// We could use multiple work trees, except our internal git doesn't yet have work-tree support.
// TODO(dbentley): migrate to work-trees.
func (db *DB) checkoutGitCommitSnapshot(sha string) (path string, err error) {
	cmds := [][]string{
		// -d removes directories. -x ignores gitignore and removes everything.
		// -f is force. -f the second time removes directories even if they're git repos themselves
		{"clean", "-f", "-f", "-d", "-x"},
		// Note: our worktree cannot be in detached head state after checkout since [Twitter] git needs a valid ref to fetch.
		//       we use scoot's tmp branch name so here subsequent fetch operations, ex: those in stream.go, can succeed.
		{"checkout", "-B", tempCheckoutBranch, sha},
	}

	for _, argv := range cmds {
		if _, err := db.dataRepo.Run(argv...); err != nil {
			return "", fmt.Errorf("Unable to run git %v: %v", argv, err)
		}
	}

	return db.dataRepo.Dir(), nil
}

func (db *DB) releaseCheckout(path string) error {
	if path == db.dataRepo.Dir() {<commit_msg>Jschiller/checkout dirty (#270)<commit_after>package gitdb

import (
	"fmt"
	"os"
	"path/filepath"

	log "github.com/Sirupsen/logrus"

	snap "github.com/scootdev/scoot/snapshot"
	"github.com/scootdev/scoot/snapshot/git/repo"
)

func (db *DB) readFileAll(id snap.ID, path string) (string, error) {
	v, err := db.parseID(id)
	if err != nil {
		return "", err
	}

	if err := v.Download(db); err != nil {
		return "", err
	}

	if v.Kind() != kindFSSnapshot {
		return "", fmt.Errorf("can only ReadFileAll from an FSSnapshot, but %v is a %v", id, v.Kind())
	}

	return db.dataRepo.Run("cat-file", "-p", fmt.Sprintf("%s:%s", v.SHA(), path))
}

// checkout creates a checkout of id.
func (db *DB) checkout(id snap.ID) (path string, err error) {
	defer func() {
		// If we're returning our repo dir, we need to keep the work tree locked, otherwise, we can unlock it.
		// Note: we defer this to capture the various places 'path' is returned.
		if path != db.dataRepo.Dir() {
			db.workTreeLock.Unlock()
		}
	}()

	v, err := db.parseID(id)
	if err != nil {
		return "", err
	}

	if err := v.Download(db); err != nil {
		return "", err
	}

	switch v.Kind() {
	case kindFSSnapshot:
		// For FSSnapshots, we make a "bare checkout".
		return db.checkoutFSSnapshot(v.SHA())
	case kindGitCommitSnapshot:
		// For GitCommitSnapshot's, we use dataRepo's work tree.
		if id == db.currentSnapID {
			log.Infof("Using cached checkout for id=%s", id)
			return db.dataRepo.Dir(), nil
		}
		path, err := db.checkoutGitCommitSnapshot(v.SHA())
		if err != nil {
			db.currentSnapID = ""
		} else {
			db.currentSnapID = id
		}
		return path, err
	default:
		return "", fmt.Errorf("cannot checkout value kind %v; id %v", v.Kind(), v.ID())
	}
}

// checkoutFSSnapshot creates a new dir with a new index and checks out exactly that tree.
func (db *DB) checkoutFSSnapshot(sha string) (path string, err error) {
	// we don't need the work tree
	indexDir, err := db.tmp.TempDir("git-index")
	if err != nil {
		return "", err
	}

	indexFilename := filepath.Join(indexDir.Dir, "index")
	defer os.RemoveAll(indexDir.Dir)

	coDir, err := db.tmp.TempDir("checkout")
	if err != nil {
		return "", err
	}

	extraEnv := []string{"GIT_INDEX_FILE=" + indexFilename, "GIT_WORK_TREE=" + coDir.Dir}

	cmd := db.dataRepo.Command("read-tree", sha)
	cmd.Env = append(cmd.Env, extraEnv...)
	_, err = db.dataRepo.RunCmd(cmd)
	if err != nil {
		return "", err
	}

	cmd = db.dataRepo.Command("checkout-index", "-a")
	cmd.Env = append(cmd.Env, extraEnv...)
	_, err = db.dataRepo.RunCmd(cmd)
	if err != nil {
		return "", err
	}

	db.checkouts[coDir.Dir] = true

	return coDir.Dir, nil
}

// checkoutGitCommitSnapshot checks out a commit into our work tree.
// We could use multiple work trees, except our internal git doesn't yet have work-tree support.
// TODO(dbentley): migrate to work-trees.
func (db *DB) checkoutGitCommitSnapshot(sha string) (path string, err error) {
	cmds := [][]string{
		// -d removes directories. -x ignores gitignore and removes everything.
		// -f is force. -f the second time removes directories even if they're git repos themselves
		{"clean", "-f", "-f", "-d", "-x"},
		// -f overrides modified files
		// -B resets or creates the named branch when checking out the given sha.
		// Note: our worktree cannot be in detached head state after checkout since [Twitter] git needs a valid ref to fetch.
		//       we use scoot's tmp branch name so here subsequent fetch operations, ex: those in stream.go, can succeed.
		{"checkout", "-fB", tempCheckoutBranch, sha},
	}

	for _, argv := range cmds {
		if _, err := db.dataRepo.Run(argv...); err != nil {
			return "", fmt.Errorf("Unable to run git %v: %v", argv, err)
		}
	}

	return db.dataRepo.Dir(), nil
}

func (db *DB) releaseCheckout(path string) error {
	if path == db.dataRepo.Dir() {
{'fraction_non_alphanumeric': 0.1525039957378796, 'fraction_numerical': 0.00039957378795950983, 'mean_word_length': 3.42226148409894, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3805538', 'n_tokens_mistral': 2798, 'n_tokens_neox': 2635, 'n_words': 839}