<commit_before>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libgit

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
	"github.com/keybase/kbfs/libfs"
	"github.com/keybase/kbfs/libkbfs"
	"github.com/pkg/errors"
	billy "gopkg.in/src-d/go-billy.v4"
	gogit "gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
	"gopkg.in/src-d/go-git.v4/plumbing/storer"
	"gopkg.in/src-d/go-git.v4/storage"
	"gopkg.in/src-d/go-git.v4/storage/filesystem"
)

const (
	kbfsRepoDir              = ".kbfs_git"
	kbfsConfigName           = "kbfs_config"
	kbfsConfigNameTemp       = "._kbfs_config"
	gitSuffixToIgnore        = ".git"
	kbfsDeletedReposDir      = ".kbfs_deleted_repos"
	minDeletedAgeForCleaning = 1 * time.Hour
	cleaningTimeLimit        = 2 * time.Second
	repoGCLockFileName       = ".gc"
)

// This character set is what Github supports in repo names.  It's
// probably to avoid any problems when cloning onto filesystems that
// have different Unicode decompression schemes
// (https://en.wikipedia.org/wiki/Unicode_equivalence).  There's no
// internal reason to be so restrictive, but it probably makes sense
// to start off more restrictive and then relax things later as we
// test.
var repoNameRE = regexp.MustCompile(`^([a-zA-Z0-9][a-zA-Z0-9_\.-]*)$`)

func checkValidRepoName(repoName string, config libkbfs.Config) bool {
	return len(repoName) >= 1 &&
		uint32(len(repoName)) <= config.MaxNameBytes() &&
		(os.Getenv("KBFS_GIT_REPONAME_SKIP_CHECK") != "" ||
			repoNameRE.MatchString(repoName))
}

// For the common "repo doesn't exist" case, use the error type that the client can recognize.
func castNoSuchNameError(err error, repoName string) error {
	switch errors.Cause(err).(type) {
	case libkbfs.NoSuchNameError:
		return libkb.RepoDoesntExistError{
			Name: repoName,
		}
	default:
		return err
	}
}

func recursiveDelete(
	ctx context.Context, fs *libfs.FS, fi os.FileInfo) error {
	if !fi.IsDir() {
		// Delete regular files and symlinks directly.
		return fs.Remove(fi.Name())
	}

	subdirFS, err := fs.Chroot(fi.Name())
	if err != nil {
		return err
	}

	children, err := subdirFS.ReadDir("/")
	if err != nil {
		return err
	}
	for _, childFI := range children {
		err := recursiveDelete(ctx, subdirFS.(*libfs.FS), childFI)
		if err != nil {
			return err
		}
	}

	return fs.Remove(fi.Name())
}

// CleanOldDeletedRepos completely removes any "deleted" repos that
// have been deleted for longer than `minDeletedAgeForCleaning`.  The
// caller is responsible for syncing any data to disk, if desired.
func CleanOldDeletedRepos(
	ctx context.Context, config libkbfs.Config,
	tlfHandle *libkbfs.TlfHandle) (err error) {
	fs, err := libfs.NewFS(
		ctx, config, tlfHandle, path.Join(kbfsRepoDir, kbfsDeletedReposDir),
		"" /* uniq ID isn't used for removals */, keybase1.MDPriorityGit)
	switch errors.Cause(err).(type) {
	case libkbfs.NoSuchNameError:
		// Nothing to clean.
		return nil
	case nil:
	default:
		return err
	}

	deletedRepos, err := fs.ReadDir("/")
	if err != nil {
		return err
	}

	if len(deletedRepos) == 0 {
		return nil
	}

	log := config.MakeLogger("")
	now := config.Clock().Now()

	log.CDebugf(ctx, "Checking %d deleted repos for cleaning in %s",
		len(deletedRepos), tlfHandle.GetCanonicalPath())
	defer func() {
		log.CDebugf(ctx, "Done checking deleted repos: %+v", err)
	}()
	for _, fi := range deletedRepos {
		parts := strings.Split(fi.Name(), "-")
		if len(parts) < 2 {
			log.CDebugf(ctx,
				"Ignoring deleted repo name with wrong format: %s", fi.Name())
			continue
		}

		deletedTimeUnixNano, err := strconv.ParseInt(
			parts[len(parts)-1], 10, 64)
		if err != nil {
			log.CDebugf(ctx,
				"Ignoring deleted repo name with wrong format: %s: %+v",
				fi.Name(), err)
			continue
		}

		deletedTime := time.Unix(0, deletedTimeUnixNano)
		if deletedTime.Add(minDeletedAgeForCleaning).After(now) {
			// Repo was deleted too recently.
			continue
		}

		log.CDebugf(ctx, "Cleaning deleted repo %s", fi.Name())
		err = recursiveDelete(ctx, fs, fi)
		if err != nil {
			return err
		}
	}
	return nil
}

// CleanOldDeletedReposTimeLimited is the same as
// `CleanOldDeletedRepos`, except it limits the time spent on
// cleaning, deleting as much data as possible within the given time
// limit (without returning an error).
func CleanOldDeletedReposTimeLimited(
	ctx context.Context, config libkbfs.Config,
	tlfHandle *libkbfs.TlfHandle) error {
	ctx, cancel := context.WithTimeout(ctx, cleaningTimeLimit)
	defer cancel()
	err := CleanOldDeletedRepos(ctx, config, tlfHandle)
	if errors.Cause(err) == context.DeadlineExceeded {
		return nil
	}
	return err
}

// UpdateRepoMD lets the Keybase service know that a repo's MD has
// been updated.
func UpdateRepoMD(ctx context.Context, config libkbfs.Config,
	tlfHandle *libkbfs.TlfHandle, fs billy.Filesystem,
	commitsByRef map[plumbing.ReferenceName][]*object.Commit) error {
	if len(commitsByRef) == 0 {
		commitsByRef = map[plumbing.ReferenceName][]*object.Commit{"": nil}
	}
	folder := tlfHandle.ToFavorite().ToKBFolder(false)

	// Get the user-formatted repo name.
	f, err := fs.Open(kbfsConfigName)
	if err != nil {
		return err
	}
	defer f.Close()
	buf, err := ioutil.ReadAll(f)<commit_msg>libgit.repo: Add a comment explaining some nil handling.<commit_after>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libgit

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/protocol/keybase1"
	"github.com/keybase/kbfs/libfs"
	"github.com/keybase/kbfs/libkbfs"
	"github.com/pkg/errors"
	billy "gopkg.in/src-d/go-billy.v4"
	gogit "gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
	"gopkg.in/src-d/go-git.v4/plumbing/storer"
	"gopkg.in/src-d/go-git.v4/storage"
	"gopkg.in/src-d/go-git.v4/storage/filesystem"
)

const (
	kbfsRepoDir              = ".kbfs_git"
	kbfsConfigName           = "kbfs_config"
	kbfsConfigNameTemp       = "._kbfs_config"
	gitSuffixToIgnore        = ".git"
	kbfsDeletedReposDir      = ".kbfs_deleted_repos"
	minDeletedAgeForCleaning = 1 * time.Hour
	cleaningTimeLimit        = 2 * time.Second
	repoGCLockFileName       = ".gc"
)

// This character set is what Github supports in repo names.  It's
// probably to avoid any problems when cloning onto filesystems that
// have different Unicode decompression schemes
// (https://en.wikipedia.org/wiki/Unicode_equivalence).  There's no
// internal reason to be so restrictive, but it probably makes sense
// to start off more restrictive and then relax things later as we
// test.
var repoNameRE = regexp.MustCompile(`^([a-zA-Z0-9][a-zA-Z0-9_\.-]*)$`)

func checkValidRepoName(repoName string, config libkbfs.Config) bool {
	return len(repoName) >= 1 &&
		uint32(len(repoName)) <= config.MaxNameBytes() &&
		(os.Getenv("KBFS_GIT_REPONAME_SKIP_CHECK") != "" ||
			repoNameRE.MatchString(repoName))
}

// For the common "repo doesn't exist" case, use the error type that the client can recognize.
func castNoSuchNameError(err error, repoName string) error {
	switch errors.Cause(err).(type) {
	case libkbfs.NoSuchNameError:
		return libkb.RepoDoesntExistError{
			Name: repoName,
		}
	default:
		return err
	}
}

func recursiveDelete(
	ctx context.Context, fs *libfs.FS, fi os.FileInfo) error {
	if !fi.IsDir() {
		// Delete regular files and symlinks directly.
		return fs.Remove(fi.Name())
	}

	subdirFS, err := fs.Chroot(fi.Name())
	if err != nil {
		return err
	}

	children, err := subdirFS.ReadDir("/")
	if err != nil {
		return err
	}
	for _, childFI := range children {
		err := recursiveDelete(ctx, subdirFS.(*libfs.FS), childFI)
		if err != nil {
			return err
		}
	}

	return fs.Remove(fi.Name())
}

// CleanOldDeletedRepos completely removes any "deleted" repos that
// have been deleted for longer than `minDeletedAgeForCleaning`.  The
// caller is responsible for syncing any data to disk, if desired.
func CleanOldDeletedRepos(
	ctx context.Context, config libkbfs.Config,
	tlfHandle *libkbfs.TlfHandle) (err error) {
	fs, err := libfs.NewFS(
		ctx, config, tlfHandle, path.Join(kbfsRepoDir, kbfsDeletedReposDir),
		"" /* uniq ID isn't used for removals */, keybase1.MDPriorityGit)
	switch errors.Cause(err).(type) {
	case libkbfs.NoSuchNameError:
		// Nothing to clean.
		return nil
	case nil:
	default:
		return err
	}

	deletedRepos, err := fs.ReadDir("/")
	if err != nil {
		return err
	}

	if len(deletedRepos) == 0 {
		return nil
	}

	log := config.MakeLogger("")
	now := config.Clock().Now()

	log.CDebugf(ctx, "Checking %d deleted repos for cleaning in %s",
		len(deletedRepos), tlfHandle.GetCanonicalPath())
	defer func() {
		log.CDebugf(ctx, "Done checking deleted repos: %+v", err)
	}()
	for _, fi := range deletedRepos {
		parts := strings.Split(fi.Name(), "-")
		if len(parts) < 2 {
			log.CDebugf(ctx,
				"Ignoring deleted repo name with wrong format: %s", fi.Name())
			continue
		}

		deletedTimeUnixNano, err := strconv.ParseInt(
			parts[len(parts)-1], 10, 64)
		if err != nil {
			log.CDebugf(ctx,
				"Ignoring deleted repo name with wrong format: %s: %+v",
				fi.Name(), err)
			continue
		}

		deletedTime := time.Unix(0, deletedTimeUnixNano)
		if deletedTime.Add(minDeletedAgeForCleaning).After(now) {
			// Repo was deleted too recently.
			continue
		}

		log.CDebugf(ctx, "Cleaning deleted repo %s", fi.Name())
		err = recursiveDelete(ctx, fs, fi)
		if err != nil {
			return err
		}
	}
	return nil
}

// CleanOldDeletedReposTimeLimited is the same as
// `CleanOldDeletedRepos`, except it limits the time spent on
// cleaning, deleting as much data as possible within the given time
// limit (without returning an error).
func CleanOldDeletedReposTimeLimited(
	ctx context.Context, config libkbfs.Config,
	tlfHandle *libkbfs.TlfHandle) error {
	ctx, cancel := context.WithTimeout(ctx, cleaningTimeLimit)
	defer cancel()
	err := CleanOldDeletedRepos(ctx, config, tlfHandle)
	if errors.Cause(err) == context.DeadlineExceeded {
		return nil
	}
	return err
}

// UpdateRepoMD lets the Keybase service know that a repo's MD has
// been updated.
func UpdateRepoMD(ctx context.Context, config libkbfs.Config,
	tlfHandle *libkbfs.TlfHandle, fs billy.Filesystem,
	commitsByRef map[plumbing.ReferenceName][]*object.Commit) error {
	if len(commitsByRef) == 0 {
		// If an empty `commitsByRef` is passed in, ensure there is at least
		// one entry so that `PutGitMetadata` is called once. Use an empty
		// reference name and an empty commits array since there is no way to
		// retrieve that metadata in those situations (e.g. repo rename).
		commitsByRef = map[plumbing.ReferenceName][]*object.Commit{"": nil}
	}
	folder := tlfHandle.ToFavorite().ToKBFolder(false)

	// Get the user-formatted repo name.
	f, err := fs.Open(kbfsConfigName)
	if err != nil {
		return err
	}
	defer f.Close()
	buf, err := ioutil.ReadAll(f)
{'fraction_non_alphanumeric': 0.12854743170365132, 'fraction_numerical': 0.005481389797542215, 'mean_word_length': 3.746957616449853, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2523740', 'n_tokens_mistral': 4340, 'n_tokens_neox': 4127, 'n_words': 1059}