<commit_before>package vendor

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

// RemoteRepo describes a remote dvcs repository.
type RemoteRepo interface {

	// Checkout checks out a specific branch, tag, or revision.
	// The interpretation of these three values is impementation
	// specific.
	Checkout(branch, tag, revision string) (WorkingCopy, error)

	// URL returns the URL the clone was taken from. It should
	// only be called after Clone.
	URL() string
}

// WorkingCopy represents a local copy of a remote dvcs repository.
type WorkingCopy interface {

	// Dir is the root of this working copy.
	Dir() string

	// Revision returns the revision of this working copy.
	Revision() (string, error)

	// Branch returns the branch to which this working copy belongs.
	Branch() (string, error)

	// Destroy removes the working copy and cleans path to the working copy.
	Destroy() error
}

var (
	ghregex = regexp.MustCompile(`^github.com/([A-Za-z0-9-._]+)/([A-Za-z0-9-._]+)(/.+)?`)
	bbregex = regexp.MustCompile(`^bitbucket.org/([A-Za-z0-9-._]+)/([A-Za-z0-9-._]+)(/.+)?`)
	lpregex = regexp.MustCompile(`^launchpad.net/([A-Za-z0-9-._]+)(/[A-Za-z0-9-._]+)?(/.+)?`)
	genre   = regexp.MustCompile(`^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(?P<vcs>bzr|git|hg|svn))(/[A-Za-z0-9_.\-]+)*$`)
)

// DeduceRemoteRepo takes a potential import path and returns a RemoteRepo
// representing the remote location of the source of an import path.
func DeduceRemoteRepo(path string) (RemoteRepo, string, error) {
	validimport := regexp.MustCompile(`^([A-Za-z0-9-]+)(.[A-Za-z0-9-]+)+(/[A-Za-z0-9-_.]+)+$`)
	if !validimport.MatchString(path) {
		return nil, "", fmt.Errorf("%q is not a valid import path", path)
	}

	switch {
	case ghregex.MatchString(path):
		v := ghregex.FindStringSubmatch(path)
		v = append(v, "")
		repo, err := Gitrepo(fmt.Sprintf("https://github.com/%s/%s", v[1], v[2]))
		return repo, v[3], err
	case bbregex.MatchString(path):
		v := bbregex.FindStringSubmatch(path)
		v = append(v, "")
		repo, err := Gitrepo(fmt.Sprintf("https://bitbucket.org/%s/%s", v[1], v[2]))
		if err == nil {
			return repo, v[3], nil
		}
		repo, err = Hgrepo(fmt.Sprintf("https://bitbucket.org/%s/%s", v[1], v[2]))
		if err == nil {
			return repo, v[3], nil
		}
		return nil, "", fmt.Errorf("unknown repository type")
	case lpregex.MatchString(path):
		v := lpregex.FindStringSubmatch(path)
		v = append(v, "", "")
		if v[2] == "" {
			// launchpad.net/project"
			repo, err := Bzrrepo(fmt.Sprintf("https://launchpad.net/%v", v[1]))
			return repo, "", err
		}
		// launchpad.net/project/series"
		repo, err := Bzrrepo(fmt.Sprintf("https://launchpad.net/%s/%s", v[1], v[2]))
		return repo, v[3], err
	}

	// try the general syntax
	if genre.MatchString(path) {
		v := genre.FindStringSubmatch(path)
		switch v[5] {
		case "git":
			repo, err := Gitrepo("git://" + v[1])
			return repo, v[6], err
		case "hg":
			repo, err := Hgrepo("https://" + v[1])
			return repo, v[6], err
		case "bzr":
			repo, err := Bzrrepo("https://" + v[1])
			return repo, v[6], err
		default:
			return nil, "", fmt.Errorf("unknown repository type: %q", v[5])

		}
	}

	// no idea, try to resolve as a vanity import
	importpath, vcs, reporoot, err := ParseMetadata(path)
	if err != nil {
		return nil, "", err
	}
	extra := path[len(importpath):]
	switch vcs {
	case "git":
		repo, err := Gitrepo(reporoot)
		return repo, extra, err
	case "hg":
		repo, err := Hgrepo(reporoot)
		return repo, extra, err
	case "bzr":
		repo, err := Bzrrepo(reporoot)
		return repo, extra, err
	default:
		return nil, "", fmt.Errorf("unknown repository type: %q", vcs)
	}
}

// Gitrepo returns a RemoteRepo representing a remote git repository.
func Gitrepo(url string) (RemoteRepo, error) {
	if err := probeGitUrl(url); err != nil {
		return nil, err
	}
	return &gitrepo{
		url: url,
	}, nil
}

func probeGitUrl(url string) error {
	_, err := run("git", "ls-remote", "--exit-code", url, "HEAD")
	return err
}

// gitrepo is a git RemoteRepo.
type gitrepo struct {

	// remote repository url, see man 1 git-clone
	url string
}

func (g *gitrepo) URL() string {
	return g.url
}

// Checkout fetchs the remote branch, and optionaly updates to the supplied
// tag or revision. If both are supplied, an error is returned. If the branch
// is blank, then the default remote branch will be used. If the branch is
// "HEAD", an error will be returned.
func (g *gitrepo) Checkout(branch, tag, revision string) (WorkingCopy, error) {
	if branch == "HEAD" {
		return nil, fmt.Errorf("cannot update %q as it has been previously fetched with -tag or -revision. Please use gb vendor delete then fetch again.", g.url)
	}
	if !atMostOne(tag, revision) {
		return nil, fmt.Errorf("only one of tag or revision may be supplied")
	}
	dir, err := mktmp()
	if err != nil {
		return nil, err
	}

	args := []string{
		"clone",
		"-q", // silence progress report to stderr
		g.url,
		dir,
		"--single-branch",
	}
	if branch != "" {
		args = append(args, "--branch", branch)
	}

	if _, err := run("git", args...); err != nil {
		os.RemoveAll(dir)
		return nil, err
	}

	if revision != "" || tag != "" {
		if err := runOutPath(os.Stderr, dir, "git", "checkout", "-q", oneOf(revision, tag)); err != nil {
			os.RemoveAll(dir)
			return nil, err
		}
	}

	return &GitClone{
		workingcopy{
			path: dir,<commit_msg>added extra check that only one of branch and tag is supplied<commit_after>package vendor

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

// RemoteRepo describes a remote dvcs repository.
type RemoteRepo interface {

	// Checkout checks out a specific branch, tag, or revision.
	// The interpretation of these three values is impementation
	// specific.
	Checkout(branch, tag, revision string) (WorkingCopy, error)

	// URL returns the URL the clone was taken from. It should
	// only be called after Clone.
	URL() string
}

// WorkingCopy represents a local copy of a remote dvcs repository.
type WorkingCopy interface {

	// Dir is the root of this working copy.
	Dir() string

	// Revision returns the revision of this working copy.
	Revision() (string, error)

	// Branch returns the branch to which this working copy belongs.
	Branch() (string, error)

	// Destroy removes the working copy and cleans path to the working copy.
	Destroy() error
}

var (
	ghregex = regexp.MustCompile(`^github.com/([A-Za-z0-9-._]+)/([A-Za-z0-9-._]+)(/.+)?`)
	bbregex = regexp.MustCompile(`^bitbucket.org/([A-Za-z0-9-._]+)/([A-Za-z0-9-._]+)(/.+)?`)
	lpregex = regexp.MustCompile(`^launchpad.net/([A-Za-z0-9-._]+)(/[A-Za-z0-9-._]+)?(/.+)?`)
	genre   = regexp.MustCompile(`^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(?P<vcs>bzr|git|hg|svn))(/[A-Za-z0-9_.\-]+)*$`)
)

// DeduceRemoteRepo takes a potential import path and returns a RemoteRepo
// representing the remote location of the source of an import path.
func DeduceRemoteRepo(path string) (RemoteRepo, string, error) {
	validimport := regexp.MustCompile(`^([A-Za-z0-9-]+)(.[A-Za-z0-9-]+)+(/[A-Za-z0-9-_.]+)+$`)
	if !validimport.MatchString(path) {
		return nil, "", fmt.Errorf("%q is not a valid import path", path)
	}

	switch {
	case ghregex.MatchString(path):
		v := ghregex.FindStringSubmatch(path)
		v = append(v, "")
		repo, err := Gitrepo(fmt.Sprintf("https://github.com/%s/%s", v[1], v[2]))
		return repo, v[3], err
	case bbregex.MatchString(path):
		v := bbregex.FindStringSubmatch(path)
		v = append(v, "")
		repo, err := Gitrepo(fmt.Sprintf("https://bitbucket.org/%s/%s", v[1], v[2]))
		if err == nil {
			return repo, v[3], nil
		}
		repo, err = Hgrepo(fmt.Sprintf("https://bitbucket.org/%s/%s", v[1], v[2]))
		if err == nil {
			return repo, v[3], nil
		}
		return nil, "", fmt.Errorf("unknown repository type")
	case lpregex.MatchString(path):
		v := lpregex.FindStringSubmatch(path)
		v = append(v, "", "")
		if v[2] == "" {
			// launchpad.net/project"
			repo, err := Bzrrepo(fmt.Sprintf("https://launchpad.net/%v", v[1]))
			return repo, "", err
		}
		// launchpad.net/project/series"
		repo, err := Bzrrepo(fmt.Sprintf("https://launchpad.net/%s/%s", v[1], v[2]))
		return repo, v[3], err
	}

	// try the general syntax
	if genre.MatchString(path) {
		v := genre.FindStringSubmatch(path)
		switch v[5] {
		case "git":
			repo, err := Gitrepo("git://" + v[1])
			return repo, v[6], err
		case "hg":
			repo, err := Hgrepo("https://" + v[1])
			return repo, v[6], err
		case "bzr":
			repo, err := Bzrrepo("https://" + v[1])
			return repo, v[6], err
		default:
			return nil, "", fmt.Errorf("unknown repository type: %q", v[5])

		}
	}

	// no idea, try to resolve as a vanity import
	importpath, vcs, reporoot, err := ParseMetadata(path)
	if err != nil {
		return nil, "", err
	}
	extra := path[len(importpath):]
	switch vcs {
	case "git":
		repo, err := Gitrepo(reporoot)
		return repo, extra, err
	case "hg":
		repo, err := Hgrepo(reporoot)
		return repo, extra, err
	case "bzr":
		repo, err := Bzrrepo(reporoot)
		return repo, extra, err
	default:
		return nil, "", fmt.Errorf("unknown repository type: %q", vcs)
	}
}

// Gitrepo returns a RemoteRepo representing a remote git repository.
func Gitrepo(url string) (RemoteRepo, error) {
	if err := probeGitUrl(url); err != nil {
		return nil, err
	}
	return &gitrepo{
		url: url,
	}, nil
}

func probeGitUrl(url string) error {
	_, err := run("git", "ls-remote", "--exit-code", url, "HEAD")
	return err
}

// gitrepo is a git RemoteRepo.
type gitrepo struct {

	// remote repository url, see man 1 git-clone
	url string
}

func (g *gitrepo) URL() string {
	return g.url
}

// Checkout fetchs the remote branch, and optionaly updates to the supplied
// tag or revision. If both are supplied, an error is returned. If the branch
// is blank, then the default remote branch will be used. If the branch is
// "HEAD", an error will be returned.
func (g *gitrepo) Checkout(branch, tag, revision string) (WorkingCopy, error) {
	if branch == "HEAD" {
		return nil, fmt.Errorf("cannot update %q as it has been previously fetched with -tag or -revision. Please use gb vendor delete then fetch again.", g.url)
	}
	if !atMostOne(tag, revision) {
		return nil, fmt.Errorf("only one of tag or revision may be supplied")
	}
	if !atMostOne(branch, tag) {
		return nil, fmt.Errorf("only one of tag or branch may be supplied")
	}
	dir, err := mktmp()
	if err != nil {
		return nil, err
	}

	args := []string{
		"clone",
		"-q", // silence progress report to stderr
		g.url,
		dir,
		"--single-branch",
	}
	if branch != "" {
		args = append(args, "--branch", branch)
	}

	if _, err := run("git", args...); err != nil {
		os.RemoveAll(dir)
		return nil, err
	}

	if revision != "" || tag != "" {
		if err := runOutPath(os.Stderr, dir, "git", "checkout", "-q", oneOf(revision, tag)); err != nil {
			os.RemoveAll(dir)
			return nil, err
		}
	}

	return &GitClone{
		workingcopy{
			path: dir,
{'fraction_non_alphanumeric': 0.17841229426207147, 'fraction_numerical': 0.009275256888242248, 'mean_word_length': 3.4652862362971986, 'pattern_counts': {'":': 12, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 14, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1048867', 'n_tokens_mistral': 4288, 'n_tokens_neox': 4006, 'n_words': 1176}