<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"
)

const (
	GitIgnore = ".gitignore"
)

var (
	GOPATH = os.Getenv("GOPATH")

	dir = ""
)

func main() {
	log.SetFlags(0)

	if len(os.Args) < 2 {
		failAndUsage("Please specify a command")
	}
	cmd := strings.ToLower(os.Args[1])
	switch cmd {
	case "init":
		doinit()
	case "get":
		get()
	default:
		failAndUsage("Unknown command: %s", cmd)
	}
}

func doinit() {
	var err error
	dir, err = os.Getwd()
	if err != nil {
		log.Fatalf("Unable to determine current directory: %s", err)
	}

	if exists(".git") {
		log.Fatalf("%s already contains a .git folder, can't initialize gost", dir)
	}

	// Initialize a git repo
	run("git", "init")

	writeAndCommit(GitIgnore, DefaultGitIgnore, "Initialized repo with a .gitignore file")

	// Done
	log.Printf("Initialized git repo with a default README.md, please set your GOPATH to \"%s\", e.g.", dir)
	log.Printf("  export GOPATH=\"%s\"", dir)
	os.Setenv("GOPATH", dir)
}

func get() {
	requireGOPATH()

	flags := flag.NewFlagSet("get", flag.ExitOnError)
	flags.Parse(os.Args[2:])
	args := flags.Args()
	if len(args) < 1 {
		log.Fatal("Please specify a package")
	}

	pkg := args[0]
	if !isGithub(pkg) {
		log.Fatal("gost only supports packages on github.com")
	}

	branch := "master"
	if len(args) > 1 {
		branch = args[1]
	}

	fetchSubtree(pkg, branch, map[string]bool{})

	run("git", "add", "src")
	run("git", "commit", "-m", fmt.Sprintf("[gost] Added %s and its dependencies", pkg))
}

func fetchSubtree(pkg string, branch string, alreadyFetched map[string]bool) {
	// Take only the path up to the github repo
	pkgParts := strings.Split(pkg, "/")
	pkgRoot := path.Join(pkgParts[:3]...)
	if alreadyFetched[pkgRoot] {
		return
	}

	prefix := path.Join("src", pkgRoot)
	if exists(prefix) {
		log.Printf("%s already exists, declining to add as subtree", prefix)
	} else {
		run("git", "subtree", "add", "--squash",
			"--prefix", prefix,
			fmt.Sprintf("https://%s.git", pkgRoot),
			branch)
	}
	alreadyFetched[pkgRoot] = true
	fetchDeps(pkg, "master", alreadyFetched)
}

func fetchDeps(pkg string, branch string, alreadyFetched map[string]bool) {
	depsString := run("go", "list", "-f", "{{range .Deps}}{{.}} {{end}} {{range .TestImports}}{{.}} {{end}}", pkg)
	deps := parseDeps(depsString)

	nonGithubDeps := []string{}
	for _, dep := range deps {
		dep = strings.TrimSpace(dep)
		if dep == "" {
			continue
		}
		if isGithub(dep) {
			fetchSubtree(dep, branch, alreadyFetched)
		} else {
			nonGithubDeps = append(nonGithubDeps, dep)
		}
	}

	for _, dep := range nonGithubDeps {
		goGet(dep, alreadyFetched)
	}
}

func goGet(pkg string, alreadyFetched map[string]bool) {
	if alreadyFetched[pkg] {
		return
	}
	run("go", "get", pkg)
	alreadyFetched[pkg] = true
}

func writeAndCommit(file string, content string, comment string) {
	if exists(file) {
		log.Fatalf("%s already contains %s, can't initialize gost", dir, file)
	}

	err := ioutil.WriteFile(file, []byte(content), 0644)
	if err != nil {
		log.Fatalf("Unable to write %s: %s", file, err)<commit_msg>Ignoring imports of .<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"
)

const (
	GitIgnore = ".gitignore"
)

var (
	GOPATH = os.Getenv("GOPATH")

	dir = ""
)

func main() {
	log.SetFlags(0)

	if len(os.Args) < 2 {
		failAndUsage("Please specify a command")
	}
	cmd := strings.ToLower(os.Args[1])
	switch cmd {
	case "init":
		doinit()
	case "get":
		get()
	default:
		failAndUsage("Unknown command: %s", cmd)
	}
}

func doinit() {
	var err error
	dir, err = os.Getwd()
	if err != nil {
		log.Fatalf("Unable to determine current directory: %s", err)
	}

	if exists(".git") {
		log.Fatalf("%s already contains a .git folder, can't initialize gost", dir)
	}

	// Initialize a git repo
	run("git", "init")

	writeAndCommit(GitIgnore, DefaultGitIgnore, "Initialized repo with a .gitignore file")

	// Done
	log.Printf("Initialized git repo with a default README.md, please set your GOPATH to \"%s\", e.g.", dir)
	log.Printf("  export GOPATH=\"%s\"", dir)
	os.Setenv("GOPATH", dir)
}

func get() {
	requireGOPATH()

	flags := flag.NewFlagSet("get", flag.ExitOnError)
	flags.Parse(os.Args[2:])
	args := flags.Args()
	if len(args) < 1 {
		log.Fatal("Please specify a package")
	}

	pkg := args[0]
	if !isGithub(pkg) {
		log.Fatal("gost only supports packages on github.com")
	}

	branch := "master"
	if len(args) > 1 {
		branch = args[1]
	}

	fetchSubtree(pkg, branch, map[string]bool{})

	run("git", "add", "src")
	run("git", "commit", "-m", fmt.Sprintf("[gost] Added %s and its dependencies", pkg))
}

func fetchSubtree(pkg string, branch string, alreadyFetched map[string]bool) {
	// Take only the path up to the github repo
	pkgParts := strings.Split(pkg, "/")
	pkgRoot := path.Join(pkgParts[:3]...)
	if alreadyFetched[pkgRoot] {
		return
	}

	prefix := path.Join("src", pkgRoot)
	if exists(prefix) {
		log.Printf("%s already exists, declining to add as subtree", prefix)
	} else {
		run("git", "subtree", "add", "--squash",
			"--prefix", prefix,
			fmt.Sprintf("https://%s.git", pkgRoot),
			branch)
	}
	alreadyFetched[pkgRoot] = true
	fetchDeps(pkg, "master", alreadyFetched)
}

func fetchDeps(pkg string, branch string, alreadyFetched map[string]bool) {
	depsString := run("go", "list", "-f", "{{range .Deps}}{{.}} {{end}} {{range .TestImports}}{{.}} {{end}}", pkg)
	deps := parseDeps(depsString)

	nonGithubDeps := []string{}
	for _, dep := range deps {
		dep = strings.TrimSpace(dep)
		if dep == "" || dep == "." {
			continue
		}
		if isGithub(dep) {
			fetchSubtree(dep, branch, alreadyFetched)
		} else {
			nonGithubDeps = append(nonGithubDeps, dep)
		}
	}

	for _, dep := range nonGithubDeps {
		goGet(dep, alreadyFetched)
	}
}

func goGet(pkg string, alreadyFetched map[string]bool) {
	if alreadyFetched[pkg] {
		return
	}
	run("go", "get", pkg)
	alreadyFetched[pkg] = true
}

func writeAndCommit(file string, content string, comment string) {
	if exists(file) {
		log.Fatalf("%s already contains %s, can't initialize gost", dir, file)
	}

	err := ioutil.WriteFile(file, []byte(content), 0644)
	if err != nil {
		log.Fatalf("Unable to write %s: %s", file, err)
{'fraction_non_alphanumeric': 0.1754892524863651, 'fraction_numerical': 0.004170676932948348, 'mean_word_length': 4.004012841091493, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '384236', 'n_tokens_mistral': 2512, 'n_tokens_neox': 2371, 'n_words': 605}