<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"

	"golang.org/x/net/publicsuffix"
	"golang.org/x/sync/errgroup"
	"golang.org/x/tools/go/vcs"
)

type packageType int

const (
	typeGuess packageType = iota
	typeLibrary
	typeProgram
	typeLibraryProgram
	typeProgramLibrary
)

var wrapAndSort string

func passthroughEnv() []string {
	var relevantVariables = []string{
		"HOME",
		"PATH",
		"HTTP_PROXY", "http_proxy",
		"HTTPS_PROXY", "https_proxy",
		"ALL_PROXY", "all_proxy",
		"NO_PROXY", "no_proxy",
		"GIT_PROXY_COMMAND",
		"GIT_HTTP_PROXY_AUTHMETHOD",
	}
	var result []string
	for _, variable := range relevantVariables {
		if value, ok := os.LookupEnv(variable); ok {
			result = append(result, fmt.Sprintf("%s=%s", variable, value))
		}
	}
	return result
}

func findVendorDirs(dir string) ([]string, error) {
	var vendorDirs []string
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info != nil && !info.IsDir() {
			return nil // nothing to do for anything but directories
		}
		if info.Name() == ".git" ||
			info.Name() == ".hg" ||
			info.Name() == ".bzr" {
			return filepath.SkipDir
		}
		if info.Name() == "vendor" {
			rel, err := filepath.Rel(dir, path)
			if err != nil {
				return err
			}
			vendorDirs = append(vendorDirs, rel)
		}
		return nil
	})
	return vendorDirs, err
}

func downloadFile(filename, url string) error {
	dst, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer dst.Close()

	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf(resp.Status)
	}

	_, err = io.Copy(dst, resp.Body)
	if err != nil {
		return err
	}

	return nil
}

// upstream describes the upstream repo we are about to package.
type upstream struct {
	rr          *vcs.RepoRoot
	tarPath     string   // path to the downloaded or generated orig tarball tempfile
	compression string   // compression method, either "gz" or "xz"
	version     string   // Debian package upstream version number, e.g. 0.0~git20180204.1d24609
	commitIsh   string   // commit-ish corresponding to upstream version to be packaged
	remote      string   // git remote, set to short hostname if upstream git history is included
	firstMain   string   // import path of the first main package within repo, if any
	vendorDirs  []string // all vendor sub directories, relative to the repo directory
	repoDeps    []string // the repository paths of all dependencies (e.g. github.com/zyedidia/glob)
	hasGodeps   bool     // whether the Godeps/_workspace directory exists
	hasRelease  bool     // whether any release tags exist, for debian/watch
	isRelease   bool     // whether what we end up packaging is a tagged release
}

func (u *upstream) get(gopath, repo, rev string) error {
	done := make(chan struct{})
	defer close(done)
	go progressSize("go get", filepath.Join(gopath, "src"), done)

	rr, err := vcs.RepoRootForImportPath(repo, false)
	if err != nil {
		return err
	}
	u.rr = rr
	dir := filepath.Join(gopath, "src", rr.Root)
	if rev != "" {
		return rr.VCS.CreateAtRev(dir, rr.Repo, rev)
	}
	return rr.VCS.Create(dir, rr.Repo)
}

func (u *upstream) tarballFromHoster(repo string) error {
	var url string
	parts := strings.Split(repo, "/")
	if len(parts) < 3 {
		return fmt.Errorf("Unsupported hoster")
	}
	host, owner, project := parts[0], parts[1], parts[2]

	switch host {
	case "github.com":
		url = fmt.Sprintf("https://%s/%s/%s/archive/v%s.tar.%s",
			host, owner, project, u.version, u.compression)
	case "gitlab.com":
		url = fmt.Sprintf("https://%s/%s/%s/-/archive/v%s/%s-%s.tar.%s",
			host, owner, project, u.version, project, u.version, u.compression)
	default:
		return fmt.Errorf("Unsupported hoster")
	}

	done := make(chan struct{})
	go progressSize("Download", u.tarPath, done)

	log.Printf("Downloading %s", url)
	err := downloadFile(u.tarPath, url)

	close(done)

	return err
}

func (u *upstream) tar(gopath, repo string) error {
	f, err := ioutil.TempFile("", "dh-make-golang")
	if err != nil {
		return err
	}
	u.tarPath = f.Name()
	f.Close()

	if u.isRelease {
		if u.hasGodeps {
			log.Printf("Godeps/_workspace exists, not downloading tarball from hoster.")
		} else {
			u.compression = "gz"
			err := u.tarballFromHoster(repo)
			if err != nil && err.Error() == "Unsupported hoster" {<commit_msg>Download upstream tarball from GitHub or GitLab<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"

	"golang.org/x/net/publicsuffix"
	"golang.org/x/sync/errgroup"
	"golang.org/x/tools/go/vcs"
)

type packageType int

const (
	typeGuess packageType = iota
	typeLibrary
	typeProgram
	typeLibraryProgram
	typeProgramLibrary
)

var wrapAndSort string

func passthroughEnv() []string {
	var relevantVariables = []string{
		"HOME",
		"PATH",
		"HTTP_PROXY", "http_proxy",
		"HTTPS_PROXY", "https_proxy",
		"ALL_PROXY", "all_proxy",
		"NO_PROXY", "no_proxy",
		"GIT_PROXY_COMMAND",
		"GIT_HTTP_PROXY_AUTHMETHOD",
	}
	var result []string
	for _, variable := range relevantVariables {
		if value, ok := os.LookupEnv(variable); ok {
			result = append(result, fmt.Sprintf("%s=%s", variable, value))
		}
	}
	return result
}

func findVendorDirs(dir string) ([]string, error) {
	var vendorDirs []string
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info != nil && !info.IsDir() {
			return nil // nothing to do for anything but directories
		}
		if info.Name() == ".git" ||
			info.Name() == ".hg" ||
			info.Name() == ".bzr" {
			return filepath.SkipDir
		}
		if info.Name() == "vendor" {
			rel, err := filepath.Rel(dir, path)
			if err != nil {
				return err
			}
			vendorDirs = append(vendorDirs, rel)
		}
		return nil
	})
	return vendorDirs, err
}

func downloadFile(filename, url string) error {
	dst, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer dst.Close()

	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return fmt.Errorf(resp.Status)
	}

	_, err = io.Copy(dst, resp.Body)
	if err != nil {
		return err
	}

	return nil
}

// upstream describes the upstream repo we are about to package.
type upstream struct {
	rr          *vcs.RepoRoot
	tarPath     string   // path to the downloaded or generated orig tarball tempfile
	compression string   // compression method, either "gz" or "xz"
	version     string   // Debian package upstream version number, e.g. 0.0~git20180204.1d24609
	commitIsh   string   // commit-ish corresponding to upstream version to be packaged
	remote      string   // git remote, set to short hostname if upstream git history is included
	firstMain   string   // import path of the first main package within repo, if any
	vendorDirs  []string // all vendor sub directories, relative to the repo directory
	repoDeps    []string // the repository paths of all dependencies (e.g. github.com/zyedidia/glob)
	hasGodeps   bool     // whether the Godeps/_workspace directory exists
	hasRelease  bool     // whether any release tags exist, for debian/watch
	isRelease   bool     // whether what we end up packaging is a tagged release
}

func (u *upstream) get(gopath, repo, rev string) error {
	done := make(chan struct{})
	defer close(done)
	go progressSize("go get", filepath.Join(gopath, "src"), done)

	rr, err := vcs.RepoRootForImportPath(repo, false)
	if err != nil {
		return err
	}
	u.rr = rr
	dir := filepath.Join(gopath, "src", rr.Root)
	if rev != "" {
		return rr.VCS.CreateAtRev(dir, rr.Repo, rev)
	}
	return rr.VCS.Create(dir, rr.Repo)
}

func (u *upstream) tarballFromHoster() error {
	var tarURL string
	repo := strings.TrimSuffix(u.rr.Repo, ".git")
	repoU, err := url.Parse(repo)

	switch repoU.Host {
	case "github.com":
		tarURL = fmt.Sprintf("%s/archive/v%s.tar.%s",
			repo, u.version, u.compression)
	case "gitlab.com":
		parts := strings.Split(repoU.Path, "/")
		if len(parts) < 3 {
			return fmt.Errorf("Incomplete repo URL: %s", u.rr.Repo)
		}
		project := parts[2]
		tarURL = fmt.Sprintf("%s/-/archive/v%[3]s/%[2]s-%s.tar.%s",
			repo, project, u.version, u.compression)
	default:
		return fmt.Errorf("Unsupported hoster")
	}

	done := make(chan struct{})
	go progressSize("Download", u.tarPath, done)

	log.Printf("Downloading %s", tarURL)
	err = downloadFile(u.tarPath, tarURL)

	close(done)

	return err
}

func (u *upstream) tar(gopath, repo string) error {
	f, err := ioutil.TempFile("", "dh-make-golang")
	if err != nil {
		return err
	}
	u.tarPath = f.Name()
	f.Close()

	if u.isRelease {
		if u.hasGodeps {
			log.Printf("Godeps/_workspace exists, not downloading tarball from hoster.")
		} else {
			u.compression = "gz"
			err := u.tarballFromHoster()
			if err != nil && err.Error() == "Unsupported hoster" {
{'fraction_non_alphanumeric': 0.13792337590227652, 'fraction_numerical': 0.005108273181565797, 'mean_word_length': 3.4761431411530817, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6006754', 'n_tokens_mistral': 3458, 'n_tokens_neox': 3229, 'n_words': 906}