<commit_before>package golang

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"path/filepath"

	"strings"

	"github.com/golang/gddo/gosrc"
	"github.com/peterbourgon/diskv"
	"github.com/sourcegraph/httpcache"
	"github.com/sourcegraph/httpcache/diskcache"
	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/container"
	"sourcegraph.com/sourcegraph/srcgraph/dep2"
	"sourcegraph.com/sourcegraph/srcgraph/task2"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

func init() {
	dep2.RegisterLister(&Package{}, dep2.DockerLister{defaultGoVersion})
	dep2.RegisterResolver(goImportPathTargetType, defaultGoVersion)
}

func (v *goVersion) BuildLister(dir string, unit unit.SourceUnit, c *config.Repository, x *task2.Context) (*container.Command, error) {
	goConfig := v.goConfig(c)
	pkg := unit.(*Package)

	dockerfile, err := v.baseDockerfile()
	if err != nil {
		return nil, err
	}

	containerDir := filepath.Join(containerGOPATH, "src", goConfig.BaseImportPath)
	cmd := container.Command{
		Container: container.Container{
			Dockerfile: dockerfile,
			RunOptions: []string{"-v", dir + ":" + containerDir},
			Cmd:        []string{"go", "list", "-e", "-f", `{{join .Imports "\n"}}{{join .TestImports "\n"}}{{join .XTestImports "\n"}}`, pkg.ImportPath},
		},
		Transform: func(orig []byte) ([]byte, error) {
			importPaths := strings.Split(string(orig), "\n")
			var deps []*dep2.RawDependency
			for _, importPath := range importPaths {
				if importPath == "" {
					continue
				}
				deps = append(deps, &dep2.RawDependency{
					TargetType: goImportPathTargetType,
					Target:     goImportPath(importPath),
				})
			}

			return json.Marshal(deps)
		},
	}
	return &cmd, nil
}

// goImportPath represents a Go import path, such as "github.com/user/repo" or
// "net/http".
type goImportPath string

const goImportPathTargetType = "go-import-path"<commit_msg>fix dupe handling<commit_after>package golang

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"path/filepath"

	"strings"

	"github.com/golang/gddo/gosrc"
	"github.com/peterbourgon/diskv"
	"github.com/sourcegraph/httpcache"
	"github.com/sourcegraph/httpcache/diskcache"
	"sourcegraph.com/sourcegraph/srcgraph/config"
	"sourcegraph.com/sourcegraph/srcgraph/container"
	"sourcegraph.com/sourcegraph/srcgraph/dep2"
	"sourcegraph.com/sourcegraph/srcgraph/task2"
	"sourcegraph.com/sourcegraph/srcgraph/unit"
)

func init() {
	dep2.RegisterLister(&Package{}, dep2.DockerLister{defaultGoVersion})
	dep2.RegisterResolver(goImportPathTargetType, defaultGoVersion)
}

func (v *goVersion) BuildLister(dir string, unit unit.SourceUnit, c *config.Repository, x *task2.Context) (*container.Command, error) {
	goConfig := v.goConfig(c)
	pkg := unit.(*Package)

	dockerfile, err := v.baseDockerfile()
	if err != nil {
		return nil, err
	}

	containerDir := filepath.Join(containerGOPATH, "src", goConfig.BaseImportPath)
	cmd := container.Command{
		Container: container.Container{
			Dockerfile: dockerfile,
			RunOptions: []string{"-v", dir + ":" + containerDir},
			Cmd:        []string{"go", "list", "-e", "-f", `{{join .Imports "\n"}}` + "\n" + `{{join .TestImports "\n"}}` + "\n" + `{{join .XTestImports "\n"}}`, pkg.ImportPath},
		},
		Transform: func(orig []byte) ([]byte, error) {
			importPaths := strings.Split(string(orig), "\n")
			seen := make(map[string]struct{})
			var deps []*dep2.RawDependency
			for _, importPath := range importPaths {
				if importPath == "" {
					continue
				}
				if _, seen := seen[importPath]; seen {
					continue
				}
				seen[importPath] = struct{}{}
				deps = append(deps, &dep2.RawDependency{
					TargetType: goImportPathTargetType,
					Target:     goImportPath(importPath),
				})
			}

			return json.Marshal(deps)
		},
	}
	return &cmd, nil
}

// goImportPath represents a Go import path, such as "github.com/user/repo" or
// "net/http".
type goImportPath string

const goImportPathTargetType = "go-import-path"
{'fraction_non_alphanumeric': 0.15830800405268491, 'fraction_numerical': 0.004052684903748734, 'mean_word_length': 4.322102425876011, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3795239', 'n_tokens_mistral': 1507, 'n_tokens_neox': 1381, 'n_words': 262}