<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/google/zoekt"
	"github.com/google/zoekt/build"
	git "github.com/libgit2/git2go"
)

var _ = log.Println

func guessRepoURL(repoDir string) (string, error) {
	base, err := git.NewConfig()
	if err != nil {
		return "", err
	}
	cfg, err := git.OpenOndisk(base, filepath.Join(repoDir, "config"))
	if err != nil {
		return "", err
	}

	url, err := cfg.LookupString("remote.origin.url")
	if err != nil {
		return "", err
	}

	if strings.Contains(url, "googlesource.com/") {
		/// eg. https://gerrit.googlesource.com/gitiles/+/master/.buckversion
		return url + "/+/{{.Branch}}/{{.Path}}", nil
	} else if strings.Contains(url, "github.com/") {
		// eg. https://github.com/hanwen/go-fuse/blob/notify/genversion.sh
		return url + "/blob/{{.Branch}}/{{.Path}}", nil
	}

	return "", fmt.Errorf("scheme unknown for URL %s", url)
}

func main() {
	var sizeMax = flag.Int("file_limit", 128*1024, "maximum file size")
	var shardLimit = flag.Int("shard_limit", 100<<20, "maximum corpus size for a shard")
	var parallelism = flag.Int("parallelism", 4, "maximum number of parallel indexing processes.")
	var recursive = flag.Bool("recursive", false, "recurse into directories to index all git repos")
	submodules := flag.Bool("submodules", true, "if set to false, do not recurse into submodules")
	branchesStr := flag.String("branches", "master", "git branches to index. If set, arguments should be bare git repositories.")
	branchPrefix := flag.String("prefix", "refs/heads/", "prefix for branch names")

	indexDir := flag.String("index", build.DefaultDir, "index directory for *.zoekt files.")
	flag.Parse()

	opts := build.Options{
		Parallelism: *parallelism,
		SizeMax:     *sizeMax,
		ShardMax:    *shardLimit,
		IndexDir:    *indexDir,
	}
	opts.SetDefaults()

	var branches []string
	if *branchesStr != "" {
		branches = strings.Split(*branchesStr, ",")
	}

	gitRepos := map[string]string{}
	if *recursive {
		for _, arg := range flag.Args() {
			repos, err := findGitRepos(arg)
			if err != nil {
				log.Fatal(err)
			}
			for k, v := range repos {
				gitRepos[k] = v
			}
		}
	} else {
		for _, repoDir := range flag.Args() {
			if _, err := os.Lstat(filepath.Join(repoDir, ".git")); err == nil {
				repoDir = filepath.Join(repoDir, ".git")
			}
			repoDir, err := filepath.Abs(repoDir)
			if err != nil {
				log.Fatal(err)
			}

			name := filepath.Base(repoDir)
			if name == ".git" {
				name = filepath.Base(filepath.Dir(repoDir))
			}
			name = strings.TrimSuffix(name, ".git")

			gitRepos[repoDir] = name
		}
	}
	exitStatus := 0
	for dir, name := range gitRepos {
		opts.RepoName = name
		opts.RepoDir = filepath.Clean(dir)
		log.Printf("indexing %s (%s)", dir, name)
		if err := indexGitRepo(opts, *branchPrefix, branches, *submodules); err != nil {
			log.Printf("indexGitRepo(%s): %v", dir, err)
			exitStatus = 1
		}
	}
	os.Exit(exitStatus)
}

// findGitRepos finds git repositories and returns repodir => name map.
func findGitRepos(arg string) (map[string]string, error) {
	arg, err := filepath.Abs(arg)
	if err != nil {
		return nil, err
	}
	var dirs []string
	gitDirs := map[string]string{}
	if err := filepath.Walk(arg, func(name string, fi os.FileInfo, err error) error {
		fi, err = os.Lstat(filepath.Join(name, ".git"))
		if err == nil && fi.IsDir() {
			dirs = append(dirs, filepath.Join(name, ".git"))
			return filepath.SkipDir
		}

		if !strings.HasSuffix(name, ".git") || !fi.IsDir() {
			return nil
		}

		fi, err = os.Lstat(filepath.Join(name, "objects"))
		if err != nil && !fi.IsDir() {
			return nil
		}

		dirs = append(dirs, name)
		return filepath.SkipDir
	}); err != nil {
		return nil, err
	}

	for _, dir := range dirs {
		name := strings.TrimSuffix(dir, ".git")
		name = strings.TrimSuffix(name, "/")<commit_msg>Fix indexing of foo.git bare repositories.<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/google/zoekt"
	"github.com/google/zoekt/build"
	git "github.com/libgit2/git2go"
)

var _ = log.Println

func guessRepoURL(repoDir string) (string, error) {
	base, err := git.NewConfig()
	if err != nil {
		return "", err
	}
	cfg, err := git.OpenOndisk(base, filepath.Join(repoDir, "config"))
	if err != nil {
		return "", err
	}

	url, err := cfg.LookupString("remote.origin.url")
	if err != nil {
		return "", err
	}

	if strings.Contains(url, "googlesource.com/") {
		/// eg. https://gerrit.googlesource.com/gitiles/+/master/.buckversion
		return url + "/+/{{.Branch}}/{{.Path}}", nil
	} else if strings.Contains(url, "github.com/") {
		// eg. https://github.com/hanwen/go-fuse/blob/notify/genversion.sh
		return url + "/blob/{{.Branch}}/{{.Path}}", nil
	}

	return "", fmt.Errorf("scheme unknown for URL %s", url)
}

func main() {
	var sizeMax = flag.Int("file_limit", 128*1024, "maximum file size")
	var shardLimit = flag.Int("shard_limit", 100<<20, "maximum corpus size for a shard")
	var parallelism = flag.Int("parallelism", 4, "maximum number of parallel indexing processes.")
	var recursive = flag.Bool("recursive", false, "recurse into directories to index all git repos")
	submodules := flag.Bool("submodules", true, "if set to false, do not recurse into submodules")
	branchesStr := flag.String("branches", "master", "git branches to index. If set, arguments should be bare git repositories.")
	branchPrefix := flag.String("prefix", "refs/heads/", "prefix for branch names")

	indexDir := flag.String("index", build.DefaultDir, "index directory for *.zoekt files.")
	flag.Parse()

	opts := build.Options{
		Parallelism: *parallelism,
		SizeMax:     *sizeMax,
		ShardMax:    *shardLimit,
		IndexDir:    *indexDir,
	}
	opts.SetDefaults()

	var branches []string
	if *branchesStr != "" {
		branches = strings.Split(*branchesStr, ",")
	}

	gitRepos := map[string]string{}
	if *recursive {
		for _, arg := range flag.Args() {
			repos, err := findGitRepos(arg)
			if err != nil {
				log.Fatal(err)
			}
			for k, v := range repos {
				gitRepos[k] = v
			}
		}
	} else {
		for _, repoDir := range flag.Args() {
			if _, err := os.Lstat(filepath.Join(repoDir, ".git")); err == nil {
				repoDir = filepath.Join(repoDir, ".git")
			}
			repoDir, err := filepath.Abs(repoDir)
			if err != nil {
				log.Fatal(err)
			}

			name := filepath.Base(repoDir)
			if name == ".git" {
				name = filepath.Base(filepath.Dir(repoDir))
			}
			name = strings.TrimSuffix(name, ".git")

			gitRepos[repoDir] = name
		}
	}
	exitStatus := 0
	for dir, name := range gitRepos {
		opts.RepoName = name
		opts.RepoDir = filepath.Clean(dir)
		log.Printf("indexing %s (%s)", dir, name)
		if err := indexGitRepo(opts, *branchPrefix, branches, *submodules); err != nil {
			log.Printf("indexGitRepo(%s): %v", dir, err)
			exitStatus = 1
		}
	}
	os.Exit(exitStatus)
}

// findGitRepos finds git repositories and returns repodir => name map.
func findGitRepos(arg string) (map[string]string, error) {
	arg, err := filepath.Abs(arg)
	if err != nil {
		return nil, err
	}
	var dirs []string
	gitDirs := map[string]string{}
	if err := filepath.Walk(arg, func(name string, fi os.FileInfo, err error) error {
		if fi, err := os.Lstat(filepath.Join(name, ".git")); err == nil && fi.IsDir() {
			dirs = append(dirs, filepath.Join(name, ".git"))
			return filepath.SkipDir
		}

		if !strings.HasSuffix(name, ".git") || !fi.IsDir() {
			return nil
		}

		fi, err = os.Lstat(filepath.Join(name, "objects"))
		if err != nil && !fi.IsDir() {
			return nil
		}

		dirs = append(dirs, name)
		return filepath.SkipDir
	}); err != nil {
		return nil, err
	}

	for _, dir := range dirs {
		name := strings.TrimSuffix(dir, ".git")
		name = strings.TrimSuffix(name, "/")
{'fraction_non_alphanumeric': 0.15798281825281713, 'fraction_numerical': 0.005578489345085351, 'mean_word_length': 3.5364372469635628, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3279309', 'n_tokens_mistral': 3390, 'n_tokens_neox': 3091, 'n_words': 873}