<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Fetchlogs downloads build failure logs from the Go dashboard so
// they can be accessed and searched from the local file system.
//
// It organizes these logs into two directories created in the
// directory specified by the -dir flag (which typically defaults to
// ~/.cache/fetchlogs). The log/ directory contains all log files
// named the same way they are named by the dashboard (which happens
// to be the SHA-1 of their contents). The rev/ directory contains
// symlinks back to these logs named
//
//	rev/<ISO 8601 commit date>-<git revision>/<builder>
//
// Fetchlogs will reuse existing log files and revision symlinks, so
// it only has to download logs that are new since the last time it
// was run.
//
// This makes failures easily searchable with standard tools. For
// example, to list the revisions and builders with a particular
// failure, use:
//
//	grep -lR <regexp> rev | sort
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"golang.org/x/build/maintner"
	"golang.org/x/build/maintner/godata"
	"golang.org/x/build/repos"
	"golang.org/x/build/types"
)

var defaultDir = filepath.Join(xdgCacheDir(), "fetchlogs")

var (
	flagN         = flag.Int("n", 300, "limit to most recent `N` commits per repo")
	flagPar       = flag.Int("j", 5, "number of concurrent download `jobs`")
	flagDir       = flag.String("dir", defaultDir, "`directory` to save logs to")
	flagRepo      = flag.String("repo", "go", `comma-separated list of repos to fetch logs for, or "all" for all known repos`)
	flagBranch    = flag.String("branch", "", `comma-separated list of Go repo branches to fetch logs for; default branch if empty`)
	flagDashboard = flag.String("dashboard", "https://build.golang.org", `the dashboard root url`)
)

func main() {
	log.SetPrefix("fetchlogs: ")
	log.SetFlags(0)

	flag.Parse()
	if flag.NArg() != 0 {
		flag.Usage()
		os.Exit(2)
	}

	// If the top-level directory is the default XDG cache
	// directory, make sure it exists.
	if *flagDir == defaultDir {
		if err := xdgCreateDir(*flagDir); err != nil {
			log.Fatal(err)
		}
	}

	// Create directory structure.
	if err := os.Chdir(*flagDir); err != nil {
		log.Fatal(err)
	}
	ensureDir("log")
	ensureDir("rev")

	// Set up fetchers.
	fetcher := newFetcher(*flagPar)
	wg := sync.WaitGroup{}

	// Fetch dashboard pages.
	for _, repo := range parseRepoFlag() {
		for _, branch := range strings.Split(*flagBranch, ",") {
			project := repo.GoGerritProject
			haveCommits := 0
			for page := 0; haveCommits < *flagN; page++ {
				dashURL := fmt.Sprintf("%s/?mode=json&page=%d", *flagDashboard, page)
				if project != "go" {
					dashURL += "&repo=" + url.QueryEscape(repo.ImportPath)
				}
				if branch != "" {
					dashURL += "&branch=" + url.QueryEscape(branch)
				}
				index, err := fetcher.get(dashURL)
				if err != nil {
					log.Fatal(err)
				}

				var status types.BuildStatus
				if err = json.NewDecoder(index).Decode(&status); err != nil {
					log.Fatal("error unmarshalling result: ", err)
				}
				index.Close()

				if len(status.Revisions) == 0 {
					// We asked for a page of revisions and received a valid reply with none.
					// Assume that there are no more beyond this.
					break
				}

				for _, rev := range status.Revisions {
					if haveCommits >= *flagN {
						break
					}
					if rev.Repo != project {
						// The results for the "go" repo (fetched without the "&repo" query
						// parameter) empirically include some subrepo results for release
						// branches.
						//
						// Those aren't really relevant to the "go" repo — and they should be
						// included when we fetch the subrepo explicitly anyway — so filter
						// them out here.
						continue
					}
					haveCommits++

					// Create a revision directory. This way we
					// have a record of commits with no failures.
					date, err := parseRevDate(rev.Date)
					if err != nil {
						log.Fatal("malformed revision date: ", err)
					}
					var goDate time.Time
					if rev.GoRevision != "" {
						commit, err := goProject(useCached).GitCommit(rev.GoRevision)
						if err != nil {
							// A rare race is possible here: if a commit is added to the Go repo
							// after the initial maintner load, and a dashboard test run completes
							// for that commit before we're done fetching logs, the maintner data
							// might not include that commit. To rule out that possibility, refresh
							// the local maintner data before bailing out.
							commit, err = goProject(forceRefresh).GitCommit(rev.GoRevision)
							if err == nil {
								log.Fatal("invalid GoRevision: ", err)
							}
						}
						goDate = commit.CommitTime
					}
					revDir, revDirDepth := revToDir(rev.Revision, date, rev.GoRevision, goDate)<commit_msg>cmd/fetchlogs: fix a typo'd comparison from CL 408937<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Fetchlogs downloads build failure logs from the Go dashboard so
// they can be accessed and searched from the local file system.
//
// It organizes these logs into two directories created in the
// directory specified by the -dir flag (which typically defaults to
// ~/.cache/fetchlogs). The log/ directory contains all log files
// named the same way they are named by the dashboard (which happens
// to be the SHA-1 of their contents). The rev/ directory contains
// symlinks back to these logs named
//
//	rev/<ISO 8601 commit date>-<git revision>/<builder>
//
// Fetchlogs will reuse existing log files and revision symlinks, so
// it only has to download logs that are new since the last time it
// was run.
//
// This makes failures easily searchable with standard tools. For
// example, to list the revisions and builders with a particular
// failure, use:
//
//	grep -lR <regexp> rev | sort
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"golang.org/x/build/maintner"
	"golang.org/x/build/maintner/godata"
	"golang.org/x/build/repos"
	"golang.org/x/build/types"
)

var defaultDir = filepath.Join(xdgCacheDir(), "fetchlogs")

var (
	flagN         = flag.Int("n", 300, "limit to most recent `N` commits per repo")
	flagPar       = flag.Int("j", 5, "number of concurrent download `jobs`")
	flagDir       = flag.String("dir", defaultDir, "`directory` to save logs to")
	flagRepo      = flag.String("repo", "go", `comma-separated list of repos to fetch logs for, or "all" for all known repos`)
	flagBranch    = flag.String("branch", "", `comma-separated list of Go repo branches to fetch logs for; default branch if empty`)
	flagDashboard = flag.String("dashboard", "https://build.golang.org", `the dashboard root url`)
)

func main() {
	log.SetPrefix("fetchlogs: ")
	log.SetFlags(0)

	flag.Parse()
	if flag.NArg() != 0 {
		flag.Usage()
		os.Exit(2)
	}

	// If the top-level directory is the default XDG cache
	// directory, make sure it exists.
	if *flagDir == defaultDir {
		if err := xdgCreateDir(*flagDir); err != nil {
			log.Fatal(err)
		}
	}

	// Create directory structure.
	if err := os.Chdir(*flagDir); err != nil {
		log.Fatal(err)
	}
	ensureDir("log")
	ensureDir("rev")

	// Set up fetchers.
	fetcher := newFetcher(*flagPar)
	wg := sync.WaitGroup{}

	// Fetch dashboard pages.
	for _, repo := range parseRepoFlag() {
		for _, branch := range strings.Split(*flagBranch, ",") {
			project := repo.GoGerritProject
			haveCommits := 0
			for page := 0; haveCommits < *flagN; page++ {
				dashURL := fmt.Sprintf("%s/?mode=json&page=%d", *flagDashboard, page)
				if project != "go" {
					dashURL += "&repo=" + url.QueryEscape(repo.ImportPath)
				}
				if branch != "" {
					dashURL += "&branch=" + url.QueryEscape(branch)
				}
				index, err := fetcher.get(dashURL)
				if err != nil {
					log.Fatal(err)
				}

				var status types.BuildStatus
				if err = json.NewDecoder(index).Decode(&status); err != nil {
					log.Fatal("error unmarshalling result: ", err)
				}
				index.Close()

				if len(status.Revisions) == 0 {
					// We asked for a page of revisions and received a valid reply with none.
					// Assume that there are no more beyond this.
					break
				}

				for _, rev := range status.Revisions {
					if haveCommits >= *flagN {
						break
					}
					if rev.Repo != project {
						// The results for the "go" repo (fetched without the "&repo" query
						// parameter) empirically include some subrepo results for release
						// branches.
						//
						// Those aren't really relevant to the "go" repo — and they should be
						// included when we fetch the subrepo explicitly anyway — so filter
						// them out here.
						continue
					}
					haveCommits++

					// Create a revision directory. This way we
					// have a record of commits with no failures.
					date, err := parseRevDate(rev.Date)
					if err != nil {
						log.Fatal("malformed revision date: ", err)
					}
					var goDate time.Time
					if rev.GoRevision != "" {
						commit, err := goProject(useCached).GitCommit(rev.GoRevision)
						if err != nil {
							// A rare race is possible here: if a commit is added to the Go repo
							// after the initial maintner load, and a dashboard test run completes
							// for that commit before we're done fetching logs, the maintner data
							// might not include that commit. To rule out that possibility, refresh
							// the local maintner data before bailing out.
							commit, err = goProject(forceRefresh).GitCommit(rev.GoRevision)
							if err != nil {
								log.Fatal("invalid GoRevision: ", err)
							}
						}
						goDate = commit.CommitTime
					}
					revDir, revDirDepth := revToDir(rev.Revision, date, rev.GoRevision, goDate)
{'fraction_non_alphanumeric': 0.12063838223632038, 'fraction_numerical': 0.0043616177636796195, 'mean_word_length': 2.8172531214528944, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2853115', 'n_tokens_mistral': 3771, 'n_tokens_neox': 3082, 'n_words': 1122}