<commit_before>package buildbot

import (
	"encoding/json"
	"fmt"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/skia-dev/glog"

	"go.skia.org/infra/go/gitinfo"
	"go.skia.org/infra/go/util"
)

var (
	// TODO(borenet): Avoid hard-coding this list. Instead, obtain it from
	// checked-in code or the set of masters which are actually running.
	MASTER_NAMES = []string{"client.skia", "client.skia.android", "client.skia.compile", "client.skia.fyi"}
	httpGet      = util.NewTimeoutClient().Get
)

// get loads data from a buildbot JSON endpoint.
func get(url string, rv interface{}) error {
	resp, err := httpGet(url)
	if err != nil {
		return fmt.Errorf("Failed to GET %s: %v", url, err)
	}
	defer util.Close(resp.Body)
	dec := json.NewDecoder(resp.Body)
	if err := dec.Decode(rv); err != nil {
		return fmt.Errorf("Failed to decode JSON: %v", err)
	}
	return nil
}

// findCommitsRecursive is a recursive function called by findCommitsForBuild.
// It traces the history to find builds which were first included in the given
// build.
func findCommitsRecursive(commits map[string]bool, b *Build, hash string, repo *gitinfo.GitInfo) error {
	// Shortcut for empty hashes. This can happen when a commit has no
	// parents (initial commit) or when a Build has no GotRevision.
	if hash == "" {
		return nil
	}

	// Determine whether any build already includes this commit.
	n, err := GetBuildForCommit(b.Builder, b.Master, hash)
	if err != nil {
		return fmt.Errorf("Could not find build for commit %s: %v", hash, err)
	}
	// If so, stop. If the build we found is the current build, keep going,
	// since we may have already ingested data for this build but still
	// need to find accurate revision data.
	if n >= 0 && n != b.Number {
		return nil
	}

	// Add the commit.
	commits[hash] = true

	// Recurse on the commit's parents.
	c, err := repo.Details(hash)
	if err != nil {
		return fmt.Errorf("Failed to obtain details for %s: %v", hash, err)
	}
	for _, p := range c.Parents {
		// If we've already seen this parent commit, don't revisit it.
		if _, ok := commits[p]; ok {<commit_msg>Skip "bad" commits during buildbot ingestion<commit_after>package buildbot

import (
	"encoding/json"
	"fmt"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/skia-dev/glog"

	"go.skia.org/infra/go/gitinfo"
	"go.skia.org/infra/go/util"
)

var (
	// TODO(borenet): Avoid hard-coding this list. Instead, obtain it from
	// checked-in code or the set of masters which are actually running.
	MASTER_NAMES = []string{"client.skia", "client.skia.android", "client.skia.compile", "client.skia.fyi"}
	httpGet      = util.NewTimeoutClient().Get
)

// get loads data from a buildbot JSON endpoint.
func get(url string, rv interface{}) error {
	resp, err := httpGet(url)
	if err != nil {
		return fmt.Errorf("Failed to GET %s: %v", url, err)
	}
	defer util.Close(resp.Body)
	dec := json.NewDecoder(resp.Body)
	if err := dec.Decode(rv); err != nil {
		return fmt.Errorf("Failed to decode JSON: %v", err)
	}
	return nil
}

// findCommitsRecursive is a recursive function called by findCommitsForBuild.
// It traces the history to find builds which were first included in the given
// build.
func findCommitsRecursive(commits map[string]bool, b *Build, hash string, repo *gitinfo.GitInfo) error {
	// Shortcut for empty hashes. This can happen when a commit has no
	// parents (initial commit) or when a Build has no GotRevision.
	if hash == "" {
		return nil
	}

	// Determine whether any build already includes this commit.
	n, err := GetBuildForCommit(b.Builder, b.Master, hash)
	if err != nil {
		return fmt.Errorf("Could not find build for commit %s: %v", hash, err)
	}
	// If so, stop. If the build we found is the current build, keep going,
	// since we may have already ingested data for this build but still
	// need to find accurate revision data.
	if n >= 0 && n != b.Number {
		return nil
	}

	// Add the commit.
	commits[hash] = true

	// Recurse on the commit's parents.
	c, err := repo.Details(hash)
	if err != nil {
		// Special case. Commits can disappear from the repository
		// after they're picked up by the buildbots but before they're
		// ingested here. If we can't find a commit, log an error and
		// skip the commit.
		glog.Errorf("Failed to obtain details for %s: %v", hash, err)
		delete(commits, hash)
		return nil
	}
	for _, p := range c.Parents {
		// If we've already seen this parent commit, don't revisit it.
		if _, ok := commits[p]; ok {
{'fraction_non_alphanumeric': 0.12331233123312331, 'fraction_numerical': 0.00045004500450045, 'mean_word_length': 3.4405594405594404, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2058736', 'n_tokens_mistral': 1534, 'n_tokens_neox': 1469, 'n_words': 551}