<commit_before>package analysis

import (
	"fmt"
	"net/url"
	"sort"
	"sync"
	"time"

	"github.com/rcrowley/go-metrics"
	"github.com/skia-dev/glog"

	"skia.googlesource.com/buildbot.git/go/timer"
	"skia.googlesource.com/buildbot.git/go/util"
	"skia.googlesource.com/buildbot.git/golden/go/diff"
	"skia.googlesource.com/buildbot.git/golden/go/expstorage"
	"skia.googlesource.com/buildbot.git/golden/go/types"
	"skia.googlesource.com/buildbot.git/perf/go/human"
	ptypes "skia.googlesource.com/buildbot.git/perf/go/types"
)

var (
	// The number of times we've successfully loaded and processed a tile.
	runsCounter metrics.Counter

	// The number of times an error has ocurred when trying to load a tile.
	errorTileLoadingCounter metrics.Counter
)

func init() {
	runsCounter = metrics.NewRegisteredCounter("analysis.runs", metrics.DefaultRegistry)
	errorTileLoadingCounter = metrics.NewRegisteredCounter("analysis.errors", metrics.DefaultRegistry)
}

type PathToURLConverter func(string) string

// LabeledTrace stores a Trace with labels and digests. CommitIds, Digests and
// Labels are of the same length, identical indices refer to the same digest.
type LabeledTrace struct {
	Params      map[string]string
	CommitIds   []int
	Digests     []string
	Labels      []types.Label
	Id          int
	IgnoreRules []*types.IgnoreRule
}

func NewLabeledTrace(params map[string]string, capacity int, traceId int) *LabeledTrace {
	return &LabeledTrace{
		Params:      params,
		CommitIds:   make([]int, 0, capacity),
		Digests:     make([]string, 0, capacity),
		Labels:      make([]types.Label, 0, capacity),
		Id:          traceId,
		IgnoreRules: []*types.IgnoreRule{},
	}
}

// addLabledDigests adds the given tripples of commitIds, digests and labels to this LabeledTrace.
func (lt *LabeledTrace) addLabeledDigests(commitIds []int, digests []string, labels []types.Label) {
	lt.CommitIds = append(lt.CommitIds, commitIds...)
	lt.Digests = append(lt.Digests, digests...)
	lt.Labels = append(lt.Labels, labels...)
}

// addIgnoreRules attaches the ignore rules that match his trace.
func (lt *LabeledTrace) addIgnoreRules(newRules []*types.IgnoreRule) {
	lt.IgnoreRules = append(lt.IgnoreRules, newRules...)
}

// LabeledTile aggregates the traces of a tile and provides a slice of commits
// that the commitIds in LabeledTrace refer to.
// LabeledTile and LabeledTrace store the cannonical information
// extracted from the unterlying tile store. The (redundant) output data is
// derived from these.
type LabeledTile struct {
	Commits []*ptypes.Commit

	// Traces are indexed by the primary key (test name). This is somewhat
	// redundant, but this also output format.
	Traces map[string][]*LabeledTrace

	// CommitsByDigest maps a corpus and a digest to a list of commit ids.
	// i.e. CommitsByDigest[corpus][digest] -> slice with indices of Commits.
	CommitsByDigest map[string]map[string][]int

	// Keeps track of unique ids for traces within this tile.
	traceIdCounter int
}

func NewLabeledTile() *LabeledTile {
	return &LabeledTile{
		Commits:         []*ptypes.Commit{},
		CommitsByDigest: map[string]map[string][]int{},
		Traces:          map[string][]*LabeledTrace{},
		traceIdCounter:  0,
	}
}

// getLabeledTrace is a utility function that returns the testName and a labeled
// trace for the given trace (read from a TileStore). If the LabeledTrace does
// not exist it will be added.
func (t *LabeledTile) getLabeledTrace(trace ptypes.Trace) (string, *LabeledTrace) {
	params := trace.Params()
	pKey := params[types.PRIMARY_KEY_FIELD]
	if _, ok := t.Traces[pKey]; !ok {
		// Add the primary key with a single labled trace.
		t.Traces[pKey] = []*LabeledTrace{}
	}

	// Search through the traces associated witht this test.
	for _, v := range t.Traces[pKey] {
		if util.MapsEqual(v.Params, params) {
			return pKey, v
		}
	}

	// If we cannot find the trace in our set of tests we are adding a new
	// labeled trace.
	newLT := NewLabeledTrace(params, trace.Len(), t.traceIdCounter)
	t.traceIdCounter++
	t.Traces[pKey] = append(t.Traces[pKey], newLT)
	return pKey, newLT
}

// LabelCounts is an output type to hold counts for classification labels.
type LabelCounts struct {
	Unt []int `json:"unt"` // Untriaged
	Pos []int `json:"pos"` // Positive
	Neg []int `json:"neg"` // Negative
}

func newLabelCounts(length int) *LabelCounts {
	return &LabelCounts{
		Unt: make([]int, length),
		Pos: make([]int, length),
		Neg: make([]int, length),
	}
}

// GUITileCounts is an output type for the aggregated label counts.
type GUITileCounts struct {
	Commits    []*ptypes.Commit        `json:"commits"`
	Ticks      []interface{}           `json:"ticks"`
	Aggregated *LabelCounts            `json:"aggregated"`
	Counts     map[string]*LabelCounts `json:"counts"`
	AllParams  map[string][]string     `json:"allParams"`
	Query      map[string][]string     `json:"query"`
}

// AnalyzeState captures the state of a partition of the incoming data.
// When a tile is read from disk it is partitioned into two tiles: current
// and ignored. current contains everything we want to be able to review
// continuously and ignored contains all ignored traces.
// This struct is the container for one of these partitions and the derived
// information.
type AnalyzeState struct {
	// Canonical data structure to hold our information about commits, digests
	// and labels.
	Tile *LabeledTile

	// Index to query the Tile.
	Index *LabeledTileIndex

	// Output data structures that are derived from Tile.
	TileCounts  *GUITileCounts
	TestDetails *GUITestDetails
	Status      *GUIStatus
}

// Analyzer continuously manages tasks like polling for new traces
// on disk and generating diffs between images. It is the primary interface
// to be called by the HTTP frontend.
type Analyzer struct {
	expStore    expstorage.ExpectationsStore
	diffStore   diff.DiffStore
	tileStore   ptypes.TileStore
	ignoreStore types.IgnoreStore

	current *AnalyzeState
	ignored *AnalyzeState

	// converter supplied by the client of the type to convert a path to a URL
	pathToURLConverter PathToURLConverter

	// Lock to protect the expectations and current* variables.
	mutex sync.RWMutex

	// Counts the number of times the main event loop has executed.
	// This is for testing only.
	loopCounter int
}

func NewAnalyzer(expStore expstorage.ExpectationsStore, tileStore ptypes.TileStore, diffStore diff.DiffStore, ignoreStore types.IgnoreStore, puConverter PathToURLConverter, timeBetweenPolls time.Duration) *Analyzer {
	result := &Analyzer{
		expStore:           expStore,
		diffStore:          diffStore,
		tileStore:          tileStore,
		ignoreStore:        ignoreStore,
		pathToURLConverter: puConverter,

		current: &AnalyzeState{},
		ignored: &AnalyzeState{},
	}

	go result.loop(timeBetweenPolls)
	return result
}

// GetTileCounts returns an entire Tile which is a collection of 'traces' over
// a series of commits. Each trace contains the digests and their labels
// based on our knowledge about digests (expectations).
func (a *Analyzer) GetTileCounts(query map[string][]string) (*GUITileCounts, error) {
	a.mutex.RLock()
	defer a.mutex.RUnlock()

	if len(query) > 0 {
		tile, effectiveQuery := a.getSubTile(query)
		if len(effectiveQuery) > 0 {
			ret := a.getOutputCounts(tile, a.current.Index)
			ret.Query = effectiveQuery
			return ret, nil
		}
	}

	return a.current.TileCounts, nil
}

// ListTestDetails returns a list of triage details based on the supplied
// query. It's complementary to GetTestDetails which returns a single test
// detail.
// TODO(stephana): This should provide pagination since the list is potentially
// very long. If we don't add pagination, this should be merged with
// GetTestDetail.
func (a *Analyzer) ListTestDetails(query map[string][]string) (*GUITestDetails, error) {
	a.mutex.RLock()
	defer a.mutex.RUnlock()

	if len(query) == 0 {
		return a.current.TestDetails, nil
	}

	effectiveQuery := make(map[string][]string, len(query))
	foundUntriaged := a.getUntriagedTestDetails(query, effectiveQuery, true)
	tests := make([]*GUITestDetail, 0, len(foundUntriaged))

	for testName, untriaged := range foundUntriaged {
		testDetail := a.current.TestDetails.lookup(testName)
		tests = append(tests, &GUITestDetail{
			Name:      testName,
			Untriaged: untriaged,
			Positive:  testDetail.Positive,
			Negative:  testDetail.Negative,
		})
	}

	// Sort the test details.
	sort.Sort(GUITestDetailSortable(tests))

	return &GUITestDetails{
		Commits:   a.current.TestDetails.Commits,
		AllParams: a.current.Index.getAllParams(query),
		Query:     effectiveQuery,
		Tests:     tests,
	}, nil
}

// PolyGUISimple is basic info about a test. Returned from PolyListTestSimple.
type PolyGUISimple struct {
	Name      string `json:"name"`
	Diameter  int    `json:"diameter"`
	Pos       int    `json:"pos"`
	Neg       int    `json:"neg"`
	Untriaged int    `json:"untriaged"`
	Num       int    `json:"num"`
}

// PolyGUISimpleSlice is used to sort PolyGUISimples.
type PolyGUISimpleSlice []*PolyGUISimple

func (p PolyGUISimpleSlice) Len() int           { return len(p) }
func (p PolyGUISimpleSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
func (p PolyGUISimpleSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// PolyListSimple returns a highlevel list of information about each test.
func (a *Analyzer) PolyListTestSimple(query url.Values) ([]*PolyGUISimple, error) {
	a.mutex.RLock()
	defer a.mutex.RUnlock()

	ret := []*PolyGUISimple{}

	tile, err := a.tileStore.Get(0, -1)
	if err != nil {
		return nil, fmt.Errorf("Analyzer couldn't retrieve tile: %s", err)
	}

	hasQuery := len(query) > 0

	names := map[string]bool{}
	if hasQuery {
		for _, tr := range tile.Traces {
			if ptypes.Matches(tr, query) {
				if name, ok := tr.Params()[types.PRIMARY_KEY_FIELD]; ok {
					names[name] = true
				}
			}
		}
	}

	for _, t := range a.current.TestDetails.Tests {
		if hasQuery {
			if _, ok := names[t.Name]; !ok {
				continue
			}
		}

		ret = append(ret, &PolyGUISimple{
			Name:      t.Name,
			Diameter:  t.Diameter,
			Pos:       len(t.Positive),
			Neg:       len(t.Negative),
			Untriaged: len(t.Untriaged),
			Num:       len(t.Positive) + len(t.Untriaged),
		})
	}

	sort.Sort(PolyGUISimpleSlice(ret))
	return ret, nil
}<commit_msg>Remove analyzer.PolyListTestSimple, it's been replaced with summary.Summary.<commit_after>package analysis

import (
	"sort"
	"sync"
	"time"

	"github.com/rcrowley/go-metrics"
	"github.com/skia-dev/glog"

	"skia.googlesource.com/buildbot.git/go/timer"
	"skia.googlesource.com/buildbot.git/go/util"
	"skia.googlesource.com/buildbot.git/golden/go/diff"
	"skia.googlesource.com/buildbot.git/golden/go/expstorage"
	"skia.googlesource.com/buildbot.git/golden/go/types"
	"skia.googlesource.com/buildbot.git/perf/go/human"
	ptypes "skia.googlesource.com/buildbot.git/perf/go/types"
)

var (
	// The number of times we've successfully loaded and processed a tile.
	runsCounter metrics.Counter

	// The number of times an error has ocurred when trying to load a tile.
	errorTileLoadingCounter metrics.Counter
)

func init() {
	runsCounter = metrics.NewRegisteredCounter("analysis.runs", metrics.DefaultRegistry)
	errorTileLoadingCounter = metrics.NewRegisteredCounter("analysis.errors", metrics.DefaultRegistry)
}

type PathToURLConverter func(string) string

// LabeledTrace stores a Trace with labels and digests. CommitIds, Digests and
// Labels are of the same length, identical indices refer to the same digest.
type LabeledTrace struct {
	Params      map[string]string
	CommitIds   []int
	Digests     []string
	Labels      []types.Label
	Id          int
	IgnoreRules []*types.IgnoreRule
}

func NewLabeledTrace(params map[string]string, capacity int, traceId int) *LabeledTrace {
	return &LabeledTrace{
		Params:      params,
		CommitIds:   make([]int, 0, capacity),
		Digests:     make([]string, 0, capacity),
		Labels:      make([]types.Label, 0, capacity),
		Id:          traceId,
		IgnoreRules: []*types.IgnoreRule{},
	}
}

// addLabledDigests adds the given tripples of commitIds, digests and labels to this LabeledTrace.
func (lt *LabeledTrace) addLabeledDigests(commitIds []int, digests []string, labels []types.Label) {
	lt.CommitIds = append(lt.CommitIds, commitIds...)
	lt.Digests = append(lt.Digests, digests...)
	lt.Labels = append(lt.Labels, labels...)
}

// addIgnoreRules attaches the ignore rules that match his trace.
func (lt *LabeledTrace) addIgnoreRules(newRules []*types.IgnoreRule) {
	lt.IgnoreRules = append(lt.IgnoreRules, newRules...)
}

// LabeledTile aggregates the traces of a tile and provides a slice of commits
// that the commitIds in LabeledTrace refer to.
// LabeledTile and LabeledTrace store the cannonical information
// extracted from the unterlying tile store. The (redundant) output data is
// derived from these.
type LabeledTile struct {
	Commits []*ptypes.Commit

	// Traces are indexed by the primary key (test name). This is somewhat
	// redundant, but this also output format.
	Traces map[string][]*LabeledTrace

	// CommitsByDigest maps a corpus and a digest to a list of commit ids.
	// i.e. CommitsByDigest[corpus][digest] -> slice with indices of Commits.
	CommitsByDigest map[string]map[string][]int

	// Keeps track of unique ids for traces within this tile.
	traceIdCounter int
}

func NewLabeledTile() *LabeledTile {
	return &LabeledTile{
		Commits:         []*ptypes.Commit{},
		CommitsByDigest: map[string]map[string][]int{},
		Traces:          map[string][]*LabeledTrace{},
		traceIdCounter:  0,
	}
}

// getLabeledTrace is a utility function that returns the testName and a labeled
// trace for the given trace (read from a TileStore). If the LabeledTrace does
// not exist it will be added.
func (t *LabeledTile) getLabeledTrace(trace ptypes.Trace) (string, *LabeledTrace) {
	params := trace.Params()
	pKey := params[types.PRIMARY_KEY_FIELD]
	if _, ok := t.Traces[pKey]; !ok {
		// Add the primary key with a single labled trace.
		t.Traces[pKey] = []*LabeledTrace{}
	}

	// Search through the traces associated witht this test.
	for _, v := range t.Traces[pKey] {
		if util.MapsEqual(v.Params, params) {
			return pKey, v
		}
	}

	// If we cannot find the trace in our set of tests we are adding a new
	// labeled trace.
	newLT := NewLabeledTrace(params, trace.Len(), t.traceIdCounter)
	t.traceIdCounter++
	t.Traces[pKey] = append(t.Traces[pKey], newLT)
	return pKey, newLT
}

// LabelCounts is an output type to hold counts for classification labels.
type LabelCounts struct {
	Unt []int `json:"unt"` // Untriaged
	Pos []int `json:"pos"` // Positive
	Neg []int `json:"neg"` // Negative
}

func newLabelCounts(length int) *LabelCounts {
	return &LabelCounts{
		Unt: make([]int, length),
		Pos: make([]int, length),
		Neg: make([]int, length),
	}
}

// GUITileCounts is an output type for the aggregated label counts.
type GUITileCounts struct {
	Commits    []*ptypes.Commit        `json:"commits"`
	Ticks      []interface{}           `json:"ticks"`
	Aggregated *LabelCounts            `json:"aggregated"`
	Counts     map[string]*LabelCounts `json:"counts"`
	AllParams  map[string][]string     `json:"allParams"`
	Query      map[string][]string     `json:"query"`
}

// AnalyzeState captures the state of a partition of the incoming data.
// When a tile is read from disk it is partitioned into two tiles: current
// and ignored. current contains everything we want to be able to review
// continuously and ignored contains all ignored traces.
// This struct is the container for one of these partitions and the derived
// information.
type AnalyzeState struct {
	// Canonical data structure to hold our information about commits, digests
	// and labels.
	Tile *LabeledTile

	// Index to query the Tile.
	Index *LabeledTileIndex

	// Output data structures that are derived from Tile.
	TileCounts  *GUITileCounts
	TestDetails *GUITestDetails
	Status      *GUIStatus
}

// Analyzer continuously manages tasks like polling for new traces
// on disk and generating diffs between images. It is the primary interface
// to be called by the HTTP frontend.
type Analyzer struct {
	expStore    expstorage.ExpectationsStore
	diffStore   diff.DiffStore
	tileStore   ptypes.TileStore
	ignoreStore types.IgnoreStore

	current *AnalyzeState
	ignored *AnalyzeState

	// converter supplied by the client of the type to convert a path to a URL
	pathToURLConverter PathToURLConverter

	// Lock to protect the expectations and current* variables.
	mutex sync.RWMutex

	// Counts the number of times the main event loop has executed.
	// This is for testing only.
	loopCounter int
}

func NewAnalyzer(expStore expstorage.ExpectationsStore, tileStore ptypes.TileStore, diffStore diff.DiffStore, ignoreStore types.IgnoreStore, puConverter PathToURLConverter, timeBetweenPolls time.Duration) *Analyzer {
	result := &Analyzer{
		expStore:           expStore,
		diffStore:          diffStore,
		tileStore:          tileStore,
		ignoreStore:        ignoreStore,
		pathToURLConverter: puConverter,

		current: &AnalyzeState{},
		ignored: &AnalyzeState{},
	}

	go result.loop(timeBetweenPolls)
	return result
}

// GetTileCounts returns an entire Tile which is a collection of 'traces' over
// a series of commits. Each trace contains the digests and their labels
// based on our knowledge about digests (expectations).
func (a *Analyzer) GetTileCounts(query map[string][]string) (*GUITileCounts, error) {
	a.mutex.RLock()
	defer a.mutex.RUnlock()

	if len(query) > 0 {
		tile, effectiveQuery := a.getSubTile(query)
		if len(effectiveQuery) > 0 {
			ret := a.getOutputCounts(tile, a.current.Index)
			ret.Query = effectiveQuery
			return ret, nil
		}
	}

	return a.current.TileCounts, nil
}

// ListTestDetails returns a list of triage details based on the supplied
// query. It's complementary to GetTestDetails which returns a single test
// detail.
// TODO(stephana): This should provide pagination since the list is potentially
// very long. If we don't add pagination, this should be merged with
// GetTestDetail.
func (a *Analyzer) ListTestDetails(query map[string][]string) (*GUITestDetails, error) {
	a.mutex.RLock()
	defer a.mutex.RUnlock()

	if len(query) == 0 {
		return a.current.TestDetails, nil
	}

	effectiveQuery := make(map[string][]string, len(query))
	foundUntriaged := a.getUntriagedTestDetails(query, effectiveQuery, true)
	tests := make([]*GUITestDetail, 0, len(foundUntriaged))

	for testName, untriaged := range foundUntriaged {
		testDetail := a.current.TestDetails.lookup(testName)
		tests = append(tests, &GUITestDetail{
			Name:      testName,
			Untriaged: untriaged,
			Positive:  testDetail.Positive,
			Negative:  testDetail.Negative,
		})
	}

	// Sort the test details.
	sort.Sort(GUITestDetailSortable(tests))

	return &GUITestDetails{
		Commits:   a.current.TestDetails.Commits,
		AllParams: a.current.Index.getAllParams(query),
		Query:     effectiveQuery,
		Tests:     tests,
	}, nil
}
{'fraction_non_alphanumeric': 0.10706162390901877, 'fraction_numerical': 0.0010050251256281408, 'mean_word_length': 3.897927461139896, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2950564', 'n_tokens_mistral': 6294, 'n_tokens_neox': 5986, 'n_words': 1860}