<commit_before>// warmer makes sure we've pre-warmed the cache for normal queries.
//
// This is a quick fix until filediffstore does full NxM diffs for every
// untriaged digest by default.
package warmer

import (
	"sync"
	"time"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/timer"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/golden/go/digesttools"
	"go.skia.org/infra/golden/go/storage"
	"go.skia.org/infra/golden/go/summary"
	"go.skia.org/infra/golden/go/tally"
	"go.skia.org/infra/golden/go/types"
)

func Init(storages *storage.Storage, summaries *summary.Summaries, tallies *tally.Tallies) error {
	exp, err := storages.ExpectationsStore.Get()
	if err != nil {
		return err
	}
	go func() {
		oneRun := func() {
			t := timer.New("warmer one loop")
			for test, sum := range summaries.Get() {
				for _, digest := range sum.UntHashes {
					t := tallies.ByTest()[test]
					if t != nil {
						// Calculate the closest digest for the side effect of filling in the filediffstore cache.
						digesttools.ClosestDigest(test, digest, exp, t, storages.DiffStore, types.POSITIVE)
						digesttools.ClosestDigest(test, digest, exp, t, storages.DiffStore, types.NEGATIVE)
					}
				}
			}
			t.Stop()
			if newExp, err := storages.ExpectationsStore.Get(); err != nil {
				glog.Errorf("warmer: Failed to get expectations: %s", err)
			} else {
				exp = newExp
			}

			// Make sure all images are downloaded. This is necessary, because
			// the front-end doesn't get URLs (generated via DiffStore.AbsPath)
			// which ensures that the image has been downloaded.
			// TODO(stephana): Remove this once the new diffstore is in place.
			tile, err := storages.GetLastTileTrimmed(true)
			if err != nil {
				glog.Errorf("Error retrieving tile: %s", err)
			}
			tileLen := tile.LastCommitIndex() + 1
			traceDigests := make(util.StringSet, tileLen)
			for _, trace := range tile.Traces {
				gTrace := trace.(*types.GoldenTrace)
				for _, digest := range gTrace.Values {
					if digest != types.MISSING_DIGEST {
						traceDigests[digest] = true
					}
				}
			}

			digests := traceDigests.Keys()
			glog.Infof("FOUND %d digests to fetch.", len(digests))
			storages.DiffStore.AbsPath(digests)

			if err := warmTrybotDigests(storages, traceDigests); err != nil {
				glog.Errorf("Error retrieving trybot digests: %s", err)
				return
			}
		}

		oneRun()
		for _ = range time.Tick(time.Minute) {
			oneRun()
		}
	}()
	return nil
}

func warmTrybotDigests(storages *storage.Storage, traceDigests map[string]bool) error {
	issues, _, err := storages.TrybotResults.ListTrybotIssues(0, 100)
	if err != nil {
		return err
	}

	trybotDigests := util.NewStringSet()
	var wg sync.WaitGroup
	var mutex sync.Mutex
	for _, oneIssue := range issues {
		wg.Add(1)
		go func(issueID string) {
			_, tile, err := storages.TrybotResults.GetIssue(issueID, nil)
			if err != nil {
				glog.Errorf("Unable to retrieve issue %s. Got error: %s", issueID, err)
				return
			}

			for _, trace := range tile.Traces {
				gTrace := trace.(*types.GoldenTrace)
				for _, digest := range gTrace.Values {
					if !traceDigests[digest] {
						mutex.Lock()
						trybotDigests[digest] = true
						mutex.Unlock()
					}
				}
			}
			wg.Done()
		}(oneIssue.ID)
	}

	wg.Wait()
	digests := trybotDigests.Keys()
	glog.Infof("FOUND %d trybot digests to fetch.", len(digests))
	storages.DiffStore.AbsPath(digests)
	return nil
}
<commit_msg>Disable warmer which seems to crash Gold<commit_after>// warmer makes sure we've pre-warmed the cache for normal queries.
//
// This is a quick fix until filediffstore does full NxM diffs for every
// untriaged digest by default.
package warmer

import (
	"sync"
	"time"

	"github.com/skia-dev/glog"
	"go.skia.org/infra/go/timer"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/golden/go/digesttools"
	"go.skia.org/infra/golden/go/storage"
	"go.skia.org/infra/golden/go/summary"
	"go.skia.org/infra/golden/go/tally"
	"go.skia.org/infra/golden/go/types"
)

func Init(storages *storage.Storage, summaries *summary.Summaries, tallies *tally.Tallies) error {
	exp, err := storages.ExpectationsStore.Get()
	if err != nil {
		return err
	}
	go func() {
		oneRun := func() {
			t := timer.New("warmer one loop")
			for test, sum := range summaries.Get() {
				for _, digest := range sum.UntHashes {
					t := tallies.ByTest()[test]
					if t != nil {
						// Calculate the closest digest for the side effect of filling in the filediffstore cache.
						digesttools.ClosestDigest(test, digest, exp, t, storages.DiffStore, types.POSITIVE)
						digesttools.ClosestDigest(test, digest, exp, t, storages.DiffStore, types.NEGATIVE)
					}
				}
			}
			t.Stop()
			if newExp, err := storages.ExpectationsStore.Get(); err != nil {
				glog.Errorf("warmer: Failed to get expectations: %s", err)
			} else {
				exp = newExp
			}

			// Make sure all images are downloaded. This is necessary, because
			// the front-end doesn't get URLs (generated via DiffStore.AbsPath)
			// which ensures that the image has been downloaded.
			// TODO(stephana): Remove this once the new diffstore is in place.
			tile, err := storages.GetLastTileTrimmed(true)
			if err != nil {
				glog.Errorf("Error retrieving tile: %s", err)
			}
			tileLen := tile.LastCommitIndex() + 1
			traceDigests := make(util.StringSet, tileLen)
			for _, trace := range tile.Traces {
				gTrace := trace.(*types.GoldenTrace)
				for _, digest := range gTrace.Values {
					if digest != types.MISSING_DIGEST {
						traceDigests[digest] = true
					}
				}
			}

			digests := traceDigests.Keys()
			glog.Infof("FOUND %d digests to fetch.", len(digests))
			storages.DiffStore.AbsPath(digests)

			// TODO(stephana): Re-enable this once we have figured out crashes.

			// if err := warmTrybotDigests(storages, traceDigests); err != nil {
			// 	glog.Errorf("Error retrieving trybot digests: %s", err)
			// 	return
			// }
		}

		oneRun()
		for _ = range time.Tick(time.Minute) {
			oneRun()
		}
	}()
	return nil
}

func warmTrybotDigests(storages *storage.Storage, traceDigests map[string]bool) error {
	issues, _, err := storages.TrybotResults.ListTrybotIssues(0, 100)
	if err != nil {
		return err
	}

	trybotDigests := util.NewStringSet()
	var wg sync.WaitGroup
	var mutex sync.Mutex
	for _, oneIssue := range issues {
		wg.Add(1)
		go func(issueID string) {
			_, tile, err := storages.TrybotResults.GetIssue(issueID, nil)
			if err != nil {
				glog.Errorf("Unable to retrieve issue %s. Got error: %s", issueID, err)
				return
			}

			for _, trace := range tile.Traces {
				gTrace := trace.(*types.GoldenTrace)
				for _, digest := range gTrace.Values {
					if !traceDigests[digest] {
						mutex.Lock()
						trybotDigests[digest] = true
						mutex.Unlock()
					}
				}
			}
			wg.Done()
		}(oneIssue.ID)
	}

	wg.Wait()
	digests := trybotDigests.Keys()
	glog.Infof("FOUND %d trybot digests to fetch.", len(digests))
	storages.DiffStore.AbsPath(digests)
	return nil
}

{'fraction_non_alphanumeric': 0.13123472761247665, 'fraction_numerical': 0.0017248814144027599, 'mean_word_length': 3.1589958158995817, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6243634', 'n_tokens_mistral': 2886, 'n_tokens_neox': 2438, 'n_words': 613}