<commit_before>// ds is a package for using Google Cloud Datastore.
package ds

import (
	"context"
	"fmt"
	"time"

	"cloud.google.com/go/datastore"
	"golang.org/x/sync/errgroup"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
)

// Global constants.
const (
	// Maximum number of entities which may be inserted or deleted at once.
	MAX_MODIFICATIONS = 500
)

type Kind string

// Below are all the Kinds used in all applications. New Kinds should be listed
// here, and they should all have unique values, because when defining indexes
// for Cloud Datastore the index config is per project, not pre-namespace.
// Remember to add them to KindsToBackup if they are to be backed up, and push
// a new version of /ds/go/datastore_backup.
const (
	// Predict
	FAILURES     Kind = "Failures"
	FLAKY_RANGES Kind = "FlakyRanges"

	// Perf
	SHORTCUT   Kind = "Shortcut"
	ACTIVITY   Kind = "Activity"
	REGRESSION Kind = "Regression"
	ALERT      Kind = "Alert"

	// Gold
	ISSUE                  Kind = "Issue"
	TRYJOB                 Kind = "Tryjob"
	TRYJOB_RESULT          Kind = "TryjobResult"
	TRYJOB_EXP_CHANGE      Kind = "TryjobExpChange"
	TEST_DIGEST_EXP        Kind = "TryjobTestDigestExp" // TODO(stephana): Remove after migration to consolidated expectations store
	TRYJOB_TEST_DIGEST_EXP Kind = "TryjobTestDigestExp"
	MASTER_EXP_CHANGE      Kind = "MasterExpChange"
	IGNORE_RULE            Kind = "IgnoreRule"
	HELPER_RECENT_KEYS     Kind = "HelperRecentKeys"
	EXPECTATIONS_BLOB      Kind = "ExpectationsBlob"
	EXPECTATIONS_BLOB_ROOT Kind = "ExpectationsBlobRoot"

	// Android Compile
	COMPILE_TASK Kind = "CompileTask"

	// Leasing
	TASK Kind = "Task"

	// CT
	CAPTURE_SKPS_TASKS              Kind = "CaptureSkpsTasks"
	CHROMIUM_ANALYSIS_TASKS         Kind = "ChromiumAnalysisTasks"
	CHROMIUM_BUILD_TASKS            Kind = "ChromiumBuildTasks"
	CHROMIUM_PERF_TASKS             Kind = "ChromiumPerfTasks"
	LUA_SCRIPT_TASKS                Kind = "LuaScriptTasks"
	METRICS_ANALYSIS_TASKS          Kind = "MetricsAnalysisTasks"
	PIXEL_DIFF_TASKS                Kind = "PixelDiffTasks"
	RECREATE_PAGESETS_TASKS         Kind = "RecreatePageSetsTasks"
	RECREATE_WEBPAGE_ARCHIVES_TASKS Kind = "RecreateWebpageArchivesTasks"
	CLUSTER_TELEMETRY_IDS           Kind = "ClusterTelemetryIDs"

	// Autoroll
	KIND_AUTOROLL_MODE                Kind = "AutorollMode"
	KIND_AUTOROLL_MODE_ANCESTOR       Kind = "AutorollModeAncestor" // Fake; used to force strong consistency for testing's sake.
	KIND_AUTOROLL_ROLL                Kind = "AutorollRoll"
	KIND_AUTOROLL_ROLL_ANCESTOR       Kind = "AutorollRollAncestor" // Fake; used to force strong consistency for testing's sake.
	KIND_AUTOROLL_STATUS              Kind = "AutorollStatus"
	KIND_AUTOROLL_STATUS_ANCESTOR     Kind = "AutorollStatusAncestor" // Fake; used to force strong consistency for testing's sake.
	KIND_AUTOROLL_STRATEGY            Kind = "AutorollStrategy"
	KIND_AUTOROLL_STRATEGY_ANCESTOR   Kind = "AutorollStrategyAncestor" // Fake; used to force strong consistency for testing's sake.
	KIND_AUTOROLL_UNTHROTTLE          Kind = "AutorollUnthrottle"
	KIND_AUTOROLL_UNTHROTTLE_ANCESTOR Kind = "AutorollUnthrottleAncestor" // Fake; used to force strong consistency for testing's sake.

	// AlertManager
	INCIDENT_AM               Kind = "IncidentAm"
	INCIDENT_ACTIVE_PARENT_AM Kind = "IncidentActiveParentAm"
	SILENCE_ACTIVE_PARENT_AM  Kind = "SilenceActiveParentAm"
	SILENCE_AM                Kind = "SilenceAm"
)

// Namespaces that are used in production, and thus might be backed up.
const (
	// Perf
	PERF_NS                = "perf"
	PERF_ANDROID_NS        = "perf-android"
	PERF_ANDROID_MASTER_NS = "perf-androidmaster"

	// Gold
	GOLD_SKIA_PROD_NS = "gold-skia-prod"

	// Android Compile
	ANDROID_COMPILE_NS = "android-compile"

	// Leasing
	LEASING_SERVER_NS = "leasing-server"

	// CT
	CT_NS = "cluster-telemetry"

	// Autoroll
	AUTOROLL_NS          = "autoroll"
	AUTOROLL_INTERNAL_NS = "autoroll-internal"

	// AlertManager
	ALERT_MANAGER_NS = "alert-manager"
)

var (
	// KindsToBackup is a map from namespace to the list of Kinds to backup.
	// If this value is changed then remember to push a new version of /ds/go/datastore_backup.
	//
	// Note that we try to backup all kinds and all namespaces for every project
	// even if that app isn't running there, which has a better failure mode of
	// possibly backing up too much data rather than too little.
	KindsToBackup = map[string][]Kind{
		AUTOROLL_NS:            []Kind{KIND_AUTOROLL_MODE, KIND_AUTOROLL_MODE_ANCESTOR, KIND_AUTOROLL_ROLL, KIND_AUTOROLL_ROLL_ANCESTOR, KIND_AUTOROLL_STATUS, KIND_AUTOROLL_STATUS_ANCESTOR, KIND_AUTOROLL_STRATEGY, KIND_AUTOROLL_STRATEGY_ANCESTOR, KIND_AUTOROLL_UNTHROTTLE, KIND_AUTOROLL_UNTHROTTLE_ANCESTOR},
		AUTOROLL_INTERNAL_NS:   []Kind{KIND_AUTOROLL_MODE, KIND_AUTOROLL_MODE_ANCESTOR, KIND_AUTOROLL_ROLL, KIND_AUTOROLL_ROLL_ANCESTOR, KIND_AUTOROLL_STATUS, KIND_AUTOROLL_STATUS_ANCESTOR, KIND_AUTOROLL_STRATEGY, KIND_AUTOROLL_STRATEGY_ANCESTOR, KIND_AUTOROLL_UNTHROTTLE, KIND_AUTOROLL_UNTHROTTLE_ANCESTOR},
		PERF_NS:                []Kind{ACTIVITY, ALERT, REGRESSION, SHORTCUT},
		PERF_ANDROID_NS:        []Kind{ACTIVITY, ALERT, REGRESSION, SHORTCUT},
		PERF_ANDROID_MASTER_NS: []Kind{ACTIVITY, ALERT, REGRESSION, SHORTCUT},
		GOLD_SKIA_PROD_NS:      []Kind{ISSUE, TRYJOB, TRYJOB_RESULT, TRYJOB_EXP_CHANGE, TRYJOB_TEST_DIGEST_EXP, MASTER_EXP_CHANGE, IGNORE_RULE, HELPER_RECENT_KEYS, EXPECTATIONS_BLOB, EXPECTATIONS_BLOB_ROOT},
		ANDROID_COMPILE_NS:     []Kind{COMPILE_TASK},
		LEASING_SERVER_NS:      []Kind{TASK},
		CT_NS:                  []Kind{CAPTURE_SKPS_TASKS, CHROMIUM_ANALYSIS_TASKS, CHROMIUM_BUILD_TASKS, CHROMIUM_PERF_TASKS, LUA_SCRIPT_TASKS, METRICS_ANALYSIS_TASKS, PIXEL_DIFF_TASKS, RECREATE_PAGESETS_TASKS, RECREATE_WEBPAGE_ARCHIVES_TASKS, CLUSTER_TELEMETRY_IDS},
		ALERT_MANAGER_NS:       []Kind{INCIDENT_AM, INCIDENT_ACTIVE_PARENT_AM, SILENCE_AM, SILENCE_ACTIVE_PARENT_AM},
	}
)

var (
	// DS is the Cloud Datastore client. Valid after Init() has been called.<commit_msg>[infra] Add Gold namespaces to ds package<commit_after>// ds is a package for using Google Cloud Datastore.
package ds

import (
	"context"
	"fmt"
	"time"

	"cloud.google.com/go/datastore"
	"golang.org/x/sync/errgroup"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
)

// Global constants.
const (
	// Maximum number of entities which may be inserted or deleted at once.
	MAX_MODIFICATIONS = 500
)

type Kind string

// Below are all the Kinds used in all applications. New Kinds should be listed
// here, and they should all have unique values, because when defining indexes
// for Cloud Datastore the index config is per project, not pre-namespace.
// Remember to add them to KindsToBackup if they are to be backed up, and push
// a new version of /ds/go/datastore_backup.
const (
	// Predict
	FAILURES     Kind = "Failures"
	FLAKY_RANGES Kind = "FlakyRanges"

	// Perf
	SHORTCUT   Kind = "Shortcut"
	ACTIVITY   Kind = "Activity"
	REGRESSION Kind = "Regression"
	ALERT      Kind = "Alert"

	// Gold
	ISSUE                  Kind = "Issue"
	TRYJOB                 Kind = "Tryjob"
	TRYJOB_RESULT          Kind = "TryjobResult"
	TRYJOB_EXP_CHANGE      Kind = "TryjobExpChange"
	TEST_DIGEST_EXP        Kind = "TryjobTestDigestExp" // TODO(stephana): Remove after migration to consolidated expectations store
	TRYJOB_TEST_DIGEST_EXP Kind = "TryjobTestDigestExp"
	MASTER_EXP_CHANGE      Kind = "MasterExpChange"
	IGNORE_RULE            Kind = "IgnoreRule"
	HELPER_RECENT_KEYS     Kind = "HelperRecentKeys"
	EXPECTATIONS_BLOB      Kind = "ExpectationsBlob"
	EXPECTATIONS_BLOB_ROOT Kind = "ExpectationsBlobRoot"

	// Android Compile
	COMPILE_TASK Kind = "CompileTask"

	// Leasing
	TASK Kind = "Task"

	// CT
	CAPTURE_SKPS_TASKS              Kind = "CaptureSkpsTasks"
	CHROMIUM_ANALYSIS_TASKS         Kind = "ChromiumAnalysisTasks"
	CHROMIUM_BUILD_TASKS            Kind = "ChromiumBuildTasks"
	CHROMIUM_PERF_TASKS             Kind = "ChromiumPerfTasks"
	LUA_SCRIPT_TASKS                Kind = "LuaScriptTasks"
	METRICS_ANALYSIS_TASKS          Kind = "MetricsAnalysisTasks"
	PIXEL_DIFF_TASKS                Kind = "PixelDiffTasks"
	RECREATE_PAGESETS_TASKS         Kind = "RecreatePageSetsTasks"
	RECREATE_WEBPAGE_ARCHIVES_TASKS Kind = "RecreateWebpageArchivesTasks"
	CLUSTER_TELEMETRY_IDS           Kind = "ClusterTelemetryIDs"

	// Autoroll
	KIND_AUTOROLL_MODE                Kind = "AutorollMode"
	KIND_AUTOROLL_MODE_ANCESTOR       Kind = "AutorollModeAncestor" // Fake; used to force strong consistency for testing's sake.
	KIND_AUTOROLL_ROLL                Kind = "AutorollRoll"
	KIND_AUTOROLL_ROLL_ANCESTOR       Kind = "AutorollRollAncestor" // Fake; used to force strong consistency for testing's sake.
	KIND_AUTOROLL_STATUS              Kind = "AutorollStatus"
	KIND_AUTOROLL_STATUS_ANCESTOR     Kind = "AutorollStatusAncestor" // Fake; used to force strong consistency for testing's sake.
	KIND_AUTOROLL_STRATEGY            Kind = "AutorollStrategy"
	KIND_AUTOROLL_STRATEGY_ANCESTOR   Kind = "AutorollStrategyAncestor" // Fake; used to force strong consistency for testing's sake.
	KIND_AUTOROLL_UNTHROTTLE          Kind = "AutorollUnthrottle"
	KIND_AUTOROLL_UNTHROTTLE_ANCESTOR Kind = "AutorollUnthrottleAncestor" // Fake; used to force strong consistency for testing's sake.

	// AlertManager
	INCIDENT_AM               Kind = "IncidentAm"
	INCIDENT_ACTIVE_PARENT_AM Kind = "IncidentActiveParentAm"
	SILENCE_ACTIVE_PARENT_AM  Kind = "SilenceActiveParentAm"
	SILENCE_AM                Kind = "SilenceAm"
)

// Namespaces that are used in production, and thus might be backed up.
const (
	// Perf
	PERF_NS                = "perf"
	PERF_ANDROID_NS        = "perf-android"
	PERF_ANDROID_MASTER_NS = "perf-androidmaster"

	// Gold
	GOLD_CHROMEVR_NS  = "gold-chromevr"
	GOLD_LOTTIE_NS    = "gold-lottie"
	GOLD_PDFIUM_NS    = "gold-pdfium"
	GOLD_SKIA_PROD_NS = "gold-skia-prod"

	// Android Compile
	ANDROID_COMPILE_NS = "android-compile"

	// Leasing
	LEASING_SERVER_NS = "leasing-server"

	// CT
	CT_NS = "cluster-telemetry"

	// Autoroll
	AUTOROLL_NS          = "autoroll"
	AUTOROLL_INTERNAL_NS = "autoroll-internal"

	// AlertManager
	ALERT_MANAGER_NS = "alert-manager"
)

var (
	// goldKinds are the DS kinds used by Gold.
	goldKinds = []Kind{ISSUE, TRYJOB, TRYJOB_RESULT, TRYJOB_EXP_CHANGE, TRYJOB_TEST_DIGEST_EXP, MASTER_EXP_CHANGE, IGNORE_RULE, HELPER_RECENT_KEYS, EXPECTATIONS_BLOB, EXPECTATIONS_BLOB_ROOT}

	// KindsToBackup is a map from namespace to the list of Kinds to backup.
	// If this value is changed then remember to push a new version of /ds/go/datastore_backup.
	//
	// Note that we try to backup all kinds and all namespaces for every project
	// even if that app isn't running there, which has a better failure mode of
	// possibly backing up too much data rather than too little.
	KindsToBackup = map[string][]Kind{
		AUTOROLL_NS:            []Kind{KIND_AUTOROLL_MODE, KIND_AUTOROLL_MODE_ANCESTOR, KIND_AUTOROLL_ROLL, KIND_AUTOROLL_ROLL_ANCESTOR, KIND_AUTOROLL_STATUS, KIND_AUTOROLL_STATUS_ANCESTOR, KIND_AUTOROLL_STRATEGY, KIND_AUTOROLL_STRATEGY_ANCESTOR, KIND_AUTOROLL_UNTHROTTLE, KIND_AUTOROLL_UNTHROTTLE_ANCESTOR},
		AUTOROLL_INTERNAL_NS:   []Kind{KIND_AUTOROLL_MODE, KIND_AUTOROLL_MODE_ANCESTOR, KIND_AUTOROLL_ROLL, KIND_AUTOROLL_ROLL_ANCESTOR, KIND_AUTOROLL_STATUS, KIND_AUTOROLL_STATUS_ANCESTOR, KIND_AUTOROLL_STRATEGY, KIND_AUTOROLL_STRATEGY_ANCESTOR, KIND_AUTOROLL_UNTHROTTLE, KIND_AUTOROLL_UNTHROTTLE_ANCESTOR},
		PERF_NS:                []Kind{ACTIVITY, ALERT, REGRESSION, SHORTCUT},
		PERF_ANDROID_NS:        []Kind{ACTIVITY, ALERT, REGRESSION, SHORTCUT},
		PERF_ANDROID_MASTER_NS: []Kind{ACTIVITY, ALERT, REGRESSION, SHORTCUT},
		GOLD_CHROMEVR_NS:       goldKinds,
		GOLD_LOTTIE_NS:         goldKinds,
		GOLD_PDFIUM_NS:         goldKinds,
		GOLD_SKIA_PROD_NS:      goldKinds,
		ANDROID_COMPILE_NS:     []Kind{COMPILE_TASK},
		LEASING_SERVER_NS:      []Kind{TASK},
		CT_NS:                  []Kind{CAPTURE_SKPS_TASKS, CHROMIUM_ANALYSIS_TASKS, CHROMIUM_BUILD_TASKS, CHROMIUM_PERF_TASKS, LUA_SCRIPT_TASKS, METRICS_ANALYSIS_TASKS, PIXEL_DIFF_TASKS, RECREATE_PAGESETS_TASKS, RECREATE_WEBPAGE_ARCHIVES_TASKS, CLUSTER_TELEMETRY_IDS},
		ALERT_MANAGER_NS:       []Kind{INCIDENT_AM, INCIDENT_ACTIVE_PARENT_AM, SILENCE_AM, SILENCE_ACTIVE_PARENT_AM},
	}
)

var (
	// DS is the Cloud Datastore client. Valid after Init() has been called.
{'fraction_non_alphanumeric': 0.07624633431085044, 'fraction_numerical': 0.00047554886264563684, 'mean_word_length': 3.85121107266436, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '166592', 'n_tokens_mistral': 4893, 'n_tokens_neox': 4731, 'n_words': 991}