<commit_before>package repo_manager

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	"go.skia.org/infra/autoroll/go/codereview"
	"go.skia.org/infra/autoroll/go/revision"
	"go.skia.org/infra/autoroll/go/strategy"
	"go.skia.org/infra/go/cipd"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/github"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
)

const (
	cipdPackageUrlTmpl = "%s/p/%s/+/%s"

	cipdGithubTitleTmpl = "Roll %s from %s to %s"
	cipdCommitMsgTmpl   = cipdGithubTitleTmpl + `

` + COMMIT_MSG_FOOTER_TMPL
)

var (
	// Use this function to instantiate a NewGithubCipdDEPSRepoManager. This is able to be
	// overridden for testing.
	NewGithubCipdDEPSRepoManager func(context.Context, *GithubCipdDEPSRepoManagerConfig, string, string, *github.GitHub, string, string, *http.Client, codereview.CodeReview, bool) (RepoManager, error) = newGithubCipdDEPSRepoManager
)

// GithubCipdDEPSRepoManagerConfig provides configuration for the Github RepoManager.
type GithubCipdDEPSRepoManagerConfig struct {
	GithubDEPSRepoManagerConfig
	CipdAssetName string `json:"cipdAssetName"`
	CipdAssetTag  string `json:"cipdAssetTag"`
}

// Validate the config.
func (c *GithubCipdDEPSRepoManagerConfig) Validate() error {
	if c.CipdAssetName == "" {
		return fmt.Errorf("CipdAssetName is required.")
	}
	if c.CipdAssetTag == "" {
		return fmt.Errorf("CipdAssetTag is required.")
	}
	return c.GithubDEPSRepoManagerConfig.Validate()
}

// githubCipdDEPSRepoManager is a struct used by the autoroller for managing checkouts.
type githubCipdDEPSRepoManager struct {
	*githubDEPSRepoManager
	cipdAssetName string
	cipdAssetTag  string
	CipdClient    cipd.CIPDClient
}

// newGithubCipdDEPSRepoManager returns a RepoManager instance which operates in the given
// working directory and updates at the given frequency.
func newGithubCipdDEPSRepoManager(ctx context.Context, c *GithubCipdDEPSRepoManagerConfig, workdir, rollerName string, githubClient *github.GitHub, recipeCfgFile, serverURL string, httpClient *http.Client, cr codereview.CodeReview, local bool) (RepoManager, error) {
	if err := c.Validate(); err != nil {
		return nil, err
	}
	wd := path.Join(workdir, strings.TrimSuffix(path.Base(c.DepotToolsRepoManagerConfig.ParentRepo), ".git"))
	drm, err := newDepotToolsRepoManager(ctx, c.DepotToolsRepoManagerConfig, wd, recipeCfgFile, serverURL, nil, httpClient, cr, local)
	if err != nil {
		return nil, err
	}
	dr := &depsRepoManager{
		depotToolsRepoManager: drm,
	}
	if c.GithubParentPath != "" {
		dr.parentDir = path.Join(wd, c.GithubParentPath)
	}
	gr := &githubDEPSRepoManager{
		depsRepoManager: dr,
		githubClient:    githubClient,
		rollBranchName:  rollerName,
	}
	sklog.Infof("Roller name is: %s\n", rollerName)
	cipdClient, err := cipd.NewClient(httpClient, path.Join(workdir, "cipd"))
	if err != nil {
		return nil, err
	}
	gcr := &githubCipdDEPSRepoManager{
		githubDEPSRepoManager: gr,
		cipdAssetName:         c.CipdAssetName,
		cipdAssetTag:          c.CipdAssetTag,
		CipdClient:            cipdClient,
	}

	return gcr, nil
}

// See documentation for RepoManager interface.
func (rm *githubCipdDEPSRepoManager) Update(ctx context.Context) error {
	// Sync the projects.
	rm.repoMtx.Lock()
	defer rm.repoMtx.Unlock()

	sklog.Info("Updating github repository")

	// If parentDir does not exist yet then create the directory structure and
	// populate it.
	if _, err := os.Stat(rm.parentDir); err != nil {
		if os.IsNotExist(err) {
			if err := rm.createAndSyncParentWithRemoteAndBranch(ctx, "origin", rm.rollBranchName, rm.rollBranchName); err != nil {
				return fmt.Errorf("Could not create and sync %s: %s", rm.parentDir, err)
			}
			// Run gclient hooks to bring in any required binaries.
			if _, err := exec.RunCommand(ctx, &exec.Command{
				Dir:  rm.parentDir,
				Env:  rm.depotToolsEnv,
				Name: rm.gclient,
				Args: []string{"runhooks"},
			}); err != nil {
				return fmt.Errorf("Error when running gclient runhooks on %s: %s", rm.parentDir, err)
			}
		} else {
			return fmt.Errorf("Error when running os.Stat on %s: %s", rm.parentDir, err)
		}
	}

	// Check to see whether there is an upstream yet.
	remoteOutput, err := git.GitDir(rm.parentDir).Git(ctx, "remote", "show")
	if err != nil {
		return err
	}
	remoteFound := false
	remoteLines := strings.Split(remoteOutput, "\n")
	for _, remoteLine := range remoteLines {
		if remoteLine == GITHUB_UPSTREAM_REMOTE_NAME {
			remoteFound = true
			break
		}
	}
	if !remoteFound {
		if _, err := git.GitDir(rm.parentDir).Git(ctx, "remote", "add", GITHUB_UPSTREAM_REMOTE_NAME, rm.parentRepo); err != nil {
			return err
		}
	}
	// Fetch upstream.
	if _, err := git.GitDir(rm.parentDir).Git(ctx, "fetch", GITHUB_UPSTREAM_REMOTE_NAME, rm.parentBranch); err != nil {
		return err
	}
	// gclient sync to get latest version of child repo to find the next roll
	// rev from.
	if err := rm.createAndSyncParentWithRemoteAndBranch(ctx, GITHUB_UPSTREAM_REMOTE_NAME, rm.rollBranchName, rm.parentBranch); err != nil {
		return fmt.Errorf("Could not create and sync parent repo: %s", err)
	}

	// Get the last roll revision.
	lastRollRev, err := rm.getLastRollRev(ctx)
	if err != nil {
		return err
	}

	// Find the not-rolled child repo commits.
	notRolledRevs, err := getNotRolledRevs(ctx, rm.CipdClient, rm.lastRollRev, rm.cipdAssetName, rm.cipdAssetTag)
	if err != nil {
		return err
	}

	// Get the next roll revision.
	nextRollRev, err := rm.getNextRollRev(ctx, notRolledRevs, lastRollRev)
	if err != nil {
		return err
	}

	rm.infoMtx.Lock()
	defer rm.infoMtx.Unlock()
	if rm.childRepoUrl == "" {
		childRepo, err := exec.RunCwd(ctx, rm.parentDir, "git", "remote", "get-url", "origin")
		if err != nil {
			return err
		}
		rm.childRepoUrl = childRepo
	}

	rm.lastRollRev = lastRollRev
	rm.nextRollRev = nextRollRev
	rm.notRolledRevs = notRolledRevs

	sklog.Infof("lastRollRev is: %s", rm.lastRollRev)
	sklog.Infof("nextRollRev is: %s", nextRollRev)
	sklog.Infof("len(notRolledRevs): %v", len(rm.notRolledRevs))
	return nil
}

// getNotRolledRevs is a utility function that uses CIPD to find the not-yet-rolled versions of
// the specified package.
// Note: that this just finds all versions of the package between the last rolled version and the
// version currently pointed to by cipdAssetTag; we can't know whether the ref we're tracking was
// ever actually applied to any of the package instances in between.
func getNotRolledRevs(ctx context.Context, cipdClient cipd.CIPDClient, lastRollRev, cipdAssetName, cipdAssetTag string) ([]*revision.Revision, error) {
	head, err := cipdClient.ResolveVersion(ctx, cipdAssetName, cipdAssetTag)
	if err != nil {
		return nil, err
	}
	iter, err := cipdClient.ListInstances(ctx, cipdAssetName)
	if err != nil {
		return nil, err
	}
	notRolledRevs := []*revision.Revision{}
	foundHead := false
	for {
		instances, err := iter.Next(ctx, 100)
		if err != nil {
			return nil, err
		}
		if len(instances) == 0 {
			break
		}<commit_msg>[Autoroll] Truncate notRolledRevs in github_cipd_deps_repo_manager<commit_after>package repo_manager

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"path"
	"strings"
	"time"

	"go.skia.org/infra/autoroll/go/codereview"
	"go.skia.org/infra/autoroll/go/revision"
	"go.skia.org/infra/autoroll/go/strategy"
	"go.skia.org/infra/go/cipd"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/github"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
)

const (
	cipdPackageUrlTmpl = "%s/p/%s/+/%s"

	cipdGithubTitleTmpl = "Roll %s from %s to %s"
	cipdCommitMsgTmpl   = cipdGithubTitleTmpl + `

` + COMMIT_MSG_FOOTER_TMPL

	NOT_ROLLED_LIMIT = 200
)

var (
	// Use this function to instantiate a NewGithubCipdDEPSRepoManager. This is able to be
	// overridden for testing.
	NewGithubCipdDEPSRepoManager func(context.Context, *GithubCipdDEPSRepoManagerConfig, string, string, *github.GitHub, string, string, *http.Client, codereview.CodeReview, bool) (RepoManager, error) = newGithubCipdDEPSRepoManager
)

// GithubCipdDEPSRepoManagerConfig provides configuration for the Github RepoManager.
type GithubCipdDEPSRepoManagerConfig struct {
	GithubDEPSRepoManagerConfig
	CipdAssetName string `json:"cipdAssetName"`
	CipdAssetTag  string `json:"cipdAssetTag"`
}

// Validate the config.
func (c *GithubCipdDEPSRepoManagerConfig) Validate() error {
	if c.CipdAssetName == "" {
		return fmt.Errorf("CipdAssetName is required.")
	}
	if c.CipdAssetTag == "" {
		return fmt.Errorf("CipdAssetTag is required.")
	}
	return c.GithubDEPSRepoManagerConfig.Validate()
}

// githubCipdDEPSRepoManager is a struct used by the autoroller for managing checkouts.
type githubCipdDEPSRepoManager struct {
	*githubDEPSRepoManager
	cipdAssetName string
	cipdAssetTag  string
	CipdClient    cipd.CIPDClient
}

// newGithubCipdDEPSRepoManager returns a RepoManager instance which operates in the given
// working directory and updates at the given frequency.
func newGithubCipdDEPSRepoManager(ctx context.Context, c *GithubCipdDEPSRepoManagerConfig, workdir, rollerName string, githubClient *github.GitHub, recipeCfgFile, serverURL string, httpClient *http.Client, cr codereview.CodeReview, local bool) (RepoManager, error) {
	if err := c.Validate(); err != nil {
		return nil, err
	}
	wd := path.Join(workdir, strings.TrimSuffix(path.Base(c.DepotToolsRepoManagerConfig.ParentRepo), ".git"))
	drm, err := newDepotToolsRepoManager(ctx, c.DepotToolsRepoManagerConfig, wd, recipeCfgFile, serverURL, nil, httpClient, cr, local)
	if err != nil {
		return nil, err
	}
	dr := &depsRepoManager{
		depotToolsRepoManager: drm,
	}
	if c.GithubParentPath != "" {
		dr.parentDir = path.Join(wd, c.GithubParentPath)
	}
	gr := &githubDEPSRepoManager{
		depsRepoManager: dr,
		githubClient:    githubClient,
		rollBranchName:  rollerName,
	}
	sklog.Infof("Roller name is: %s\n", rollerName)
	cipdClient, err := cipd.NewClient(httpClient, path.Join(workdir, "cipd"))
	if err != nil {
		return nil, err
	}
	gcr := &githubCipdDEPSRepoManager{
		githubDEPSRepoManager: gr,
		cipdAssetName:         c.CipdAssetName,
		cipdAssetTag:          c.CipdAssetTag,
		CipdClient:            cipdClient,
	}

	return gcr, nil
}

// See documentation for RepoManager interface.
func (rm *githubCipdDEPSRepoManager) Update(ctx context.Context) error {
	// Sync the projects.
	rm.repoMtx.Lock()
	defer rm.repoMtx.Unlock()

	sklog.Info("Updating github repository")

	// If parentDir does not exist yet then create the directory structure and
	// populate it.
	if _, err := os.Stat(rm.parentDir); err != nil {
		if os.IsNotExist(err) {
			if err := rm.createAndSyncParentWithRemoteAndBranch(ctx, "origin", rm.rollBranchName, rm.rollBranchName); err != nil {
				return fmt.Errorf("Could not create and sync %s: %s", rm.parentDir, err)
			}
			// Run gclient hooks to bring in any required binaries.
			if _, err := exec.RunCommand(ctx, &exec.Command{
				Dir:  rm.parentDir,
				Env:  rm.depotToolsEnv,
				Name: rm.gclient,
				Args: []string{"runhooks"},
			}); err != nil {
				return fmt.Errorf("Error when running gclient runhooks on %s: %s", rm.parentDir, err)
			}
		} else {
			return fmt.Errorf("Error when running os.Stat on %s: %s", rm.parentDir, err)
		}
	}

	// Check to see whether there is an upstream yet.
	remoteOutput, err := git.GitDir(rm.parentDir).Git(ctx, "remote", "show")
	if err != nil {
		return err
	}
	remoteFound := false
	remoteLines := strings.Split(remoteOutput, "\n")
	for _, remoteLine := range remoteLines {
		if remoteLine == GITHUB_UPSTREAM_REMOTE_NAME {
			remoteFound = true
			break
		}
	}
	if !remoteFound {
		if _, err := git.GitDir(rm.parentDir).Git(ctx, "remote", "add", GITHUB_UPSTREAM_REMOTE_NAME, rm.parentRepo); err != nil {
			return err
		}
	}
	// Fetch upstream.
	if _, err := git.GitDir(rm.parentDir).Git(ctx, "fetch", GITHUB_UPSTREAM_REMOTE_NAME, rm.parentBranch); err != nil {
		return err
	}
	// gclient sync to get latest version of child repo to find the next roll
	// rev from.
	if err := rm.createAndSyncParentWithRemoteAndBranch(ctx, GITHUB_UPSTREAM_REMOTE_NAME, rm.rollBranchName, rm.parentBranch); err != nil {
		return fmt.Errorf("Could not create and sync parent repo: %s", err)
	}

	// Get the last roll revision.
	lastRollRev, err := rm.getLastRollRev(ctx)
	if err != nil {
		return err
	}

	// Find the not-rolled child repo commits.
	notRolledRevs, err := getNotRolledRevs(ctx, rm.CipdClient, rm.lastRollRev, rm.cipdAssetName, rm.cipdAssetTag)
	if err != nil {
		return err
	}

	// Get the next roll revision.
	nextRollRev, err := rm.getNextRollRev(ctx, notRolledRevs, lastRollRev)
	if err != nil {
		return err
	}

	rm.infoMtx.Lock()
	defer rm.infoMtx.Unlock()
	if rm.childRepoUrl == "" {
		childRepo, err := exec.RunCwd(ctx, rm.parentDir, "git", "remote", "get-url", "origin")
		if err != nil {
			return err
		}
		rm.childRepoUrl = childRepo
	}

	rm.lastRollRev = lastRollRev
	rm.nextRollRev = nextRollRev
	rm.notRolledRevs = notRolledRevs

	sklog.Infof("lastRollRev is: %s", rm.lastRollRev)
	sklog.Infof("nextRollRev is: %s", nextRollRev)
	sklog.Infof("len(notRolledRevs): %v", len(rm.notRolledRevs))
	if len(rm.notRolledRevs) > NOT_ROLLED_LIMIT {
		sklog.Infof("Truncating notRolledRevs to the last %d", NOT_ROLLED_LIMIT)
		rm.notRolledRevs = rm.notRolledRevs[(len(rm.notRolledRevs) - NOT_ROLLED_LIMIT):]
	}
	return nil
}

// getNotRolledRevs is a utility function that uses CIPD to find the not-yet-rolled versions of
// the specified package.
// Note: that this just finds all versions of the package between the last rolled version and the
// version currently pointed to by cipdAssetTag; we can't know whether the ref we're tracking was
// ever actually applied to any of the package instances in between.
func getNotRolledRevs(ctx context.Context, cipdClient cipd.CIPDClient, lastRollRev, cipdAssetName, cipdAssetTag string) ([]*revision.Revision, error) {
	head, err := cipdClient.ResolveVersion(ctx, cipdAssetName, cipdAssetTag)
	if err != nil {
		return nil, err
	}
	iter, err := cipdClient.ListInstances(ctx, cipdAssetName)
	if err != nil {
		return nil, err
	}
	notRolledRevs := []*revision.Revision{}
	foundHead := false
	for {
		instances, err := iter.Next(ctx, 100)
		if err != nil {
			return nil, err
		}
		if len(instances) == 0 {
			break
		}
{'fraction_non_alphanumeric': 0.11783617556611686, 'fraction_numerical': 0.0007688006709533128, 'mean_word_length': 4.260661764705882, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '674573', 'n_tokens_mistral': 5434, 'n_tokens_neox': 5224, 'n_words': 1277}