<commit_before>/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tide

import (
	"context"
	"errors"
	"fmt"
	"strconv"
	"time"

	"k8s.io/apimachinery/pkg/util/sets"
	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	v1 "k8s.io/test-infra/prow/apis/prowjobs/v1"
	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/gerrit/client"
	"k8s.io/test-infra/prow/git/types"
	"k8s.io/test-infra/prow/git/v2"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/tide/blockers"
	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"

	"github.com/andygrunwald/go-gerrit"
	githubql "github.com/shurcooL/githubv4"
	"github.com/sirupsen/logrus"
)

const (
	// ref:
	// https://gerrit-review.googlesource.com/Documentation/user-search.html#_search_operators.
	// Also good to know: `(repo:repo-A OR repo:repo-B)`
	gerritDefaultQueryParam = "status:open -is:wip is:submittable is:mergeable"
)

type gerritClient interface {
	QueryChangesForProject(instance, project string, lastUpdate time.Time, rateLimit int, addtionalFilters []string) ([]gerrit.ChangeInfo, error)
	GetBranchRevision(instance, project, branch string) (string, error)
}

// Enforcing interface implementation check at compile time
var _ provider = (*GerritProvider)(nil)

// GerritProvider implements provider, used by tide Controller for
// interacting directly with Gerrit.
//
// Tide Controller should only use GerritProvider for communicating with Gerrit.
type GerritProvider struct {
	cfg config.Getter
	gc  gerritClient

	pjclientset ctrlruntimeclient.Client

	cookiefilePath    string
	tokenPathOverride string

	*mergeChecker
	logger *logrus.Entry
}

func newGerritProvider(
	logger *logrus.Entry,
	cfg config.Getter,
	pjclientset ctrlruntimeclient.Client,
	mergeChecker *mergeChecker,
	cookiefilePath string,
	tokenPathOverride string,
) *GerritProvider {
	gerritClient, err := client.NewClient(nil)
	if err != nil {
		logrus.WithError(err).Fatal("Error creating gerrit client.")
	}

	gp := &GerritProvider{
		logger:            logger,
		cfg:               cfg,
		pjclientset:       pjclientset,
		gc:                gerritClient,
		cookiefilePath:    cookiefilePath,
		tokenPathOverride: tokenPathOverride,
		mergeChecker:      mergeChecker,
	}

	gp.applyGlobalConfig(cfg, gerritClient, cookiefilePath, tokenPathOverride)

	return gp
}

func (p *GerritProvider) applyGlobalConfig(cfg config.Getter, gerritClient *client.Client, cookiefilePath, tokenPathOverride string) {
	p.applyGlobalConfigOnce(cfg, gerritClient, cookiefilePath, tokenPathOverride)

	go func() {
		for {
			p.applyGlobalConfigOnce(cfg, gerritClient, cookiefilePath, tokenPathOverride)
			// No need to spin constantly, give it a break. It's ok that config change has one second delay.
			time.Sleep(time.Second)
		}
	}()
}

func (p *GerritProvider) applyGlobalConfigOnce(cfg config.Getter, gerritClient *client.Client, cookiefilePath, tokenPathOverride string) {
	gerritCfg := cfg().Tide.Gerrit
	if gerritCfg == nil {
		return
	}

	if err := gerritClient.UpdateClients(gerritCfg.Queries.AllRepos()); err != nil {
		logrus.WithError(err).Error("Updating clients.")
	}
	// Authenticate creates a goroutine for rotating token secrets when called the first
	// time, afterwards it only authenticate once.
	gerritClient.Authenticate(cookiefilePath, tokenPathOverride)
}

// Query returns all PRs from configured gerrit org/repos.
func (p *GerritProvider) Query() (map[string]CodeReviewCommon, error) {
	// lastUpdate is used by gerrit adapter for achieving incremental query. In
	// tide case we want to get everything so use default time.Time, which
	// should be 1970,1,1.
	var lastUpdate time.Time

	res := make(map[string]CodeReviewCommon)
	// This is querying serially, which would safely guard against quota issues.
	// TODO(chaodai): parallize this to boot the performance.
	for instance, projs := range p.cfg().Tide.Gerrit.Queries.AllRepos() {
		for projName := range projs {
			changes, err := p.gc.QueryChangesForProject(instance, projName, lastUpdate, p.cfg().Gerrit.RateLimit, []string{gerritDefaultQueryParam})
			if err != nil {
				p.logger.WithFields(logrus.Fields{"instance": instance, "project": projName}).WithError(err).Error("Querying gerrit project for changes.")
				continue
			}
			for _, pr := range changes {
				crc := CodeReviewCommonFromGerrit(&pr, instance)
				res[prKey(crc)] = *crc
			}
		}
	}

	return res, nil
}
<commit_msg>Tide: Gerrit controller uses common auth refresher<commit_after>/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tide

import (
	"context"
	"errors"
	"fmt"
	"strconv"
	"time"

	"k8s.io/apimachinery/pkg/util/sets"
	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	v1 "k8s.io/test-infra/prow/apis/prowjobs/v1"
	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/gerrit/client"
	"k8s.io/test-infra/prow/git/types"
	"k8s.io/test-infra/prow/git/v2"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/tide/blockers"
	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"

	"github.com/andygrunwald/go-gerrit"
	githubql "github.com/shurcooL/githubv4"
	"github.com/sirupsen/logrus"
)

const (
	// ref:
	// https://gerrit-review.googlesource.com/Documentation/user-search.html#_search_operators.
	// Also good to know: `(repo:repo-A OR repo:repo-B)`
	gerritDefaultQueryParam = "status:open -is:wip is:submittable is:mergeable"
)

type gerritClient interface {
	QueryChangesForProject(instance, project string, lastUpdate time.Time, rateLimit int, addtionalFilters []string) ([]gerrit.ChangeInfo, error)
	GetBranchRevision(instance, project, branch string) (string, error)
}

// Enforcing interface implementation check at compile time
var _ provider = (*GerritProvider)(nil)

// GerritProvider implements provider, used by tide Controller for
// interacting directly with Gerrit.
//
// Tide Controller should only use GerritProvider for communicating with Gerrit.
type GerritProvider struct {
	cfg config.Getter
	gc  gerritClient

	pjclientset ctrlruntimeclient.Client

	cookiefilePath    string
	tokenPathOverride string

	*mergeChecker
	logger *logrus.Entry
}

func newGerritProvider(
	logger *logrus.Entry,
	cfg config.Getter,
	pjclientset ctrlruntimeclient.Client,
	mergeChecker *mergeChecker,
	cookiefilePath string,
	tokenPathOverride string,
) *GerritProvider {
	gerritClient, err := client.NewClient(nil)
	if err != nil {
		logrus.WithError(err).Fatal("Error creating gerrit client.")
	}
	orgRepoConfigGetter := func() *config.GerritOrgRepoConfigs {
		return &cfg().Tide.Gerrit.Queries
	}
	gerritClient.ApplyGlobalConfig(orgRepoConfigGetter, nil, cookiefilePath, tokenPathOverride, nil)

	return &GerritProvider{
		logger:            logger,
		cfg:               cfg,
		pjclientset:       pjclientset,
		gc:                gerritClient,
		cookiefilePath:    cookiefilePath,
		tokenPathOverride: tokenPathOverride,
		mergeChecker:      mergeChecker,
	}
}

// Query returns all PRs from configured gerrit org/repos.
func (p *GerritProvider) Query() (map[string]CodeReviewCommon, error) {
	// lastUpdate is used by gerrit adapter for achieving incremental query. In
	// tide case we want to get everything so use default time.Time, which
	// should be 1970,1,1.
	var lastUpdate time.Time

	res := make(map[string]CodeReviewCommon)
	// This is querying serially, which would safely guard against quota issues.
	// TODO(chaodai): parallize this to boot the performance.
	for instance, projs := range p.cfg().Tide.Gerrit.Queries.AllRepos() {
		for projName := range projs {
			changes, err := p.gc.QueryChangesForProject(instance, projName, lastUpdate, p.cfg().Gerrit.RateLimit, []string{gerritDefaultQueryParam})
			if err != nil {
				p.logger.WithFields(logrus.Fields{"instance": instance, "project": projName}).WithError(err).Error("Querying gerrit project for changes.")
				continue
			}
			for _, pr := range changes {
				crc := CodeReviewCommonFromGerrit(&pr, instance)
				res[prKey(crc)] = *crc
			}
		}
	}

	return res, nil
}

{'fraction_non_alphanumeric': 0.10461504326603062, 'fraction_numerical': 0.006434435322831151, 'mean_word_length': 4.385304659498208, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6095215', 'n_tokens_mistral': 3140, 'n_tokens_neox': 2945, 'n_words': 777}