<commit_before>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package refresher

import (
	"context"

	"github.com/google/go-github/v26/github"

	"istio.io/bots/policybot/handlers/githubwebhook"
	"istio.io/bots/policybot/pkg/blobstorage"
	"istio.io/bots/policybot/pkg/config"
	"istio.io/bots/policybot/pkg/gh"
	gatherer "istio.io/bots/policybot/pkg/resultgatherer"
	"istio.io/bots/policybot/pkg/storage"
	"istio.io/bots/policybot/pkg/storage/cache"
	"istio.io/pkg/log"
)

// Updates the DB based on incoming GitHub webhook events.
type Refresher struct {
	cache *cache.Cache
	store storage.Store
	gc    *gh.ThrottledClient
	bs    blobstorage.Store
	reg   *config.Registry
}

var scope = log.RegisterScope("refresher", "Dynamic database refresher", 0)

func NewRefresher(cache *cache.Cache, store storage.Store, bs blobstorage.Store, gc *gh.ThrottledClient, reg *config.Registry) githubwebhook.Filter {
	return &Refresher{
		cache: cache,
		store: store,
		bs:    bs,
		gc:    gc,
		reg:   reg,
	}
}

// accept an event arriving from GitHub
func (r *Refresher) Handle(context context.Context, event interface{}) {
	switch p := event.(type) {
	case *github.IssuesEvent:
		scope.Infof("Received IssuesEvent: %s, %d, %s", p.GetRepo().GetFullName(), p.GetIssue().GetNumber(), p.GetAction())

		if _, ok := r.reg.SingleRecord(RecordType, p.GetRepo().GetFullName()); !ok {
			scope.Infof("Ignoring issue %d from repo %s since there aren't matching refreshers", p.GetIssue().GetNumber(), p.GetRepo().GetFullName())
			return
		}

		issue := gh.ConvertIssue(
			p.GetRepo().GetOwner().GetLogin(),
			p.GetRepo().GetName(),
			p.GetIssue())
		issues := []*storage.Issue{issue}
		if err := r.cache.WriteIssues(context, issues); err != nil {
			scope.Errorf(err.Error())
			return
		}

		event := &storage.IssueEvent{
			OrgLogin:    issue.OrgLogin,
			RepoName:    issue.RepoName,
			IssueNumber: issue.IssueNumber,
			CreatedAt:   p.GetIssue().GetUpdatedAt(),
			Actor:       p.GetSender().GetLogin(),
			Action:      p.GetAction(),
		}

		events := []*storage.IssueEvent{event}
		if err := r.store.WriteIssueEvents(context, events); err != nil {
			scope.Error(err.Error())
			return
		}

		r.syncUsers(context, issue.Author)
		r.syncUsers(context, issue.Assignees...)

	case *github.IssueCommentEvent:
		scope.Infof("Received IssueCommentEvent: %s, %d, %s", p.GetRepo().GetFullName(), p.GetIssue().GetNumber(), p.GetAction())

		if _, ok := r.reg.SingleRecord(RecordType, p.GetRepo().GetFullName()); !ok {
			scope.Infof("Ignoring issue comment for issue %d from repo %s since there are no matching refreshers", p.GetIssue().GetNumber(), p.GetRepo().GetFullName())
			return
		}

		issueComment := gh.ConvertIssueComment(
			p.GetRepo().GetOwner().GetLogin(),
			p.GetRepo().GetName(),
			p.GetIssue().GetNumber(),
			p.GetComment())
		issueComments := []*storage.IssueComment{issueComment}
		if err := r.cache.WriteIssueComments(context, issueComments); err != nil {
			scope.Error(err.Error())
			return
		}

		event := &storage.IssueCommentEvent{
			OrgLogin:       issueComment.OrgLogin,
			RepoName:       issueComment.RepoName,
			IssueNumber:    issueComment.IssueNumber,
			IssueCommentID: p.GetComment().GetID(),
			CreatedAt:      p.GetComment().GetUpdatedAt(),
			Actor:          p.GetSender().GetLogin(),
			Action:         p.GetAction(),
		}

		events := []*storage.IssueCommentEvent{event}
		if err := r.store.WriteIssueCommentEvents(context, events); err != nil {
			scope.Error(err.Error())
			return
		}

		r.syncUsers(context, issueComment.Author)

	case *github.PullRequestEvent:
		scope.Infof("Received PullRequestEvent: %s, %d, %s", p.GetRepo().GetFullName(), p.GetNumber(), p.GetAction())

		action := p.GetAction()

		rec, ok := r.reg.SingleRecord(RecordType, p.GetRepo().GetFullName())
		if ok {
			ref := rec.(*TestOutputRecord)
			orgLogin := p.GetRepo().GetOwner().GetLogin()
			repoName := p.GetRepo().GetName()
			prNum := p.GetNumber()

			tg := gatherer.TestResultGatherer{
				Client:           r.bs,
				BucketName:       ref.BucketName,
				PreSubmitPrefix:  ref.PreSubmitTestPath,
				PostSubmitPrefix: ref.PostSubmitTestPath,
			}

			testResults, err := tg.CheckTestResultsForPr(context, orgLogin, repoName, string(prNum))
			if err != nil {
				scope.Errorf("Unable to get test result for PR %d in repo %s: %v", prNum, p.GetRepo().GetFullName(), err)
			} else if err = r.cache.WriteTestResults(context, testResults); err != nil {
				scope.Errorf("Unable to write test results: %v", err)
			}
		}

		if action == "opened" || action == "edited" {
			opt := &github.ListOptions{
				PerPage: 100,
			}

			// get the set of files comprising this PR since the payload didn't supply them
			var allFiles []string
			for {
				files, resp, err := r.gc.ThrottledCall(func(client *github.Client) (interface{}, *github.Response, error) {
					return client.PullRequests.ListFiles(context, p.GetRepo().GetOwner().GetLogin(), p.GetRepo().GetName(), p.GetNumber(), opt)
				})

				if err != nil {
					scope.Errorf("Unable to list all files for pull request %d in repo %s: %v\n", p.GetNumber(), p.GetRepo().GetFullName(), err)
					return
				}

				for _, f := range files.([]*github.CommitFile) {
					allFiles = append(allFiles, f.GetFilename())
				}

				if resp.NextPage == 0 {
					break
				}

				opt.Page = resp.NextPage
			}<commit_msg>Nagger should refresh data on updates (#145)<commit_after>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package refresher

import (
	"context"

	"github.com/google/go-github/v26/github"

	"istio.io/bots/policybot/handlers/githubwebhook"
	"istio.io/bots/policybot/pkg/blobstorage"
	"istio.io/bots/policybot/pkg/config"
	"istio.io/bots/policybot/pkg/gh"
	gatherer "istio.io/bots/policybot/pkg/resultgatherer"
	"istio.io/bots/policybot/pkg/storage"
	"istio.io/bots/policybot/pkg/storage/cache"
	"istio.io/pkg/log"
)

// Updates the DB based on incoming GitHub webhook events.
type Refresher struct {
	cache *cache.Cache
	store storage.Store
	gc    *gh.ThrottledClient
	bs    blobstorage.Store
	reg   *config.Registry
}

var scope = log.RegisterScope("refresher", "Dynamic database refresher", 0)

func NewRefresher(cache *cache.Cache, store storage.Store, bs blobstorage.Store, gc *gh.ThrottledClient, reg *config.Registry) githubwebhook.Filter {
	return &Refresher{
		cache: cache,
		store: store,
		bs:    bs,
		gc:    gc,
		reg:   reg,
	}
}

// accept an event arriving from GitHub
func (r *Refresher) Handle(context context.Context, event interface{}) {
	switch p := event.(type) {
	case *github.IssuesEvent:
		scope.Infof("Received IssuesEvent: %s, %d, %s", p.GetRepo().GetFullName(), p.GetIssue().GetNumber(), p.GetAction())

		if _, ok := r.reg.SingleRecord(RecordType, p.GetRepo().GetFullName()); !ok {
			scope.Infof("Ignoring issue %d from repo %s since there aren't matching refreshers", p.GetIssue().GetNumber(), p.GetRepo().GetFullName())
			return
		}

		issue := gh.ConvertIssue(
			p.GetRepo().GetOwner().GetLogin(),
			p.GetRepo().GetName(),
			p.GetIssue())
		issues := []*storage.Issue{issue}
		if err := r.cache.WriteIssues(context, issues); err != nil {
			scope.Errorf(err.Error())
			return
		}

		event := &storage.IssueEvent{
			OrgLogin:    issue.OrgLogin,
			RepoName:    issue.RepoName,
			IssueNumber: issue.IssueNumber,
			CreatedAt:   p.GetIssue().GetUpdatedAt(),
			Actor:       p.GetSender().GetLogin(),
			Action:      p.GetAction(),
		}

		events := []*storage.IssueEvent{event}
		if err := r.store.WriteIssueEvents(context, events); err != nil {
			scope.Error(err.Error())
			return
		}

		r.syncUsers(context, issue.Author)
		r.syncUsers(context, issue.Assignees...)

	case *github.IssueCommentEvent:
		scope.Infof("Received IssueCommentEvent: %s, %d, %s", p.GetRepo().GetFullName(), p.GetIssue().GetNumber(), p.GetAction())

		if _, ok := r.reg.SingleRecord(RecordType, p.GetRepo().GetFullName()); !ok {
			scope.Infof("Ignoring issue comment for issue %d from repo %s since there are no matching refreshers", p.GetIssue().GetNumber(), p.GetRepo().GetFullName())
			return
		}

		issueComment := gh.ConvertIssueComment(
			p.GetRepo().GetOwner().GetLogin(),
			p.GetRepo().GetName(),
			p.GetIssue().GetNumber(),
			p.GetComment())
		issueComments := []*storage.IssueComment{issueComment}
		if err := r.cache.WriteIssueComments(context, issueComments); err != nil {
			scope.Error(err.Error())
			return
		}

		event := &storage.IssueCommentEvent{
			OrgLogin:       issueComment.OrgLogin,
			RepoName:       issueComment.RepoName,
			IssueNumber:    issueComment.IssueNumber,
			IssueCommentID: p.GetComment().GetID(),
			CreatedAt:      p.GetComment().GetUpdatedAt(),
			Actor:          p.GetSender().GetLogin(),
			Action:         p.GetAction(),
		}

		events := []*storage.IssueCommentEvent{event}
		if err := r.store.WriteIssueCommentEvents(context, events); err != nil {
			scope.Error(err.Error())
			return
		}

		r.syncUsers(context, issueComment.Author)

	case *github.PullRequestEvent:
		scope.Infof("Received PullRequestEvent: %s, %d, %s", p.GetRepo().GetFullName(), p.GetNumber(), p.GetAction())

		action := p.GetAction()

		rec, ok := r.reg.SingleRecord(RecordType, p.GetRepo().GetFullName())
		if ok {
			ref := rec.(*TestOutputRecord)
			orgLogin := p.GetRepo().GetOwner().GetLogin()
			repoName := p.GetRepo().GetName()
			prNum := p.GetNumber()

			tg := gatherer.TestResultGatherer{
				Client:           r.bs,
				BucketName:       ref.BucketName,
				PreSubmitPrefix:  ref.PreSubmitTestPath,
				PostSubmitPrefix: ref.PostSubmitTestPath,
			}

			testResults, err := tg.CheckTestResultsForPr(context, orgLogin, repoName, string(prNum))
			if err != nil {
				scope.Errorf("Unable to get test result for PR %d in repo %s: %v", prNum, p.GetRepo().GetFullName(), err)
			} else if err = r.cache.WriteTestResults(context, testResults); err != nil {
				scope.Errorf("Unable to write test results: %v", err)
			}
		}

		if action == "opened" || action == "edited" || action == "synchronize" {
			opt := &github.ListOptions{
				PerPage: 100,
			}

			// get the set of files comprising this PR since the payload didn't supply them
			var allFiles []string
			for {
				files, resp, err := r.gc.ThrottledCall(func(client *github.Client) (interface{}, *github.Response, error) {
					return client.PullRequests.ListFiles(context, p.GetRepo().GetOwner().GetLogin(), p.GetRepo().GetName(), p.GetNumber(), opt)
				})

				if err != nil {
					scope.Errorf("Unable to list all files for pull request %d in repo %s: %v\n", p.GetNumber(), p.GetRepo().GetFullName(), err)
					return
				}

				for _, f := range files.([]*github.CommitFile) {
					allFiles = append(allFiles, f.GetFilename())
				}

				if resp.NextPage == 0 {
					break
				}

				opt.Page = resp.NextPage
			}
{'fraction_non_alphanumeric': 0.14702911967682208, 'fraction_numerical': 0.0027773102171351626, 'mean_word_length': 3.654524089306698, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3746215', 'n_tokens_mistral': 4403, 'n_tokens_neox': 3897, 'n_words': 934}