<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildbucket

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"time"

	"go.chromium.org/luci/buildbucket/bbperms"
	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/server/auth"
	"go.chromium.org/luci/server/auth/realms"
	"go.chromium.org/luci/server/caching"
	"go.chromium.org/luci/server/caching/layered"

	"go.chromium.org/luci/milo/common"
	"go.chromium.org/luci/milo/frontend/ui"
)

const (
	// Keep the builders in cache for 10 mins to speed up repeated page loads and
	// reduce stress on buildbucket side.
	// But this also means newly added/removed builders would take 10 mins to
	// propagate.
	// Cache duration can be adjusted if needed.
	cacheDuration = 10 * time.Minute

	// Refresh the builders cache if the cache TTL falls below this threshold.
	cacheRefreshThreshold = cacheDuration - time.Minute
)

var buildbucketBuildersCache = layered.Cache{
	ProcessLRUCache: caching.RegisterLRUCache(64),
	GlobalNamespace: "buildbucket-builders-v3",
	Marshal:         json.Marshal,
	Unmarshal: func(blob []byte) (interface{}, error) {
		res := make([]*buildbucketpb.BuilderID, 0)
		err := json.Unmarshal(blob, &res)
		return res, err
	},
}

// getAllBuilders returns all cached buildbucket builders. If the cache expired,
// refresh it with Milo's credential.
func getAllBuilders(c context.Context, host string, opt ...layered.Option) ([]*buildbucketpb.BuilderID, error) {
	builders, err := buildbucketBuildersCache.GetOrCreate(c, host, func() (v interface{}, exp time.Duration, err error) {
		start := time.Now()

		buildersClient, err := ProdBuildersClientFactory(c, host, auth.AsSelf)
		if err != nil {
			return nil, 0, err
		}

		// Get all the Builder IDs from buildbucket.
		bids := make([]*buildbucketpb.BuilderID, 0)
		req := &buildbucketpb.ListBuildersRequest{PageSize: 1000}
		for {
			r, err := buildersClient.ListBuilders(c, req)
			if err != nil {
				return nil, 0, err
			}

			for _, builder := range r.Builders {
				bids = append(bids, builder.Id)
			}

			if r.NextPageToken == "" {
				break
			}
			req.PageToken = r.NextPageToken
		}

		logging.Infof(c, "listing all builders from buildbucket took %v", time.Since(start))

		return bids, cacheDuration, nil
	})
	if err != nil {
		return nil, err
	}

	return builders.([]*buildbucketpb.BuilderID), nil
}

// filterVisibleBuilders returns a list of builders that are visible to the
// current user.
func filterVisibleBuilders(c context.Context, builders []*buildbucketpb.BuilderID) ([]*buildbucketpb.BuilderID, error) {<commit_msg>[milo] fix missing opt after rebase<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildbucket

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"time"

	"go.chromium.org/luci/buildbucket/bbperms"
	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/server/auth"
	"go.chromium.org/luci/server/auth/realms"
	"go.chromium.org/luci/server/caching"
	"go.chromium.org/luci/server/caching/layered"

	"go.chromium.org/luci/milo/common"
	"go.chromium.org/luci/milo/frontend/ui"
)

const (
	// Keep the builders in cache for 10 mins to speed up repeated page loads and
	// reduce stress on buildbucket side.
	// But this also means newly added/removed builders would take 10 mins to
	// propagate.
	// Cache duration can be adjusted if needed.
	cacheDuration = 10 * time.Minute

	// Refresh the builders cache if the cache TTL falls below this threshold.
	cacheRefreshThreshold = cacheDuration - time.Minute
)

var buildbucketBuildersCache = layered.Cache{
	ProcessLRUCache: caching.RegisterLRUCache(64),
	GlobalNamespace: "buildbucket-builders-v3",
	Marshal:         json.Marshal,
	Unmarshal: func(blob []byte) (interface{}, error) {
		res := make([]*buildbucketpb.BuilderID, 0)
		err := json.Unmarshal(blob, &res)
		return res, err
	},
}

// getAllBuilders returns all cached buildbucket builders. If the cache expired,
// refresh it with Milo's credential.
func getAllBuilders(c context.Context, host string, opt ...layered.Option) ([]*buildbucketpb.BuilderID, error) {
	builders, err := buildbucketBuildersCache.GetOrCreate(c, host, func() (v interface{}, exp time.Duration, err error) {
		start := time.Now()

		buildersClient, err := ProdBuildersClientFactory(c, host, auth.AsSelf)
		if err != nil {
			return nil, 0, err
		}

		// Get all the Builder IDs from buildbucket.
		bids := make([]*buildbucketpb.BuilderID, 0)
		req := &buildbucketpb.ListBuildersRequest{PageSize: 1000}
		for {
			r, err := buildersClient.ListBuilders(c, req)
			if err != nil {
				return nil, 0, err
			}

			for _, builder := range r.Builders {
				bids = append(bids, builder.Id)
			}

			if r.NextPageToken == "" {
				break
			}
			req.PageToken = r.NextPageToken
		}

		logging.Infof(c, "listing all builders from buildbucket took %v", time.Since(start))

		return bids, cacheDuration, nil
	}, opt...)
	if err != nil {
		return nil, err
	}

	return builders.([]*buildbucketpb.BuilderID), nil
}

// filterVisibleBuilders returns a list of builders that are visible to the
// current user.
func filterVisibleBuilders(c context.Context, builders []*buildbucketpb.BuilderID) ([]*buildbucketpb.BuilderID, error) {
{'fraction_non_alphanumeric': 0.1166407465007776, 'fraction_numerical': 0.007776049766718507, 'mean_word_length': 3.806427503736921, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '961854', 'n_tokens_mistral': 2228, 'n_tokens_neox': 2014, 'n_words': 630}