<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildstore

import (
	"time"

	"golang.org/x/net/context"

	"go.chromium.org/gae/service/datastore"
	"go.chromium.org/luci/common/auth/identity"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/sync/parallel"
	"go.chromium.org/luci/milo/api/buildbot"
	"go.chromium.org/luci/milo/common"
	"go.chromium.org/luci/server/auth"
)

// Master is buildbot.Master plus extra storage-level information.
type Master struct {
	buildbot.Master
	Internal bool
	Modified time.Time
}

// GetMaster fetches a master.
//
// If refreshState is true, refreshes individual builds from the datastore
// and the list of Cached builds.
//
// If any of the master's builders is emulated, the returned Master
// does not have any slave information or pending build states.
//
// Does not check access.
func GetMaster(c context.Context, name string, refreshState bool) (*Master, error) {
	entity := masterEntity{Name: name}
	err := datastore.Get(c, &entity)
	if err == datastore.ErrNoSuchEntity {
		return nil, errors.New("master not found", common.CodeNotFound)
	}
	if err != nil {
		return nil, err
	}

	m, err := entity.decode()
	if err != nil {
		return nil, err
	}
	if !refreshState {
		return m, nil
	}

	emulation := false
	for builder := range m.Builders {
		opt, err := GetEmulationOptions(c, name, builder)
		if err != nil {
			return nil, err
		}
		if opt != nil {
			emulation = true
			break
		}
	}
	if emulation {
		// Emulation does not support this Slaves field.
		m.Slaves = nil
		for _, b := range m.Builders {
			b.Slaves = nil
			b.PendingBuildStates = nil
		}
	} else {
		var refreshBuilds []*buildEntity
		for _, slave := range m.Slaves {
			for _, b := range slave.Runningbuilds {
				refreshBuilds = append(refreshBuilds, (*buildEntity)(b))
			}
		}
		if err = datastore.Get(c, refreshBuilds); err != nil {
			return nil, errors.Annotate(err, "refresh builds").Err()
		}
	}

	// Inject cached builds information.
	return m, parallel.FanOutIn(func(work chan<- func() error) {
		for builderName, builder := range m.Builders {
			builderName := builderName
			builder := builder
			work <- func() error {
				// Get the most recent 50 buildNums on the builder to simulate what the
				// cachedBuilds field looks like from the real buildbot master json.
				q := Query{
					Master:   name,
					Builder:  builderName,
					Finished: Yes,
					Limit:    50,

					NoAnnotationFetch: true,
					KeyOnly:           true,
				}
				res, err := GetBuilds(c, q)
				if err != nil {
					return err
				}
				builder.CachedBuilds = make([]int, len(res.Builds))
				for i, b := range res.Builds {
					builder.CachedBuilds[i] = b.Number
				}

				opt, err := GetEmulationOptions(c, name, builderName)
				if err != nil {
					return err
				}
				if opt != nil {
					// Pending buildbot builds do not have a number yet
					// so it is hard to tell whether they WILL be before
					// or after opt.StartFrom. Just ignore them.
					builder.PendingBuilds = 0
					// But keep current ones because they have numbers
					// and it is easier to decide whether should be included in the
					// response or not.
					buildbotCurrent := builder.CurrentBuilds
					builder.CurrentBuilds = nil

					// Load all incomplete builds and add only those after
					// opt.StartFrom.
					q := Query{
						Master:   name,
						Builder:  builderName,
						Finished: No,

						// we will ignore buildbot builds
						KeyOnly:           true,
						NoAnnotationFetch: true,
					}
					res, err := GetBuilds(c, q)
					if err != nil {
						return err
					}
					for _, b := range res.Builds {
						if b.Number < int(opt.StartFrom) {
							// the rest of res.Builds are buildbot builds
							// ignore them.
							break
						}
						if b.Times.Start.IsZero() {
							builder.PendingBuilds++
						} else {
							builder.CurrentBuilds = append(builder.CurrentBuilds, b.Number)
						}
					}
					for _, num := range buildbotCurrent {
						if num < int(opt.StartFrom) {
							builder.CurrentBuilds = append(builder.CurrentBuilds, num)
						}
					}
				}

				return nil
			}
		}
	})
}

// AllMasters returns all buildbot masters.
func AllMasters(c context.Context, checkAccess bool) ([]*Master, error) {
	const batchSize = int32(500)
	allowInternal := isAllowedInternal(c)
	masters := make([]*Master, 0, batchSize)
	q := datastore.NewQuery(masterKind)
	err := datastore.RunBatch(c, batchSize, q, func(e *masterEntity) error {
		if !checkAccess || allowInternal || !e.Internal {
			m, err := e.decode()
			if err != nil {
				return err
			}
			masters = append(masters, m)
		}
		return nil
	})
	return masters, err
}

// ExpireCallback is called when a build is marked as expired.
type ExpireCallback func(b *buildbot.Build, reason string)

// SaveMaster persists the master in the storage.
//
// Expires all incomplete builds in the datastore associated with this master
// and
//   - associated with builders not declared in master.Builders
//   - OR or not "current" from this master's perspective and >=20min stale.
func SaveMaster(c context.Context, master *buildbot.Master,
	internal bool, expireCallback ExpireCallback) error {<commit_msg>[milo] avoid calling IsMember in cron<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildstore

import (
	"time"

	"golang.org/x/net/context"

	"go.chromium.org/gae/service/datastore"
	"go.chromium.org/luci/common/auth/identity"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/sync/parallel"
	"go.chromium.org/luci/milo/api/buildbot"
	"go.chromium.org/luci/milo/common"
	"go.chromium.org/luci/server/auth"
)

// Master is buildbot.Master plus extra storage-level information.
type Master struct {
	buildbot.Master
	Internal bool
	Modified time.Time
}

// GetMaster fetches a master.
//
// If refreshState is true, refreshes individual builds from the datastore
// and the list of Cached builds.
//
// If any of the master's builders is emulated, the returned Master
// does not have any slave information or pending build states.
//
// Does not check access.
func GetMaster(c context.Context, name string, refreshState bool) (*Master, error) {
	entity := masterEntity{Name: name}
	err := datastore.Get(c, &entity)
	if err == datastore.ErrNoSuchEntity {
		return nil, errors.New("master not found", common.CodeNotFound)
	}
	if err != nil {
		return nil, err
	}

	m, err := entity.decode()
	if err != nil {
		return nil, err
	}
	if !refreshState {
		return m, nil
	}

	emulation := false
	for builder := range m.Builders {
		opt, err := GetEmulationOptions(c, name, builder)
		if err != nil {
			return nil, err
		}
		if opt != nil {
			emulation = true
			break
		}
	}
	if emulation {
		// Emulation does not support this Slaves field.
		m.Slaves = nil
		for _, b := range m.Builders {
			b.Slaves = nil
			b.PendingBuildStates = nil
		}
	} else {
		var refreshBuilds []*buildEntity
		for _, slave := range m.Slaves {
			for _, b := range slave.Runningbuilds {
				refreshBuilds = append(refreshBuilds, (*buildEntity)(b))
			}
		}
		if err = datastore.Get(c, refreshBuilds); err != nil {
			return nil, errors.Annotate(err, "refresh builds").Err()
		}
	}

	// Inject cached builds information.
	return m, parallel.FanOutIn(func(work chan<- func() error) {
		for builderName, builder := range m.Builders {
			builderName := builderName
			builder := builder
			work <- func() error {
				// Get the most recent 50 buildNums on the builder to simulate what the
				// cachedBuilds field looks like from the real buildbot master json.
				q := Query{
					Master:   name,
					Builder:  builderName,
					Finished: Yes,
					Limit:    50,

					NoAnnotationFetch: true,
					KeyOnly:           true,
				}
				res, err := GetBuilds(c, q)
				if err != nil {
					return err
				}
				builder.CachedBuilds = make([]int, len(res.Builds))
				for i, b := range res.Builds {
					builder.CachedBuilds[i] = b.Number
				}

				opt, err := GetEmulationOptions(c, name, builderName)
				if err != nil {
					return err
				}
				if opt != nil {
					// Pending buildbot builds do not have a number yet
					// so it is hard to tell whether they WILL be before
					// or after opt.StartFrom. Just ignore them.
					builder.PendingBuilds = 0
					// But keep current ones because they have numbers
					// and it is easier to decide whether should be included in the
					// response or not.
					buildbotCurrent := builder.CurrentBuilds
					builder.CurrentBuilds = nil

					// Load all incomplete builds and add only those after
					// opt.StartFrom.
					q := Query{
						Master:   name,
						Builder:  builderName,
						Finished: No,

						// we will ignore buildbot builds
						KeyOnly:           true,
						NoAnnotationFetch: true,
					}
					res, err := GetBuilds(c, q)
					if err != nil {
						return err
					}
					for _, b := range res.Builds {
						if b.Number < int(opt.StartFrom) {
							// the rest of res.Builds are buildbot builds
							// ignore them.
							break
						}
						if b.Times.Start.IsZero() {
							builder.PendingBuilds++
						} else {
							builder.CurrentBuilds = append(builder.CurrentBuilds, b.Number)
						}
					}
					for _, num := range buildbotCurrent {
						if num < int(opt.StartFrom) {
							builder.CurrentBuilds = append(builder.CurrentBuilds, num)
						}
					}
				}

				return nil
			}
		}
	})
}

// AllMasters returns all buildbot masters.
func AllMasters(c context.Context, checkAccess bool) ([]*Master, error) {
	const batchSize = int32(500)
	masters := make([]*Master, 0, batchSize)
	q := datastore.NewQuery(masterKind)

	// note: avoid calling isAllowedInternal is checkAccess is false.
	// checkAccess is usually false in cron jobs where there is no auth state,
	// so isAllowedInternal call would unconditionally log an error.
	allowInternal := !checkAccess || isAllowedInternal(c)
	err := datastore.RunBatch(c, batchSize, q, func(e *masterEntity) error {
		if allowInternal || !e.Internal {
			m, err := e.decode()
			if err != nil {
				return err
			}
			masters = append(masters, m)
		}
		return nil
	})
	return masters, err
}

// ExpireCallback is called when a build is marked as expired.
type ExpireCallback func(b *buildbot.Build, reason string)

// SaveMaster persists the master in the storage.
//
// Expires all incomplete builds in the datastore associated with this master
// and
//   - associated with builders not declared in master.Builders
//   - OR or not "current" from this master's perspective and >=20min stale.
func SaveMaster(c context.Context, master *buildbot.Master,
	internal bool, expireCallback ExpireCallback) error {
{'fraction_non_alphanumeric': 0.10611902766135792, 'fraction_numerical': 0.003520536462699078, 'mean_word_length': 2.797262889879058, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4883753', 'n_tokens_mistral': 4437, 'n_tokens_neox': 3674, 'n_words': 1236}