<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build

import (
	"context"
	"fmt"
	"io"
	"sync"
	"sync/atomic"

	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/timestamppb"

	bbpb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/buildbucket/protoutil"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/sync/dispatcher"
	"go.chromium.org/luci/logdog/client/butlerlib/streamclient"
	ldTypes "go.chromium.org/luci/logdog/common/types"
)

// State is the state of the current Build.
//
// This is properly initialized with the Start function, and as long as it isn't
// "End"ed, you can manipulate it with the State's various methods.
//
// The State is preserved in the context.Context for use with the ScheduleStep
// and StartStep functions. These will add a new manipulatable step to the build
// State.
//
// All manipulations to the build State will result in an invocation of the
// configured Send function (see OptSend).
type State struct {
	ctx       context.Context
	ctxCloser func()

	// copyExclusionMu is "backwards", intentionally.
	//
	// The lock is held in "WRITE" mode in order to do `proto.Clone` on buildPb,
	// since the Clone operation actually can write metadata to the struct, and is
	// not safe with concurrent writes to the proto message.
	//
	// The lock is held in "READ" mode for all other mutations to buildPb; The
	// library has other mutexes to protect indivitual portions of the buildPb
	// from concurrent modification.
	//
	// This is done to allow e.g. multiple Steps to be mutated concurrently, but
	// allow `proto.Clone` to proceed safely.
	copyExclusionMu sync.RWMutex
	buildPb         *bbpb.Build
	// buildPbVers updated/read with sync/atomic while holding copyExclusionMu in
	// either WRITE/READ mode.
	buildPbVers int64
	// buildPbVersSent only updated when copyExclusionMu is held in WRITE mode.
	buildPbVersSent int64

	sendCh dispatcher.Channel

	logsink    *streamclient.Client
	logNames   nameTracker
	logClosers map[string]func() error

	stepsMu   sync.Mutex
	stepNames nameTracker
}

var _ Loggable = (*State)(nil)

// Start is the entrypoint to this library.
//
// This function clones `initial` as the basis of all state updates (see
// OptSend) and MakePropertyReader declarations. This also initializes the build
// State in `ctx` and returns the manipulable State object.
//
// Start may print information and exit the program immediately if various
// command-line options, such as `--help`, are passed. Use OptSuppressExit() to
// avoid this.
//
// You must End the returned State. To automatically map errors and panics to
// their correct visual representation, End the State like:
//
//    var err error
//    state, ctx := build.Start(ctx, initialBuild, ...)
//    defer func() { state.End(err) }()
//
//    err = opThatErrsOrPanics(ctx)
//
// NOTE: A panic will still crash the program as usual. This does NOT
// `recover()` the panic. Please use conventional Go error handling and control
// flow mechanisms.
func Start(ctx context.Context, initial *bbpb.Build, opts ...StartOption) (*State, context.Context) {
	if initial == nil {
		initial = &bbpb.Build{}
	}

	ret := &State{
		buildPb:    proto.Clone(initial).(*bbpb.Build),
		logClosers: map[string]func() error{},
	}
	ret.ctx, ret.ctxCloser = context.WithCancel(ctx)

	for _, opt := range opts {
		opt(ret)
	}

	// initialize proto sections which other code in this module assumes exist.
	proto.Merge(ret.buildPb, &bbpb.Build{
		Output: &bbpb.Build_Output{},
	})

	// in case our buildPb is unstarted, start it now.
	if ret.buildPb.StartTime == nil {
		ret.buildPb.StartTime = timestamppb.New(clock.Now(ctx))
		ret.buildPb.Status = bbpb.Status_STARTED
	}

	// initialize all log names already in ret.buildPb; likely this includes
	// stdout/stderr which may already be populated by our parent process, such as
	// `bbagent`.
	for _, l := range ret.buildPb.Output.Logs {
		ret.logNames.resolveName(l.Name)
	}

	return ret, setState(ctx, ctxState{ret, nil})
}

// End sets the build's final status, according to `err` (See ExtractStatus).
//
// End will also be able to set INFRA_FAILURE status and log additional
// information if the program is panic'ing.
//
// End must be invoked like:
//
//    var err error
//    state, ctx := build.Start(ctx, initialBuild, ...)
//    defer func() { state.End(err) }()
//
//    err = opThatErrsOrPanics(ctx)
//
// NOTE: A panic will still crash the program as usual. This does NOT
// `recover()` the panic. Please use conventional Go error handling and control
// flow mechanisms.
func (s *State) End(err error) {
	var message string
	s.mutate(func() bool {
		s.buildPb.Status, message = computePanicStatus(err)
		s.buildPb.EndTime = timestamppb.New(clock.Now(s.ctx))

		// TODO(iannucci): handle closing logs

		return true<commit_msg>[luciexe/build] Fix missed log closers for State.<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build

import (
	"context"
	"fmt"
	"io"
	"sync"
	"sync/atomic"

	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/timestamppb"

	bbpb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/buildbucket/protoutil"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/sync/dispatcher"
	"go.chromium.org/luci/logdog/client/butlerlib/streamclient"
	ldTypes "go.chromium.org/luci/logdog/common/types"
)

// State is the state of the current Build.
//
// This is properly initialized with the Start function, and as long as it isn't
// "End"ed, you can manipulate it with the State's various methods.
//
// The State is preserved in the context.Context for use with the ScheduleStep
// and StartStep functions. These will add a new manipulatable step to the build
// State.
//
// All manipulations to the build State will result in an invocation of the
// configured Send function (see OptSend).
type State struct {
	ctx       context.Context
	ctxCloser func()

	// copyExclusionMu is "backwards", intentionally.
	//
	// The lock is held in "WRITE" mode in order to do `proto.Clone` on buildPb,
	// since the Clone operation actually can write metadata to the struct, and is
	// not safe with concurrent writes to the proto message.
	//
	// The lock is held in "READ" mode for all other mutations to buildPb; The
	// library has other mutexes to protect indivitual portions of the buildPb
	// from concurrent modification.
	//
	// This is done to allow e.g. multiple Steps to be mutated concurrently, but
	// allow `proto.Clone` to proceed safely.
	copyExclusionMu sync.RWMutex
	buildPb         *bbpb.Build
	// buildPbVers updated/read with sync/atomic while holding copyExclusionMu in
	// either WRITE/READ mode.
	buildPbVers int64
	// buildPbVersSent only updated when copyExclusionMu is held in WRITE mode.
	buildPbVersSent int64

	sendCh dispatcher.Channel

	logsink    *streamclient.Client
	logNames   nameTracker
	logClosers map[string]func() error

	stepsMu   sync.Mutex
	stepNames nameTracker
}

var _ Loggable = (*State)(nil)

// Start is the entrypoint to this library.
//
// This function clones `initial` as the basis of all state updates (see
// OptSend) and MakePropertyReader declarations. This also initializes the build
// State in `ctx` and returns the manipulable State object.
//
// Start may print information and exit the program immediately if various
// command-line options, such as `--help`, are passed. Use OptSuppressExit() to
// avoid this.
//
// You must End the returned State. To automatically map errors and panics to
// their correct visual representation, End the State like:
//
//    var err error
//    state, ctx := build.Start(ctx, initialBuild, ...)
//    defer func() { state.End(err) }()
//
//    err = opThatErrsOrPanics(ctx)
//
// NOTE: A panic will still crash the program as usual. This does NOT
// `recover()` the panic. Please use conventional Go error handling and control
// flow mechanisms.
func Start(ctx context.Context, initial *bbpb.Build, opts ...StartOption) (*State, context.Context) {
	if initial == nil {
		initial = &bbpb.Build{}
	}

	ret := &State{
		buildPb:    proto.Clone(initial).(*bbpb.Build),
		logClosers: map[string]func() error{},
	}
	ret.ctx, ret.ctxCloser = context.WithCancel(ctx)

	for _, opt := range opts {
		opt(ret)
	}

	// initialize proto sections which other code in this module assumes exist.
	proto.Merge(ret.buildPb, &bbpb.Build{
		Output: &bbpb.Build_Output{},
	})

	// in case our buildPb is unstarted, start it now.
	if ret.buildPb.StartTime == nil {
		ret.buildPb.StartTime = timestamppb.New(clock.Now(ctx))
		ret.buildPb.Status = bbpb.Status_STARTED
	}

	// initialize all log names already in ret.buildPb; likely this includes
	// stdout/stderr which may already be populated by our parent process, such as
	// `bbagent`.
	for _, l := range ret.buildPb.Output.Logs {
		ret.logNames.resolveName(l.Name)
	}

	return ret, setState(ctx, ctxState{ret, nil})
}

// End sets the build's final status, according to `err` (See ExtractStatus).
//
// End will also be able to set INFRA_FAILURE status and log additional
// information if the program is panic'ing.
//
// End must be invoked like:
//
//    var err error
//    state, ctx := build.Start(ctx, initialBuild, ...)
//    defer func() { state.End(err) }()
//
//    err = opThatErrsOrPanics(ctx)
//
// NOTE: A panic will still crash the program as usual. This does NOT
// `recover()` the panic. Please use conventional Go error handling and control
// flow mechanisms.
func (s *State) End(err error) {
	var message string
	s.mutate(func() bool {
		s.buildPb.Status, message = computePanicStatus(err)
		s.buildPb.EndTime = timestamppb.New(clock.Now(s.ctx))

		for logName, closer := range s.logClosers {
			if err := closer(); err != nil {
				logging.Warningf(s.ctx, "error closing log %q: %s", logName, err)
			}
		}
		s.logClosers = nil

		return true
{'fraction_non_alphanumeric': 0.10906642728904847, 'fraction_numerical': 0.0021543985637342907, 'mean_word_length': 3.426301152165276, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1422168', 'n_tokens_mistral': 3552, 'n_tokens_neox': 3380, 'n_words': 1266}