<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package common

import (
	"context"
	"strings"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/server/tq"
)

// MostSevereError returns the most severe error in order of
// non-transient => transient => nil.
//
// Walks over potentially recursive errors.MultiError errors only.
//
// Returns only singular errors or nil if input was nil.
func MostSevereError(err error) error {
	if err == nil {
		return nil
	}
	errs, ok := err.(errors.MultiError)
	if !ok {
		return err
	}
	var firstTrans error
	for _, err := range errs {
		switch err = MostSevereError(err); {
		case err == nil:
		case !transient.Tag.In(err):
			return err
		case firstTrans == nil:
			firstTrans = err
		}
	}
	return firstTrans
}

// TQifyError does final error processing before returning from a TQ handler.
//
// * logs error stack,
// * non-transient errors are tagged with tq.Fatal to avoid retries.
func TQifyError(ctx context.Context, err error) error {
	if err == nil {
		return nil
	}
	logError(
		ctx,
		err,
		64*1024, // Cloud Logging hard limit is 256KB.

		// These packages are not useful in CV tests:
		"github.com/smartystreets/goconvey/convey",
		"github.com/jtolds/gls",
		// These packages are not useful in production:
		"go.chromium.org/luci/server",
		"go.chromium.org/luci/server/tq",
	)
	if !transient.Tag.In(err) {
		err = tq.Fatal.Apply(err)
	}
	return err
}

func logError(ctx context.Context, err error, maxBytesPerEntry int, excludePackages ...string) {
	// TODO(tandrii): upstream this fork of errors.Log.
	log := logging.Get(ctx)
	lines := errors.RenderStack(err, excludePackages...)
	for len(lines) > 0 {
		bytes := 0
		batch := lines
		for i, l := range lines {
			bytes += len(l)
			// Always allow first line into the batch to ensure progress, even if it's
			// too big. Practically, this should never happen, but if it doesn't, we
			// shouldn't be looping forever.
			if bytes > maxBytesPerEntry && i > 0 {
				batch = lines[:i] // excluding i-th.
				break
			}
		}
		lines = lines[len(batch):]
		log.Errorf("%s", strings.Join(batch, "\n"))
	}
}<commit_msg>cv: skip server/router package in error stacks.<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package common

import (
	"context"
	"strings"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/server/tq"
)

// MostSevereError returns the most severe error in order of
// non-transient => transient => nil.
//
// Walks over potentially recursive errors.MultiError errors only.
//
// Returns only singular errors or nil if input was nil.
func MostSevereError(err error) error {
	if err == nil {
		return nil
	}
	errs, ok := err.(errors.MultiError)
	if !ok {
		return err
	}
	var firstTrans error
	for _, err := range errs {
		switch err = MostSevereError(err); {
		case err == nil:
		case !transient.Tag.In(err):
			return err
		case firstTrans == nil:
			firstTrans = err
		}
	}
	return firstTrans
}

// TQifyError does final error processing before returning from a TQ handler.
//
// * logs error stack,
// * non-transient errors are tagged with tq.Fatal to avoid retries.
func TQifyError(ctx context.Context, err error) error {
	if err == nil {
		return nil
	}
	logError(
		ctx,
		err,
		64*1024, // Cloud Logging hard limit is 256KB.

		// These packages are not useful in CV tests:
		"github.com/smartystreets/goconvey/convey",
		"github.com/jtolds/gls",
		// These packages are not useful in production:
		"go.chromium.org/luci/server",
		"go.chromium.org/luci/server/tq",
		"go.chromium.org/luci/server/router",
	)
	if !transient.Tag.In(err) {
		err = tq.Fatal.Apply(err)
	}
	return err
}

func logError(ctx context.Context, err error, maxBytesPerEntry int, excludePackages ...string) {
	// TODO(tandrii): upstream this fork of errors.Log.
	log := logging.Get(ctx)
	lines := errors.RenderStack(err, excludePackages...)
	for len(lines) > 0 {
		bytes := 0
		batch := lines
		for i, l := range lines {
			bytes += len(l)
			// Always allow first line into the batch to ensure progress, even if it's
			// too big. Practically, this should never happen, but if it doesn't, we
			// shouldn't be looping forever.
			if bytes > maxBytesPerEntry && i > 0 {
				batch = lines[:i] // excluding i-th.
				break
			}
		}
		lines = lines[len(batch):]
		log.Errorf("%s", strings.Join(batch, "\n"))
	}
}
{'fraction_non_alphanumeric': 0.11949573863636363, 'fraction_numerical': 0.007102272727272727, 'mean_word_length': 3.2194756554307116, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6787172', 'n_tokens_mistral': 1982, 'n_tokens_neox': 1809, 'n_words': 625}