<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package decorate

import (
	"fmt"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/sirupsen/logrus"
	coreapi "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/validation"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	"k8s.io/test-infra/prow/clonerefs"
	"k8s.io/test-infra/prow/entrypoint"
	"k8s.io/test-infra/prow/gcsupload"
	"k8s.io/test-infra/prow/initupload"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/pod-utils/clone"
	"k8s.io/test-infra/prow/pod-utils/downwardapi"
	"k8s.io/test-infra/prow/pod-utils/wrapper"
	"k8s.io/test-infra/prow/sidecar"
)

const (
	logMountName            = "logs"
	logMountPath            = "/logs"
	artifactsEnv            = "ARTIFACTS"
	artifactsPath           = logMountPath + "/artifacts"
	codeMountName           = "code"
	codeMountPath           = "/home/prow/go"
	gopathEnv               = "GOPATH"
	toolsMountName          = "tools"
	toolsMountPath          = "/tools"
	gcsCredentialsMountName = "gcs-credentials"
	gcsCredentialsMountPath = "/secrets/gcs"
	s3CredentialsMountName  = "s3-credentials"
	s3CredentialsMountPath  = "/secrets/s3-storage"
	outputMountName         = "output"
	outputMountPath         = "/output"
	oauthTokenFilename      = "oauth-token"
)

// Labels returns a string slice with label consts from kube.
func Labels() []string {
	return []string{kube.ProwJobTypeLabel, kube.CreatedByProw, kube.ProwJobIDLabel}
}

// VolumeMounts returns a string set with *MountName consts in it.
func VolumeMounts() sets.String {
	return sets.NewString(logMountName, codeMountName, toolsMountName, gcsCredentialsMountName, s3CredentialsMountName)
}

// VolumeMountsOnTestContainer returns a string set with *MountName consts in it which are applied to the test container.
func VolumeMountsOnTestContainer() sets.String {
	return sets.NewString(logMountName, codeMountName, toolsMountName)
}

// VolumeMountPathsOnTestContainer returns a string set with *MountPath consts in it which are applied to the test container.
func VolumeMountPathsOnTestContainer() sets.String {
	return sets.NewString(logMountPath, codeMountPath, toolsMountPath)
}

// PodUtilsContainerNames returns a string set with pod utility container name consts in it.
func PodUtilsContainerNames() sets.String {
	return sets.NewString(cloneRefsName, initUploadName, entrypointName, sidecarName)
}

// LabelsAndAnnotationsForSpec returns a minimal set of labels to add to prowjobs or its owned resources.
//
// User-provided extraLabels and extraAnnotations values will take precedence over auto-provided values.
func LabelsAndAnnotationsForSpec(spec prowapi.ProwJobSpec, extraLabels, extraAnnotations map[string]string) (map[string]string, map[string]string) {
	log := logrus.WithFields(logrus.Fields{
		"job": spec.Job,
		"id":  extraLabels[kube.ProwBuildIDLabel],
	})
	labels := map[string]string{
		kube.CreatedByProw:    "true",
		kube.ProwJobTypeLabel: string(spec.Type),
	}
	annotations := map[string]string{}
	for key, value := range map[string]string{
		kube.ProwJobAnnotation: spec.Job,
		kube.ContextAnnotation: spec.Context,
	} {
		maybeTruncated := value
		if len(value) > validation.LabelValueMaxLength {
			// TODO(fejta): consider truncating middle rather than end.
			maybeTruncated = strings.TrimRight(value[:validation.LabelValueMaxLength], ".-")
			log.WithFields(logrus.Fields{
				"key":            key,
				"value":          value,
				"maybeTruncated": maybeTruncated,
			}).Info("Cannot use full value, will truncate.")
		}
		labels[key] = maybeTruncated
		annotations[key] = value<commit_msg>Prow job decoration: also trim underscore after truncating<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package decorate

import (
	"fmt"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/sirupsen/logrus"
	coreapi "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/validation"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	"k8s.io/test-infra/prow/clonerefs"
	"k8s.io/test-infra/prow/entrypoint"
	"k8s.io/test-infra/prow/gcsupload"
	"k8s.io/test-infra/prow/initupload"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/pod-utils/clone"
	"k8s.io/test-infra/prow/pod-utils/downwardapi"
	"k8s.io/test-infra/prow/pod-utils/wrapper"
	"k8s.io/test-infra/prow/sidecar"
)

const (
	logMountName            = "logs"
	logMountPath            = "/logs"
	artifactsEnv            = "ARTIFACTS"
	artifactsPath           = logMountPath + "/artifacts"
	codeMountName           = "code"
	codeMountPath           = "/home/prow/go"
	gopathEnv               = "GOPATH"
	toolsMountName          = "tools"
	toolsMountPath          = "/tools"
	gcsCredentialsMountName = "gcs-credentials"
	gcsCredentialsMountPath = "/secrets/gcs"
	s3CredentialsMountName  = "s3-credentials"
	s3CredentialsMountPath  = "/secrets/s3-storage"
	outputMountName         = "output"
	outputMountPath         = "/output"
	oauthTokenFilename      = "oauth-token"
)

// Labels returns a string slice with label consts from kube.
func Labels() []string {
	return []string{kube.ProwJobTypeLabel, kube.CreatedByProw, kube.ProwJobIDLabel}
}

// VolumeMounts returns a string set with *MountName consts in it.
func VolumeMounts() sets.String {
	return sets.NewString(logMountName, codeMountName, toolsMountName, gcsCredentialsMountName, s3CredentialsMountName)
}

// VolumeMountsOnTestContainer returns a string set with *MountName consts in it which are applied to the test container.
func VolumeMountsOnTestContainer() sets.String {
	return sets.NewString(logMountName, codeMountName, toolsMountName)
}

// VolumeMountPathsOnTestContainer returns a string set with *MountPath consts in it which are applied to the test container.
func VolumeMountPathsOnTestContainer() sets.String {
	return sets.NewString(logMountPath, codeMountPath, toolsMountPath)
}

// PodUtilsContainerNames returns a string set with pod utility container name consts in it.
func PodUtilsContainerNames() sets.String {
	return sets.NewString(cloneRefsName, initUploadName, entrypointName, sidecarName)
}

// LabelsAndAnnotationsForSpec returns a minimal set of labels to add to prowjobs or its owned resources.
//
// User-provided extraLabels and extraAnnotations values will take precedence over auto-provided values.
func LabelsAndAnnotationsForSpec(spec prowapi.ProwJobSpec, extraLabels, extraAnnotations map[string]string) (map[string]string, map[string]string) {
	log := logrus.WithFields(logrus.Fields{
		"job": spec.Job,
		"id":  extraLabels[kube.ProwBuildIDLabel],
	})
	labels := map[string]string{
		kube.CreatedByProw:    "true",
		kube.ProwJobTypeLabel: string(spec.Type),
	}
	annotations := map[string]string{}
	for key, value := range map[string]string{
		kube.ProwJobAnnotation: spec.Job,
		kube.ContextAnnotation: spec.Context,
	} {
		maybeTruncated := value
		if len(value) > validation.LabelValueMaxLength {
			// TODO(fejta): consider truncating middle rather than end.
			maybeTruncated = strings.TrimRight(value[:validation.LabelValueMaxLength], "._-")
			log.WithFields(logrus.Fields{
				"key":            key,
				"value":          value,
				"maybeTruncated": maybeTruncated,
			}).Info("Cannot use full value, will truncate.")
		}
		labels[key] = maybeTruncated
		annotations[key] = value
{'fraction_non_alphanumeric': 0.10404962546816479, 'fraction_numerical': 0.007256554307116105, 'mean_word_length': 4.160024154589372, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '228421', 'n_tokens_mistral': 2928, 'n_tokens_neox': 2814, 'n_words': 684}