<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package decorate

import (
	"fmt"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/sirupsen/logrus"
	coreapi "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/validation"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	"k8s.io/test-infra/prow/clonerefs"
	"k8s.io/test-infra/prow/entrypoint"
	"k8s.io/test-infra/prow/gcsupload"
	"k8s.io/test-infra/prow/initupload"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/pod-utils/clone"
	"k8s.io/test-infra/prow/pod-utils/downwardapi"
	"k8s.io/test-infra/prow/pod-utils/wrapper"
	"k8s.io/test-infra/prow/sidecar"
)

const (
	logMountName            = "logs"
	logMountPath            = "/logs"
	artifactsEnv            = "ARTIFACTS"
	artifactsPath           = logMountPath + "/artifacts"
	codeMountName           = "code"
	codeMountPath           = "/home/prow/go"
	gopathEnv               = "GOPATH"
	toolsMountName          = "tools"
	toolsMountPath          = "/tools"
	gcsCredentialsMountName = "gcs-credentials"
	gcsCredentialsMountPath = "/secrets/gcs"
	outputMountName         = "output"
	outputMountPath         = "/output"
)

// Labels returns a string slice with label consts from kube.
func Labels() []string {
	return []string{kube.ProwJobTypeLabel, kube.CreatedByProw, kube.ProwJobIDLabel}
}

// VolumeMounts returns a string slice with *MountName consts in it.
func VolumeMounts() []string {
	return []string{logMountName, codeMountName, toolsMountName, gcsCredentialsMountName}
}

// VolumeMountPaths returns a string slice with *MountPath consts in it.
func VolumeMountPaths() []string {
	return []string{logMountPath, codeMountPath, toolsMountPath, gcsCredentialsMountPath}
}

// LabelsAndAnnotationsForSpec returns a minimal set of labels to add to prowjobs or its owned resources.
//
// User-provided extraLabels and extraAnnotations values will take precedence over auto-provided values.
func LabelsAndAnnotationsForSpec(spec prowapi.ProwJobSpec, extraLabels, extraAnnotations map[string]string) (map[string]string, map[string]string) {
	jobNameForLabel := spec.Job
	if len(jobNameForLabel) > validation.LabelValueMaxLength {
		// TODO(fejta): consider truncating middle rather than end.
		jobNameForLabel = strings.TrimRight(spec.Job[:validation.LabelValueMaxLength], ".-")
		logrus.WithFields(logrus.Fields{
			"job":       spec.Job,
			"key":       kube.ProwJobAnnotation,
			"value":     spec.Job,
			"truncated": jobNameForLabel,
		}).Info("Cannot use full job name, will truncate.")
	}
	labels := map[string]string{
		kube.CreatedByProw:     "true",
		kube.ProwJobTypeLabel:  string(spec.Type),
		kube.ProwJobAnnotation: jobNameForLabel,
	}
	if spec.Type != prowapi.PeriodicJob && spec.Refs != nil {
		labels[kube.OrgLabel] = spec.Refs.Org
		labels[kube.RepoLabel] = spec.Refs.Repo
		if len(spec.Refs.Pulls) > 0 {
			labels[kube.PullLabel] = strconv.Itoa(spec.Refs.Pulls[0].Number)
		}
	}

	for k, v := range extraLabels {
		labels[k] = v
	}

	// let's validate labels
	for key, value := range labels {
		if errs := validation.IsValidLabelValue(value); len(errs) > 0 {
			// try to use basename of a path, if path contains invalid //
			base := filepath.Base(value)
			if errs := validation.IsValidLabelValue(base); len(errs) == 0 {
				labels[key] = base
				continue
			}
			logrus.WithFields(logrus.Fields{
				"key":    key,
				"value":  value,
				"errors": errs,
			}).Warn("Removing invalid label")
			delete(labels, key)
		}
	}

	annotations := map[string]string{
		kube.ProwJobAnnotation: spec.Job,
	}
	for k, v := range extraAnnotations {
		annotations[k] = v
	}

	return labels, annotations
}

// LabelsAndAnnotationsForJob returns a standard set of labels to add to pod/build/etc resources.
func LabelsAndAnnotationsForJob(pj prowapi.ProwJob) (map[string]string, map[string]string) {
	var extraLabels map[string]string
	if extraLabels = pj.ObjectMeta.Labels; extraLabels == nil {
		extraLabels = map[string]string{}
	}
	var extraAnnotations map[string]string
	if extraAnnotations = pj.ObjectMeta.Annotations; extraAnnotations == nil {
		extraAnnotations = map[string]string{}
	}
	extraLabels[kube.ProwJobIDLabel] = pj.ObjectMeta.Name
	extraLabels[kube.ProwBuildIDLabel] = pj.Status.BuildID
	return LabelsAndAnnotationsForSpec(pj.Spec, extraLabels, extraAnnotations)
}

// ProwJobToPod converts a ProwJob to a Pod that will run the tests.
func ProwJobToPod(pj prowapi.ProwJob, buildID string) (*coreapi.Pod, error) {
	return ProwJobToPodLocal(pj, buildID, "")
}

// ProwJobToPodLocal converts a ProwJob to a Pod that will run the tests.
// If an output directory is specified, files are copied to the dir instead of uploading to GCS if
// decoration is configured.
func ProwJobToPodLocal(pj prowapi.ProwJob, buildID string, outputDir string) (*coreapi.Pod, error) {
	if pj.Spec.PodSpec == nil {
		return nil, fmt.Errorf("prowjob %q lacks a pod spec", pj.Name)
	}

	rawEnv, err := downwardapi.EnvForSpec(downwardapi.NewJobSpec(pj.Spec, buildID, pj.Name))
	if err != nil {
		return nil, err
	}

	spec := pj.Spec.PodSpec.DeepCopy()
	spec.RestartPolicy = "Never"
	spec.Containers[0].Name = kube.TestContainerName

	// if the user has not provided a serviceaccount to use or explicitly
	// requested mounting the default token, we treat the unset value as
	// false, while kubernetes treats it as true if it is unset because
	// it was added in v1.6
	if spec.AutomountServiceAccountToken == nil && spec.ServiceAccountName == "" {
		myFalse := false
		spec.AutomountServiceAccountToken = &myFalse
	}

	if pj.Spec.DecorationConfig == nil {
		spec.Containers[0].Env = append(spec.Containers[0].Env, KubeEnv(rawEnv)...)
	} else {
		if err := decorate(spec, &pj, rawEnv, outputDir); err != nil {
			return nil, fmt.Errorf("error decorating podspec: %v", err)
		}
	}

	podLabels, annotations := LabelsAndAnnotationsForJob(pj)
	return &coreapi.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:        pj.ObjectMeta.Name,
			Labels:      podLabels,
			Annotations: annotations,
		},
		Spec: *spec,
	}, nil
}

const cloneLogPath = "clone.json"

// CloneLogPath returns the path to the clone log file in the volume mount.
// CloneLogPath returns the path to the clone log file in the volume mount.
func CloneLogPath(logMount coreapi.VolumeMount) string {
	return filepath.Join(logMount.MountPath, cloneLogPath)
}

// Exposed for testing
const (
	cloneRefsName    = "clonerefs"
	cloneRefsCommand = "/clonerefs"
)

// cloneEnv encodes clonerefs Options into json and puts it into an environment variable
func cloneEnv(opt clonerefs.Options) ([]coreapi.EnvVar, error) {
	// TODO(fejta): use flags
	cloneConfigEnv, err := clonerefs.Encode(opt)
	if err != nil {
		return nil, err
	}
	return KubeEnv(map[string]string{clonerefs.JSONConfigEnvVar: cloneConfigEnv}), nil
}

// tmpVolume creates an emptyDir volume and mount for a tmp folder
// This is e.g. used by CloneRefs to store the known hosts file
func tmpVolume(name string) (coreapi.Volume, coreapi.VolumeMount) {
	v := coreapi.Volume{
		Name: name,
		VolumeSource: coreapi.VolumeSource{
			EmptyDir: &coreapi.EmptyDirVolumeSource{},
		},
	}

	vm := coreapi.VolumeMount{
		Name:      name,
		MountPath: "/tmp",
		ReadOnly:  false,
	}

	return v, vm
}

// sshVolume converts a secret holding ssh keys into the corresponding volume and mount.
//
// This is used by CloneRefs to attach the mount to the clonerefs container.
func sshVolume(secret string) (coreapi.Volume, coreapi.VolumeMount) {
	var sshKeyMode int32 = 0400 // this is octal, so symbolic ref is `u+r`
	name := strings.Join([]string{"ssh-keys", secret}, "-")
	mountPath := path.Join("/secrets/ssh", secret)
	v := coreapi.Volume{
		Name: name,
		VolumeSource: coreapi.VolumeSource{
			Secret: &coreapi.SecretVolumeSource{
				SecretName:  secret,
				DefaultMode: &sshKeyMode,
			},
		},
	}

	vm := coreapi.VolumeMount{
		Name:      name,
		MountPath: mountPath,
		ReadOnly:  true,
	}

	return v, vm
}

// cookiefileVolumes converts a secret holding cookies into the corresponding volume and mount.
//
// Secret can be of the form secret-name/base-name or just secret-name.
// Here secret-name refers to the kubernetes secret volume to mount, and base-name refers to the key in the secret
// where the cookies are stored. The secret-name pattern is equivalent to secret-name/secret-name.
//
// This is used by CloneRefs to attach the mount to the clonerefs container.
// The returned string value is the path to the cookiefile for use with --cookiefile.
func cookiefileVolume(secret string) (coreapi.Volume, coreapi.VolumeMount, string) {
	// Separate secret-name/key-in-secret
	parts := strings.SplitN(secret, "/", 2)
	cookieSecret := parts[0]
	var base string
	if len(parts) == 1 {
		base = parts[0] // Assume key-in-secret == secret-name
	} else {
		base = parts[1]
	}
	var cookiefileMode int32 = 0400 // u+r
	vol := coreapi.Volume{
		Name: "cookiefile",
		VolumeSource: coreapi.VolumeSource{
			Secret: &coreapi.SecretVolumeSource{
				SecretName:  cookieSecret,
				DefaultMode: &cookiefileMode,
			},
		},
	}
	mount := coreapi.VolumeMount{
		Name:      vol.Name,
		MountPath: "/secrets/cookiefile", // append base to flag
		ReadOnly:  true,
	}
	return vol, mount, path.Join(mount.MountPath, base)
}

// CloneRefs constructs the container and volumes necessary to clone the refs requested by the ProwJob.
//
// The container checks out repositories specified by the ProwJob Refs to `codeMount`.
// A log of what it checked out is written to `clone.json` in `logMount`.
//
// The container may need to mount SSH keys and/or cookiefiles in order to access private refs.
// CloneRefs returns a list of volumes containing these secrets required by the container.
func CloneRefs(pj prowapi.ProwJob, codeMount, logMount coreapi.VolumeMount) (*coreapi.Container, []prowapi.Refs, []coreapi.Volume, error) {
	if pj.Spec.DecorationConfig == nil {
		return nil, nil, nil, nil
	}
	if skip := pj.Spec.DecorationConfig.SkipCloning; skip != nil && *skip {
		return nil, nil, nil, nil
	}
	var cloneVolumes []coreapi.Volume
	var refs []prowapi.Refs // Do not return []*prowapi.Refs which we do not own
	if pj.Spec.Refs != nil {
		refs = append(refs, *pj.Spec.Refs)
	}
	for _, r := range pj.Spec.ExtraRefs {
		refs = append(refs, r)
	}
	if len(refs) == 0 { // nothing to clone
		return nil, nil, nil, nil
	}
	if codeMount.Name == "" || codeMount.MountPath == "" {
		return nil, nil, nil, fmt.Errorf("codeMount must set Name and MountPath")
	}
	if logMount.Name == "" || logMount.MountPath == "" {
		return nil, nil, nil, fmt.Errorf("logMount must set Name and MountPath")
	}

	var cloneMounts []coreapi.VolumeMount
	var sshKeyPaths []string
	for _, secret := range pj.Spec.DecorationConfig.SSHKeySecrets {
		volume, mount := sshVolume(secret)
		cloneMounts = append(cloneMounts, mount)
		sshKeyPaths = append(sshKeyPaths, mount.MountPath)
		cloneVolumes = append(cloneVolumes, volume)
	}
	volume, mount := tmpVolume("clonerefs-tmp")
	cloneMounts = append(cloneMounts, mount)
	cloneVolumes = append(cloneVolumes, volume)

	var cloneArgs []string
	var cookiefilePath string

	if cp := pj.Spec.DecorationConfig.CookiefileSecret; cp != "" {
		v, vm, vp := cookiefileVolume(cp)
		cloneMounts = append(cloneMounts, vm)
		cloneVolumes = append(cloneVolumes, v)
		cookiefilePath = vp
		cloneArgs = append(cloneArgs, "--cookiefile="+cookiefilePath)
	}

	env, err := cloneEnv(clonerefs.Options{
		CookiePath:       cookiefilePath,
		GitRefs:          refs,
		GitUserEmail:     clonerefs.DefaultGitUserEmail,
		GitUserName:      clonerefs.DefaultGitUserName,
		HostFingerprints: pj.Spec.DecorationConfig.SSHHostFingerprints,
		KeyFiles:         sshKeyPaths,
		Log:              CloneLogPath(logMount),
		SrcRoot:          codeMount.MountPath,
	})
	if err != nil {
		return nil, nil, nil, fmt.Errorf("clone env: %v", err)
	}

	container := coreapi.Container{
		Name:         cloneRefsName,
		Image:        pj.Spec.DecorationConfig.UtilityImages.CloneRefs,
		Command:      []string{cloneRefsCommand},<commit_msg>mount the oauth token from a secret in podspec<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package decorate

import (
	"fmt"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/sirupsen/logrus"
	coreapi "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/validation"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	"k8s.io/test-infra/prow/clonerefs"
	"k8s.io/test-infra/prow/entrypoint"
	"k8s.io/test-infra/prow/gcsupload"
	"k8s.io/test-infra/prow/initupload"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/pod-utils/clone"
	"k8s.io/test-infra/prow/pod-utils/downwardapi"
	"k8s.io/test-infra/prow/pod-utils/wrapper"
	"k8s.io/test-infra/prow/sidecar"
)

const (
	logMountName            = "logs"
	logMountPath            = "/logs"
	artifactsEnv            = "ARTIFACTS"
	artifactsPath           = logMountPath + "/artifacts"
	codeMountName           = "code"
	codeMountPath           = "/home/prow/go"
	gopathEnv               = "GOPATH"
	toolsMountName          = "tools"
	toolsMountPath          = "/tools"
	gcsCredentialsMountName = "gcs-credentials"
	gcsCredentialsMountPath = "/secrets/gcs"
	outputMountName         = "output"
	outputMountPath         = "/output"
	oauthTokenFilename      = "oauth-token"
)

// Labels returns a string slice with label consts from kube.
func Labels() []string {
	return []string{kube.ProwJobTypeLabel, kube.CreatedByProw, kube.ProwJobIDLabel}
}

// VolumeMounts returns a string slice with *MountName consts in it.
func VolumeMounts() []string {
	return []string{logMountName, codeMountName, toolsMountName, gcsCredentialsMountName}
}

// VolumeMountPaths returns a string slice with *MountPath consts in it.
func VolumeMountPaths() []string {
	return []string{logMountPath, codeMountPath, toolsMountPath, gcsCredentialsMountPath}
}

// LabelsAndAnnotationsForSpec returns a minimal set of labels to add to prowjobs or its owned resources.
//
// User-provided extraLabels and extraAnnotations values will take precedence over auto-provided values.
func LabelsAndAnnotationsForSpec(spec prowapi.ProwJobSpec, extraLabels, extraAnnotations map[string]string) (map[string]string, map[string]string) {
	jobNameForLabel := spec.Job
	if len(jobNameForLabel) > validation.LabelValueMaxLength {
		// TODO(fejta): consider truncating middle rather than end.
		jobNameForLabel = strings.TrimRight(spec.Job[:validation.LabelValueMaxLength], ".-")
		logrus.WithFields(logrus.Fields{
			"job":       spec.Job,
			"key":       kube.ProwJobAnnotation,
			"value":     spec.Job,
			"truncated": jobNameForLabel,
		}).Info("Cannot use full job name, will truncate.")
	}
	labels := map[string]string{
		kube.CreatedByProw:     "true",
		kube.ProwJobTypeLabel:  string(spec.Type),
		kube.ProwJobAnnotation: jobNameForLabel,
	}
	if spec.Type != prowapi.PeriodicJob && spec.Refs != nil {
		labels[kube.OrgLabel] = spec.Refs.Org
		labels[kube.RepoLabel] = spec.Refs.Repo
		if len(spec.Refs.Pulls) > 0 {
			labels[kube.PullLabel] = strconv.Itoa(spec.Refs.Pulls[0].Number)
		}
	}

	for k, v := range extraLabels {
		labels[k] = v
	}

	// let's validate labels
	for key, value := range labels {
		if errs := validation.IsValidLabelValue(value); len(errs) > 0 {
			// try to use basename of a path, if path contains invalid //
			base := filepath.Base(value)
			if errs := validation.IsValidLabelValue(base); len(errs) == 0 {
				labels[key] = base
				continue
			}
			logrus.WithFields(logrus.Fields{
				"key":    key,
				"value":  value,
				"errors": errs,
			}).Warn("Removing invalid label")
			delete(labels, key)
		}
	}

	annotations := map[string]string{
		kube.ProwJobAnnotation: spec.Job,
	}
	for k, v := range extraAnnotations {
		annotations[k] = v
	}

	return labels, annotations
}

// LabelsAndAnnotationsForJob returns a standard set of labels to add to pod/build/etc resources.
func LabelsAndAnnotationsForJob(pj prowapi.ProwJob) (map[string]string, map[string]string) {
	var extraLabels map[string]string
	if extraLabels = pj.ObjectMeta.Labels; extraLabels == nil {
		extraLabels = map[string]string{}
	}
	var extraAnnotations map[string]string
	if extraAnnotations = pj.ObjectMeta.Annotations; extraAnnotations == nil {
		extraAnnotations = map[string]string{}
	}
	extraLabels[kube.ProwJobIDLabel] = pj.ObjectMeta.Name
	extraLabels[kube.ProwBuildIDLabel] = pj.Status.BuildID
	return LabelsAndAnnotationsForSpec(pj.Spec, extraLabels, extraAnnotations)
}

// ProwJobToPod converts a ProwJob to a Pod that will run the tests.
func ProwJobToPod(pj prowapi.ProwJob, buildID string) (*coreapi.Pod, error) {
	return ProwJobToPodLocal(pj, buildID, "")
}

// ProwJobToPodLocal converts a ProwJob to a Pod that will run the tests.
// If an output directory is specified, files are copied to the dir instead of uploading to GCS if
// decoration is configured.
func ProwJobToPodLocal(pj prowapi.ProwJob, buildID string, outputDir string) (*coreapi.Pod, error) {
	if pj.Spec.PodSpec == nil {
		return nil, fmt.Errorf("prowjob %q lacks a pod spec", pj.Name)
	}

	rawEnv, err := downwardapi.EnvForSpec(downwardapi.NewJobSpec(pj.Spec, buildID, pj.Name))
	if err != nil {
		return nil, err
	}

	spec := pj.Spec.PodSpec.DeepCopy()
	spec.RestartPolicy = "Never"
	spec.Containers[0].Name = kube.TestContainerName

	// if the user has not provided a serviceaccount to use or explicitly
	// requested mounting the default token, we treat the unset value as
	// false, while kubernetes treats it as true if it is unset because
	// it was added in v1.6
	if spec.AutomountServiceAccountToken == nil && spec.ServiceAccountName == "" {
		myFalse := false
		spec.AutomountServiceAccountToken = &myFalse
	}

	if pj.Spec.DecorationConfig == nil {
		spec.Containers[0].Env = append(spec.Containers[0].Env, KubeEnv(rawEnv)...)
	} else {
		if err := decorate(spec, &pj, rawEnv, outputDir); err != nil {
			return nil, fmt.Errorf("error decorating podspec: %v", err)
		}
	}

	podLabels, annotations := LabelsAndAnnotationsForJob(pj)
	return &coreapi.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:        pj.ObjectMeta.Name,
			Labels:      podLabels,
			Annotations: annotations,
		},
		Spec: *spec,
	}, nil
}

const cloneLogPath = "clone.json"

// CloneLogPath returns the path to the clone log file in the volume mount.
// CloneLogPath returns the path to the clone log file in the volume mount.
func CloneLogPath(logMount coreapi.VolumeMount) string {
	return filepath.Join(logMount.MountPath, cloneLogPath)
}

// Exposed for testing
const (
	cloneRefsName    = "clonerefs"
	cloneRefsCommand = "/clonerefs"
)

// cloneEnv encodes clonerefs Options into json and puts it into an environment variable
func cloneEnv(opt clonerefs.Options) ([]coreapi.EnvVar, error) {
	// TODO(fejta): use flags
	cloneConfigEnv, err := clonerefs.Encode(opt)
	if err != nil {
		return nil, err
	}
	return KubeEnv(map[string]string{clonerefs.JSONConfigEnvVar: cloneConfigEnv}), nil
}

// tmpVolume creates an emptyDir volume and mount for a tmp folder
// This is e.g. used by CloneRefs to store the known hosts file
func tmpVolume(name string) (coreapi.Volume, coreapi.VolumeMount) {
	v := coreapi.Volume{
		Name: name,
		VolumeSource: coreapi.VolumeSource{
			EmptyDir: &coreapi.EmptyDirVolumeSource{},
		},
	}

	vm := coreapi.VolumeMount{
		Name:      name,
		MountPath: "/tmp",
		ReadOnly:  false,
	}

	return v, vm
}

func oauthVolume(secret, key string) (coreapi.Volume, coreapi.VolumeMount) {
	name := strings.Join([]string{"oauth-secret", secret}, "-")
	return coreapi.Volume{
			Name: name,
			VolumeSource: coreapi.VolumeSource{
				Secret: &coreapi.SecretVolumeSource{
					SecretName: secret,
					Items: []coreapi.KeyToPath{{
						Key:  key,
						Path: fmt.Sprintf("./%s", oauthTokenFilename),
					}},
				},
			},
		}, coreapi.VolumeMount{
			Name:      name,
			MountPath: "/secrets/oauth",
			ReadOnly:  true,
		}
}

// sshVolume converts a secret holding ssh keys into the corresponding volume and mount.
//
// This is used by CloneRefs to attach the mount to the clonerefs container.
func sshVolume(secret string) (coreapi.Volume, coreapi.VolumeMount) {
	var sshKeyMode int32 = 0400 // this is octal, so symbolic ref is `u+r`
	name := strings.Join([]string{"ssh-keys", secret}, "-")
	mountPath := path.Join("/secrets/ssh", secret)
	v := coreapi.Volume{
		Name: name,
		VolumeSource: coreapi.VolumeSource{
			Secret: &coreapi.SecretVolumeSource{
				SecretName:  secret,
				DefaultMode: &sshKeyMode,
			},
		},
	}

	vm := coreapi.VolumeMount{
		Name:      name,
		MountPath: mountPath,
		ReadOnly:  true,
	}

	return v, vm
}

// cookiefileVolumes converts a secret holding cookies into the corresponding volume and mount.
//
// Secret can be of the form secret-name/base-name or just secret-name.
// Here secret-name refers to the kubernetes secret volume to mount, and base-name refers to the key in the secret
// where the cookies are stored. The secret-name pattern is equivalent to secret-name/secret-name.
//
// This is used by CloneRefs to attach the mount to the clonerefs container.
// The returned string value is the path to the cookiefile for use with --cookiefile.
func cookiefileVolume(secret string) (coreapi.Volume, coreapi.VolumeMount, string) {
	// Separate secret-name/key-in-secret
	parts := strings.SplitN(secret, "/", 2)
	cookieSecret := parts[0]
	var base string
	if len(parts) == 1 {
		base = parts[0] // Assume key-in-secret == secret-name
	} else {
		base = parts[1]
	}
	var cookiefileMode int32 = 0400 // u+r
	vol := coreapi.Volume{
		Name: "cookiefile",
		VolumeSource: coreapi.VolumeSource{
			Secret: &coreapi.SecretVolumeSource{
				SecretName:  cookieSecret,
				DefaultMode: &cookiefileMode,
			},
		},
	}
	mount := coreapi.VolumeMount{
		Name:      vol.Name,
		MountPath: "/secrets/cookiefile", // append base to flag
		ReadOnly:  true,
	}
	return vol, mount, path.Join(mount.MountPath, base)
}

// CloneRefs constructs the container and volumes necessary to clone the refs requested by the ProwJob.
//
// The container checks out repositories specified by the ProwJob Refs to `codeMount`.
// A log of what it checked out is written to `clone.json` in `logMount`.
//
// The container may need to mount SSH keys and/or cookiefiles in order to access private refs.
// CloneRefs returns a list of volumes containing these secrets required by the container.
func CloneRefs(pj prowapi.ProwJob, codeMount, logMount coreapi.VolumeMount) (*coreapi.Container, []prowapi.Refs, []coreapi.Volume, error) {
	if pj.Spec.DecorationConfig == nil {
		return nil, nil, nil, nil
	}
	if skip := pj.Spec.DecorationConfig.SkipCloning; skip != nil && *skip {
		return nil, nil, nil, nil
	}
	var cloneVolumes []coreapi.Volume
	var refs []prowapi.Refs // Do not return []*prowapi.Refs which we do not own
	if pj.Spec.Refs != nil {
		refs = append(refs, *pj.Spec.Refs)
	}
	for _, r := range pj.Spec.ExtraRefs {
		refs = append(refs, r)
	}
	if len(refs) == 0 { // nothing to clone
		return nil, nil, nil, nil
	}
	if codeMount.Name == "" || codeMount.MountPath == "" {
		return nil, nil, nil, fmt.Errorf("codeMount must set Name and MountPath")
	}
	if logMount.Name == "" || logMount.MountPath == "" {
		return nil, nil, nil, fmt.Errorf("logMount must set Name and MountPath")
	}

	var cloneMounts []coreapi.VolumeMount
	var sshKeyPaths []string
	for _, secret := range pj.Spec.DecorationConfig.SSHKeySecrets {
		volume, mount := sshVolume(secret)
		cloneMounts = append(cloneMounts, mount)
		sshKeyPaths = append(sshKeyPaths, mount.MountPath)
		cloneVolumes = append(cloneVolumes, volume)
	}

	var oauthMountPath string
	if pj.Spec.DecorationConfig.OauthTokenSecret != nil {
		oauthVolume, oauthMount := oauthVolume(pj.Spec.DecorationConfig.OauthTokenSecret.Name, pj.Spec.DecorationConfig.OauthTokenSecret.Key)
		cloneMounts = append(cloneMounts, oauthMount)
		oauthMountPath = filepath.Join(oauthMount.MountPath, oauthTokenFilename)
		cloneVolumes = append(cloneVolumes, oauthVolume)
	}

	volume, mount := tmpVolume("clonerefs-tmp")
	cloneMounts = append(cloneMounts, mount)
	cloneVolumes = append(cloneVolumes, volume)

	var cloneArgs []string
	var cookiefilePath string

	if cp := pj.Spec.DecorationConfig.CookiefileSecret; cp != "" {
		v, vm, vp := cookiefileVolume(cp)
		cloneMounts = append(cloneMounts, vm)
		cloneVolumes = append(cloneVolumes, v)
		cookiefilePath = vp
		cloneArgs = append(cloneArgs, "--cookiefile="+cookiefilePath)
	}

	env, err := cloneEnv(clonerefs.Options{
		CookiePath:       cookiefilePath,
		GitRefs:          refs,
		GitUserEmail:     clonerefs.DefaultGitUserEmail,
		GitUserName:      clonerefs.DefaultGitUserName,
		HostFingerprints: pj.Spec.DecorationConfig.SSHHostFingerprints,
		KeyFiles:         sshKeyPaths,
		Log:              CloneLogPath(logMount),
		SrcRoot:          codeMount.MountPath,
		OauthTokenFile:   oauthMountPath,
	})
	if err != nil {
		return nil, nil, nil, fmt.Errorf("clone env: %v", err)
	}

	container := coreapi.Container{
		Name:         cloneRefsName,
		Image:        pj.Spec.DecorationConfig.UtilityImages.CloneRefs,
		Command:      []string{cloneRefsCommand},
{'fraction_non_alphanumeric': 0.11237921326942099, 'fraction_numerical': 0.004032564863425398, 'mean_word_length': 3.9015476412455716, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3849111', 'n_tokens_mistral': 9287, 'n_tokens_neox': 8725, 'n_words': 2391}