<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/util/sets"
	"sigs.k8s.io/yaml"

	"k8s.io/test-infra/prow/config/org"
	"k8s.io/test-infra/prow/config/secret"
	"k8s.io/test-infra/prow/errorutil"
	"k8s.io/test-infra/prow/flagutil"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/logrusutil"
)

const (
	defaultMinAdmins = 5
	defaultDelta     = 0.25
	defaultTokens    = 300
	defaultBurst     = 100
)

type options struct {
	config            string
	confirm           bool
	dump              string
	dumpFull          bool
	jobConfig         string
	maximumDelta      float64
	minAdmins         int
	requireSelf       bool
	requiredAdmins    flagutil.Strings
	fixOrg            bool
	fixOrgMembers     bool
	fixTeamMembers    bool
	fixTeams          bool
	fixTeamRepos      bool
	ignoreSecretTeams bool
	github            flagutil.GitHubOptions
	tokenBurst        int
	tokensPerHour     int
	logLevel          string
}

func parseOptions() options {
	var o options
	if err := o.parseArgs(flag.CommandLine, os.Args[1:]); err != nil {
		logrus.Fatalf("Invalid flags: %v", err)
	}
	return o
}

func (o *options) parseArgs(flags *flag.FlagSet, args []string) error {
	o.requiredAdmins = flagutil.NewStrings()
	flags.Var(&o.requiredAdmins, "required-admins", "Ensure config specifies these users as admins")
	flags.IntVar(&o.minAdmins, "min-admins", defaultMinAdmins, "Ensure config specifies at least this many admins")
	flags.BoolVar(&o.requireSelf, "require-self", true, "Ensure --github-token-path user is an admin")
	flags.Float64Var(&o.maximumDelta, "maximum-removal-delta", defaultDelta, "Fail if config removes more than this fraction of current members")
	flags.StringVar(&o.config, "config-path", "", "Path to org config.yaml")
	flags.StringVar(&o.jobConfig, "job-config-path", "", "Path to prow job configs.")
	flags.BoolVar(&o.confirm, "confirm", false, "Mutate github if set")
	flags.IntVar(&o.tokensPerHour, "tokens", defaultTokens, "Throttle hourly token consumption (0 to disable)")
	flags.IntVar(&o.tokenBurst, "token-burst", defaultBurst, "Allow consuming a subset of hourly tokens in a short burst")
	flags.StringVar(&o.dump, "dump", "", "Output current config of this org if set")
	flags.BoolVar(&o.dumpFull, "dump-full", false, "Output current config of the org as a valid input config file instead of a snippet")
	flags.BoolVar(&o.ignoreSecretTeams, "ignore-secret-teams", false, "Do not dump or update secret teams if set")
	flags.BoolVar(&o.fixOrg, "fix-org", false, "Change org metadata if set")
	flags.BoolVar(&o.fixOrgMembers, "fix-org-members", false, "Add/remove org members if set")
	flags.BoolVar(&o.fixTeams, "fix-teams", false, "Create/delete/update teams if set")
	flags.BoolVar(&o.fixTeamMembers, "fix-team-members", false, "Add/remove team members if set")
	flags.BoolVar(&o.fixTeamRepos, "fix-team-repos", false, "Add/remove team permissions on repos if set")
	flags.StringVar(&o.logLevel, "log-level", logrus.InfoLevel.String(), fmt.Sprintf("Logging level, one of %v", logrus.AllLevels))
	o.github.AddFlags(flags)
	if err := flags.Parse(args); err != nil {
		return err
	}
	if err := o.github.Validate(!o.confirm); err != nil {
		return err
	}
	if o.tokensPerHour > 0 && o.tokenBurst >= o.tokensPerHour {
		return fmt.Errorf("--tokens=%d must exceed --token-burst=%d", o.tokensPerHour, o.tokenBurst)
	}

	if o.minAdmins < 2 {
		return fmt.Errorf("--min-admins=%d must be at least 2", o.minAdmins)
	}
	if o.maximumDelta > 1 || o.maximumDelta < 0 {
		return fmt.Errorf("--maximum-removal-delta=%f must be a non-negative number less than 1.0", o.maximumDelta)
	}

	if o.confirm && o.dump != "" {
		return fmt.Errorf("--confirm cannot be used with --dump=%s", o.dump)
	}
	if o.config == "" && o.dump == "" {
		return errors.New("--config-path or --dump required")
	}
	if o.config != "" && o.dump != "" {
		return fmt.Errorf("--config-path=%s and --dump=%s cannot both be set", o.config, o.dump)
	}

	if o.jobConfig != "" {
		logrus.Warn("--job-config-path is deprecated and unused, stop using it before July 2019")
	}

	if o.dumpFull && o.dump == "" {
		return errors.New("--dump-full can't be used without --dump")
	}

	if o.fixTeamMembers && !o.fixTeams {
		return fmt.Errorf("--fix-team-members requires --fix-teams")
	}

	if o.fixTeamRepos && !o.fixTeams {
		return fmt.Errorf("--fix-team-repos requires --fix-teams")
	}

	level, err := logrus.ParseLevel(o.logLevel)
	if err != nil {
		return fmt.Errorf("--log-level invalid: %v", err)
	}
	logrus.SetLevel(level)

	return nil<commit_msg>Remove `--job-config-path` option from peribolos<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/util/sets"
	"sigs.k8s.io/yaml"

	"k8s.io/test-infra/prow/config/org"
	"k8s.io/test-infra/prow/config/secret"
	"k8s.io/test-infra/prow/errorutil"
	"k8s.io/test-infra/prow/flagutil"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/logrusutil"
)

const (
	defaultMinAdmins = 5
	defaultDelta     = 0.25
	defaultTokens    = 300
	defaultBurst     = 100
)

type options struct {
	config            string
	confirm           bool
	dump              string
	dumpFull          bool
	maximumDelta      float64
	minAdmins         int
	requireSelf       bool
	requiredAdmins    flagutil.Strings
	fixOrg            bool
	fixOrgMembers     bool
	fixTeamMembers    bool
	fixTeams          bool
	fixTeamRepos      bool
	ignoreSecretTeams bool
	github            flagutil.GitHubOptions
	tokenBurst        int
	tokensPerHour     int
	logLevel          string
}

func parseOptions() options {
	var o options
	if err := o.parseArgs(flag.CommandLine, os.Args[1:]); err != nil {
		logrus.Fatalf("Invalid flags: %v", err)
	}
	return o
}

func (o *options) parseArgs(flags *flag.FlagSet, args []string) error {
	o.requiredAdmins = flagutil.NewStrings()
	flags.Var(&o.requiredAdmins, "required-admins", "Ensure config specifies these users as admins")
	flags.IntVar(&o.minAdmins, "min-admins", defaultMinAdmins, "Ensure config specifies at least this many admins")
	flags.BoolVar(&o.requireSelf, "require-self", true, "Ensure --github-token-path user is an admin")
	flags.Float64Var(&o.maximumDelta, "maximum-removal-delta", defaultDelta, "Fail if config removes more than this fraction of current members")
	flags.StringVar(&o.config, "config-path", "", "Path to org config.yaml")
	flags.BoolVar(&o.confirm, "confirm", false, "Mutate github if set")
	flags.IntVar(&o.tokensPerHour, "tokens", defaultTokens, "Throttle hourly token consumption (0 to disable)")
	flags.IntVar(&o.tokenBurst, "token-burst", defaultBurst, "Allow consuming a subset of hourly tokens in a short burst")
	flags.StringVar(&o.dump, "dump", "", "Output current config of this org if set")
	flags.BoolVar(&o.dumpFull, "dump-full", false, "Output current config of the org as a valid input config file instead of a snippet")
	flags.BoolVar(&o.ignoreSecretTeams, "ignore-secret-teams", false, "Do not dump or update secret teams if set")
	flags.BoolVar(&o.fixOrg, "fix-org", false, "Change org metadata if set")
	flags.BoolVar(&o.fixOrgMembers, "fix-org-members", false, "Add/remove org members if set")
	flags.BoolVar(&o.fixTeams, "fix-teams", false, "Create/delete/update teams if set")
	flags.BoolVar(&o.fixTeamMembers, "fix-team-members", false, "Add/remove team members if set")
	flags.BoolVar(&o.fixTeamRepos, "fix-team-repos", false, "Add/remove team permissions on repos if set")
	flags.StringVar(&o.logLevel, "log-level", logrus.InfoLevel.String(), fmt.Sprintf("Logging level, one of %v", logrus.AllLevels))
	o.github.AddFlags(flags)
	if err := flags.Parse(args); err != nil {
		return err
	}
	if err := o.github.Validate(!o.confirm); err != nil {
		return err
	}
	if o.tokensPerHour > 0 && o.tokenBurst >= o.tokensPerHour {
		return fmt.Errorf("--tokens=%d must exceed --token-burst=%d", o.tokensPerHour, o.tokenBurst)
	}

	if o.minAdmins < 2 {
		return fmt.Errorf("--min-admins=%d must be at least 2", o.minAdmins)
	}
	if o.maximumDelta > 1 || o.maximumDelta < 0 {
		return fmt.Errorf("--maximum-removal-delta=%f must be a non-negative number less than 1.0", o.maximumDelta)
	}

	if o.confirm && o.dump != "" {
		return fmt.Errorf("--confirm cannot be used with --dump=%s", o.dump)
	}
	if o.config == "" && o.dump == "" {
		return errors.New("--config-path or --dump required")
	}
	if o.config != "" && o.dump != "" {
		return fmt.Errorf("--config-path=%s and --dump=%s cannot both be set", o.config, o.dump)
	}

	if o.dumpFull && o.dump == "" {
		return errors.New("--dump-full can't be used without --dump")
	}

	if o.fixTeamMembers && !o.fixTeams {
		return fmt.Errorf("--fix-team-members requires --fix-teams")
	}

	if o.fixTeamRepos && !o.fixTeams {
		return fmt.Errorf("--fix-team-repos requires --fix-teams")
	}

	level, err := logrus.ParseLevel(o.logLevel)
	if err != nil {
		return fmt.Errorf("--log-level invalid: %v", err)
	}
	logrus.SetLevel(level)

	return nil
{'fraction_non_alphanumeric': 0.1350450803606429, 'fraction_numerical': 0.008036064288514309, 'mean_word_length': 3.711449676823638, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7527212', 'n_tokens_mistral': 3632, 'n_tokens_neox': 3469, 'n_words': 975}