<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"

	"bufio"
	"io/ioutil"
	"os"
	"strings"

	"github.com/google/go-github/github"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"golang.org/x/oauth2"
)

var (
	ghsyncCmd = &cobra.Command{
		Use:           "ghsync",
		Short:         "ghsync syncs configuration across GitHub repositories",
		Long:          "TODO",
		SilenceErrors: true,
		SilenceUsage:  true,
		RunE:          run,
	}
	cfg struct {
		Labels map[string]string `mapstructure:"labels"`
	}
	configFile string
)

func init() {
	ghsyncCmd.PersistentFlags().String("token", "", "GitHub token to use for API authentication")
	must(viper.BindPFlag("token", ghsyncCmd.PersistentFlags().Lookup("token")))
	must(viper.BindEnv("token", "GITHUB_TOKEN"))

	ghsyncCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "ghsync.yaml", "Configuration file in YAML format")

	log.SetFlags(0)
}

func run(cmd *cobra.Command, args []string) error {
	viper.SetConfigFile(configFile)
	if err := viper.ReadInConfig(); err != nil {
		abort(fmt.Errorf("Error reading config file: %s", err))
	}
	if err := viper.Unmarshal(&cfg); err != nil {
		abort(fmt.Errorf("Error parsing configuration: %s", err))
	}

	var httpClient *http.Client
	if token := viper.GetString("token"); token != "" {
		ts := oauth2.StaticTokenSource(
			&oauth2.Token{AccessToken: token},
		)
		httpClient = oauth2.NewClient(oauth2.NoContext, ts)
	}
	c := github.NewClient(httpClient)

	repoReader := bufio.NewReader(os.Stdin)
	repoData, err := ioutil.ReadAll(repoReader)
	if err != nil {
		return fmt.Errorf("Error reading repository list: %s", err)
	}
	repos := strings.Split(strings.Trim(string(repoData), " \n"), "\n")

	if err := syncLabels(cfg.Labels, repos, c); err != nil {
		return fmt.Errorf("Error syncing repositories: %s", err)
	}

	return nil
}

func syncLabels(labels map[string]string, repos []string, c *github.Client) error {
	for _, repo := range repos {
		log.Printf("Processing repository %q", repo)
		repoParts := strings.Split(repo, "/")
		if len(repoParts) < 2 {
			return fmt.Errorf("repository %q is missing the owner. Required format: <owner>/<repository>", repo)
		}

		repoLabels := map[string]string{}
		page := 0

		for {
			repoLabelsPage, resp, err := c.Issues.ListLabels(repoParts[0], repoParts[1], &github.ListOptions{
				Page: page,
			})
			if err != nil {
				return fmt.Errorf("could not list existing labels of repo %q: %s", repo, err)
			}

			for _, repoLabel := range repoLabelsPage {
				repoLabels[*repoLabel.Name] = *repoLabel.Color
			}

			if resp.NextPage == 0 {
				break
			}
			page = resp.NextPage
		}

		log.Printf("  Found %d labels", len(repoLabels))

		for label, color := range labels {
			if color[0] == '#' {
				color = color[1:]
			}
			if len(color) != 6 {
				return fmt.Errorf("color %q of label %q for repo %q is in an invalid format. Colors need to be formatted as six hexadecimal digits", color, label, repo)
			}

			repoLabelColor, ok := repoLabels[label]
			if !ok {
				log.Printf("  Creating label %q: %q", label, color)
				_, _, err := c.Issues.CreateLabel(repoParts[0], repoParts[1], &github.Label{
					Name:  github.String(label),
					Color: github.String(color),
				})
				if err != nil {
					return fmt.Errorf("could not create label %q in repo %q: %s", label, repo, err)
				}
			} else if repoLabelColor != color {
				log.Printf("  Updating label %q: %q", label, color)
				_, _, err := c.Issues.EditLabel(repoParts[0], repoParts[1], label, &github.Label{
					Name:  github.String(label),
					Color: github.String(color),
				})
				if err != nil {
					return fmt.Errorf("could not update label %q in repo %q: %s", label, repo, err)
				}
			} else {
				log.Printf("  Label %q: %q already exists", label, color)
			}
		}
	}
	return nil
}

func must(err error) {
	if err != nil {
		abort(err)
	}
}
<commit_msg>Refactoring: extract functions<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"

	"bufio"
	"io/ioutil"
	"os"
	"strings"

	"github.com/google/go-github/github"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"golang.org/x/oauth2"
)

var (
	ghsyncCmd = &cobra.Command{
		Use:           "ghsync",
		Short:         "ghsync syncs configuration across GitHub repositories",
		Long:          "TODO",
		SilenceErrors: true,
		SilenceUsage:  true,
		RunE:          run,
	}
	cfg struct {
		Labels map[string]string `mapstructure:"labels"`
	}
	configFile string
)

func init() {
	ghsyncCmd.PersistentFlags().String("token", "", "GitHub token to use for API authentication")
	must(viper.BindPFlag("token", ghsyncCmd.PersistentFlags().Lookup("token")))
	must(viper.BindEnv("token", "GITHUB_TOKEN"))

	ghsyncCmd.PersistentFlags().StringVarP(&configFile, "config", "c", "ghsync.yaml", "Configuration file in YAML format")

	log.SetFlags(0)
}

func run(cmd *cobra.Command, args []string) error {
	viper.SetConfigFile(configFile)
	if err := viper.ReadInConfig(); err != nil {
		abort(fmt.Errorf("Error reading config file: %s", err))
	}
	if err := viper.Unmarshal(&cfg); err != nil {
		abort(fmt.Errorf("Error parsing configuration: %s", err))
	}

	var httpClient *http.Client
	if token := viper.GetString("token"); token != "" {
		ts := oauth2.StaticTokenSource(
			&oauth2.Token{AccessToken: token},
		)
		httpClient = oauth2.NewClient(oauth2.NoContext, ts)
	}
	c := github.NewClient(httpClient)

	repoReader := bufio.NewReader(os.Stdin)
	repoData, err := ioutil.ReadAll(repoReader)
	if err != nil {
		return fmt.Errorf("Error reading repository list: %s", err)
	}
	repos := strings.Split(strings.Trim(string(repoData), " \n"), "\n")

	if err := syncLabels(cfg.Labels, repos, c); err != nil {
		return fmt.Errorf("Error syncing repositories: %s", err)
	}

	return nil
}

func syncLabels(labels map[string]string, repos []string, c *github.Client) error {
	for _, repo := range repos {
		log.Printf("Processing repository %q", repo)
		repoParts := strings.Split(repo, "/")
		if len(repoParts) < 2 {
			return fmt.Errorf("repository %q is missing the owner. Required format: <owner>/<repository>", repo)
		}

		repoLabels, err := fetchLabels(repoParts[0], repoParts[1], c)
		if err != nil {
			return err
		}

		log.Printf("  Found %d labels", len(repoLabels))

		for label, color := range labels {
			if err := ensureLabel(label, color, repoParts[0], repoParts[1], repoLabels, c); err != nil {
				return err
			}
		}
	}
	return nil
}

func fetchLabels(owner, repo string, c *github.Client) (map[string]string, error) {
	repoLabels := map[string]string{}
	page := 0
	for {
		repoLabelsPage, resp, err := c.Issues.ListLabels(owner, repo, &github.ListOptions{
			Page: page,
		})
		if err != nil {
			return nil, fmt.Errorf("could not list existing labels of repo %q: %s", repo, err)
		}

		for _, repoLabel := range repoLabelsPage {
			repoLabels[*repoLabel.Name] = *repoLabel.Color
		}

		if resp.NextPage == 0 {
			break
		}
		page = resp.NextPage
	}
	return repoLabels, nil
}

func ensureLabel(label, color, owner, repo string, existingLabels map[string]string, c *github.Client) error {
	if color[0] == '#' {
		color = color[1:]
	}
	if len(color) != 6 {
		return fmt.Errorf("color %q of label %q for repo \"%s/%s\" is in an invalid format. Colors need to be formatted as six hexadecimal digits", color, label, owner, repo)
	}

	repoLabelColor, ok := existingLabels[label]
	if !ok {
		log.Printf("  Creating label %q: %q", label, color)
		_, _, err := c.Issues.CreateLabel(owner, repo, &github.Label{
			Name:  github.String(label),
			Color: github.String(color),
		})
		if err != nil {
			return fmt.Errorf("could not create label %q in repo \"%s/%s\": %s", label, owner, repo, err)
		}
	} else if repoLabelColor != color {
		log.Printf("  Updating label %q: %q", label, color)
		_, _, err := c.Issues.EditLabel(owner, repo, label, &github.Label{
			Name:  github.String(label),
			Color: github.String(color),
		})
		if err != nil {
			return fmt.Errorf("could not update label %q in repo \"%s/%s\": %s", label, owner, repo, err)
		}
	} else {
		log.Printf("  Label %q: %q already exists", label, color)
	}
	return nil
}

func must(err error) {
	if err != nil {
		abort(err)
	}
}

{'fraction_non_alphanumeric': 0.1555310817425355, 'fraction_numerical': 0.005139500734214391, 'mean_word_length': 3.705238917674151, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7071642', 'n_tokens_mistral': 3169, 'n_tokens_neox': 2886, 'n_words': 748}