<commit_before>package main

import (
	"flag"
	"io/ioutil"
	"os"
	"path"

	log "github.com/Sirupsen/logrus"
)

var logLevel int

var showsApp []byte
var showApp []byte
var seasonApp []byte
var episodeApp []byte

func init() {
	const (
		logLevelUsage = "Set log level (0,1,2,3,4,5, higher is more logging)."
	)

	flag.IntVar(&logLevel, "log-level", int(log.ErrorLevel), logLevelUsage)
}

type commonEpisode struct {
	Number  int    `json:"number"`
	Name    string `json:"name"`
	Summary string `json:"summary"`
	Image   struct {
		Medium   string `json:"medium"`
		Original string `json:"original"`
	} `json:"image"`
}

func findMatchingShow(file os.FileInfo) *TvMazeShow {
	contextLogger := log.WithField("file", file.Name())
	tvMaze := TvMazeClient{
		URLTemplate: tvMazeURLTemplate,
		logger:      contextLogger,
	}

	tvMazeShow, err := tvMaze.Find(file.Name())
	if err != nil || tvMazeShow == nil {
		contextLogger.Debug("No match")
		return nil
	}
	contextLogger.WithField("show", tvMazeShow.Name).Debug("Found match")

	return tvMazeShow
}

func unique(list []int) []int {
	unique := []int{}
	for _, item := range list {
		found := false
		for _, uniqueItem := range unique {
			if item == uniqueItem {
				found = true
			}
		}
		if !found {
			unique = append(unique, item)
		}
	}

	return unique
}

func seasons(show *TvMazeShow) []int {
	seasons := []int{}
	for _, episode := range show.Embedded.Episodes {
		seasons = append(seasons, int(episode.Season))
	}

	return unique(seasons)
}

func loadShowsApp() error {
	var err error
	showsApp, err = loadApp("apps/shows.html")()
	return err
}

func loadShowApp() error {
	var err error
	showApp, err = loadApp("apps/show.html")()
	return err
}

func loadSeasonApp() error {
	var err error
	seasonApp, err = loadApp("apps/season.html")()
	return err
}

func loadEpisodeApp() error {
	var err error
	episodeApp, err = loadApp("apps/episode.html")()
	return err
}

func loadApp(fileName string) func() ([]byte, error) {
	return func() ([]byte, error) {
		data, err := ioutil.ReadFile(fileName)
		if err != nil {
			log.WithField("err", err).Errorf("Error opening %s", fileName)
			return nil, err
		}
		return data, nil
	}
}

func main() {
	flag.Parse()
	log.SetLevel(log.Level(logLevel))

	if err := loadShowsApp(); err != nil {
		return
	}

	if err := loadShowApp(); err != nil {
		return
	}

	if err := loadSeasonApp(); err != nil {
		return
	}

	if err := loadEpisodeApp(); err != nil {
		return
	}

	dir, err := os.Getwd()
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
		}).Fatal("Error getting working directory")
	}

	if err := os.Chdir(path.Join(dir, flag.Args()[0])); err != nil {
		log.WithFields(log.Fields{
			"err":  err,
			"root": path.Join(dir, flag.Args()[0]),
		}).Fatal("Error changing working dir")
	}

	files, err := ioutil.ReadDir(".")
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
		}).Fatal("Error initializing Fetcher")
	}

	shows := []ShowInList{}
	for _, file := range files {
		if !file.IsDir() {<commit_msg>cli: tell user it needs to supply 1 arg.<commit_after>package main

import (
	"flag"
	"io/ioutil"
	"os"
	"path"

	log "github.com/Sirupsen/logrus"
)

var logLevel int

var showsApp []byte
var showApp []byte
var seasonApp []byte
var episodeApp []byte

func init() {
	const (
		logLevelUsage = "Set log level (0,1,2,3,4,5, higher is more logging)."
	)

	flag.IntVar(&logLevel, "log-level", int(log.ErrorLevel), logLevelUsage)
}

type commonEpisode struct {
	Number  int    `json:"number"`
	Name    string `json:"name"`
	Summary string `json:"summary"`
	Image   struct {
		Medium   string `json:"medium"`
		Original string `json:"original"`
	} `json:"image"`
}

func findMatchingShow(file os.FileInfo) *TvMazeShow {
	contextLogger := log.WithField("file", file.Name())
	tvMaze := TvMazeClient{
		URLTemplate: tvMazeURLTemplate,
		logger:      contextLogger,
	}

	tvMazeShow, err := tvMaze.Find(file.Name())
	if err != nil || tvMazeShow == nil {
		contextLogger.Debug("No match")
		return nil
	}
	contextLogger.WithField("show", tvMazeShow.Name).Debug("Found match")

	return tvMazeShow
}

func unique(list []int) []int {
	unique := []int{}
	for _, item := range list {
		found := false
		for _, uniqueItem := range unique {
			if item == uniqueItem {
				found = true
			}
		}
		if !found {
			unique = append(unique, item)
		}
	}

	return unique
}

func seasons(show *TvMazeShow) []int {
	seasons := []int{}
	for _, episode := range show.Embedded.Episodes {
		seasons = append(seasons, int(episode.Season))
	}

	return unique(seasons)
}

func loadShowsApp() error {
	var err error
	showsApp, err = loadApp("apps/shows.html")()
	return err
}

func loadShowApp() error {
	var err error
	showApp, err = loadApp("apps/show.html")()
	return err
}

func loadSeasonApp() error {
	var err error
	seasonApp, err = loadApp("apps/season.html")()
	return err
}

func loadEpisodeApp() error {
	var err error
	episodeApp, err = loadApp("apps/episode.html")()
	return err
}

func loadApp(fileName string) func() ([]byte, error) {
	return func() ([]byte, error) {
		data, err := ioutil.ReadFile(fileName)
		if err != nil {
			log.WithField("err", err).Errorf("Error opening %s", fileName)
			return nil, err
		}
		return data, nil
	}
}

func main() {
	flag.Parse()
	log.SetLevel(log.Level(logLevel))

	if err := loadShowsApp(); err != nil {
		return
	}

	if err := loadShowApp(); err != nil {
		return
	}

	if err := loadSeasonApp(); err != nil {
		return
	}

	if err := loadEpisodeApp(); err != nil {
		return
	}

	dir, err := os.Getwd()
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
		}).Fatal("Error getting working directory")
	}

	if len(flag.Args()) != 1 {
		log.Fatal("Require one argument pointing to media path")
	}

	if err := os.Chdir(path.Join(dir, flag.Args()[0])); err != nil {
		log.WithFields(log.Fields{
			"err":  err,
			"root": path.Join(dir, flag.Args()[0]),
		}).Fatal("Error changing working dir")
	}

	files, err := ioutil.ReadDir(".")
	if err != nil {
		log.WithFields(log.Fields{
			"err": err,
		}).Fatal("Error initializing Fetcher")
	}

	shows := []ShowInList{}
	for _, file := range files {
		if !file.IsDir() {
{'fraction_non_alphanumeric': 0.15914257875933746, 'fraction_numerical': 0.002923026956804157, 'mean_word_length': 3.76702786377709, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4366573', 'n_tokens_mistral': 2388, 'n_tokens_neox': 2238, 'n_words': 582}