<commit_before>package main

import (
	"errors"
	"flag"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
)

type options struct {
	workersCount int
	bucketName, source,
	cacheFile string
	dryRun, verbose, quiet, doCache, doUpload, gzipHTML bool
}

var opts *options

var appEnv string

var say func(...string)

// Order matters: first hit, first served.
// TODO: Make this configurable somehow, so that end users can provide their own mappings.
var r = regexp.MustCompile
var customHeadersDef = []pathToHeaders{
	pathToHeaders{r("index\\.html"),
		headersDef{ContentEncoding: "gzip", CacheControl: "max-age=1800"}},
	pathToHeaders{r("articole.*\\.html$"),
		headersDef{ContentEncoding: "gzip", CacheControl: "max-age=86400"}},
	pathToHeaders{r("[^/]*\\.html$"),
		headersDef{ContentEncoding: "gzip", CacheControl: "max-age=3600"}},
	pathToHeaders{r("\\.xml$"),
		headersDef{ContentEncoding: "gzip", CacheControl: "max-age=1800"}},
	pathToHeaders{r("\\.ico$"),
		headersDef{ContentEncoding: "gzip", CacheControl: "max-age=31536000"}},
	pathToHeaders{r("\\.(js|css)$"),
		headersDef{ContentEncoding: "gzip", CacheControl: "max-age=31536000"}},
	pathToHeaders{r("images/articole/.*(jpg|JPG|png|PNG)$"),
		headersDef{CacheControl: "max-age=31536000"}},
	pathToHeaders{r("\\.(jpg|JPG|png|PNG)$"),
		headersDef{CacheControl: "max-age=31536000"}},
}

// processCmdLineFlags wraps the command line flags handling.
func processCmdLineFlags(opts *options) {
	flag.IntVar(&opts.workersCount, "workers", 42, "No. of workers/threads to use for S3 uploads")
	flag.StringVar(&opts.bucketName, "bucket", "", "S3 bucket to upload files to")
	flag.StringVar(&opts.source, "source", "output", "Source folder for files to be uploaded to S3")
	flag.StringVar(&opts.cacheFile, "cachefile", filepath.Join(".go3up.txt"), "Location of the cache file")
	flag.BoolVar(&opts.dryRun, "dry", false, "Dry run (no upload/cache update)")
	flag.BoolVar(&opts.verbose, "verbose", false, "Print the name of the files as they are uploaded")
	flag.BoolVar(&opts.quiet, "quiet", false, "Print only warnings and errors")
	flag.BoolVar(&opts.doUpload, "upload", true, "Do perform an upload")
	flag.BoolVar(&opts.doCache, "cache", true, "Do update the cache")
	flag.BoolVar(&opts.gzipHTML, "gzip", true, "Gzip HTML files")
	flag.Parse()
}

// validateCmdLineFlags validates some of the flags, mostly paths. Defers actual validation to validateCmdLineFlag()
func validateCmdLineFlags(opts *options) (err error) {
	flags := map[string]string{"Bucket Name": opts.bucketName, "Source": opts.source, "Cache file": opts.cacheFile}
	for label, val := range flags {
		if err = validateCmdLineFlag(label, val); err != nil {
			return
		}
	}

	return
}

// validateCmdLineFlag handles the actual validation of flags.
func validateCmdLineFlag(label, val string) (err error) {
	switch label {
	case "Bucket Name":
		if val == "" {
			return errors.New(label + " is not set")
		}
	default:
		_, err = os.Stat(val)
	}

	return
}

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	opts = new(options)
	processCmdLineFlags(opts)
	appEnv = "production"
	say = loggerGen()
}
<commit_msg>Very minor cleanup<commit_after>package main

import (
	"errors"
	"flag"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
)

type options struct {
	workersCount int
	bucketName, source,
	cacheFile string
	dryRun, verbose, quiet, doCache, doUpload, gzipHTML bool
}

var opts *options

var appEnv string

var say func(...string)

// Order matters: first hit, first served.
// TODO: Make this configurable somehow, so that end users can provide their own mappings.
var r = regexp.MustCompile
var customHeadersDef = []pathToHeaders{
	{r("index\\.html"), headersDef{ContentEncoding: "gzip", CacheControl: "max-age=1800"}},
	{r("articole.*\\.html$"), headersDef{ContentEncoding: "gzip", CacheControl: "max-age=86400"}},
	{r("[^/]*\\.html$"), headersDef{ContentEncoding: "gzip", CacheControl: "max-age=3600"}},
	{r("\\.xml$"), headersDef{ContentEncoding: "gzip", CacheControl: "max-age=1800"}},
	{r("\\.ico$"), headersDef{ContentEncoding: "gzip", CacheControl: "max-age=31536000"}},
	{r("\\.(js|css)$"), headersDef{ContentEncoding: "gzip", CacheControl: "max-age=31536000"}},
	{r("images/articole/.*(jpg|JPG|png|PNG)$"), headersDef{CacheControl: "max-age=31536000"}},
	{r("\\.(jpg|JPG|png|PNG)$"), headersDef{CacheControl: "max-age=31536000"}},
}

// processCmdLineFlags wraps the command line flags handling.
func processCmdLineFlags(opts *options) {
	flag.IntVar(&opts.workersCount, "workers", 42, "No. of workers/threads to use for S3 uploads")
	flag.StringVar(&opts.bucketName, "bucket", "", "S3 bucket to upload files to")
	flag.StringVar(&opts.source, "source", "output", "Source folder for files to be uploaded to S3")
	flag.StringVar(&opts.cacheFile, "cachefile", filepath.Join(".go3up.txt"), "Location of the cache file")
	flag.BoolVar(&opts.dryRun, "dry", false, "Dry run (no upload/cache update)")
	flag.BoolVar(&opts.verbose, "verbose", false, "Print the name of the files as they are uploaded")
	flag.BoolVar(&opts.quiet, "quiet", false, "Print only warnings and errors")
	flag.BoolVar(&opts.doUpload, "upload", true, "Do perform an upload")
	flag.BoolVar(&opts.doCache, "cache", true, "Do update the cache")
	flag.BoolVar(&opts.gzipHTML, "gzip", true, "Gzip HTML files")
	flag.Parse()
}

// validateCmdLineFlags validates some of the flags, mostly paths. Defers actual validation to validateCmdLineFlag()
func validateCmdLineFlags(opts *options) (err error) {
	flags := map[string]string{"Bucket Name": opts.bucketName, "Source": opts.source, "Cache file": opts.cacheFile}
	for label, val := range flags {
		if err = validateCmdLineFlag(label, val); err != nil {
			return
		}
	}

	return
}

// validateCmdLineFlag handles the actual validation of flags.
func validateCmdLineFlag(label, val string) (err error) {
	switch label {
	case "Bucket Name":
		if val == "" {
			return errors.New(label + " is not set")
		}
	default:
		_, err = os.Stat(val)
	}

	return
}

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	opts = new(options)
	processCmdLineFlags(opts)
	appEnv = "production"
	say = loggerGen()
}

{'fraction_non_alphanumeric': 0.16104319478402607, 'fraction_numerical': 0.017929910350448247, 'mean_word_length': 5.332301341589267, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3237516', 'n_tokens_mistral': 2187, 'n_tokens_neox': 2064, 'n_words': 550}