<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc"
	"github.com/concourse/worker"
	"github.com/concourse/worker/beacon"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
)

// overridden via linker flags
var Version = "0.0.0-dev"

type WorkerCommand struct {
	JSON beacon.FileFlag `long:"json"`

	GardenAddr      string   `long:"garden-addr"`
	BaggageclaimURL string   `long:"baggageclaim-url"`
	Platform        string   `long:"platform"`
	Tags            []string `long:"tag"`
	Team            string   `long:"team"`
	Name            string   `long:"name"`
	Version         string   `long:"version"`
	CertsPath       *string  `long:"certs_path"`
	HTTPProxyURL    string   `long:"http_proxy_url"`
	HTTPSProxyURL   string   `long:"https_proxy_url"`
	NoProxy         string   `long:"no_proxy"`

	BeaconConfig beacon.Config `group:"Beacon Configuration" namespace:"beacon"`
}

func main() {
	var cmd WorkerCommand

	parser := flags.NewParser(&cmd, flags.HelpFlag|flags.PassDoubleDash)
	parser.NamespaceDelimiter = "-"

	_, err := parser.Parse()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	var atcWorker atc.Worker
	if cmd.JSON != "" {
		workerJSON, err := ioutil.ReadFile(string(cmd.JSON))
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}

		err = json.Unmarshal(workerJSON, &atcWorker)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}

	} else {
		atcWorker = atc.Worker{
			GardenAddr:      cmd.GardenAddr,
			BaggageclaimURL: cmd.BaggageclaimURL,
			Platform:        cmd.Platform,
			Tags:            cmd.Tags,
			Team:            cmd.Team,
			Name:            cmd.Name,
			Version:         cmd.Version,
			CertsPath:       cmd.CertsPath,
			HTTPProxyURL:    cmd.HTTPProxyURL,
			HTTPSProxyURL:   cmd.HTTPSProxyURL,
			NoProxy:         cmd.NoProxy,
		}
	}

	logger := lager.NewLogger("worker")
	//TODO support changing the log level
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))
	runner := worker.BeaconRunner(logger.Session("beacon"), atcWorker, cmd.BeaconConfig)

	select {
	case err := <-ifrit.Invoke(runner).Wait():
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}

}<commit_msg>Allow log level configuration via flags<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/concourse/atc"
	"github.com/concourse/flag"
	"github.com/concourse/worker"
	"github.com/concourse/worker/beacon"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
)

// overridden via linker flags
var Version = "0.0.0-dev"

type WorkerCommand struct {
	JSON beacon.FileFlag `long:"json"`

	GardenAddr      string   `long:"garden-addr"`
	BaggageclaimURL string   `long:"baggageclaim-url"`
	Platform        string   `long:"platform"`
	Tags            []string `long:"tag"`
	Team            string   `long:"team"`
	Name            string   `long:"name"`
	Version         string   `long:"version"`
	CertsPath       *string  `long:"certs_path"`
	HTTPProxyURL    string   `long:"http_proxy_url"`
	HTTPSProxyURL   string   `long:"https_proxy_url"`
	NoProxy         string   `long:"no_proxy"`
	Logger          flag.Lager
	BeaconConfig    beacon.Config `group:"Beacon Configuration" namespace:"beacon"`
}

func main() {
	var cmd WorkerCommand

	parser := flags.NewParser(&cmd, flags.HelpFlag|flags.PassDoubleDash)
	parser.NamespaceDelimiter = "-"

	_, err := parser.Parse()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	var atcWorker atc.Worker
	if cmd.JSON != "" {
		workerJSON, err := ioutil.ReadFile(string(cmd.JSON))
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}

		err = json.Unmarshal(workerJSON, &atcWorker)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}

	} else {
		atcWorker = atc.Worker{
			GardenAddr:      cmd.GardenAddr,
			BaggageclaimURL: cmd.BaggageclaimURL,
			Platform:        cmd.Platform,
			Tags:            cmd.Tags,
			Team:            cmd.Team,
			Name:            cmd.Name,
			Version:         cmd.Version,
			CertsPath:       cmd.CertsPath,
			HTTPProxyURL:    cmd.HTTPProxyURL,
			HTTPSProxyURL:   cmd.HTTPSProxyURL,
			NoProxy:         cmd.NoProxy,
		}
	}

	logger, _ := cmd.Logger.Logger("beacon")
	runner := worker.BeaconRunner(logger, atcWorker, cmd.BeaconConfig)

	select {
	case err := <-ifrit.Invoke(runner).Wait():
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}

}
{'fraction_non_alphanumeric': 0.13488372093023257, 'fraction_numerical': 0.0031007751937984496, 'mean_word_length': 2.9753521126760565, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2178992', 'n_tokens_mistral': 1760, 'n_tokens_neox': 1632, 'n_words': 307}