<commit_before>package main

import (
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
	"os"
	"time"
)

var log = logrus.New()

type Rotatee struct {
	setting RotateeSetting
}

func newRotatee(setting RotateeSetting) *Rotatee {
	return &Rotatee{setting: setting}
}

type RotateeSetting struct {
	args    []string
	verbose bool
}

func setupEventPipe(arg string) EventPipe {
	pipe := NewEventPipe()
	pipe.Add(NewTimer(DetectSeries(arg, time.Now())))
	pipe.Add(NewFormatEval(arg))
	pipe.Add(NewRoller())
	pipe.Add(NewWriter())
	return pipe
}

func (r *Rotatee) start() {
	log.WithFields(logrus.Fields{"Rotatee": r}).Debug("Start rotatee")
	// init pipe
	pipeGroup := NewEventPipeGroup()
	for _, arg := range r.setting.args {
		pipeGroup.Add(setupEventPipe(arg))
	}
	pipeGroup.Start()
	// init first destination
	pipeGroup.Broadcast(NewWriteTarget())
	// reader loop
	reader := os.Stdin
	readBuf := make([]byte, 1024)
	for {
		len, err := reader.Read(readBuf)
		if err != nil {
			log.Panic("Writer goroutine IO failed")
		}
		// do copy because 'content' is shared among goroutine(s)
		content := make([]byte, len)
		copy(content, readBuf[:len])
		pipeGroup.Broadcast(NewPayload(content))
		// Also, write to stdout
		os.Stdout.Write(content)
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "rotatee"
	app.Usage = "advanced tee, advanced input rotation"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "verbose logging to stderr",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "enable debug mode (very verbose logging to stderr)",
		},
	}
	app.Action = func(c *cli.Context) error {
		verbose := c.Bool("verbose")
		// setup logger
		log.Out = os.Stderr
		if verbose {
			log.Level = logrus.InfoLevel
		} else {
			log.Level = logrus.WarnLevel
		}
		if c.Bool("debug") {
			log.Level = logrus.DebugLevel
		}
		log.WithFields(logrus.Fields{"Args": c.Args()}).Debug("Parsed input arguments")
		// start rotatee
		rotatee := newRotatee(RotateeSetting{
			args:    c.Args(),
			verbose: verbose,
		})
		rotatee.start()
		return nil
	}
	app.Run(os.Args)
}<commit_msg>make setupLogger to func<commit_after>package main

import (
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
	"os"
	"time"
)

var log = logrus.New()

type Rotatee struct {
	setting RotateeSetting
}

func newRotatee(setting RotateeSetting) *Rotatee {
	return &Rotatee{setting: setting}
}

type RotateeSetting struct {
	args    []string
	verbose bool
}

func setupEventPipe(arg string) EventPipe {
	pipe := NewEventPipe()
	pipe.Add(NewTimer(DetectSeries(arg, time.Now())))
	pipe.Add(NewFormatEval(arg))
	pipe.Add(NewRoller())
	pipe.Add(NewWriter())
	return pipe
}

func (r *Rotatee) start() {
	log.WithFields(logrus.Fields{"Rotatee": r}).Debug("Start rotatee")
	// init pipe
	pipeGroup := NewEventPipeGroup()
	for _, arg := range r.setting.args {
		pipeGroup.Add(setupEventPipe(arg))
	}
	pipeGroup.Start()
	// init first destination
	pipeGroup.Broadcast(NewWriteTarget())
	// reader loop
	reader := os.Stdin
	readBuf := make([]byte, 1024)
	for {
		len, err := reader.Read(readBuf)
		if err != nil {
			log.Panic("Writer goroutine IO failed")
		}
		// do copy because 'content' is shared among goroutine(s)
		content := make([]byte, len)
		copy(content, readBuf[:len])
		pipeGroup.Broadcast(NewPayload(content))
		// Also, write to stdout
		os.Stdout.Write(content)
	}
}

func setupLogger(verboe bool, debug bool) {
	log.Out = os.Stderr
	if verbose {
		log.Level = logrus.InfoLevel
	} else {
		log.Level = logrus.WarnLevel
	}
	if debug {
		log.Level = logrus.DebugLevel
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "rotatee"
	app.Usage = "advanced tee, advanced input rotation"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "verbose logging to stderr",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "enable debug mode (very verbose logging to stderr)",
		},
	}
	app.Action = func(c *cli.Context) error {
		verbose, debug := c.Bool("verbose"), c.Bool("debug")
		setupLogger(verbose, debug)
		log.WithFields(logrus.Fields{"Args": c.Args()}).Debug("Parsed input arguments")
		rotatee := newRotatee(RotateeSetting{
			args:    c.Args(),
			verbose: verbose,
		})
		rotatee.start()
		return nil
	}
	app.Run(os.Args)
}
{'fraction_non_alphanumeric': 0.14769520225776106, 'fraction_numerical': 0.0018814675446848542, 'mean_word_length': 3.9281575898030128, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1109868', 'n_tokens_mistral': 1678, 'n_tokens_neox': 1569, 'n_words': 350}