<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/apex/log"
	"github.com/apex/log/handlers/cli"
	"github.com/apex/log/handlers/multi"
	"github.com/segmentio/ecs-logs-go"
	"github.com/segmentio/ecs-logs/lib"

	_ "github.com/segmentio/ecs-logs/lib/cloudwatchlogs"
	_ "github.com/segmentio/ecs-logs/lib/datadog"
	_ "github.com/segmentio/ecs-logs/lib/loggly"
	_ "github.com/segmentio/ecs-logs/lib/statsd"
	_ "github.com/segmentio/ecs-logs/lib/syslog"
)

type source struct {
	lib.Source
	name string
}

type destination struct {
	lib.Destination
	name string
}

type reader struct {
	lib.Reader
	name string
}

func main() {
	var err error
	var src string
	var dst string
	var hostname string
	var level = lib.LogLevel(log.InfoLevel)
	var maxBytes int
	var maxCount int
	var flushTimeout time.Duration
	var cacheTimeout time.Duration

	hostname, _ = os.Hostname()
	log.SetLevel(log.DebugLevel)
	log.SetHandler(cli.New(os.Stdout))

	flag.StringVar(&src, "src", "stdin", "A comma separated list of log sources from which messages will be read ["+strings.Join(lib.SourcesAvailable(), ", ")+"]")
	flag.StringVar(&dst, "dst", "stdout", "A comma separated list of log destinations to which messages will be written ["+strings.Join(lib.DestinationsAvailable(), ", ")+"]")
	flag.StringVar(&hostname, "hostname", hostname, "The hostname advertised by ecs-logs")
	flag.Var(&level, "log-level", "The minimum level of log messages shown by ecs-logs")
	flag.IntVar(&maxBytes, "max-batch-bytes", 1000000, "The maximum size in bytes of a message batch")
	flag.IntVar(&maxCount, "max-batch-size", 10000, "The maximum number of messages in a batch")
	flag.DurationVar(&flushTimeout, "flush-timeout", 5*time.Second, "How often messages will be flushed")
	flag.DurationVar(&cacheTimeout, "cache-timeout", 5*time.Minute, "How to wait before clearing unused internal cache")
	flag.Parse()

	var store = lib.NewStore()
	var sources []source
	var readers []reader
	var dests []destination

	if sources = getSources(strings.Split(src, ",")); len(sources) == 0 {
		log.Fatal("no or invalid log sources")
	}

	if dests = getDestinations(strings.Split(dst, ",")); len(dests) == 0 {
		log.Fatal("no or invalid log destinations")
	}

	if readers, err = openSources(sources); err != nil {
		log.WithError(err).Fatal("failed to open log sources readers")
	}

	join := &sync.WaitGroup{}

	limits := lib.StreamLimits{
		MaxCount: maxCount,
		MaxBytes: maxBytes,<commit_msg>check for the hostname<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/apex/log"
	"github.com/apex/log/handlers/cli"
	"github.com/apex/log/handlers/multi"
	"github.com/segmentio/ecs-logs-go"
	"github.com/segmentio/ecs-logs/lib"

	_ "github.com/segmentio/ecs-logs/lib/cloudwatchlogs"
	_ "github.com/segmentio/ecs-logs/lib/datadog"
	_ "github.com/segmentio/ecs-logs/lib/loggly"
	_ "github.com/segmentio/ecs-logs/lib/statsd"
	_ "github.com/segmentio/ecs-logs/lib/syslog"
)

type source struct {
	lib.Source
	name string
}

type destination struct {
	lib.Destination
	name string
}

type reader struct {
	lib.Reader
	name string
}

func main() {
	var err error
	var src string
	var dst string
	var hostname string
	var level = lib.LogLevel(log.InfoLevel)
	var maxBytes int
	var maxCount int
	var flushTimeout time.Duration
	var cacheTimeout time.Duration

	hostname, _ = os.Hostname()
	log.SetLevel(log.DebugLevel)
	log.SetHandler(cli.New(os.Stdout))

	flag.StringVar(&src, "src", "stdin", "A comma separated list of log sources from which messages will be read ["+strings.Join(lib.SourcesAvailable(), ", ")+"]")
	flag.StringVar(&dst, "dst", "stdout", "A comma separated list of log destinations to which messages will be written ["+strings.Join(lib.DestinationsAvailable(), ", ")+"]")
	flag.StringVar(&hostname, "hostname", hostname, "The hostname advertised by ecs-logs")
	flag.Var(&level, "log-level", "The minimum level of log messages shown by ecs-logs")
	flag.IntVar(&maxBytes, "max-batch-bytes", 1000000, "The maximum size in bytes of a message batch")
	flag.IntVar(&maxCount, "max-batch-size", 10000, "The maximum number of messages in a batch")
	flag.DurationVar(&flushTimeout, "flush-timeout", 5*time.Second, "How often messages will be flushed")
	flag.DurationVar(&cacheTimeout, "cache-timeout", 5*time.Minute, "How to wait before clearing unused internal cache")
	flag.Parse()

	var store = lib.NewStore()
	var sources []source
	var readers []reader
	var dests []destination

	if len(hostname) == 0 {
		log.Fatal("no hostname configured")
	}

	if sources = getSources(strings.Split(src, ",")); len(sources) == 0 {
		log.Fatal("no or invalid log sources")
	}

	if dests = getDestinations(strings.Split(dst, ",")); len(dests) == 0 {
		log.Fatal("no or invalid log destinations")
	}

	if readers, err = openSources(sources); err != nil {
		log.WithError(err).Fatal("failed to open log sources readers")
	}

	join := &sync.WaitGroup{}

	limits := lib.StreamLimits{
		MaxCount: maxCount,
		MaxBytes: maxBytes,
{'fraction_non_alphanumeric': 0.14159811985898943, 'fraction_numerical': 0.006462984723854289, 'mean_word_length': 4.655592469545958, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2378173', 'n_tokens_mistral': 1837, 'n_tokens_neox': 1761, 'n_words': 452}