<commit_before>package tailer

import (
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/hpcloud/tail"

	"github.com/LeKovr/go-base/log"
	"github.com/LeKovr/webtail/worker"
)

// Config defines local application flags
type Config struct {
	Root        string `long:"root"  default:"log/"  description:"Root directory for log files"`
	Bytes       int64  `long:"bytes" default:"5000"  description:"tail from the last Nth location"`
	Lines       int    `long:"lines" default:"100"   description:"keep N old lines for new consumers"`
	MaxLineSize int    `long:"split" default:"180"   description:"min line size for split"`
	Poll        bool   `long:"poll"  description:"use polling, instead of inotify"`
	ListCache   int    `long:"cache"       default:"2"       description:"Time to cache file listing (sec)"`
}

type tailer struct {
	Buffer [][]byte

	// Unregister requests from clients.
	Unregister chan bool

	Incomplete bool
}

type WorkerHub struct {
	Log     log.Logger
	Config  Config
	workers map[string]*tailer
	index   worker.IndexStore
}

// New creates tailer
func New(logger log.Logger, cfg Config) (*WorkerHub, error) {
	return &WorkerHub{
		Config: cfg,
		Log:    logger,

		workers: make(map[string]*tailer),
		index:   make(worker.IndexStore),
	}, nil
}

func (wh *WorkerHub) WorkerExists(channel string) bool {
	_, ok := wh.workers[channel]
	return ok
}

func (wh *WorkerHub) WorkerRun(channel string, out chan *worker.Message) error {

	config := tail.Config{
		Follow: true,
		ReOpen: true,
	}
	cfg := wh.Config
	filename := path.Join(cfg.Root, channel)
	config.MaxLineSize = cfg.MaxLineSize
	config.Poll = cfg.Poll
	lineIncomlete := false

	if cfg.Bytes != 0 {
		fi, err := os.Stat(filename)
		if err != nil {
			return err<commit_msg>fix: check if root exists<commit_after>package tailer

import (
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/hpcloud/tail"

	"github.com/LeKovr/go-base/log"
	"github.com/LeKovr/webtail/worker"
)

// Config defines local application flags
type Config struct {
	Root        string `long:"root"  default:"log/"  description:"Root directory for log files"`
	Bytes       int64  `long:"bytes" default:"5000"  description:"tail from the last Nth location"`
	Lines       int    `long:"lines" default:"100"   description:"keep N old lines for new consumers"`
	MaxLineSize int    `long:"split" default:"180"   description:"min line size for split"`
	Poll        bool   `long:"poll"  description:"use polling, instead of inotify"`
	ListCache   int    `long:"cache"       default:"2"       description:"Time to cache file listing (sec)"`
}

type tailer struct {
	Buffer [][]byte

	// Unregister requests from clients.
	Unregister chan bool

	Incomplete bool
}

type WorkerHub struct {
	Log     log.Logger
	Config  Config
	workers map[string]*tailer
	index   worker.IndexStore
}

// New creates tailer
func New(logger log.Logger, cfg Config) (*WorkerHub, error) {
	_, err := os.Stat(cfg.Root)
	if err != nil {
		return nil, err
	}
	return &WorkerHub{
		Config: cfg,
		Log:    logger,

		workers: make(map[string]*tailer),
		index:   make(worker.IndexStore),
	}, nil
}

func (wh *WorkerHub) WorkerExists(channel string) bool {
	_, ok := wh.workers[channel]
	return ok
}

func (wh *WorkerHub) WorkerRun(channel string, out chan *worker.Message) error {

	config := tail.Config{
		Follow: true,
		ReOpen: true,
	}
	cfg := wh.Config
	filename := path.Join(cfg.Root, channel)
	config.MaxLineSize = cfg.MaxLineSize
	config.Poll = cfg.Poll
	lineIncomlete := false

	if cfg.Bytes != 0 {
		fi, err := os.Stat(filename)
		if err != nil {
			return err
{'fraction_non_alphanumeric': 0.12254495159059474, 'fraction_numerical': 0.007745504840940526, 'mean_word_length': 3.6658064516129034, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5064626', 'n_tokens_mistral': 1260, 'n_tokens_neox': 1200, 'n_words': 349}