<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
)

const version = "0.2.0rc3"

// exit code
const (
	ValidExit = iota
	ErrInitialize
	ErrRun
)

type option struct {
	version bool
	root    string
	word    string
	verbose bool
	abort   bool
	out     string
	force   bool
	total   bool

	// specify target file types
	types string

	// ignores
	ignoreDirs  string
	ignoreBases string
	ignoreTypes string

	trim bool
	add  uint64

	maxRune int

	nworker uint64
	sync    bool
	cache   bool
}

var opt = &option{}

// TODO: consider default ignores
// Default Ignores
var (
	IgnoreDirs = []string{
		".git",
		".cache",
	}
	IgnoreBases = []string{}
	IgnoreTypes = []string{
		".pgp", ".ttf", ".pdf",
		".jpg", ".jpeg", ".png", ".ico", ".gif",
		".mp4",
		".mp3", ".ogg", ".wav", ".au",
		".so", ".mo", ".a", ".o", ".pyc", ".exe", ".efi",
		".gz", ".xz", ".tar", ".bz2", ".db", ".tgz", ".zip",
	}
)

func init() {
	flag.BoolVar(&opt.version, "version", false, "print version")
	flag.StringVar(&opt.root, "root", "", "specify search root directory")
	flag.StringVar(&opt.word, "word", "TODO: ", "specify search word")
	flag.StringVar(&opt.out, "out", "", "specify output file")
	flag.BoolVar(&opt.force, "force", false, "accept overwrite for \"-out\"")
	flag.BoolVar(&opt.total, "total", false, "prints total number of contents")

	sep := string(filepath.ListSeparator)
	flag.StringVar(&opt.types, "types", "", "specify filetypes. separator is '"+sep+"'")
	flag.StringVar(&opt.ignoreDirs, "ignore-dirs", strings.Join(IgnoreDirs, sep), "specify ignore directories. separator is '"+sep+"'")
	flag.StringVar(&opt.ignoreBases, "ignore-Bases", strings.Join(IgnoreBases, sep), "specify ignore Bases. separator is '"+sep+"'")
	flag.StringVar(&opt.ignoreTypes, "ignore-types", strings.Join(IgnoreTypes, sep), "specify ignore file types. separator is '"+sep+"'")

	flag.BoolVar(&opt.trim, "trim", false, "trim the word on output")
	flag.Uint64Var(&opt.add, "add", 0, "specify number of lines of after find the word")

	flag.IntVar(&opt.maxRune, "max", 256, "specify characters limit")
	flag.BoolVar(&opt.verbose, "verbose", false, "output of log messages")
	flag.BoolVar(&opt.abort, "abort", false, "if exists errors then abort process")

	flag.Uint64Var(&opt.nworker, "nworker", 0, "specify limitation of goriutine")
	flag.BoolVar(&opt.sync, "sync", false, "for debug: run on sync")
	flag.BoolVar(&opt.cache, "cache", false, "use data cache")
}

func run(w, errw io.Writer, opt *option) (exitCode int) {
	// version
	if opt.version {
		fmt.Fprintln(w, "todogotcha version "+version)
		return
	}

	// abs for root
	fullpath, err := filepath.Abs(opt.root)
	if err != nil {
		fmt.Fprintln(errw, err)
		exitCode = ErrInitialize
		return
	}
	opt.root = fullpath

	// out to file<commit_msg>fix flag -ignore-Bases to -ignore-bases  Changes to be committed: 	modified:   todogotcha/main.go<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
)

const version = "0.2.0rc3"

// exit code
const (
	ValidExit = iota
	ErrInitialize
	ErrRun
)

type option struct {
	version bool
	root    string
	word    string
	verbose bool
	abort   bool
	out     string
	force   bool
	total   bool

	// specify target file types
	types string

	// ignores
	ignoreDirs  string
	ignoreBases string
	ignoreTypes string

	trim bool
	add  uint64

	maxRune int

	nworker uint64
	sync    bool
	cache   bool
}

var opt = &option{}

// TODO: consider default ignores
// Default Ignores
var (
	IgnoreDirs = []string{
		".git",
		".cache",
	}
	IgnoreBases = []string{}
	IgnoreTypes = []string{
		".pgp", ".ttf", ".pdf",
		".jpg", ".jpeg", ".png", ".ico", ".gif",
		".mp4",
		".mp3", ".ogg", ".wav", ".au",
		".so", ".mo", ".a", ".o", ".pyc", ".exe", ".efi",
		".gz", ".xz", ".tar", ".bz2", ".db", ".tgz", ".zip",
	}
)

func init() {
	flag.BoolVar(&opt.version, "version", false, "print version")
	flag.StringVar(&opt.root, "root", "", "specify search root directory")
	flag.StringVar(&opt.word, "word", "TODO: ", "specify search word")
	flag.StringVar(&opt.out, "out", "", "specify output file")
	flag.BoolVar(&opt.force, "force", false, "accept overwrite for \"-out\"")
	flag.BoolVar(&opt.total, "total", false, "prints total number of contents")

	sep := string(filepath.ListSeparator)
	flag.StringVar(&opt.types, "types", "", "specify filetypes. separator is '"+sep+"'")
	flag.StringVar(&opt.ignoreDirs, "ignore-dirs", strings.Join(IgnoreDirs, sep), "specify ignore directories. separator is '"+sep+"'")
	flag.StringVar(&opt.ignoreBases, "ignore-bases", strings.Join(IgnoreBases, sep), "specify ignore basenames. separator is '"+sep+"'")
	flag.StringVar(&opt.ignoreTypes, "ignore-types", strings.Join(IgnoreTypes, sep), "specify ignore file types. separator is '"+sep+"'")

	flag.BoolVar(&opt.trim, "trim", false, "trim the word on output")
	flag.Uint64Var(&opt.add, "add", 0, "specify number of lines of after find the word")

	flag.IntVar(&opt.maxRune, "max", 256, "specify characters limit")
	flag.BoolVar(&opt.verbose, "verbose", false, "output of log messages")
	flag.BoolVar(&opt.abort, "abort", false, "if exists errors then abort process")

	flag.Uint64Var(&opt.nworker, "nworker", 0, "specify limitation of goriutine")
	flag.BoolVar(&opt.sync, "sync", false, "for debug: run on sync")
	flag.BoolVar(&opt.cache, "cache", false, "use data cache")
}

func run(w, errw io.Writer, opt *option) (exitCode int) {
	// version
	if opt.version {
		fmt.Fprintln(w, "todogotcha version "+version)
		return
	}

	// abs for root
	fullpath, err := filepath.Abs(opt.root)
	if err != nil {
		fmt.Fprintln(errw, err)
		exitCode = ErrInitialize
		return
	}
	opt.root = fullpath

	// out to file
{'fraction_non_alphanumeric': 0.1771886989884897, 'fraction_numerical': 0.006975933031042902, 'mean_word_length': 4.017497812773403, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '466032', 'n_tokens_mistral': 2147, 'n_tokens_neox': 2063, 'n_words': 581}