<commit_before>package sentence

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/ikawaha/kagome/v2/filter"
)

// subcommand property
const (
	CommandName  = "sentence"
	Description  = `tiny sentence splitter`
	usageMessage = "%s\n"
)

// ErrorWriter writes to stderr
var (
	ErrorWriter = os.Stderr
)

// options
type option struct {
	file    string
	flagSet *flag.FlagSet
}

// ContinueOnError ErrorHandling // Return a descriptive error.
// ExitOnError                   // Call os.Exit(2).
// PanicOnError                  // Call panic with a descriptive error.flag.ContinueOnError
func newOption(w io.Writer, eh flag.ErrorHandling) (o *option) {
	o = &option{
		flagSet: flag.NewFlagSet(CommandName, eh),
	}
	// option settings
	o.flagSet.SetOutput(w)
	o.flagSet.StringVar(&o.file, "file", "", "input file")
	return
}

func (o *option) parse(args []string) error {
	if err := o.flagSet.Parse(args); err != nil {
		return err
	}
	return nil
}

// OptionCheck receives a slice of args and returns an error if it was not successfully parsed
func OptionCheck(args []string) error {
	opt := newOption(ioutil.Discard, flag.ContinueOnError)
	if err := opt.parse(args); err != nil {
		return fmt.Errorf("%v, %w", CommandName, err)
	}
	return nil
}

// command main
func command(opt *option) error {
	fp := os.Stdin
	if opt.file != "" {
		var err error
		fp, err = os.Open(opt.file)
		if err != nil {
			return err
		}
		defer func() {
			_ = fp.Close()
		}()
	}<commit_msg>Fix if-return: redundant if ...; err != nil check, just return error instead. (revive)<commit_after>package sentence

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/ikawaha/kagome/v2/filter"
)

// subcommand property
const (
	CommandName  = "sentence"
	Description  = `tiny sentence splitter`
	usageMessage = "%s\n"
)

// ErrorWriter writes to stderr
var (
	ErrorWriter = os.Stderr
)

// options
type option struct {
	file    string
	flagSet *flag.FlagSet
}

// ContinueOnError ErrorHandling // Return a descriptive error.
// ExitOnError                   // Call os.Exit(2).
// PanicOnError                  // Call panic with a descriptive error.flag.ContinueOnError
func newOption(w io.Writer, eh flag.ErrorHandling) (o *option) {
	o = &option{
		flagSet: flag.NewFlagSet(CommandName, eh),
	}
	// option settings
	o.flagSet.SetOutput(w)
	o.flagSet.StringVar(&o.file, "file", "", "input file")
	return
}

func (o *option) parse(args []string) error {
	return o.flagSet.Parse(args)
}

// OptionCheck receives a slice of args and returns an error if it was not successfully parsed
func OptionCheck(args []string) error {
	opt := newOption(ioutil.Discard, flag.ContinueOnError)
	if err := opt.parse(args); err != nil {
		return fmt.Errorf("%v, %w", CommandName, err)
	}
	return nil
}

// command main
func command(opt *option) error {
	fp := os.Stdin
	if opt.file != "" {
		var err error
		fp, err = os.Open(opt.file)
		if err != nil {
			return err
		}
		defer func() {
			_ = fp.Close()
		}()
	}
{'fraction_non_alphanumeric': 0.13636363636363635, 'fraction_numerical': 0.0013271400132714001, 'mean_word_length': 3.164364640883978, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3055323', 'n_tokens_mistral': 1115, 'n_tokens_neox': 1057, 'n_words': 296}