<commit_before>// main.go
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/transform"
	"io/ioutil"
	"os"
	"runtime"
)

func main() {
	var text string
	var err error
	var encode string
	var debug bool
	var interactive bool
	var default_encoding string

	if runtime.GOOS == "windows" {
		default_encoding = "sjis"
	} else {
		default_encoding = "utf-8"
	}
	flag.StringVar(&encode, "e", default_encoding, "encoding")
	flag.BoolVar(&debug, "d", false, "debug mode")
	flag.BoolVar(&interactive, "i", false, "interactive mode")

	flag.Parse()

	if interactive == true {
		fmt.Print("> ")
	} else if len(flag.Args()) == 0 {
		text, err = readPipe()
	} else if flag.Arg(0) == "-" {
		text, err = readStdin()
	} else {
		text, err = readFileByArg(flag.Arg(0))
	}
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	if interactive == false {
		text, err := transEnc(text, encode)
		if err != nil {
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(1)
		}
		s, d := Dajarep(text)
		for i := 0; i < len(s); i++ {
			if !debug {
				fmt.Println(s[i])
			} else {
				fmt.Println(s[i] + "[" + d[i] + "]")
			}
		}
	} else {
		//interactive mode
		s := bufio.NewScanner(os.Stdin)
		for s.Scan() {
			if s.Err() != nil {
				fmt.Fprintln(os.Stderr, s.Err())
				os.Exit(1)
			}
			if s.Text() == "" {
				break
			}
			text, err := transEnc(s.Text(), encode)
			if err != nil {
				fmt.Fprintln(os.Stderr, err.Error())
				os.Exit(1)
			}
			_, d := Dajarep(text)
			if len(d) > 0 {
				for i := 0; i < len(d); i++ {
					fmt.Println("-> " + d[i])
				}
			} else {
				fmt.Println("")
			}
			fmt.Print("> ")
		}
	}
}

func readPipe() (string, error) {<commit_msg>Bugfix<commit_after>// main.go
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/transform"
	"io/ioutil"
	"os"
	"runtime"

	"github.com/kurehajime/dajarep"
)

func main() {
	var text string
	var err error
	var encode string
	var debug bool
	var interactive bool
	var default_encoding string

	if runtime.GOOS == "windows" {
		default_encoding = "sjis"
	} else {
		default_encoding = "utf-8"
	}
	flag.StringVar(&encode, "e", default_encoding, "encoding")
	flag.BoolVar(&debug, "d", false, "debug mode")
	flag.BoolVar(&interactive, "i", false, "interactive mode")

	flag.Parse()

	if interactive == true {
		fmt.Print("> ")
	} else if len(flag.Args()) == 0 {
		text, err = readPipe()
	} else if flag.Arg(0) == "-" {
		text, err = readStdin()
	} else {
		text, err = readFileByArg(flag.Arg(0))
	}
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	if interactive == false {
		text, err := transEnc(text, encode)
		if err != nil {
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(1)
		}
		s, d := dajarep.Dajarep(text)
		for i := 0; i < len(s); i++ {
			if !debug {
				fmt.Println(s[i])
			} else {
				fmt.Println(s[i] + "[" + d[i] + "]")
			}
		}
	} else {
		//interactive mode
		s := bufio.NewScanner(os.Stdin)
		for s.Scan() {
			if s.Err() != nil {
				fmt.Fprintln(os.Stderr, s.Err())
				os.Exit(1)
			}
			if s.Text() == "" {
				break
			}
			text, err := transEnc(s.Text(), encode)
			if err != nil {
				fmt.Fprintln(os.Stderr, err.Error())
				os.Exit(1)
			}
			_, d := dajarep.Dajarep(text)
			if len(d) > 0 {
				for i := 0; i < len(d); i++ {
					fmt.Println("-> " + d[i])
				}
			} else {
				fmt.Println("")
			}
			fmt.Print("> ")
		}
	}
}

func readPipe() (string, error) {
{'fraction_non_alphanumeric': 0.1936127744510978, 'fraction_numerical': 0.006273167949814656, 'mean_word_length': 2.743863393810032, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4248736', 'n_tokens_mistral': 1682, 'n_tokens_neox': 1481, 'n_words': 317}