<commit_before>package grep

import (
	"bufio"
	"code.google.com/p/mahonia"
	"github.com/monochromegane/the_platinum_searcher/search/file"
	"github.com/monochromegane/the_platinum_searcher/search/option"
	"github.com/monochromegane/the_platinum_searcher/search/print"
	"os"
	"strings"
)

type Params struct {
	Path, Pattern, Encode string
}

type Grepper struct {
	In     chan *Params
	Out    chan *print.Params
	Option *option.Option
}

func (self *Grepper) ConcurrentGrep() {
	sem := make(chan bool, self.Option.Proc)
	for arg := range self.In {
		sem <- true
		go self.Grep(arg.Path, arg.Encode, arg.Pattern, sem)
	}
	for {
		if len(sem) == 0 {
			break
		}
	}
	close(self.Out)
}

func (self *Grepper) Grep(path, encode, pattern string, finish chan bool) {

	fh, err := os.Open(path)
	f := bufio.NewReader(fh)
	if err != nil {
		panic(err)
	}
	buf := make([]byte, 1024)
	decoder := mahonia.NewDecoder(encode)

	m := make([]*print.Match, 0)

	var lineNum = 1
	for {
		buf, _, err = f.ReadLine()
		if err != nil {
			break
		}

		s := string(buf)
		if decoder != nil && encode != file.UTF8 && encode != file.ASCII {
			s = decoder.ConvertString(s)
		}
		if strings.Contains(s, pattern) {
			m = append(m, &print.Match{lineNum, s})
		}
		lineNum++
	}
	self.Out <- &print.Params{pattern, path, m}
	fh.Close()
	<-finish
}<commit_msg>Import "code.google.com/p/go.text/encoding" instead of "code.google.com/p/mahonia".<commit_after>package grep

import (
	"bufio"
	"code.google.com/p/go.text/encoding/japanese"
	"code.google.com/p/go.text/transform"
	"github.com/monochromegane/the_platinum_searcher/search/file"
	"github.com/monochromegane/the_platinum_searcher/search/option"
	"github.com/monochromegane/the_platinum_searcher/search/print"
	"os"
	"strings"
)

type Params struct {
	Path, Pattern, Encode string
}

type Grepper struct {
	In     chan *Params
	Out    chan *print.Params
	Option *option.Option
}

func (self *Grepper) ConcurrentGrep() {
	sem := make(chan bool, self.Option.Proc)
	for arg := range self.In {
		sem <- true
		go self.Grep(arg.Path, arg.Encode, arg.Pattern, sem)
	}
	for {
		if len(sem) == 0 {
			break
		}
	}
	close(self.Out)
}

func getDecoder(encode string) transform.Transformer {
	switch encode {
	case file.EUCJP:
		return japanese.EUCJP.NewDecoder()
	case file.SHIFTJIS:
		return japanese.ShiftJIS.NewDecoder()
	}
	return nil
}

func (self *Grepper) Grep(path, encode, pattern string, finish chan bool) {
	fh, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	var f *bufio.Reader
	if dec := getDecoder(encode); dec != nil {
		f = bufio.NewReader(transform.NewReader(fh, dec))
	} else {
		f = bufio.NewReader(fh)
	}

	var buf []byte
	m := make([]*print.Match, 0)
	var lineNum = 1
	for {
		buf, _, err = f.ReadLine()
		if err != nil {
			break
		}

		s := string(buf)
		if strings.Contains(s, pattern) {
			m = append(m, &print.Match{lineNum, s})
		}
		lineNum++
	}
	self.Out <- &print.Params{pattern, path, m}
	fh.Close()
	<-finish
}
{'fraction_non_alphanumeric': 0.14988211519029976, 'fraction_numerical': 0.00370495116200741, 'mean_word_length': 3.5762711864406778, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '766496', 'n_tokens_mistral': 1267, 'n_tokens_neox': 1171, 'n_words': 254}