<commit_before>package grep

import (
	"github.com/monochromegane/the_platinum_searcher/search/file"
	"github.com/monochromegane/the_platinum_searcher/search/option"
	"github.com/monochromegane/the_platinum_searcher/search/pattern"
	"github.com/monochromegane/the_platinum_searcher/search/print"
	"os"
	"testing"
)

type Assert struct {
	path, pattern, fileType, match string
}

var Asserts = []Assert{
	Assert{"ascii.txt", "go", file.ASCII, "go test"},
	Assert{"ja/euc-jp.txt", "go", file.EUCJP, "go テスト"},
	Assert{"ja/shift_jis.txt", "go", file.SHIFTJIS, "go テスト"},
	Assert{"ja/utf8.txt", "go", file.UTF8, "go テスト"},
	Assert{"ja/broken_euc-jp.txt", "go", file.EUCJP, "go テスト"},
	Assert{"ja/broken_shift_jis.txt", "go", file.SHIFTJIS, "go テスト"},
	Assert{"ja/broken_utf8.txt", "go", file.UTF8, "go テスト"},
}

func TestGrep(t *testing.T) {

	for _, g := range Asserts {
		in := make(chan *Params)
		out := make(chan *print.Params)
		grepper := Grepper{in, out, &option.Option{Proc: 1}}

		pattern, _ := pattern.NewPattern(g.pattern, "", false, false, false)
		sem := make(chan bool, 1)
		sem <- true
		go grepper.Grep("../../files/"+g.path, g.fileType, pattern, sem)
		o := <-out
		if o.Path != "../../files/"+g.path {
			t.Errorf("It should be equal ../../files/%s.", g.path)
		}
		if o.Matches[0].Match() != g.match {
			t.Errorf("%s should be equal %s", g.path, g.match)
		}
	}
}

func TestGrepWithStream(t *testing.T) {
	fh, err := os.Open("../../files/ascii.txt")
	if err != nil {
		panic(err)
	}
	tempStdin := os.Stdin
	os.Stdin = fh
	defer func() { os.Stdin = tempStdin }()
	g := Assert{"", "go", file.ASCII, "go test"}
	in := make(chan *Params)
	out := make(chan *print.Params)
	grepper := Grepper{in, out, &option.Option{Proc: 1, SearchStream: true}}

	pattern, _ := pattern.NewPattern(g.pattern, "", false, false, false)
	sem := make(chan bool, 1)
	sem <- true
	go grepper.Grep(g.path, g.fileType, pattern, sem)
	o := <-out
	if o.Path != g.path {
		t.Errorf("It should be equal %s.", g.path)
	}
	if o.Matches[0].Match() != g.match {
		t.Errorf("%s should be equal %s", g.path, g.match)
	}
}

func receive(in chan *Params, params *Params) {
	in <- params
	close(in)
}<commit_msg>Fixed a wrong test.<commit_after>package grep

import (
	"os"
	"testing"

	"github.com/monochromegane/the_platinum_searcher"
	"github.com/monochromegane/the_platinum_searcher/search/option"
	"github.com/monochromegane/the_platinum_searcher/search/pattern"
	"github.com/monochromegane/the_platinum_searcher/search/print"
)

type Assert struct {
	path, pattern, fileType, match string
}

var Asserts = []Assert{
	Assert{"ascii.txt", "go", the_platinum_searcher.ASCII, "go test"},
	Assert{"ja/euc-jp.txt", "go", the_platinum_searcher.EUCJP, "go テスト"},
	Assert{"ja/shift_jis.txt", "go", the_platinum_searcher.SHIFTJIS, "go テスト"},
	Assert{"ja/utf8.txt", "go", the_platinum_searcher.UTF8, "go テスト"},
	Assert{"ja/broken_euc-jp.txt", "go", the_platinum_searcher.EUCJP, "go テスト"},
	Assert{"ja/broken_shift_jis.txt", "go", the_platinum_searcher.SHIFTJIS, "go テスト"},
	Assert{"ja/broken_utf8.txt", "go", the_platinum_searcher.UTF8, "go テスト"},
}

func TestGrep(t *testing.T) {

	for _, g := range Asserts {
		in := make(chan *Params)
		out := make(chan *print.Params)
		grepper := Grepper{in, out, &option.Option{Proc: 1}}

		pattern, _ := pattern.NewPattern(g.pattern, "", false, false, false)
		sem := make(chan bool, 1)
		sem <- true
		go grepper.Grep("../../files/"+g.path, g.fileType, pattern, sem)
		o := <-out
		if o.Path != "../../files/"+g.path {
			t.Errorf("It should be equal ../../files/%s.", g.path)
		}
		if o.Matches[0].Match() != g.match {
			t.Errorf("%s should be equal %s", g.path, g.match)
		}
	}
}

func TestGrepWithStream(t *testing.T) {
	fh, err := os.Open("../../files/ascii.txt")
	if err != nil {
		panic(err)
	}
	tempStdin := os.Stdin
	os.Stdin = fh
	defer func() { os.Stdin = tempStdin }()
	g := Assert{"", "go", the_platinum_searcher.ASCII, "go test"}
	in := make(chan *Params)
	out := make(chan *print.Params)
	grepper := Grepper{in, out, &option.Option{Proc: 1, SearchStream: true}}

	pattern, _ := pattern.NewPattern(g.pattern, "", false, false, false)
	sem := make(chan bool, 1)
	sem <- true
	go grepper.Grep(g.path, g.fileType, pattern, sem)
	o := <-out
	if o.Path != g.path {
		t.Errorf("It should be equal %s.", g.path)
	}
	if o.Matches[0].Match() != g.match {
		t.Errorf("%s should be equal %s", g.path, g.match)
	}
}

func receive(in chan *Params, params *Params) {
	in <- params
	close(in)
}
{'fraction_non_alphanumeric': 0.19564734621363536, 'fraction_numerical': 0.0044414834554741284, 'mean_word_length': 3.982300884955752, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1844984', 'n_tokens_mistral': 1927, 'n_tokens_neox': 1822, 'n_words': 389}