<commit_before>package main

import (
	"fmt"
	"reflect"
	"strings"
	"testing"
)

type ScanResult struct {
	scan  bool
	text  string
	isErr bool
}

func ppResults(results []ScanResult) string {
	lines := make([]string, len(results))
	for i := 0; i < len(results); i++ {
		lines[i] = fmt.Sprintf("%#v", results[i])
		lines[i] = strings.TrimPrefix(lines[i], "main.ScanResult")
	}
	return strings.Join(lines, "\n")
}

type ScanTest struct {
	selector Selector
	src      string
	result   []ScanResult
}

type DummyAll struct {
	dropHeaders bool
}

func (d *DummyAll) DropHeaders() bool {
	return d.dropHeaders
}

func (d *DummyAll) ParseHeaders(headers []string) error {
	return nil
}

func (d *DummyAll) Select(recode []string) ([]string, error) {
	return recode, nil
}

var scanTests = []ScanTest{
	{
		selector: &DummyAll{dropHeaders: false},
		src: `
1,1,1
2,4,8
`[1:],
		result: []ScanResult{
			{scan: true, text: "1\t1\t1", isErr: false},
			{scan: true, text: "2\t4\t8", isErr: false},
			{scan: false, text: "", isErr: false},
		},
	},
	{
		selector: &DummyAll{dropHeaders: false},
		src: `
1,2,3
1,2,3,4,5,6
`[1:],
		result: []ScanResult{
			{scan: true, text: "1\t2\t3", isErr: false},
			{scan: false, text: "", isErr: true},
			{scan: false, text: "", isErr: true},
		},
	},
	{
		selector: &DummyAll{dropHeaders: true},
		src: `
a,b,c
1,1,1
2,4,8
`[1:],
		result: []ScanResult{
			{scan: true, text: "1\t1\t1", isErr: false},
			{scan: true, text: "2\t4\t8", isErr: false},
			{scan: false, text: "", isErr: false},
		},
	},
	{
		selector: &DummyAll{dropHeaders: true},
		src: `
a,b,c
1,2,3
1,2,3,4,5,6
`[1:],
		result: []ScanResult{
			{scan: true, text: "1\t2\t3", isErr: false},
			{scan: false, text: "", isErr: true},
			{scan: false, text: "", isErr: true},
		},
	},
}

func TestScan(t *testing.T) {
	for _, test := range scanTests {
		r := strings.NewReader(test.src)
		c := NewCSVScanner(test.selector, r)

		expect := test.result
		actual := make([]ScanResult, len(test.result))
		for i := 0; i < len(test.result); i++ {
			scan := c.Scan()
			actual[i] = ScanResult{
				scan:  scan,
				text:  c.Text(),
				isErr: c.Err() != nil,
			}
		}
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("src:\n%sgot:\n%s\nwant:\n%s",
				test.src,
				ppResults(actual), ppResults(expect))
		}
	}
}
<commit_msg>Add case of no inputs<commit_after>package main

import (
	"fmt"
	"reflect"
	"strings"
	"testing"
)

type ScanResult struct {
	scan  bool
	text  string
	isErr bool
}

func ppResults(results []ScanResult) string {
	lines := make([]string, len(results))
	for i := 0; i < len(results); i++ {
		lines[i] = fmt.Sprintf("%#v", results[i])
		lines[i] = strings.TrimPrefix(lines[i], "main.ScanResult")
	}
	return strings.Join(lines, "\n")
}

type ScanTest struct {
	selector Selector
	src      string
	result   []ScanResult
}

type DummyAll struct {
	dropHeaders bool
}

func (d *DummyAll) DropHeaders() bool {
	return d.dropHeaders
}

func (d *DummyAll) ParseHeaders(headers []string) error {
	return nil
}

func (d *DummyAll) Select(recode []string) ([]string, error) {
	return recode, nil
}

var scanTests = []ScanTest{
	{
		selector: &DummyAll{dropHeaders: false},
		src: `
`[1:],
		result: []ScanResult{
			{scan: false, text: "", isErr: false},
		},
	},
	{
		selector: &DummyAll{dropHeaders: false},
		src: `
1,1,1
2,4,8
`[1:],
		result: []ScanResult{
			{scan: true, text: "1\t1\t1", isErr: false},
			{scan: true, text: "2\t4\t8", isErr: false},
			{scan: false, text: "", isErr: false},
		},
	},
	{
		selector: &DummyAll{dropHeaders: false},
		src: `
1,2,3
1,2,3,4,5,6
`[1:],
		result: []ScanResult{
			{scan: true, text: "1\t2\t3", isErr: false},
			{scan: false, text: "", isErr: true},
			{scan: false, text: "", isErr: true},
		},
	},
	{
		selector: &DummyAll{dropHeaders: true},
		src: `
`[1:],
		result: []ScanResult{
			{scan: false, text: "", isErr: false},
		},
	},
	{
		selector: &DummyAll{dropHeaders: true},
		src: `
a,b,c
`[1:],
		result: []ScanResult{
			{scan: false, text: "", isErr: false},
		},
	},
	{
		selector: &DummyAll{dropHeaders: true},
		src: `
a,b,c
1,1,1
2,4,8
`[1:],
		result: []ScanResult{
			{scan: true, text: "1\t1\t1", isErr: false},
			{scan: true, text: "2\t4\t8", isErr: false},
			{scan: false, text: "", isErr: false},
		},
	},
	{
		selector: &DummyAll{dropHeaders: true},
		src: `
a,b,c
1,2,3
1,2,3,4,5,6
`[1:],
		result: []ScanResult{
			{scan: true, text: "1\t2\t3", isErr: false},
			{scan: false, text: "", isErr: true},
			{scan: false, text: "", isErr: true},
		},
	},
}

func TestScan(t *testing.T) {
	for _, test := range scanTests {
		r := strings.NewReader(test.src)
		c := NewCSVScanner(test.selector, r)

		expect := test.result
		actual := make([]ScanResult, len(test.result))
		for i := 0; i < len(test.result); i++ {
			scan := c.Scan()
			actual[i] = ScanResult{
				scan:  scan,
				text:  c.Text(),
				isErr: c.Err() != nil,
			}
		}
		if !reflect.DeepEqual(actual, expect) {
			t.Errorf("src:\n%sgot:\n%s\nwant:\n%s",
				test.src,
				ppResults(actual), ppResults(expect))
		}
	}
}

{'fraction_non_alphanumeric': 0.20256663376110562, 'fraction_numerical': 0.021915103652517275, 'mean_word_length': 3.6477064220183486, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2589961', 'n_tokens_mistral': 2300, 'n_tokens_neox': 2121, 'n_words': 491}