<commit_before>package main

import (
	"regexp"
	"strconv"
)

var FIELD = regexp.MustCompile(`(?:[^,\\]|\\.)*`)

func parseIndexesList(list string) ([]int, error) {
	fields := FIELD.FindAllString(list, -1)

	nums := make([]int, len(fields))
	for i, field := range fields {
		num, err := strconv.Atoi(field)
		if err != nil {
			return nil, err
		}
		nums[i] = num
	}
	return nums, nil
}<commit_msg>Implement CSVScanner as rough<commit_after>package main

import (
	"encoding/csv"
	"io"
	"regexp"
	"strconv"
	"strings"
)

var FIELD = regexp.MustCompile(`(?:[^,\\]|\\.)*`)

func parseIndexesList(list string) ([]int, error) {
	fields := FIELD.FindAllString(list, -1)

	nums := make([]int, len(fields))
	for i, field := range fields {
		num, err := strconv.Atoi(field)
		if err != nil {
			return nil, err
		}
		nums[i] = num
	}
	return nums, nil
}

type CSVScanner struct {
	indexes []int
	text    string
	err     error
	reader  *csv.Reader
}

func NewCSVScanner(indexes []int, r io.Reader) *CSVScanner {
	return &CSVScanner{
		indexes: indexes,
		reader:  csv.NewReader(r),
	}
}

func (c *CSVScanner) Err() error {
	if c.err == io.EOF {
		return nil
	}
	return c.err
}

func (c *CSVScanner) Bytes() []byte {
	return []byte(c.text)
}

func (c *CSVScanner) Text() string {
	return c.text
}

func (c *CSVScanner) Scan() bool {
	if c.err != nil {
		return false
	}

	recode, err := c.reader.Read()
	if err != nil {
		c.err = err
		c.text = ""
		return false
	}

	var values []string
	for _, index := range c.indexes {
		values = append(values, recode[index])
	}
	c.text = strings.Join(values, "\t")

	return true
}
{'fraction_non_alphanumeric': 0.1678060907395898, 'fraction_numerical': 0.001243008079552517, 'mean_word_length': 3.398907103825137, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3148109', 'n_tokens_mistral': 685, 'n_tokens_neox': 650, 'n_words': 167}