<commit_before>package inputs

import (
	"bufio"
	"encoding/csv"
	"fmt"
	"io"
	"strings"

	"github.com/subparlabs/bonjourno/log"
)

// MessageBuilder - Turns raw data into a list of messages
type MessageBuilder <-chan []string

func CSVField(fieldIndex int, source DataSource) MessageBuilder {
	c := make(chan []string)

	go func() {
		for {
			reader := csv.NewReader(strings.NewReader(<-source))
			reader.TrimLeadingSpace = true
			reader.FieldsPerRecord = -1

			var values []string

			// Ignore first line - comment
			reader.Read()
			for {
				if record, err := reader.Read(); err == io.EOF {
					break
				} else if err != nil {
					log.Error("Error reading CSV data", "err", err)
				} else if len(record) <= fieldIndex {
					log.Error("Error reading CSV data: index out of bounds", "# fields in record", record, "index", fieldIndex)
				} else {
					values = append(values, record[fieldIndex])
				}
			}
			if len(values) == 0 {
				log.Error("Didn't get any values from CSV")
			} else {
				c <- values
			}
		}
	}()
<commit_msg>Log # csv records read<commit_after>package inputs

import (
	"bufio"
	"encoding/csv"
	"fmt"
	"io"
	"strings"

	"github.com/subparlabs/bonjourno/log"
)

// MessageBuilder - Turns raw data into a list of messages
type MessageBuilder <-chan []string

func CSVField(fieldIndex int, source DataSource) MessageBuilder {
	c := make(chan []string)

	go func() {
		for {
			reader := csv.NewReader(strings.NewReader(<-source))
			reader.TrimLeadingSpace = true
			reader.FieldsPerRecord = -1

			var values []string

			// Ignore first line - comment
			reader.Read()
			for {
				if record, err := reader.Read(); err == io.EOF {
					break
				} else if err != nil {
					log.Error("Error reading CSV data", "err", err)
				} else if len(record) <= fieldIndex {
					log.Error("Error reading CSV data: index out of bounds", "# fields in record", record, "index", fieldIndex)
				} else {
					values = append(values, record[fieldIndex])
				}
			}
			if len(values) == 0 {
				log.Error("Didn't get any values from CSV")
			} else {
				log.Info("Read CSV data", "num values", len(values))
				c <- values
			}
		}
	}()

{'fraction_non_alphanumeric': 0.13703703703703704, 'fraction_numerical': 0.001851851851851852, 'mean_word_length': 2.9870848708487086, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7628627', 'n_tokens_mistral': 862, 'n_tokens_neox': 702, 'n_words': 209}