<commit_before>package scipipe

import (
	"bufio"
	"log"
	"os"
)

// FileReader takes a file path on its FilePath in-port, and returns the file
// content as []byte on its out-port Out
type FileReader struct {
	Process
	FilePath chan string
	Out      chan []byte
}

// Instantiate a new FileReader
func NewFileReader() *FileReader {
	return &FileReader{
		FilePath: make(chan string),
		Out:      make(chan []byte, BUFSIZE),
	}
}

// Run the FileReader
func (proc *FileReader) Run() {
	defer close(proc.Out)

	file, err := os.Open(<-proc.FilePath)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scan := bufio.NewScanner(file)
	for scan.Scan() {
		proc.Out <- append([]byte(nil), scan.Bytes()...)
	}
	if scan.Err() != nil {
		log.Fatal(scan.Err())
	}
}<commit_msg>API Change: FileReader.Out -> .OutLine<commit_after>package scipipe

import (
	"bufio"
	"log"
	"os"
)

// FileReader takes a file path on its FilePath in-port, and returns the file
// content as []byte on its out-port Out
type FileReader struct {
	Process
	FilePath chan string
	OutLine  chan []byte
}

// Instantiate a new FileReader
func NewFileReader() *FileReader {
	return &FileReader{
		FilePath: make(chan string),
		OutLine:  make(chan []byte, BUFSIZE),
	}
}

// Run the FileReader
func (proc *FileReader) Run() {
	defer close(proc.OutLine)

	file, err := os.Open(<-proc.FilePath)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scan := bufio.NewScanner(file)
	for scan.Scan() {
		proc.OutLine <- append(append([]byte(nil), scan.Bytes()...), []byte("\n")...)
	}
	if scan.Err() != nil {
		log.Fatal(scan.Err())
	}
}
{'fraction_non_alphanumeric': 0.15422885572139303, 'fraction_numerical': 0.0, 'mean_word_length': 3.5323943661971833, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7098801', 'n_tokens_mistral': 608, 'n_tokens_neox': 580, 'n_words': 166}