<commit_before>package utils

import (
	"bufio"
	"os"
	"io"
)

type lineIterator struct {
	file   *os.File
	buffer *bufio.Reader
}

func NewLineIterator(filename string) (*lineIterator, error) {

	// Open file
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	fileReader := bufio.NewReader(file)

	return &lineIterator{file, fileReader}, nil
}

func readLine(r *bufio.Reader) (string, error) {
	var (
		isPrefix bool = true
		err error = nil
		line, ln []byte
	)
	for isPrefix && err == nil {
		line, isPrefix, err = r.ReadLine()
		ln = append(ln, line...)
	}
	return string(ln), err
}

/**
 * Return the next line and if is last.
 * return line string: The next line in the file.
 * return isLast bool: If this line is the last in the file.
 */
func (self *lineIterator) Next() (line string, isLast bool) {
	line, err := readLine(self.buffer)
	if err == io.EOF {
		return line, true
	}
	return line, false
}

func (self *lineIterator) Close() {
	self.file.Close()
}<commit_msg>Change iterator<commit_after>package utils

import (
	"bufio"
	"os"
	"io"
)

type lineIterator struct {
	file    *os.File
	buffer  *bufio.Reader
	hasNext bool
}

func NewLineIterator(filename string) (*lineIterator, error) {

	// Open file
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	fileReader := bufio.NewReader(file)

	return &lineIterator{file, fileReader, true}, nil
}

func readLine(r *bufio.Reader) (string, error) {
	var (
		isPrefix bool = true
		err error = nil
		line, ln []byte
	)
	for isPrefix && err == nil {
		line, isPrefix, err = r.ReadLine()
		ln = append(ln, line...)
	}
	return string(ln), err
}

func (self *lineIterator) Next() string {
	line, err := readLine(self.buffer)
	if err == io.EOF { self.hasNext = false }
	return line
}

func (self *lineIterator) HasNext() bool {
	return self.hasNext
}

func (self *lineIterator) Close() {
	self.file.Close()
}
{'fraction_non_alphanumeric': 0.13602941176470587, 'fraction_numerical': 0.0, 'mean_word_length': 3.5574162679425836, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '492154', 'n_tokens_mistral': 705, 'n_tokens_neox': 685, 'n_words': 213}