<commit_before>package main

import (
	"bufio"
	"os"
)

func initRIO() *RObject {
	obj := &RObject{}
	obj.name = "RIO"
	obj.ivars = make(map[string]Object)
	obj.class = nil
	obj.methods = make(map[string]*RMethod)

	// RString method initialization
	obj.methods["readlines"] = &RMethod{gofunc: RIO_readlines}

	return obj
}

// IO.readlines(filename)
// [RString]
func RIO_readlines(vm *GobiesVM, receiver Object, v []Object) Object {
	filename := v[0].(*RObject).val.str

	obj := RArray_new(vm, receiver, nil)

	input, _ := os.Open(filename)
	scanner := bufio.NewScanner(input)

	for scanner.Scan() {
		dummy_obj := make([]Object, 1, 1)
		dummy_obj[0] = scanner.Text()
		rstr := RString_new(vm, receiver, dummy_obj)
		dummy_obj[0] = rstr
		RArray_append(vm, obj, dummy_obj)
	}

	return obj
}<commit_msg>[IO] revise readlines() for large files<commit_after>package main

import (
	"bufio"
	"os"
)

func initRIO() *RObject {
	obj := &RObject{}
	obj.name = "RIO"
	obj.ivars = make(map[string]Object)
	obj.class = nil
	obj.methods = make(map[string]*RMethod)

	// RString method initialization
	obj.methods["readlines"] = &RMethod{gofunc: RIO_readlines}

	return obj
}

// IO.readlines(filename)
// [RString]
func RIO_readlines(vm *GobiesVM, receiver Object, v []Object) Object {
	filename := v[0].(*RObject).val.str

	obj := RArray_new(vm, receiver, nil)

	input, _ := os.Open(filename)
	defer input.Close()

	reader := bufio.NewReader(input)

	line, err := reader.ReadString('\n')
	for ; err == nil; line, err = reader.ReadString('\n') {
		dummy_obj := make([]Object, 1, 1)
		dummy_obj[0] = line
		rstr := RString_new(vm, receiver, dummy_obj)
		dummy_obj[0] = rstr
		RArray_append(vm, obj, dummy_obj)
	}

	return obj
}
{'fraction_non_alphanumeric': 0.1607454863133372, 'fraction_numerical': 0.005824111822947, 'mean_word_length': 4.453968253968254, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7603290', 'n_tokens_mistral': 695, 'n_tokens_neox': 664, 'n_words': 144}