<commit_before>package utilities

import (
	"bufio"
	"fmt"
	"io"
	"math"
	"os"
)

func ReadFileToIntArray(fileName string, array []int) (int, error) {
	fileDesc, err := os.Open(fileName)
	if err != nil {
		return 0, err
	}
	defer fileDesc.Close()

	var numbersLen = 0
	var number int
	for {
		_, err := fmt.Fscanf(fileDesc, "%d\n", &number)
		if err != nil {
			if err == io.EOF {
				return numbersLen, nil
			}
			fmt.Println(err)
			return 0, err
		}
		array[numbersLen] = number
		numbersLen++
	}
}

func ReadFileToStringArray(fileName string, array []string) (int, error) {
	fileDesc, err := os.Open(fileName)
	if err != nil {
		return 0, err
	}
	defer fileDesc.Close()

	var linesLen = 0
	var scanner = bufio.NewScanner(fileDesc)
	for scanner.Scan() {
		array[linesLen] = scanner.Text()
		linesLen++
	}

	err = scanner.Err()
	if err != nil {
		return 0, err
	}

	return linesLen, nil
}

type stringLineHandler func(string)

func StreamFileAsStringLines(fileName string, handler stringLineHandler) error {
	fileDesc, err := os.Open(fileName)
	if err != nil {
		return err
	}
	defer fileDesc.Close()

	var scanner = bufio.NewScanner(fileDesc)
	for scanner.Scan() {
		handler(scanner.Text())
	}

	err = scanner.Err()
	if err != nil {
		return err
	}

	return nil
}

func ScanString(content string, format string, a ...interface{}) {
	_, err := fmt.Sscanf(content, format, a...)
	if err != nil {
		fmt.Println("Failed to parse string content: "+content, err)<commit_msg>Small changes in the Go utilities<commit_after>package utilities

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"strconv"
	"strings"
)

func ReadFileToIntArray(fileName string, array []int) (int, error) {
	fileDesc, err := os.Open(fileName)
	if err != nil {
		return 0, err
	}
	defer fileDesc.Close()

	var numbersLen = 0
	var number int
	for {
		_, err := fmt.Fscanf(fileDesc, "%d\n", &number)
		if err != nil {
			if err == io.EOF {
				return numbersLen, nil
			}
			fmt.Println(err)
			return 0, err
		}
		array[numbersLen] = number
		numbersLen++
	}
}

func ReadFileToString(fileName string) (string, error) {
	var fileContent, err = ioutil.ReadFile(fileName)
	if err != nil {
		return "", err
	}
	return string(fileContent), nil
}

func ReadFileToStringArray(fileName string, array []string) (int, error) {
	fileDesc, err := os.Open(fileName)
	if err != nil {
		return 0, err
	}
	defer fileDesc.Close()

	var linesLen = 0
	var scanner = bufio.NewScanner(fileDesc)
	for scanner.Scan() {
		array[linesLen] = scanner.Text()
		linesLen++
	}

	err = scanner.Err()
	if err != nil {
		return 0, err
	}

	return linesLen, nil
}

func SplitCsvStringToIntArray(csvString string, targetArray []int) int {
	var stringSplits = strings.Split(csvString, ",")
	for stringSplitsIndex := range stringSplits {
		var parsedInt, err = strconv.ParseInt(stringSplits[stringSplitsIndex], 10, 0)
		if err != nil {
			fmt.Printf("Failed to parse string to int: [%s]", stringSplits[stringSplitsIndex])
		} else {
			targetArray[stringSplitsIndex] = int(parsedInt)
		}
	}
	return len(stringSplits)
}

type stringLineHandler func(string)

func StreamFileAsStringLines(fileName string, handler stringLineHandler) error {
	fileDesc, err := os.Open(fileName)
	if err != nil {
		return err
	}
	defer fileDesc.Close()

	var scanner = bufio.NewScanner(fileDesc)
	for scanner.Scan() {
		handler(scanner.Text())
	}

	err = scanner.Err()
	if err != nil {
		return err
	}

	return nil
}

func ScanString(content string, format string, a ...interface{}) {
	_, err := fmt.Sscanf(content, format, a...)
	if err != nil {
		fmt.Println("Failed to parse string content: "+content, err)
{'fraction_non_alphanumeric': 0.13274580005508124, 'fraction_numerical': 0.004131093362709997, 'mean_word_length': 3.79155672823219, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7209517', 'n_tokens_mistral': 1413, 'n_tokens_neox': 1306, 'n_words': 355}