<commit_before>package cmd

import (
	"os"
	"log"
	"bufio"
	"strconv"
)

func readFiles(args []string, f func(int64, int64)) {
	yFile, err := os.Open(args[0])
	if err != nil {
		log.Fatal("You must provide a file for actual values")
	}
	yHatFile, err := os.Open(args[1])
	if err != nil {
		log.Fatal("You must provide a value with predictions")
	}

	yScanner := bufio.NewScanner(bufio.NewReader(yFile))

	yHatScanner := bufio.NewScanner(bufio.NewReader(yHatFile))
	for {
		yScan := yScanner.Scan()
		if !yScan {
			if yScanner.Err() != nil {
				log.Fatal("Scanner error", yScanner.Err())
			}
			break
		}
		yHatScan := yHatScanner.Scan()
		if !yHatScan {
			if yHatScanner.Err() != nil {
				log.Fatal("Scanner error", yHatScanner.Err())
			}
			break

		}

		yText := yScanner.Text()
		yHatText := yHatScanner.Text()
		y, err := strconv.ParseInt(yText, 10, 8)
		if err != nil {
			log.Fatal("Error parsing int", err)
		}
		yHat, err := strconv.ParseInt(yHatText, 10, 8)
		if err != nil {
			log.Fatal("Error parsing int", err)
		}
		f(y, yHat)
	}
}<commit_msg>Splits lines on space and uses only the first column<commit_after>package cmd

import (
	"os"
	"log"
	"bufio"
	"strconv"
	"strings"
)

func readFiles(args []string, f func(int64, int64)) {
	yFile, err := os.Open(args[0])
	if err != nil {
		log.Fatal("You must provide a file for actual values")
	}
	yHatFile, err := os.Open(args[1])
	if err != nil {
		log.Fatal("You must provide a value with predictions")
	}

	yScanner := bufio.NewScanner(bufio.NewReader(yFile))

	yHatScanner := bufio.NewScanner(bufio.NewReader(yHatFile))
	for {
		yScan := yScanner.Scan()
		if !yScan {
			if yScanner.Err() != nil {
				log.Fatal("Scanner error", yScanner.Err())
			}
			break
		}
		yHatScan := yHatScanner.Scan()
		if !yHatScan {
			if yHatScanner.Err() != nil {
				log.Fatal("Scanner error", yHatScanner.Err())
			}
			break

		}

		yText := yScanner.Text()
		columnsY := strings.SplitN(yText, " ", 1)
		yHatText := yHatScanner.Text()
		columnsYHat := strings.SplitN(yHatText, " ", 1)
		y, err := strconv.ParseInt(columnsY[0], 10, 8)
		if err != nil {
			log.Fatal("Error parsing int", err)
		}
		yHat, err := strconv.ParseInt(columnsYHat[0], 10, 8)
		if err != nil {
			log.Fatal("Error parsing int", err)
		}
		f(y, yHat)
	}
}
{'fraction_non_alphanumeric': 0.16126205083260298, 'fraction_numerical': 0.012269938650306749, 'mean_word_length': 3.4416342412451364, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4063160', 'n_tokens_mistral': 1057, 'n_tokens_neox': 932, 'n_words': 210}