<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

func main() {
	sourceFile, err := ioutil.ReadFile(os.Args[1])
	var s string
	if err == nil {
		s = string(sourceFile)
		fmt.Println(s)
	}
	splitFile := strings.Split(s, "\n")
	lineSlice := createLines(splitFile)
	debugFile, err := os.Create("testfiles/c++/debug.cpp")
	if err != nil {
		panic(err)
	}
	for i := 0; i < len(lineSlice); i++ {
		_, err := debugFile.WriteString(splitFile[i] + "\n")
		if err != nil {
			panic(err)
		}
		AddPrint(debugFile, lineSlice[i].s)
	}
}

type lineType struct {
	s string
	code int
}

func AddPrint(file *os.File, s string) {
	file.WriteString("std::cout << \"" + s + "\" std::endl;\n")
}

func createLines(stringSlice []string) ([]lineType) {
	var lineSlice []lineType
	for i := 0; i < len(stringSlice); i++ {
		lineSlice = append(lineSlice, lineType{stringSlice[i], 0})
	}
	return lineSlice
}<commit_msg>Removing newlines before parsing<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

func main() {
	sourceFile, err := ioutil.ReadFile(os.Args[1])
	var s string
	if err == nil {
		s = string(sourceFile)
		fmt.Println(s)
	}
	splitFile := strings.Split(s, "\n")
	splitFile = removeNewlines(splitFile)
	lineSlice := createLines(splitFile)
	debugFile, err := os.Create("testfiles/c++/debug.cpp")
	if err != nil {
		panic(err)
	}
	for i := 0; i < len(lineSlice); i++ {
		_, err := debugFile.WriteString(splitFile[i] + "\n")
		if err != nil {
			panic(err)
		}
		AddPrint(debugFile, lineSlice[i].s)
	}
}

type lineType struct {
	s    string
	code int
}

func AddPrint(file *os.File, s string) {
	file.WriteString("std::cout << \"" + s + "\" std::endl;\n")
}

func createLines(stringSlice []string) []lineType {
	var lineSlice []lineType
	for i := 0; i < len(stringSlice); i++ {
		lineSlice = append(lineSlice, lineType{stringSlice[i], 0})
	}
	return lineSlice
}

func removeNewlines(stringSlice []string) []string {
	var newSlice []string
	for i := 0; i < len(stringSlice); i++ {
		if stringSlice[i] != "" && stringSlice[i][0] != 10 {
			newSlice = append(newSlice, stringSlice[i])
		}
	}
	return newSlice
}
{'fraction_non_alphanumeric': 0.17529137529137528, 'fraction_numerical': 0.005594405594405594, 'mean_word_length': 3.9906976744186045, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '266753', 'n_tokens_mistral': 869, 'n_tokens_neox': 833, 'n_words': 191}