<commit_before>package hvsc

import (
	"bufio"
	"log"
	"os"
)

const SongInfoFile = "DOCUMENTS/STIL.txt"

func readSTIL() {
	file, err := os.Open(hvscPathTo(SongInfoFile))
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	var tune *SidTune
	var info []string
	var tuneIndex = -1

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		if line[0] == '/' && line[len(line)-4:] == ".sid" {
			tuneIndex = tuneIndexByPath(line) //, tuneIndex+1)
			if tuneIndex < 0 {
				log.Fatalf("Unknown path in file %s: %s", SongInfoFile, line)
			}
			if tune != nil {
				tune.Info = info
			}
			tune = &Tunes[tuneIndex]
			info = make([]string, 0)
		} else {
			info = append(info, line)
		}
	}
}

func tuneIndexByPath(path string) int {
	for i, tune := range Tunes {
		if tune.Path == path {
			return i
		}
	}
	return -1
}
<commit_msg>Fixed STIL parsing to skip folder info.<commit_after>package hvsc

import (
	"bufio"
	"log"
	"os"
)

const SongInfoFile = "DOCUMENTS/STIL.txt"

func readSTIL() {
	file, err := os.Open(hvscPathTo(SongInfoFile))
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	var tune *SidTune
	var info []string
	var tuneIndex = -1

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		if line[0] == '/' {
			if tune != nil {
				tune.Info = info
				tune = nil
			}
			if line[len(line)-4:] == ".sid" {
				tuneIndex = tuneIndexByPath(line) //, tuneIndex+1)
				if tuneIndex < 0 {
					log.Fatalf("Unknown path in file %s: %s", SongInfoFile, line)
				}
				tune = &Tunes[tuneIndex]
				info = make([]string, 0)
			}
		} else if tune != nil {
			info = append(info, line)
		}
	}
}

func tuneIndexByPath(path string) int {
	for i, tune := range Tunes {
		if tune.Path == path {
			return i
		}
	}
	return -1
}

{'fraction_non_alphanumeric': 0.16230366492146597, 'fraction_numerical': 0.009424083769633508, 'mean_word_length': 2.8450704225352115, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6363078', 'n_tokens_mistral': 876, 'n_tokens_neox': 769, 'n_words': 188}