<commit_before>package strings

import (
	"github.com/losinggeneration/hge-go/hge"
	. "github.com/losinggeneration/hge-go/legacy"
	"strings"
)

const (
	strHEADERTAG   = "[HGESTRINGTABLE]"
	strFORMATERROR = "String table %s has incorrect format."
)

type StringTable struct {
	stringsMap map[string]string
}

func NewStringTable(filename string) *StringTable {
	st := new(StringTable)
	hge := Create(hge.VERSION)

	st.stringsMap = make(map[string]string)

	f := hge.ResourceLoadString(filename)

	if f == nil || !strings.HasPrefix(*f, strHEADERTAG) {
		hge.System_Log(strFORMATERROR, filename)
		return nil
	}

	var (
		inComment, inIdentifier, inValue bool
		identifier, value                string
	)

	reader := strings.NewReader(*f)
	_, e := reader.Seek(int64(len(strHEADERTAG)), 0)

	if e != nil {
		hge.System_Log("Unable to seek past header tag")
		return nil
	}

	for b, e := reader.ReadByte(); e == nil; b, e = reader.ReadByte() {
		// we ignore whitespace
		if b == '\n' {
			inComment = false
			inIdentifier = false
			continue
		}

		// just continue if we're in a comment
		if inComment {
			continue<commit_msg>Update strings to use new API<commit_after>package strings

import (
	"github.com/losinggeneration/hge-go/hge"
	"strings"
)

const (
	strHEADERTAG   = "[HGESTRINGTABLE]"
	strFORMATERROR = "String table %s has incorrect format."
)

type StringTable struct {
	stringsMap map[string]string
}

func NewStringTable(filename string) *StringTable {
	st := new(StringTable)

	st.stringsMap = make(map[string]string)

	f := hge.LoadString(filename)

	if f == nil || !strings.HasPrefix(*f, strHEADERTAG) {
		hge.Log(strFORMATERROR, filename)
		return nil
	}

	var (
		inComment, inIdentifier, inValue bool
		identifier, value                string
	)

	reader := strings.NewReader(*f)
	_, e := reader.Seek(int64(len(strHEADERTAG)), 0)

	if e != nil {
		hge.Log("Unable to seek past header tag")
		return nil
	}

	for b, e := reader.ReadByte(); e == nil; b, e = reader.ReadByte() {
		// we ignore whitespace
		if b == '\n' {
			inComment = false
			inIdentifier = false
			continue
		}

		// just continue if we're in a comment
		if inComment {
			continue
{'fraction_non_alphanumeric': 0.12298479963150621, 'fraction_numerical': 0.0027637033625057578, 'mean_word_length': 3.563025210084034, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '893114', 'n_tokens_mistral': 810, 'n_tokens_neox': 742, 'n_words': 193}