<commit_before>"""
Hearthstone Strings file

File format: TSV. Lines starting with `#` are ignored.
Key is always `TAG`
"""
import csv
from typing import Dict

import hearthstone_data


StringsRow = Dict[str, str]
StringsDict = Dict[str, StringsRow]

_cache: Dict[str, StringsDict] = {}


def load(fp) -> StringsDict:
	reader = csv.DictReader(
		filter(lambda row: row.strip() and not row.startswith("#"), fp),
		delimiter="\t"
	)
	stripped_rows = [{k: v for k, v in row.items() if v} for row in reader]
	return {stripped_row.pop("TAG"): stripped_row for stripped_row in stripped_rows}


def load_globalstrings(locale="enUS") -> StringsDict:
	path: str = hearthstone_data.get_strings_file(locale, filename="GLOBAL.txt")
	if path not in _cache:
		with open(path, "r", encoding="utf-8-sig") as f:
			_cache[path] = load(f)

	return _cache[path]
<commit_msg>Skip blank lines in strings files<commit_after>"""
Hearthstone Strings file

File format: TSV. Lines starting with `#` are ignored.
Key is always `TAG`
"""
import csv
from typing import Dict

import hearthstone_data


StringsRow = Dict[str, str]
StringsDict = Dict[str, StringsRow]

_cache: Dict[str, StringsDict] = {}


def load(fp) -> StringsDict:
	reader = csv.DictReader(
		filter(lambda row: row.strip() and not row.startswith("#"), fp),
		delimiter="\t"
	)
	stripped_rows = [{k: v for k, v in row.items() if k and v} for row in reader]
	return {
		stripped_row.pop("TAG"): stripped_row for stripped_row in stripped_rows
		if stripped_row
	}


def load_globalstrings(locale="enUS") -> StringsDict:
	path: str = hearthstone_data.get_strings_file(locale, filename="GLOBAL.txt")
	if path not in _cache:
		with open(path, "r", encoding="utf-8-sig") as f:
			_cache[path] = load(f)

	return _cache[path]

{'fraction_non_alphanumeric': 0.13075611142694712, 'fraction_numerical': 0.0011370096645821489, 'mean_word_length': 4.847176079734219, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5418357', 'n_tokens_mistral': 650, 'n_tokens_neox': 623, 'n_words': 187}