<commit_before>using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using NLog;
using YamlDotNet.Serialization;

namespace Mitternacht.Services.Impl {
	public class StringService : IMService {
		public const string StringsPath = @"locales";
		public const string FilenameRegex = @"(.+)\.yml$";

		private readonly ImmutableDictionary<string, ImmutableDictionary<string, ImmutableDictionary<string, string>>> _responseStrings;

		// Used as failsafe in case response key doesn't exist in the selected or default language.
		private readonly CultureInfo   _fallbackCultureInfo = new CultureInfo("de-DE");
		private readonly ILocalization _localization;
		private readonly Logger        _logger = LogManager.GetCurrentClassLogger();

		public StringService(ILocalization loc) {
			var log = LogManager.GetCurrentClassLogger();
			_localization = loc;

			var sw          = Stopwatch.StartNew();
			var localesDict = new Dictionary<string, ImmutableDictionary<string, ImmutableDictionary<string, string>>>();
			var localeFiles = Directory.GetFiles(StringsPath)
								.Select(filename => (Filename: filename, Match: Regex.Match(Path.GetFileName(filename), FilenameRegex)))
								.Where((fnm) => fnm.Match.Success)
								.Select(fnm => (Filename: fnm.Filename, Locale: fnm.Match.Groups[1].Value))
								.ToArray();
			
			var deserializer = new Deserializer();

			foreach(var (filename, locale) in localeFiles) {
				var langDict = deserializer.Deserialize<Dictionary<string, Dictionary<string, string>>>(File.ReadAllText(filename));
				localesDict.Add(locale.ToLowerInvariant(), langDict.ToImmutableDictionary(k => k.Key, v => v.Value.ToImmutableDictionary()));
			}

			_responseStrings = localesDict.ToImmutableDictionary();
			sw.Stop();

			log.Info($"Loaded {_responseStrings.Count} locales in {sw.Elapsed.TotalSeconds:F2}s");
		}

		private string GetString(string moduleName, string key, CultureInfo cultureInfo) {
			if(!_responseStrings.TryGetValue(cultureInfo.Name.ToLowerInvariant(), out var moduleStrings)) return null;

			if(moduleStrings.TryGetValue(moduleName, out var strings)) {
				strings.TryGetValue(key, out var val);
				return val;
			}
			return null;
		}
<commit_msg>Add event UnknownKeyRequested in StringService.<commit_after>using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using NLog;
using YamlDotNet.Serialization;

namespace Mitternacht.Services.Impl {
	public class StringService : IMService {
		public const string StringsPath = @"locales";
		public const string FilenameRegex = @"(.+)\.yml$";

		private readonly ImmutableDictionary<string, ImmutableDictionary<string, ImmutableDictionary<string, string>>> _responseStrings;

		// Used as failsafe in case response key doesn't exist in the selected or default language.
		private readonly CultureInfo   _fallbackCultureInfo = new CultureInfo("de-DE");
		private readonly ILocalization _localization;
		private readonly Logger        _logger = LogManager.GetCurrentClassLogger();

		public event Action<string, string, CultureInfo> UnknownKeyRequested = delegate{ };

		public StringService(ILocalization loc) {
			var log = LogManager.GetCurrentClassLogger();
			_localization = loc;

			var sw          = Stopwatch.StartNew();
			var localesDict = new Dictionary<string, ImmutableDictionary<string, ImmutableDictionary<string, string>>>();
			var localeFiles = Directory.GetFiles(StringsPath)
								.Select(filename => (Filename: filename, Match: Regex.Match(Path.GetFileName(filename), FilenameRegex)))
								.Where((fnm) => fnm.Match.Success)
								.Select(fnm => (Filename: fnm.Filename, Locale: fnm.Match.Groups[1].Value))
								.ToArray();
			
			var deserializer = new Deserializer();

			foreach(var (filename, locale) in localeFiles) {
				var langDict = deserializer.Deserialize<Dictionary<string, Dictionary<string, string>>>(File.ReadAllText(filename));
				localesDict.Add(locale.ToLowerInvariant(), langDict.ToImmutableDictionary(k => k.Key, v => v.Value.ToImmutableDictionary()));
			}

			_responseStrings = localesDict.ToImmutableDictionary();
			sw.Stop();

			log.Info($"Loaded {_responseStrings.Count} locales in {sw.Elapsed.TotalSeconds:F2}s");
		}

		private string GetString(string moduleName, string key, CultureInfo cultureInfo) {
			if(_responseStrings.TryGetValue(cultureInfo.Name.ToLowerInvariant(), out var moduleStrings) && moduleStrings.TryGetValue(moduleName, out var strings)) {
				strings.TryGetValue(key, out var val);
				return val;
			} else {
				UnknownKeyRequested(moduleName, key, cultureInfo);
				return null;
			}
		}

{'fraction_non_alphanumeric': 0.10806385591485879, 'fraction_numerical': 0.0008186655751125665, 'mean_word_length': 5.011070110701107, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5210598', 'n_tokens_mistral': 1560, 'n_tokens_neox': 1356, 'n_words': 335}