<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Glyssen.Character;
using SIL.Xml;
using System.Text.RegularExpressions;

namespace DevTools.TermTranslator
{
	public class Processor
	{
		private static readonly List<string> LanguagesToProcess = new List<string>{"En", "Es", "Fr", "Pt", "zh-Hans", "zh-Hant"}; 

		public static void Process()
		{
			IEnumerable<Glyssen.Character.CharacterVerse> quoteInfo = ControlCharacterVerseData.Singleton.GetAllQuoteInfo();

			SortedSet<string> names = new SortedSet<string>();
			names.UnionWith(quoteInfo.Select(t => t.NonlocalizedAlias));
			names.UnionWith(quoteInfo.Select(t => t.Character));
			names.Remove("");
			names.Remove(null);

			BiblicalTermsLocalizations englishTermsList = XmlSerializationHelper.DeserializeFromFile<BiblicalTermsLocalizations>("..\\..\\Resources\\BiblicalTermsEn.xml");
			
			foreach (string langAbbr in LanguagesToProcess)
			{
				BiblicalTermsLocalizations localTermsList = XmlSerializationHelper.DeserializeFromFile<BiblicalTermsLocalizations>("..\\..\\Resources\\BiblicalTerms" + langAbbr + ".xml");

				string modifiedLangAbbr = Char.ToLowerInvariant(langAbbr[0]) + langAbbr.Substring(1);

				string outputFileName = "..\\..\\..\\DistFiles\\localization\\Glyssen." + modifiedLangAbbr + ".tmx";

				TmxFormat newTmx;
				if (File.Exists(outputFileName))
				{
					newTmx = XmlSerializationHelper.DeserializeFromFile<TmxFormat>(outputFileName);

					var tus = newTmx.Body.Tus;

					tus.RemoveAll(t => t.Tuid.StartsWith("CharacterName."));
				}<commit_msg>Fix problem introduced with previous commit (PG-206)<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Glyssen.Character;
using SIL.Xml;
using System.Text.RegularExpressions;

namespace DevTools.TermTranslator
{
	public class Processor
	{
		private static readonly List<string> LanguagesToProcess = new List<string>{"En", "Es", "Fr", "Pt", "zh-Hans", "zh-Hant"}; 

		public static void Process()
		{
			IEnumerable<Glyssen.Character.CharacterVerse> quoteInfo = ControlCharacterVerseData.Singleton.GetAllQuoteInfo();

			SortedSet<string> names = new SortedSet<string>();
			names.UnionWith(quoteInfo.Select(t => t.Alias));
			names.UnionWith(quoteInfo.Select(t => t.Character));
			names.Remove("");
			names.Remove(null);

			BiblicalTermsLocalizations englishTermsList = XmlSerializationHelper.DeserializeFromFile<BiblicalTermsLocalizations>("..\\..\\Resources\\BiblicalTermsEn.xml");
			
			foreach (string langAbbr in LanguagesToProcess)
			{
				BiblicalTermsLocalizations localTermsList = XmlSerializationHelper.DeserializeFromFile<BiblicalTermsLocalizations>("..\\..\\Resources\\BiblicalTerms" + langAbbr + ".xml");

				string modifiedLangAbbr = Char.ToLowerInvariant(langAbbr[0]) + langAbbr.Substring(1);

				string outputFileName = "..\\..\\..\\DistFiles\\localization\\Glyssen." + modifiedLangAbbr + ".tmx";

				TmxFormat newTmx;
				if (File.Exists(outputFileName))
				{
					newTmx = XmlSerializationHelper.DeserializeFromFile<TmxFormat>(outputFileName);

					var tus = newTmx.Body.Tus;

					tus.RemoveAll(t => t.Tuid.StartsWith("CharacterName."));
				}
{'fraction_non_alphanumeric': 0.13488372093023257, 'fraction_numerical': 0.0021705426356589145, 'mean_word_length': 4.886861313868613, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2474336', 'n_tokens_mistral': 1144, 'n_tokens_neox': 1005, 'n_words': 168}