<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Xml;
using System.Xml.Linq;
using SIL.Extensions;
using SIL.Keyboarding;
using SIL.Migration;
using SIL.WritingSystems.Migration.WritingSystemsLdmlV0To1Migration;
using SIL.Xml;

namespace SIL.WritingSystems.Migration.WritingSystemsLdmlV2To3Migration
{
	/// <summary>
	/// This class is used to migrate an LdmlFile from LDML palaso version 2 to 3. 
	/// Also note that the files are not written until all writing systems have been migrated in order to deal correctly
	/// with duplicate Ieft Language tags that might result from migration.
	/// </summary>
	internal class LdmlVersion2MigrationStrategy : MigrationStrategyBase
	{
		private readonly List<LdmlMigrationInfo> _migrationInfo;
		private readonly Dictionary<string, WritingSystemDefinitionV3> _writingSystemsV3;
		private readonly Action<int, IEnumerable<LdmlMigrationInfo>> _migrationHandler;
		private readonly IAuditTrail _auditLog;

		private static readonly XNamespace FW = "urn://fieldworks.sil.org/ldmlExtensions/v1";

		public LdmlVersion2MigrationStrategy(Action<int, IEnumerable<LdmlMigrationInfo>> migrationHandler, IAuditTrail auditLog) :
			base(2, 3)
		{
			_migrationInfo = new List<LdmlMigrationInfo>();
			_writingSystemsV3 = new Dictionary<string, WritingSystemDefinitionV3>();
			_migrationHandler = migrationHandler;
			_auditLog = auditLog;
		}

		public override void Migrate(string sourceFilePath, string destinationFilePath)
		{
			string sourceFileName = Path.GetFileName(sourceFilePath);

			var writingSystemDefinitionV1 = new WritingSystemDefinitionV1();
			new LdmlAdaptorV1().Read(sourceFilePath, writingSystemDefinitionV1);

			string abbreviation = writingSystemDefinitionV1.Abbreviation;
			float defaultFontSize = writingSystemDefinitionV1.DefaultFontSize;
			string keyboard = writingSystemDefinitionV1.Keyboard;
			string spellCheckingId = writingSystemDefinitionV1.SpellCheckingId;
			string defaultFontName = writingSystemDefinitionV1.DefaultFontName;
			string languageName = writingSystemDefinitionV1.LanguageName.IsOneOf("Unknown Language", "Language Not Listed") ? string.Empty : writingSystemDefinitionV1.LanguageName;
			string variant, privateUse;
			IetfLanguageTagHelper.SplitVariantAndPrivateUse(writingSystemDefinitionV1.Variant, out variant, out privateUse);
			var langTagCleaner = new IetfLanguageTagCleaner(writingSystemDefinitionV1.Language, writingSystemDefinitionV1.Script, writingSystemDefinitionV1.Region,
				variant, privateUse);
			langTagCleaner.Clean();
			string langTag = IetfLanguageTagHelper.Canonicalize(langTagCleaner.GetCompleteTag());
			bool isGraphiteEnabled = false;
			string legacyMapping = string.Empty;
			string scriptName = string.Empty;
			string regionName = string.Empty;
			string variantName = string.Empty;

			// Migrate fields from legacy fw namespace, and then remove fw namespace
			XElement ldmlElem = XElement.Load(sourceFilePath);
			XElement fwElem = ldmlElem.Elements("special").FirstOrDefault(e => !string.IsNullOrEmpty((string) e.Attribute(XNamespace.Xmlns + "fw")));
			if (fwElem != null)
			{
				XElement graphiteEnabledElem = fwElem.Element(FW + "graphiteEnabled");
				if (graphiteEnabledElem != null)
				{
					if (!bool.TryParse((string) graphiteEnabledElem.Attribute("value"), out isGraphiteEnabled))
						isGraphiteEnabled = false;
				}

				// LegacyMapping
				XElement legacyMappingElem = fwElem.Element(FW + "legacyMapping");
				if (legacyMappingElem != null)
					legacyMapping = (string) legacyMappingElem.Attribute("value");

				// ScriptName
				XElement scriptNameElem = fwElem.Element(FW + "scriptName");
				if (scriptNameElem != null)
					scriptName = (string) scriptNameElem.Attribute("value");

				// RegionName
				XElement regionNameElem = fwElem.Element(FW + "regionName");
				if (regionNameElem != null)
					regionName = (string) regionNameElem.Attribute("value");

				// VariantName
				// Intentionally only checking the first variant
				XElement variantNameElem = fwElem.Element(FW + "variantName");
				if (variantNameElem != null)
					variantName = (string) variantNameElem.Attribute("value");
			}

			// Record the details for use in PostMigrate where we change the file name to match the ieft language tag where we can.
			var migrationInfo = new LdmlMigrationInfo(sourceFileName)
				{
					IetfLanguageTagBeforeMigration = writingSystemDefinitionV1.Bcp47Tag,
					IetfLanguageTagAfterMigration = langTag,
					RemovedPropertiesSetter = ws =>
					{
						if (!string.IsNullOrEmpty(abbreviation))
							ws.Abbreviation = abbreviation;
						if (defaultFontSize != 0)
							ws.DefaultFontSize = defaultFontSize;
						if (!string.IsNullOrEmpty(keyboard))
							ws.Keyboard = keyboard;
						if (!string.IsNullOrEmpty(spellCheckingId))
							ws.SpellCheckingId = spellCheckingId;
						if (!string.IsNullOrEmpty(defaultFontName))
							ws.DefaultFont = ws.Fonts[defaultFontName];
						if (!string.IsNullOrEmpty(languageName))
							ws.Language = new LanguageSubtag(ws.Language, languageName);
						ws.IsGraphiteEnabled = isGraphiteEnabled;
						if (!string.IsNullOrEmpty(legacyMapping))
							ws.LegacyMapping = legacyMapping;
						if (!string.IsNullOrEmpty(scriptName) && ws.Script != null && ws.Script.IsPrivateUse)
							ws.Script = new ScriptSubtag(ws.Script, scriptName);
						if (!string.IsNullOrEmpty(regionName) && ws.Region != null && ws.Region.IsPrivateUse)
							ws.Region = new RegionSubtag(ws.Region, regionName);
						if (!string.IsNullOrEmpty(variantName))
						{
							int index = ws.Variants.IndexOf(v => v.IsPrivateUse);
							if (index > -1)
								ws.Variants[index] = new VariantSubtag(ws.Variants[index], variantName);
						}
					}
				};

			_migrationInfo.Add(migrationInfo);

			// misc properties
			var writingSystemDefinitionV3 = new WritingSystemDefinitionV3
			{
				RightToLeftScript = writingSystemDefinitionV1.RightToLeftScript,
				VersionDescription = writingSystemDefinitionV1.VersionDescription,
				WindowsLcid = writingSystemDefinitionV1.WindowsLcid,
				DateModified = DateTime.Now,
			};

			// font
			FontDefinition fd = null;
			if (!string.IsNullOrEmpty(defaultFontName))
			{
				fd = new FontDefinition(writingSystemDefinitionV1.DefaultFontName);
				writingSystemDefinitionV3.Fonts.Add(fd);
			}

			// known keyboards
			foreach (KeyboardDefinitionV1 kd in writingSystemDefinitionV1.KnownKeyboards)
			{
				string id = string.IsNullOrEmpty(kd.Locale) ? kd.Layout : string.Format("{0}_{1}", kd.Locale, kd.Layout);
				writingSystemDefinitionV3.KnownKeyboards.Add(Keyboard.Controller.CreateKeyboard(id, KeyboardFormat.Unknown, Enumerable.Empty<string>()));
			}

			// Convert sort rules to collation definition of standard type
			CollationDefinition cd = null;
			switch (writingSystemDefinitionV1.SortUsing)
			{
				case WritingSystemDefinitionV1.SortRulesType.CustomSimple:
					cd = new SimpleCollationDefinition("standard") {SimpleRules = writingSystemDefinitionV1.SortRules};
					break;
				case WritingSystemDefinitionV1.SortRulesType.OtherLanguage:
					if (!string.IsNullOrEmpty(writingSystemDefinitionV1.SortRules))
						cd = new IcuCollationDefinition("standard") {Imports = {new IcuCollationImport(writingSystemDefinitionV1.SortRules)}};
					break;
				case WritingSystemDefinitionV1.SortRulesType.CustomICU:
					cd = new IcuCollationDefinition("standard") {IcuRules = writingSystemDefinitionV1.SortRules};
					break;
				case WritingSystemDefinitionV1.SortRulesType.DefaultOrdering:
					cd = new IcuCollationDefinition("standard");
					break;
			}
			if (cd != null)
				writingSystemDefinitionV3.DefaultCollation = cd;

			// IETF language tag<commit_msg>Look for existing keyboards during V2-to-V3 LDML migration<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Xml;
using System.Xml.Linq;
using SIL.Extensions;
using SIL.Keyboarding;
using SIL.Migration;
using SIL.WritingSystems.Migration.WritingSystemsLdmlV0To1Migration;
using SIL.Xml;

namespace SIL.WritingSystems.Migration.WritingSystemsLdmlV2To3Migration
{
	/// <summary>
	/// This class is used to migrate an LdmlFile from LDML palaso version 2 to 3. 
	/// Also note that the files are not written until all writing systems have been migrated in order to deal correctly
	/// with duplicate Ieft Language tags that might result from migration.
	/// </summary>
	internal class LdmlVersion2MigrationStrategy : MigrationStrategyBase
	{
		private readonly List<LdmlMigrationInfo> _migrationInfo;
		private readonly Dictionary<string, WritingSystemDefinitionV3> _writingSystemsV3;
		private readonly Action<int, IEnumerable<LdmlMigrationInfo>> _migrationHandler;
		private readonly IAuditTrail _auditLog;

		private static readonly XNamespace FW = "urn://fieldworks.sil.org/ldmlExtensions/v1";

		public LdmlVersion2MigrationStrategy(Action<int, IEnumerable<LdmlMigrationInfo>> migrationHandler, IAuditTrail auditLog) :
			base(2, 3)
		{
			_migrationInfo = new List<LdmlMigrationInfo>();
			_writingSystemsV3 = new Dictionary<string, WritingSystemDefinitionV3>();
			_migrationHandler = migrationHandler;
			_auditLog = auditLog;
		}

		public override void Migrate(string sourceFilePath, string destinationFilePath)
		{
			string sourceFileName = Path.GetFileName(sourceFilePath);

			var writingSystemDefinitionV1 = new WritingSystemDefinitionV1();
			new LdmlAdaptorV1().Read(sourceFilePath, writingSystemDefinitionV1);

			string abbreviation = writingSystemDefinitionV1.Abbreviation;
			float defaultFontSize = writingSystemDefinitionV1.DefaultFontSize;
			string keyboard = writingSystemDefinitionV1.Keyboard;
			string spellCheckingId = writingSystemDefinitionV1.SpellCheckingId;
			string defaultFontName = writingSystemDefinitionV1.DefaultFontName;
			string languageName = writingSystemDefinitionV1.LanguageName.IsOneOf("Unknown Language", "Language Not Listed") ? string.Empty : writingSystemDefinitionV1.LanguageName;
			string variant, privateUse;
			IetfLanguageTagHelper.SplitVariantAndPrivateUse(writingSystemDefinitionV1.Variant, out variant, out privateUse);
			var langTagCleaner = new IetfLanguageTagCleaner(writingSystemDefinitionV1.Language, writingSystemDefinitionV1.Script, writingSystemDefinitionV1.Region,
				variant, privateUse);
			langTagCleaner.Clean();
			string langTag = IetfLanguageTagHelper.Canonicalize(langTagCleaner.GetCompleteTag());
			bool isGraphiteEnabled = false;
			string legacyMapping = string.Empty;
			string scriptName = string.Empty;
			string regionName = string.Empty;
			string variantName = string.Empty;

			// Migrate fields from legacy fw namespace, and then remove fw namespace
			XElement ldmlElem = XElement.Load(sourceFilePath);
			XElement fwElem = ldmlElem.Elements("special").FirstOrDefault(e => !string.IsNullOrEmpty((string) e.Attribute(XNamespace.Xmlns + "fw")));
			if (fwElem != null)
			{
				XElement graphiteEnabledElem = fwElem.Element(FW + "graphiteEnabled");
				if (graphiteEnabledElem != null)
				{
					if (!bool.TryParse((string) graphiteEnabledElem.Attribute("value"), out isGraphiteEnabled))
						isGraphiteEnabled = false;
				}

				// LegacyMapping
				XElement legacyMappingElem = fwElem.Element(FW + "legacyMapping");
				if (legacyMappingElem != null)
					legacyMapping = (string) legacyMappingElem.Attribute("value");

				// ScriptName
				XElement scriptNameElem = fwElem.Element(FW + "scriptName");
				if (scriptNameElem != null)
					scriptName = (string) scriptNameElem.Attribute("value");

				// RegionName
				XElement regionNameElem = fwElem.Element(FW + "regionName");
				if (regionNameElem != null)
					regionName = (string) regionNameElem.Attribute("value");

				// VariantName
				// Intentionally only checking the first variant
				XElement variantNameElem = fwElem.Element(FW + "variantName");
				if (variantNameElem != null)
					variantName = (string) variantNameElem.Attribute("value");
			}

			// Record the details for use in PostMigrate where we change the file name to match the ieft language tag where we can.
			var migrationInfo = new LdmlMigrationInfo(sourceFileName)
				{
					IetfLanguageTagBeforeMigration = writingSystemDefinitionV1.Bcp47Tag,
					IetfLanguageTagAfterMigration = langTag,
					RemovedPropertiesSetter = ws =>
					{
						if (!string.IsNullOrEmpty(abbreviation))
							ws.Abbreviation = abbreviation;
						if (defaultFontSize != 0)
							ws.DefaultFontSize = defaultFontSize;
						if (!string.IsNullOrEmpty(keyboard))
							ws.Keyboard = keyboard;
						if (!string.IsNullOrEmpty(spellCheckingId))
							ws.SpellCheckingId = spellCheckingId;
						if (!string.IsNullOrEmpty(defaultFontName))
							ws.DefaultFont = ws.Fonts[defaultFontName];
						if (!string.IsNullOrEmpty(languageName))
							ws.Language = new LanguageSubtag(ws.Language, languageName);
						ws.IsGraphiteEnabled = isGraphiteEnabled;
						if (!string.IsNullOrEmpty(legacyMapping))
							ws.LegacyMapping = legacyMapping;
						if (!string.IsNullOrEmpty(scriptName) && ws.Script != null && ws.Script.IsPrivateUse)
							ws.Script = new ScriptSubtag(ws.Script, scriptName);
						if (!string.IsNullOrEmpty(regionName) && ws.Region != null && ws.Region.IsPrivateUse)
							ws.Region = new RegionSubtag(ws.Region, regionName);
						if (!string.IsNullOrEmpty(variantName))
						{
							int index = ws.Variants.IndexOf(v => v.IsPrivateUse);
							if (index > -1)
								ws.Variants[index] = new VariantSubtag(ws.Variants[index], variantName);
						}
					}
				};

			_migrationInfo.Add(migrationInfo);

			// misc properties
			var writingSystemDefinitionV3 = new WritingSystemDefinitionV3
			{
				RightToLeftScript = writingSystemDefinitionV1.RightToLeftScript,
				VersionDescription = writingSystemDefinitionV1.VersionDescription,
				WindowsLcid = writingSystemDefinitionV1.WindowsLcid,
				DateModified = DateTime.Now,
			};

			// font
			FontDefinition fd = null;
			if (!string.IsNullOrEmpty(defaultFontName))
			{
				fd = new FontDefinition(writingSystemDefinitionV1.DefaultFontName);
				writingSystemDefinitionV3.Fonts.Add(fd);
			}

			// known keyboards
			foreach (KeyboardDefinitionV1 keyboardV1 in writingSystemDefinitionV1.KnownKeyboards)
			{
				string id = string.IsNullOrEmpty(keyboardV1.Locale) ? keyboardV1.Layout : string.Format("{0}_{1}", keyboardV1.Locale, keyboardV1.Layout);
				IKeyboardDefinition keyboardV3;
				if (!Keyboard.Controller.TryGetKeyboard(id, out keyboardV3))
					keyboardV3 = Keyboard.Controller.CreateKeyboard(id, KeyboardFormat.Unknown, Enumerable.Empty<string>());
				writingSystemDefinitionV3.KnownKeyboards.Add(keyboardV3);
			}

			// Convert sort rules to collation definition of standard type
			CollationDefinition cd = null;
			switch (writingSystemDefinitionV1.SortUsing)
			{
				case WritingSystemDefinitionV1.SortRulesType.CustomSimple:
					cd = new SimpleCollationDefinition("standard") {SimpleRules = writingSystemDefinitionV1.SortRules};
					break;
				case WritingSystemDefinitionV1.SortRulesType.OtherLanguage:
					if (!string.IsNullOrEmpty(writingSystemDefinitionV1.SortRules))
						cd = new IcuCollationDefinition("standard") {Imports = {new IcuCollationImport(writingSystemDefinitionV1.SortRules)}};
					break;
				case WritingSystemDefinitionV1.SortRulesType.CustomICU:
					cd = new IcuCollationDefinition("standard") {IcuRules = writingSystemDefinitionV1.SortRules};
					break;
				case WritingSystemDefinitionV1.SortRulesType.DefaultOrdering:
					cd = new IcuCollationDefinition("standard");
					break;
			}
			if (cd != null)
				writingSystemDefinitionV3.DefaultCollation = cd;

			// IETF language tag
{'fraction_non_alphanumeric': 0.09089176829268293, 'fraction_numerical': 0.007939532520325204, 'mean_word_length': 4.599217638691323, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6351015', 'n_tokens_mistral': 5424, 'n_tokens_neox': 4503, 'n_words': 962}