<commit_before>﻿// Copyright (c) 2015 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
using System;
using System.IO;
using System.Collections.Generic;
using SIL.FieldWorks.FDO;

namespace LfMerge
{
	public class PartOfSpeechConverter
	{
		public FdoCache cache;
		private IPartOfSpeechRepository posRepo;
		private IPartOfSpeechFactory posFactory;

		public PartOfSpeechConverter(FdoCache fdoCache)
		{
			cache = fdoCache;
			posRepo = cache.ServiceLocator.GetInstance<IPartOfSpeechRepository>();
			posFactory = cache.ServiceLocator.GetInstance<IPartOfSpeechFactory>();
		}

		public static Lazy<Stream> GoldEticXml = new Lazy<Stream>(() =>
			typeof(PartOfSpeechConverter).Assembly.GetManifestResourceStream(typeof(PartOfSpeechConverter), "GOLDEtic.xml")
		);
		public static Lazy<List<GoldEticItem>> GoldEticItems = new Lazy<List<GoldEticItem>>(() =>
			GoldEticXmlParser.ParseXml(GoldEticXml.Value)
		);

		private static IEnumerable<GoldEticItem> FlattenGoldEticItems(List<GoldEticItem> topLevel)
		{
			foreach (GoldEticItem item in GoldEticItems.Value)
			{
				yield return item;
				foreach (GoldEticItem subItem in FlattenGoldEticItems(item.Subitems))
					yield return subItem;
			}
		}
		public static Lazy<IEnumerable<GoldEticItem>> FlattenedGoldEticItems = new Lazy<IEnumerable<GoldEticItem>>(() =>
			FlattenGoldEticItems(GoldEticItems.Value)
		);

		public static string FromGuid(Guid guid, bool flat=false)
		{
			return FromGuid(guid.ToString(), flat);
		}

		public static string FromGuid(string guid, bool flat=false)
		{
			string result;
			Dictionary<string, string> lookupTable = (flat) ?
				PartOfSpeechMasterList.FlatPoSGuids :
				PartOfSpeechMasterList.HierarchicalPoSGuids;
			if (lookupTable.TryGetValue(guid, out result))
				return result;
			// This GUID not found? Hmmm, try something else maybe.
			// TODO: Implement "get name instead" code. Maybe in a different function, though.
			// TODO: Actually, just fold this function into the other one since this won't ever need to be called separately.
			return null;
		}

		public static IPartOfSpeech FromMSA(IMoMorphSynAnalysis msa)
		{
			switch (msa.GetType().Name)
			{
			case "MoDerivAffMsa":
				// TODO: Turn this into a log message, and try to make the log message a little clearer to non-linguists, if possible.
				Console.WriteLine("For derivational affix {0}, arbitrarily picking \"To\" part of speech instead of the \"From\" part of speech.", msa.GetGlossOfFirstSense());
				return ((IMoDerivAffMsa)msa).ToPartOfSpeechRA;
			case "MoDerivStepMsa":
				return ((IMoDerivStepMsa)msa).PartOfSpeechRA;
			case "MoInflAffMsa":
				return ((IMoInflAffMsa)msa).PartOfSpeechRA;
			case "MoStemMsa":
				return ((IMoStemMsa)msa).PartOfSpeechRA;
			case "MoUnclassifiedAffixMsa":
				return ((IMoUnclassifiedAffixMsa)msa).PartOfSpeechRA;
			default:
				// TODO: Make this a log message, not Console.WriteLine
				Console.WriteLine("Got MSA of unknown type {0}", msa.GetType().Name);
				return null;
			}
		}

		public static void SetPartOfSpeech(IMoMorphSynAnalysis msa, IPartOfSpeech pos)
		{
			switch (msa.GetType().Name)
			{
			case "MoDerivAffMsa":
				// TODO: Turn this into a log message, and try to make the log message a little clearer to non-linguists, if possible.
				Console.WriteLine("For derivational affix {0}, arbitrarily picking \"To\" part of speech instead of the \"From\" part of speech.", msa.GetGlossOfFirstSense());
				((IMoDerivAffMsa)msa).ToPartOfSpeechRA = pos;
				break;
			case "MoDerivStepMsa":
				((IMoDerivStepMsa)msa).PartOfSpeechRA = pos;
				break;
			case "MoInflAffMsa":
				((IMoInflAffMsa)msa).PartOfSpeechRA = pos;
				break;
			case "MoStemMsa":
				((IMoStemMsa)msa).PartOfSpeechRA = pos;
				break;
			case "MoUnclassifiedAffixMsa":
				((IMoUnclassifiedAffixMsa)msa).PartOfSpeechRA = pos;
				break;
			default:
				// TODO: Make this a log message, not Console.WriteLine
				Console.WriteLine("Got MSA of unknown type {0}", msa.GetType().Name);
				return;
			}
		}

		public IPartOfSpeech FromName(string name, string wsToSearch = "en")
		{
			string guidStr;
			Guid guid = Guid.Empty;
			if (PartOfSpeechMasterList.HierarchicalPoSGuids.TryGetValue(name, out guidStr))
			{
			}
			else if (PartOfSpeechMasterList.FlatPoSGuids.TryGetValue(name, out guidStr))
			{
			}
			else
			{
				foreach (GoldEticItem item in FlattenedGoldEticItems.Value)
				{
					if (item.ORCDelimitedNameByWs(wsToSearch) == name || item.NameByWs(wsToSearch) == name)
					{
						guidStr = item.Guid;
						break;
					}
				}
			}
			if (guidStr != null)
				Guid.TryParse(guidStr, out guid);
			if (guid != Guid.Empty)
			{
				IPartOfSpeech result;
				if (posRepo.TryGetObject(guid, out result))
					return result;
				// Not found? Fall through to creation.
			}
			// Whether or not we have an "official" GUID, we can now create a PartOfSpeech
			return posFactory.Create(guid, cache.LanguageProject.PartsOfSpeechOA);
		}
	}
}<commit_msg>When searching GOLDEtic.xml, fall back to English<commit_after>﻿// Copyright (c) 2015 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
using System;
using System.IO;
using System.Collections.Generic;
using SIL.FieldWorks.FDO;

namespace LfMerge
{
	public class PartOfSpeechConverter
	{
		public FdoCache cache;
		private IPartOfSpeechRepository posRepo;
		private IPartOfSpeechFactory posFactory;

		public PartOfSpeechConverter(FdoCache fdoCache)
		{
			cache = fdoCache;
			posRepo = cache.ServiceLocator.GetInstance<IPartOfSpeechRepository>();
			posFactory = cache.ServiceLocator.GetInstance<IPartOfSpeechFactory>();
		}

		public static Lazy<Stream> GoldEticXml = new Lazy<Stream>(() =>
			typeof(PartOfSpeechConverter).Assembly.GetManifestResourceStream(typeof(PartOfSpeechConverter), "GOLDEtic.xml")
		);
		public static Lazy<List<GoldEticItem>> GoldEticItems = new Lazy<List<GoldEticItem>>(() =>
			GoldEticXmlParser.ParseXml(GoldEticXml.Value)
		);

		private static IEnumerable<GoldEticItem> FlattenGoldEticItems(List<GoldEticItem> topLevel)
		{
			foreach (GoldEticItem item in GoldEticItems.Value)
			{
				yield return item;
				foreach (GoldEticItem subItem in FlattenGoldEticItems(item.Subitems))
					yield return subItem;
			}
		}
		public static Lazy<IEnumerable<GoldEticItem>> FlattenedGoldEticItems = new Lazy<IEnumerable<GoldEticItem>>(() =>
			FlattenGoldEticItems(GoldEticItems.Value)
		);

		public static string FromGuid(Guid guid, bool flat=false)
		{
			return FromGuid(guid.ToString(), flat);
		}

		public static string FromGuid(string guid, bool flat=false)
		{
			string result;
			Dictionary<string, string> lookupTable = (flat) ?
				PartOfSpeechMasterList.FlatPoSGuids :
				PartOfSpeechMasterList.HierarchicalPoSGuids;
			if (lookupTable.TryGetValue(guid, out result))
				return result;
			// This GUID not found? Hmmm, try something else maybe.
			// TODO: Implement "get name instead" code. Maybe in a different function, though.
			// TODO: Actually, just fold this function into the other one since this won't ever need to be called separately.
			return null;
		}

		public static IPartOfSpeech FromMSA(IMoMorphSynAnalysis msa)
		{
			switch (msa.GetType().Name)
			{
			case "MoDerivAffMsa":
				// TODO: Turn this into a log message, and try to make the log message a little clearer to non-linguists, if possible.
				Console.WriteLine("For derivational affix {0}, arbitrarily picking \"To\" part of speech instead of the \"From\" part of speech.", msa.GetGlossOfFirstSense());
				return ((IMoDerivAffMsa)msa).ToPartOfSpeechRA;
			case "MoDerivStepMsa":
				return ((IMoDerivStepMsa)msa).PartOfSpeechRA;
			case "MoInflAffMsa":
				return ((IMoInflAffMsa)msa).PartOfSpeechRA;
			case "MoStemMsa":
				return ((IMoStemMsa)msa).PartOfSpeechRA;
			case "MoUnclassifiedAffixMsa":
				return ((IMoUnclassifiedAffixMsa)msa).PartOfSpeechRA;
			default:
				// TODO: Make this a log message, not Console.WriteLine
				Console.WriteLine("Got MSA of unknown type {0}", msa.GetType().Name);
				return null;
			}
		}

		public static void SetPartOfSpeech(IMoMorphSynAnalysis msa, IPartOfSpeech pos)
		{
			switch (msa.GetType().Name)
			{
			case "MoDerivAffMsa":
				// TODO: Turn this into a log message, and try to make the log message a little clearer to non-linguists, if possible.
				Console.WriteLine("For derivational affix {0}, arbitrarily picking \"To\" part of speech instead of the \"From\" part of speech.", msa.GetGlossOfFirstSense());
				((IMoDerivAffMsa)msa).ToPartOfSpeechRA = pos;
				break;
			case "MoDerivStepMsa":
				((IMoDerivStepMsa)msa).PartOfSpeechRA = pos;
				break;
			case "MoInflAffMsa":
				((IMoInflAffMsa)msa).PartOfSpeechRA = pos;
				break;
			case "MoStemMsa":
				((IMoStemMsa)msa).PartOfSpeechRA = pos;
				break;
			case "MoUnclassifiedAffixMsa":
				((IMoUnclassifiedAffixMsa)msa).PartOfSpeechRA = pos;
				break;
			default:
				// TODO: Make this a log message, not Console.WriteLine
				Console.WriteLine("Got MSA of unknown type {0}", msa.GetType().Name);
				return;
			}
		}

		private string FindGuidInGoldEtic(string searchTerm, string wsToSearch)
		{
			foreach (GoldEticItem item in FlattenedGoldEticItems.Value)
				if (item.ORCDelimitedNameByWs(wsToSearch) == searchTerm || item.NameByWs(wsToSearch) == searchTerm)
					return item.Guid;
		}

		public IPartOfSpeech FromName(string name, string wsToSearch = "en", string fallbackWs = "en")
		{
			string guidStr;
			Guid guid = Guid.Empty;
			PartOfSpeechMasterList.HierarchicalPoSGuids.TryGetValue(name, out guidStr);
			if (guidStr == null)
				PartOfSpeechMasterList.FlatPoSGuids.TryGetValue(name, out guidStr);
			if (guidStr == null)
				FindGuidInGoldEtic(name, wsToSearch);
			if (guidStr == null)
				FindGuidInGoldEtic(name, fallbackWs);
			if (guidStr != null)
				Guid.TryParse(guidStr, out guid);
			if (guid != Guid.Empty)
			{
				IPartOfSpeech result;
				if (posRepo.TryGetObject(guid, out result))
					return result;
				// Not found? Fall through to creation.
			}
			// Whether or not we have an "official" GUID, we can now create a PartOfSpeech
			return posFactory.Create(guid, cache.LanguageProject.PartsOfSpeechOA);
		}
	}
}
{'fraction_non_alphanumeric': 0.10233690360272639, 'fraction_numerical': 0.001557935735150925, 'mean_word_length': 4.117588440458396, 'pattern_counts': {'":': 20, '<': 33, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5541529', 'n_tokens_mistral': 4065, 'n_tokens_neox': 3408, 'n_words': 845}