<commit_before>﻿// Copyright (c) 2016 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)

using LfMerge.LanguageForge.Model;
using System;
using System.IO;
using System.Linq;
using System.Collections.Generic;
using SIL.FieldWorks.FDO;

namespace LfMerge.DataConverters
{
	public class ConvertMongoToFdoPartsOfSpeech
	{
		private FdoCache _cache;
		private IPartOfSpeechRepository _posRepo;
		private IPartOfSpeechFactory _posFactory;

		// TODO: Should we also pass in an LfProject object? And make the POSConverter responsible for
		// updating the LF part of speech list as needed? Consider, and refactor if the answer is yes.
		public ConvertMongoToFdoPartsOfSpeech(FdoCache fdoCache)
		{
			_cache = fdoCache;
			_posRepo = _cache.ServiceLocator.GetInstance<IPartOfSpeechRepository>();
			_posFactory = _cache.ServiceLocator.GetInstance<IPartOfSpeechFactory>();
		}

		// TODO: Do we need the GOLDEtic.xml parsing code any longer?
		// ... Yes, until LF gains the ability to create new grammar entries *properly*. Until then,
		// we'll have to keep this code around so we can create proper PoSes in FW as needed.
		// Once LF gains that feature, we can remove the whole GOLDEtic parsing code.

		public static Lazy<Stream> GoldEticXml = new Lazy<Stream>(() =>
			typeof(MainClass).Assembly.GetManifestResourceStream(typeof(MainClass), "GOLDEtic.xml")
		);
		public static Lazy<List<GoldEticItem>> GoldEticItems = new Lazy<List<GoldEticItem>>(() =>
			GoldEticXmlParser.ParseXml(GoldEticXml.Value)
		);

		private static IEnumerable<GoldEticItem> FlattenGoldEticItems(IEnumerable<GoldEticItem> topLevel)
		{
			foreach (GoldEticItem item in topLevel)
			{
				yield return item;
				foreach (GoldEticItem subItem in FlattenGoldEticItems(item.Subitems))
					yield return subItem;
			}
		}
		public static Lazy<GoldEticItem[]> FlattenedGoldEticItems = new Lazy<GoldEticItem[]>(() =>
			FlattenGoldEticItems(GoldEticItems.Value).ToArray()
		);
		public static Lazy<Dictionary<string, GoldEticItem>> ItemsByGuidStr = new Lazy<Dictionary<string, GoldEticItem>>(() =>
			FlattenedGoldEticItems.Value.ToDictionary(item => item.Guid, item => item)
		);

		public string NameFromGuid(Guid guid, bool flat=false)
		{
			return NameFromGuidStr(guid.ToString(), flat);
		}

		public string NameFromGuidStr(string guidStr, bool flat=false)
		{
			string result;
			Dictionary<string, string> lookupTable = flat ?
				PartOfSpeechMasterList.FlatPosNames :
				PartOfSpeechMasterList.HierarchicalPosNames;
			if (lookupTable.TryGetValue(guidStr, out result))
				return result;
			return null;
		}

		public IPartOfSpeech FromGuid(Guid guid)
		{
			IPartOfSpeech result;
			if (TryGetPos(guid, out result))
				return result;
			string name = NameFromGuidStr(guid.ToString(), flat:false);
			if (name == null) return null; // If it's not a well-known name, we'll need to create an entry
			return CreateFromWellKnownGuid(guid);
		}

		public IPartOfSpeech FromGuidStr(string guidStr)
		{
			IPartOfSpeech result;
			if (TryGetPos(guidStr, out result))
				return result;
			string name = NameFromGuidStr(guidStr, flat:false);
			if (name == null) return null;
			// We know we have a well-known name, so we should use it. TODO: Or should we?
			Guid guid;
			if (Guid.TryParse(guidStr, out guid))
				return CreateFromWellKnownGuid(guid);
			// Really shouldn't get here at all, but if we do...
			throw new ArgumentException(String.Format("Well-known GUID {0} didn't parse", guidStr), "guidStr");
			// TODO: Turn that into a log message and don't actually throw an exception.
		}

		public IPartOfSpeech FromGuidStrOrName(string guidStr, string name, string userWs = "en", string fallbackWs = "en")
		{
			IPartOfSpeech posFromGuid = FromGuidStr(guidStr);
			if (posFromGuid != null) return posFromGuid;
			return FromAbbrevAndName(name, userWs, fallbackWs);
		}

		public static void SetPartOfSpeech(IMoMorphSynAnalysis msa, IPartOfSpeech pos, IPartOfSpeech secondaryPos = null)
		{
			if (msa == null)
			{
				Console.WriteLine("msa is null!"); // TODO: Turn this into proper log message
				// throw new ArgumentNullException("msa");
				return; // TODO: Or throw an ArgumentNullException?
			}
			if (pos == null)
			{
				Console.WriteLine("pos is null!"); // TODO: Turn this into proper log message
				// throw new ArgumentNullException("pos");
				return; // TODO: Or throw an ArgumentNullException?
			}
			//Console.WriteLine("Setting part of speech {0} ({1}) in msa {2}", pos.NameHierarchyString, pos.Guid, msa.Guid);
			// TODO: Is the below switch statement REALLY complete? Or do we need to do more?
			// See FdoFactoryAdditions.cs, lines 1603-1698: perhaps we should be using factories and SandboxMSA objects?
			switch (msa.ClassID)
			{
			case MoDerivAffMsaTags.kClassId:
				((IMoDerivAffMsa)msa).FromPartOfSpeechRA = pos;
				if (secondaryPos != null)
					((IMoDerivAffMsa)msa).ToPartOfSpeechRA = secondaryPos;
				break;
			case MoDerivStepMsaTags.kClassId:
				((IMoDerivStepMsa)msa).PartOfSpeechRA = pos;
				break;
			case MoInflAffMsaTags.kClassId:
				((IMoInflAffMsa)msa).PartOfSpeechRA = pos;
				break;
			case MoStemMsaTags.kClassId:
				((IMoStemMsa)msa).PartOfSpeechRA = pos;
				break;
			case MoUnclassifiedAffixMsaTags.kClassId:
				((IMoUnclassifiedAffixMsa)msa).PartOfSpeechRA = pos;
				break;
			default:
				// TODO: Make this a log message, not Console.WriteLine
				Console.WriteLine("Got MSA of unknown type {0}", msa.GetType().Name);
				return;
			}
		}

		private GoldEticItem FindGoldEticItem(string searchTerm, string wsToSearch)
		{
			foreach (GoldEticItem item in FlattenedGoldEticItems.Value)
			{
				if (item.ORCDelimitedAbbrevByWs(wsToSearch) == searchTerm || item.AbbrevByWs(wsToSearch) == searchTerm)
					return item;
				if (item.ORCDelimitedNameByWs(wsToSearch) == searchTerm || item.NameByWs(wsToSearch) == searchTerm)
					return item;
			}
			return null;
		}

		private void PopulateWellKnownPos(IPartOfSpeech pos, GoldEticItem item)
		// Creating, assigning parent, etc., is handled elsewhere. This function only sets text values.
		{
			foreach (var kv in item.Abbrevs)
			{
				int wsId = _cache.WritingSystemFactory.GetWsFromStr(kv.Key);
				if (wsId != 0)
					pos.Abbreviation.set_String(wsId, kv.Value);
			}
			foreach (var kv in item.Definitions)
			{
				int wsId = _cache.WritingSystemFactory.GetWsFromStr(kv.Key);
				if (wsId != 0)
					pos.Description.set_String(wsId, kv.Value);
			}
			foreach (var kv in item.Terms)
			{
				int wsId = _cache.WritingSystemFactory.GetWsFromStr(kv.Key);
				if (wsId != 0)
					pos.Name.set_String(wsId, kv.Value);
			}
		}

		private void PopulateCustomPos(IPartOfSpeech pos, string finalName, string userWs)
		{
			int wsId = _cache.WritingSystemFactory.GetWsFromStr(userWs);
			pos.Name.set_String(wsId, finalName);
		}

		private bool TryGetPos(Guid guid, out IPartOfSpeech result)
		{
			return _posRepo.TryGetObject(guid, out result);
		}

		private bool TryGetPos(string guidStr, out IPartOfSpeech result)
		{
			Guid guid = Guid.Empty;
			if (!Guid.TryParse(guidStr, out guid) || guid == Guid.Empty)<commit_msg>Removed more unused code<commit_after>﻿// Copyright (c) 2016 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)

using LfMerge.LanguageForge.Model;
using System;
using System.IO;
using System.Linq;
using System.Collections.Generic;
using SIL.FieldWorks.FDO;

namespace LfMerge.DataConverters
{
	public class ConvertMongoToFdoPartsOfSpeech
	{
		private FdoCache _cache;
		private IPartOfSpeechRepository _posRepo;
		private IPartOfSpeechFactory _posFactory;

		// TODO: Should we also pass in an LfProject object? And make the POSConverter responsible for
		// updating the LF part of speech list as needed? Consider, and refactor if the answer is yes.
		public ConvertMongoToFdoPartsOfSpeech(FdoCache fdoCache)
		{
			_cache = fdoCache;
			_posRepo = _cache.ServiceLocator.GetInstance<IPartOfSpeechRepository>();
			_posFactory = _cache.ServiceLocator.GetInstance<IPartOfSpeechFactory>();
		}

		// TODO: Do we need the GOLDEtic.xml parsing code any longer?
		// ... Yes, until LF gains the ability to create new grammar entries *properly*. Until then,
		// we'll have to keep this code around so we can create proper PoSes in FW as needed.
		// Once LF gains that feature, we can remove the whole GOLDEtic parsing code.

		public static Lazy<Stream> GoldEticXml = new Lazy<Stream>(() =>
			typeof(MainClass).Assembly.GetManifestResourceStream(typeof(MainClass), "GOLDEtic.xml")
		);
		public static Lazy<List<GoldEticItem>> GoldEticItems = new Lazy<List<GoldEticItem>>(() =>
			GoldEticXmlParser.ParseXml(GoldEticXml.Value)
		);

		private static IEnumerable<GoldEticItem> FlattenGoldEticItems(IEnumerable<GoldEticItem> topLevel)
		{
			foreach (GoldEticItem item in topLevel)
			{
				yield return item;
				foreach (GoldEticItem subItem in FlattenGoldEticItems(item.Subitems))
					yield return subItem;
			}
		}
		public static Lazy<GoldEticItem[]> FlattenedGoldEticItems = new Lazy<GoldEticItem[]>(() =>
			FlattenGoldEticItems(GoldEticItems.Value).ToArray()
		);
		public static Lazy<Dictionary<string, GoldEticItem>> ItemsByGuidStr = new Lazy<Dictionary<string, GoldEticItem>>(() =>
			FlattenedGoldEticItems.Value.ToDictionary(item => item.Guid, item => item)
		);

		public static void SetPartOfSpeech(IMoMorphSynAnalysis msa, IPartOfSpeech pos, IPartOfSpeech secondaryPos = null)
		{
			if (msa == null)
			{
				Console.WriteLine("msa is null!"); // TODO: Turn this into proper log message
				// throw new ArgumentNullException("msa");
				return; // TODO: Or throw an ArgumentNullException?
			}
			if (pos == null)
			{
				Console.WriteLine("pos is null!"); // TODO: Turn this into proper log message
				// throw new ArgumentNullException("pos");
				return; // TODO: Or throw an ArgumentNullException?
			}
			//Console.WriteLine("Setting part of speech {0} ({1}) in msa {2}", pos.NameHierarchyString, pos.Guid, msa.Guid);
			// TODO: Is the below switch statement REALLY complete? Or do we need to do more?
			// See FdoFactoryAdditions.cs, lines 1603-1698: perhaps we should be using factories and SandboxMSA objects?
			switch (msa.ClassID)
			{
			case MoDerivAffMsaTags.kClassId:
				((IMoDerivAffMsa)msa).FromPartOfSpeechRA = pos;
				if (secondaryPos != null)
					((IMoDerivAffMsa)msa).ToPartOfSpeechRA = secondaryPos;
				break;
			case MoDerivStepMsaTags.kClassId:
				((IMoDerivStepMsa)msa).PartOfSpeechRA = pos;
				break;
			case MoInflAffMsaTags.kClassId:
				((IMoInflAffMsa)msa).PartOfSpeechRA = pos;
				break;
			case MoStemMsaTags.kClassId:
				((IMoStemMsa)msa).PartOfSpeechRA = pos;
				break;
			case MoUnclassifiedAffixMsaTags.kClassId:
				((IMoUnclassifiedAffixMsa)msa).PartOfSpeechRA = pos;
				break;
			default:
				// TODO: Make this a log message, not Console.WriteLine
				Console.WriteLine("Got MSA of unknown type {0}", msa.GetType().Name);
				return;
			}
		}

		private GoldEticItem FindGoldEticItem(string searchTerm, string wsToSearch)
		{
			foreach (GoldEticItem item in FlattenedGoldEticItems.Value)
			{
				if (item.ORCDelimitedAbbrevByWs(wsToSearch) == searchTerm || item.AbbrevByWs(wsToSearch) == searchTerm)
					return item;
				if (item.ORCDelimitedNameByWs(wsToSearch) == searchTerm || item.NameByWs(wsToSearch) == searchTerm)
					return item;
			}
			return null;
		}

		private void PopulateWellKnownPos(IPartOfSpeech pos, GoldEticItem item)
		// Creating, assigning parent, etc., is handled elsewhere. This function only sets text values.
		{
			foreach (var kv in item.Abbrevs)
			{
				int wsId = _cache.WritingSystemFactory.GetWsFromStr(kv.Key);
				if (wsId != 0)
					pos.Abbreviation.set_String(wsId, kv.Value);
			}
			foreach (var kv in item.Definitions)
			{
				int wsId = _cache.WritingSystemFactory.GetWsFromStr(kv.Key);
				if (wsId != 0)
					pos.Description.set_String(wsId, kv.Value);
			}
			foreach (var kv in item.Terms)
			{
				int wsId = _cache.WritingSystemFactory.GetWsFromStr(kv.Key);
				if (wsId != 0)
					pos.Name.set_String(wsId, kv.Value);
			}
		}

		private bool TryGetPos(string guidStr, out IPartOfSpeech result)
		{
			Guid guid = Guid.Empty;
			if (!Guid.TryParse(guidStr, out guid) || guid == Guid.Empty)
{'fraction_non_alphanumeric': 0.10146570572515994, 'fraction_numerical': 0.0031581504575269254, 'mean_word_length': 4.053191489361702, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6344235', 'n_tokens_mistral': 4808, 'n_tokens_neox': 4028, 'n_words': 1063}