<commit_before>﻿// Copyright (c) 2016 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
using SIL.FieldWorks.Common.COMInterfaces;
using SIL.CoreImpl;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace LfMerge.Core.DataConverters
{
	public class ConvertFdoToMongoTsStrings
	{
		public static string HtmlEncode(string decoded)
		{
			// System.Net.WebUtility.HtmlEncode and HtmlDecode are over-zealous (we do NOT want non-Roman characters
			// encoded, for example). So we have to write our own. Thankfully, it isn't hard at all.
			if (decoded == null)
				return null;
			return decoded.Replace("&", "&amp;").Replace("<", "&lt;").Replace(">", "&gt;");
		}

		public static string HexEncode(string decoded)
		{
			// System.Net.WebUtility.HtmlEncode and HtmlDecode are ALSO under-zealous! In the case of ObjData properties,
			// they can (and often do) contain null bytes and ASCII control characters (U+0000 through U+001F), which HtmlEncode
			// and HtmlDecode don't escape. So for ObjData, we convert it to hex and store it that way.

			char[] chars = decoded.ToCharArray();
			StringBuilder result = new StringBuilder(chars.Length * 4);
			foreach (char c in chars)
			{
				result.Append(((UInt16)c).ToString("X4"));
			}
			return result.ToString();
		}

		public static string TextFromTsString(ITsString tss, ILgWritingSystemFactory wsf)
		{
			// This will replace SafeTsStringText, and will actually deal with <span> elements

			// Spec: the TsString properties ktptWs (an int property) and ktptNamedStyle (a string property)
			// will be replaced with lang="(wsStr)" and class="styleName_PropValue". Other properties, that
			// LF doesn't process, will be preserved in the class attribute as well, as follows:
			//
			// 1) Determine whether the property is an int property or a string property.
			// 2) Call either IntPropertyName() or StringPropertyName() to get its name.
			// 3) Create a class string in format "propC_N_NAME_VALUE" where:
			//      - C is "i" for int props or "s" for string props
			//      - N is the property number
			//      - NAME is, of course, the property name
			//      - VALUE is the property's value, with two special cases:
			//          * in string properties, spaces are first replaced by "_SPACE_"
			//          * in int properties, there's a value and a "variation", and VALUE is both of them separated by _
			//
			// Result: class="propi_1_ktptWs_1_0 props_1_ktptFontFamily_Times_SPACE_New_SPACE_Roman"

			if (tss == null)
				return null;
			int[] intPropsToSkip = new int[] { (int)FwTextPropType.ktptWs };
			int[] strPropsToSkip = new int[] { (int)FwTextPropType.ktptNamedStyle };
			int mainWs = tss.get_WritingSystem(0);
			var resultBuilder = new StringBuilder();
			for (int i = 0, n = tss.RunCount; i < n; i++)
			{
				string runText = HtmlEncode(tss.get_RunText(i));
				ITsTextProps props = tss.get_Properties(i);
				// int ignored;
				// int ws = props.GetIntPropValues((int)FwTextPropType.ktptWs, out ignored);
				int ws = tss.get_WritingSystem(i);  // Simpler than GetIntPropValues((int)FwTextPropType.ktptWs)
				string namedStyle = props.GetStrPropValue((int)FwTextPropType.ktptNamedStyle);
				List<string> classes = ClassesFromTsTextProps(props, intPropsToSkip, strPropsToSkip);

				bool needSpan = false;
				string langAttr = null;
				string classAttr = null;
				if (ws != mainWs)
				{
					needSpan = true;
					langAttr = String.Format(" lang=\"{0}\"", wsf.GetStrFromWs(ws));
				}
				if (namedStyle != null)
				{
					needSpan = true;
					classes.Insert(0, String.Format("styleName_{0}", namedStyle.Replace(" ", "_SPACE_")));
				}
				if (classes.Count > 0)
				{
					needSpan = true;
					classAttr = String.Format(" class=\"{0}\"", String.Join(" ", classes));
				}

				if (needSpan)
				{
					var spanBuilder = new StringBuilder();
					spanBuilder.Append("<span");
					if (langAttr != null)
						spanBuilder.Append(langAttr);
					if (classAttr != null)
						spanBuilder.Append(classAttr);
					spanBuilder.Append(">");
					spanBuilder.Append(runText);
					spanBuilder.Append("</span>");
					resultBuilder.Append(spanBuilder.ToString());
				}
				else
					resultBuilder.Append(runText);
			}
			string result = resultBuilder.ToString();
			if (String.IsNullOrEmpty(result))
				result = null; // We prefer nulls rather than empty strings
			return result.Normalize(NormalizationForm.FormC);
		}

		public static List<string> ClassesFromTsTextProps(ITsTextProps props, int[] intPropsToSkip, int[] strPropsToSkip)
		{
			var classes = new List<string>();
			for (int i = 0, n = props.IntPropCount; i < n; i++)
			{
				int propNum;
				int variation;
				int propValue = props.GetIntProp(i, out propNum, out variation);
				if (intPropsToSkip.Contains(propNum))
					continue;
				string className = String.Format("propi_{0}_{1}_{2}_{3}", propNum, IntPropertyName(propNum), propValue, variation);
				classes.Add(className);
			}
			for (int i = 0, n = props.StrPropCount; i < n; i++)
			{
				int propNum;
				string propValue = props.GetStrProp(i, out propNum);
				if (propNum == (int)FwTextStringProp.kstpObjData)
				{
					// Object data can have arbitrary bytes, including null.
					propValue = HexEncode(propValue);
				}
				// In any other property type, the only problematic value is a space character.
				propValue = propValue.Replace(" ", "_SPACE_");
				string className = String.Format("props_{0}_{1}_{2}", propNum, StringPropertyName(propNum), propValue);<commit_msg>Fix NullReferenceException on empty TsStrings<commit_after>﻿// Copyright (c) 2016 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
using SIL.FieldWorks.Common.COMInterfaces;
using SIL.CoreImpl;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace LfMerge.Core.DataConverters
{
	public class ConvertFdoToMongoTsStrings
	{
		public static string HtmlEncode(string decoded)
		{
			// System.Net.WebUtility.HtmlEncode and HtmlDecode are over-zealous (we do NOT want non-Roman characters
			// encoded, for example). So we have to write our own. Thankfully, it isn't hard at all.
			if (decoded == null)
				return null;
			return decoded.Replace("&", "&amp;").Replace("<", "&lt;").Replace(">", "&gt;");
		}

		public static string HexEncode(string decoded)
		{
			// System.Net.WebUtility.HtmlEncode and HtmlDecode are ALSO under-zealous! In the case of ObjData properties,
			// they can (and often do) contain null bytes and ASCII control characters (U+0000 through U+001F), which HtmlEncode
			// and HtmlDecode don't escape. So for ObjData, we convert it to hex and store it that way.

			char[] chars = decoded.ToCharArray();
			StringBuilder result = new StringBuilder(chars.Length * 4);
			foreach (char c in chars)
			{
				result.Append(((UInt16)c).ToString("X4"));
			}
			return result.ToString();
		}

		public static string TextFromTsString(ITsString tss, ILgWritingSystemFactory wsf)
		{
			// This will replace SafeTsStringText, and will actually deal with <span> elements

			// Spec: the TsString properties ktptWs (an int property) and ktptNamedStyle (a string property)
			// will be replaced with lang="(wsStr)" and class="styleName_PropValue". Other properties, that
			// LF doesn't process, will be preserved in the class attribute as well, as follows:
			//
			// 1) Determine whether the property is an int property or a string property.
			// 2) Call either IntPropertyName() or StringPropertyName() to get its name.
			// 3) Create a class string in format "propC_N_NAME_VALUE" where:
			//      - C is "i" for int props or "s" for string props
			//      - N is the property number
			//      - NAME is, of course, the property name
			//      - VALUE is the property's value, with two special cases:
			//          * in string properties, spaces are first replaced by "_SPACE_"
			//          * in int properties, there's a value and a "variation", and VALUE is both of them separated by _
			//
			// Result: class="propi_1_ktptWs_1_0 props_1_ktptFontFamily_Times_SPACE_New_SPACE_Roman"

			if (tss == null)
				return null;
			int[] intPropsToSkip = new int[] { (int)FwTextPropType.ktptWs };
			int[] strPropsToSkip = new int[] { (int)FwTextPropType.ktptNamedStyle };
			int mainWs = tss.get_WritingSystem(0);
			var resultBuilder = new StringBuilder();
			for (int i = 0, n = tss.RunCount; i < n; i++)
			{
				string runText = HtmlEncode(tss.get_RunText(i));
				ITsTextProps props = tss.get_Properties(i);
				// int ignored;
				// int ws = props.GetIntPropValues((int)FwTextPropType.ktptWs, out ignored);
				int ws = tss.get_WritingSystem(i);  // Simpler than GetIntPropValues((int)FwTextPropType.ktptWs)
				string namedStyle = props.GetStrPropValue((int)FwTextPropType.ktptNamedStyle);
				List<string> classes = ClassesFromTsTextProps(props, intPropsToSkip, strPropsToSkip);

				bool needSpan = false;
				string langAttr = null;
				string classAttr = null;
				if (ws != mainWs)
				{
					needSpan = true;
					langAttr = String.Format(" lang=\"{0}\"", wsf.GetStrFromWs(ws));
				}
				if (namedStyle != null)
				{
					needSpan = true;
					classes.Insert(0, String.Format("styleName_{0}", namedStyle.Replace(" ", "_SPACE_")));
				}
				if (classes.Count > 0)
				{
					needSpan = true;
					classAttr = String.Format(" class=\"{0}\"", String.Join(" ", classes));
				}

				if (needSpan)
				{
					var spanBuilder = new StringBuilder();
					spanBuilder.Append("<span");
					if (langAttr != null)
						spanBuilder.Append(langAttr);
					if (classAttr != null)
						spanBuilder.Append(classAttr);
					spanBuilder.Append(">");
					spanBuilder.Append(runText);
					spanBuilder.Append("</span>");
					resultBuilder.Append(spanBuilder.ToString());
				}
				else
					resultBuilder.Append(runText);
			}
			string result = resultBuilder.ToString();
			if (String.IsNullOrEmpty(result))
			{
				return null; // We prefer nulls rather than empty strings
			}
			else
			{
				return result.Normalize(NormalizationForm.FormC);
			}
		}

		public static List<string> ClassesFromTsTextProps(ITsTextProps props, int[] intPropsToSkip, int[] strPropsToSkip)
		{
			var classes = new List<string>();
			for (int i = 0, n = props.IntPropCount; i < n; i++)
			{
				int propNum;
				int variation;
				int propValue = props.GetIntProp(i, out propNum, out variation);
				if (intPropsToSkip.Contains(propNum))
					continue;
				string className = String.Format("propi_{0}_{1}_{2}_{3}", propNum, IntPropertyName(propNum), propValue, variation);
				classes.Add(className);
			}
			for (int i = 0, n = props.StrPropCount; i < n; i++)
			{
				int propNum;
				string propValue = props.GetStrProp(i, out propNum);
				if (propNum == (int)FwTextStringProp.kstpObjData)
				{
					// Object data can have arbitrary bytes, including null.
					propValue = HexEncode(propValue);
				}
				// In any other property type, the only problematic value is a space character.
				propValue = propValue.Replace(" ", "_SPACE_");
				string className = String.Format("props_{0}_{1}_{2}", propNum, StringPropertyName(propNum), propValue);
{'fraction_non_alphanumeric': 0.11874944488853362, 'fraction_numerical': 0.006750155431210587, 'mean_word_length': 3.427841132520645, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1461564', 'n_tokens_mistral': 4199, 'n_tokens_neox': 3451, 'n_words': 1036}