<commit_before>using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Icu;
using SIL.Extensions;

namespace SIL.WritingSystems
{
	/// <summary>
	/// This static utility class contains various methods for processing IETF language tags. Currently,
	/// there are no methods for accessing extended language and extension subtags.
	/// </summary>
	public static class IetfLanguageTag
	{
		private const string PrivateUseExpr = "[xX](-" + PrivateUseSubExpr + ")+";
		private const string PrivateUseSubExpr = "[a-zA-Z0-9]{1,8}";
		// according to RFC-5646, a primary language subtag can be anywhere from 2 to 8 characters in length,
		// at this point only ISO 639 codes are allowed, which are all 2 to 3 characters in length, so we
		// use the more practical constraint of 2 to 3 characters, which allows private use ICU locales with
		// only a language defined (i.e. "xkal") to not match the regex.
		private const string LanguageExpr = "[a-zA-Z]{2,8}(-[a-zA-Z]{3}){0,3}";
		private const string ScriptExpr = "[a-zA-Z]{4}";
		private const string RegionExpr = "[a-zA-Z]{2}|[0-9]{3}";
		private const string VariantSubExpr = "[0-9][a-zA-Z0-9]{3}|[a-zA-Z0-9]{5,8}";
		private const string VariantExpr = "(" + VariantSubExpr + ")(-(" + VariantSubExpr + "))*";
		private const string ExtensionExpr = "[a-wyzA-WYZ](-([a-zA-Z0-9]{2,8})+)+";

		private const string IcuTagExpr = "(\\A(?'privateuse'" + PrivateUseExpr + ")\\z)"
			+ "|(\\A(?'language'" + LanguageExpr + ")"
			+ "(-(?'script'" + ScriptExpr + "))?"
			+ "(-(?'region'" + RegionExpr + "))?"
			+ "(-(?'variant'" + VariantExpr + "))?"
			+ "(-(?'extension'" + ExtensionExpr + "))?"
			+ "(-(?'privateuse'" + PrivateUseExpr + "))?\\z)";

		private const string LangTagExpr = "(\\A(?'privateuse'" + PrivateUseExpr + ")\\z)"
			+ "|(\\A(?'language'" + LanguageExpr + ")"
			+ "(-(?'script'" + ScriptExpr + "))?"
			+ "(-(?'region'" + RegionExpr + "))?"
			+ "(-(?'variant'" + VariantExpr + "))?"
			+ "(-(?'extension'" + ExtensionExpr + "))?"
			+ "(-(?'privateuse'" + PrivateUseExpr + "))?\\z)";

		private static readonly Regex IcuTagPattern;
		private static readonly Regex LangTagPattern;
		private static readonly Regex LangPattern;
		private static readonly Regex ScriptPattern;
		private static readonly Regex RegionPattern;
		private static readonly Regex PrivateUsePattern;

		static IetfLanguageTag()
		{
			IcuTagPattern = new Regex(IcuTagExpr, RegexOptions.ExplicitCapture);
			LangTagPattern = new Regex(LangTagExpr, RegexOptions.ExplicitCapture);
			LangPattern = new Regex("\\A(" + LanguageExpr + ")\\z", RegexOptions.ExplicitCapture);
			ScriptPattern = new Regex("\\A(" + ScriptExpr + ")\\z", RegexOptions.ExplicitCapture);
			RegionPattern = new Regex("\\A(" + RegionExpr + ")\\z", RegexOptions.ExplicitCapture);
			PrivateUsePattern = new Regex("\\A(" + PrivateUseSubExpr + ")\\z", RegexOptions.ExplicitCapture);
		}

		/// <summary>
		/// Return Variant Subtags based on the code, looking up and returning subtags for standard codes or creating new ones for custom (private use) codes
		/// </summary>
		/// <param name="variantCodes">Variant code</param>
		/// <param name="variantSubtags">return as variantsubtags</param>
		/// <param name="variantNames">comma separated list of variant names to use with each private use code in variantCodes</param>
		/// <returns></returns>
		public static bool TryGetVariantSubtags(string variantCodes, out IEnumerable<VariantSubtag> variantSubtags, string variantNames = "")
		{
			if (string.IsNullOrEmpty(variantCodes))
			{
				variantSubtags = Enumerable.Empty<VariantSubtag>();
				return true;
			}

			string standardVariantCodes, privateUseVariantCodes;
			SplitVariantAndPrivateUse(variantCodes, out standardVariantCodes, out privateUseVariantCodes);
			var variantSubtagsList = new List<VariantSubtag>();
			foreach (string standardCode in standardVariantCodes.Split(new[] {'-'}, StringSplitOptions.RemoveEmptyEntries))
			{
				VariantSubtag variantSubtag;
				if (StandardSubtags.RegisteredVariants.TryGet(standardCode, out variantSubtag))
				{
					variantSubtagsList.Add(variantSubtag);
				}
				else
				{
					variantSubtags = null;
					return false;
				}
			}

			var variantName = variantNames.Split(',');
			int index = 0;
			foreach (string privateUseCode in privateUseVariantCodes.Split(new[] { '-' }, StringSplitOptions.RemoveEmptyEntries))
			{
				VariantSubtag variantSubtag;
				if (!StandardSubtags.CommonPrivateUseVariants.TryGet(privateUseCode, out variantSubtag))
				{
					if (!string.IsNullOrEmpty(variantNames))
					{
						variantSubtag = new VariantSubtag(privateUseCode, variantName[index]);
						index++;
					}
					else
						variantSubtag = new VariantSubtag(privateUseCode);
				}
				variantSubtagsList.Add(variantSubtag);
			}
			variantSubtags = variantSubtagsList;
			return true;
		}

		public static string GetVariantCodes(IEnumerable<VariantSubtag> variantSubtags)<commit_msg>Fix LT-19153: Unable to create fully customized writing system<commit_after>using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Icu;
using SIL.Extensions;

namespace SIL.WritingSystems
{
	/// <summary>
	/// This static utility class contains various methods for processing IETF language tags. Currently,
	/// there are no methods for accessing extended language and extension subtags.
	/// </summary>
	public static class IetfLanguageTag
	{
		private const string PrivateUseExpr = "[xX](-" + PrivateUseSubExpr + ")+";
		private const string PrivateUseSubExpr = "[a-zA-Z0-9]{1,8}";
		// according to RFC-5646, a primary language subtag can be anywhere from 2 to 8 characters in length,
		// at this point only ISO 639 codes are allowed, which are all 2 to 3 characters in length, so we
		// use the more practical constraint of 2 to 3 characters, which allows private use ICU locales with
		// only a language defined (i.e. "xkal") to not match the regex.
		private const string LanguageExpr = "[a-zA-Z]{2,8}(-[a-zA-Z]{3}){0,3}";
		private const string ScriptExpr = "[a-zA-Z]{4}";
		private const string RegionExpr = "[a-zA-Z]{2}|[0-9]{3}";
		private const string VariantSubExpr = "[0-9][a-zA-Z0-9]{3}|[a-zA-Z0-9]{5,8}";
		private const string VariantExpr = "(" + VariantSubExpr + ")(-(" + VariantSubExpr + "))*";
		private const string ExtensionExpr = "[a-wyzA-WYZ](-([a-zA-Z0-9]{2,8})+)+";

		private const string IcuTagExpr = "(\\A(?'privateuse'" + PrivateUseExpr + ")\\z)"
			+ "|(\\A(?'language'" + LanguageExpr + ")"
			+ "(-(?'script'" + ScriptExpr + "))?"
			+ "(-(?'region'" + RegionExpr + "))?"
			+ "(-(?'variant'" + VariantExpr + "))?"
			+ "(-(?'extension'" + ExtensionExpr + "))?"
			+ "(-(?'privateuse'" + PrivateUseExpr + "))?\\z)";

		private const string LangTagExpr = "(\\A(?'privateuse'" + PrivateUseExpr + ")\\z)"
			+ "|(\\A(?'language'" + LanguageExpr + ")"
			+ "(-(?'script'" + ScriptExpr + "))?"
			+ "(-(?'region'" + RegionExpr + "))?"
			+ "(-(?'variant'" + VariantExpr + "))?"
			+ "(-(?'extension'" + ExtensionExpr + "))?"
			+ "(-(?'privateuse'" + PrivateUseExpr + "))?\\z)";

		private static readonly Regex IcuTagPattern;
		private static readonly Regex LangTagPattern;
		private static readonly Regex LangPattern;
		private static readonly Regex ScriptPattern;
		private static readonly Regex RegionPattern;
		private static readonly Regex PrivateUsePattern;

		static IetfLanguageTag()
		{
			IcuTagPattern = new Regex(IcuTagExpr, RegexOptions.ExplicitCapture);
			LangTagPattern = new Regex(LangTagExpr, RegexOptions.ExplicitCapture);
			LangPattern = new Regex("\\A(" + LanguageExpr + ")\\z", RegexOptions.ExplicitCapture);
			ScriptPattern = new Regex("\\A(" + ScriptExpr + ")\\z", RegexOptions.ExplicitCapture);
			RegionPattern = new Regex("\\A(" + RegionExpr + ")\\z", RegexOptions.ExplicitCapture);
			PrivateUsePattern = new Regex("\\A(" + PrivateUseSubExpr + ")\\z", RegexOptions.ExplicitCapture);
		}

		/// <summary>
		/// Return Variant Subtags based on the code, looking up and returning subtags for standard codes or creating new ones for custom (private use) codes
		/// </summary>
		/// <param name="variantCodes">Variant code</param>
		/// <param name="variantSubtags">return as variantsubtags</param>
		/// <param name="variantNames">comma separated list of variant names to use with each private use code in variantCodes</param>
		/// <returns></returns>
		public static bool TryGetVariantSubtags(string variantCodes, out IEnumerable<VariantSubtag> variantSubtags, string variantNames = "")
		{
			if (string.IsNullOrEmpty(variantCodes))
			{
				variantSubtags = Enumerable.Empty<VariantSubtag>();
				return true;
			}

			string standardVariantCodes, privateUseVariantCodes;
			SplitVariantAndPrivateUse(variantCodes, out standardVariantCodes, out privateUseVariantCodes);
			var variantSubtagsList = new List<VariantSubtag>();
			foreach (string standardCode in standardVariantCodes.Split(new[] {'-'}, StringSplitOptions.RemoveEmptyEntries))
			{
				VariantSubtag variantSubtag;
				if (StandardSubtags.RegisteredVariants.TryGet(standardCode, out variantSubtag))
				{
					variantSubtagsList.Add(variantSubtag);
				}
				else
				{
					variantSubtags = null;
					return false;
				}
			}

			var variantName = variantNames.Split(',');
			int index = 0;
			foreach (string privateUseCode in privateUseVariantCodes.Split(new[] { '-' }, StringSplitOptions.RemoveEmptyEntries))
			{
				VariantSubtag variantSubtag;
				if (!StandardSubtags.CommonPrivateUseVariants.TryGet(privateUseCode, out variantSubtag))
				{
					if (!string.IsNullOrEmpty(variantNames) && index < variantName.Length)
					{
						variantSubtag = new VariantSubtag(privateUseCode, variantName[index]);
						index++;
					}
					else
						variantSubtag = new VariantSubtag(privateUseCode);
				}
				variantSubtagsList.Add(variantSubtag);
			}
			variantSubtags = variantSubtagsList;
			return true;
		}

		public static string GetVariantCodes(IEnumerable<VariantSubtag> variantSubtags)
{'fraction_non_alphanumeric': 0.14575299119944626, 'fraction_numerical': 0.008602788490062296, 'mean_word_length': 4.168114460909555, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3936502', 'n_tokens_mistral': 3511, 'n_tokens_neox': 3075, 'n_words': 793}