<commit_before>using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Text;
using Icu.Collation;
using Spart;
using Spart.Actions;
using Spart.Parsers;
using Spart.Parsers.NonTerminal;
using Spart.Parsers.Primitives;
using Spart.Scanners;
using Debugger=Spart.Debug.Debugger;

namespace Palaso.WritingSystems.Collation
{
	public class SimpleRulesCollator : ICollator
	{
		private RuleBasedCollator _collator;

		public SimpleRulesCollator(string rules)
		{
			string icuRules = ConvertToIcuRules(rules);
			_collator = new RuleBasedCollator(icuRules);
		}

		static public string ConvertToIcuRules(string rules)
		{
			// parse:
			// ShoeboxRules ::= (<PrimaryRule> | <whitespace>) ('\n' (<PrimaryRule> | <whitespace>))*
			// PrimaryRule ::= <whitespace>* (<SecondaryRule> | <TertiaryRule>) (<whitespace> (<SecondaryRule> | <TertiaryRule>))*
			// TertiaryRule ::= <whitespace>* '(' <whitespace>* <CollatingElement> <whitespace>* ')' <whitespace>*
			// whitespace ::= (' ' | '\t')+
			// CollatingElement ::= any character other than ' \n\t()'
<commit_msg>Change error message when icu dlls not found<commit_after>using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Text;
using Icu.Collation;
using Spart;
using Spart.Actions;
using Spart.Parsers;
using Spart.Parsers.NonTerminal;
using Spart.Parsers.Primitives;
using Spart.Scanners;
using Debugger=Spart.Debug.Debugger;

namespace Palaso.WritingSystems.Collation
{
	public class SimpleRulesCollator : ICollator
	{
		private RuleBasedCollator _collator;

		public SimpleRulesCollator(string rules)
		{
			string icuRules = ConvertToIcuRules(rules);
			try
			{
				_collator = new RuleBasedCollator(icuRules);
			}
			catch (DllNotFoundException e)
			{
				throw new NotSupportedException("Currently SimpleRulesCollator uses Icu and thus requires the ICU dlls to be present", e);
			}
		}

		static public string ConvertToIcuRules(string rules)
		{
			// parse:
			// ShoeboxRules ::= (<PrimaryRule> | <whitespace>) ('\n' (<PrimaryRule> | <whitespace>))*
			// PrimaryRule ::= <whitespace>* (<SecondaryRule> | <TertiaryRule>) (<whitespace> (<SecondaryRule> | <TertiaryRule>))*
			// TertiaryRule ::= <whitespace>* '(' <whitespace>* <CollatingElement> <whitespace>* ')' <whitespace>*
			// whitespace ::= (' ' | '\t')+
			// CollatingElement ::= any character other than ' \n\t()'

{'fraction_non_alphanumeric': 0.13344182262001628, 'fraction_numerical': 0.0, 'mean_word_length': 4.925301204819277, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5624433', 'n_tokens_mistral': 904, 'n_tokens_neox': 799, 'n_words': 181}