<commit_before>using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Xml;
using Palaso.WritingSystems;

namespace Palaso.WritingSystems
{
	public class LdmlInFolderWritingSystemStore : WritingSystemStoreBase
	{
		private const string _kExtension = ".ldml";
		private string _path;

		/// <summary>
		/// Use the default repository
		/// </summary>
		public LdmlInFolderWritingSystemStore()
		{
			string p =
				Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "SIL");
			Directory.CreateDirectory(p);
			p = Path.Combine(p, "WritingSystemRepository");
			Directory.CreateDirectory(p);
			PathToWritingSystems = p;
		}

		/// <summary>
		/// use a special path for the repository
		/// </summary>
		/// <param name="path"></param>
		public LdmlInFolderWritingSystemStore(string path)
		{
			PathToWritingSystems = path;
		}

		public string PathToWritingSystems
		{
			get
			{
				return _path;
			}
			set
			{
				_path = value;
				if (!Directory.Exists(_path))
				{
					string parent = Directory.GetParent(_path).FullName;
					if (!Directory.Exists(parent))
					{
						throw new ApplicationException(
							"The writing system repository cannot be created because its parent folder, " + parent +
							", does not exist.");
					}
					Directory.CreateDirectory(_path);
				}
				LoadAllDefinitions();
			}
		}

		private string GetFilePath(WritingSystemDefinition ws)
		{
			return Path.Combine(PathToWritingSystems, GetFileName(ws));
		}

		private string GetFilePathFromIdentifier(string identifier)
		{
			return Path.Combine(PathToWritingSystems, GetFileNameFromIdentifier(identifier));
		}

		public string GetFileName(WritingSystemDefinition ws)
		{
			return GetFileNameFromIdentifier(ws.RFC5646);
		}

		private static string GetFileNameFromIdentifier(string identifier)
		{
			if (string.IsNullOrEmpty(identifier))
			{
				return "unknown"+_kExtension;
			}
			return identifier + _kExtension;
		}

		public void LoadAllDefinitions()
		{
			Clear();

			List<WritingSystemDefinition> loadedWritingSystems = ReadAndDisambiguateWritingSystems();
			SafelyRenameFilesAndUpdateStoreIdsToMatchWritingSystems(loadedWritingSystems);

			foreach (WritingSystemDefinition ws in loadedWritingSystems)
			{
				SaveDefinition(ws);
				ws.Modified = false;
			}

			AddActiveOSLanguages();
		}

		private List<WritingSystemDefinition> ReadAndDisambiguateWritingSystems()
		{
			List<WritingSystemDefinition> loadedWritingSystems = new List<WritingSystemDefinition>();
			foreach (string filePath in Directory.GetFiles(_path, "*.ldml"))
			{
				WritingSystemDefinition wsFromFile;
				try
				{
					wsFromFile = GetWritingSystemFromLdml(filePath);
				}
				catch (Exception
#if DEBUG
					error
#endif
					)
				{
#if DEBUG
					throw new ApplicationException("problem loading " + filePath, error);
#endif
				}
				WritingSystemDefinition writingSystemWithIdenticalRfc5646Tag =
					loadedWritingSystems.Find(ws => ws.Rfc5646TagOnLoad.CompleteTag == wsFromFile.Rfc5646TagOnLoad.CompleteTag);

				if (writingSystemWithIdenticalRfc5646Tag != null)<commit_msg>Removed try-catch as we really want to know if something is wrong with someone's writing systems.<commit_after>using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Xml;
using Palaso.WritingSystems;

namespace Palaso.WritingSystems
{
	public class LdmlInFolderWritingSystemStore : WritingSystemStoreBase
	{
		private const string _kExtension = ".ldml";
		private string _path;

		/// <summary>
		/// Use the default repository
		/// </summary>
		public LdmlInFolderWritingSystemStore()
		{
			string p =
				Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "SIL");
			Directory.CreateDirectory(p);
			p = Path.Combine(p, "WritingSystemRepository");
			Directory.CreateDirectory(p);
			PathToWritingSystems = p;
		}

		/// <summary>
		/// use a special path for the repository
		/// </summary>
		/// <param name="path"></param>
		public LdmlInFolderWritingSystemStore(string path)
		{
			PathToWritingSystems = path;
		}

		public string PathToWritingSystems
		{
			get
			{
				return _path;
			}
			set
			{
				_path = value;
				if (!Directory.Exists(_path))
				{
					string parent = Directory.GetParent(_path).FullName;
					if (!Directory.Exists(parent))
					{
						throw new ApplicationException(
							"The writing system repository cannot be created because its parent folder, " + parent +
							", does not exist.");
					}
					Directory.CreateDirectory(_path);
				}
				LoadAllDefinitions();
			}
		}

		private string GetFilePath(WritingSystemDefinition ws)
		{
			return Path.Combine(PathToWritingSystems, GetFileName(ws));
		}

		private string GetFilePathFromIdentifier(string identifier)
		{
			return Path.Combine(PathToWritingSystems, GetFileNameFromIdentifier(identifier));
		}

		public string GetFileName(WritingSystemDefinition ws)
		{
			return GetFileNameFromIdentifier(ws.RFC5646);
		}

		private static string GetFileNameFromIdentifier(string identifier)
		{
			if (string.IsNullOrEmpty(identifier))
			{
				return "unknown"+_kExtension;
			}
			return identifier + _kExtension;
		}

		public void LoadAllDefinitions()
		{
			Clear();

			List<WritingSystemDefinition> loadedWritingSystems = ReadAndDisambiguateWritingSystems();
			SafelyRenameFilesAndUpdateStoreIdsToMatchWritingSystems(loadedWritingSystems);

			foreach (WritingSystemDefinition ws in loadedWritingSystems)
			{
				SaveDefinition(ws);
				ws.Modified = false;
			}

			AddActiveOSLanguages();
		}

		private List<WritingSystemDefinition> ReadAndDisambiguateWritingSystems()
		{
			List<WritingSystemDefinition> loadedWritingSystems = new List<WritingSystemDefinition>();
			foreach (string filePath in Directory.GetFiles(_path, "*.ldml"))
			{
				WritingSystemDefinition wsFromFile;
				wsFromFile = GetWritingSystemFromLdml(filePath);
				WritingSystemDefinition writingSystemWithIdenticalRfc5646Tag =
					loadedWritingSystems.Find(ws => ws.Rfc5646TagOnLoad.CompleteTag == wsFromFile.Rfc5646TagOnLoad.CompleteTag);

				if (writingSystemWithIdenticalRfc5646Tag != null)
{'fraction_non_alphanumeric': 0.09217921952786254, 'fraction_numerical': 0.006423638991488678, 'mean_word_length': 4.134377576257213, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3400402', 'n_tokens_mistral': 2283, 'n_tokens_neox': 1774, 'n_words': 374}