<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace markdom.cs.Utils {
	public class EntityInfo {
		private static readonly IDictionary<string, int[]> _codepointsByEntityName;
		private static readonly IDictionary<int, string> _entityNameByCodepoint;

		static EntityInfo() {
			_codepointsByEntityName = new Dictionary<string, int[]>();
			_entityNameByCodepoint = new Dictionary<int, string>();

			// EntityInfo.bin follows a very simple format designed to be easy to read.
			// Each entity is defined in EntityInfo.bin as:
			// 
			// 1. 8 bits denoting the number of characters in the entity name
			// 2. A sequence of 8 bit ASCII characters, the entity name
			// 3. 8 bits denoting the number of codepoints in the entity value
			// 4. A sequence of 32 bit UTF32 codepoints, the entity value
			var readBuffer = new byte[64];
			var embeddedResourceName = typeof(EntityInfo).Namespace + ".EntityInfo.bin";
			using(var stream = typeof(EntityInfo).Assembly.GetManifestResourceStream(embeddedResourceName))
			do {
				// Read the number of 8-bit characters in the entity name
				var entityNameLength = stream.ReadByte();
				if(-1 == entityNameLength) break;

				// Read the entity's name
				stream.Read(readBuffer, 0, entityNameLength);
				var entityName = new string(readBuffer.Take(entityNameLength).Select(b => (char)b).ToArray());

				// Get the number of codepoints for the entity
				var numCodepoints = stream.ReadByte();

				// Read 32-bit UTF32 codepoints into an array
				var entityCodepoints = new int[numCodepoints];
				for(var i = 0; i < numCodepoints; i++) {
					stream.Read(readBuffer, 0, 4);
					entityCodepoints[i] = (readBuffer[0] << 24) | (readBuffer[1] << 16) | (readBuffer[2] << 8) | readBuffer[3];
				}

				_codepointsByEntityName.Add(entityName, entityCodepoints);
				if(1 == numCodepoints)
					_entityNameByCodepoint[entityCodepoints[0]] = entityName;
			} while(true);
		}

		/// <summary>
		/// Returns true if the provided <paramref name="name"/> is a valid entity name.
		/// </summary>
		public static bool IsEntityName(string name) {
			return _codepointsByEntityName.ContainsKey(name);
		}
	}
}<commit_msg>Added EntityInfo::TryGetEntityNameByCodepoint(int, out string):bool<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace markdom.cs.Utils {
	public class EntityInfo {
		private static readonly IDictionary<string, int[]> _codepointsByEntityName;
		private static readonly IDictionary<int, string> _entityNameByCodepoint;

		static EntityInfo() {
			_codepointsByEntityName = new Dictionary<string, int[]>();
			_entityNameByCodepoint = new Dictionary<int, string>();

			// EntityInfo.bin follows a very simple format designed to be easy to read.
			// Each entity is defined in EntityInfo.bin as:
			// 
			// 1. 8 bits denoting the number of characters in the entity name
			// 2. A sequence of 8 bit ASCII characters, the entity name
			// 3. 8 bits denoting the number of codepoints in the entity value
			// 4. A sequence of 32 bit UTF32 codepoints, the entity value
			var readBuffer = new byte[64];
			var embeddedResourceName = typeof(EntityInfo).Namespace + ".EntityInfo.bin";
			using(var stream = typeof(EntityInfo).Assembly.GetManifestResourceStream(embeddedResourceName))
			do {
				// Read the number of 8-bit characters in the entity name
				var entityNameLength = stream.ReadByte();
				if(-1 == entityNameLength) break;

				// Read the entity's name
				stream.Read(readBuffer, 0, entityNameLength);
				var entityName = new string(readBuffer.Take(entityNameLength).Select(b => (char)b).ToArray());

				// Get the number of codepoints for the entity
				var numCodepoints = stream.ReadByte();

				// Read 32-bit UTF32 codepoints into an array
				var entityCodepoints = new int[numCodepoints];
				for(var i = 0; i < numCodepoints; i++) {
					stream.Read(readBuffer, 0, 4);
					entityCodepoints[i] = (readBuffer[0] << 24) | (readBuffer[1] << 16) | (readBuffer[2] << 8) | readBuffer[3];
				}

				_codepointsByEntityName.Add(entityName, entityCodepoints);
				if(1 == numCodepoints)
					_entityNameByCodepoint[entityCodepoints[0]] = entityName;
			} while(true);
		}

		/// <summary>
		/// Returns true if the provided <paramref name="name"/> is a valid entity name.
		/// </summary>
		public static bool IsEntityName(string name) {
			return _codepointsByEntityName.ContainsKey(name);
		}

		public static bool TryGetEntityNameByCodepoint(int codepoint, out string entityName) {
			return _entityNameByCodepoint.TryGetValue(codepoint, out entityName);
		}
	}
}
{'fraction_non_alphanumeric': 0.1067524115755627, 'fraction_numerical': 0.01457663451232583, 'mean_word_length': 3.88586387434555, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1065758', 'n_tokens_mistral': 1610, 'n_tokens_neox': 1447, 'n_words': 429}