<commit_before>﻿using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;

using Rant.Stringes;

namespace Rant.Vocabulary
{
	internal static class DicLexer
	{
		public static IEnumerable<Token<DicTokenType>> Tokenize(string data)
		{
			var reader = new StringReader(data);

			while (true)
			{
				int nextChar = reader.Read();
				if (nextChar == -1)
				{
					yield return new Token<DicTokenType>(DicTokenType.EOF, "");
					yield break;
				}

				char currentChar = (char)nextChar;
				if (char.IsWhiteSpace(currentChar))
					continue;

				switch (currentChar)
				{
					// directive
					case '#':
						{
							string value = ReadRestOfLine(reader);
							yield return new Token<DicTokenType>(DicTokenType.Directive, value.Trim());
						}
						break;
					// entry or diffentry
					case '>':
						{
							int peekedChar = reader.Peek();
							if (peekedChar == -1)
								yield break;

							bool diffmark = false;
							// it's a diffentry
							if ((char)peekedChar == '>')
							{
								diffmark = true;
								reader.Read();
							}

							string value = ReadRestOfLine(reader, diffmark);
							yield return new Token<DicTokenType>(diffmark ? DicTokenType.DiffEntry : DicTokenType.Entry, value.Trim());
						}
						break;
					// property
					case '|':
						{
							string value = ReadRestOfLine(reader);
							yield return new Token<DicTokenType>(DicTokenType.Property, value.Trim());
						}
						break;
					default:
						throw new System.Exception("Unexpected character in dictionary.");
				}
			}
		}

		private static string ReadRestOfLine(StringReader reader, bool isDiffmark = false)
		{
			int nextChar;
			char currentChar;
			string value = "";

			// read to the end of the line
			while (true)
			{
				nextChar = reader.Read();
				if (nextChar == -1)
					break;
				currentChar = (char)nextChar;
				// we've made it to the end
				if ((!isDiffmark && currentChar == '|') || currentChar == '>' || currentChar == '\n' || currentChar == '#')
					break;
				value = string.Concat(value, currentChar);
			}

			return value;
		}
	}

	internal enum DicTokenType
	{
		Directive,
		Entry,
		DiffEntry,
		Property,
		Ignore,
		EOF
	}
}<commit_msg>Allow inline dictionary entries.<commit_after>﻿using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;

using Rant.Stringes;

namespace Rant.Vocabulary
{
	internal static class DicLexer
	{
		public static IEnumerable<Token<DicTokenType>> Tokenize(string data)
		{
			var reader = new StringReader(data);

			while (true)
			{
				int nextChar = reader.Read();
				if (nextChar == -1)
				{
					yield return new Token<DicTokenType>(DicTokenType.EOF, "");
					yield break;
				}

				char currentChar = (char)nextChar;
				if (char.IsWhiteSpace(currentChar))
					continue;

				switch (currentChar)
				{
					// directive
					case '#':
						{
							string value = ReadRestOfLine(reader);
							yield return new Token<DicTokenType>(DicTokenType.Directive, value.Trim());
						}
						break;
					// entry or diffentry
					case '>':
						{
							int peekedChar = reader.Peek();
							if (peekedChar == -1)
								yield break;

							bool diffmark = false;
							// it's a diffentry
							if ((char)peekedChar == '>')
							{
								diffmark = true;
								reader.Read();
							}

							string value = ReadRestOfLine(reader, diffmark);
							yield return new Token<DicTokenType>(diffmark ? DicTokenType.DiffEntry : DicTokenType.Entry, value.Trim());
						}
						break;
					// property
					case '|':
						{
							string value = ReadRestOfLine(reader);
							yield return new Token<DicTokenType>(DicTokenType.Property, value.Trim());
						}
						break;
					default:
						throw new System.Exception("Unexpected character in dictionary.");
				}
			}
		}

		private static string ReadRestOfLine(StringReader reader, bool isDiffmark = false)
		{
			int nextChar;
			char currentChar;
			string value = "";
			bool pipeAllowed = !isDiffmark;

			// read to the end of the line
			while (true)
			{
				nextChar = reader.Peek();
				if (nextChar == -1)
					break;
				currentChar = (char)nextChar;
				if (isDiffmark && char.IsWhiteSpace(currentChar))
					pipeAllowed = true;

				// we've made it to the end
				if ((pipeAllowed && currentChar == '|') || currentChar == '>' || currentChar == '\n' || currentChar == '#')
					break;
				reader.Read();
				value = string.Concat(value, currentChar);
			}

			return value;
		}
	}

	internal enum DicTokenType
	{
		Directive,
		Entry,
		DiffEntry,
		Property,
		Ignore,
		EOF
	}
}
{'fraction_non_alphanumeric': 0.11999133636560537, 'fraction_numerical': 0.001299545159194282, 'mean_word_length': 2.4721804511278194, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6930039', 'n_tokens_mistral': 2051, 'n_tokens_neox': 1441, 'n_words': 348}