<commit_before>﻿using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using Lens.Lexer;
using Lens.Parser;
using Lens.SyntaxTree;
using NUnit.Framework;

namespace Lens.Test
{
	internal class TestBase
	{
		protected static void Test(string src, object value, bool testConstants = false)
		{
			Assert.AreEqual(value, Compile(src, new LensCompilerOptions { UnrollConstants = true, AllowSave = true }));
			if (testConstants)
				Assert.AreEqual(value, Compile(src));
		}

		protected static void Test(IEnumerable<NodeBase> nodes, object value, bool testConstants = false)
		{
			Assert.AreEqual(value, Compile(nodes, new LensCompilerOptions {UnrollConstants = true}));
			if (testConstants)
				Assert.AreEqual(value, Compile(nodes));
		}

		protected static void TestError(string src, string bareMsg)
		{
			var ex = Assert.Throws<LensCompilerException>(() => Compile(src));

			if (!bareMsg.Contains('{'))
			{
				Assert.AreEqual(bareMsg, ex.Message);
			}
			else
			{
				var regex = new Regex(regexFromMsg(bareMsg));
				Assert.IsTrue(
					regex.IsMatch(ex.Message),
					"Message does not match!\n\n  Expected: {0}\n  Actual: {1}\n",
					bareMsg.Substring(0, 70) + "...",
					ex.Message.Substring(0, 70) + "..."
				);
			}
		}

		protected static void Test(string src, object value, LensCompilerOptions opts)
		{
			Assert.AreEqual(value, Compile(src, opts));
		}

		protected static void Test(IEnumerable<NodeBase> nodes, object value, LensCompilerOptions opts)
		{
			Assert.AreEqual(value, Compile(nodes, opts));
		}

		protected static void TestParser(string source, params NodeBase[] expected)
		{
			Assert.AreEqual(expected, Parse(source).ToArray());
		}

		protected static IEnumerable<NodeBase> Parse(string source)
		{
			var lexer = new LensLexer(source);
			var parser = new LensParser(lexer.Lexems);
			return parser.Nodes;
		}

		protected static object Compile(string src, LensCompilerOptions opts = null)
		{
			opts = opts ?? new LensCompilerOptions { AllowSave = true };
			return new LensCompiler(opts).Run(src);
		}

		protected static object Compile(IEnumerable<NodeBase> nodes, LensCompilerOptions opts = null)
		{
			opts = opts ?? new LensCompilerOptions { AllowSave = true };
			return new LensCompiler(opts).Run(nodes);
		}

		private static Regex _MessageWildcardRegex = new Regex(@"\{[0-9]\}", RegexOptions.Compiled);
		private static Regex _MessageSymbolRegex = new Regex(@"[\-\[\]\/\(\)\*\+\?\.\\\^\$\|]", RegexOptions.Compiled);

		private static string regexFromMsg(string msg)
		{
			msg = _MessageSymbolRegex.Replace(msg, @"\$1");
			msg = _MessageWildcardRegex.Replace(msg, "(.+)");
			return "$" + msg + "^";
		}
	}
}
<commit_msg>Фикс для теста на сообщение об ошибке.<commit_after>﻿using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using Lens.Lexer;
using Lens.Parser;
using Lens.SyntaxTree;
using NUnit.Framework;

namespace Lens.Test
{
	internal class TestBase
	{
		protected static void Test(string src, object value, bool testConstants = false)
		{
			Assert.AreEqual(value, Compile(src, new LensCompilerOptions { UnrollConstants = true, AllowSave = true }));
			if (testConstants)
				Assert.AreEqual(value, Compile(src));
		}

		protected static void Test(IEnumerable<NodeBase> nodes, object value, bool testConstants = false)
		{
			Assert.AreEqual(value, Compile(nodes, new LensCompilerOptions {UnrollConstants = true}));
			if (testConstants)
				Assert.AreEqual(value, Compile(nodes));
		}

		protected static void TestError(string src, string bareMsg)
		{
			var ex = Assert.Throws<LensCompilerException>(() => Compile(src));

			if (!bareMsg.Contains('{'))
			{
				Assert.AreEqual(bareMsg, ex.Message);
			}
			else
			{
				var pattern = regexFromMsg(bareMsg);
				var regex = new Regex(pattern);
				Assert.IsTrue(
					regex.IsMatch(ex.Message),
					"Message does not match!\n\n  Expected: {0}\n  Actual: {1}\n",
					bareMsg.Substring(0, 70) + "...",
					ex.Message.Substring(0, 70) + "..."
				);
			}
		}

		protected static void Test(string src, object value, LensCompilerOptions opts)
		{
			Assert.AreEqual(value, Compile(src, opts));
		}

		protected static void Test(IEnumerable<NodeBase> nodes, object value, LensCompilerOptions opts)
		{
			Assert.AreEqual(value, Compile(nodes, opts));
		}

		protected static void TestParser(string source, params NodeBase[] expected)
		{
			Assert.AreEqual(expected, Parse(source).ToArray());
		}

		protected static IEnumerable<NodeBase> Parse(string source)
		{
			var lexer = new LensLexer(source);
			var parser = new LensParser(lexer.Lexems);
			return parser.Nodes;
		}

		protected static object Compile(string src, LensCompilerOptions opts = null)
		{
			opts = opts ?? new LensCompilerOptions { AllowSave = true };
			return new LensCompiler(opts).Run(src);
		}

		protected static object Compile(IEnumerable<NodeBase> nodes, LensCompilerOptions opts = null)
		{
			opts = opts ?? new LensCompilerOptions { AllowSave = true };
			return new LensCompiler(opts).Run(nodes);
		}

		private static Regex _MessageWildcardRegex = new Regex(@"\{[0-9]\}", RegexOptions.Compiled);
		private static Regex _MessageSymbolRegex = new Regex(@"(?<s>[\-\[\]\/\(\)\*\+\?\.\\\^\$\|])", RegexOptions.Compiled);

		private static string regexFromMsg(string msg)
		{
			msg = _MessageSymbolRegex.Replace(msg, @"\${s}");
			msg = _MessageWildcardRegex.Replace(msg, "(.+)");
			return "^" + msg + "$";
		}
	}
}

{'fraction_non_alphanumeric': 0.1330225887414844, 'fraction_numerical': 0.003764790247400502, 'mean_word_length': 3.727966101694915, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6234316', 'n_tokens_mistral': 2048, 'n_tokens_neox': 1751, 'n_words': 417}