<commit_before>using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using Faithlife.Parsing;

namespace Facility.Definition.Fsd
{
	internal static class FsdParsers
	{
		public static ServiceInfo ParseDefinition(ServiceDefinitionText source, IReadOnlyDictionary<string, FsdRemarksSection> remarksSections) =>
			DefinitionParser(new Context(source, remarksSections)).Parse(source.Text);

		private static IParser<string> CommentParser { get; } =
			Parser.Regex(@"//([^\r\n]*)(\r\n?|\n|$)").Select(x => x.Groups[1].ToString());

		private static IParser<string> CommentOrWhiteSpaceParser { get; } =
			CommentParser.Or(Parser.WhiteSpace.AtLeastOnce().Success(""));

		private static IParser<T> CommentedToken<T>(this IParser<T> parser) =>
			parser.PrecededBy(CommentOrWhiteSpaceParser.Many()).TrimEnd();

		private static IParser<Positioned<string>> PunctuationParser(string token) =>
			Parser.String(token, StringComparison.Ordinal).Positioned().CommentedToken().Named("'" + token + "'");

		private static IParser<Positioned<string>> NameParser { get; } =
			Parser.Regex(@"[a-zA-Z_][0-9a-zA-Z_]*").Select(x => x.ToString()).Positioned().CommentedToken();

		private static IParser<Positioned<string>> KeywordParser(params string[] keywords) =>
			NameParser.Where(x => keywords.Contains(x.Value)).Named(string.Join(" or ", keywords.Select(x => "'" + x + "'")));

		private static IParser<IReadOnlyList<T>> Delimited<T>(this IParser<T> parser, string delimiter) =>
			parser.Delimited(PunctuationParser(delimiter));

		private static IParser<IReadOnlyList<T>> DelimitedAllowTrailing<T>(this IParser<T> parser, string delimiter) =>
			parser.DelimitedAllowTrailing(PunctuationParser(delimiter));

		private static IParser<T> Bracketed<T>(this IParser<T> parser, string openBracket, string closeBracket) =>
			parser.Bracketed(PunctuationParser(openBracket), PunctuationParser(closeBracket));

		private static IParser<Match> AttributeParameterValueParser { get; } =
			Parser.Regex(@"""(([^""\\]+|\\[""\\/bfnrt]|\\u[0-9a-fA-f]{4})*)""|([0-9a-zA-Z.+_-]+)");

		private static IParser<ServiceAttributeParameterInfo> AttributeParameterParser(Context context) =>
			from name in NameParser.Named("parameter name")
			from colon in PunctuationParser(":")
			from value in AttributeParameterValueParser.Named("parameter value").Positioned()
			select new ServiceAttributeParameterInfo(name.Value, TryParseAttributeParameterValue(value.Value),
				context.GetPart(ServicePartKind.Name, name),
				context.GetPart(ServicePartKind.Value, value));

		private static IParser<ServiceAttributeInfo> AttributeParser(Context context) =>
			from name in NameParser.Named("attribute name")
			from parameters in AttributeParameterParser(context).Delimited(",").Bracketed("(", ")").OrDefault()
			select new ServiceAttributeInfo(name.Value, parameters,
				context.GetPart(ServicePartKind.Name, name));

		private static IParser<ServiceAttributeInfo> RequiredParser { get; } =
			from bang in PunctuationParser("!")
			select new ServiceAttributeInfo("required");<commit_msg>Allow backslashes in attribute values for regex validation<commit_after>using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using Faithlife.Parsing;

namespace Facility.Definition.Fsd
{
	internal static class FsdParsers
	{
		public static ServiceInfo ParseDefinition(ServiceDefinitionText source, IReadOnlyDictionary<string, FsdRemarksSection> remarksSections) =>
			DefinitionParser(new Context(source, remarksSections)).Parse(source.Text);

		private static IParser<string> CommentParser { get; } =
			Parser.Regex(@"//([^\r\n]*)(\r\n?|\n|$)").Select(x => x.Groups[1].ToString());

		private static IParser<string> CommentOrWhiteSpaceParser { get; } =
			CommentParser.Or(Parser.WhiteSpace.AtLeastOnce().Success(""));

		private static IParser<T> CommentedToken<T>(this IParser<T> parser) =>
			parser.PrecededBy(CommentOrWhiteSpaceParser.Many()).TrimEnd();

		private static IParser<Positioned<string>> PunctuationParser(string token) =>
			Parser.String(token, StringComparison.Ordinal).Positioned().CommentedToken().Named("'" + token + "'");

		private static IParser<Positioned<string>> NameParser { get; } =
			Parser.Regex(@"[a-zA-Z_][0-9a-zA-Z_]*").Select(x => x.ToString()).Positioned().CommentedToken();

		private static IParser<Positioned<string>> KeywordParser(params string[] keywords) =>
			NameParser.Where(x => keywords.Contains(x.Value)).Named(string.Join(" or ", keywords.Select(x => "'" + x + "'")));

		private static IParser<IReadOnlyList<T>> Delimited<T>(this IParser<T> parser, string delimiter) =>
			parser.Delimited(PunctuationParser(delimiter));

		private static IParser<IReadOnlyList<T>> DelimitedAllowTrailing<T>(this IParser<T> parser, string delimiter) =>
			parser.DelimitedAllowTrailing(PunctuationParser(delimiter));

		private static IParser<T> Bracketed<T>(this IParser<T> parser, string openBracket, string closeBracket) =>
			parser.Bracketed(PunctuationParser(openBracket), PunctuationParser(closeBracket));

		private static IParser<Match> AttributeParameterValueParser { get; } =
			Parser.Regex(@"""(([^""]+|\\[""\\/bfnrt]|\\u[0-9a-fA-f]{4})*)""|([0-9a-zA-Z.+_-]+)");

		private static IParser<ServiceAttributeParameterInfo> AttributeParameterParser(Context context) =>
			from name in NameParser.Named("parameter name")
			from colon in PunctuationParser(":")
			from value in AttributeParameterValueParser.Named("parameter value").Positioned()
			select new ServiceAttributeParameterInfo(name.Value, TryParseAttributeParameterValue(value.Value),
				context.GetPart(ServicePartKind.Name, name),
				context.GetPart(ServicePartKind.Value, value));

		private static IParser<ServiceAttributeInfo> AttributeParser(Context context) =>
			from name in NameParser.Named("attribute name")
			from parameters in AttributeParameterParser(context).Delimited(",").Bracketed("(", ")").OrDefault()
			select new ServiceAttributeInfo(name.Value, parameters,
				context.GetPart(ServicePartKind.Name, name));

		private static IParser<ServiceAttributeInfo> RequiredParser { get; } =
			from bang in PunctuationParser("!")
			select new ServiceAttributeInfo("required");
{'fraction_non_alphanumeric': 0.14396456256921372, 'fraction_numerical': 0.0025312450561619996, 'mean_word_length': 6.6352657004830915, 'pattern_counts': {'":': 2, '<': 57, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3300962', 'n_tokens_mistral': 1992, 'n_tokens_neox': 1832, 'n_words': 348}