<commit_before>﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Web;

namespace Mustachio
{
    /// <summary>
    /// The main entry point for this library. Use the static "Parse" methods to create template functions.
    /// Functions are safe for reuse, so you may parse and cache the resulting function.
    /// </summary>
    public class Parser
    {
        /// <summary>
        /// This function is now obsolete. You should use the function accepting a ParsingOptions type instead.
        /// Parse the template content, producing a function that can be used to apply variables to the template. 
        /// The provided function can be reused (i.e. no state will "leak" from one application of the function to the next).
        /// Default parsing options will be used.
        /// </summary>
        /// <param name="template">The content of the template to be parsed.</param>
        /// <param name="disableContentEscaping">In some cases, content should not be escaped (such as when rendering text bodies and subjects in emails). 
        /// By default, we use content escaping, but this parameter allows it to be disabled.</param>
        /// Other parsing options have default values.
        /// <returns></returns>
        [Obsolete]
        public static Func<IDictionary<String, object>, String> Parse(string template, bool disableContentEscaping = false)
        {
            return Parse(template, new ParsingOptions { DisableContentSafety = disableContentEscaping });
        }

        /// <summary>
        /// Parse the template content, producing a function that can be used to apply variables to the template. 
        /// The provided function can be reused (i.e. no state will "leak" from one application of the function to the next).
        /// </summary>
        /// <param name="template">The content of the template to be parsed.</param>
        /// <param name="options">Options for configuring/extending the parser.</param>
        /// <returns></returns>
        public static Func<IDictionary<String, object>, String> Parse(string template, ParsingOptions options)
        {
            var tokensQueue = GetTokensQueue(template, options);
            var internalTemplate = Parse(tokensQueue, options);
            return (model) =>
            {
                var retval = new StringBuilder();
                var context = new ContextObject()
                {
                    Value = model,
                    Key = ""
                };
                internalTemplate(retval, context);
                return retval.ToString();
            };
        }

        private static Queue<TokenTuple> GetTokensQueue(string template, ParsingOptions options)
        {
            var tokenizeResult = Tokenizer.Tokenize(template, options);
            if (tokenizeResult.Errors.Any())
            {
                var innerExceptions = tokenizeResult.Errors.OrderBy(k => k.LineNumber).ThenBy(k => k.CharacterOnLine).ToArray();
                throw new AggregateException(innerExceptions);
            }
            return new Queue<TokenTuple>(tokenizeResult.Tokens);
        }

        /// <summary>
        /// This function is now obsolete. You should use the function accepting a ParsingOptions type instead.
        /// Parse the template, and capture paths used in the template to determine a suitable structure for the required model.
        /// Default parsing options will be used.
        /// </summary>
        /// <param name="templateSource">The template content to parse.</param>
        /// <param name="disableContentEscaping">In some cases, content should not be escaped (such as when rendering text bodies and subjects in emails). 
        /// By default, we use content escaping, but this parameter allows it to be disabled.</param>
        /// Other parsing options have default values.
        /// <returns></returns>
        [Obsolete]
        public static ExtendedParseInformation ParseWithModelInference(string templateSource, bool disableContentEscaping = false)
        {
            return ParseWithModelInference(templateSource, new ParsingOptions { DisableContentSafety = disableContentEscaping });
        }

        /// <summary>
        /// Parse the template, and capture paths used in the template to determine a suitable structure for the required model.
        /// </summary>
        /// <param name="templateSource">The template content to parse.</param>
        /// <param name="options">Options for configuring/extending the parser.</param>
        /// <returns></returns>
        public static ExtendedParseInformation ParseWithModelInference(string templateSource, ParsingOptions options)
        {
            var tokensQueue = GetTokensQueue(templateSource, options);
            var inferredModel = new InferredTemplateModel();

            var internalTemplate = Parse(tokensQueue, options, inferredModel);
            Func<IDictionary<String, object>, String> template = (model) =>
            {
                var retval = new StringBuilder();
                var context = new ContextObject()
                {
                    Value = model,
                    Key = ""<commit_msg>Make small readjustments to the public API<commit_after>﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Web;

namespace Mustachio
{
    /// <summary>
    /// The main entry point for this library. Use the static "Parse" methods to create template functions.
    /// Functions are safe for reuse, so you may parse and cache the resulting function.
    /// </summary>
    public class Parser
    {
        /// <summary>
        /// Parse the template content, producing a function that can be used to apply variables to the template. 
        /// The provided function can be reused (i.e. no state will "leak" from one application of the function to the next).
        /// Default parsing options will be used.
        /// </summary>
        /// <param name="template">The content of the template to be parsed.</param>
        /// <returns></returns>
        public static Func<IDictionary<String, object>, String> Parse(string template)
        {
            return Parse(template, new ParsingOptions());
        }

        /// <summary>
        /// This function is now obsolete. You should use the function accepting a ParsingOptions type instead.
        /// Parse the template content, producing a function that can be used to apply variables to the template. 
        /// The provided function can be reused (i.e. no state will "leak" from one application of the function to the next).
        /// Other parsing options will have default values.
        /// </summary>
        /// <param name="template">The content of the template to be parsed.</param>
        /// <param name="disableContentEscaping">In some cases, content should not be escaped (such as when rendering text bodies and subjects in emails). 
        /// By default, we use content escaping, but this parameter allows it to be disabled.</param>
        /// <returns></returns>
        [Obsolete]
        public static Func<IDictionary<String, object>, String> Parse(string template, bool disableContentEscaping)
        {
            return Parse(template, new ParsingOptions { DisableContentSafety = disableContentEscaping });
        }

        /// <summary>
        /// Parse the template content, producing a function that can be used to apply variables to the template. 
        /// The provided function can be reused (i.e. no state will "leak" from one application of the function to the next).
        /// </summary>
        /// <param name="template">The content of the template to be parsed.</param>
        /// <param name="options">Options for configuring/extending the parser.</param>
        /// <returns></returns>
        public static Func<IDictionary<String, object>, String> Parse(string template, ParsingOptions options)
        {
            var tokensQueue = GetTokensQueue(template, options);
            var internalTemplate = Parse(tokensQueue, options);
            return (model) =>
            {
                var retval = new StringBuilder();
                var context = new ContextObject()
                {
                    Value = model,
                    Key = ""
                };
                internalTemplate(retval, context);
                return retval.ToString();
            };
        }

        private static Queue<TokenTuple> GetTokensQueue(string template, ParsingOptions options)
        {
            var tokenizeResult = Tokenizer.Tokenize(template, options);
            if (tokenizeResult.Errors.Any())
            {
                var innerExceptions = tokenizeResult.Errors.OrderBy(k => k.LineNumber).ThenBy(k => k.CharacterOnLine).ToArray();
                throw new AggregateException(innerExceptions);
            }
            return new Queue<TokenTuple>(tokenizeResult.Tokens);
        }

        /// <summary>
        /// Parse the template, and capture paths used in the template to determine a suitable structure for the required model.
        /// Default parsing options will be used.
        /// </summary>
        /// <param name="templateSource">The template content to parse.</param>
        /// <returns></returns>
        public static ExtendedParseInformation ParseWithModelInference(string templateSource)
        {
            return ParseWithModelInference(templateSource, new ParsingOptions());
        }

        /// <summary>
        /// This function is now obsolete. You should use the function accepting a ParsingOptions type instead.
        /// Parse the template, and capture paths used in the template to determine a suitable structure for the required model.
        /// Other parsing options will have default values.
        /// </summary>
        /// <param name="templateSource">The template content to parse.</param>
        /// <param name="disableContentEscaping">In some cases, content should not be escaped (such as when rendering text bodies and subjects in emails). 
        /// By default, we use content escaping, but this parameter allows it to be disabled.</param>
        /// <returns></returns>
        [Obsolete]
        public static ExtendedParseInformation ParseWithModelInference(string templateSource, bool disableContentEscaping)
        {
            return ParseWithModelInference(templateSource, new ParsingOptions { DisableContentSafety = disableContentEscaping });
        }

        /// <summary>
        /// Parse the template, and capture paths used in the template to determine a suitable structure for the required model.
        /// </summary>
        /// <param name="templateSource">The template content to parse.</param>
        /// <param name="options">Options for configuring/extending the parser.</param>
        /// <returns></returns>
        public static ExtendedParseInformation ParseWithModelInference(string templateSource, ParsingOptions options)
        {
            var tokensQueue = GetTokensQueue(templateSource, options);
            var inferredModel = new InferredTemplateModel();

            var internalTemplate = Parse(tokensQueue, options, inferredModel);
            Func<IDictionary<String, object>, String> template = (model) =>
            {
                var retval = new StringBuilder();
                var context = new ContextObject()
                {
                    Value = model,
                    Key = ""
{'fraction_non_alphanumeric': 0.0915122791900043, 'fraction_numerical': 0.0, 'mean_word_length': 2.2509803921568627, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 109, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5152113', 'n_tokens_mistral': 2668, 'n_tokens_neox': 2587, 'n_words': 1025}