<commit_before>﻿using System.Collections.Generic;
using System.Text.RegularExpressions;
using System.IO;
using System.Text;

namespace Nustache.Core
{
    public class Scanner
    {
        private static readonly Regex _markerRegex = new Regex(@"\{\{([\{]?[^}]+?\}?)\}\}");
        // Remove standalone lines (lines which have only a non-variable expression on them)
        private static readonly Regex _standaloneRegex = new Regex(
            @"(^|\r?\n)[\r\t\v ]*({\{\s*[#\/!\<\>^]+[^}]+?\}\})[\r\t\v ]*(\r?\n|$)");
        // Special case for standalone lines that have multiple non-variable expressions with only newlines separating them
        // E.g.: {{#begin}}\n{{/end}}\n   (this is straight from the mustache specs)
        private static readonly Regex _standaloneSpecialCaseRegex = new Regex(
            @"(^|\r?\n)[\r\t\v ]*({\{\s*[#\/!\<\>^]+[^}]+?\}\})[\r\t\v\n ]*({\{\s*[#\/!\<\>^]+[^}]+?\}\})[\r\t\v ]*(\r?\n|$)");

        public IEnumerable<Part> Scan(string template)
        {
            int i = 0;
            Match m;

            // remove standalone expressions before parsing as this greatly simplifies things.  
            // See https://github.com/defunkt/mustache/blob/master/lib/mustache/parser.rb for how complex parsing is without this
            template = _standaloneRegex.Replace(template, match => match.Groups[1].Value + match.Groups[2].Value);
            template = _standaloneSpecialCaseRegex.Replace(template, match => match.Groups[1].Value + match.Groups[2].Value + match.Groups[3].Value);
            
            while (true)
            {
                if ((m = _markerRegex.Match(template, i)).Success)
                {
                    string literal = template.Substring(i, m.Index - i);

                    if (literal != "")
                    {
                        yield return new LiteralText(literal);
                    }

                    string marker = m.Groups[1].Value;

                    marker = marker.Trim();

                    if (marker[0] == '#')
                    {
                        yield return new Block(marker.Substring(1).Trim());
                    }
                    else if (marker[0] == '^')
                    {
                        yield return new InvertedBlock(marker.Substring(1).Trim());
                    }
                    else if (marker[0] == '<')
                    {
                        yield return new TemplateDefinition(marker.Substring(1).Trim());
                    }<commit_msg>delimiters<commit_after>﻿using System.Collections.Generic;
using System.Text.RegularExpressions;
using System.IO;
using System.Text;

namespace Nustache.Core
{
    public class Scanner
    {
        private static readonly Regex _markerRegex = new Regex(@"\{\{([\{]?[^}]+?\}?)\}\}");
        // Remove standalone lines (lines which have only a non-variable expression on them)
        private static readonly Regex _standaloneRegex = new Regex(
            @"(^|\r?\n)[\r\t\v ]*({\{\s*[#\/!\<\>^=]+[^}]+?\}\})[\r\t\v ]*(\r?\n|$)");

        // Special case for standalone lines that have multiple non-variable expressions with only newlines separating them
        // E.g.: {{#begin}}\n{{/end}}\n   (this is straight from the mustache specs)
        private static readonly Regex _standaloneSpecialCaseRegex = new Regex(
            @"(^|\r?\n)[\r\t\v ]*({\{\s*[#\/!\<\>^]+[^}]+?\}\})[\r\t\v\n ]*({\{\s*[#\/!\<\>^]+[^}]+?\}\})[\r\t\v ]*(\r?\n|$)");

        private static readonly Regex _delimitersRegex = new Regex(@"^=\s*(\S+)\s+(\S+)\s*=$");

        public IEnumerable<Part> Scan(string template)
        {
            int i = 0;
            Match m;

            // remove standalone expressions before parsing as this greatly simplifies things.  
            // See https://github.com/defunkt/mustache/blob/master/lib/mustache/parser.rb for how complex parsing is without this
            template = _standaloneRegex.Replace(template, match => match.Groups[1].Value + match.Groups[2].Value);
            template = _standaloneSpecialCaseRegex.Replace(template,
                match => match.Groups[1].Value + match.Groups[2].Value + match.Groups[3].Value);

            var regex = _markerRegex;

            while (true)
            {
                if ((m = regex.Match(template, i)).Success)
                {
                    string literal = template.Substring(i, m.Index - i);

                    if (literal != "")
                    {
                        yield return new LiteralText(literal);
                    }

                    string marker = m.Groups[1].Value;

                    marker = marker.Trim();

                    if (marker[0] == '=')
                    {
                        var delimiters = _delimitersRegex.Match(marker);

                        if (delimiters.Success)
                        {
                            var start = delimiters.Groups[1].Value;
                            var end = delimiters.Groups[2].Value;
                            regex = new Regex(Regex.Escape(start) + "(.+)" + Regex.Escape(end));
                        }
                    }
                    else if (marker[0] == '#')
                    {
                        yield return new Block(marker.Substring(1).Trim());
                    }
                    else if (marker[0] == '^')
                    {
                        yield return new InvertedBlock(marker.Substring(1).Trim());
                    }
                    else if (marker[0] == '<')
                    {
                        yield return new TemplateDefinition(marker.Substring(1).Trim());
                    }
{'fraction_non_alphanumeric': 0.15570021111893034, 'fraction_numerical': 0.00510204081632653, 'mean_word_length': 1.5334224598930482, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3591333', 'n_tokens_mistral': 1626, 'n_tokens_neox': 1509, 'n_words': 344}