<commit_before>﻿using System.Text.RegularExpressions;
using System.Linq;
using System;
using System.Collections.Generic;
using System.Text;

namespace WikipediaAvsAnTrieExtractor {
    public partial class RegexTextUtils {
        //Note: regexes are NOT static and shared between threads because of... http://stackoverflow.com/questions/7585087/multithreaded-use-of-regex
        //This code is bottlenecked by regexes, so this really matters, here.


        public string StripWikiMarkup(string wikiMarkedUpText) {
            return markupToReplaceRegex.Replace(CutBraces(markupToStripRegex.Replace(wikiMarkedUpText, "")), m => m.Groups["txt"].Value);
        }

        static string CutBraces(string wikiMarkedUpText) {
            int numOpen = 0;
            int nextOpen = wikiMarkedUpText.IndexOf("{{", StringComparison.Ordinal);
            int nextClose = wikiMarkedUpText.IndexOf("}}", StringComparison.Ordinal);
            nextOpen = nextOpen == -1 ? int.MaxValue : nextOpen;
            nextClose = nextClose == -1 ? int.MaxValue : nextClose;
            var sb = new StringBuilder();
            int startAt = 0;
            while (true) {
                if (nextOpen < nextClose) {
                    if (numOpen == 0)
                        sb.Append(wikiMarkedUpText.Substring(startAt, nextOpen - startAt));
                    numOpen++;
                    nextOpen = wikiMarkedUpText.IndexOf("{{", nextOpen + 2, StringComparison.Ordinal);
                    nextOpen = nextOpen == -1 ? int.MaxValue : nextOpen;
                } else if (nextClose < nextOpen) {
                    //if (numOpen == 0) 
                    //throw new Exception("Invalid Wiki Text: unbalanced braces");
                    numOpen = Math.Max(numOpen - 1, 0);
                    if (numOpen == 0)
                        startAt = nextClose + 2;
                    nextClose = wikiMarkedUpText.IndexOf("}}", nextClose + 2, StringComparison.Ordinal);
                    nextClose = nextClose == -1 ? int.MaxValue : nextClose;
                } else if (numOpen == 0) { // nextOpen==nextClose implies both are not present.
                    sb.Append(wikiMarkedUpText.Substring(startAt, wikiMarkedUpText.Length - startAt));
                    break;
                } else {
                    //	throw new Exception("Invalid Wiki Text: unbalanced braces");
                    break;
                }
            }

            return sb.ToString();
        }


        static readonly string[] markupToStripRegexes = {
            @"(?>'')'*",
            @"(?><)(!--([^-]|-[^-]|--[^>])*-->|([mM][aA][tT][hH]|[rR][eE][fF]|[sS][mM][aA][lL][lL]).*?(/>|</([mM][aA][tT][hH]|[rR][eE][fF]|[sS][mM][aA][lL][lL])>))",
            @"^((?>#)[rR][eE][dD][iI][rR][eE][cC][tT].*$|(?>\*)\**|(?>=)=*)",
            @"(?<=&)(?>[aA])[mM][pP];",
            @"&(?>[nN])([bB][sS][pP]|[dD][aA][sS][hH]);",
            @"=+ *$",
            @"\{\|([^\|]|\|[^\}])*\|\}",
            @"</?[a-zA-Z]+( [^>]*?)?/?>",
        };
        readonly Regex markupToStripRegex = new Regex(string.Join("|", markupToStripRegexes.Select(r => "(" + r + ")").ToArray()), options);

        static readonly string[] markupToReplaceRegexes = {
            @"(?>\[\[([^\[:\|\]]+):)([^\[\]]|\[\[[^\[\]]*\]\])*\]\]",
			@"\[([^ \[\]]+( (?<txt>[^\[\]]*))?|\[((?<txt>:[^\[\]]*)|(?<txt>[^\[\]:\|]*)|[^\[\]:\|]*\|(?<txt>[^\[\]]*))\])\]",
        };
        readonly Regex markupToReplaceRegex = new Regex(string.Join("|", markupToReplaceRegexes.Select(r => "(" + r + ")").ToArray()), options);
    }
}
<commit_msg>Optimize markup stripping regexes<commit_after>﻿using System.Text.RegularExpressions;
using System.Linq;
using System;
using System.Collections.Generic;
using System.Text;

namespace WikipediaAvsAnTrieExtractor {
    public partial class RegexTextUtils {
        //Note: regexes are NOT static and shared between threads because of... http://stackoverflow.com/questions/7585087/multithreaded-use-of-regex
        //This code is bottlenecked by regexes, so this really matters, here.


        public string StripWikiMarkup(string wikiMarkedUpText)
        {
            var withoutNonTrivialMarkup = markupToStripRegex.Replace(wikiMarkedUpText, "");
            var withoutBraces = CutBraces(withoutNonTrivialMarkup);
            var plainTextMarkupReplacedByContent =
                markupToReplaceRegex.Replace(withoutBraces, m => m.Groups["txt"].Value);
            return plainTextMarkupReplacedByContent;
        }

        static string CutBraces(string wikiMarkedUpText) {
            int numOpen = 0;
            int nextOpen = wikiMarkedUpText.IndexOf("{{", StringComparison.Ordinal);
            int nextClose = wikiMarkedUpText.IndexOf("}}", StringComparison.Ordinal);
            nextOpen = nextOpen == -1 ? int.MaxValue : nextOpen;
            nextClose = nextClose == -1 ? int.MaxValue : nextClose;
            var sb = new StringBuilder();
            int startAt = 0;
            while (true) {
                if (nextOpen < nextClose) {
                    if (numOpen == 0)
                        sb.Append(wikiMarkedUpText.Substring(startAt, nextOpen - startAt));
                    numOpen++;
                    nextOpen = wikiMarkedUpText.IndexOf("{{", nextOpen + 2, StringComparison.Ordinal);
                    nextOpen = nextOpen == -1 ? int.MaxValue : nextOpen;
                } else if (nextClose < nextOpen) {
                    //if (numOpen == 0) 
                    //throw new Exception("Invalid Wiki Text: unbalanced braces");
                    numOpen = Math.Max(numOpen - 1, 0);
                    if (numOpen == 0)
                        startAt = nextClose + 2;
                    nextClose = wikiMarkedUpText.IndexOf("}}", nextClose + 2, StringComparison.Ordinal);
                    nextClose = nextClose == -1 ? int.MaxValue : nextClose;
                } else if (numOpen == 0) { // nextOpen==nextClose implies both are not present.
                    sb.Append(wikiMarkedUpText.Substring(startAt, wikiMarkedUpText.Length - startAt));
                    break;
                } else {
                    //	throw new Exception("Invalid Wiki Text: unbalanced braces");
                    break;
                }
            }

            return sb.ToString();
        }


        static readonly string[] markupToStripRegexes = {
            @"(?>'''*)",
            @"(?><)(!--([^-]|-[^-]|--[^>])*-->|([mM][aA][tT][hH]|[rR][eE][fF]|[sS][mM][aA][lL][lL]).*?(/>|</([mM][aA][tT][hH]|[rR][eE][fF]|[sS][mM][aA][lL][lL])>)|/?[a-zA-Z]+[^>]*>)",
            @"^((?>#)[rR][eE][dD][iI][rR][eE][cC][tT].*$|(?>\*)\**|(?>=)=*)",
            @"(?>=+ *$)",
            @"(?<=&)[aA][mM][pP];",
            @"&[nN](?>[bB][sS][pP]|[dD][aA][sS][hH]);",
            @"\{\|(?>(?>[^\|]|\|[^\}])*)\|\}",
        };
        readonly Regex markupToStripRegex0 = new Regex(string.Join("|", markupToStripRegexes.Select(r => "(" + r + ")")), options);

        readonly Regex markupToStripRegex = new Regex(@"
(?>
'''*
|<(
  !--.*?-->
  |(
    [mM][aA][tT][hH]
    |[rR][eE][fF]
    |[sS][mM][aA][lL][lL]
    ).*?(
      />
      |</(
        [mM][aA][tT][hH]
        |[rR][eE][fF]
        |[sS][mM][aA][lL][lL]
        )>
      )
  |[^>]*>
  )
|=+[ ]*$
|&[nN]([bB][sS][pP]|[dD][aA][sS][hH]);
|\{\|([^\|]|\|[^\}])*\|\}
|(?<=&)[aA][mM][pP];
|^(\#[rR][eE][dD][iI][rR][eE][cC][tT][^\n]*|\*+|=+)
)
", options | RegexOptions.IgnorePatternWhitespace);

        readonly Regex markupToReplaceRegex = new Regex(@"
  \[
    (
      [^ \[\]]+
        ([ ](?<txt>[^\[\]]*))?
      |\[(
        (?<txt>:[^\[\]]*)
        |(?<txt>[^\[\]:\|]*)
        |[^\[\]:\|]*\|(?<txt>[^\[\]]*)
        |([^\[:\|\]]+):
		([^\[\]]|\[\[[^\[\]]*\]\])*   #support one level of nesting.
        )
        \]
    )
  \]
", options | RegexOptions.IgnorePatternWhitespace);
    }
}

{'fraction_non_alphanumeric': 0.20843734398402397, 'fraction_numerical': 0.005616575137294059, 'mean_word_length': 1.9340900768949103, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5323814', 'n_tokens_mistral': 2786, 'n_tokens_neox': 2575, 'n_words': 412}