<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Xml;
using NCalc;
using Newtonsoft.Json;

namespace SuperXml
{
    public static class Extensions
    {
        static Extensions()
        {
            IsExpressionRegex = new Regex("(?<={{).*?(?=}})");
            ForEachRegex = 
                new Regex(@"^\s*([a-zA-Z_]+[\w]*)\s+in\s+(([a-zA-Z][\w]*(\.[a-zA-Z][\w]*)*)|\[(.+)(,\s*.+)*\])\s*$");
            IsXmlTagClosedRegex = new Regex(@"^<.*\/\s*>.*$", RegexOptions.Singleline);
        }

        private static readonly Regex IsExpressionRegex;
        private static readonly Regex ForEachRegex;
        private static readonly Regex IsXmlTagClosedRegex;

        public static XmlNode Compile(this XmlNode node, Dictionary<string, dynamic> s)
        {
            var content = node.CompileString(s);
            var doc = new XmlDocument();
            doc.LoadXml(content);
            return doc.DocumentElement;
        }

        private static string CompileString(this XmlNode node, Dictionary<string, dynamic> s)
        {
            if (!node.If(s)) return "";

            var xml = "";
            var nodeMarkUp = "";

            foreach (var expansion in node.ForEach(s))
            {
                if (node.NodeType == XmlNodeType.Element)
                {
                    var isExpression = node.Name == "Expression";

                    xml = !isExpression
                        ? node.OuterXml.Substring(0, node.OuterXml.IndexOf(">", StringComparison.Ordinal) + 1)
                        : "";
                    nodeMarkUp = xml;

                    if (!isExpression)
                        foreach (Match e in IsExpressionRegex.Matches(xml))
                            xml = xml.Replace("{{" + e.Value + "}}",
                                EvaluateExpression(e.Value, expansion.Scope));

                    xml = node.ChildNodes.Cast<XmlNode>()
                        .Aggregate(xml, (current, child) => current + child.CompileString(expansion.Scope));
                }
                else
                {
                    foreach (Match e in IsExpressionRegex.Matches(node.InnerText))
                    {
                        xml = node.InnerText.Replace("{{" + e.Value + "}}", EvaluateExpression(e.Value, expansion.Scope));
                    }   
                }
            }
            return xml + (IsXmlTagClosedRegex.IsMatch(nodeMarkUp) ? "" : "</" + node.Name + ">");
        }

        private static IEnumerable<ExpandedNode> ForEach(this XmlNode node, Dictionary<string, dynamic> s)
        {
            if (node == null) yield break;
            var exp = node.ReadAttrAsString("ForEach");
            if (exp == null)
            {
                yield return new ExpandedNode { Node = node, Scope = s };
                yield break;
            }

            exp = exp.Replace(Environment.NewLine, "");
                
            if (!ForEachRegex.IsMatch(exp))
                throw new Exception(
                    "Compilation Error: ForEach was expecting an expression like " +
                    "'varName in [value1, value2, value3..., valueN]'");

            var match = ForEachRegex.Match(exp);
            var repeater = match.Groups[1].ToString();
            var scopeName = match.Groups[3].ToString();
            var scopeRoot = scopeName.Split('.')[0];
            var items = scopeName.Length > 0
                ? (s.ContainsKey(scopeRoot) ? NavigateTo(s[scopeRoot], scopeName) : new int[] { })
                : JsonConvert.DeserializeObject<dynamic>(match.Groups[2].ToString());

            var cs = s.ToDictionary(x => x.Key, x => x.Value);

            Action<string, dynamic> addToScope = (key, value) =>
            {
                if (cs.ContainsKey(key)) cs[key] = value;
                else cs.Add(key, value);
            };

            var i = 0;
            foreach (var item in items)
            {
                addToScope(repeater, item);
                addToScope("$index", i++);
                yield return new ExpandedNode { Node = node, Scope = cs };
            }
        }

        private static bool If(this XmlNode node, Dictionary<string, dynamic> s)
        {
            if (node == null) return false;
            var exp = node.ReadAttr("If");
            if (exp == null) return true;

            var eval = EvaluateExpression(exp, s);

            bool res;
            var couldConvert = bool.TryParse(eval, out res);

            return couldConvert && res;
        }

        public static string EvaluateExpression(string expression, Dictionary<string, dynamic> scope)
        {
            var originalExpression = expression;
#if(DEBUG)
            var starti = DateTime.Now;
#endif
            var p = 0;
            var parameters = new Dictionary<string, object>();
            var regex = new Regex(@"[\s|&=!<>+\-*/%^(]([A-Za-z_$]\w*(\.[A-Za-z_][\w()]*)*)");

            foreach (Match match in regex.Matches(" " + expression))
            {
                var g = match.Groups[1].Value;
                var varName = g.Split('.')[0].Trim();
                dynamic varValue = scope.ContainsKey(varName)
                    ? NavigateTo(scope[varName], g)
                    : "";
                expression = expression.Replace(g, "[p" + p + "]");
                parameters.Add("p" + p, varValue);
                p++;
            }
            if (string.IsNullOrWhiteSpace(expression)) return "";<commit_msg>Removed Diagnostics<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Xml;
using NCalc;
using Newtonsoft.Json;

namespace SuperXml
{
    public static class Extensions
    {
        static Extensions()
        {
            IsExpressionRegex = new Regex("(?<={{).*?(?=}})");
            ForEachRegex = 
                new Regex(@"^\s*([a-zA-Z_]+[\w]*)\s+in\s+(([a-zA-Z][\w]*(\.[a-zA-Z][\w]*)*)|\[(.+)(,\s*.+)*\])\s*$");
            IsXmlTagClosedRegex = new Regex(@"^<.*\/\s*>.*$", RegexOptions.Singleline);
        }

        private static readonly Regex IsExpressionRegex;
        private static readonly Regex ForEachRegex;
        private static readonly Regex IsXmlTagClosedRegex;

        public static XmlNode Compile(this XmlNode node, Dictionary<string, dynamic> s)
        {
            var content = node.CompileString(s);
            var doc = new XmlDocument();
            doc.LoadXml(content);
            return doc.DocumentElement;
        }

        private static string CompileString(this XmlNode node, Dictionary<string, dynamic> s)
        {
            if (!node.If(s)) return "";

            var xml = "";
            var nodeMarkUp = "";

            foreach (var expansion in node.ForEach(s))
            {
                if (node.NodeType == XmlNodeType.Element)
                {
                    var isExpression = node.Name == "Expression";

                    xml = !isExpression
                        ? node.OuterXml.Substring(0, node.OuterXml.IndexOf(">", StringComparison.Ordinal) + 1)
                        : "";
                    nodeMarkUp = xml;

                    if (!isExpression)
                        foreach (Match e in IsExpressionRegex.Matches(xml))
                            xml = xml.Replace("{{" + e.Value + "}}",
                                EvaluateExpression(e.Value, expansion.Scope));

                    xml = node.ChildNodes.Cast<XmlNode>()
                        .Aggregate(xml, (current, child) => current + child.CompileString(expansion.Scope));
                }
                else
                {
                    foreach (Match e in IsExpressionRegex.Matches(node.InnerText))
                    {
                        xml = node.InnerText.Replace("{{" + e.Value + "}}", EvaluateExpression(e.Value, expansion.Scope));
                    }   
                }
            }
            return xml + (IsXmlTagClosedRegex.IsMatch(nodeMarkUp) ? "" : "</" + node.Name + ">");
        }

        private static IEnumerable<ExpandedNode> ForEach(this XmlNode node, Dictionary<string, dynamic> s)
        {
            if (node == null) yield break;
            var exp = node.ReadAttrAsString("ForEach");
            if (exp == null)
            {
                yield return new ExpandedNode { Node = node, Scope = s };
                yield break;
            }

            exp = exp.Replace(Environment.NewLine, "");
                
            if (!ForEachRegex.IsMatch(exp))
                throw new Exception(
                    "Compilation Error: ForEach was expecting an expression like " +
                    "'varName in [value1, value2, value3..., valueN]'");

            var match = ForEachRegex.Match(exp);
            var repeater = match.Groups[1].ToString();
            var scopeName = match.Groups[3].ToString();
            var scopeRoot = scopeName.Split('.')[0];
            var items = scopeName.Length > 0
                ? (s.ContainsKey(scopeRoot) ? NavigateTo(s[scopeRoot], scopeName) : new int[] { })
                : JsonConvert.DeserializeObject<dynamic>(match.Groups[2].ToString());

            var cs = s.ToDictionary(x => x.Key, x => x.Value);

            Action<string, dynamic> addToScope = (key, value) =>
            {
                if (cs.ContainsKey(key)) cs[key] = value;
                else cs.Add(key, value);
            };

            var i = 0;
            foreach (var item in items)
            {
                addToScope(repeater, item);
                addToScope("$index", i++);
                yield return new ExpandedNode { Node = node, Scope = cs };
            }
        }

        private static bool If(this XmlNode node, Dictionary<string, dynamic> s)
        {
            if (node == null) return false;
            var exp = node.ReadAttr("If");
            if (exp == null) return true;

            var eval = EvaluateExpression(exp, s);

            bool res;
            var couldConvert = bool.TryParse(eval, out res);

            return couldConvert && res;
        }

        public static string EvaluateExpression(string expression, Dictionary<string, dynamic> scope)
        {
            var originalExpression = expression;
            var p = 0;
            var parameters = new Dictionary<string, object>();
            var regex = new Regex(@"[\s|&=!<>+\-*/%^(]([A-Za-z_$]\w*(\.[A-Za-z_][\w()]*)*)");

            foreach (Match match in regex.Matches(" " + expression))
            {
                var g = match.Groups[1].Value;
                var varName = g.Split('.')[0].Trim();
                dynamic varValue = scope.ContainsKey(varName)
                    ? NavigateTo(scope[varName], g)
                    : "";
                expression = expression.Replace(g, "[p" + p + "]");
                parameters.Add("p" + p, varValue);
                p++;
            }
            if (string.IsNullOrWhiteSpace(expression)) return "";
{'fraction_non_alphanumeric': 0.13029345372460496, 'fraction_numerical': 0.002528216704288939, 'mean_word_length': 1.552661903664439, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5287012', 'n_tokens_mistral': 3051, 'n_tokens_neox': 2872, 'n_words': 674}