<commit_before>﻿using DocumentFormat.OpenXml;
using DocumentFormat.OpenXml.Wordprocessing;
using Signum.Engine.DynamicQuery;
using Signum.Engine.Templating;
using Signum.Engine.Translation;
using Signum.Entities;
using Signum.Entities.DynamicQuery;
using Signum.Entities.Reflection;
using Signum.Entities.UserAssets;
using Signum.Entities.Word;
using Signum.Utilities;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml;
using System.Globalization;
using Signum.Utilities.DataStructures;
using System.IO;

namespace Signum.Engine.Word
{
    public class MatchNode : Run
    {
        public Match Match;

        public MatchNode(Match match)
        {
            this.Match = match;
        }

        public override string ToString()
        {
            return "Match " + Match.ToString();
        }

        public override string LocalName
        {
            get { return this.GetType().Name; }
        }

        public override void WriteTo(System.Xml.XmlWriter xmlWriter)
        {
            var tempText = new Text(Match.ToString());

            this.AppendChild(tempText);
            base.WriteTo(xmlWriter);
            this.RemoveChild(tempText);
        }
    }

    public abstract class BaseNode : Run
    {
        public BaseNode() { }

        public BaseNode(BaseNode original)
        {
            this.SetAttributes(original.GetAttributes().ToList());
            foreach (var item in original.ChildElements)
            {
                this.AppendChild(item.CloneNode(true));
            }
        }

        public abstract void FillTokens(List<QueryToken> tokens);

        public override string LocalName
        {
            get { return this.GetType().Name; }
        }

        internal protected abstract void RenderNode(WordTemplateParameters p);

        internal protected abstract void RenderTemplate(ScopedDictionary<string, ValueProviderBase> variables);

        public override string ToString()
        {
            return this.GetType().Name;
        }

        public abstract override OpenXmlElement CloneNode(bool deep);

        public abstract void Synchronize(SyncronizationContext sc);
    }

    public class TokenNode : BaseNode
    {
        public readonly ValueProviderBase ValueProvider;
        public readonly string Format;

        internal TokenNode(ValueProviderBase valueProvider, string format)
        {
            this.ValueProvider = valueProvider;
            this.Format = format;
        }

        internal TokenNode(TokenNode original) : base(original)
        {
            this.ValueProvider = original.ValueProvider;
            this.Format = original.Format;
        }

        public override void FillTokens(List<QueryToken> tokens)
        {
            ValueProvider.FillQueryTokens(tokens);
        }

        internal protected override void RenderNode(WordTemplateParameters p)
        {
            object obj = ValueProvider.GetValue(p);
            string text = obj is Enum ? ((Enum)obj).NiceToString() :
                obj is IFormattable ? ((IFormattable)obj).ToString(Format ?? ValueProvider.Format, p.Culture) :
                obj.TryToString();

            this.ReplaceBy(new Run(this.RunProperties.TryDo(prop => prop.Remove()), new Text(text)));
        }

        protected internal override void RenderTemplate(ScopedDictionary<string, ValueProviderBase> variables)
        {
            var str = "@" + this.ValueProvider.ToString(variables, Format.HasText() ? (":" + Format) : null);

            this.ReplaceBy(new Run(this.RunProperties.TryDo(prop => prop.Remove()), new Text(str)));
        }

        public override void WriteTo(System.Xml.XmlWriter xmlWriter)
        {
            var tempText = new Text(ValueProvider.ToString());

            this.AppendChild(tempText);
            base.WriteTo(xmlWriter);
            this.RemoveChild(tempText);
        }

        public override OpenXmlElement CloneNode(bool deep)
        {
            return new TokenNode(this);
        }

        public override void Synchronize(SyncronizationContext sc)
        {
            ValueProvider.Synchronize(sc, "@");

            ValueProvider.Declare(sc.Variables);
        }
    }

    public class DeclareNode : BaseNode
    {
        public readonly ValueProviderBase ValueProvider;

        internal DeclareNode(ValueProviderBase valueProvider, Action<bool, string> addError)
        {
            if (!valueProvider.Variable.HasText())
                addError(true, "declare{0} should end with 'as $someVariable'".FormatWith(valueProvider.ToString()));

            this.ValueProvider = valueProvider;
        }

        public DeclareNode(DeclareNode original)
            : base(original)
        {
            this.ValueProvider = original.ValueProvider;
        }

        public override void WriteTo(System.Xml.XmlWriter xmlWriter)
        {
            var tempText = new Text(ValueProvider.ToString() ?? "Error!");

            this.AppendChild(tempText);
            base.WriteTo(xmlWriter);
            this.RemoveChild(tempText);
        }

        public override void FillTokens(List<QueryToken> tokens)
        {
        }

        public override OpenXmlElement CloneNode(bool deep)
        {
            return new DeclareNode(this);
        }

        protected internal override void RenderNode(WordTemplateParameters p)
        {
            this.Remove();
        }

        protected internal override void RenderTemplate(ScopedDictionary<string, ValueProviderBase> variables)
        {
            string str = "@declare" + ValueProvider.ToString(variables, null);

            this.ReplaceBy(new Run(this.RunProperties.TryDo(prop => prop.Remove()), new Text(str)));

            ValueProvider.Declare(variables);
        }

        public override void Synchronize(SyncronizationContext sc)
        {
            ValueProvider.Synchronize(sc, "@declare");
        }
    }


    public class BlockNode : BaseNode
    {
        public BlockNode() { }

        public BlockNode(BlockNode original) : base(original) { }

        public override OpenXmlElement CloneNode(bool deep)
        {
            return new BlockNode(this);
        }

        public override void FillTokens(List<QueryToken> tokens)
        {
            foreach (var item in this.Descendants<BaseNode>().ToList())
            {
                item.FillTokens(tokens);
            }
        }

        protected internal override void RenderNode(WordTemplateParameters p)
        {
            foreach (var item in this.Descendants<BaseNode>().ToList())
            {
                item.RenderNode(p);
            }

            var parent = this.Parent;
            int index = parent.ChildElements.IndexOf(this);
            parent.RemoveChild(this);
  
            foreach (var item in this.ChildElements.ToList())
            {
                item.Remove();
                parent.InsertAt(item, index++);
            }   
        }

        protected internal override void RenderTemplate(ScopedDictionary<string, ValueProviderBase> variables)
        {
            foreach (var item in this.Descendants<BaseNode>().ToList())
            {
                item.RenderTemplate(variables);
            }
        }

        public override void Synchronize(SyncronizationContext sc)
        {
            foreach (var item in this.Descendants<BaseNode>().ToList())
            {
                item.Synchronize(sc);
            }
        }
    }

    public abstract class BlockContainerNode : BaseNode
    {
        public BlockContainerNode() { }

        public BlockContainerNode(BlockContainerNode original) : base(original) { }

        public static string UserString(Type type)
        {
            if (type == typeof(ForeachNode))
                return "foreach";

            if (type == typeof(IfNode))
                return "if";

            if (type == typeof(AnyNode))
                return "any";

            return "block";
        }

        protected internal abstract void ReplaceBlock();

        protected void NormalizeInterval(ref MatchNodePair first, ref MatchNodePair last, MatchNode errorHintParent)
        {
            if (first.MatchNode == last.MatchNode)
                throw new ArgumentException("first and last are the same node");

            var chainFirst = ((OpenXmlElement)first.MatchNode).Follow(a => a.Parent).Reverse().ToList();
            var chainLast = ((OpenXmlElement)last.MatchNode).Follow(a => a.Parent).Reverse().ToList();

            var result = chainFirst.Zip(chainLast, (f, l) => new { f, l }).First(a => a.f != a.l);
            AssertNotImportant(chainFirst, result.f, errorHintParent, first.MatchNode, last.MatchNode);
            AssertNotImportant(chainLast, result.l, errorHintParent, last.MatchNode, first.MatchNode);

            first.AscendantNode = result.f;
            last.AscendantNode = result.l;
        }

        private void AssertNotImportant(List<OpenXmlElement> chain, OpenXmlElement openXmlElement, MatchNode errorHintParent, MatchNode errorHint1, MatchNode errorHint2)
        {
            var index = chain.IndexOf(openXmlElement);

            for (int i = index; i < chain.Count; i++)
            {
                var current = chain[i];
                var next = i == chain.Count - 1 ? null : chain[i + 1];

                var important = current.ChildElements.Where(c => c != next && IsImportant(c));

                if (important.Any())
                    throw new InvalidOperationException("Node {0} is not at the same level than {1}{2}. Important nodes could be removed close to {0}:\r\n{3}".FormatWith(
                        errorHint1.Match,
                        errorHint2.Match,
                        errorHintParent != errorHint1 && errorHintParent != errorHint2 ? " in " + errorHintParent.Match : "",
                        current.NiceToString()));
            }
        }

        private bool IsImportant(OpenXmlElement c)
        {
            if (c is Paragraph)
                return true;

            if (c is Run)
            {
                var text = c.ChildElements.Where(a => !(a is RunProperties)).Only() as Text;

                if (text != null && string.IsNullOrWhiteSpace(text.Text))
                    return false;

                return true; 
            }

            return false;
        }

        protected static List<OpenXmlElement> NodesBetween(MatchNodePair first, MatchNodePair last)
        {
            var parent = first.CommonParent(last);

            int indexFirst = parent.ChildElements.IndexOf(first.AscendantNode);
            if (indexFirst == -1)
                throw new InvalidOperationException("Element not found");

            int indexLast = parent.ChildElements.IndexOf(last.AscendantNode);
            if (indexLast == -1)<commit_msg>clean liens with DeclareNode only<commit_after>﻿using DocumentFormat.OpenXml;
using DocumentFormat.OpenXml.Wordprocessing;
using Signum.Engine.DynamicQuery;
using Signum.Engine.Templating;
using Signum.Engine.Translation;
using Signum.Entities;
using Signum.Entities.DynamicQuery;
using Signum.Entities.Reflection;
using Signum.Entities.UserAssets;
using Signum.Entities.Word;
using Signum.Utilities;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml;
using System.Globalization;
using Signum.Utilities.DataStructures;
using System.IO;

namespace Signum.Engine.Word
{
    public class MatchNode : Run
    {
        public Match Match;

        public MatchNode(Match match)
        {
            this.Match = match;
        }

        public override string ToString()
        {
            return "Match " + Match.ToString();
        }

        public override string LocalName
        {
            get { return this.GetType().Name; }
        }

        public override void WriteTo(System.Xml.XmlWriter xmlWriter)
        {
            var tempText = new Text(Match.ToString());

            this.AppendChild(tempText);
            base.WriteTo(xmlWriter);
            this.RemoveChild(tempText);
        }
    }

    public abstract class BaseNode : Run
    {
        public BaseNode() { }

        public BaseNode(BaseNode original)
        {
            this.SetAttributes(original.GetAttributes().ToList());
            foreach (var item in original.ChildElements)
            {
                this.AppendChild(item.CloneNode(true));
            }
        }

        public abstract void FillTokens(List<QueryToken> tokens);

        public override string LocalName
        {
            get { return this.GetType().Name; }
        }

        internal protected abstract void RenderNode(WordTemplateParameters p);

        internal protected abstract void RenderTemplate(ScopedDictionary<string, ValueProviderBase> variables);

        public override string ToString()
        {
            return this.GetType().Name;
        }

        public abstract override OpenXmlElement CloneNode(bool deep);

        public abstract void Synchronize(SyncronizationContext sc);
    }

    public class TokenNode : BaseNode
    {
        public readonly ValueProviderBase ValueProvider;
        public readonly string Format;

        internal TokenNode(ValueProviderBase valueProvider, string format)
        {
            this.ValueProvider = valueProvider;
            this.Format = format;
        }

        internal TokenNode(TokenNode original) : base(original)
        {
            this.ValueProvider = original.ValueProvider;
            this.Format = original.Format;
        }

        public override void FillTokens(List<QueryToken> tokens)
        {
            ValueProvider.FillQueryTokens(tokens);
        }

        internal protected override void RenderNode(WordTemplateParameters p)
        {
            object obj = ValueProvider.GetValue(p);
            string text = obj is Enum ? ((Enum)obj).NiceToString() :
                obj is IFormattable ? ((IFormattable)obj).ToString(Format ?? ValueProvider.Format, p.Culture) :
                obj.TryToString();

            this.ReplaceBy(new Run(this.RunProperties.TryDo(prop => prop.Remove()), new Text(text)));
        }

        protected internal override void RenderTemplate(ScopedDictionary<string, ValueProviderBase> variables)
        {
            var str = "@" + this.ValueProvider.ToString(variables, Format.HasText() ? (":" + Format) : null);

            this.ReplaceBy(new Run(this.RunProperties.TryDo(prop => prop.Remove()), new Text(str)));
        }

        public override void WriteTo(System.Xml.XmlWriter xmlWriter)
        {
            var tempText = new Text(ValueProvider.ToString());

            this.AppendChild(tempText);
            base.WriteTo(xmlWriter);
            this.RemoveChild(tempText);
        }

        public override OpenXmlElement CloneNode(bool deep)
        {
            return new TokenNode(this);
        }

        public override void Synchronize(SyncronizationContext sc)
        {
            ValueProvider.Synchronize(sc, "@");

            ValueProvider.Declare(sc.Variables);
        }
    }

    public class DeclareNode : BaseNode
    {
        public readonly ValueProviderBase ValueProvider;

        internal DeclareNode(ValueProviderBase valueProvider, Action<bool, string> addError)
        {
            if (!valueProvider.Variable.HasText())
                addError(true, "declare{0} should end with 'as $someVariable'".FormatWith(valueProvider.ToString()));

            this.ValueProvider = valueProvider;
        }

        public DeclareNode(DeclareNode original)
            : base(original)
        {
            this.ValueProvider = original.ValueProvider;
        }

        public override void WriteTo(System.Xml.XmlWriter xmlWriter)
        {
            var tempText = new Text(ValueProvider.ToString() ?? "Error!");

            this.AppendChild(tempText);
            base.WriteTo(xmlWriter);
            this.RemoveChild(tempText);
        }

        public override void FillTokens(List<QueryToken> tokens)
        {
        }

        public override OpenXmlElement CloneNode(bool deep)
        {
            return new DeclareNode(this);
        }

        protected internal override void RenderNode(WordTemplateParameters p)
        {
            if (this.Parent is Paragraph && !this.Parent.ChildElements.Any(a => BlockContainerNode.IsImportant (a) && a != this))
                this.Parent.Remove();
            else
                this.Remove();
        }

        protected internal override void RenderTemplate(ScopedDictionary<string, ValueProviderBase> variables)
        {
            string str = "@declare" + ValueProvider.ToString(variables, null);

            this.ReplaceBy(new Run(this.RunProperties.TryDo(prop => prop.Remove()), new Text(str)));

            ValueProvider.Declare(variables);
        }

        public override void Synchronize(SyncronizationContext sc)
        {
            ValueProvider.Synchronize(sc, "@declare");
        }
    }


    public class BlockNode : BaseNode
    {
        public BlockNode() { }

        public BlockNode(BlockNode original) : base(original) { }

        public override OpenXmlElement CloneNode(bool deep)
        {
            return new BlockNode(this);
        }

        public override void FillTokens(List<QueryToken> tokens)
        {
            foreach (var item in this.Descendants<BaseNode>().ToList())
            {
                item.FillTokens(tokens);
            }
        }

        protected internal override void RenderNode(WordTemplateParameters p)
        {
            foreach (var item in this.Descendants<BaseNode>().ToList())
            {
                item.RenderNode(p);
            }

            var parent = this.Parent;
            int index = parent.ChildElements.IndexOf(this);
            parent.RemoveChild(this);
  
            foreach (var item in this.ChildElements.ToList())
            {
                item.Remove();
                parent.InsertAt(item, index++);
            }   
        }

        protected internal override void RenderTemplate(ScopedDictionary<string, ValueProviderBase> variables)
        {
            foreach (var item in this.Descendants<BaseNode>().ToList())
            {
                item.RenderTemplate(variables);
            }
        }

        public override void Synchronize(SyncronizationContext sc)
        {
            foreach (var item in this.Descendants<BaseNode>().ToList())
            {
                item.Synchronize(sc);
            }
        }
    }

    public abstract class BlockContainerNode : BaseNode
    {
        public BlockContainerNode() { }

        public BlockContainerNode(BlockContainerNode original) : base(original) { }

        public static string UserString(Type type)
        {
            if (type == typeof(ForeachNode))
                return "foreach";

            if (type == typeof(IfNode))
                return "if";

            if (type == typeof(AnyNode))
                return "any";

            return "block";
        }

        protected internal abstract void ReplaceBlock();

        protected void NormalizeInterval(ref MatchNodePair first, ref MatchNodePair last, MatchNode errorHintParent)
        {
            if (first.MatchNode == last.MatchNode)
                throw new ArgumentException("first and last are the same node");

            var chainFirst = ((OpenXmlElement)first.MatchNode).Follow(a => a.Parent).Reverse().ToList();
            var chainLast = ((OpenXmlElement)last.MatchNode).Follow(a => a.Parent).Reverse().ToList();

            var result = chainFirst.Zip(chainLast, (f, l) => new { f, l }).First(a => a.f != a.l);
            AssertNotImportant(chainFirst, result.f, errorHintParent, first.MatchNode, last.MatchNode);
            AssertNotImportant(chainLast, result.l, errorHintParent, last.MatchNode, first.MatchNode);

            first.AscendantNode = result.f;
            last.AscendantNode = result.l;
        }

        private void AssertNotImportant(List<OpenXmlElement> chain, OpenXmlElement openXmlElement, MatchNode errorHintParent, MatchNode errorHint1, MatchNode errorHint2)
        {
            var index = chain.IndexOf(openXmlElement);

            for (int i = index; i < chain.Count; i++)
            {
                var current = chain[i];
                var next = i == chain.Count - 1 ? null : chain[i + 1];

                var important = current.ChildElements.Where(c => c != next && IsImportant(c));

                if (important.Any())
                    throw new InvalidOperationException("Node {0} is not at the same level than {1}{2}. Important nodes could be removed close to {0}:\r\n{3}".FormatWith(
                        errorHint1.Match,
                        errorHint2.Match,
                        errorHintParent != errorHint1 && errorHintParent != errorHint2 ? " in " + errorHintParent.Match : "",
                        current.NiceToString()));
            }
        }

        public static bool IsImportant(OpenXmlElement c)
        {
            if (c is Paragraph)
                return true;

            if (c is Run)
            {
                var text = c.ChildElements.Where(a => !(a is RunProperties)).Only() as Text;

                if (text != null && string.IsNullOrWhiteSpace(text.Text))
                    return false;

                return true; 
            }

            if (c is BaseNode)
                return true;

            return false;
        }

        protected static List<OpenXmlElement> NodesBetween(MatchNodePair first, MatchNodePair last)
        {
            var parent = first.CommonParent(last);

            int indexFirst = parent.ChildElements.IndexOf(first.AscendantNode);
            if (indexFirst == -1)
                throw new InvalidOperationException("Element not found");

            int indexLast = parent.ChildElements.IndexOf(last.AscendantNode);
            if (indexLast == -1)
{'fraction_non_alphanumeric': 0.09205095998564508, 'fraction_numerical': 0.0014354925533823794, 'mean_word_length': 2.008908084761776, 'pattern_counts': {'":': 2, '<': 35, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6607653', 'n_tokens_mistral': 5688, 'n_tokens_neox': 5488, 'n_words': 1307}