<commit_before>﻿using Antlr4.Runtime;
using Antlr4.Runtime.Tree;
using MetaDslx.Core;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace MetaDslx.Compiler
{
    public class Antlr4DefaultNameProvider : DefaultNameProvider
    {
        public override string GetName(object node)
        {
            IParseTree parseTree = node as IParseTree;
            if (parseTree == null) return null;
            string name = parseTree.GetText();
            if (name != null && name.StartsWith("@")) name = name.Substring(1);
            return name;
        }

        public override object GetValue(object node, Type type)
        {
            IParseTree parseTree = node as IParseTree;
            if (parseTree == null) return null;
            string text = parseTree.GetText();
            if (text.Length >= 3 && text.StartsWith("@\'") && text.EndsWith("\'"))
            {
                return text.Substring(2, text.Length - 3).Replace("\'\'", "\'");
            }
            else if (text.Length >= 2 && text.StartsWith("\'") && text.EndsWith("\'"))
            {
                return Regex.Unescape(text.Substring(1, text.Length - 2));
            }
            else if (text.Length >= 3 && text.StartsWith("@\"") && text.EndsWith("\""))
            {
                return text.Substring(2, text.Length - 3).Replace("\"\"", "\"");
            }
            else if (text.Length >= 2 && text.StartsWith("\"") && text.EndsWith("\""))
            {
                return Regex.Unescape(text.Substring(1, text.Length - 2));
            }
            return base.GetValue(text, type);
        }

        public override TextSpan GetTreeNodeTextSpan(object node)
        {
            return new Antlr4TextSpan(node);
        }
    }

    public class Antlr4DefaultTriviaProvider : DefaultTriviaProvider
    {
        private IAntlr4Compiler compiler;
        private int channel;

        public Antlr4DefaultTriviaProvider(IAntlr4Compiler compiler, int channel = -1)
        {
            this.compiler = compiler;
            this.channel = channel;
            this.CleanTrivia = true;
            this.TrimTriviaLines = true;
            this.ConcatTriviaLines = false;
            this.TriviaStartTokens = new List<string>();
            this.TriviaEndTokens = new List<string>();
            this.TriviaLineStartTokens = new List<string>();
            this.TriviaLineEndTokens = new List<string>();
            this.TriviaStartTokens.Add("///");
            this.TriviaStartTokens.Add("/**");
            this.TriviaStartTokens.Add("//");
            this.TriviaStartTokens.Add("/*");
            this.TriviaEndTokens.Add("*/");
            this.TriviaLineStartTokens.Add("*");
        }

        public bool CleanTrivia { get; set; }
        public bool TrimTriviaLines { get; set; }
        public bool ConcatTriviaLines { get; set; }
        public List<string> TriviaStartTokens { get; private set; }
        public List<string> TriviaEndTokens { get; private set; }
        public List<string> TriviaLineStartTokens { get; private set; }
        public List<string> TriviaLineEndTokens { get; private set; }

        protected virtual string CleanTriviaText(string text)
        {
            if (text == null) return text;
            text = text.Trim();
            foreach (var token in this.TriviaStartTokens)
            {
                if (text.StartsWith(token))
                {
                    text = text.Substring(token.Length);
                    break;
                }
            }
            foreach (var token in this.TriviaEndTokens)
            {
                if (text.EndsWith(token))
                {
                    text = text.Substring(0, text.Length - token.Length);
                    break;
                }
            }
            if (this.TrimTriviaLines)
            {
                text = text.Trim();
            }
            MemoryStream stream = new MemoryStream();
            StreamWriter writer = new StreamWriter(stream);
            writer.Write(text);
            writer.Flush();
            stream.Position = 0;
            StringBuilder sb = new StringBuilder();
            using (StreamReader reader = new StreamReader(stream))
            {
                while(!reader.EndOfStream)
                {
                    string line = reader.ReadLine();
                    if (line != null)
                    {
                        foreach (var token in this.TriviaLineStartTokens)
                        {
                            if (line.StartsWith(token))
                            {
                                line = line.Substring(token.Length);
                                break;
                            }
                        }
                        foreach (var token in this.TriviaLineEndTokens)
                        {
                            if (line.EndsWith(token))
                            {
                                line = line.Substring(token.Length);
                                break;
                            }
                        }
                        if (this.TrimTriviaLines)
                        {
                            line = line.Trim();
                        }
                        if (this.ConcatTriviaLines)
                        {
                            if (!string.IsNullOrWhiteSpace(line))
                            {
                                if (sb.Length != 0)
                                {
                                    sb.Append(" ");
                                }
                                sb.Append(line);
                            }
                        }
                        else<commit_msg>Synced with Java implementation<commit_after>﻿using Antlr4.Runtime;
using Antlr4.Runtime.Tree;
using MetaDslx.Core;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace MetaDslx.Compiler
{
    public class Antlr4DefaultNameProvider : DefaultNameProvider
    {
        public override string GetName(object node)
        {
            IParseTree parseTree = node as IParseTree;
            if (parseTree == null) return null;
            string name = parseTree.GetText();
            if (name != null && name.StartsWith("@")) name = name.Substring(1);
            return name;
        }

        public override object GetValue(object node, Type type)
        {
            IParseTree parseTree = node as IParseTree;
            if (parseTree == null) return null;
            string text = parseTree.GetText();
            if (text.Length >= 3 && text.StartsWith("@\'") && text.EndsWith("\'"))
            {
                return text.Substring(2, text.Length - 3).Replace("\'\'", "\'");
            }
            else if (text.Length >= 2 && text.StartsWith("\'") && text.EndsWith("\'"))
            {
                return Regex.Unescape(text.Substring(1, text.Length - 2));
            }
            else if (text.Length >= 3 && text.StartsWith("@\"") && text.EndsWith("\""))
            {
                return text.Substring(2, text.Length - 3).Replace("\"\"", "\"");
            }
            else if (text.Length >= 2 && text.StartsWith("\"") && text.EndsWith("\""))
            {
                return Regex.Unescape(text.Substring(1, text.Length - 2));
            }
            return base.GetValue(text, type);
        }

        public override TextSpan GetTreeNodeTextSpan(object node)
        {
            return new Antlr4TextSpan(node);
        }
    }

    public class Antlr4DefaultTriviaProvider : DefaultTriviaProvider
    {
        private IAntlr4Compiler compiler;
        private int channel;

        public Antlr4DefaultTriviaProvider(IAntlr4Compiler compiler, int channel = -1)
        {
            this.compiler = compiler;
            this.channel = channel;
            this.CleanTrivia = true;
            this.TrimTriviaLines = true;
            this.ConcatTriviaLines = false;
            this.TriviaStartTokens = new List<string>();
            this.TriviaEndTokens = new List<string>();
            this.TriviaLineStartTokens = new List<string>();
            this.TriviaLineEndTokens = new List<string>();
            this.TriviaStartTokens.Add("///");
            this.TriviaStartTokens.Add("/**");
            this.TriviaStartTokens.Add("//");
            this.TriviaStartTokens.Add("/*");
            this.TriviaEndTokens.Add("*/");
            this.TriviaLineStartTokens.Add("*");
        }

        public bool CleanTrivia { get; set; }
        public bool TrimTriviaLines { get; set; }
        public bool ConcatTriviaLines { get; set; }
        public List<string> TriviaStartTokens { get; private set; }
        public List<string> TriviaEndTokens { get; private set; }
        public List<string> TriviaLineStartTokens { get; private set; }
        public List<string> TriviaLineEndTokens { get; private set; }

        protected virtual string CleanTriviaText(string text)
        {
            if (text == null) return text;
            text = text.Trim();
            foreach (var token in this.TriviaStartTokens)
            {
                if (text.StartsWith(token))
                {
                    text = text.Substring(token.Length);
                    break;
                }
            }
            foreach (var token in this.TriviaEndTokens)
            {
                if (text.EndsWith(token))
                {
                    text = text.Substring(0, text.Length - token.Length);
                    break;
                }
            }
            if (this.TrimTriviaLines)
            {
                text = text.Trim();
            }
            MemoryStream stream = new MemoryStream();
            StreamWriter writer = new StreamWriter(stream);
            writer.Write(text);
            writer.Flush();
            stream.Position = 0;
            StringBuilder sb = new StringBuilder();
            using (StreamReader reader = new StreamReader(stream))
            {
                while(!reader.EndOfStream)
                {
                    string line = reader.ReadLine();
                    if (line != null)
                    {
                        foreach (var token in this.TriviaLineStartTokens)
                        {
                            if (line.StartsWith(token))
                            {
                                line = line.Substring(token.Length);
                                break;
                            }
                        }
                        foreach (var token in this.TriviaLineEndTokens)
                        {
                            if (line.EndsWith(token))
                            {
                                line = line.Substring(0, line.Length - token.Length);
                                break;
                            }
                        }
                        if (this.TrimTriviaLines)
                        {
                            line = line.Trim();
                        }
                        if (this.ConcatTriviaLines)
                        {
                            if (!string.IsNullOrWhiteSpace(line))
                            {
                                if (sb.Length != 0)
                                {
                                    sb.Append(" ");
                                }
                                sb.Append(line);
                            }
                        }
                        else
{'fraction_non_alphanumeric': 0.1003818413237166, 'fraction_numerical': 0.004327535002121341, 'mean_word_length': 1.2083567547311223, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4734340', 'n_tokens_mistral': 2984, 'n_tokens_neox': 2928, 'n_words': 614}