<commit_before>﻿using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

namespace Take.Blip.Builder.Utils
{
    public class VariableReplacer : IVariableReplacer
    {
        private static readonly Regex TextVariablesRegex = new Regex(@"{{([a-zA-Z0-9\.@]+)}}", RegexOptions.Compiled);

        public async Task<string> ReplaceAsync(string value, IContext context, CancellationToken cancellationToken)
        {
            var variableValues = new Dictionary<string, string>();
            foreach (Match match in TextVariablesRegex.Matches(value))
            {
                var variableName = match.Groups[1].Value;
                if (variableValues.ContainsKey(variableName)) continue;
                var variableValue = await context.GetVariableAsync(variableName, cancellationToken);
                variableValues.Add(variableName, EscapeString(variableValue));
            }
            
            if (variableValues.Count > 0)
            {
                value = TextVariablesRegex.Replace(value, match => variableValues[match.Groups[1].Value]);
            }

            return value;
        }

        /// <summary>
        /// https://stackoverflow.com/a/39527149/704742
        /// </summary>
        /// <param name="src"></param>
        /// <returns></returns>
        private static string EscapeString(string src)
        {
            var sb = new StringBuilder();

            int start = 0;
            for (int i = 0; i < src.Length; i++)
                if (NeedEscape(src, i))
                {
                    sb.Append(src, start, i - start);
                    switch (src[i])
                    {
                        case '\b': sb.Append("\\b"); break;
                        case '\f': sb.Append("\\f"); break;
                        case '\n': sb.Append("\\n"); break;
                        case '\r': sb.Append("\\r"); break;
                        case '\t': sb.Append("\\t"); break;
                        case '\"': sb.Append("\\\""); break;
                        case '\\': sb.Append("\\\\"); break;
                        case '/': sb.Append("\\/"); break;
                        default:
                            sb.Append("\\u");
                            sb.Append(((int)src[i]).ToString("x04"));
                            break;
                    }
                    start = i + 1;<commit_msg>Handling null variable values on escaping<commit_after>﻿using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

namespace Take.Blip.Builder.Utils
{
    public class VariableReplacer : IVariableReplacer
    {
        private static readonly Regex TextVariablesRegex = new Regex(@"{{([a-zA-Z0-9\.@]+)}}", RegexOptions.Compiled);

        public async Task<string> ReplaceAsync(string value, IContext context, CancellationToken cancellationToken)
        {
            var variableValues = new Dictionary<string, string>();
            foreach (Match match in TextVariablesRegex.Matches(value))
            {
                var variableName = match.Groups[1].Value;
                if (variableValues.ContainsKey(variableName)) continue;
                var variableValue = await context.GetVariableAsync(variableName, cancellationToken);
                variableValues.Add(variableName, EscapeString(variableValue));
            }
            
            if (variableValues.Count > 0)
            {
                value = TextVariablesRegex.Replace(value, match => variableValues[match.Groups[1].Value]);
            }

            return value;
        }

        /// <summary>
        /// https://stackoverflow.com/a/39527149/704742
        /// </summary>
        /// <param name="src"></param>
        /// <returns></returns>
        private static string EscapeString(string src)
        {
            if (string.IsNullOrWhiteSpace(src)) return src;

            var sb = new StringBuilder();

            int start = 0;
            for (int i = 0; i < src.Length; i++)
                if (NeedEscape(src, i))
                {
                    sb.Append(src, start, i - start);
                    switch (src[i])
                    {
                        case '\b': sb.Append("\\b"); break;
                        case '\f': sb.Append("\\f"); break;
                        case '\n': sb.Append("\\n"); break;
                        case '\r': sb.Append("\\r"); break;
                        case '\t': sb.Append("\\t"); break;
                        case '\"': sb.Append("\\\""); break;
                        case '\\': sb.Append("\\\\"); break;
                        case '/': sb.Append("\\/"); break;
                        default:
                            sb.Append("\\u");
                            sb.Append(((int)src[i]).ToString("x04"));
                            break;
                    }
                    start = i + 1;
{'fraction_non_alphanumeric': 0.1329882540314553, 'fraction_numerical': 0.009556042205853077, 'mean_word_length': 1.4270531400966184, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2551478', 'n_tokens_mistral': 1303, 'n_tokens_neox': 1220, 'n_words': 278}