<commit_before>﻿using System;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text.RegularExpressions;
using System.Web;

namespace Quobject.EngineIoClientDotNet.Modules
{
    public static class Global
    {
        public static string EncodeURIComponent(string str)
        {
            //http://stackoverflow.com/a/4550600/1109316
            return Uri.EscapeDataString(str);
        }

        public static string DecodeURIComponent(string str)
        {
			return Uri.UnescapeDataString(str);
        }

        public static string CallerName(string caller = "", int number = 0, string path = "")
        {
            var s = path.Split('\\');
            var fileName = s.LastOrDefault();
            if (path.Contains("SocketIoClientDotNet.Tests"))
            {
                path = "SocketIoClientDotNet.Tests";
            }
            else if (path.Contains("SocketIoClientDotNet"))
            {
                path = "SocketIoClientDotNet";
            }
            else if (path.Contains("EngineIoClientDotNet"))
            {
                path = "EngineIoClientDotNet";
            }

            return string.Format("{0}-{1}:{2}#{3}",path, fileName, caller, number);
        }

        //from http://stackoverflow.com/questions/8767103/how-to-remove-invalid-code-points-from-a-string
        public static string StripInvalidUnicodeCharacters(string str)
        {
            var invalidCharactersRegex = new Regex("([\ud800-\udbff](?![\udc00-\udfff]))|((?<![\ud800-\udbff])[\udc00-\udfff])");
            return invalidCharactersRegex.Replace(str, "");<commit_msg>Tabs to spaces<commit_after>﻿using System;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text.RegularExpressions;
using System.Web;

namespace Quobject.EngineIoClientDotNet.Modules
{
    public static class Global
    {
        public static string EncodeURIComponent(string str)
        {
            //http://stackoverflow.com/a/4550600/1109316
            return Uri.EscapeDataString(str);
        }

        public static string DecodeURIComponent(string str)
        {
            return Uri.UnescapeDataString(str);
        }

        public static string CallerName(string caller = "", int number = 0, string path = "")
        {
            var s = path.Split('\\');
            var fileName = s.LastOrDefault();
            if (path.Contains("SocketIoClientDotNet.Tests"))
            {
                path = "SocketIoClientDotNet.Tests";
            }
            else if (path.Contains("SocketIoClientDotNet"))
            {
                path = "SocketIoClientDotNet";
            }
            else if (path.Contains("EngineIoClientDotNet"))
            {
                path = "EngineIoClientDotNet";
            }

            return string.Format("{0}-{1}:{2}#{3}",path, fileName, caller, number);
        }

        //from http://stackoverflow.com/questions/8767103/how-to-remove-invalid-code-points-from-a-string
        public static string StripInvalidUnicodeCharacters(string str)
        {
            var invalidCharactersRegex = new Regex("([\ud800-\udbff](?![\udc00-\udfff]))|((?<![\ud800-\udbff])[\udc00-\udfff])");
            return invalidCharactersRegex.Replace(str, "");
{'fraction_non_alphanumeric': 0.12307692307692308, 'fraction_numerical': 0.022153846153846152, 'mean_word_length': 2.119961612284069, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3014045', 'n_tokens_mistral': 948, 'n_tokens_neox': 907, 'n_words': 160}