<commit_before>﻿using System;
using System.Buffers;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using DuoVia.FuzzyStrings;
using HomoglyphConverter;
using Microsoft.Extensions.Caching.Memory;

namespace CompatBot.Utils
{
    public static class StringUtils
    {
        private static readonly Encoding Latin8BitEncoding = Encoding.GetEncodings()
                                                                 .FirstOrDefault(e => e.CodePage == 1250 || e.CodePage == 1252 || e.CodePage == 28591)?
                                                                 .GetEncoding()
                                                             ?? Encoding.ASCII;
        private static readonly Encoding Utf8 = new UTF8Encoding(false);
        private static readonly MemoryCache FuzzyPairCache = new MemoryCache(new MemoryCacheOptions {ExpirationScanFrequency = TimeSpan.FromMinutes(10)});
        private const char StrikeThroughChar = '\u0336'; // 0x0335 = short dash, 0x0336 = long dash, 0x0337 = short slash, 0x0338 = long slash

        private static readonly HashSet<char> SpaceCharacters = new HashSet<char>
        {
            '\u00a0',
            '\u2002', '\u2003', '\u2004', '\u2005', '\u2006',
            '\u2007', '\u2008', '\u2009', '\u200a', '\u200b',
            '\u200c', '\u200d', '\u200e', '\u200f',
            '\u2028', '\u2029', '\u202a', '\u202b', '\u202c',
            '\u202c', '\u202d', '\u202e', '\u202f',
            '\u205f', '\u2060', '\u2061', '\u2062', '\u2063',
            '\u2064', '\u2065', '\u2066', '\u2067', '\u2068',
            '\u2069', '\u206a', '\u206b', '\u206c', '\u206d',
            '\u206e', '\u206f',
            '\u3000', '\u303f',
        };

        public static string StripMarks(this string str)
        {
            return str?.Replace("(R)", "", StringComparison.InvariantCultureIgnoreCase)
                .Replace("®", "", StringComparison.InvariantCultureIgnoreCase)
                .Replace("(TM)", "", StringComparison.InvariantCultureIgnoreCase)
                .Replace("™", "", StringComparison.InvariantCultureIgnoreCase);
        }

        public static string StripQuotes(this string str)
        {
            if (str == null || str.Length < 2)
                return str;

            if (str.StartsWith('"') && str.EndsWith('"'))
                return str.Substring(1, str.Length - 2);
            return str;
        }

        public static string TrimEager(this string str)
        {
            if (string.IsNullOrEmpty(str))
                return str;

            int start, end;
            for (start = 0; start < str.Length; start++)
            {
                if (char.IsWhiteSpace(str, start) || IsFormat(str[start]))
                    continue;

                if (char.IsHighSurrogate(str, start)
                    && char.GetUnicodeCategory(str, start) == UnicodeCategory.OtherNotAssigned
                    && str[start] >= 0xdb40) // this will check if the surrogate pair is >= E0000 (see https://en.wikipedia.org/wiki/UTF-16#U+010000_to_U+10FFFF)
                    continue;

                if (char.IsLowSurrogate(str, start))
                    continue;

                break;
            }

            for (end = str.Length - 1; end >= start; end--)
            {
                if (char.IsWhiteSpace(str, end) || IsFormat(str[end]))
                    continue;

                if (char.IsLowSurrogate(str, end)
                    && end > start
                    && char.IsHighSurrogate(str, end - 1)
                    && char.GetUnicodeCategory(str, end - 1) == UnicodeCategory.OtherNotAssigned
                    && str[end-1] >= 0xdb40)
                    continue;

                if (char.IsHighSurrogate(str, end)
                    && char.GetUnicodeCategory(str, end) == UnicodeCategory.OtherNotAssigned
                    && str[end] >= 0xdb40)
                    continue;

                break;
            }

            return CreateTrimmedString(str, start, end);
        }

        public static string AsString(this ReadOnlySequence<byte> buffer, Encoding encoding = null)
        {
            encoding = encoding ?? Latin8BitEncoding;
            if (buffer.IsSingleSegment)
                return encoding.GetString(buffer.First.Span);

            void Splice(Span<char> span, ReadOnlySequence<byte> sequence)
            {
                foreach (var segment in sequence)
                {
                    encoding.GetChars(segment.Span, span);
                    span = span.Slice(segment.Length);
                }
            }
            return string.Create((int)buffer.Length, buffer, Splice);
        }

        public static string ToUtf8(this string str)
        {
            return Utf8.GetString(Latin8BitEncoding.GetBytes(str));
        }

        public static string ToLatin8BitEncoding(this string str)
        {
            try
            {
                return Latin8BitEncoding.GetString(Utf8.GetBytes(str));
            }
            catch (Exception e)
            {
                Config.Log.Error(e, $"Failed to decode string from {Latin8BitEncoding.EncodingName} to {Utf8.EncodingName}");
                return str;
            }
        }

        public static string GetSuffix(long num) => num == 1 ? "" : "s";

        public static string FixSpaces(this string text) => text?.Replace(" ", " \u206a").Replace(Environment.NewLine, "\n");

        public static int GetVisibleLength(this string s)
        {
            if (string.IsNullOrEmpty(s))<commit_msg>workaround discord bullshittery, once again<commit_after>﻿using System;
using System.Buffers;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using DuoVia.FuzzyStrings;
using HomoglyphConverter;
using Microsoft.Extensions.Caching.Memory;

namespace CompatBot.Utils
{
    public static class StringUtils
    {
        private static readonly Encoding Latin8BitEncoding = Encoding.GetEncodings()
                                                                 .FirstOrDefault(e => e.CodePage == 1250 || e.CodePage == 1252 || e.CodePage == 28591)?
                                                                 .GetEncoding()
                                                             ?? Encoding.ASCII;
        private static readonly Encoding Utf8 = new UTF8Encoding(false);
        private static readonly MemoryCache FuzzyPairCache = new MemoryCache(new MemoryCacheOptions {ExpirationScanFrequency = TimeSpan.FromMinutes(10)});
        private const char StrikeThroughChar = '\u0336'; // 0x0335 = short dash, 0x0336 = long dash, 0x0337 = short slash, 0x0338 = long slash

        private static readonly HashSet<char> SpaceCharacters = new HashSet<char>
        {
            '\u00a0',
            '\u2002', '\u2003', '\u2004', '\u2005', '\u2006',
            '\u2007', '\u2008', '\u2009', '\u200a', '\u200b',
            '\u200c', '\u200d', '\u200e', '\u200f',
            '\u2028', '\u2029', '\u202a', '\u202b', '\u202c',
            '\u202c', '\u202d', '\u202e', '\u202f',
            '\u205f', '\u2060', '\u2061', '\u2062', '\u2063',
            '\u2064', '\u2065', '\u2066', '\u2067', '\u2068',
            '\u2069', '\u206a', '\u206b', '\u206c', '\u206d',
            '\u206e', '\u206f',
            '\u3000', '\u303f',
        };

        public static string StripMarks(this string str)
        {
            return str?.Replace("(R)", "", StringComparison.InvariantCultureIgnoreCase)
                .Replace("®", "", StringComparison.InvariantCultureIgnoreCase)
                .Replace("(TM)", "", StringComparison.InvariantCultureIgnoreCase)
                .Replace("™", "", StringComparison.InvariantCultureIgnoreCase);
        }

        public static string StripQuotes(this string str)
        {
            if (str == null || str.Length < 2)
                return str;

            if (str.StartsWith('"') && str.EndsWith('"'))
                return str.Substring(1, str.Length - 2);
            return str;
        }

        public static string TrimEager(this string str)
        {
            if (string.IsNullOrEmpty(str))
                return str;

            int start, end;
            for (start = 0; start < str.Length; start++)
            {
                if (char.IsWhiteSpace(str, start) || IsFormat(str[start]))
                    continue;

                if (char.IsHighSurrogate(str, start)
                    && char.GetUnicodeCategory(str, start) == UnicodeCategory.OtherNotAssigned
                    && str[start] >= 0xdb40) // this will check if the surrogate pair is >= E0000 (see https://en.wikipedia.org/wiki/UTF-16#U+010000_to_U+10FFFF)
                    continue;

                if (char.IsLowSurrogate(str, start))
                    continue;

                break;
            }

            for (end = str.Length - 1; end >= start; end--)
            {
                if (char.IsWhiteSpace(str, end) || IsFormat(str[end]))
                    continue;

                if (char.IsLowSurrogate(str, end)
                    && end > start
                    && char.IsHighSurrogate(str, end - 1)
                    && char.GetUnicodeCategory(str, end - 1) == UnicodeCategory.OtherNotAssigned
                    && str[end-1] >= 0xdb40)
                    continue;

                if (char.IsHighSurrogate(str, end)
                    && char.GetUnicodeCategory(str, end) == UnicodeCategory.OtherNotAssigned
                    && str[end] >= 0xdb40)
                    continue;

                break;
            }

            return CreateTrimmedString(str, start, end);
        }

        public static string AsString(this ReadOnlySequence<byte> buffer, Encoding encoding = null)
        {
            encoding = encoding ?? Latin8BitEncoding;
            if (buffer.IsSingleSegment)
                return encoding.GetString(buffer.First.Span);

            void Splice(Span<char> span, ReadOnlySequence<byte> sequence)
            {
                foreach (var segment in sequence)
                {
                    encoding.GetChars(segment.Span, span);
                    span = span.Slice(segment.Length);
                }
            }
            return string.Create((int)buffer.Length, buffer, Splice);
        }

        public static string ToUtf8(this string str)
        {
            return Utf8.GetString(Latin8BitEncoding.GetBytes(str));
        }

        public static string ToLatin8BitEncoding(this string str)
        {
            try
            {
                return Latin8BitEncoding.GetString(Utf8.GetBytes(str));
            }
            catch (Exception e)
            {
                Config.Log.Error(e, $"Failed to decode string from {Latin8BitEncoding.EncodingName} to {Utf8.EncodingName}");
                return str;
            }
        }

        public static string GetSuffix(long num) => num == 1 ? "" : "s";

        public static string FixSpaces(this string text) => text?.Replace(" ", " \u206a")
            .Replace("`", "\u206a`")
            .Replace(Environment.NewLine, "\n");

        public static int GetVisibleLength(this string s)
        {
            if (string.IsNullOrEmpty(s))
{'fraction_non_alphanumeric': 0.1191783247742164, 'fraction_numerical': 0.042057729768018415, 'mean_word_length': 1.627966496044672, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 31, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3735297', 'n_tokens_mistral': 3417, 'n_tokens_neox': 2981, 'n_words': 686}