<commit_before>﻿namespace Code.Library
{
    using System;
    using System.Collections.Generic;
    using System.Globalization;
    using System.Linq;
    using System.Net;
    using System.Security.Cryptography;
    using System.Text;
    using System.Text.RegularExpressions;
    using System.Threading;

    /// <summary>
    /// All String extensions.
    /// </summary>
    public static class StringExtensions
    {
        private static readonly char[] CleanForXssChars = "*?(){}[];:%<>/\\|&'\"".ToCharArray();
        private static MD5CryptoServiceProvider md5CryptoServiceProvider = null;

        /// <summary>
        /// Clean the content by replacing special characters/HTML tags.
        /// </summary>
        /// <param name="content">
        /// The content.
        /// </param>
        /// <param name="removeHtml">
        /// The remove html.
        /// </param>
        /// <returns>
        /// The <see cref="string"/>.
        /// </returns>
        public static string Clean(this string content, bool removeHtml = true)
        {
            if (removeHtml)
            {
                content = content.StripHtml();
            }

            content = content.Replace("\\", string.Empty).Replace("|", string.Empty).Replace("(", string.Empty)
                .Replace(")", string.Empty).Replace("[", string.Empty).Replace("]", string.Empty)
                .Replace("*", string.Empty).Replace("?", string.Empty).Replace("}", string.Empty)
                .Replace("{", string.Empty).Replace("^", string.Empty).Replace("+", string.Empty);

            var words = content.Split(new[] { ' ', '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
            var sb = new StringBuilder();
            foreach (var word in
                words.Select(t => t.Trim()).Where(word => word.Length >= 1))
            {
                sb.AppendFormat("{0} ", word);
            }

            return sb.ToString().Trim();
        }

        /// <summary>
        /// Cleans string to aid in preventing xss attacks.
        /// </summary>
        /// <param name="input">Input string.</param>
        /// <param name="ignoreFromClean">Characters to ignore during clean.</param>
        /// <returns>Cleaned string.</returns>
        public static string CleanForXss(this string input, params char[] ignoreFromClean)
        {
            // remove any html
            input = input.StripHtml();

            // strip out any potential chars involved with XSS
            return input.ExceptChars(new HashSet<char>(CleanForXssChars.Except(ignoreFromClean)));
        }

        public static string EnsureEndsWith(this string input, char value)
        {
            return input.EndsWith(value.ToString(CultureInfo.InvariantCulture)) ? input : input + value;
        }

        public static string EnsureEndsWith(this string input, string toEndWith)
        {
            return input.EndsWith(toEndWith.ToString(CultureInfo.InvariantCulture)) ? input : input + toEndWith;
        }

        public static string EnsureStartsWith(this string input, char value)
        {
            return input.StartsWith(value.ToString(CultureInfo.InvariantCulture)) ? input : value + input;
        }

        public static string ExceptChars(this string str, HashSet<char> toExclude)
        {
            var sb = new StringBuilder(str.Length);
            foreach (var c in str.Where(c => toExclude.Contains(c) == false))
            {
                sb.Append(c);
            }

            return sb.ToString();
        }

        /// <summary>
        /// Returns default value if string is null or whitespace.
        /// </summary>
        /// <param name="str">Input string.</param><commit_msg>Get Last N characters from a string<commit_after>﻿namespace Code.Library
{
    using System;
    using System.Collections.Generic;
    using System.Globalization;
    using System.Linq;
    using System.Net;
    using System.Security.Cryptography;
    using System.Text;
    using System.Text.RegularExpressions;
    using System.Threading;

    /// <summary>
    /// All String extensions.
    /// </summary>
    public static class StringExtensions
    {
        private static readonly char[] CleanForXssChars = "*?(){}[];:%<>/\\|&'\"".ToCharArray();
        private static MD5CryptoServiceProvider md5CryptoServiceProvider = null;

        /// <summary>
        /// Clean the content by replacing special characters/HTML tags.
        /// </summary>
        /// <param name="content">
        /// The content.
        /// </param>
        /// <param name="removeHtml">
        /// The remove html.
        /// </param>
        /// <returns>
        /// The <see cref="string"/>.
        /// </returns>
        public static string Clean(this string content, bool removeHtml = true)
        {
            if (removeHtml)
            {
                content = content.StripHtml();
            }

            content = content.Replace("\\", string.Empty).Replace("|", string.Empty).Replace("(", string.Empty)
                .Replace(")", string.Empty).Replace("[", string.Empty).Replace("]", string.Empty)
                .Replace("*", string.Empty).Replace("?", string.Empty).Replace("}", string.Empty)
                .Replace("{", string.Empty).Replace("^", string.Empty).Replace("+", string.Empty);

            var words = content.Split(new[] { ' ', '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
            var sb = new StringBuilder();
            foreach (var word in
                words.Select(t => t.Trim()).Where(word => word.Length >= 1))
            {
                sb.AppendFormat("{0} ", word);
            }

            return sb.ToString().Trim();
        }

        /// <summary>
        /// Cleans string to aid in preventing xss attacks.
        /// </summary>
        /// <param name="input">Input string.</param>
        /// <param name="ignoreFromClean">Characters to ignore during clean.</param>
        /// <returns>Cleaned string.</returns>
        public static string CleanForXss(this string input, params char[] ignoreFromClean)
        {
            // remove any html
            input = input.StripHtml();

            // strip out any potential chars involved with XSS
            return input.ExceptChars(new HashSet<char>(CleanForXssChars.Except(ignoreFromClean)));
        }

        public static string EnsureEndsWith(this string input, char value)
        {
            return input.EndsWith(value.ToString(CultureInfo.InvariantCulture)) ? input : input + value;
        }

        public static string EnsureEndsWith(this string input, string toEndWith)
        {
            return input.EndsWith(toEndWith.ToString(CultureInfo.InvariantCulture)) ? input : input + toEndWith;
        }

        public static string EnsureStartsWith(this string input, char value)
        {
            return input.StartsWith(value.ToString(CultureInfo.InvariantCulture)) ? input : value + input;
        }

        public static string ExceptChars(this string str, HashSet<char> toExclude)
        {
            var sb = new StringBuilder(str.Length);
            foreach (var c in str.Where(c => toExclude.Contains(c) == false))
            {
                sb.Append(c);
            }

            return sb.ToString();
        }

        public static string GetLast(this string source, int numberOfChars)
        {
            if (numberOfChars >= source.Length)
                return source;
            return source.Substring(source.Length - numberOfChars);
        }

        /// <summary>
        /// Returns default value if string is null or whitespace.
        /// </summary>
        /// <param name="str">Input string.</param>
{'fraction_non_alphanumeric': 0.13360166190599845, 'fraction_numerical': 0.001038691249026227, 'mean_word_length': 1.9244495064540623, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4221938', 'n_tokens_mistral': 1989, 'n_tokens_neox': 1949, 'n_words': 499}