<commit_before>using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Runtime.Serialization.Json;
using System.Text;
using System.Text.RegularExpressions;
//using Newtonsoft.Json;

namespace Extensions
{
    public static class StringExtensions
    {
        //private static readonly JsonSerializerSettings _JsonDotNetSettingsDeserialize = new JsonSerializerSettings()
        //{
        //    DateFormatHandling = DateFormatHandling.IsoDateFormat, // DEFAULT
        //    DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate, // Ignore default value properties on serializer, populate on deserialize
        //    MissingMemberHandling = MissingMemberHandling.Ignore, // DEFAULT
        //    NullValueHandling = NullValueHandling.Ignore, // Ignore null values
        //    ObjectCreationHandling = ObjectCreationHandling.Auto, // DEFAULT
        //    ReferenceLoopHandling = ReferenceLoopHandling.Ignore, // Ignore Reference Loops
        //};

        #region Conversion

        public static T ToEnum<T>(this string input)
        {
            if (typeof(T).BaseType != typeof(Enum))
            {
                throw new InvalidCastException();
            }
            if (Enum.IsDefined(typeof(T), input) == false)
            {
                throw new InvalidCastException();
            }
            return (T)Enum.Parse(typeof(T), input);
        }

        #endregion

        #region Hashing

        public static byte[] GenerateHash(this string input)
        {
            return Encoding.Unicode.GetBytes(input).GenerateHash();
        }

        public static string GenerateHashBase64String(string input)
        {
            return System.Convert.ToBase64String(GenerateHash(input));
        }

        #endregion

        #region Manipulation

        public static string SafeSubstring(this string input, int length = 0, int startIndex = 0)
        {
            // If null or empty is passed, just return input
            if (string.IsNullOrEmpty(input)) return input;

            // Length is optional, set it to input length if zero
            var _length = length > 0 ? length : input.Length;

            // Make sure startIndex plus length is possible
            if (input.Length >= (startIndex + _length)) return input.Substring(startIndex, _length);

            // Make sure startIndex does not exceed input length
            if (input.Length > startIndex) return input.Substring(startIndex);
            else throw new ApplicationException(string.Format("Start Index ({0}) cannot exceed string input Length ({1})", startIndex, input.Length));
        }

        public static string SanitizeHtmlString(this string input)
        {
            if (string.IsNullOrEmpty(input)) return null;

            // replace line breaks
            input = Regex.Replace(input, @"\r\n?|\n|\u000d|\u000a", @"<br/>");

            // replace horizontal tabs
            input = Regex.Replace(input, @"\t|\u0009", @"&emsp;");

            // replace reserved chars
            input = Regex.Replace(input, @"\|", @"");

            // remove unit separator
            input = input.Replace((char)(0x1F), ' ');

            return input;
        }

        public static string SanitizeXmlString(this string xml)
        {
            if (string.IsNullOrEmpty(xml)) return null;
<commit_msg>Added SafeTrim methods to StringExtensions, still need to bring in unit tests<commit_after>using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Runtime.Serialization.Json;
using System.Text;
using System.Text.RegularExpressions;
//using Newtonsoft.Json;

namespace Extensions
{
    public static class StringExtensions
    {
        //private static readonly JsonSerializerSettings _JsonDotNetSettingsDeserialize = new JsonSerializerSettings()
        //{
        //    DateFormatHandling = DateFormatHandling.IsoDateFormat, // DEFAULT
        //    DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate, // Ignore default value properties on serializer, populate on deserialize
        //    MissingMemberHandling = MissingMemberHandling.Ignore, // DEFAULT
        //    NullValueHandling = NullValueHandling.Ignore, // Ignore null values
        //    ObjectCreationHandling = ObjectCreationHandling.Auto, // DEFAULT
        //    ReferenceLoopHandling = ReferenceLoopHandling.Ignore, // Ignore Reference Loops
        //};

        #region Conversion

        public static T ToEnum<T>(this string input)
        {
            if (typeof(T).BaseType != typeof(Enum))
            {
                throw new InvalidCastException();
            }
            if (Enum.IsDefined(typeof(T), input) == false)
            {
                throw new InvalidCastException();
            }
            return (T)Enum.Parse(typeof(T), input);
        }

        #endregion

        #region Hashing

        public static byte[] GenerateHash(this string input)
        {
            return Encoding.Unicode.GetBytes(input).GenerateHash();
        }

        public static string GenerateHashBase64String(string input)
        {
            return System.Convert.ToBase64String(GenerateHash(input));
        }

        #endregion

        #region Manipulation

        public static string SafeSubstring(this string input, int length = 0, int startIndex = 0)
        {
            // If null or empty is passed, just return input
            if (string.IsNullOrEmpty(input)) return input;

            // Length is optional, set it to input length if zero
            var _length = length > 0 ? length : input.Length;

            // Make sure startIndex plus length is possible
            if (input.Length >= (startIndex + _length)) return input.Substring(startIndex, _length);

            // Make sure startIndex does not exceed input length
            if (input.Length > startIndex) return input.Substring(startIndex);
            else throw new ApplicationException(string.Format("Start Index ({0}) cannot exceed string input Length ({1})", startIndex, input.Length));
        }

        public static string SafeTrim(this string input)
        {
            if (input == null) return input;
            return input.Trim();
        }

        public static string SafeTrimEnd(this string input)
        {
            if (input == null) return input;
            return input.TrimEnd();
        }

        public static string SafeTrimStart(this string input)
        {
            if (input == null) return input;
            return input.TrimStart();
        }

        public static string SanitizeHtmlString(this string input)
        {
            if (string.IsNullOrEmpty(input)) return null;

            // replace line breaks
            input = Regex.Replace(input, @"\r\n?|\n|\u000d|\u000a", @"<br/>");

            // replace horizontal tabs
            input = Regex.Replace(input, @"\t|\u0009", @"&emsp;");

            // replace reserved chars
            input = Regex.Replace(input, @"\|", @"");

            // remove unit separator
            input = input.Replace((char)(0x1F), ' ');

            return input;
        }

        public static string SanitizeXmlString(this string xml)
        {
            if (string.IsNullOrEmpty(xml)) return null;

{'fraction_non_alphanumeric': 0.09377548246806197, 'fraction_numerical': 0.0057080728458820335, 'mean_word_length': 2.1314893617021275, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5857642', 'n_tokens_mistral': 1881, 'n_tokens_neox': 1802, 'n_words': 520}