<commit_before>﻿using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using Newtonsoft.Json;
using Suigetsu.Core.Common;
using Suigetsu.Core.Serialization;
using Suigetsu.Core.Util;

namespace Suigetsu.Core.Extensions
{
    /// <summary>
    ///     Extensions for the <see cref="T:System.String" /> class.
    /// </summary>
    public static class StringExtensions
    {
        private static object FromJson(this string str, Type type,
                                       JsonSerializerSettingsWrapperParameters settings = null)
            => JsonConvert.DeserializeObject(str, type, new JsonSerializerSettingsWrapper(settings));

        /// <summary>
        ///     Deserializes the given json string into a <see langword="object" /> of the given
        ///     <typeparamref name="T" /> type.
        /// </summary>
        public static T FromJson<T>(this string str, JsonSerializerSettingsWrapperParameters settings = null)
            => (T)str.FromJson(typeof(T), settings);

        /// <summary>
        ///     Tests if the <paramref name="str" /> is empty.
        /// </summary>
        public static bool IsEmpty(this string str) => string.IsNullOrEmpty(str);

        /// <summary>
        ///     Wraps the <paramref name="str" /> with the given <paramref name="edge" />.
        /// </summary>
        public static string Wrap(this string str, string edge) => Wrap(str, edge, edge);

        /// <summary>
        ///     Wraps the <paramref name="str" /> with the given <paramref name="before" /> and
        ///     <paramref name="after" /> parameters.
        /// </summary>
        public static string Wrap(this string str, string before, string after) => before + str + after;

        public static int ToInt(this string text, NumberStyles style = NumberStyles.Integer) => int.Parse(text, style);

        public static long ToLong(this string text) => long.Parse(text);

        public static IList<int> HexToDigits(this string hex, int digitLength = StringUtils.DefaultHexDigitLength)
        {
            var bytes = new List<int>();

            for(var i = 0; i < hex.Length; i += digitLength)
            {
                if((hex.Length - i) >= digitLength)
                {
                    bytes.Add(hex.Substring(i, digitLength).Trim().ToInt(NumberStyles.HexNumber));
                }
            }

            return bytes;
        }

        public static string HexToString(this string hex, int digitLength)
            => new string(hex.HexToDigits(digitLength).ToArray().Convert<char>());

        public static string HexToString(this string hex) => hex.HexToString(StringUtils.DefaultHexDigitLength);

        public static IEnumerable<byte> HexToBytes(this string hex) => hex.HexToDigits(2).ToArray().Convert<byte>();

        private static IEnumerable<string> GraphemeClusters(this string str)
        {
            var enumerator = StringInfo.GetTextElementEnumerator(str);
            while(enumerator.MoveNext())
            {
                yield return (string)enumerator.Current;
            }
        }

        public static string Reverse(this string str)
            => string.Join(string.Empty, str.GraphemeClusters().Reverse().ToArray());

        public static string GetNumbers(this string text) => Regex.Replace(text, @"(\D)", string.Empty);

        public static string ToUtf8(this string text) => text.ToUtf8(Encoding.GetEncoding(1252));

        private static string ToUtf8(this string text, Encoding fromEncoding)
            => Encoding.UTF8.GetString(fromEncoding.GetBytes(text));

        public static string FixNewLine(this string text)
        {
            return Regex.Replace("", @"(\r\n?|\n)", Environment.NewLine);
        }
    }
}<commit_msg>newline fix on linux<commit_after>﻿using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
using Newtonsoft.Json;
using Suigetsu.Core.Common;
using Suigetsu.Core.Serialization;
using Suigetsu.Core.Util;

namespace Suigetsu.Core.Extensions
{
    /// <summary>
    ///     Extensions for the <see cref="T:System.String" /> class.
    /// </summary>
    public static class StringExtensions
    {
        private static object FromJson(this string str, Type type,
                                       JsonSerializerSettingsWrapperParameters settings = null)
            => JsonConvert.DeserializeObject(str, type, new JsonSerializerSettingsWrapper(settings));

        /// <summary>
        ///     Deserializes the given json string into a <see langword="object" /> of the given
        ///     <typeparamref name="T" /> type.
        /// </summary>
        public static T FromJson<T>(this string str, JsonSerializerSettingsWrapperParameters settings = null)
            => (T)str.FromJson(typeof(T), settings);

        /// <summary>
        ///     Tests if the <paramref name="str" /> is empty.
        /// </summary>
        public static bool IsEmpty(this string str) => string.IsNullOrEmpty(str);

        /// <summary>
        ///     Wraps the <paramref name="str" /> with the given <paramref name="edge" />.
        /// </summary>
        public static string Wrap(this string str, string edge) => Wrap(str, edge, edge);

        /// <summary>
        ///     Wraps the <paramref name="str" /> with the given <paramref name="before" /> and
        ///     <paramref name="after" /> parameters.
        /// </summary>
        public static string Wrap(this string str, string before, string after) => before + str + after;

        public static int ToInt(this string text, NumberStyles style = NumberStyles.Integer) => int.Parse(text, style);

        public static long ToLong(this string text) => long.Parse(text);

        public static IList<int> HexToDigits(this string hex, int digitLength = StringUtils.DefaultHexDigitLength)
        {
            var bytes = new List<int>();

            for(var i = 0; i < hex.Length; i += digitLength)
            {
                if((hex.Length - i) >= digitLength)
                {
                    bytes.Add(hex.Substring(i, digitLength).Trim().ToInt(NumberStyles.HexNumber));
                }
            }

            return bytes;
        }

        public static string HexToString(this string hex, int digitLength)
            => new string(hex.HexToDigits(digitLength).ToArray().Convert<char>());

        public static string HexToString(this string hex) => hex.HexToString(StringUtils.DefaultHexDigitLength);

        public static IEnumerable<byte> HexToBytes(this string hex) => hex.HexToDigits(2).ToArray().Convert<byte>();

        private static IEnumerable<string> GraphemeClusters(this string str)
        {
            var enumerator = StringInfo.GetTextElementEnumerator(str);
            while(enumerator.MoveNext())
            {
                yield return (string)enumerator.Current;
            }
        }

        public static string Reverse(this string str)
            => string.Join(string.Empty, str.GraphemeClusters().Reverse().ToArray());

        public static string GetNumbers(this string text) => Regex.Replace(text, @"(\D)", string.Empty);

        public static string ToUtf8(this string text) => text.ToUtf8(Encoding.GetEncoding(1252));

        private static string ToUtf8(this string text, Encoding fromEncoding)
            => Encoding.UTF8.GetString(fromEncoding.GetBytes(text));

        public static string FixNewLine(this string text) => Regex.Replace(text, @"(\r\n?|\n)", Environment.NewLine);
    }
}
{'fraction_non_alphanumeric': 0.11861048738607846, 'fraction_numerical': 0.002641658961828028, 'mean_word_length': 2.380357142857143, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4733119', 'n_tokens_mistral': 2071, 'n_tokens_neox': 2006, 'n_words': 534}