<commit_before>﻿using System;
using System.Collections.Generic;
using System.Text;

namespace Buzzbox_Common
{
    static class Extensions
    {
        //Randomize a given lists order.
        //Source: http://stackoverflow.com/questions/273313/randomize-a-listt-in-c-sharp
        private static readonly Random Rng = new Random();
        
        public static void Shuffle<T>(this IList<T> list)
        {
            var n = list.Count;
            while (n > 1)
            {
                n--;
                var k = Rng.Next(n + 1);
                T value = list[k];
                list[k] = list[n];
                list[n] = value;
            }
        }

        public static string CapitalizeOnlyFirstLetter(this string str)
        {
            return char.ToUpper(str[0]) + str.Substring(1).ToLower();
        }
    }
}<commit_msg>Added a string extension method to remove markup like <b> </i> and [x]<commit_after>﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;

namespace Buzzbox_Common
{
    static class Extensions
    {
        //Randomize a given lists order.
        //Source: http://stackoverflow.com/questions/273313/randomize-a-listt-in-c-sharp
        private static readonly Random Rng = new Random();
        
        public static void Shuffle<T>(this IList<T> list)
        {
            var n = list.Count;
            while (n > 1)
            {
                n--;
                var k = Rng.Next(n + 1);
                T value = list[k];
                list[k] = list[n];
                list[n] = value;
            }
        }

        public static string CapitalizeOnlyFirstLetter(this string str)
        {
            return char.ToUpper(str[0]) + str.Substring(1).ToLower();
        }

        public static string RemoveMarkup(this string str)
        {
            var output = Regex.Replace(str, @"</?.>", "");
            output = Regex.Replace(output, @"\[.\]", "");
            
            return output;
        }
    }
}
{'fraction_non_alphanumeric': 0.1142156862745098, 'fraction_numerical': 0.00980392156862745, 'mean_word_length': 1.570528967254408, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1540669', 'n_tokens_mistral': 606, 'n_tokens_neox': 572, 'n_words': 143}