<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace Ext.NET
{
    public static class EnumerableExtension
    {
        public static void Each<T>(this IEnumerable<T> enumerable, Action<T> action)
        {
            foreach (var item in enumerable)
            {
                action(item);
            }
        }

        public static IEnumerable<T> Grep<T>(this IEnumerable<T> enumerable, string pattern)
        {
            foreach (var item in enumerable)
            {
                if (item.ToString().Contains(pattern))
                {
                    yield return item;
                }
            }
        }

        public static IEnumerable<T> EGrep<T>(this IEnumerable<T> enumerable, string pattern, RegexOptions option = RegexOptions.None)
        {
            var regex = new Regex(pattern, option);
            foreach (var item in enumerable)
            {
                if (regex.IsMatch(item.ToString()))
                {
                    yield return item;
                }
            }
        }
    }
}
<commit_msg>Add Shuffle<commit_after>﻿using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;

namespace Ext.NET
{
    public static class EnumerableExtension
    {
        public static void Each<T>(this IEnumerable<T> enumerable, Action<T> action)
        {
            foreach (var item in enumerable)
            {
                action(item);
            }
        }

        public static IEnumerable<T> Grep<T>(this IEnumerable<T> enumerable, string pattern)
        {
            foreach (var item in enumerable)
            {
                if (item.ToString().Contains(pattern))
                {
                    yield return item;
                }
            }
        }

        public static IEnumerable<T> EGrep<T>(this IEnumerable<T> enumerable, string pattern, RegexOptions option = RegexOptions.None)
        {
            var regex = new Regex(pattern, option);
            foreach (var item in enumerable)
            {
                if (regex.IsMatch(item.ToString()))
                {
                    yield return item;
                }
            }
        }

        public static IEnumerable<T> Shuffle<T>(this IEnumerable<T> source)
        {
            var random = new Random();
            return source.ShuffleIterator(random);
        }

        public static IEnumerable<T> Shuffle<T>(this IEnumerable<T> source, Random random)
        {
            return source.ShuffleIterator(random);
        }

        private static IEnumerable<T> ShuffleIterator<T>(this IEnumerable<T> source, Random random)
        {
            var buffer = new List<T>(source);

            for (int i = 0; i < buffer.Count; ++i)
            {
                int j = random.Next(i, buffer.Count);
                yield return buffer[j];

                buffer[j] = buffer[i];
            }
        }
    }
}

{'fraction_non_alphanumeric': 0.08910235177210997, 'fraction_numerical': 0.0003312355084465055, 'mean_word_length': 1.5, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2620846', 'n_tokens_mistral': 795, 'n_tokens_neox': 758, 'n_words': 195}