<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
#if !SILVERLIGHT
using System.Collections.Specialized;
#endif

namespace SuperSocket.ClientEngine
{
    public static class BinaryUtil
    {
        public static int IndexOf<T>(this IList<T> source, T target, int pos, int length)
            where T : IEquatable<T>
        {
            for (int i = pos; i < pos + length; i++)
            {
                if (source[i].Equals(target))
                    return i;
            }

            return -1;
        }

        public static int? SearchMark<T>(this IList<T> source, T[] mark)
            where T : IEquatable<T>
        {
            return SearchMark(source, 0, source.Count, mark, 0);
        }

        public static int? SearchMark<T>(this IList<T> source, int offset, int length, T[] mark)
            where T : IEquatable<T>
        {
            return SearchMark(source, offset, length, mark, 0);
        }

        public static int? SearchMark<T>(this IList<T> source, int offset, int length, T[] mark, int matched)
            where T : IEquatable<T>
        {
            int pos = offset;
            int endOffset = offset + length - 1;
            int matchCount = matched;

            while (true)
            {
                pos = source.IndexOf(mark[matchCount], pos, length - pos + offset);

                if (pos < 0)
                    return null;

                matchCount += 1;

                for (int i = matchCount; i < mark.Length; i++)
                {
                    int checkPos = pos + i;

                    if (checkPos > endOffset)
                    {
                        //found end, return matched chars count
                        return (0 - matchCount);
                    }

                    if (!source[checkPos].Equals(mark[i]))
                        break;

                    matchCount++;
                }

                if (matchCount == mark.Length)
                    return pos;

                //Reset next round read pos
                pos += 1;
                //clear matched chars count
                matchCount = 0;
            }
        }

        public static int SearchMark<T>(this IList<T> source, int offset, int length, SearchMarkState<T> searchState)
            where T : IEquatable<T>
        {
            int? result = source.SearchMark(offset, length, searchState.Mark, searchState.Matched);

            if (!result.HasValue)
            {
                searchState.Matched = 0;
                return -1;
            }

            if (result.Value < 0)
            {
                searchState.Matched = 0 - result.Value;
                return -1;
            }

            searchState.Matched = 0;
            return result.Value;
        }

        public static int StartsWith<T>(this IList<T> source, T[] mark)
            where T : IEquatable<T>
        {
            return source.StartsWith(0, source.Count, mark);
        }

        public static int StartsWith<T>(this IList<T> source, int offset, int length, T[] mark)
            where T : IEquatable<T>
        {
            int pos = offset;
            int endOffset = offset + length - 1;

            for (int i = 0; i < mark.Length; i++)
            {
                int checkPos = pos + i;

                if (checkPos > endOffset)
                    return i;

                if (!source[checkPos].Equals(mark[i]))
                    return -1;
            }

            return mark.Length;
        }

        public static bool EndsWith<T>(this IList<T> source, T[] mark)
            where T : IEquatable<T>
        {
            return source.EndsWith(0, source.Count, mark);
        }

        public static bool EndsWith<T>(this IList<T> source, int offset, int length, T[] mark)
            where T : IEquatable<T>
        {
            if (mark.Length > length)
                return false;

            for (int i = 0; i < Math.Min(length, mark.Length); i++)
            {
                if (!mark[i].Equals(source[offset + length - mark.Length + i]))
                    return false;
            }

            return true;
        }

        public static T[] CloneRange<T>(this T[] source, int offset, int length)
        {
            T[] target = new T[length];
            Array.Copy(source, offset, target, 0, length);
            return target;
        }

        private static Random m_Random = new Random();

        public static T[] RandomOrder<T>(this T[] source)
        {
            var n = source.Length / 2;

            for (var i = 0; i < n; i++)
            {
                var x = m_Random.Next(0, source.Length - 1);
                var y = m_Random.Next(0, source.Length - 1);

                if (x == y)
                    continue;

                var t = source[y];

                //swap position
                source[y] = source[x];
                source[x] = t;
            }

            return source;
        }

        public static string GetValue(this NameValueCollection collection, string key)
        {
            return GetValue(collection, key, string.Empty);
        }

        public static string GetValue(this NameValueCollection collection, string key, string defaultValue)
        {
            if (string.IsNullOrEmpty(key))
                throw new ArgumentNullException("key");

            if (collection == null)
                return defaultValue;

            var e = collection[key];

            if (e == null)
                return defaultValue;

            return e;
        }
    }
}
<commit_msg>mainline: fixed the stateful search bug in ClientEngine<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
#if !SILVERLIGHT
using System.Collections.Specialized;
#endif

namespace SuperSocket.ClientEngine
{
    public static class BinaryUtil
    {
        public static int IndexOf<T>(this IList<T> source, T target, int pos, int length)
            where T : IEquatable<T>
        {
            for (int i = pos; i < pos + length; i++)
            {
                if (source[i].Equals(target))
                    return i;
            }

            return -1;
        }

        public static int? SearchMark<T>(this IList<T> source, T[] mark)
            where T : IEquatable<T>
        {
            return SearchMark(source, 0, source.Count, mark, 0);
        }

        public static int? SearchMark<T>(this IList<T> source, int offset, int length, T[] mark)
            where T : IEquatable<T>
        {
            return SearchMark(source, offset, length, mark, 0);
        }

        public static int? SearchMark<T>(this IList<T> source, int offset, int length, T[] mark, int matched)
            where T : IEquatable<T>
        {
            int pos = offset;
            int endOffset = offset + length - 1;
            int matchCount = matched;

            if (matched > 0)
            {
                for (int i = matchCount; i < mark.Length; i++)
                {
                    if (!source[pos++].Equals(mark[i]))
                        break;

                    matchCount++;
                }

                if (matchCount == mark.Length)
                    return 0;

                pos = offset;
                matchCount = 0;
            }

            while (true)
            {
                pos = source.IndexOf(mark[matchCount], pos, length - pos + offset);

                if (pos < 0)
                    return null;

                matchCount += 1;

                for (int i = matchCount; i < mark.Length; i++)
                {
                    int checkPos = pos + i;

                    if (checkPos > endOffset)
                    {
                        //found end, return matched chars count
                        return (0 - matchCount);
                    }

                    if (!source[checkPos].Equals(mark[i]))
                        break;

                    matchCount++;
                }

                if (matchCount == mark.Length)
                    return pos;

                //Reset next round read pos
                pos += 1;
                //clear matched chars count
                matchCount = 0;
            }
        }

        public static int SearchMark<T>(this IList<T> source, int offset, int length, SearchMarkState<T> searchState)
            where T : IEquatable<T>
        {
            int? result = source.SearchMark(offset, length, searchState.Mark, searchState.Matched);

            if (!result.HasValue)
            {
                searchState.Matched = 0;
                return -1;
            }

            if (result.Value < 0)
            {
                searchState.Matched = 0 - result.Value;
                return -1;
            }

            searchState.Matched = 0;
            return result.Value;
        }

        public static int StartsWith<T>(this IList<T> source, T[] mark)
            where T : IEquatable<T>
        {
            return source.StartsWith(0, source.Count, mark);
        }

        public static int StartsWith<T>(this IList<T> source, int offset, int length, T[] mark)
            where T : IEquatable<T>
        {
            int pos = offset;
            int endOffset = offset + length - 1;

            for (int i = 0; i < mark.Length; i++)
            {
                int checkPos = pos + i;

                if (checkPos > endOffset)
                    return i;

                if (!source[checkPos].Equals(mark[i]))
                    return -1;
            }

            return mark.Length;
        }

        public static bool EndsWith<T>(this IList<T> source, T[] mark)
            where T : IEquatable<T>
        {
            return source.EndsWith(0, source.Count, mark);
        }

        public static bool EndsWith<T>(this IList<T> source, int offset, int length, T[] mark)
            where T : IEquatable<T>
        {
            if (mark.Length > length)
                return false;

            for (int i = 0; i < Math.Min(length, mark.Length); i++)
            {
                if (!mark[i].Equals(source[offset + length - mark.Length + i]))
                    return false;
            }

            return true;
        }

        public static T[] CloneRange<T>(this T[] source, int offset, int length)
        {
            T[] target = new T[length];
            Array.Copy(source, offset, target, 0, length);
            return target;
        }

        private static Random m_Random = new Random();

        public static T[] RandomOrder<T>(this T[] source)
        {
            var n = source.Length / 2;

            for (var i = 0; i < n; i++)
            {
                var x = m_Random.Next(0, source.Length - 1);
                var y = m_Random.Next(0, source.Length - 1);

                if (x == y)
                    continue;

                var t = source[y];

                //swap position
                source[y] = source[x];
                source[x] = t;
            }

            return source;
        }

        public static string GetValue(this NameValueCollection collection, string key)
        {
            return GetValue(collection, key, string.Empty);
        }

        public static string GetValue(this NameValueCollection collection, string key, string defaultValue)
        {
            if (string.IsNullOrEmpty(key))
                throw new ArgumentNullException("key");

            if (collection == null)
                return defaultValue;

            var e = collection[key];

            if (e == null)
                return defaultValue;

            return e;
        }
    }
}

{'fraction_non_alphanumeric': 0.09354175236528178, 'fraction_numerical': 0.005018510900863842, 'mean_word_length': 1.138258575197889, 'pattern_counts': {'":': 0, '<': 78, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1991041', 'n_tokens_mistral': 3329, 'n_tokens_neox': 2997, 'n_words': 902}