<commit_before>using System;
using System.Buffers;
using System.Collections.Immutable;
using System.Linq;
using System.Reflection;
using System.Runtime.CompilerServices;

namespace Pidgin
{
    /// <summary>
    /// A mutable struct! Careful!
    /// </summary>
    internal struct PooledList<T>
    {
        // If T is not primitive, we should clear out the pooled arrays so as not to leak objects.
        // Ideally I'd look at typeof(T).IsManagedType but there's no such thing.
        private static readonly bool _needsClear = !typeof(T).GetTypeInfo().IsPrimitive;

        private const int InitialCapacity = 16;
        private T[] _items;
        private int _size;

        public int Count
        {
            [MethodImpl(MethodImplOptions.AggressiveInlining)]
            get
            {
                return _size;
            }
        }

        public PooledList(int initialCapacity)
        {
            _items = ArrayPool<T>.Shared.Rent(initialCapacity);
            _size = 0;
        }

        public T this[int index]
        {
            [MethodImpl(MethodImplOptions.AggressiveInlining)]
            get
            {
                if (index >= _size)
                {
                    ThrowArgumentOutOfRangeException(nameof(index));
                }
                return _items[index];
            }
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public void Add(T item)
        {
            GrowIfNecessary(1);
            _items[_size] = item;
            _size++;
        }
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public void AddRange(ImmutableArray<T> items)
        {
            GrowIfNecessary(items.Length);
            items.CopyTo(_items, _size);
            _size += items.Length;
        }
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public void AddRange(ReadOnlySpan<T> items)
        {
            GrowIfNecessary(items.Length);
            items.CopyTo(_items.AsSpan().Slice(_size));
            _size += items.Length;
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public T Pop()
        {
            if (_size == 0)
            {
                ThrowInvalidOperationException();
            }
            _size -= 1;
            var result = _items[_size];
            if (_needsClear)
            {
                _items[_size] = default;
            }
            return result;
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public void Shrink(int newCount)
        {
            if (newCount > _size || newCount < 0)
            {
                ThrowArgumentOutOfRangeException(nameof(newCount));
            }
            if (_needsClear)
            {
                _items.AsSpan().Slice(newCount).Clear();
            }<commit_msg>Don't clear the PooledList's buffer every time you remove an item, just when you return the buffer to the pool.<commit_after>using System;
using System.Buffers;
using System.Collections.Immutable;
using System.Linq;
using System.Reflection;
using System.Runtime.CompilerServices;

namespace Pidgin
{
    /// <summary>
    /// A mutable struct! Careful!
    /// </summary>
    internal struct PooledList<T>
    {
        // If T is not primitive, we should clear out the pooled arrays so as not to leak objects.
        // Ideally I'd look at typeof(T).IsManagedType but there's no such thing.
        private static readonly bool _needsClear = !typeof(T).GetTypeInfo().IsPrimitive;

        private const int InitialCapacity = 16;
        private T[] _items;
        private int _size;

        public int Count
        {
            [MethodImpl(MethodImplOptions.AggressiveInlining)]
            get
            {
                return _size;
            }
        }

        public PooledList(int initialCapacity)
        {
            _items = ArrayPool<T>.Shared.Rent(initialCapacity);
            _size = 0;
        }

        public T this[int index]
        {
            [MethodImpl(MethodImplOptions.AggressiveInlining)]
            get
            {
                if (index >= _size)
                {
                    ThrowArgumentOutOfRangeException(nameof(index));
                }
                return _items[index];
            }
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public void Add(T item)
        {
            GrowIfNecessary(1);
            _items[_size] = item;
            _size++;
        }
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public void AddRange(ImmutableArray<T> items)
        {
            GrowIfNecessary(items.Length);
            items.CopyTo(_items, _size);
            _size += items.Length;
        }
        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public void AddRange(ReadOnlySpan<T> items)
        {
            GrowIfNecessary(items.Length);
            items.CopyTo(_items.AsSpan().Slice(_size));
            _size += items.Length;
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public T Pop()
        {
            if (_size == 0)
            {
                ThrowInvalidOperationException();
            }
            _size--;
            return _items[_size];
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public void Shrink(int newCount)
        {
            if (newCount > _size || newCount < 0)
            {
                ThrowArgumentOutOfRangeException(nameof(newCount));
            }
{'fraction_non_alphanumeric': 0.08756756756756756, 'fraction_numerical': 0.0023423423423423423, 'mean_word_length': 1.6024378809188935, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4398634', 'n_tokens_mistral': 1520, 'n_tokens_neox': 1427, 'n_words': 318}