<commit_before>using System;
using System.Buffers;

namespace Pidgin.ParseStates
{
    internal abstract class BufferedParseState<TToken> : BaseParseState<TToken, Positioned<int>>
    {
        private static readonly int InitialCapacity =
            typeof(TToken).Equals(typeof(char))
                ? 4096 / sizeof(char)
                : 4096;

        private int _bufSize = 0;
        private int _bufPos = 0;
        private SourcePos _sourcePos = new SourcePos(1, 1);

        // alloc on first use
        private TToken[] _buffer = null;
        // no need for locking, it (shouldn't be) possible
        // for multiple threads to access the same ParseState
        private TToken[] Buffer 
        {
            get
            {
                if (_buffer == null)
                {
                    _buffer = ArrayPool<TToken>.Shared.Rent(InitialCapacity);
                }
                return _buffer;
            }
        }
        private Maybe<TToken> _current;
        private readonly Func<TToken, SourcePos, SourcePos> _calculatePos;

        protected BufferedParseState(Func<TToken, SourcePos, SourcePos> calculatePos)
        {
            _calculatePos = calculatePos;
        }

        public sealed override Maybe<TToken> Peek() => _current;
        public sealed override SourcePos SourcePos => _sourcePos;

        protected abstract Maybe<TToken> AdvanceInput();
        public sealed override void Advance()
        {
            if (_current.HasValue)
            {
                _sourcePos = _calculatePos(_current.GetValueOrDefault(), _sourcePos);
            }
            if (_bufPos < _bufSize - 1)
            {
                // we are re-reading previously buffered input
                _bufPos++;
                _current = Maybe.Just(Buffer[_bufPos]);
                return;
            }
            _bufPos = _bufSize;  // just to be sure
            if (!IsBuffering)
            {
                // we're at the end of the buffer,
                // and we're not currently writing to the buffer,
                // so the buffer can be discarded
                _bufPos = _bufSize = 0;
                ArrayPool<TToken>.Shared.Return(Buffer);
                _buffer = null;
            }
            else if (_current.HasValue)
            {
                // don't forget the old value
                PushBuf(_current.GetValueOrDefault());
            }

            _current = AdvanceInput();
        }

        protected sealed override Positioned<int> GetBookmark()
            => new Positioned<int>(_bufPos, _sourcePos);
        protected sealed override void Rewind(Positioned<int> bookmark)
        {
            if (_bufPos == _bufSize && _current.HasValue)
            {
                // we were writing to the buffer before the rewind,
                // so don't forget to store the last value
                PushBuf(_current.GetValueOrDefault());
            }
            _bufPos = bookmark.Value;
            _current = _bufPos >= _bufSize
                ? Maybe.Nothing<TToken>()
                : Maybe.Just(Buffer[_bufPos]);
            _sourcePos = bookmark.Pos;
        }
<commit_msg>don't rent a buffer just to return it immediately<commit_after>using System;
using System.Buffers;

namespace Pidgin.ParseStates
{
    internal abstract class BufferedParseState<TToken> : BaseParseState<TToken, Positioned<int>>
    {
        private static readonly int InitialCapacity =
            typeof(TToken).Equals(typeof(char))
                ? 4096 / sizeof(char)
                : 4096;

        private int _bufSize = 0;
        private int _bufPos = 0;
        private SourcePos _sourcePos = new SourcePos(1, 1);

        // alloc on first use
        private TToken[] _buffer = null;
        // no need for locking, it (shouldn't be) possible
        // for multiple threads to access the same ParseState
        private TToken[] Buffer 
        {
            get
            {
                if (_buffer == null)
                {
                    _buffer = ArrayPool<TToken>.Shared.Rent(InitialCapacity);
                }
                return _buffer;
            }
        }
        private Maybe<TToken> _current;
        private readonly Func<TToken, SourcePos, SourcePos> _calculatePos;

        protected BufferedParseState(Func<TToken, SourcePos, SourcePos> calculatePos)
        {
            _calculatePos = calculatePos;
        }

        public sealed override Maybe<TToken> Peek() => _current;
        public sealed override SourcePos SourcePos => _sourcePos;

        protected abstract Maybe<TToken> AdvanceInput();
        public sealed override void Advance()
        {
            if (_current.HasValue)
            {
                _sourcePos = _calculatePos(_current.GetValueOrDefault(), _sourcePos);
            }
            if (_bufPos < _bufSize - 1)
            {
                // we are re-reading previously buffered input
                _bufPos++;
                _current = Maybe.Just(Buffer[_bufPos]);
                return;
            }
            _bufPos = _bufSize;  // just to be sure
            if (!IsBuffering)
            {
                if (_buffer != null)
                {
                    // we're at the end of the buffer,
                    // and we're not currently writing to the buffer,
                    // so the buffer can be discarded
                    _bufPos = _bufSize = 0;
                    ArrayPool<TToken>.Shared.Return(_buffer);
                    _buffer = null;
                }
            }
            else if (_current.HasValue)
            {
                // don't forget the old value
                PushBuf(_current.GetValueOrDefault());
            }

            _current = AdvanceInput();
        }

        protected sealed override Positioned<int> GetBookmark()
            => new Positioned<int>(_bufPos, _sourcePos);
        protected sealed override void Rewind(Positioned<int> bookmark)
        {
            if (_bufPos == _bufSize && _current.HasValue)
            {
                // we were writing to the buffer before the rewind,
                // so don't forget to store the last value
                PushBuf(_current.GetValueOrDefault());
            }
            _bufPos = bookmark.Value;
            _current = _bufPos >= _bufSize
                ? Maybe.Nothing<TToken>()
                : Maybe.Just(Buffer[_bufPos]);
            _sourcePos = bookmark.Pos;
        }

{'fraction_non_alphanumeric': 0.07971014492753623, 'fraction_numerical': 0.004316990440949738, 'mean_word_length': 1.4712380952380952, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '809603', 'n_tokens_mistral': 1690, 'n_tokens_neox': 1567, 'n_words': 445}