<commit_before>﻿// Needed for NET35 (BigInteger)

using System;
using System.Collections.Generic;
using Theraot.Collections;

namespace Theraot.Core
{
    /// <summary>
    ///     A class to extract information from strings.
    /// </summary>
    /// <remarks>
    ///     An alternative for StringReader for all your parsing needs.
    ///     StringProcessor is NOT thread safe. Do not share instances without locking.
    /// </remarks>
    public class StringProcessor
    {
        // Note: it is assumed that _length = _string.Length - if this weren't true extra check would be needed in every case of IndexOf or IndexOfAny
        private readonly int _length;

        private int _position;

        /// <summary>
        ///     Creates a new instance of StringProcessor.
        /// </summary>
        /// <param name="str">The string to process.</param>
        /// <exception cref="ArgumentNullException">The string is null.</exception>
        public StringProcessor(string str)
        {
            String = str ?? throw new ArgumentNullException(nameof(str), "The string is null.");
            _length = str.Length;
            _position = 0;
        }

        /// <summary>
        ///     Gets the number of characters yet to process.
        /// </summary>
        public int Count => _length - _position;

        /// <summary>
        ///     Gets a value that indicates whether the current position is at the end of the string.
        /// </summary>
        public bool EndOfString => _position == _length;

        public bool Greedy { get; set; }

        /// <summary>
        ///     Gets or sets the current position withing the underlying string.
        /// </summary>
        /// <exception cref="ArgumentOutOfRangeException">
        ///     The position must be greater than zero and less or equal to the length of
        ///     the underlying string.
        /// </exception>
        public int Position
        {
            get => _position;

            set
            {
                if (value > 0 && value <= _length)
                {
                    _position = value;
                }
                else
                {
                    throw new ArgumentOutOfRangeException(nameof(value), "The position must be greater than zero and less or equal to the length of the underlying string.");
                }
            }
        }

        /// <summary><commit_msg>FIX: off by one error in StringProcessor<commit_after>﻿// Needed for NET35 (BigInteger)

using System;
using System.Collections.Generic;
using Theraot.Collections;

namespace Theraot.Core
{
    /// <summary>
    ///     A class to extract information from strings.
    /// </summary>
    /// <remarks>
    ///     An alternative for StringReader for all your parsing needs.
    ///     StringProcessor is NOT thread safe. Do not share instances without locking.
    /// </remarks>
    public class StringProcessor
    {
        // Note: it is assumed that _length = _string.Length - if this weren't true extra check would be needed in every case of IndexOf or IndexOfAny
        private readonly int _length;

        private int _position;

        /// <summary>
        ///     Creates a new instance of StringProcessor.
        /// </summary>
        /// <param name="str">The string to process.</param>
        /// <exception cref="ArgumentNullException">The string is null.</exception>
        public StringProcessor(string str)
        {
            String = str ?? throw new ArgumentNullException(nameof(str), "The string is null.");
            _length = str.Length;
            _position = 0;
        }

        /// <summary>
        ///     Gets the number of characters yet to process.
        /// </summary>
        public int Count => _length - _position;

        /// <summary>
        ///     Gets a value that indicates whether the current position is at the end of the string.
        /// </summary>
        public bool EndOfString => _position == _length;

        public bool Greedy { get; set; }

        /// <summary>
        ///     Gets or sets the current position withing the underlying string.
        /// </summary>
        /// <exception cref="ArgumentOutOfRangeException">
        ///     The position must be greater or equal to zero and less or equal to the length of
        ///     the underlying string.
        /// </exception>
        public int Position
        {
            get => _position;

            set
            {
                if (value >= 0 && value <= _length)
                {
                    _position = value;
                }
                else
                {
                    throw new ArgumentOutOfRangeException(nameof(value), "The position must be greater or equal to zero and less or equal to the length of the underlying string.");
                }
            }
        }

        /// <summary>
{'fraction_non_alphanumeric': 0.09584013050570962, 'fraction_numerical': 0.0016313213703099511, 'mean_word_length': 1.5938656795346378, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5822401', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1153, 'n_words': 455}