<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace PineTree.Language.Lexer
{
    public class PineTreeLexer
    {
        private static List<string> _keywords = new List<string>()
        {
            "if",
            "else",
            "while",
            "for",

            "new",

            "class",

            "var",
            "void",

            "is",

            "import",

            "true",
            "false",

            "null",

            "public",
            "private",
            "protected",

            "static",

            "return",
        };

        private StringBuilder _builder;
        private int _index;
        private int _lineIndex;
        private int _lineNumber;
        private string _source;
        private Position _start;

        /// <summary>
        /// Gets the size of the currently loaded Source Code.
        /// </summary>
        public int Size => _source.Length;

        private char _ch => _source.CharAt(_index);

        private char _last => _source.CharAt(_index - 1);

        private char _next => _source.CharAt(_index + 1);

        /// <summary>
        /// Initializes a new instance of the <see cref="PineTreeLexer"/> with the given source code.
        /// </summary>
        /// <param name="sourceCode"></param>
        public PineTreeLexer(string sourceCode)
        {
            Reset(sourceCode);
        }

        /// <summary>
        /// Initializes a new instance of the <see cref="PineTreeLexer"/> with no source code.
        /// </summary>
        public PineTreeLexer()
            : this(string.Empty)
        {
        }

        /// <summary>
        /// Clears the current source code regardless of the progress of the lexing process.
        /// </summary>
        public void Clear()
        {
            _source = string.Empty;
        }

        /// <summary>
        /// Feeds new source code into this <see cref="PineTreeLexer"/>.
        /// </summary>
        /// <remarks>
        /// If the lexer has reached the end of file, the cached source code will be removed.
        /// </remarks><commit_msg>Fix bug where Lexer would not Reset on Clear.<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace PineTree.Language.Lexer
{
    public class PineTreeLexer
    {
        private static List<string> _keywords = new List<string>()
        {
            "if",
            "else",
            "while",
            "for",

            "new",

            "class",

            "var",
            "void",

            "is",

            "import",

            "true",
            "false",

            "null",

            "public",
            "private",
            "protected",

            "static",

            "return",
        };

        private StringBuilder _builder;
        private int _index;
        private int _lineIndex;
        private int _lineNumber;
        private string _source;
        private Position _start;

        /// <summary>
        /// Gets the size of the currently loaded Source Code.
        /// </summary>
        public int Size => _source.Length;

        private char _ch => _source.CharAt(_index);

        private char _last => _source.CharAt(_index - 1);

        private char _next => _source.CharAt(_index + 1);

        /// <summary>
        /// Initializes a new instance of the <see cref="PineTreeLexer"/> with the given source code.
        /// </summary>
        /// <param name="sourceCode"></param>
        public PineTreeLexer(string sourceCode)
        {
            Reset(sourceCode);
        }

        /// <summary>
        /// Initializes a new instance of the <see cref="PineTreeLexer"/> with no source code.
        /// </summary>
        public PineTreeLexer()
            : this(string.Empty)
        {
        }

        /// <summary>
        /// Clears the current source code regardless of the progress of the lexing process.
        /// </summary>
        public void Clear()
        {
            Reset(string.Empty);
        }

        /// <summary>
        /// Feeds new source code into this <see cref="PineTreeLexer"/>.
        /// </summary>
        /// <remarks>
        /// If the lexer has reached the end of file, the cached source code will be removed.
        /// </remarks>
{'fraction_non_alphanumeric': 0.11518560179977502, 'fraction_numerical': 0.0008998875140607424, 'mean_word_length': 1.4455445544554455, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1546389', 'n_tokens_mistral': 1194, 'n_tokens_neox': 1131, 'n_words': 350}