<commit_before>﻿namespace Text.Scanning.Core
{
    /// <summary>A-Z / a-z</summary>
    public class AlphaLexer : Lexer<AlphaToken>
    {
        public AlphaLexer(ITextScanner scanner)
            : base(scanner)
        {
        }

        public override AlphaToken Read()
        {
            var context = this.Scanner.GetContext();
            AlphaToken token;
            if (this.TryRead(out token))
            {
                return token;
            }

            throw new SyntaxErrorException("Expected 'ALPHA'", context);
        }

        public override bool TryRead(out AlphaToken token)
        {
            token = default(AlphaToken);
            var context = this.Scanner.GetContext();
            for (int i = 'A'; i <= 'Z'; i++)
            {
                var c = (char) i;
                if (this.Scanner.TryMatch(c))
                {
                    token = new AlphaToken(c, context);
                    return true;
                }
            }

            for (int i = 'a'; i <= 'z'; i++)
            {
                var c = (char)i;
                if (this.Scanner.TryMatch(c))
                {
                    token = new AlphaToken(c, context);
                    return true;
                }
            }

            return false;
        }
    }
}
<commit_msg>Refactoring 'AlphaLexer'<commit_after>﻿namespace Text.Scanning.Core
{
    /// <summary>A-Z / a-z</summary>
    public class AlphaLexer : Lexer<AlphaToken>
    {
        public AlphaLexer(ITextScanner scanner)
            : base(scanner)
        {
        }

        public override AlphaToken Read()
        {
            var context = this.Scanner.GetContext();
            AlphaToken token;
            if (this.TryRead(out token))
            {
                return token;
            }

            throw new SyntaxErrorException("Expected 'ALPHA'", context);
        }

        public override bool TryRead(out AlphaToken token)
        {
            var context = this.Scanner.GetContext();
            return this.TryReadLowercase(out token, context) || this.TryReadUppercase(out token, context);
        }

        private bool TryReadUppercase(out AlphaToken token, ITextContext context)
        {
            for (var c = 'A'; c <= 'Z'; c++)
            {
                if (this.Scanner.TryMatch(c))
                {
                    token = new AlphaToken(c, context);
                    return true;
                }
            }

            token = default(AlphaToken);
            return false;
        }

        private bool TryReadLowercase(out AlphaToken token, ITextContext context)
        {
            for (var c = 'a'; c <= 'z'; c++)
            {
                if (this.Scanner.TryMatch(c))
                {
                    token = new AlphaToken(c, context);
                    return true;
                }
            }

            token = default(AlphaToken);
            return false;
        }
    }
}

{'fraction_non_alphanumeric': 0.09986595174262734, 'fraction_numerical': 0.0, 'mean_word_length': 1.185212298682284, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2259653', 'n_tokens_mistral': 809, 'n_tokens_neox': 767, 'n_words': 181}