<commit_before>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using JetBrains.Annotations;
using Txt.Core;

namespace Txt.ABNF
{
    /// <summary>
    ///     Provides methods for reading a terminal value using the specified casing rules.
    /// </summary>
    public class TerminalLexer : Lexer<Terminal>
    {
        [DebuggerBrowsable(SwitchOnBuild.DebuggerBrowsableState)]
        private readonly IEqualityComparer<string> comparer;

        [DebuggerBrowsable(SwitchOnBuild.DebuggerBrowsableState)]
        private readonly string terminal;

        public TerminalLexer([NotNull] string terminal, [NotNull] IEqualityComparer<string> comparer)
        {
            if (terminal == null)
            {
                throw new ArgumentNullException(nameof(terminal));
            }
            if (comparer == null)
            {
                throw new ArgumentNullException(nameof(comparer));
            }
            this.terminal = terminal;
            this.comparer = comparer;
        }

        protected override IReadResult<Terminal> ReadImpl(ITextScanner scanner, ITextContext context)
        {
            var result = scanner.TryMatch(terminal, comparer);
            if (result.Success)
            {
                return new ReadResult<Terminal>(new Terminal(result.Text, context));
            }
            return new ReadResult<Terminal>(SyntaxError.FromMatchResult(result, context));
        }
    }
}
<commit_msg>Don't allow empty strings in TerminalLexer<commit_after>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using JetBrains.Annotations;
using Txt.Core;

namespace Txt.ABNF
{
    /// <summary>
    ///     Provides methods for reading a terminal value using the specified casing rules.
    /// </summary>
    public class TerminalLexer : Lexer<Terminal>
    {
        [DebuggerBrowsable(SwitchOnBuild.DebuggerBrowsableState)]
        private readonly IEqualityComparer<string> comparer;

        [DebuggerBrowsable(SwitchOnBuild.DebuggerBrowsableState)]
        private readonly string terminal;

        public TerminalLexer([NotNull] string terminal, [NotNull] IEqualityComparer<string> comparer)
        {
            if (terminal == null)
            {
                throw new ArgumentNullException(nameof(terminal));
            }
            if (comparer == null)
            {
                throw new ArgumentNullException(nameof(comparer));
            }
            if (string.IsNullOrEmpty(terminal))
            {
                throw new ArgumentException("Value cannot be null or empty.", nameof(terminal));
            }
            this.terminal = terminal;
            this.comparer = comparer;
        }

        protected override IReadResult<Terminal> ReadImpl(ITextScanner scanner, ITextContext context)
        {
            var result = scanner.TryMatch(terminal, comparer);
            if (result.Success)
            {
                return new ReadResult<Terminal>(new Terminal(result.Text, context));
            }
            return new ReadResult<Terminal>(SyntaxError.FromMatchResult(result, context));
        }
    }
}

{'fraction_non_alphanumeric': 0.08528852251109702, 'fraction_numerical': 0.0, 'mean_word_length': 2.2492276004119462, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6412564', 'n_tokens_mistral': 826, 'n_tokens_neox': 794, 'n_words': 187}