<commit_before>﻿using System;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace Pliant.Tests.Unit
{
    [TestClass]
    public class LexerTests
    {
        [TestMethod]
        public void Test_Lexer_Scan_Emits_Three_Tokens()
        {
            var input = "three tokens";
            //  S   ->  S W
            //  S   ->  S L
            //  S   ->  <null>
            //  W   ->  W \s
            //  W   ->  \s
            //  L   ->  L [a-z]
            //  L   ->  [a-z]
            var grammar = new GrammarBuilder("S", p=>p
                    .Production("S", r=>r
                        .Rule("S", "W")
                        .Rule("S", "L")
                        .Lambda())
                    .Production("W", r=>r
                        .Rule(new WhitespaceTerminal(), "W`"))
                    .Production("W`", r=>r
                        .Rule(new WhitespaceTerminal(), "W`")
                        .Lambda())
                    .Production("L", r=>r
                        .Rule(new RangeTerminal('a', 'z'), "L`"))
                    .Production("L`", r=>r
                        .Rule(new RangeTerminal('a', 'z'), "L`")
                        .Lambda()))
                .GetGrammar();

            var lexer = new Lexer(grammar);

            int tokenCount = 0;
            var tokenObserver = new Observer<IToken>(t=>
            {
                tokenCount++;
            });
            lexer.Subscribe(tokenObserver);

            for (int i = 0; i < input.Length; i++)
            {
                char c = input[i];
                if (!lexer.Scan(c))
                    break;
            }

            Assert.AreEqual(3, tokenCount);
        }
    }
}
<commit_msg>updates lexer test to use deterministic grammer<commit_after>﻿using System;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace Pliant.Tests.Unit
{
    [TestClass]
    public class LexerTests
    {
        [TestMethod]
        public void Test_Lexer_Scan_Emits_Three_Tokens()
        {
            var input = "three tokens";
            //  S   ->  S W
            //  S   ->  S L
            //  S   ->  <null>
            //  W   ->  W \s
            //  W   ->  \s
            //  L   ->  L [a-z]
            //  L   ->  [a-z]
            var grammar = new GrammarBuilder("S", p=>p
                    .Production("S", r=>r
                        .Rule("S", "W")
                        .Rule("S", "L")
                        .Lambda())
                    .Production("W", r=>r
                        .Rule(new WhitespaceTerminal(), "W")
                        .Lambda())
                    .Production("L", r=>r
                        .Rule(new RangeTerminal('a', 'z'), "L")
                        .Lambda()))
                .GetGrammar();

            var lexer = new Lexer(grammar);

            int tokenCount = 0;
            var tokenObserver = new Observer<IToken>(t=>
            {
                tokenCount++;
            });
            lexer.Subscribe(tokenObserver);

            for (int i = 0; i < input.Length; i++)
            {
                char c = input[i];
                if (!lexer.Scan(c))
                    break;
            }

            Assert.AreEqual(3, tokenCount);
        }
    }
}

{'fraction_non_alphanumeric': 0.1292517006802721, 'fraction_numerical': 0.001774622892635315, 'mean_word_length': 0.8851727982162765, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7471367', 'n_tokens_mistral': 1011, 'n_tokens_neox': 903, 'n_words': 197}