<commit_before>﻿using System;
using Lury.Compiling.Lexer;
using Lury.Compiling.Utils;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace UnitTest
{
    [TestClass]
    public class TokenTest
    {
        const string name = "dummyName";
        const string nameShort = "n";
        const string regex = "dummyRegex";
        const string source = "dummyText";
        const int index = 2;
        const int length = 3;
        static CharPosition position;
        static RegexTokenEntry entry, entryShort;

        [ClassInitialize]
        public static void ClassInitialize(TestContext context)
        {
            position = new CharPosition(1, 3);
            entry = new RegexTokenEntry(name, regex);
            entryShort = new RegexTokenEntry(nameShort, regex);
        }

        [TestMethod]
        public void EntryTest()
        {
            Token token = new Token(entry, source, index, length);
            Assert.AreEqual(entry, token.Entry);
        }

        [TestMethod]
        public void TextTest()
        {
            Token token = new Token(entry, source, index, length);
            Assert.AreEqual("mmy", token.Text);
        }

        [TestMethod]
        public void IndexTest()
        {
            Token token = new Token(entry, source, index, length);
            Assert.AreEqual(index, token.Index);
        }

        [TestMethod]
        public void PositionTest()
        {
            Token token = new Token(entry, source, index, length);
            Assert.AreEqual(position, token.Position);
        }

        [TestMethod]
        public void ToStringTest()
        {
            Token token = new Token(entry, source, index, length);
            string tokenString = token.ToString();

            Assert.IsTrue(tokenString.Contains(position.Line.ToString()));
            Assert.IsTrue(tokenString.Contains(position.Column.ToString()));
            Assert.IsTrue(tokenString.Contains(entry.Name));
            Assert.IsTrue(tokenString.Contains("mmy"));
        }

        [TestMethod]
        public void ToStringTestShort()
        {
            Token token = new Token(entryShort, source, index, length);
            string tokenString = token.ToString();

            Assert.IsTrue(tokenString.Contains(position.Line.ToString()));
            Assert.IsTrue(tokenString.Contains(position.Column.ToString()));
            Assert.IsTrue(tokenString.Contains(entryShort.Name));
        }
    }
}
<commit_msg>Add a test method 'TokenTest#ConstructorErrorTest'<commit_after>﻿using System;
using Lury.Compiling.Lexer;
using Lury.Compiling.Utils;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace UnitTest
{
    [TestClass]
    public class TokenTest
    {
        const string name = "dummyName";
        const string nameShort = "n";
        const string regex = "dummyRegex";
        const string source = "dummyText";
        const int index = 2;
        const int length = 3;
        static CharPosition position;
        static RegexTokenEntry entry, entryShort;

        [ClassInitialize]
        public static void ClassInitialize(TestContext context)
        {
            position = new CharPosition(1, 3);
            entry = new RegexTokenEntry(name, regex);
            entryShort = new RegexTokenEntry(nameShort, regex);
        }

        [TestMethod]
        public void EntryTest()
        {
            Token token = new Token(entry, source, index, length);
            Assert.AreEqual(entry, token.Entry);
        }

        [TestMethod]
        public void TextTest()
        {
            Token token = new Token(entry, source, index, length);
            Assert.AreEqual("mmy", token.Text);
        }

        [TestMethod]
        public void IndexTest()
        {
            Token token = new Token(entry, source, index, length);
            Assert.AreEqual(index, token.Index);
        }

        [TestMethod]
        public void PositionTest()
        {
            Token token = new Token(entry, source, index, length);
            Assert.AreEqual(position, token.Position);
        }

        [TestMethod]
        public void ToStringTest()
        {
            Token token = new Token(entry, source, index, length);
            string tokenString = token.ToString();

            Assert.IsTrue(tokenString.Contains(position.Line.ToString()));
            Assert.IsTrue(tokenString.Contains(position.Column.ToString()));
            Assert.IsTrue(tokenString.Contains(entry.Name));
            Assert.IsTrue(tokenString.Contains("mmy"));
        }

        [TestMethod]
        public void ToStringTestShort()
        {
            Token token = new Token(entryShort, source, index, length);
            string tokenString = token.ToString();

            Assert.IsTrue(tokenString.Contains(position.Line.ToString()));
            Assert.IsTrue(tokenString.Contains(position.Column.ToString()));
            Assert.IsTrue(tokenString.Contains(entryShort.Name));
        }

        [TestMethod]
        [ExpectedException(typeof(ArgumentOutOfRangeException))]
        public void ConstructorErrorTest()
        {
            Token token = new Token(entry, source, source.Length, 1);
        }
    }
}

{'fraction_non_alphanumeric': 0.09975014414760715, 'fraction_numerical': 0.0017297712857966558, 'mean_word_length': 2.093935790725327, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1163526', 'n_tokens_mistral': 1340, 'n_tokens_neox': 1313, 'n_words': 316}