<commit_before>using System.Text;

namespace Parsley.Tests;

class TextTests
{
    public void CanPeekAheadNCharacters()
    {
        var empty = new Text("");
        empty.Peek(0).ShouldBe("");
        empty.Peek(1).ShouldBe("");

        var abc = new Text("abc");
        abc.Peek(0).ShouldBe("");
        abc.Peek(1).ShouldBe("a");
        abc.Peek(2).ShouldBe("ab");
        abc.Peek(3).ShouldBe("abc");
        abc.Peek(4).ShouldBe("abc");
        abc.Peek(100).ShouldBe("abc");
    }

    public void CanAdvanceAheadNCharactersWithSnapshotBacktracking()
    {
        var empty = new Text("");

        empty.Advance(0);
        empty.ToString().ShouldBe("");

        empty.Advance(1);
        empty.ToString().ShouldBe("");

        var abc = new Text("abc");

        abc.Advance(0);
        abc.ToString().ShouldBe("abc");

        var snapshot = abc.Snapshot();
        abc.Advance(1);
        abc.ToString().ShouldBe("bc");

        abc.Restore(snapshot);
        abc.Advance(2);
        abc.ToString().ShouldBe("c");

        abc.Restore(snapshot);
        abc.Advance(3);
        abc.ToString().ShouldBe("");

        abc.Restore(snapshot);
        abc.Advance(4);
        abc.ToString().ShouldBe("");

        abc.Restore(snapshot);
        abc.Advance(100);
        abc.ToString().ShouldBe("");
    }

    public void DetectsTheEndOfInput()
    {
        new Text("!").EndOfInput.ShouldBeFalse();
        new Text("").EndOfInput.ShouldBeTrue();
    }

    public void CanMatchLeadingCharactersByPredicate()
    {
        Predicate<char> letters = char.IsLetter;
        Predicate<char> digits = char.IsDigit;
        Predicate<char> alphanumerics = char.IsLetterOrDigit;

        var empty = new Text("");
        empty.TryMatch(letters, out var value).ShouldBe(false);
        value.ShouldBe("");

        var abc123 = new Text("abc123");
        abc123.TryMatch(digits, out value).ShouldBe(false);
        value.ShouldBe("");
        abc123.TryMatch(letters, out value).ShouldBe(true);
        value.ShouldBe("abc");
        abc123.TryMatch(alphanumerics, out value).ShouldBe(true);
        value.ShouldBe("abc123");

        abc123.Advance(2);
        abc123.TryMatch(digits, out value).ShouldBe(false);
        value.ShouldBe("");
        abc123.TryMatch(letters, out value).ShouldBe(true);
        value.ShouldBe("c");
        abc123.TryMatch(alphanumerics, out value).ShouldBe(true);
        value.ShouldBe("c123");

        abc123.Advance(1);
        abc123.TryMatch(digits, out value).ShouldBe(true);
        value.ShouldBe("123");
        abc123.TryMatch(letters, out value).ShouldBe(false);
        value.ShouldBe("");
        abc123.TryMatch(alphanumerics, out value).ShouldBe(true);
        value.ShouldBe("123");

        abc123.Advance(3);
        abc123.TryMatch(digits, out value).ShouldBe(false);
        value.ShouldBe("");
        abc123.TryMatch(letters, out value).ShouldBe(false);
        value.ShouldBe("");
        abc123.TryMatch(alphanumerics, out value).ShouldBe(false);
        value.ShouldBe("");
    }

    public void CanGetCurrentPosition()
    {
        var empty = new Text("");
        empty.Advance(0);empty.Position.ShouldBe(new Position(1, 1));
        empty.Advance(1);empty.Position.ShouldBe(new Position(1, 1));

        var lines = new StringBuilder()
            .Append("Line 1\n")//Index 0-5, \n
            .Append("Line 2\n")//Index 7-12, \n
            .Append("Line 3\n");//Index 14-19, \n
        var list = new Text(lines.ToString());

        var snapshot = list.Snapshot();
        list.Advance(0);
        list.Position.ShouldBe(new Position(1, 1));

        list.Restore(snapshot);
        list.Advance(5);
        list.Position.ShouldBe(new Position(1, 6));

        list.Restore(snapshot);
        list.Advance(6);
        list.Position.ShouldBe(new Position(1, 7));


        list.Restore(snapshot);
        list.Advance(7);
        list.Position.ShouldBe(new Position(2, 1));

        list.Restore(snapshot);
        list.Advance(12);
        list.Position.ShouldBe(new Position(2, 6));

        list.Restore(snapshot);
        list.Advance(13);
        list.Position.ShouldBe(new Position(2, 7));


        list.Restore(snapshot);
        list.Advance(14);
        list.Position.ShouldBe(new Position(3, 1));

        list.Restore(snapshot);
        list.Advance(19);
        list.Position.ShouldBe(new Position(3, 6));

        list.Restore(snapshot);
        list.Advance(20);
        list.Position.ShouldBe(new Position(3, 7));


        list.Restore(snapshot);
        list.Advance(21);
        list.Position.ShouldBe(new Position(4, 1));

        list.Restore(snapshot);
        list.Advance(1000);
        list.Position.ShouldBe(new Position(4, 1));
    }
}
<commit_msg>Add a defensive .ToString() on Peek()'s tests. These are already strings at this time, so the conversion is excessive, but this will keep them from breaking when the return type changes to ReadOnlySpan<char>.<commit_after>using System.Text;

namespace Parsley.Tests;

class TextTests
{
    public void CanPeekAheadNCharacters()
    {
        var empty = new Text("");
        empty.Peek(0).ToString().ShouldBe("");
        empty.Peek(1).ToString().ShouldBe("");

        var abc = new Text("abc");
        abc.Peek(0).ToString().ShouldBe("");
        abc.Peek(1).ToString().ShouldBe("a");
        abc.Peek(2).ToString().ShouldBe("ab");
        abc.Peek(3).ToString().ShouldBe("abc");
        abc.Peek(4).ToString().ShouldBe("abc");
        abc.Peek(100).ToString().ShouldBe("abc");
    }

    public void CanAdvanceAheadNCharactersWithSnapshotBacktracking()
    {
        var empty = new Text("");

        empty.Advance(0);
        empty.ToString().ShouldBe("");

        empty.Advance(1);
        empty.ToString().ShouldBe("");

        var abc = new Text("abc");

        abc.Advance(0);
        abc.ToString().ShouldBe("abc");

        var snapshot = abc.Snapshot();
        abc.Advance(1);
        abc.ToString().ShouldBe("bc");

        abc.Restore(snapshot);
        abc.Advance(2);
        abc.ToString().ShouldBe("c");

        abc.Restore(snapshot);
        abc.Advance(3);
        abc.ToString().ShouldBe("");

        abc.Restore(snapshot);
        abc.Advance(4);
        abc.ToString().ShouldBe("");

        abc.Restore(snapshot);
        abc.Advance(100);
        abc.ToString().ShouldBe("");
    }

    public void DetectsTheEndOfInput()
    {
        new Text("!").EndOfInput.ShouldBeFalse();
        new Text("").EndOfInput.ShouldBeTrue();
    }

    public void CanMatchLeadingCharactersByPredicate()
    {
        Predicate<char> letters = char.IsLetter;
        Predicate<char> digits = char.IsDigit;
        Predicate<char> alphanumerics = char.IsLetterOrDigit;

        var empty = new Text("");
        empty.TryMatch(letters, out var value).ShouldBe(false);
        value.ShouldBe("");

        var abc123 = new Text("abc123");
        abc123.TryMatch(digits, out value).ShouldBe(false);
        value.ShouldBe("");
        abc123.TryMatch(letters, out value).ShouldBe(true);
        value.ShouldBe("abc");
        abc123.TryMatch(alphanumerics, out value).ShouldBe(true);
        value.ShouldBe("abc123");

        abc123.Advance(2);
        abc123.TryMatch(digits, out value).ShouldBe(false);
        value.ShouldBe("");
        abc123.TryMatch(letters, out value).ShouldBe(true);
        value.ShouldBe("c");
        abc123.TryMatch(alphanumerics, out value).ShouldBe(true);
        value.ShouldBe("c123");

        abc123.Advance(1);
        abc123.TryMatch(digits, out value).ShouldBe(true);
        value.ShouldBe("123");
        abc123.TryMatch(letters, out value).ShouldBe(false);
        value.ShouldBe("");
        abc123.TryMatch(alphanumerics, out value).ShouldBe(true);
        value.ShouldBe("123");

        abc123.Advance(3);
        abc123.TryMatch(digits, out value).ShouldBe(false);
        value.ShouldBe("");
        abc123.TryMatch(letters, out value).ShouldBe(false);
        value.ShouldBe("");
        abc123.TryMatch(alphanumerics, out value).ShouldBe(false);
        value.ShouldBe("");
    }

    public void CanGetCurrentPosition()
    {
        var empty = new Text("");
        empty.Advance(0);empty.Position.ShouldBe(new Position(1, 1));
        empty.Advance(1);empty.Position.ShouldBe(new Position(1, 1));

        var lines = new StringBuilder()
            .Append("Line 1\n")//Index 0-5, \n
            .Append("Line 2\n")//Index 7-12, \n
            .Append("Line 3\n");//Index 14-19, \n
        var list = new Text(lines.ToString());

        var snapshot = list.Snapshot();
        list.Advance(0);
        list.Position.ShouldBe(new Position(1, 1));

        list.Restore(snapshot);
        list.Advance(5);
        list.Position.ShouldBe(new Position(1, 6));

        list.Restore(snapshot);
        list.Advance(6);
        list.Position.ShouldBe(new Position(1, 7));


        list.Restore(snapshot);
        list.Advance(7);
        list.Position.ShouldBe(new Position(2, 1));

        list.Restore(snapshot);
        list.Advance(12);
        list.Position.ShouldBe(new Position(2, 6));

        list.Restore(snapshot);
        list.Advance(13);
        list.Position.ShouldBe(new Position(2, 7));


        list.Restore(snapshot);
        list.Advance(14);
        list.Position.ShouldBe(new Position(3, 1));

        list.Restore(snapshot);
        list.Advance(19);
        list.Position.ShouldBe(new Position(3, 6));

        list.Restore(snapshot);
        list.Advance(20);
        list.Position.ShouldBe(new Position(3, 7));


        list.Restore(snapshot);
        list.Advance(21);
        list.Position.ShouldBe(new Position(4, 1));

        list.Restore(snapshot);
        list.Advance(1000);
        list.Position.ShouldBe(new Position(4, 1));
    }
}

{'fraction_non_alphanumeric': 0.15500867966915144, 'fraction_numerical': 0.029817216379046257, 'mean_word_length': 2.4173063503140266, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '338269', 'n_tokens_mistral': 3197, 'n_tokens_neox': 3030, 'n_words': 496}