<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Uri.Grammar
{
    using Text.Scanning;

    public class PathRootlessLexer : Lexer<PathRootless>
    {
        private readonly ILexer<SegmentNonZero> segmentNonZeroLexer;

        private readonly ILexer<Segment> segmentLexer;

        public PathRootlessLexer()
            : this(new SegmentNonZeroLexer(), new SegmentLexer())
        {
        }

        public PathRootlessLexer(ILexer<SegmentNonZero> segmentNonZeroLexer, ILexer<Segment> segmentLexer)
            : base("path-rootless")
        {
            this.segmentNonZeroLexer = segmentNonZeroLexer;
            this.segmentLexer = segmentLexer;
        }

        public override bool TryRead(ITextScanner scanner, out PathRootless element)
        {
            if (scanner.EndOfInput)
            {
                element = default(PathRootless);
                return false;
            }

            var context = scanner.GetContext();
            SegmentNonZero segmentNonZero;
            if (!this.segmentNonZeroLexer.TryRead(scanner, out segmentNonZero))
            {
                element = default(PathRootless);
                return false;
            }

            var segments = new List<Sequence<Element, Segment>>();
            var innerContext = scanner.GetContext();
            while (scanner.TryMatch('/'))
            {
                var terminal = new Element('/', innerContext);
                Segment segment;
                if (this.segmentLexer.TryRead(scanner, out segment))
                {
                    var sequence = new Sequence<Element, Segment>(terminal, segment, innerContext);
                    segments.Add(sequence);
                }
                else
                {
                    scanner.PutBack('/');
                    break;
                }

                innerContext = scanner.GetContext();
            }

            element = new PathRootless(segmentNonZero, segments, context);
            return true;
        }
    }
}
<commit_msg>Adding missing invariants to 'PathRootlessLexer'<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Uri.Grammar
{
    using System.Diagnostics.Contracts;

    using Text.Scanning;

    public class PathRootlessLexer : Lexer<PathRootless>
    {
        private readonly ILexer<SegmentNonZero> segmentNonZeroLexer;

        private readonly ILexer<Segment> segmentLexer;

        public PathRootlessLexer()
            : this(new SegmentNonZeroLexer(), new SegmentLexer())
        {
        }

        public PathRootlessLexer(ILexer<SegmentNonZero> segmentNonZeroLexer, ILexer<Segment> segmentLexer)
            : base("path-rootless")
        {
            this.segmentNonZeroLexer = segmentNonZeroLexer;
            this.segmentLexer = segmentLexer;
        }

        public override bool TryRead(ITextScanner scanner, out PathRootless element)
        {
            if (scanner.EndOfInput)
            {
                element = default(PathRootless);
                return false;
            }

            var context = scanner.GetContext();
            SegmentNonZero segmentNonZero;
            if (!this.segmentNonZeroLexer.TryRead(scanner, out segmentNonZero))
            {
                element = default(PathRootless);
                return false;
            }

            var segments = new List<Sequence<Element, Segment>>();
            var innerContext = scanner.GetContext();
            while (scanner.TryMatch('/'))
            {
                var terminal = new Element('/', innerContext);
                Segment segment;
                if (this.segmentLexer.TryRead(scanner, out segment))
                {
                    var sequence = new Sequence<Element, Segment>(terminal, segment, innerContext);
                    segments.Add(sequence);
                }
                else
                {
                    scanner.PutBack('/');
                    break;
                }

                innerContext = scanner.GetContext();
            }

            element = new PathRootless(segmentNonZero, segments, context);
            return true;
        }

        [ContractInvariantMethod]
        private void ObjectInvariant()
        {
            Contract.Invariant(this.segmentNonZeroLexer != null);
            Contract.Invariant(this.segmentLexer != null);
        }
    }
}

{'fraction_non_alphanumeric': 0.08327840035157108, 'fraction_numerical': 0.0, 'mean_word_length': 1.7014836795252226, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4722572', 'n_tokens_mistral': 1215, 'n_tokens_neox': 1113, 'n_words': 234}