<commit_before>﻿namespace Uri.Grammar
{
    using System.Collections.Generic;
    using System.Diagnostics.Contracts;

    using Text.Scanning;

    public class PathNoSchemeLexer : Lexer<PathNoScheme>
    {
         private readonly ILexer<SegmentNonZeroNoColons> segmentNonZeroNoColonsLexer;

        private readonly ILexer<Segment> segmentLexer;

        public PathNoSchemeLexer()
            : this(new SegmentNonZeroNoColonsLexer(), new SegmentLexer())
        {
        }

        public PathNoSchemeLexer(ILexer<SegmentNonZeroNoColons> segmentNonZeroNoColonsLexer, ILexer<Segment> segmentLexer)
            : base("path-noscheme")
        {
            this.segmentNonZeroNoColonsLexer = segmentNonZeroNoColonsLexer;
            this.segmentLexer = segmentLexer;
        }

        public override bool TryRead(ITextScanner scanner, out PathNoScheme element)
        {
            if (scanner.EndOfInput)
            {
                element = default(PathNoScheme);
                return false;
            }

            var context = scanner.GetContext();
            SegmentNonZeroNoColons segmentNonZeroNoColons;
            if (!this.segmentNonZeroNoColonsLexer.TryRead(scanner, out segmentNonZeroNoColons))
            {
                element = default(PathNoScheme);
                return false;
            }

            var segments = new List<Sequence<Element, Segment>>();
            var innerContext = scanner.GetContext();
            while (scanner.TryMatch('/'))
            {
                var terminal = new Element('/', innerContext);
                Segment segment;
                if (this.segmentLexer.TryRead(scanner, out segment))
                {
                    var sequence = new Sequence<Element, Segment>(terminal, segment, innerContext);
                    segments.Add(sequence);
                }
                else
                {
                    scanner.PutBack('/');
                    break;
                }
<commit_msg>Fixing a bug in 'PathNoSchemeLexer'<commit_after>﻿namespace Uri.Grammar
{
    using System.Collections.Generic;
    using System.Diagnostics.Contracts;

    using Text.Scanning;

    public class PathNoSchemeLexer : Lexer<PathNoScheme>
    {
         private readonly ILexer<SegmentNonZeroNoColons> segmentNonZeroNoColonsLexer;

        private readonly ILexer<Segment> segmentLexer;

        public PathNoSchemeLexer()
            : this(new SegmentNonZeroNoColonsLexer(), new SegmentLexer())
        {
        }

        public PathNoSchemeLexer(ILexer<SegmentNonZeroNoColons> segmentNonZeroNoColonsLexer, ILexer<Segment> segmentLexer)
            : base("path-noscheme")
        {
            this.segmentNonZeroNoColonsLexer = segmentNonZeroNoColonsLexer;
            this.segmentLexer = segmentLexer;
        }

        public override bool TryRead(ITextScanner scanner, out PathNoScheme element)
        {
            if (scanner.EndOfInput)
            {
                element = default(PathNoScheme);
                return false;
            }

            var context = scanner.GetContext();
            SegmentNonZeroNoColons segmentNonZeroNoColons;
            if (!this.segmentNonZeroNoColonsLexer.TryRead(scanner, out segmentNonZeroNoColons))
            {
                element = default(PathNoScheme);
                return false;
            }

            var segments = new List<Sequence<Element, Segment>>();
            var innerContext = scanner.GetContext();
            while (!scanner.EndOfInput && scanner.TryMatch('/'))
            {
                var terminal = new Element('/', innerContext);
                Segment segment;
                if (this.segmentLexer.TryRead(scanner, out segment))
                {
                    var sequence = new Sequence<Element, Segment>(terminal, segment, innerContext);
                    segments.Add(sequence);
                }
                else
                {
                    scanner.PutBack('/');
                    break;
                }

{'fraction_non_alphanumeric': 0.07892777364110201, 'fraction_numerical': 0.0, 'mean_word_length': 1.790858725761773, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6798465', 'n_tokens_mistral': 1100, 'n_tokens_neox': 1007, 'n_words': 195}