<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;

using NQuery.Text;

namespace NQuery.Syntax
{
    internal sealed class Parser
    {
        private readonly TextBuffer _textBuffer;
        private readonly SyntaxTree _syntaxTree;
        private readonly List<SyntaxToken> _tokens = new List<SyntaxToken>();
        private int _tokenIndex;

        public Parser(TextBuffer textBuffer, SyntaxTree syntaxTree)
        {
            _textBuffer = textBuffer;
            _syntaxTree = syntaxTree;
            LexAllTokens(textBuffer);
        }

        private SyntaxToken Current
        {
            get { return Peek(0); }
        }

        private SyntaxToken Lookahead
        {
            get { return Peek(1); }
        }

        private void LexAllTokens(TextBuffer textBuffer)
        {
            var lexer = new Lexer(_syntaxTree, textBuffer);
            var badTokens = new List<SyntaxToken>();

            SyntaxToken token;
            do
            {
                token = lexer.Lex();

                // Skip any bad tokens.

                badTokens.Clear();
                while (token.Kind == SyntaxKind.BadToken)
                {
                    badTokens.Add(token);
                    token = lexer.Lex();
                }

                if (badTokens.Count > 0)
                {
                    var trivia = new[] { CreateSkippedTokensTrivia(badTokens) }
                                 .Concat(token.LeadingTrivia).ToArray();
                    token = token.WithLeadingTrivia(trivia);
                }

                _tokens.Add(token);
            } while (token.Kind != SyntaxKind.EndOfFileToken);
        }

        private SyntaxToken Peek(int offset)
        {
            var i = Math.Min(_tokenIndex + offset, _tokens.Count - 1);
            return _tokens[i];
        }

        private SyntaxToken NextToken()
        {
            var result = Current;
            _tokenIndex = Math.Min(_tokenIndex + 1, _tokens.Count - 1);
            return result;
        }

        private SyntaxToken NextTokenAsKeyword()
        {
            var result = NextToken();
            return result.Kind.IsKeyword()
                       ? result
                       : result.WithKind(result.ContextualKind);
        }

        private SyntaxToken NextTokenIf(SyntaxKind kind)
        {
            return Current.Kind == kind
                       ? NextToken()
                       : null;
        }

        private SyntaxToken Match(SyntaxKind kind)
        {
            return Current.Kind == kind
                       ? NextToken()
                       : CreateMissingToken(kind);
        }

        private void SkipTokens(Func<SyntaxToken, bool> stopPredicate)
        {
            if (stopPredicate(Current))
                return;

            var tokens = new List<SyntaxToken>();
            do
            {
                tokens.Add(NextToken());
            } while (!stopPredicate(Current));

            var current = _tokens[_tokenIndex];
            var skippedTokensTrivia = CreateSkippedTokensTrivia(tokens);

            var leadingTrivia = new List<SyntaxTrivia>(current.LeadingTrivia.Count + 1);
            leadingTrivia.Add(skippedTokensTrivia);
            leadingTrivia.AddRange(current.LeadingTrivia);

            _tokens[_tokenIndex] = current.WithLeadingTrivia(leadingTrivia);
        }

        private SyntaxToken CreateMissingToken(SyntaxKind kind)
        {
            var missingTokenSpan = new TextSpan(Current.FullSpan.Start, 0);
            var diagnosticSpan = GetDiagnosticSpanForMissingToken();
            var diagnostics = new List<Diagnostic>(1);
            diagnostics.ReportTokenExpected(diagnosticSpan, Current, kind);
            return new SyntaxToken(_syntaxTree, kind, SyntaxKind.BadToken, true, missingTokenSpan, string.Empty, null, new SyntaxTrivia[0], new SyntaxTrivia[0], diagnostics);
        }

        private TextSpan GetDiagnosticSpanForMissingToken()
        {
            if (_tokenIndex > 0)
            {
                var previousToken = _tokens[_tokenIndex - 1];
                var previousTokenLine = _textBuffer.GetLineFromPosition(previousToken.Span.End);
                var currentTokenLine = _textBuffer.GetLineFromPosition(Current.Span.Start);
                if (currentTokenLine != previousTokenLine)
                    return new TextSpan(previousToken.Span.End, 2);
            }<commit_msg>Refine error recovery strategy when parsing identifiers<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;

using NQuery.Text;

namespace NQuery.Syntax
{
    internal sealed class Parser
    {
        private readonly TextBuffer _textBuffer;
        private readonly SyntaxTree _syntaxTree;
        private readonly List<SyntaxToken> _tokens = new List<SyntaxToken>();
        private int _tokenIndex;

        public Parser(TextBuffer textBuffer, SyntaxTree syntaxTree)
        {
            _textBuffer = textBuffer;
            _syntaxTree = syntaxTree;
            LexAllTokens(textBuffer);
        }

        private SyntaxToken Current
        {
            get { return Peek(0); }
        }

        private SyntaxToken Lookahead
        {
            get { return Peek(1); }
        }

        private void LexAllTokens(TextBuffer textBuffer)
        {
            var lexer = new Lexer(_syntaxTree, textBuffer);
            var badTokens = new List<SyntaxToken>();

            SyntaxToken token;
            do
            {
                token = lexer.Lex();

                // Skip any bad tokens.

                badTokens.Clear();
                while (token.Kind == SyntaxKind.BadToken)
                {
                    badTokens.Add(token);
                    token = lexer.Lex();
                }

                if (badTokens.Count > 0)
                {
                    var trivia = new[] { CreateSkippedTokensTrivia(badTokens) }
                                 .Concat(token.LeadingTrivia).ToArray();
                    token = token.WithLeadingTrivia(trivia);
                }

                _tokens.Add(token);
            } while (token.Kind != SyntaxKind.EndOfFileToken);
        }

        private SyntaxToken Peek(int offset)
        {
            var i = Math.Min(_tokenIndex + offset, _tokens.Count - 1);
            return _tokens[i];
        }

        private SyntaxToken NextToken()
        {
            var result = Current;
            _tokenIndex = Math.Min(_tokenIndex + 1, _tokens.Count - 1);
            return result;
        }

        private SyntaxToken NextTokenAsKeyword()
        {
            var result = NextToken();
            return result.Kind.IsKeyword()
                       ? result
                       : result.WithKind(result.ContextualKind);
        }

        private SyntaxToken NextTokenIf(SyntaxKind kind)
        {
            return Current.Kind == kind
                       ? NextToken()
                       : null;
        }

        private SyntaxToken Match(SyntaxKind kind)
        {
            // Normally, our behavior is to insert missing tokens if the current token doesn't
            // match our expected token. The root of the parser will make sure that after we've
            // completed parsing the main element (i.e. query or expression) the only remaining
            // token is EOF.
            //
            // If there are any tokens left, we report an error saying "we expected EOF but
            // found <token>". That EOF token will contain all remaining tokens as skipped
            // token trivia.
            //
            // However, there are cases where we expect an identifier but the currently typed
            // identifier is a keyword.
            //
            // For example, consider this case:
            //
            //     SELECT  e.FirstName + ' ' + e.LastName AS Full|
            //     FROM    Employees e
            //
            // The developer is typing "FullName" but as soon as he completes the second
            // "l" the syntax is completely messed up.
            //
            // Here is another example:
            //
            //     SELECT   *
            //     FROM     Or|
            //
            // In this case, the develope wants to type "Orders" but as soon as the "r" is
            // typed the completion will stop working because the context of the "Or"
            // keyword is no longer part of the table reference but part of the skipped
            // token trivia contained in the EOF token.
            //
            // Both cases can be fixed by skipping the current token and inserting a missing
            // token that contains the skipped token as trivia.
            //
            // You may wonder why we shouldn't do this for all cases where an identifier is
            // expeted but a keyword is found.
            //
            // Consider this case:
            //
            //     SELECT   o.|
            //     FROM     Orders
            //
            // If we were always eating the curent token, we would skip the FROM token which
            // would cause the completion provider to think we've typed "o.FROM|" which is not
            // what we want. So in essence we want to disambiguate this case:
            //
            //     SELECT   o.|
            //     FROM     Orders
            //
            // from:
            //
            //     SELECT   o.From|
            //
            // The easiest is to use the line information -- if the current token is on the
            // same line as the previous, then we skip (second case). Otherwise we insert
            // a new token (frist case).

            if (kind == SyntaxKind.IdentifierToken && Current.Kind.IsKeyword() && IsPreviousTokenOnSameLine())
                return SkipAndInsertMissingToken(SyntaxKind.IdentifierToken);

            return Current.Kind == kind
                       ? NextToken()
                       : InsertMissingToken(kind);
        }

        private bool IsPreviousTokenOnSameLine()
        {
            var previous = Peek(-1);
            if (previous == null)
                return true;

            var previousLine = _textBuffer.GetLineNumberFromPosition(previous.Span.Start);
            var currentLine = _textBuffer.GetLineNumberFromPosition(Current.Span.Start);
            return previousLine == currentLine;
        }

        private void SkipTokens(Func<SyntaxToken, bool> stopPredicate)
        {
            if (stopPredicate(Current))
                return;

            var tokens = new List<SyntaxToken>();
            do
            {
                tokens.Add(NextToken());
            } while (!stopPredicate(Current));

            var current = _tokens[_tokenIndex];
            var skippedTokensTrivia = CreateSkippedTokensTrivia(tokens);

            var leadingTrivia = new List<SyntaxTrivia>(current.LeadingTrivia.Count + 1);
            leadingTrivia.Add(skippedTokensTrivia);
            leadingTrivia.AddRange(current.LeadingTrivia);

            _tokens[_tokenIndex] = current.WithLeadingTrivia(leadingTrivia);
        }

        private SyntaxToken InsertMissingToken(SyntaxKind kind)
        {
            var missingTokenSpan = new TextSpan(Current.FullSpan.Start, 0);
            var diagnosticSpan = GetDiagnosticSpanForMissingToken();
            var diagnostics = new List<Diagnostic>(1);
            diagnostics.ReportTokenExpected(diagnosticSpan, Current, kind);
            return new SyntaxToken(_syntaxTree, kind, SyntaxKind.BadToken, true, missingTokenSpan, string.Empty, null, new SyntaxTrivia[0], new SyntaxTrivia[0], diagnostics);
        }

        private SyntaxToken SkipAndInsertMissingToken(SyntaxKind kind)
        {
            var skippedToken = Current;
            var skippedTokensTrivia = new[] { CreateSkippedTokensTrivia(new[] { skippedToken }) };
            NextToken();

            var missingTokenSpan = new TextSpan(Current.FullSpan.Start, 0);
            var diagnosticSpan = GetDiagnosticSpanForMissingToken();
            var diagnostics = new List<Diagnostic>(1);
            diagnostics.ReportTokenExpected(diagnosticSpan, skippedToken, kind);
            return new SyntaxToken(_syntaxTree, kind, SyntaxKind.BadToken, true, missingTokenSpan, string.Empty, null, skippedTokensTrivia, new SyntaxTrivia[0], diagnostics);
        }

        private TextSpan GetDiagnosticSpanForMissingToken()
        {
            if (_tokenIndex > 0)
            {
                var previousToken = _tokens[_tokenIndex - 1];
                var previousTokenLine = _textBuffer.GetLineFromPosition(previousToken.Span.End);
                var currentTokenLine = _textBuffer.GetLineFromPosition(Current.Span.Start);
                if (currentTokenLine != previousTokenLine)
                    return new TextSpan(previousToken.Span.End, 2);
            }
{'fraction_non_alphanumeric': 0.07802129958326902, 'fraction_numerical': 0.0024695169007562896, 'mean_word_length': 1.585078795132655, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '636424', 'n_tokens_mistral': 3275, 'n_tokens_neox': 3126, 'n_words': 854}