<commit_before>﻿' Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

Imports System
Imports System.Collections.Generic
Imports System.Linq
Imports System.Runtime.CompilerServices
Imports System.Runtime.InteropServices
Imports System.Text
Imports System.Threading
Imports Microsoft.CodeAnalysis
Imports Microsoft.CodeAnalysis.Shared.Collections
Imports Microsoft.CodeAnalysis.Text
Imports Microsoft.CodeAnalysis.VisualBasic
Imports Microsoft.CodeAnalysis.VisualBasic.Symbols
Imports Microsoft.CodeAnalysis.VisualBasic.Syntax
Imports Microsoft.CodeAnalysis.VisualBasic.Utilities

Namespace Microsoft.CodeAnalysis.VisualBasic.Extensions
    Friend Module SyntaxTreeExtensions
        ''' <summary>
        ''' Finds the token being touched by this position. Unlike the normal FindTrivia helper, this helper will prefer
        ''' trivia to the left rather than the right if the position is on the border.
        ''' </summary>
        ''' <param name="syntaxTree">The syntaxTree to search.</param>
        ''' <param name="position">The position to find trivia.</param>
        <Extension()>
        Public Function FindTriviaToLeft(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As SyntaxTrivia
            Return FindTriviaToLeft(syntaxTree.GetRoot(cancellationToken), position)
        End Function

        Private Function FindTriviaToLeft(nodeOrToken As SyntaxNodeOrToken, position As Integer) As SyntaxTrivia
            For Each child In nodeOrToken.ChildNodesAndTokens().Reverse()
                If child.FullSpan.Start < position AndAlso position <= child.FullSpan.End Then
                    If child.IsNode Then
                        Return FindTriviaToLeft(child, position)
                    Else
                        Dim triviaList = child.GetLeadingTrivia().Concat(child.GetTrailingTrivia()).Reverse()

                        For Each trivia In triviaList
                            If trivia.SpanStart < position AndAlso position <= child.FullSpan.End Then
                                Return trivia
                            End If
                        Next
                    End If
                End If
            Next

            Return Nothing
        End Function

        <Extension()>
        Public Function IsInNonUserCode(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As Boolean
            Return _
                syntaxTree.IsEntirelyWithinComment(position, cancellationToken) OrElse
                syntaxTree.IsEntirelyWithinStringOrCharOrNumericLiteral(position, cancellationToken) OrElse
                syntaxTree.IsInInactiveRegion(position, cancellationToken) OrElse
                syntaxTree.IsWithinPartialMethodDeclaration(position, cancellationToken)
        End Function

        <Extension()>
        Public Function IsWithinPartialMethodDeclaration(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As Boolean
            Dim token = syntaxTree.FindTokenOnLeftOfPosition(position, cancellationToken)
            Dim declaration = token.GetAncestor(Of MethodStatementSyntax)
            If declaration IsNot Nothing AndAlso declaration.Modifiers.Any(SyntaxKind.PartialKeyword) Then
                Return True
            End If

            Dim block = token.GetAncestor(Of MethodBlockSyntax)
            If block IsNot Nothing AndAlso block.BlockStatement.Modifiers.Any(SyntaxKind.PartialKeyword) Then
                Return True
            End If

            Return False
        End Function

        <Extension()>
        Public Function IsEntirelyWithinComment(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As Boolean
            Dim trivia = syntaxTree.FindTriviaToLeft(position, cancellationToken)

            If trivia.IsKind(SyntaxKind.CommentTrivia, SyntaxKind.DocumentationCommentTrivia) AndAlso trivia.SpanStart <> position Then
                Return True
            End If

            Return False
        End Function

        <Extension()>
        Public Function IsEntirelyWithinStringOrCharOrNumericLiteral(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As Boolean
            Dim token = syntaxTree.FindTokenOnLeftOfPosition(position, cancellationToken, includeDirectives:=True, includeDocumentationComments:=True)

            If Not token.IsKind(SyntaxKind.StringLiteralToken, SyntaxKind.CharacterLiteralToken, SyntaxKind.DecimalLiteralToken, SyntaxKind.IntegerLiteralToken,
                                SyntaxKind.DateLiteralToken, SyntaxKind.FloatingLiteralToken) Then
                Return False
            End If

            ' Check if it's within a completed token.
            If token.SpanStart < position AndAlso position < token.Span.End Then
                Return True
            End If

            ' Check if it's a token that was started, but not ended
            Dim startLength = 1
            If token.IsKind(SyntaxKind.CharacterLiteralToken) Then
                startLength = 2
            End If

            Dim lastChar = If(token.IsKind(SyntaxKind.CharacterLiteralToken), "'", """")
            Return _
                position = token.Span.End AndAlso
                 (token.Span.Length = startLength OrElse
                  (token.Span.Length > startLength AndAlso Not token.ToString().EndsWith(lastChar, StringComparison.Ordinal)))
        End Function

        <Extension()>
        Public Function IsInInactiveRegion(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As Boolean
            Contract.ThrowIfNull(syntaxTree)

            ' cases:
            ' $ is EOF

            ' #IF false Then
            '    |

            ' #IF false Then
            '    |$

            ' #IF false Then
            ' |

            ' #IF false Then
            ' |$

            If SyntaxTree.FindTriviaToLeft(position, cancellationToken).Kind = SyntaxKind.DisabledTextTrivia Then
                Return True
            End If

            ' TODO : insert point at the same line as preprocessor?
            Return False
        End Function

        <Extension()>
        Public Function IsInSkippedText(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As Boolean
            Dim trivia = syntaxTree.FindTriviaToLeft(position, cancellationToken)

            Return trivia.IsKind(SyntaxKind.SkippedTokensTrivia)
        End Function

        ''' <summary>
        ''' If the position is inside of token, return that token; otherwise, return the token to right.
        ''' </summary>
        <Extension()>
        Public Function FindTokenOnRightOfPosition(<commit_msg>Fix for Issue  #5528 Removes recursion from the function.<commit_after>﻿' Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

Imports System
Imports System.Collections.Generic
Imports System.Linq
Imports System.Runtime.CompilerServices
Imports System.Runtime.InteropServices
Imports System.Text
Imports System.Threading
Imports Microsoft.CodeAnalysis
Imports Microsoft.CodeAnalysis.Shared.Collections
Imports Microsoft.CodeAnalysis.Text
Imports Microsoft.CodeAnalysis.VisualBasic
Imports Microsoft.CodeAnalysis.VisualBasic.Symbols
Imports Microsoft.CodeAnalysis.VisualBasic.Syntax
Imports Microsoft.CodeAnalysis.VisualBasic.Utilities

Namespace Microsoft.CodeAnalysis.VisualBasic.Extensions
    Friend Module SyntaxTreeExtensions
        ''' <summary>
        ''' Finds the token being touched by this position. Unlike the normal FindTrivia helper, this helper will prefer
        ''' trivia to the left rather than the right if the position is on the border.
        ''' </summary>
        ''' <param name="syntaxTree">The syntaxTree to search.</param>
        ''' <param name="position">The position to find trivia.</param>
        <Extension()>
        Public Function FindTriviaToLeft(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As SyntaxTrivia
            Return FindTriviaToLeft(syntaxTree.GetRoot(cancellationToken), position)
        End Function

        Private Function FindTriviaToLeft(nodeOrToken As SyntaxNodeOrToken, position As Integer) As SyntaxTrivia
recurse:
            For Each child In nodeOrToken.ChildNodesAndTokens().Reverse()
                If (child.FullSpan.Start < position) AndAlso (position <= child.FullSpan.End) Then
                    If child.IsNode Then
                        nodeOrToken = child
                        GoTo recurse
                    Else
                        Dim triviaList = child.GetLeadingTrivia().Concat(child.GetTrailingTrivia()).Reverse()
                        For Each trivia In triviaList
                            If (trivia.SpanStart < position) AndAlso (position <= child.FullSpan.End) Then
                                Return trivia
                            End If
                        Next
                    End If
                End If
            Next
            Return Nothing
        End Function

        <Extension()>
        Public Function IsInNonUserCode(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As Boolean
            Return _
                syntaxTree.IsEntirelyWithinComment(position, cancellationToken) OrElse
                syntaxTree.IsEntirelyWithinStringOrCharOrNumericLiteral(position, cancellationToken) OrElse
                syntaxTree.IsInInactiveRegion(position, cancellationToken) OrElse
                syntaxTree.IsWithinPartialMethodDeclaration(position, cancellationToken)
        End Function

        <Extension()>
        Public Function IsWithinPartialMethodDeclaration(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As Boolean
            Dim token = syntaxTree.FindTokenOnLeftOfPosition(position, cancellationToken)
            Dim declaration = token.GetAncestor(Of MethodStatementSyntax)
            If declaration IsNot Nothing AndAlso declaration.Modifiers.Any(SyntaxKind.PartialKeyword) Then
                Return True
            End If

            Dim block = token.GetAncestor(Of MethodBlockSyntax)
            If block IsNot Nothing AndAlso block.BlockStatement.Modifiers.Any(SyntaxKind.PartialKeyword) Then
                Return True
            End If

            Return False
        End Function

        <Extension()>
        Public Function IsEntirelyWithinComment(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As Boolean
            Dim trivia = syntaxTree.FindTriviaToLeft(position, cancellationToken)

            If trivia.IsKind(SyntaxKind.CommentTrivia, SyntaxKind.DocumentationCommentTrivia) AndAlso trivia.SpanStart <> position Then
                Return True
            End If

            Return False
        End Function

        <Extension()>
        Public Function IsEntirelyWithinStringOrCharOrNumericLiteral(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As Boolean
            Dim token = syntaxTree.FindTokenOnLeftOfPosition(position, cancellationToken, includeDirectives:=True, includeDocumentationComments:=True)

            If Not token.IsKind(SyntaxKind.StringLiteralToken, SyntaxKind.CharacterLiteralToken, SyntaxKind.DecimalLiteralToken, SyntaxKind.IntegerLiteralToken,
                                SyntaxKind.DateLiteralToken, SyntaxKind.FloatingLiteralToken) Then
                Return False
            End If

            ' Check if it's within a completed token.
            If token.SpanStart < position AndAlso position < token.Span.End Then
                Return True
            End If

            ' Check if it's a token that was started, but not ended
            Dim startLength = 1
            If token.IsKind(SyntaxKind.CharacterLiteralToken) Then
                startLength = 2
            End If

            Dim lastChar = If(token.IsKind(SyntaxKind.CharacterLiteralToken), "'", """")
            Return _
                position = token.Span.End AndAlso
                 (token.Span.Length = startLength OrElse
                  (token.Span.Length > startLength AndAlso Not token.ToString().EndsWith(lastChar, StringComparison.Ordinal)))
        End Function

        <Extension()>
        Public Function IsInInactiveRegion(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As Boolean
            Contract.ThrowIfNull(syntaxTree)

            ' cases:
            ' $ is EOF

            ' #IF false Then
            '    |

            ' #IF false Then
            '    |$

            ' #IF false Then
            ' |

            ' #IF false Then
            ' |$

            If syntaxTree.FindTriviaToLeft(position, cancellationToken).Kind = SyntaxKind.DisabledTextTrivia Then
                Return True
            End If

            ' TODO : insert point at the same line as preprocessor?
            Return False
        End Function

        <Extension()>
        Public Function IsInSkippedText(syntaxTree As SyntaxTree, position As Integer, cancellationToken As CancellationToken) As Boolean
            Dim trivia = syntaxTree.FindTriviaToLeft(position, cancellationToken)

            Return trivia.IsKind(SyntaxKind.SkippedTokensTrivia)
        End Function

        ''' <summary>
        ''' If the position is inside of token, return that token; otherwise, return the token to right.
        ''' </summary>
        <Extension()>
        Public Function FindTokenOnRightOfPosition(
{'fraction_non_alphanumeric': 0.05674015178894109, 'fraction_numerical': 0.0008673653776653415, 'mean_word_length': 2.363150218765192, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5326755', 'n_tokens_mistral': 3340, 'n_tokens_neox': 3183, 'n_words': 963}