<commit_before>module ts.formatting {
    var scanner = createScanner(ScriptTarget.ES5, /*skipTrivia*/ false);

    export interface FormattingScanner {
        advance(): void;
        isOnToken(): boolean;
        readTokenInfo(n: Node): TokenInfo;
        lastTrailingTriviaWasNewLine(): boolean;
        close(): void;
    }

    export function getFormattingScanner(sourceFile: SourceFile, enclosingNode: Node, range: TextRange): FormattingScanner {

        scanner.setText(sourceFile.text);
        scanner.setTextPos(enclosingNode.pos);

        var wasNewLine: boolean = true;
        var leadingTrivia: TextRangeWithKind[];
        var trailingTrivia: TextRangeWithKind[];
        var savedStartPos: number;
        var lastTokenInfo: TokenInfo;

        return {
            advance: advance,
            readTokenInfo: readTokenInfo,
            isOnToken: isOnToken,
            lastTrailingTriviaWasNewLine: lastTrailingTriviaWasNewLine,
            close: () => scanner.setText(undefined)
        }


        function advance(): void {
            lastTokenInfo = undefined;
            var isStarted = scanner.getStartPos() !== enclosingNode.pos;

            // accumulate leading trivia and token
            if (isStarted) {
                if (trailingTrivia) {
                    Debug.assert(trailingTrivia.length !== 0);
                    wasNewLine = trailingTrivia[trailingTrivia.length - 1].kind === SyntaxKind.NewLineTrivia;
                }
                else {
                    wasNewLine = false;
                }
            }

            leadingTrivia = undefined;
            trailingTrivia = undefined;

            if (!isStarted) {
                scanner.scan();
            }

            var t: SyntaxKind;
            var startPos = scanner.getStartPos();

            while (startPos < range.end) {
                var t = scanner.getToken();
                if (!isTrivia(t)) {
                    break;
                }

                // consume leading trivia
                scanner.scan();
                var item = {
                    pos: startPos,
                    end: scanner.getStartPos(),
                    kind: t
                }

                startPos = scanner.getStartPos();

                if (!leadingTrivia) {
                    leadingTrivia = [];
                }
                leadingTrivia.push(item);
            }

            savedStartPos = scanner.getStartPos();
        }

        function startsWithGreaterThanToken(t: SyntaxKind): boolean {
            switch(t) {
                case SyntaxKind.GreaterThanEqualsToken:
                case SyntaxKind.GreaterThanGreaterThanEqualsToken:
                case SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken:
                case SyntaxKind.GreaterThanGreaterThanGreaterThanToken:
                case SyntaxKind.GreaterThanGreaterThanToken:
                    return true;
                default:
                    return false;
            }
        }

        function readTokenInfo(n: Node): TokenInfo {
            if (!isOnToken()) {
                return {
                    leadingTrivia: leadingTrivia,
                    trailingTrivia: undefined,
                    token: undefined
                };

            }
            if (lastTokenInfo) {
                //return lastTokenInfo;
            }

            if (scanner.getStartPos() !== savedStartPos) {
                scanner.setTextPos(savedStartPos);
                scanner.scan();
            }

            var current = scanner.getToken();
            var endPos: number;
            if (n.kind === SyntaxKind.BinaryExpression && startsWithGreaterThanToken((<BinaryExpression>n).operator) && current === SyntaxKind.GreaterThanToken) {
                current = scanner.reScanGreaterToken();
                Debug.assert((<BinaryExpression>n).operator === current);
            }
            else if (n.kind === SyntaxKind.RegularExpressionLiteral && current === SyntaxKind.SlashToken) {
                current = scanner.reScanSlashToken();
                Debug.assert(n.kind === current);
            }

            endPos = scanner.getTextPos();

            var token: TextRangeWithKind = {
                pos: scanner.getStartPos(),
                end: scanner.getTextPos(),
                kind: current
            }

            while(scanner.getStartPos() < range.end) {
                current = scanner.scan();
                if (!isTrivia(current)) {
                    break;
                }
                var trivia = {
                    pos: scanner.getStartPos(),
                    end: scanner.getTextPos(),
                    kind: current
                };

                if (!trailingTrivia) {
                    trailingTrivia = [];
                }

                trailingTrivia.push(trivia);

                if (current === SyntaxKind.NewLineTrivia) {
                    // move past new line
                    scanner.scan();
                    break;
                }
            }

            return lastTokenInfo = {
                leadingTrivia: leadingTrivia,
                trailingTrivia: trailingTrivia,
                token: token
            }
        }

        function isOnToken(): boolean {
            var current = (lastTokenInfo && lastTokenInfo.token.kind) ||  scanner.getToken();
            var startPos = (lastTokenInfo && lastTokenInfo.token.pos) || scanner.getStartPos();
            return startPos < range.end && current !== SyntaxKind.EndOfFileToken && !isTrivia(current);
        }

        function lastTrailingTriviaWasNewLine(): boolean {
            return wasNewLine;
        }
    }
}<commit_msg>return last scanned token if possible<commit_after>module ts.formatting {
    var scanner = createScanner(ScriptTarget.ES5, /*skipTrivia*/ false);

    export interface FormattingScanner {
        advance(): void;
        isOnToken(): boolean;
        readTokenInfo(n: Node): TokenInfo;
        lastTrailingTriviaWasNewLine(): boolean;
        close(): void;
    }

    const enum ScanAction{
        Normal,
        RescanGreaterThanToken,
        RescanSlashToken
    }

    export function getFormattingScanner(sourceFile: SourceFile, enclosingNode: Node, range: TextRange): FormattingScanner {

        scanner.setText(sourceFile.text);
        scanner.setTextPos(enclosingNode.pos);

        var wasNewLine: boolean = true;
        var leadingTrivia: TextRangeWithKind[];
        var trailingTrivia: TextRangeWithKind[];
        
        var savedStartPos: number;
        var lastScanAction: ScanAction;
        var lastTokenInfo: TokenInfo;

        return {
            advance: advance,
            readTokenInfo: readTokenInfo,
            isOnToken: isOnToken,
            lastTrailingTriviaWasNewLine: () => wasNewLine,
            close: () => {
                lastTokenInfo = undefined;
                scanner.setText(undefined);
            }
        }

        function advance(): void {
            lastTokenInfo = undefined;
            var isStarted = scanner.getStartPos() !== enclosingNode.pos;

            if (isStarted) {
                if (trailingTrivia) {
                    Debug.assert(trailingTrivia.length !== 0);
                    wasNewLine = trailingTrivia[trailingTrivia.length - 1].kind === SyntaxKind.NewLineTrivia;
                }
                else {
                    wasNewLine = false;
                }
            }

            leadingTrivia = undefined;
            trailingTrivia = undefined;

            if (!isStarted) {
                scanner.scan();
            }

            var t: SyntaxKind;
            var startPos = scanner.getStartPos();

            while (startPos < range.end) {
                var t = scanner.getToken();
                if (!isTrivia(t)) {
                    break;
                }

                // consume leading trivia
                scanner.scan();
                var item = {
                    pos: startPos,
                    end: scanner.getStartPos(),
                    kind: t
                }

                startPos = scanner.getStartPos();

                if (!leadingTrivia) {
                    leadingTrivia = [];
                }
                leadingTrivia.push(item);
            }

            savedStartPos = scanner.getStartPos();
        }

        function shouldRescanGreaterThanToken(container: Node): boolean {
            if (container.kind !== SyntaxKind.BinaryExpression) {
                return false;
            }
            switch ((<BinaryExpression>container).operator) {
                case SyntaxKind.GreaterThanEqualsToken:
                case SyntaxKind.GreaterThanGreaterThanEqualsToken:
                case SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken:
                case SyntaxKind.GreaterThanGreaterThanGreaterThanToken:
                case SyntaxKind.GreaterThanGreaterThanToken:
                    return true;
            }

            return false;
        }

        function shouldRescanSlashToken(container: Node): boolean {
            return container.kind === SyntaxKind.RegularExpressionLiteral;
        }

        function startsWithSlashToken(t: SyntaxKind): boolean {
            return t === SyntaxKind.SlashToken || t === SyntaxKind.SlashEqualsToken;
        }

        function readTokenInfo(n: Node): TokenInfo {
            if (!isOnToken()) {
                return {
                    leadingTrivia: leadingTrivia,
                    trailingTrivia: undefined,
                    token: undefined
                };

            }

            if (lastTokenInfo) {
                if (shouldRescanGreaterThanToken(n)) {
                    if (lastScanAction === ScanAction.RescanGreaterThanToken) {
                        return lastTokenInfo;
                    }
                }
                else if (shouldRescanSlashToken(n)) {
                    if (lastScanAction === ScanAction.RescanSlashToken) {
                        return lastTokenInfo;
                    }
                }
                else if (lastScanAction === ScanAction.Normal) {
                    return lastTokenInfo;
                }
            }

            if (scanner.getStartPos() !== savedStartPos) {
                scanner.setTextPos(savedStartPos);
                scanner.scan();
            }

            var currentToken = scanner.getToken();
            var endPos: number;

            if (currentToken === SyntaxKind.GreaterThanToken && shouldRescanGreaterThanToken(n)) {
                currentToken = scanner.reScanGreaterToken();
                Debug.assert((<BinaryExpression>n).operator === currentToken);
                lastScanAction = ScanAction.RescanGreaterThanToken;
            }
            else if (n.kind === SyntaxKind.RegularExpressionLiteral && startsWithSlashToken(currentToken)) {
                currentToken = scanner.reScanSlashToken();
                Debug.assert(n.kind === currentToken);
                lastScanAction = ScanAction.RescanSlashToken;
            }
            else {
                lastScanAction = ScanAction.Normal;
            }

            endPos = scanner.getTextPos();

            var token: TextRangeWithKind = {
                pos: scanner.getStartPos(),
                end: scanner.getTextPos(),
                kind: currentToken
            }

            while(scanner.getStartPos() < range.end) {
                currentToken = scanner.scan();
                if (!isTrivia(currentToken)) {
                    break;
                }
                var trivia = {
                    pos: scanner.getStartPos(),
                    end: scanner.getTextPos(),
                    kind: currentToken
                };

                if (!trailingTrivia) {
                    trailingTrivia = [];
                }

                trailingTrivia.push(trivia);

                if (currentToken === SyntaxKind.NewLineTrivia) {
                    // move past new line
                    scanner.scan();
                    break;
                }
            }

            return lastTokenInfo = {
                leadingTrivia: leadingTrivia,
                trailingTrivia: trailingTrivia,
                token: token
            }
        }

        function isOnToken(): boolean {
            var current = (lastTokenInfo && lastTokenInfo.token.kind) ||  scanner.getToken();
            var startPos = (lastTokenInfo && lastTokenInfo.token.pos) || scanner.getStartPos();
            return startPos < range.end && current !== SyntaxKind.EndOfFileToken && !isTrivia(current);
        }
    }
}
{'fraction_non_alphanumeric': 0.0865906409752261, 'fraction_numerical': 0.00047188360204482896, 'mean_word_length': 1.3902255639097745, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7579869', 'n_tokens_mistral': 3309, 'n_tokens_neox': 3046, 'n_words': 584}