<commit_before>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import * as vscode from 'vscode';
import { ITypeScriptServiceClient, ExecConfig, ServerResponse } from '../typescriptService';
import * as Proto from '../protocol';
import { VersionDependentRegistration } from '../utils/dependentRegistration';
import API from '../utils/api';

// all constants are const
import { TokenType, TokenModifier, TokenEncodingConsts, VersionRequirement } from 'typescript-vscode-sh-plugin/lib/constants';

const minTypeScriptVersion = API.fromVersionString(`${VersionRequirement.major}.${VersionRequirement.minor}`);

export function register(selector: vscode.DocumentSelector, client: ITypeScriptServiceClient) {
	return new VersionDependentRegistration(client, minTypeScriptVersion, () => {
		const provider = new DocumentSemanticTokensProvider(client);
		return vscode.Disposable.from(
			vscode.languages.registerDocumentSemanticTokensProvider(selector, provider, provider.getLegend()),
			vscode.languages.registerDocumentRangeSemanticTokensProvider(selector, provider, provider.getLegend()),
		);
	});
}

/**
 * Prototype of a DocumentSemanticTokensProvider, relying on the experimental `encodedSemanticClassifications-full` request from the TypeScript server.
 * As the results retured by the TypeScript server are limited, we also add a Typescript plugin (typescript-vscode-sh-plugin) to enrich the returned token.
 * See https://github.com/aeschli/typescript-vscode-sh-plugin.
 */
class DocumentSemanticTokensProvider implements vscode.DocumentSemanticTokensProvider, vscode.DocumentRangeSemanticTokensProvider {

	constructor(private readonly client: ITypeScriptServiceClient) {
	}

	getLegend(): vscode.SemanticTokensLegend {
		return new vscode.SemanticTokensLegend(tokenTypes, tokenModifiers);
	}

	async provideDocumentSemanticTokens(document: vscode.TextDocument, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
		const file = this.client.toOpenedFilePath(document);
		if (!file) {
			return null;
		}
		return this._provideSemanticTokens(document, { file, start: 0, length: document.getText().length }, token);
	}

	async provideDocumentRangeSemanticTokens(document: vscode.TextDocument, range: vscode.Range, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
		const file = this.client.toOpenedFilePath(document);
		if (!file) {
			return null;
		}
		const start = document.offsetAt(range.start);
		const length = document.offsetAt(range.end) - start;
		return this._provideSemanticTokens(document, { file, start, length }, token);
	}

	async _provideSemanticTokens(document: vscode.TextDocument, requestArg: ExperimentalProtocol.EncodedSemanticClassificationsRequestArgs, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
		const file = this.client.toOpenedFilePath(document);
		if (!file) {
			return null;
		}

		let versionBeforeRequest = document.version;

		const response = await (this.client as ExperimentalProtocol.IExtendedTypeScriptServiceClient).execute('encodedSemanticClassifications-full', requestArg, token);
		if (response.type !== 'response' || !response.body) {
			return null;
		}

		const versionAfterRequest = document.version;

		if (versionBeforeRequest !== versionAfterRequest) {
			// cannot convert result's offsets to (line;col) values correctly
			// a new request will come in soon...<commit_msg>Disable semantic highlighting in large JS/TS files. Fixes #94321<commit_after>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import * as vscode from 'vscode';
import { ITypeScriptServiceClient, ExecConfig, ServerResponse } from '../typescriptService';
import * as Proto from '../protocol';
import { VersionDependentRegistration } from '../utils/dependentRegistration';
import API from '../utils/api';

// all constants are const
import { TokenType, TokenModifier, TokenEncodingConsts, VersionRequirement } from 'typescript-vscode-sh-plugin/lib/constants';

const minTypeScriptVersion = API.fromVersionString(`${VersionRequirement.major}.${VersionRequirement.minor}`);

export function register(selector: vscode.DocumentSelector, client: ITypeScriptServiceClient) {
	return new VersionDependentRegistration(client, minTypeScriptVersion, () => {
		const provider = new DocumentSemanticTokensProvider(client);
		return vscode.Disposable.from(
			vscode.languages.registerDocumentSemanticTokensProvider(selector, provider, provider.getLegend()),
			vscode.languages.registerDocumentRangeSemanticTokensProvider(selector, provider, provider.getLegend()),
		);
	});
}


// as we don't do deltas, for performance reasons, don't compute semantic tokens for documents above that limit
const CONTENT_LENGTH_LIMIT = 100000;

/**
 * Prototype of a DocumentSemanticTokensProvider, relying on the experimental `encodedSemanticClassifications-full` request from the TypeScript server.
 * As the results retured by the TypeScript server are limited, we also add a Typescript plugin (typescript-vscode-sh-plugin) to enrich the returned token.
 * See https://github.com/aeschli/typescript-vscode-sh-plugin.
 */
class DocumentSemanticTokensProvider implements vscode.DocumentSemanticTokensProvider, vscode.DocumentRangeSemanticTokensProvider {

	constructor(private readonly client: ITypeScriptServiceClient) {
	}

	getLegend(): vscode.SemanticTokensLegend {
		return new vscode.SemanticTokensLegend(tokenTypes, tokenModifiers);
	}

	async provideDocumentSemanticTokens(document: vscode.TextDocument, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
		const file = this.client.toOpenedFilePath(document);
		if (!file || document.getText().length > CONTENT_LENGTH_LIMIT) {
			return null;
		}
		return this._provideSemanticTokens(document, { file, start: 0, length: document.getText().length }, token);
	}

	async provideDocumentRangeSemanticTokens(document: vscode.TextDocument, range: vscode.Range, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
		const file = this.client.toOpenedFilePath(document);
		if (!file || document.getText().length > CONTENT_LENGTH_LIMIT) {
			return null;
		}
		const start = document.offsetAt(range.start);
		const length = document.offsetAt(range.end) - start;
		return this._provideSemanticTokens(document, { file, start, length }, token);
	}

	async _provideSemanticTokens(document: vscode.TextDocument, requestArg: ExperimentalProtocol.EncodedSemanticClassificationsRequestArgs, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
		const file = this.client.toOpenedFilePath(document);
		if (!file) {
			return null;
		}

		let versionBeforeRequest = document.version;

		const response = await (this.client as ExperimentalProtocol.IExtendedTypeScriptServiceClient).execute('encodedSemanticClassifications-full', requestArg, token);
		if (response.type !== 'response' || !response.body) {
			return null;
		}

		const versionAfterRequest = document.version;

		if (versionBeforeRequest !== versionAfterRequest) {
			// cannot convert result's offsets to (line;col) values correctly
			// a new request will come in soon...
{'fraction_non_alphanumeric': 0.14354128915350764, 'fraction_numerical': 0.0016826300802485115, 'mean_word_length': 4.3548163548163545, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1793921', 'n_tokens_mistral': 2130, 'n_tokens_neox': 2044, 'n_words': 513}