<commit_before>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import { MainContext, MainThreadLanguagesShape, IMainContext, ExtHostLanguagesShape } from './extHost.protocol';
import type * as vscode from 'vscode';
import { ExtHostDocuments } from 'vs/workbench/api/common/extHostDocuments';
import * as typeConvert from 'vs/workbench/api/common/extHostTypeConverters';
import { StandardTokenType, Range, Position, LanguageStatusSeverity } from 'vs/workbench/api/common/extHostTypes';
import Severity from 'vs/base/common/severity';
import { disposableTimeout } from 'vs/base/common/async';
import { DisposableStore, IDisposable } from 'vs/base/common/lifecycle';
import { IExtensionDescription } from 'vs/platform/extensions/common/extensions';
import { CommandsConverter } from 'vs/workbench/api/common/extHostCommands';
import { IURITransformer } from 'vs/base/common/uriIpc';

export class ExtHostLanguages implements ExtHostLanguagesShape {

	private readonly _proxy: MainThreadLanguagesShape;

	private _languageIds: string[] = [];

	constructor(
		mainContext: IMainContext,
		private readonly _documents: ExtHostDocuments,
		private readonly _commands: CommandsConverter,
		private readonly _uriTransformer: IURITransformer | undefined
	) {
		this._proxy = mainContext.getProxy(MainContext.MainThreadLanguages);
	}

	$acceptLanguageIds(ids: string[]): void {
		this._languageIds = ids;
	}

	async getLanguages(): Promise<string[]> {
		return this._languageIds.slice(0);
	}

	async changeLanguage(uri: vscode.Uri, languageId: string): Promise<vscode.TextDocument> {
		await this._proxy.$changeLanguage(uri, languageId);
		const data = this._documents.getDocumentData(uri);
		if (!data) {
			throw new Error(`document '${uri.toString}' NOT found`);
		}
		return data.document;
	}

	async tokenAtPosition(document: vscode.TextDocument, position: vscode.Position): Promise<vscode.TokenInformation> {
		const versionNow = document.version;
		const pos = typeConvert.Position.from(position);<commit_msg>fix error message for https://github.com/microsoft/vscode/issues/143031<commit_after>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import { MainContext, MainThreadLanguagesShape, IMainContext, ExtHostLanguagesShape } from './extHost.protocol';
import type * as vscode from 'vscode';
import { ExtHostDocuments } from 'vs/workbench/api/common/extHostDocuments';
import * as typeConvert from 'vs/workbench/api/common/extHostTypeConverters';
import { StandardTokenType, Range, Position, LanguageStatusSeverity } from 'vs/workbench/api/common/extHostTypes';
import Severity from 'vs/base/common/severity';
import { disposableTimeout } from 'vs/base/common/async';
import { DisposableStore, IDisposable } from 'vs/base/common/lifecycle';
import { IExtensionDescription } from 'vs/platform/extensions/common/extensions';
import { CommandsConverter } from 'vs/workbench/api/common/extHostCommands';
import { IURITransformer } from 'vs/base/common/uriIpc';

export class ExtHostLanguages implements ExtHostLanguagesShape {

	private readonly _proxy: MainThreadLanguagesShape;

	private _languageIds: string[] = [];

	constructor(
		mainContext: IMainContext,
		private readonly _documents: ExtHostDocuments,
		private readonly _commands: CommandsConverter,
		private readonly _uriTransformer: IURITransformer | undefined
	) {
		this._proxy = mainContext.getProxy(MainContext.MainThreadLanguages);
	}

	$acceptLanguageIds(ids: string[]): void {
		this._languageIds = ids;
	}

	async getLanguages(): Promise<string[]> {
		return this._languageIds.slice(0);
	}

	async changeLanguage(uri: vscode.Uri, languageId: string): Promise<vscode.TextDocument> {
		await this._proxy.$changeLanguage(uri, languageId);
		const data = this._documents.getDocumentData(uri);
		if (!data) {
			throw new Error(`document '${uri.toString()}' NOT found`);
		}
		return data.document;
	}

	async tokenAtPosition(document: vscode.TextDocument, position: vscode.Position): Promise<vscode.TokenInformation> {
		const versionNow = document.version;
		const pos = typeConvert.Position.from(position);
{'fraction_non_alphanumeric': 0.17905982905982906, 'fraction_numerical': 0.0017094017094017094, 'mean_word_length': 3.6716566866267466, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1554519', 'n_tokens_mistral': 1314, 'n_tokens_neox': 1249, 'n_words': 303}