<commit_before>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import { URI, UriComponents } from 'vs/base/common/uri';
import { mixin } from 'vs/base/common/objects';
import * as vscode from 'vscode';
import * as typeConvert from 'vs/workbench/api/common/extHostTypeConverters';
import { Range, Disposable, CompletionList, SnippetString, CodeActionKind, SymbolInformation, DocumentSymbol, SemanticColoringArea, SemanticColoring } from 'vs/workbench/api/common/extHostTypes';
import { ISingleEditOperation } from 'vs/editor/common/model';
import * as modes from 'vs/editor/common/modes';
import { ExtHostDocuments } from 'vs/workbench/api/common/extHostDocuments';
import { ExtHostCommands, CommandsConverter } from 'vs/workbench/api/common/extHostCommands';
import { ExtHostDiagnostics } from 'vs/workbench/api/common/extHostDiagnostics';
import { asPromise } from 'vs/base/common/async';
import * as extHostProtocol from './extHost.protocol';
import { regExpLeadsToEndlessLoop, regExpFlags } from 'vs/base/common/strings';
import { IPosition } from 'vs/editor/common/core/position';
import { IRange, Range as EditorRange } from 'vs/editor/common/core/range';
import { isFalsyOrEmpty, isNonEmptyArray, coalesce, asArray } from 'vs/base/common/arrays';
import { isObject } from 'vs/base/common/types';
import { ISelection, Selection } from 'vs/editor/common/core/selection';
import { ILogService } from 'vs/platform/log/common/log';
import { CancellationToken } from 'vs/base/common/cancellation';
import { ExtensionIdentifier, IExtensionDescription } from 'vs/platform/extensions/common/extensions';
import { IURITransformer } from 'vs/base/common/uriIpc';
import { DisposableStore, dispose } from 'vs/base/common/lifecycle';
import { VSBuffer } from 'vs/base/common/buffer';
import { encodeSemanticTokensDto, ISemanticTokensDto } from 'vs/workbench/api/common/shared/semanticTokens';

// --- adapter

class DocumentSymbolAdapter {

	private _documents: ExtHostDocuments;
	private _provider: vscode.DocumentSymbolProvider;

	constructor(documents: ExtHostDocuments, provider: vscode.DocumentSymbolProvider) {
		this._documents = documents;
		this._provider = provider;
	}

	provideDocumentSymbols(resource: URI, token: CancellationToken): Promise<modes.DocumentSymbol[] | undefined> {
		const doc = this._documents.getDocument(resource);
		return asPromise(() => this._provider.provideDocumentSymbols(doc, token)).then(value => {
			if (isFalsyOrEmpty(value)) {
				return undefined;
			} else if (value![0] instanceof DocumentSymbol) {
				return (<DocumentSymbol[]>value).map(typeConvert.DocumentSymbol.from);
			} else {
				return DocumentSymbolAdapter._asDocumentSymbolTree(<SymbolInformation[]>value);
			}
		});
	}

	private static _asDocumentSymbolTree(infos: SymbolInformation[]): modes.DocumentSymbol[] {
		// first sort by start (and end) and then loop over all elements
		// and build a tree based on containment.
		infos = infos.slice(0).sort((a, b) => {
			let res = a.location.range.start.compareTo(b.location.range.start);
			if (res === 0) {
				res = b.location.range.end.compareTo(a.location.range.end);
			}
			return res;
		});
		const res: modes.DocumentSymbol[] = [];
		const parentStack: modes.DocumentSymbol[] = [];
		for (const info of infos) {
			const element: modes.DocumentSymbol = {
				name: info.name || '!!MISSING: name!!',
				kind: typeConvert.SymbolKind.from(info.kind),
				tags: info.tags ? info.tags.map(typeConvert.SymbolTag.from) : [],
				detail: '',
				containerName: info.containerName,
				range: typeConvert.Range.from(info.location.range),
				selectionRange: typeConvert.Range.from(info.location.range),
				children: []
			};

			while (true) {
				if (parentStack.length === 0) {
					parentStack.push(element);
					res.push(element);
					break;
				}
				const parent = parentStack[parentStack.length - 1];
				if (EditorRange.containsRange(parent.range, element.range) && !EditorRange.equalsRange(parent.range, element.range)) {
					if (parent.children) {
						parent.children.push(element);
					}
					parentStack.push(element);
					break;
				}
				parentStack.pop();
			}
		}
		return res;
	}
}

class CodeLensAdapter {

	private static _badCmd: vscode.Command = { command: 'missing', title: '!!MISSING: command!!' };

	private readonly _cache = new Cache<vscode.CodeLens>('CodeLens');
	private readonly _disposables = new Map<number, DisposableStore>();

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _commands: CommandsConverter,
		private readonly _provider: vscode.CodeLensProvider
	) { }

	provideCodeLenses(resource: URI, token: CancellationToken): Promise<extHostProtocol.ICodeLensListDto | undefined> {
		const doc = this._documents.getDocument(resource);

		return asPromise(() => this._provider.provideCodeLenses(doc, token)).then(lenses => {

			if (!lenses || token.isCancellationRequested) {
				return undefined;
			}

			const cacheId = this._cache.add(lenses);
			const disposables = new DisposableStore();
			this._disposables.set(cacheId, disposables);

			const result: extHostProtocol.ICodeLensListDto = {
				cacheId,
				lenses: [],
			};

			for (let i = 0; i < lenses.length; i++) {
				result.lenses.push({
					cacheId: [cacheId, i],
					range: typeConvert.Range.from(lenses[i].range),
					command: this._commands.toInternal(lenses[i].command, disposables)
				});
			}

			return result;
		});
	}

	resolveCodeLens(symbol: extHostProtocol.ICodeLensDto, token: CancellationToken): Promise<extHostProtocol.ICodeLensDto | undefined> {

		const lens = symbol.cacheId && this._cache.get(...symbol.cacheId);
		if (!lens) {
			return Promise.resolve(undefined);
		}

		let resolve: Promise<vscode.CodeLens | undefined | null>;
		if (typeof this._provider.resolveCodeLens !== 'function' || lens.isResolved) {
			resolve = Promise.resolve(lens);
		} else {
			resolve = asPromise(() => this._provider.resolveCodeLens!(lens, token));
		}

		return resolve.then(newLens => {
			if (token.isCancellationRequested) {
				return undefined;
			}

			const disposables = symbol.cacheId && this._disposables.get(symbol.cacheId[0]);
			if (!disposables) {
				// We've already been disposed of
				return undefined;
			}

			newLens = newLens || lens;
			symbol.command = this._commands.toInternal(newLens.command || CodeLensAdapter._badCmd, disposables);
			return symbol;
		});
	}

	releaseCodeLenses(cachedId: number): void {
		dispose(this._disposables.get(cachedId));
		this._disposables.delete(cachedId);
		this._cache.delete(cachedId);
	}
}

function convertToLocationLinks(value: vscode.Definition): modes.LocationLink[] {
	return value ? asArray(value).map(typeConvert.DefinitionLink.from) : [];
}

class DefinitionAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.DefinitionProvider
	) { }

	provideDefinition(resource: URI, position: IPosition, token: CancellationToken): Promise<modes.LocationLink[]> {
		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);
		return asPromise(() => this._provider.provideDefinition(doc, pos, token)).then(convertToLocationLinks);
	}
}

class DeclarationAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.DeclarationProvider
	) { }

	provideDeclaration(resource: URI, position: IPosition, token: CancellationToken): Promise<modes.LocationLink[]> {
		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);
		return asPromise(() => this._provider.provideDeclaration(doc, pos, token)).then(convertToLocationLinks);
	}
}

class ImplementationAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.ImplementationProvider
	) { }

	provideImplementation(resource: URI, position: IPosition, token: CancellationToken): Promise<modes.LocationLink[]> {
		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);
		return asPromise(() => this._provider.provideImplementation(doc, pos, token)).then(convertToLocationLinks);
	}
}

class TypeDefinitionAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.TypeDefinitionProvider
	) { }

	provideTypeDefinition(resource: URI, position: IPosition, token: CancellationToken): Promise<modes.LocationLink[]> {
		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);
		return asPromise(() => this._provider.provideTypeDefinition(doc, pos, token)).then(convertToLocationLinks);
	}
}

class HoverAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.HoverProvider,
	) { }

	public provideHover(resource: URI, position: IPosition, token: CancellationToken): Promise<modes.Hover | undefined> {

		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);

		return asPromise(() => this._provider.provideHover(doc, pos, token)).then(value => {
			if (!value || isFalsyOrEmpty(value.contents)) {
				return undefined;
			}
			if (!value.range) {
				value.range = doc.getWordRangeAtPosition(pos);
			}
			if (!value.range) {
				value.range = new Range(pos, pos);
			}

			return typeConvert.Hover.from(value);
		});
	}
}

class DocumentHighlightAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.DocumentHighlightProvider
	) { }

	provideDocumentHighlights(resource: URI, position: IPosition, token: CancellationToken): Promise<modes.DocumentHighlight[] | undefined> {

		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);

		return asPromise(() => this._provider.provideDocumentHighlights(doc, pos, token)).then(value => {
			if (Array.isArray(value)) {
				return value.map(typeConvert.DocumentHighlight.from);
			}
			return undefined;
		});
	}
}

class ReferenceAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.ReferenceProvider
	) { }

	provideReferences(resource: URI, position: IPosition, context: modes.ReferenceContext, token: CancellationToken): Promise<modes.Location[] | undefined> {
		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);

		return asPromise(() => this._provider.provideReferences(doc, pos, context, token)).then(value => {
			if (Array.isArray(value)) {
				return value.map(typeConvert.location.from);
			}
			return undefined;
		});
	}
}

export interface CustomCodeAction extends extHostProtocol.ICodeActionDto {
	_isSynthetic?: boolean;
}

class CodeActionAdapter {
	private static readonly _maxCodeActionsPerFile: number = 1000;

	private readonly _cache = new Cache<vscode.CodeAction | vscode.Command>('CodeAction');
	private readonly _disposables = new Map<number, DisposableStore>();

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _commands: CommandsConverter,
		private readonly _diagnostics: ExtHostDiagnostics,
		private readonly _provider: vscode.CodeActionProvider,
		private readonly _logService: ILogService,
		private readonly _extensionId: ExtensionIdentifier
	) { }

	provideCodeActions(resource: URI, rangeOrSelection: IRange | ISelection, context: modes.CodeActionContext, token: CancellationToken): Promise<extHostProtocol.ICodeActionListDto | undefined> {

		const doc = this._documents.getDocument(resource);
		const ran = Selection.isISelection(rangeOrSelection)
			? <vscode.Selection>typeConvert.Selection.to(rangeOrSelection)
			: <vscode.Range>typeConvert.Range.to(rangeOrSelection);
		const allDiagnostics: vscode.Diagnostic[] = [];

		for (const diagnostic of this._diagnostics.getDiagnostics(resource)) {
			if (ran.intersection(diagnostic.range)) {
				const newLen = allDiagnostics.push(diagnostic);
				if (newLen > CodeActionAdapter._maxCodeActionsPerFile) {
					break;
				}
			}
		}

		const codeActionContext: vscode.CodeActionContext = {
			diagnostics: allDiagnostics,
			only: context.only ? new CodeActionKind(context.only) : undefined
		};

		return asPromise(() => this._provider.provideCodeActions(doc, ran, codeActionContext, token)).then((commandsOrActions): extHostProtocol.ICodeActionListDto | undefined => {
			if (!isNonEmptyArray(commandsOrActions) || token.isCancellationRequested) {
				return undefined;
			}

			const cacheId = this._cache.add(commandsOrActions);
			const disposables = new DisposableStore();
			this._disposables.set(cacheId, disposables);

			const actions: CustomCodeAction[] = [];
			for (const candidate of commandsOrActions) {
				if (!candidate) {
					continue;
				}
				if (CodeActionAdapter._isCommand(candidate)) {
					// old school: synthetic code action
					actions.push({
						_isSynthetic: true,
						title: candidate.title,
						command: this._commands.toInternal(candidate, disposables),
					});
				} else {
					if (codeActionContext.only) {
						if (!candidate.kind) {
							this._logService.warn(`${this._extensionId.value} - Code actions of kind '${codeActionContext.only.value} 'requested but returned code action does not have a 'kind'. Code action will be dropped. Please set 'CodeAction.kind'.`);
						} else if (!codeActionContext.only.contains(candidate.kind)) {
							this._logService.warn(`${this._extensionId.value} - Code actions of kind '${codeActionContext.only.value} 'requested but returned code action is of kind '${candidate.kind.value}'. Code action will be dropped. Please check 'CodeActionContext.only' to only return requested code actions.`);
						}
					}

					// new school: convert code action
					actions.push({
						title: candidate.title,
						command: candidate.command && this._commands.toInternal(candidate.command, disposables),
						diagnostics: candidate.diagnostics && candidate.diagnostics.map(typeConvert.Diagnostic.from),
						edit: candidate.edit && typeConvert.WorkspaceEdit.from(candidate.edit),
						kind: candidate.kind && candidate.kind.value,
						isPreferred: candidate.isPreferred,
					});
				}
			}

			return { cacheId, actions };
		});
	}

	public releaseCodeActions(cachedId: number): void {
		dispose(this._disposables.get(cachedId));
		this._disposables.delete(cachedId);
		this._cache.delete(cachedId);
	}

	private static _isCommand(thing: any): thing is vscode.Command {
		return typeof (<vscode.Command>thing).command === 'string' && typeof (<vscode.Command>thing).title === 'string';
	}
}

class DocumentFormattingAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.DocumentFormattingEditProvider
	) { }

	provideDocumentFormattingEdits(resource: URI, options: modes.FormattingOptions, token: CancellationToken): Promise<ISingleEditOperation[] | undefined> {

		const document = this._documents.getDocument(resource);

		return asPromise(() => this._provider.provideDocumentFormattingEdits(document, <any>options, token)).then(value => {
			if (Array.isArray(value)) {
				return value.map(typeConvert.TextEdit.from);
			}
			return undefined;
		});
	}
}

class RangeFormattingAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.DocumentRangeFormattingEditProvider
	) { }

	provideDocumentRangeFormattingEdits(resource: URI, range: IRange, options: modes.FormattingOptions, token: CancellationToken): Promise<ISingleEditOperation[] | undefined> {

		const document = this._documents.getDocument(resource);
		const ran = typeConvert.Range.to(range);

		return asPromise(() => this._provider.provideDocumentRangeFormattingEdits(document, ran, <any>options, token)).then(value => {
			if (Array.isArray(value)) {
				return value.map(typeConvert.TextEdit.from);
			}
			return undefined;
		});
	}
}

class OnTypeFormattingAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.OnTypeFormattingEditProvider
	) { }

	autoFormatTriggerCharacters: string[] = []; // not here

	provideOnTypeFormattingEdits(resource: URI, position: IPosition, ch: string, options: modes.FormattingOptions, token: CancellationToken): Promise<ISingleEditOperation[] | undefined> {

		const document = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);

		return asPromise(() => this._provider.provideOnTypeFormattingEdits(document, pos, ch, <any>options, token)).then(value => {
			if (Array.isArray(value)) {
				return value.map(typeConvert.TextEdit.from);
			}
			return undefined;
		});
	}
}

class NavigateTypeAdapter {

	private readonly _symbolCache = new Map<number, vscode.SymbolInformation>();
	private readonly _resultCache = new Map<number, [number, number]>();
	private readonly _provider: vscode.WorkspaceSymbolProvider;

	constructor(provider: vscode.WorkspaceSymbolProvider) {
		this._provider = provider;
	}

	provideWorkspaceSymbols(search: string, token: CancellationToken): Promise<extHostProtocol.IWorkspaceSymbolsDto> {
		const result: extHostProtocol.IWorkspaceSymbolsDto = extHostProtocol.IdObject.mixin({ symbols: [] });
		return asPromise(() => this._provider.provideWorkspaceSymbols(search, token)).then(value => {
			if (isNonEmptyArray(value)) {
				for (const item of value) {
					if (!item) {
						// drop
						continue;
					}
					if (!item.name) {
						console.warn('INVALID SymbolInformation, lacks name', item);
						continue;
					}
					const symbol = extHostProtocol.IdObject.mixin(typeConvert.WorkspaceSymbol.from(item));
					this._symbolCache.set(symbol._id!, item);
					result.symbols.push(symbol);
				}
			}
		}).then(() => {
			if (result.symbols.length > 0) {
				this._resultCache.set(result._id!, [result.symbols[0]._id!, result.symbols[result.symbols.length - 1]._id!]);
			}
			return result;
		});
	}

	async resolveWorkspaceSymbol(symbol: extHostProtocol.IWorkspaceSymbolDto, token: CancellationToken): Promise<extHostProtocol.IWorkspaceSymbolDto | undefined> {
		if (typeof this._provider.resolveWorkspaceSymbol !== 'function') {
			return symbol;
		}

		const item = this._symbolCache.get(symbol._id!);
		if (item) {
			const value = await asPromise(() => this._provider.resolveWorkspaceSymbol!(item, token));
			return value && mixin(symbol, typeConvert.WorkspaceSymbol.from(value), true);
		}
		return undefined;
	}

	releaseWorkspaceSymbols(id: number): any {
		const range = this._resultCache.get(id);
		if (range) {
			for (let [from, to] = range; from <= to; from++) {
				this._symbolCache.delete(from);
			}
			this._resultCache.delete(id);
		}
	}
}

class RenameAdapter {

	static supportsResolving(provider: vscode.RenameProvider): boolean {
		return typeof provider.prepareRename === 'function';
	}

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.RenameProvider
	) { }

	provideRenameEdits(resource: URI, position: IPosition, newName: string, token: CancellationToken): Promise<extHostProtocol.IWorkspaceEditDto | undefined> {

		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);

		return asPromise(() => this._provider.provideRenameEdits(doc, pos, newName, token)).then(value => {
			if (!value) {
				return undefined;
			}
			return typeConvert.WorkspaceEdit.from(value);
		}, err => {
			const rejectReason = RenameAdapter._asMessage(err);
			if (rejectReason) {
				return <extHostProtocol.IWorkspaceEditDto>{ rejectReason, edits: undefined! };
			} else {
				// generic error
				return Promise.reject<extHostProtocol.IWorkspaceEditDto>(err);
			}
		});
	}

	resolveRenameLocation(resource: URI, position: IPosition, token: CancellationToken): Promise<(modes.RenameLocation & modes.Rejection) | undefined> {
		if (typeof this._provider.prepareRename !== 'function') {
			return Promise.resolve(undefined);
		}

		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);

		return asPromise(() => this._provider.prepareRename!(doc, pos, token)).then(rangeOrLocation => {

			let range: vscode.Range | undefined;
			let text: string | undefined;
			if (Range.isRange(rangeOrLocation)) {
				range = rangeOrLocation;
				text = doc.getText(rangeOrLocation);

			} else if (isObject(rangeOrLocation)) {
				range = rangeOrLocation.range;
				text = rangeOrLocation.placeholder;
			}

			if (!range) {
				return undefined;
			}
			if (range.start.line > pos.line || range.end.line < pos.line) {
				console.warn('INVALID rename location: position line must be within range start/end lines');
				return undefined;
			}
			return { range: typeConvert.Range.from(range), text };
		}, err => {
			const rejectReason = RenameAdapter._asMessage(err);
			if (rejectReason) {
				return <modes.RenameLocation & modes.Rejection>{ rejectReason, range: undefined!, text: undefined! };
			} else {
				return Promise.reject<any>(err);
			}
		});
	}

	private static _asMessage(err: any): string | undefined {
		if (typeof err === 'string') {
			return err;
		} else if (err instanceof Error && typeof err.message === 'string') {
			return err.message;
		} else {
			return undefined;
		}
	}
}

const enum SemanticColoringConstants {
	/**
	 * Let's aim at having 8KB buffers if possible...
	 * So that would be 8192 / (5 * 4) = 409.6 tokens per area
	 */
	DesiredTokensPerArea = 400,

	/**
	 * If the semantic coloring result contains a single area and the area contains
	 * more than 600 tokens, then split the area into mulitple areas.
	 */
	SplitSingleAreaTokenCountThreshold = 600,

	/**
	 * Try to keep the total number of areas under 1024 if possible,
	 * simply compensate by having more tokens per area...
	 */
	DesiredMaxAreas = 1024
}

class SemanticColoringAdapter {

	private readonly _previousResults = new Map<number, vscode.SemanticColoring>();
	private _nextResultId = 1;

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.SemanticColoringProvider,
	) { }

	provideSemanticColoring(resource: URI, previousSemanticColoringResultId: number, token: CancellationToken): Promise<VSBuffer | null> {
		const doc = this._documents.getDocument(resource);

		return asPromise(() => this._provider.provideSemanticColoring(doc, token)).then(value => {
			if (!value) {
				return null;
			}

			const previousResult = (previousSemanticColoringResultId !== 0 ? this._previousResults.get(previousSemanticColoringResultId) : null);
			if (previousResult) {
				this._previousResults.delete(previousSemanticColoringResultId);
				return this._deltaEncode(previousResult, value);
			}

			return this._encode(value);
		});
	}

	async releaseSemanticColoring(semanticColoringResultId: number): Promise<void> {
		this._previousResults.delete(semanticColoringResultId);
	}

	private _deltaEncode(previousResult: vscode.SemanticColoring, currentResult: vscode.SemanticColoring): VSBuffer {
		console.log(previousResult);
		console.log(currentResult);
		throw new Error(`TODO - Not implemented`);
	}

	private _encode(result: vscode.SemanticColoring): VSBuffer {
		if (result.areas.length === 1) {
			const tokenCount = (result.areas[0].data.length / 5) | 0;
			if (tokenCount > SemanticColoringConstants.SplitSingleAreaTokenCountThreshold) {
				// This result contains a single area which contains over `SplitSingleAreaTokenCountThreshold` tokens
				// so let's split tokens into multiple areas...
				result = this._splitIntoMultipleAreas(result.areas[0]);
			}
		}

		const myId = this._nextResultId++;
		this._previousResults.set(myId, result);
		const dto: ISemanticTokensDto = {
			id: myId,
			areas: result.areas.map(area => {
				return {
					type: 'full',
					line: area.line,
					data: area.data
				};
			})
		};
		return encodeSemanticTokensDto(dto);
	}

	private _splitIntoMultipleAreas(area: vscode.SemanticColoringArea): SemanticColoring {
		const srcAreaLine = area.line;
		const srcAreaData = area.data;
		const tokenCount = (srcAreaData.length / 5) | 0;
		const tokensPerArea = Math.max(Math.ceil(tokenCount / SemanticColoringConstants.DesiredMaxAreas), SemanticColoringConstants.DesiredTokensPerArea);
<commit_msg>Implement identity based diffing for smart semantic tokens providers<commit_after>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import { URI, UriComponents } from 'vs/base/common/uri';
import { mixin } from 'vs/base/common/objects';
import * as vscode from 'vscode';
import * as typeConvert from 'vs/workbench/api/common/extHostTypeConverters';
import { Range, Disposable, CompletionList, SnippetString, CodeActionKind, SymbolInformation, DocumentSymbol, SemanticColoringArea, SemanticColoring } from 'vs/workbench/api/common/extHostTypes';
import { ISingleEditOperation } from 'vs/editor/common/model';
import * as modes from 'vs/editor/common/modes';
import { ExtHostDocuments } from 'vs/workbench/api/common/extHostDocuments';
import { ExtHostCommands, CommandsConverter } from 'vs/workbench/api/common/extHostCommands';
import { ExtHostDiagnostics } from 'vs/workbench/api/common/extHostDiagnostics';
import { asPromise } from 'vs/base/common/async';
import * as extHostProtocol from './extHost.protocol';
import { regExpLeadsToEndlessLoop, regExpFlags } from 'vs/base/common/strings';
import { IPosition } from 'vs/editor/common/core/position';
import { IRange, Range as EditorRange } from 'vs/editor/common/core/range';
import { isFalsyOrEmpty, isNonEmptyArray, coalesce, asArray } from 'vs/base/common/arrays';
import { isObject } from 'vs/base/common/types';
import { ISelection, Selection } from 'vs/editor/common/core/selection';
import { ILogService } from 'vs/platform/log/common/log';
import { CancellationToken } from 'vs/base/common/cancellation';
import { ExtensionIdentifier, IExtensionDescription } from 'vs/platform/extensions/common/extensions';
import { IURITransformer } from 'vs/base/common/uriIpc';
import { DisposableStore, dispose } from 'vs/base/common/lifecycle';
import { VSBuffer } from 'vs/base/common/buffer';
import { encodeSemanticTokensDto, ISemanticTokensDto, ISemanticTokensFullAreaDto, ISemanticTokensDeltaAreaDto } from 'vs/workbench/api/common/shared/semanticTokens';

// --- adapter

class DocumentSymbolAdapter {

	private _documents: ExtHostDocuments;
	private _provider: vscode.DocumentSymbolProvider;

	constructor(documents: ExtHostDocuments, provider: vscode.DocumentSymbolProvider) {
		this._documents = documents;
		this._provider = provider;
	}

	provideDocumentSymbols(resource: URI, token: CancellationToken): Promise<modes.DocumentSymbol[] | undefined> {
		const doc = this._documents.getDocument(resource);
		return asPromise(() => this._provider.provideDocumentSymbols(doc, token)).then(value => {
			if (isFalsyOrEmpty(value)) {
				return undefined;
			} else if (value![0] instanceof DocumentSymbol) {
				return (<DocumentSymbol[]>value).map(typeConvert.DocumentSymbol.from);
			} else {
				return DocumentSymbolAdapter._asDocumentSymbolTree(<SymbolInformation[]>value);
			}
		});
	}

	private static _asDocumentSymbolTree(infos: SymbolInformation[]): modes.DocumentSymbol[] {
		// first sort by start (and end) and then loop over all elements
		// and build a tree based on containment.
		infos = infos.slice(0).sort((a, b) => {
			let res = a.location.range.start.compareTo(b.location.range.start);
			if (res === 0) {
				res = b.location.range.end.compareTo(a.location.range.end);
			}
			return res;
		});
		const res: modes.DocumentSymbol[] = [];
		const parentStack: modes.DocumentSymbol[] = [];
		for (const info of infos) {
			const element: modes.DocumentSymbol = {
				name: info.name || '!!MISSING: name!!',
				kind: typeConvert.SymbolKind.from(info.kind),
				tags: info.tags ? info.tags.map(typeConvert.SymbolTag.from) : [],
				detail: '',
				containerName: info.containerName,
				range: typeConvert.Range.from(info.location.range),
				selectionRange: typeConvert.Range.from(info.location.range),
				children: []
			};

			while (true) {
				if (parentStack.length === 0) {
					parentStack.push(element);
					res.push(element);
					break;
				}
				const parent = parentStack[parentStack.length - 1];
				if (EditorRange.containsRange(parent.range, element.range) && !EditorRange.equalsRange(parent.range, element.range)) {
					if (parent.children) {
						parent.children.push(element);
					}
					parentStack.push(element);
					break;
				}
				parentStack.pop();
			}
		}
		return res;
	}
}

class CodeLensAdapter {

	private static _badCmd: vscode.Command = { command: 'missing', title: '!!MISSING: command!!' };

	private readonly _cache = new Cache<vscode.CodeLens>('CodeLens');
	private readonly _disposables = new Map<number, DisposableStore>();

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _commands: CommandsConverter,
		private readonly _provider: vscode.CodeLensProvider
	) { }

	provideCodeLenses(resource: URI, token: CancellationToken): Promise<extHostProtocol.ICodeLensListDto | undefined> {
		const doc = this._documents.getDocument(resource);

		return asPromise(() => this._provider.provideCodeLenses(doc, token)).then(lenses => {

			if (!lenses || token.isCancellationRequested) {
				return undefined;
			}

			const cacheId = this._cache.add(lenses);
			const disposables = new DisposableStore();
			this._disposables.set(cacheId, disposables);

			const result: extHostProtocol.ICodeLensListDto = {
				cacheId,
				lenses: [],
			};

			for (let i = 0; i < lenses.length; i++) {
				result.lenses.push({
					cacheId: [cacheId, i],
					range: typeConvert.Range.from(lenses[i].range),
					command: this._commands.toInternal(lenses[i].command, disposables)
				});
			}

			return result;
		});
	}

	resolveCodeLens(symbol: extHostProtocol.ICodeLensDto, token: CancellationToken): Promise<extHostProtocol.ICodeLensDto | undefined> {

		const lens = symbol.cacheId && this._cache.get(...symbol.cacheId);
		if (!lens) {
			return Promise.resolve(undefined);
		}

		let resolve: Promise<vscode.CodeLens | undefined | null>;
		if (typeof this._provider.resolveCodeLens !== 'function' || lens.isResolved) {
			resolve = Promise.resolve(lens);
		} else {
			resolve = asPromise(() => this._provider.resolveCodeLens!(lens, token));
		}

		return resolve.then(newLens => {
			if (token.isCancellationRequested) {
				return undefined;
			}

			const disposables = symbol.cacheId && this._disposables.get(symbol.cacheId[0]);
			if (!disposables) {
				// We've already been disposed of
				return undefined;
			}

			newLens = newLens || lens;
			symbol.command = this._commands.toInternal(newLens.command || CodeLensAdapter._badCmd, disposables);
			return symbol;
		});
	}

	releaseCodeLenses(cachedId: number): void {
		dispose(this._disposables.get(cachedId));
		this._disposables.delete(cachedId);
		this._cache.delete(cachedId);
	}
}

function convertToLocationLinks(value: vscode.Definition): modes.LocationLink[] {
	return value ? asArray(value).map(typeConvert.DefinitionLink.from) : [];
}

class DefinitionAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.DefinitionProvider
	) { }

	provideDefinition(resource: URI, position: IPosition, token: CancellationToken): Promise<modes.LocationLink[]> {
		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);
		return asPromise(() => this._provider.provideDefinition(doc, pos, token)).then(convertToLocationLinks);
	}
}

class DeclarationAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.DeclarationProvider
	) { }

	provideDeclaration(resource: URI, position: IPosition, token: CancellationToken): Promise<modes.LocationLink[]> {
		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);
		return asPromise(() => this._provider.provideDeclaration(doc, pos, token)).then(convertToLocationLinks);
	}
}

class ImplementationAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.ImplementationProvider
	) { }

	provideImplementation(resource: URI, position: IPosition, token: CancellationToken): Promise<modes.LocationLink[]> {
		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);
		return asPromise(() => this._provider.provideImplementation(doc, pos, token)).then(convertToLocationLinks);
	}
}

class TypeDefinitionAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.TypeDefinitionProvider
	) { }

	provideTypeDefinition(resource: URI, position: IPosition, token: CancellationToken): Promise<modes.LocationLink[]> {
		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);
		return asPromise(() => this._provider.provideTypeDefinition(doc, pos, token)).then(convertToLocationLinks);
	}
}

class HoverAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.HoverProvider,
	) { }

	public provideHover(resource: URI, position: IPosition, token: CancellationToken): Promise<modes.Hover | undefined> {

		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);

		return asPromise(() => this._provider.provideHover(doc, pos, token)).then(value => {
			if (!value || isFalsyOrEmpty(value.contents)) {
				return undefined;
			}
			if (!value.range) {
				value.range = doc.getWordRangeAtPosition(pos);
			}
			if (!value.range) {
				value.range = new Range(pos, pos);
			}

			return typeConvert.Hover.from(value);
		});
	}
}

class DocumentHighlightAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.DocumentHighlightProvider
	) { }

	provideDocumentHighlights(resource: URI, position: IPosition, token: CancellationToken): Promise<modes.DocumentHighlight[] | undefined> {

		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);

		return asPromise(() => this._provider.provideDocumentHighlights(doc, pos, token)).then(value => {
			if (Array.isArray(value)) {
				return value.map(typeConvert.DocumentHighlight.from);
			}
			return undefined;
		});
	}
}

class ReferenceAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.ReferenceProvider
	) { }

	provideReferences(resource: URI, position: IPosition, context: modes.ReferenceContext, token: CancellationToken): Promise<modes.Location[] | undefined> {
		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);

		return asPromise(() => this._provider.provideReferences(doc, pos, context, token)).then(value => {
			if (Array.isArray(value)) {
				return value.map(typeConvert.location.from);
			}
			return undefined;
		});
	}
}

export interface CustomCodeAction extends extHostProtocol.ICodeActionDto {
	_isSynthetic?: boolean;
}

class CodeActionAdapter {
	private static readonly _maxCodeActionsPerFile: number = 1000;

	private readonly _cache = new Cache<vscode.CodeAction | vscode.Command>('CodeAction');
	private readonly _disposables = new Map<number, DisposableStore>();

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _commands: CommandsConverter,
		private readonly _diagnostics: ExtHostDiagnostics,
		private readonly _provider: vscode.CodeActionProvider,
		private readonly _logService: ILogService,
		private readonly _extensionId: ExtensionIdentifier
	) { }

	provideCodeActions(resource: URI, rangeOrSelection: IRange | ISelection, context: modes.CodeActionContext, token: CancellationToken): Promise<extHostProtocol.ICodeActionListDto | undefined> {

		const doc = this._documents.getDocument(resource);
		const ran = Selection.isISelection(rangeOrSelection)
			? <vscode.Selection>typeConvert.Selection.to(rangeOrSelection)
			: <vscode.Range>typeConvert.Range.to(rangeOrSelection);
		const allDiagnostics: vscode.Diagnostic[] = [];

		for (const diagnostic of this._diagnostics.getDiagnostics(resource)) {
			if (ran.intersection(diagnostic.range)) {
				const newLen = allDiagnostics.push(diagnostic);
				if (newLen > CodeActionAdapter._maxCodeActionsPerFile) {
					break;
				}
			}
		}

		const codeActionContext: vscode.CodeActionContext = {
			diagnostics: allDiagnostics,
			only: context.only ? new CodeActionKind(context.only) : undefined
		};

		return asPromise(() => this._provider.provideCodeActions(doc, ran, codeActionContext, token)).then((commandsOrActions): extHostProtocol.ICodeActionListDto | undefined => {
			if (!isNonEmptyArray(commandsOrActions) || token.isCancellationRequested) {
				return undefined;
			}

			const cacheId = this._cache.add(commandsOrActions);
			const disposables = new DisposableStore();
			this._disposables.set(cacheId, disposables);

			const actions: CustomCodeAction[] = [];
			for (const candidate of commandsOrActions) {
				if (!candidate) {
					continue;
				}
				if (CodeActionAdapter._isCommand(candidate)) {
					// old school: synthetic code action
					actions.push({
						_isSynthetic: true,
						title: candidate.title,
						command: this._commands.toInternal(candidate, disposables),
					});
				} else {
					if (codeActionContext.only) {
						if (!candidate.kind) {
							this._logService.warn(`${this._extensionId.value} - Code actions of kind '${codeActionContext.only.value} 'requested but returned code action does not have a 'kind'. Code action will be dropped. Please set 'CodeAction.kind'.`);
						} else if (!codeActionContext.only.contains(candidate.kind)) {
							this._logService.warn(`${this._extensionId.value} - Code actions of kind '${codeActionContext.only.value} 'requested but returned code action is of kind '${candidate.kind.value}'. Code action will be dropped. Please check 'CodeActionContext.only' to only return requested code actions.`);
						}
					}

					// new school: convert code action
					actions.push({
						title: candidate.title,
						command: candidate.command && this._commands.toInternal(candidate.command, disposables),
						diagnostics: candidate.diagnostics && candidate.diagnostics.map(typeConvert.Diagnostic.from),
						edit: candidate.edit && typeConvert.WorkspaceEdit.from(candidate.edit),
						kind: candidate.kind && candidate.kind.value,
						isPreferred: candidate.isPreferred,
					});
				}
			}

			return { cacheId, actions };
		});
	}

	public releaseCodeActions(cachedId: number): void {
		dispose(this._disposables.get(cachedId));
		this._disposables.delete(cachedId);
		this._cache.delete(cachedId);
	}

	private static _isCommand(thing: any): thing is vscode.Command {
		return typeof (<vscode.Command>thing).command === 'string' && typeof (<vscode.Command>thing).title === 'string';
	}
}

class DocumentFormattingAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.DocumentFormattingEditProvider
	) { }

	provideDocumentFormattingEdits(resource: URI, options: modes.FormattingOptions, token: CancellationToken): Promise<ISingleEditOperation[] | undefined> {

		const document = this._documents.getDocument(resource);

		return asPromise(() => this._provider.provideDocumentFormattingEdits(document, <any>options, token)).then(value => {
			if (Array.isArray(value)) {
				return value.map(typeConvert.TextEdit.from);
			}
			return undefined;
		});
	}
}

class RangeFormattingAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.DocumentRangeFormattingEditProvider
	) { }

	provideDocumentRangeFormattingEdits(resource: URI, range: IRange, options: modes.FormattingOptions, token: CancellationToken): Promise<ISingleEditOperation[] | undefined> {

		const document = this._documents.getDocument(resource);
		const ran = typeConvert.Range.to(range);

		return asPromise(() => this._provider.provideDocumentRangeFormattingEdits(document, ran, <any>options, token)).then(value => {
			if (Array.isArray(value)) {
				return value.map(typeConvert.TextEdit.from);
			}
			return undefined;
		});
	}
}

class OnTypeFormattingAdapter {

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.OnTypeFormattingEditProvider
	) { }

	autoFormatTriggerCharacters: string[] = []; // not here

	provideOnTypeFormattingEdits(resource: URI, position: IPosition, ch: string, options: modes.FormattingOptions, token: CancellationToken): Promise<ISingleEditOperation[] | undefined> {

		const document = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);

		return asPromise(() => this._provider.provideOnTypeFormattingEdits(document, pos, ch, <any>options, token)).then(value => {
			if (Array.isArray(value)) {
				return value.map(typeConvert.TextEdit.from);
			}
			return undefined;
		});
	}
}

class NavigateTypeAdapter {

	private readonly _symbolCache = new Map<number, vscode.SymbolInformation>();
	private readonly _resultCache = new Map<number, [number, number]>();
	private readonly _provider: vscode.WorkspaceSymbolProvider;

	constructor(provider: vscode.WorkspaceSymbolProvider) {
		this._provider = provider;
	}

	provideWorkspaceSymbols(search: string, token: CancellationToken): Promise<extHostProtocol.IWorkspaceSymbolsDto> {
		const result: extHostProtocol.IWorkspaceSymbolsDto = extHostProtocol.IdObject.mixin({ symbols: [] });
		return asPromise(() => this._provider.provideWorkspaceSymbols(search, token)).then(value => {
			if (isNonEmptyArray(value)) {
				for (const item of value) {
					if (!item) {
						// drop
						continue;
					}
					if (!item.name) {
						console.warn('INVALID SymbolInformation, lacks name', item);
						continue;
					}
					const symbol = extHostProtocol.IdObject.mixin(typeConvert.WorkspaceSymbol.from(item));
					this._symbolCache.set(symbol._id!, item);
					result.symbols.push(symbol);
				}
			}
		}).then(() => {
			if (result.symbols.length > 0) {
				this._resultCache.set(result._id!, [result.symbols[0]._id!, result.symbols[result.symbols.length - 1]._id!]);
			}
			return result;
		});
	}

	async resolveWorkspaceSymbol(symbol: extHostProtocol.IWorkspaceSymbolDto, token: CancellationToken): Promise<extHostProtocol.IWorkspaceSymbolDto | undefined> {
		if (typeof this._provider.resolveWorkspaceSymbol !== 'function') {
			return symbol;
		}

		const item = this._symbolCache.get(symbol._id!);
		if (item) {
			const value = await asPromise(() => this._provider.resolveWorkspaceSymbol!(item, token));
			return value && mixin(symbol, typeConvert.WorkspaceSymbol.from(value), true);
		}
		return undefined;
	}

	releaseWorkspaceSymbols(id: number): any {
		const range = this._resultCache.get(id);
		if (range) {
			for (let [from, to] = range; from <= to; from++) {
				this._symbolCache.delete(from);
			}
			this._resultCache.delete(id);
		}
	}
}

class RenameAdapter {

	static supportsResolving(provider: vscode.RenameProvider): boolean {
		return typeof provider.prepareRename === 'function';
	}

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.RenameProvider
	) { }

	provideRenameEdits(resource: URI, position: IPosition, newName: string, token: CancellationToken): Promise<extHostProtocol.IWorkspaceEditDto | undefined> {

		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);

		return asPromise(() => this._provider.provideRenameEdits(doc, pos, newName, token)).then(value => {
			if (!value) {
				return undefined;
			}
			return typeConvert.WorkspaceEdit.from(value);
		}, err => {
			const rejectReason = RenameAdapter._asMessage(err);
			if (rejectReason) {
				return <extHostProtocol.IWorkspaceEditDto>{ rejectReason, edits: undefined! };
			} else {
				// generic error
				return Promise.reject<extHostProtocol.IWorkspaceEditDto>(err);
			}
		});
	}

	resolveRenameLocation(resource: URI, position: IPosition, token: CancellationToken): Promise<(modes.RenameLocation & modes.Rejection) | undefined> {
		if (typeof this._provider.prepareRename !== 'function') {
			return Promise.resolve(undefined);
		}

		const doc = this._documents.getDocument(resource);
		const pos = typeConvert.Position.to(position);

		return asPromise(() => this._provider.prepareRename!(doc, pos, token)).then(rangeOrLocation => {

			let range: vscode.Range | undefined;
			let text: string | undefined;
			if (Range.isRange(rangeOrLocation)) {
				range = rangeOrLocation;
				text = doc.getText(rangeOrLocation);

			} else if (isObject(rangeOrLocation)) {
				range = rangeOrLocation.range;
				text = rangeOrLocation.placeholder;
			}

			if (!range) {
				return undefined;
			}
			if (range.start.line > pos.line || range.end.line < pos.line) {
				console.warn('INVALID rename location: position line must be within range start/end lines');
				return undefined;
			}
			return { range: typeConvert.Range.from(range), text };
		}, err => {
			const rejectReason = RenameAdapter._asMessage(err);
			if (rejectReason) {
				return <modes.RenameLocation & modes.Rejection>{ rejectReason, range: undefined!, text: undefined! };
			} else {
				return Promise.reject<any>(err);
			}
		});
	}

	private static _asMessage(err: any): string | undefined {
		if (typeof err === 'string') {
			return err;
		} else if (err instanceof Error && typeof err.message === 'string') {
			return err.message;
		} else {
			return undefined;
		}
	}
}

const enum SemanticColoringConstants {
	/**
	 * Let's aim at having 8KB buffers if possible...
	 * So that would be 8192 / (5 * 4) = 409.6 tokens per area
	 */
	DesiredTokensPerArea = 400,

	/**
	 * If the semantic coloring result contains a single area and the area contains
	 * more than 600 tokens, then split the area into mulitple areas.
	 */
	SplitSingleAreaTokenCountThreshold = 600,

	/**
	 * Try to keep the total number of areas under 1024 if possible,
	 * simply compensate by having more tokens per area...
	 */
	DesiredMaxAreas = 1024
}

class SemanticColoringAdapter {

	private readonly _previousResults = new Map<number, vscode.SemanticColoring>();
	private _nextResultId = 1;

	constructor(
		private readonly _documents: ExtHostDocuments,
		private readonly _provider: vscode.SemanticColoringProvider,
	) { }

	provideSemanticColoring(resource: URI, previousSemanticColoringResultId: number, token: CancellationToken): Promise<VSBuffer | null> {
		const doc = this._documents.getDocument(resource);

		return asPromise(() => this._provider.provideSemanticColoring(doc, token)).then(value => {
			if (!value) {
				return null;
			}

			const oldResult = (previousSemanticColoringResultId !== 0 ? this._previousResults.get(previousSemanticColoringResultId) : null);
			if (oldResult) {
				this._previousResults.delete(previousSemanticColoringResultId);
				return this._deltaEncode(oldResult, value);
			}

			return this._encode(value);
		});
	}

	async releaseSemanticColoring(semanticColoringResultId: number): Promise<void> {
		this._previousResults.delete(semanticColoringResultId);
	}

	private _deltaEncode(oldResult: vscode.SemanticColoring, newResult: vscode.SemanticColoring): VSBuffer {
		if (newResult.areas.length > 1) {
			// this is a fancy provider which is smart enough to break things into good areas
			// we therefore try to match old areas by identity
			const oldAreas = new Map<Uint32Array, number>();
			for (let i = 0, len = oldResult.areas.length; i < len; i++) {
				const oldArea = oldResult.areas[i];
				oldAreas.set(oldArea.data, i);
			}

			let areas: (ISemanticTokensFullAreaDto | ISemanticTokensDeltaAreaDto)[] = [];
			for (let i = 0, len = newResult.areas.length; i < len; i++) {
				const newArea = newResult.areas[i];
				if (oldAreas.has(newArea.data)) {
					// great! we can reuse this area
					const oldIndex = oldAreas.get(newArea.data)!;
					areas.push({
						type: 'delta',
						line: newArea.line,
						oldIndex: oldIndex
					});
				} else {
					areas.push({
						type: 'full',
						line: newArea.line,
						data: newArea.data
					});
				}
			}

			return this._saveResultAndEncode(newResult, areas);
		}

		// Try to find appearences of `oldResult` areas inside `newResult`.

		console.log(oldResult);
		console.log(newResult);
		throw new Error(`TODO - Not implemented`);
	}

	// /**
	//  * Note: Please do not use arrays.equals (for perf reasons)
	//  */
	// private static _areasEquals(a: SemanticColoringArea, b: SemanticColoringArea): boolean {
	// 	if (a.line !== b.line) {
	// 		return false;
	// 	}
	// 	const aData = a.data;
	// 	const bData = b.data;
	// 	if (aData === bData) {
	// 		return true;
	// 	}
	// 	if (aData.length !== bData.length) {
	// 		return false;
	// 	}
	// 	for (let i = 0, len = aData.length; i < len; i++) {
	// 		if (aData[i] !== bData[i]) {
	// 			return false;
	// 		}
	// 	}
	// 	return true;
	// }

	private _encode(result: vscode.SemanticColoring): VSBuffer {
		if (result.areas.length === 1) {
			const tokenCount = (result.areas[0].data.length / 5) | 0;
			if (tokenCount > SemanticColoringConstants.SplitSingleAreaTokenCountThreshold) {
				// This result contains a single area which contains over `SplitSingleAreaTokenCountThreshold` tokens
				// so let's split tokens into multiple areas...
				result = SemanticColoringAdapter._splitIntoMultipleAreas(result.areas[0]);
			}
		}

		return this._saveResultAndEncode(result, result.areas.map(area => {
			return {
				type: 'full',
				line: area.line,
				data: area.data
			};
		}));
	}

	private _saveResultAndEncode(result: vscode.SemanticColoring, areas: (ISemanticTokensFullAreaDto | ISemanticTokensDeltaAreaDto)[]): VSBuffer {
		const myId = this._nextResultId++;
		this._previousResults.set(myId, result);
		const dto: ISemanticTokensDto = {
			id: myId,
			areas: areas
		};
		return encodeSemanticTokensDto(dto);
	}

	private static _splitIntoMultipleAreas(area: vscode.SemanticColoringArea): SemanticColoring {
		const srcAreaLine = area.line;
		const srcAreaData = area.data;
		const tokenCount = (srcAreaData.length / 5) | 0;
		const tokensPerArea = Math.max(Math.ceil(tokenCount / SemanticColoringConstants.DesiredMaxAreas), SemanticColoringConstants.DesiredTokensPerArea);

{'fraction_non_alphanumeric': 0.1266441260968984, 'fraction_numerical': 0.0021107354349483064, 'mean_word_length': 4.411167512690355, 'pattern_counts': {'":': 0, '<': 95, '<?xml version=': 0, '>': 171, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4819668', 'n_tokens_mistral': 17131, 'n_tokens_neox': 15022, 'n_words': 3314}