<commit_before>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import { IBuiltinExtensionsScannerService, IScannedExtension, ExtensionType, IExtensionIdentifier, ITranslatedScannedExtension } from 'vs/platform/extensions/common/extensions';
import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService';
import { IWebExtensionsScannerService } from 'vs/workbench/services/extensionManagement/common/extensionManagement';
import { isWeb } from 'vs/base/common/platform';
import { registerSingleton } from 'vs/platform/instantiation/common/extensions';
import { joinPath } from 'vs/base/common/resources';
import { URI, UriComponents } from 'vs/base/common/uri';
import { IFileService } from 'vs/platform/files/common/files';
import { Queue } from 'vs/base/common/async';
import { VSBuffer } from 'vs/base/common/buffer';
import { asText, isSuccess, IRequestService } from 'vs/platform/request/common/request';
import { ILogService } from 'vs/platform/log/common/log';
import { CancellationToken } from 'vs/base/common/cancellation';
import { IGalleryExtension, INSTALL_ERROR_NOT_SUPPORTED } from 'vs/platform/extensionManagement/common/extensionManagement';
import { groupByExtension, areSameExtensions, getGalleryExtensionId } from 'vs/platform/extensionManagement/common/extensionManagementUtil';
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
import type { IStaticExtension } from 'vs/workbench/workbench.web.api';
import { Disposable } from 'vs/base/common/lifecycle';
import { Event } from 'vs/base/common/event';
import { localizeManifest } from 'vs/platform/extensionManagement/common/extensionNls';
import { localize } from 'vs/nls';
import * as semver from 'vs/base/common/semver/semver';

interface IUserExtension {
	identifier: IExtensionIdentifier;
	version: string;
	location: URI;
	readmeUri?: URI;
	changelogUri?: URI;
	packageNLSUri?: URI;
}

interface IStoredUserExtension {
	identifier: IExtensionIdentifier;
	version: string;
	location: UriComponents;
	readmeUri?: UriComponents;
	changelogUri?: UriComponents;
	packageNLSUri?: UriComponents;
}

export class WebExtensionsScannerService extends Disposable implements IWebExtensionsScannerService {

	declare readonly _serviceBrand: undefined;

	private readonly systemExtensionsPromise: Promise<IScannedExtension[]> = Promise.resolve([]);
	private readonly defaultExtensionsPromise: Promise<IScannedExtension[]> = Promise.resolve([]);
	private readonly extensionsResource: URI | undefined = undefined;
	private readonly userExtensionsResourceLimiter: Queue<IUserExtension[]> = new Queue<IUserExtension[]>();

	private userExtensionsPromise: Promise<IScannedExtension[]> | undefined;

	constructor(
		@IWorkbenchEnvironmentService private readonly environmentService: IWorkbenchEnvironmentService,
		@IBuiltinExtensionsScannerService private readonly builtinExtensionsScannerService: IBuiltinExtensionsScannerService,
		@IFileService private readonly fileService: IFileService,
		@IRequestService private readonly requestService: IRequestService,
		@ILogService private readonly logService: ILogService,
		@IConfigurationService private readonly configurationService: IConfigurationService,
	) {
		super();
		if (isWeb) {
			this.extensionsResource = joinPath(environmentService.userRoamingDataHome, 'extensions.json');
			this.systemExtensionsPromise = this.readSystemExtensions();
			this.defaultExtensionsPromise = this.readDefaultExtensions();
			if (this.extensionsResource) {
				this._register(Event.filter(this.fileService.onDidFilesChange, e => e.contains(this.extensionsResource!))(() => this.userExtensionsPromise = undefined));
			}
		}
	}

	private async readSystemExtensions(): Promise<IScannedExtension[]> {
		const extensions = await this.builtinExtensionsScannerService.scanBuiltinExtensions();
		return extensions.concat(this.getStaticExtensions(true));
	}

	/**
	 * All extensions defined via `staticExtensions`
	 */
	private getStaticExtensions(builtin: boolean): IScannedExtension[] {
		const staticExtensions = this.environmentService.options && Array.isArray(this.environmentService.options.staticExtensions) ? this.environmentService.options.staticExtensions : [];
		const result: IScannedExtension[] = [];
		for (const e of staticExtensions) {
			if (Boolean(e.isBuiltin) === builtin) {
				const scannedExtension = this.parseStaticExtension(e, builtin);
				if (scannedExtension) {
					result.push(scannedExtension);
				}
			}
		}
		return result;
	}

	private async readDefaultExtensions(): Promise<IScannedExtension[]> {
		const defaultUserWebExtensions = await this.readDefaultUserWebExtensions();
		const extensions: IScannedExtension[] = [];
		for (const e of defaultUserWebExtensions) {
			const scannedExtension = this.parseStaticExtension(e, false);
			if (scannedExtension) {
				extensions.push(scannedExtension);
			}
		}
		return extensions.concat(this.getStaticExtensions(false));
	}

	private parseStaticExtension(e: IStaticExtension, builtin: boolean): IScannedExtension | null {
		try {
			return {
				identifier: { id: getGalleryExtensionId(e.packageJSON.publisher, e.packageJSON.name) },
				location: e.extensionLocation,
				type: builtin ? ExtensionType.System : ExtensionType.User,
				packageJSON: e.packageJSON,
			};
		} catch (error) {
			this.logService.error(`Error while parsing extension ${e.extensionLocation.toString()}`);
			this.logService.error(error);
		}
		return null;
	}

	private async readDefaultUserWebExtensions(): Promise<IStaticExtension[]> {
		const result: IStaticExtension[] = [];
		const defaultUserWebExtensions = this.configurationService.getValue<{ location: string }[]>('_extensions.defaultUserWebExtensions') || [];
		for (const webExtension of defaultUserWebExtensions) {
			const extensionLocation = URI.parse(webExtension.location);
			const manifestLocation = joinPath(extensionLocation, 'package.json');
			const context = await this.requestService.request({ type: 'GET', url: manifestLocation.toString(true) }, CancellationToken.None);
			if (!isSuccess(context)) {
				this.logService.warn('Skipped default user web extension as there is an error while fetching manifest', manifestLocation);
				continue;
			}
			const content = await asText(context);
			if (!content) {
				this.logService.warn('Skipped default user web extension as there is manifest is not found', manifestLocation);
				continue;
			}
			const packageJSON = JSON.parse(content);
			result.push({
				packageJSON,
				extensionLocation,
			});
		}
		return result;
	}

	async scanExtensions(type?: ExtensionType): Promise<IScannedExtension[]> {
		const extensions = [];
		if (type === undefined || type === ExtensionType.System) {
			const systemExtensions = await this.systemExtensionsPromise;
			extensions.push(...systemExtensions);
		}
		if (type === undefined || type === ExtensionType.User) {
			const staticExtensions = await this.defaultExtensionsPromise;
			extensions.push(...staticExtensions);<commit_msg>Fix #111573<commit_after>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import { IBuiltinExtensionsScannerService, IScannedExtension, ExtensionType, IExtensionIdentifier, ITranslatedScannedExtension } from 'vs/platform/extensions/common/extensions';
import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService';
import { IWebExtensionsScannerService } from 'vs/workbench/services/extensionManagement/common/extensionManagement';
import { isWeb } from 'vs/base/common/platform';
import { registerSingleton } from 'vs/platform/instantiation/common/extensions';
import { joinPath } from 'vs/base/common/resources';
import { URI, UriComponents } from 'vs/base/common/uri';
import { IFileService } from 'vs/platform/files/common/files';
import { Queue } from 'vs/base/common/async';
import { VSBuffer } from 'vs/base/common/buffer';
import { asText, isSuccess, IRequestService } from 'vs/platform/request/common/request';
import { ILogService } from 'vs/platform/log/common/log';
import { CancellationToken } from 'vs/base/common/cancellation';
import { IGalleryExtension, INSTALL_ERROR_NOT_SUPPORTED } from 'vs/platform/extensionManagement/common/extensionManagement';
import { groupByExtension, areSameExtensions, getGalleryExtensionId } from 'vs/platform/extensionManagement/common/extensionManagementUtil';
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
import type { IStaticExtension } from 'vs/workbench/workbench.web.api';
import { Disposable } from 'vs/base/common/lifecycle';
import { Event } from 'vs/base/common/event';
import { localizeManifest } from 'vs/platform/extensionManagement/common/extensionNls';
import { localize } from 'vs/nls';
import * as semver from 'vs/base/common/semver/semver';

interface IUserExtension {
	identifier: IExtensionIdentifier;
	version: string;
	location: URI;
	readmeUri?: URI;
	changelogUri?: URI;
	packageNLSUri?: URI;
}

interface IStoredUserExtension {
	identifier: IExtensionIdentifier;
	version: string;
	location: UriComponents;
	readmeUri?: UriComponents;
	changelogUri?: UriComponents;
	packageNLSUri?: UriComponents;
}

export class WebExtensionsScannerService extends Disposable implements IWebExtensionsScannerService {

	declare readonly _serviceBrand: undefined;

	private readonly systemExtensionsPromise: Promise<IScannedExtension[]> = Promise.resolve([]);
	private readonly defaultExtensionsPromise: Promise<IScannedExtension[]> = Promise.resolve([]);
	private readonly extensionsResource: URI | undefined = undefined;
	private readonly userExtensionsResourceLimiter: Queue<IUserExtension[]> = new Queue<IUserExtension[]>();

	private userExtensionsPromise: Promise<IScannedExtension[]> | undefined;

	constructor(
		@IWorkbenchEnvironmentService private readonly environmentService: IWorkbenchEnvironmentService,
		@IBuiltinExtensionsScannerService private readonly builtinExtensionsScannerService: IBuiltinExtensionsScannerService,
		@IFileService private readonly fileService: IFileService,
		@IRequestService private readonly requestService: IRequestService,
		@ILogService private readonly logService: ILogService,
		@IConfigurationService private readonly configurationService: IConfigurationService,
	) {
		super();
		if (isWeb) {
			this.extensionsResource = joinPath(environmentService.userRoamingDataHome, 'extensions.json');
			this.systemExtensionsPromise = this.readSystemExtensions();
			this.defaultExtensionsPromise = this.readDefaultExtensions();
			if (this.extensionsResource) {
				this._register(Event.filter(this.fileService.onDidFilesChange, e => e.contains(this.extensionsResource!))(() => this.userExtensionsPromise = undefined));
			}
		}
	}

	private async readSystemExtensions(): Promise<IScannedExtension[]> {
		const extensions = await this.builtinExtensionsScannerService.scanBuiltinExtensions();
		return extensions.concat(this.getStaticExtensions(true));
	}

	/**
	 * All extensions defined via `staticExtensions`
	 */
	private getStaticExtensions(builtin: boolean): IScannedExtension[] {
		const staticExtensions = this.environmentService.options && Array.isArray(this.environmentService.options.staticExtensions) ? this.environmentService.options.staticExtensions : [];
		const result: IScannedExtension[] = [];
		for (const e of staticExtensions) {
			if (Boolean(e.isBuiltin) === builtin) {
				const scannedExtension = this.parseStaticExtension(e, builtin);
				if (scannedExtension) {
					result.push(scannedExtension);
				}
			}
		}
		return result;
	}

	private async readDefaultExtensions(): Promise<IScannedExtension[]> {
		const defaultUserWebExtensions = await this.readDefaultUserWebExtensions();
		const extensions: IScannedExtension[] = [];
		for (const e of defaultUserWebExtensions) {
			const scannedExtension = this.parseStaticExtension(e, false);
			if (scannedExtension) {
				extensions.push(scannedExtension);
			}
		}
		return extensions.concat(this.getStaticExtensions(false));
	}

	private parseStaticExtension(e: IStaticExtension, builtin: boolean): IScannedExtension | null {
		try {
			return {
				identifier: { id: getGalleryExtensionId(e.packageJSON.publisher, e.packageJSON.name) },
				location: e.extensionLocation,
				type: builtin ? ExtensionType.System : ExtensionType.User,
				packageJSON: e.packageJSON,
			};
		} catch (error) {
			this.logService.error(`Error while parsing extension ${e.extensionLocation.toString()}`);
			this.logService.error(error);
		}
		return null;
	}

	private async readDefaultUserWebExtensions(): Promise<IStaticExtension[]> {
		const result: IStaticExtension[] = [];
		const defaultUserWebExtensions = this.configurationService.getValue<{ location: string }[]>('_extensions.defaultUserWebExtensions') || [];
		for (const webExtension of defaultUserWebExtensions) {
			try {
				const extensionLocation = URI.parse(webExtension.location);
				const manifestLocation = joinPath(extensionLocation, 'package.json');
				const context = await this.requestService.request({ type: 'GET', url: manifestLocation.toString(true) }, CancellationToken.None);
				if (!isSuccess(context)) {
					this.logService.warn('Skipped default user web extension as there is an error while fetching manifest', manifestLocation);
					continue;
				}
				const content = await asText(context);
				if (!content) {
					this.logService.warn('Skipped default user web extension as there is manifest is not found', manifestLocation);
					continue;
				}
				const packageJSON = JSON.parse(content);
				result.push({
					packageJSON,
					extensionLocation,
				});
			} catch (error) {
				this.logService.warn('Skipped default user web extension as there is an error while fetching manifest', webExtension);
			}
		}
		return result;
	}

	async scanExtensions(type?: ExtensionType): Promise<IScannedExtension[]> {
		const extensions = [];
		if (type === undefined || type === ExtensionType.System) {
			const systemExtensions = await this.systemExtensionsPromise;
			extensions.push(...systemExtensions);
		}
		if (type === undefined || type === ExtensionType.User) {
			const staticExtensions = await this.defaultExtensionsPromise;
			extensions.push(...staticExtensions);
{'fraction_non_alphanumeric': 0.12847807047092905, 'fraction_numerical': 0.0004042309506164522, 'mean_word_length': 4.959052589321558, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2374541', 'n_tokens_mistral': 4183, 'n_tokens_neox': 3840, 'n_words': 897}