<commit_before>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/
'use strict';

import { PPromise, TPromise } from 'vs/base/common/winjs.base';
import uri from 'vs/base/common/uri';
import objects = require('vs/base/common/objects');
import strings = require('vs/base/common/strings');
import { getNextTickChannel } from 'vs/base/parts/ipc/common/ipc';
import { Client, IIPCOptions } from 'vs/base/parts/ipc/node/ipc.cp';
import { IProgress, LineMatch, FileMatch, ISearchComplete, ISearchProgressItem, QueryType, IFileMatch, ISearchQuery, ISearchConfiguration, ISearchService, pathIncludedInQuery, ISearchResultProvider } from 'vs/platform/search/common/search';
import { IUntitledEditorService } from 'vs/workbench/services/untitled/common/untitledEditorService';
import { IModelService } from 'vs/editor/common/services/modelService';
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
import { IRawSearch, ISerializedSearchComplete, ISerializedSearchProgressItem, ISerializedFileMatch, IRawSearchService, ITelemetryEvent } from './search';
import { ISearchChannel, SearchChannelClient } from './searchIpc';
import { IEnvironmentService, IDebugParams } from 'vs/platform/environment/common/environment';
import { ResourceMap } from 'vs/base/common/map';
import { IDisposable } from 'vs/base/common/lifecycle';
import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
import { onUnexpectedError } from 'vs/base/common/errors';
import { Schemas } from 'vs/base/common/network';
import { ILogService } from 'vs/platform/log/common/log';

export class SearchService implements ISearchService {
	public _serviceBrand: any;

	private diskSearch: DiskSearch;
	private readonly searchProvider: ISearchResultProvider[] = [];
	private forwardingTelemetry: PPromise<void, ITelemetryEvent>;

	constructor(
		@IModelService private modelService: IModelService,
		@IUntitledEditorService private untitledEditorService: IUntitledEditorService,
		@IEnvironmentService environmentService: IEnvironmentService,
		@ITelemetryService private telemetryService: ITelemetryService,
		@IConfigurationService private configurationService: IConfigurationService,
		@ILogService private logService: ILogService
	) {
		this.diskSearch = new DiskSearch(!environmentService.isBuilt || environmentService.verbose, /*timeout=*/undefined, environmentService.debugSearch);
		this.registerSearchResultProvider(this.diskSearch);
	}

	public registerSearchResultProvider(provider: ISearchResultProvider): IDisposable {
		this.searchProvider.push(provider);
		return {
			dispose: () => {
				const idx = this.searchProvider.indexOf(provider);
				if (idx >= 0) {
					this.searchProvider.splice(idx, 1);
				}
			}
		};
	}

	public extendQuery(query: ISearchQuery): void {
		const configuration = this.configurationService.getValue<ISearchConfiguration>();

		// Configuration: Encoding
		if (!query.fileEncoding) {
			const fileEncoding = configuration && configuration.files && configuration.files.encoding;
			query.fileEncoding = fileEncoding;
		}

		// Configuration: File Excludes
		if (!query.disregardExcludeSettings) {
			const fileExcludes = objects.deepClone(configuration && configuration.files && configuration.files.exclude);
			if (fileExcludes) {
				if (!query.excludePattern) {
					query.excludePattern = fileExcludes;
				} else {
					objects.mixin(query.excludePattern, fileExcludes, false /* no overwrite */);
				}
			}
		}
	}

	public search(query: ISearchQuery): PPromise<ISearchComplete, ISearchProgressItem> {
		this.forwardTelemetry();

		let combinedPromise: TPromise<void>;

		return new PPromise<ISearchComplete, ISearchProgressItem>((onComplete, onError, onProgress) => {

			// Get local results from dirty/untitled
			const localResults = this.getLocalResults(query);

			// Allow caller to register progress callback
			process.nextTick(() => localResults.values().filter((res) => !!res).forEach(onProgress));

			const providerPromises = this.searchProvider.map(provider => TPromise.wrap(provider.search(query)).then(e => e,
				err => {
					// TODO@joh
					// single provider fail. fail all?
					onError(err);
				},
				progress => {
					if (progress.resource) {
						// Match
						if (!localResults.has(progress.resource)) { // don't override local results
							onProgress(progress);
						}
					} else {
						// Progress
						onProgress(<IProgress>progress);
					}

					if (progress.message) {
						this.logService.info('SearchService#search', progress.message);
					}
				}
			));

			combinedPromise = TPromise.join(providerPromises).then(values => {

				const result: ISearchComplete = {
					limitHit: false,
					results: [],
					stats: undefined
				};

				// TODO@joh
				// sorting, disjunct results
				for (const value of values) {
					if (!value) {
						continue;
					}
					// TODO@joh individual stats/limit
					result.stats = value.stats || result.stats;
					result.limitHit = value.limitHit || result.limitHit;

					for (const match of value.results) {
						if (!localResults.has(match.resource)) {
							result.results.push(match);
						}
					}
				}<commit_msg>Change log levels for SearchService to not show up by default - too noisy<commit_after>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/
'use strict';

import { PPromise, TPromise } from 'vs/base/common/winjs.base';
import uri from 'vs/base/common/uri';
import objects = require('vs/base/common/objects');
import strings = require('vs/base/common/strings');
import { getNextTickChannel } from 'vs/base/parts/ipc/common/ipc';
import { Client, IIPCOptions } from 'vs/base/parts/ipc/node/ipc.cp';
import { IProgress, LineMatch, FileMatch, ISearchComplete, ISearchProgressItem, QueryType, IFileMatch, ISearchQuery, ISearchConfiguration, ISearchService, pathIncludedInQuery, ISearchResultProvider } from 'vs/platform/search/common/search';
import { IUntitledEditorService } from 'vs/workbench/services/untitled/common/untitledEditorService';
import { IModelService } from 'vs/editor/common/services/modelService';
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
import { IRawSearch, ISerializedSearchComplete, ISerializedSearchProgressItem, ISerializedFileMatch, IRawSearchService, ITelemetryEvent } from './search';
import { ISearchChannel, SearchChannelClient } from './searchIpc';
import { IEnvironmentService, IDebugParams } from 'vs/platform/environment/common/environment';
import { ResourceMap } from 'vs/base/common/map';
import { IDisposable } from 'vs/base/common/lifecycle';
import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
import { onUnexpectedError } from 'vs/base/common/errors';
import { Schemas } from 'vs/base/common/network';
import { ILogService } from 'vs/platform/log/common/log';

export class SearchService implements ISearchService {
	public _serviceBrand: any;

	private diskSearch: DiskSearch;
	private readonly searchProvider: ISearchResultProvider[] = [];
	private forwardingTelemetry: PPromise<void, ITelemetryEvent>;

	constructor(
		@IModelService private modelService: IModelService,
		@IUntitledEditorService private untitledEditorService: IUntitledEditorService,
		@IEnvironmentService environmentService: IEnvironmentService,
		@ITelemetryService private telemetryService: ITelemetryService,
		@IConfigurationService private configurationService: IConfigurationService,
		@ILogService private logService: ILogService
	) {
		this.diskSearch = new DiskSearch(!environmentService.isBuilt || environmentService.verbose, /*timeout=*/undefined, environmentService.debugSearch);
		this.registerSearchResultProvider(this.diskSearch);
	}

	public registerSearchResultProvider(provider: ISearchResultProvider): IDisposable {
		this.searchProvider.push(provider);
		return {
			dispose: () => {
				const idx = this.searchProvider.indexOf(provider);
				if (idx >= 0) {
					this.searchProvider.splice(idx, 1);
				}
			}
		};
	}

	public extendQuery(query: ISearchQuery): void {
		const configuration = this.configurationService.getValue<ISearchConfiguration>();

		// Configuration: Encoding
		if (!query.fileEncoding) {
			const fileEncoding = configuration && configuration.files && configuration.files.encoding;
			query.fileEncoding = fileEncoding;
		}

		// Configuration: File Excludes
		if (!query.disregardExcludeSettings) {
			const fileExcludes = objects.deepClone(configuration && configuration.files && configuration.files.exclude);
			if (fileExcludes) {
				if (!query.excludePattern) {
					query.excludePattern = fileExcludes;
				} else {
					objects.mixin(query.excludePattern, fileExcludes, false /* no overwrite */);
				}
			}
		}
	}

	public search(query: ISearchQuery): PPromise<ISearchComplete, ISearchProgressItem> {
		this.forwardTelemetry();

		let combinedPromise: TPromise<void>;

		return new PPromise<ISearchComplete, ISearchProgressItem>((onComplete, onError, onProgress) => {

			// Get local results from dirty/untitled
			const localResults = this.getLocalResults(query);

			// Allow caller to register progress callback
			process.nextTick(() => localResults.values().filter((res) => !!res).forEach(onProgress));

			this.logService.trace('SearchService#search', JSON.stringify(query));
			const providerPromises = this.searchProvider.map(provider => TPromise.wrap(provider.search(query)).then(e => e,
				err => {
					// TODO@joh
					// single provider fail. fail all?
					onError(err);
				},
				progress => {
					if (progress.resource) {
						// Match
						if (!localResults.has(progress.resource)) { // don't override local results
							onProgress(progress);
						}
					} else {
						// Progress
						onProgress(<IProgress>progress);
					}

					if (progress.message) {
						this.logService.debug('SearchService#search', progress.message);
					}
				}
			));

			combinedPromise = TPromise.join(providerPromises).then(values => {

				const result: ISearchComplete = {
					limitHit: false,
					results: [],
					stats: undefined
				};

				// TODO@joh
				// sorting, disjunct results
				for (const value of values) {
					if (!value) {
						continue;
					}
					// TODO@joh individual stats/limit
					result.stats = value.stats || result.stats;
					result.limitHit = value.limitHit || result.limitHit;

					for (const match of value.results) {
						if (!localResults.has(match.resource)) {
							result.results.push(match);
						}
					}
				}
{'fraction_non_alphanumeric': 0.14385046053819758, 'fraction_numerical': 0.00036120642947444465, 'mean_word_length': 3.6987696224013575, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1458082', 'n_tokens_mistral': 3516, 'n_tokens_neox': 2969, 'n_words': 664}