<commit_before>import * as fs from 'mz/fs';
import * as path from 'path';
import { LanguageClient } from './lang-handler';
import glob = require('glob');
import iterate from 'iterare';
import { Span } from 'opentracing';
import Semaphore from 'semaphore-async-await';
import { InMemoryFileSystem } from './memfs';
import { normalizeDir, path2uri, toUnixPath, uri2path } from './util';

export interface FileSystem {
	/**
	 * Returns all files in the workspace under base
	 *
	 * @param base A URI under which to search, resolved relative to the rootUri
	 * @return A promise that is fulfilled with an array of URIs
	 */
	getWorkspaceFiles(base?: string, childOf?: Span): Promise<Iterable<string>>;

	/**
	 * Returns the content of a text document
	 *
	 * @param uri The URI of the text document, resolved relative to the rootUri
	 * @return A promise that is fulfilled with the text document content
	 */
	getTextDocumentContent(uri: string, childOf?: Span): Promise<string>;
}

export class RemoteFileSystem implements FileSystem {

	constructor(private client: LanguageClient) {}

	/**
	 * The files request is sent from the server to the client to request a list of all files in the workspace or inside the directory of the base parameter, if given.
	 * A language server can use the result to index files by filtering and doing a content request for each text document of interest.
	 */
	async getWorkspaceFiles(base?: string, childOf = new Span()): Promise<Iterable<string>> {
		return iterate(await this.client.workspaceXfiles({ base }, childOf))
			.map(textDocument => textDocument.uri);
	}

	/**
	 * The content request is sent from the server to the client to request the current content of any text document. This allows language servers to operate without accessing the file system directly.
	 */
	async getTextDocumentContent(uri: string, childOf = new Span()): Promise<string> {
		const textDocument = await this.client.textDocumentXcontent({ textDocument: { uri } }, childOf);
		return textDocument.text;
	}
}

export class LocalFileSystem implements FileSystem {

	/**
	 * @param rootPath The root directory path that relative URIs should be resolved to
	 */
	constructor(private rootPath: string) {}

	/**
	 * Converts the URI to an absolute path
	 */
	protected resolveUriToPath(uri: string): string {
		return toUnixPath(path.resolve(this.rootPath, uri2path(uri)));
	}

	async getWorkspaceFiles(base?: string): Promise<Iterable<string>> {
		// Even if no base provided, still need to call resolveUriToPath which may be overridden
		const root = this.resolveUriToPath(base || path2uri('', this.rootPath));
		const baseUri = path2uri('', normalizeDir(root)) + '/';
		const files = await new Promise<string[]>((resolve, reject) => {
			glob('*', { cwd: root, nodir: true, matchBase: true }, (err, matches) => err ? reject(err) : resolve(matches));
		});
		return iterate(files).map(file => baseUri + file.split('/').map(encodeURIComponent).join('/'));
	}

	async getTextDocumentContent(uri: string): Promise<string> {
		return fs.readFile(this.resolveUriToPath(uri), 'utf8');
	}
}

/**
 * Synchronizes a remote file system to an in-memory file system
 *
 * TODO: Implement Disposable with Disposer
 */
export class FileSystemUpdater {

	/**
	 * Promise for a pending or fulfilled structure fetch
	 */
	private structureFetch?: Promise<void>;

	/**
	 * Map from URI to Promise of pending or fulfilled content fetch
	 */
	private fetches = new Map<string, Promise<void>>();

	/**
	 * Limits concurrent fetches to not fetch thousands of files in parallel
	 */
	private concurrencyLimit = new Semaphore(100);

	constructor(private remoteFs: FileSystem, private inMemoryFs: InMemoryFileSystem) {}

	/**
	 * Fetches the file content for the given URI and adds the content to the in-memory file system
	 *
	 * @param uri URI of the file to fetch
	 * @param childOf A parent span for tracing
	 */
	async fetch(uri: string, childOf = new Span()): Promise<void> {
		// Limit concurrent fetches
		const promise = this.concurrencyLimit.execute(async () => {
			try {
				const content = await this.remoteFs.getTextDocumentContent(uri);
				this.inMemoryFs.add(uri, content);
				this.inMemoryFs.getContent(uri);
			} catch (err) {
				this.fetches.delete(uri);
				throw err;
			}
		});
		this.fetches.set(uri, promise);
		return promise;
	}

	/**
	 * Returns a promise that is resolved when the given URI has been fetched (at least once) to the in-memory file system.
	 * This function cannot be cancelled because multiple callers get the result of the same operation.
	 *
	 * @param uri URI of the file to ensure
	 * @param childOf A parent span for tracing
	 */
	async ensure(uri: string, childOf = new Span()): Promise<void> {
		const span = childOf.tracer().startSpan('Ensure file content', { childOf });
		span.addTags({ uri });
		try {
			return await (this.fetches.get(uri) || this.fetch(uri, span));
		} catch (err) {
			span.setTag('error', true);
			span.log({ 'event': 'error', 'error.object': err, 'message': err.message, 'stack': err.stack });
			throw err;
		} finally {
			span.finish();
		}
	}

	/**
	 * Fetches the file/directory structure for the given directory from the remote file system and saves it in the in-memory file system
	 *
	 * @param childOf A parent span for tracing
	 */
	fetchStructure(childOf = new Span()): Promise<void> {
		const promise = (async () => {
			const span = childOf.tracer().startSpan('Fetch workspace structure', { childOf });
			try {
				const uris = await this.remoteFs.getWorkspaceFiles(undefined, span);
				for (const uri of uris) {
					this.inMemoryFs.add(uri);
				}
			} catch (err) {
				this.structureFetch = undefined;
				span.setTag('error', true);
				span.log({ 'event': 'error', 'error.object': err, 'message': err.message, 'stack': err.stack });
				throw err;
			} finally {
				span.finish();
			}
		})();
		this.structureFetch = promise;
		return promise;
	}

	/**
	 * Returns a promise that is resolved as soon as the file/directory structure for the given directory has been synced
	 * from the remote file system to the in-memory file system (at least once)
	 *
	 * @param childOf A parent span for tracing
	 */
	async ensureStructure(childOf = new Span()) {
		const span = childOf.tracer().startSpan('Ensure workspace structure', { childOf });
		try {
			return await (this.structureFetch || this.fetchStructure(span));
		} catch (err) {
			span.setTag('error', true);
			span.log({ 'event': 'error', 'error.object': err, 'message': err.message, 'stack': err.stack });
			throw err;
		} finally {
			span.finish();
		}
	}

	/**
	 * Invalidates the structure fetch cache.
	 * The next call to `ensureStructure` will do a refetch.
	 */
	invalidateStructure() {
		this.structureFetch = undefined;
	}
}<commit_msg>Add method for content invalidation<commit_after>import * as fs from 'mz/fs';
import * as path from 'path';
import { LanguageClient } from './lang-handler';
import glob = require('glob');
import iterate from 'iterare';
import { Span } from 'opentracing';
import Semaphore from 'semaphore-async-await';
import { InMemoryFileSystem } from './memfs';
import { normalizeDir, path2uri, toUnixPath, uri2path } from './util';

export interface FileSystem {
	/**
	 * Returns all files in the workspace under base
	 *
	 * @param base A URI under which to search, resolved relative to the rootUri
	 * @return A promise that is fulfilled with an array of URIs
	 */
	getWorkspaceFiles(base?: string, childOf?: Span): Promise<Iterable<string>>;

	/**
	 * Returns the content of a text document
	 *
	 * @param uri The URI of the text document, resolved relative to the rootUri
	 * @return A promise that is fulfilled with the text document content
	 */
	getTextDocumentContent(uri: string, childOf?: Span): Promise<string>;
}

export class RemoteFileSystem implements FileSystem {

	constructor(private client: LanguageClient) {}

	/**
	 * The files request is sent from the server to the client to request a list of all files in the workspace or inside the directory of the base parameter, if given.
	 * A language server can use the result to index files by filtering and doing a content request for each text document of interest.
	 */
	async getWorkspaceFiles(base?: string, childOf = new Span()): Promise<Iterable<string>> {
		return iterate(await this.client.workspaceXfiles({ base }, childOf))
			.map(textDocument => textDocument.uri);
	}

	/**
	 * The content request is sent from the server to the client to request the current content of any text document. This allows language servers to operate without accessing the file system directly.
	 */
	async getTextDocumentContent(uri: string, childOf = new Span()): Promise<string> {
		const textDocument = await this.client.textDocumentXcontent({ textDocument: { uri } }, childOf);
		return textDocument.text;
	}
}

export class LocalFileSystem implements FileSystem {

	/**
	 * @param rootPath The root directory path that relative URIs should be resolved to
	 */
	constructor(private rootPath: string) {}

	/**
	 * Converts the URI to an absolute path
	 */
	protected resolveUriToPath(uri: string): string {
		return toUnixPath(path.resolve(this.rootPath, uri2path(uri)));
	}

	async getWorkspaceFiles(base?: string): Promise<Iterable<string>> {
		// Even if no base provided, still need to call resolveUriToPath which may be overridden
		const root = this.resolveUriToPath(base || path2uri('', this.rootPath));
		const baseUri = path2uri('', normalizeDir(root)) + '/';
		const files = await new Promise<string[]>((resolve, reject) => {
			glob('*', { cwd: root, nodir: true, matchBase: true }, (err, matches) => err ? reject(err) : resolve(matches));
		});
		return iterate(files).map(file => baseUri + file.split('/').map(encodeURIComponent).join('/'));
	}

	async getTextDocumentContent(uri: string): Promise<string> {
		return fs.readFile(this.resolveUriToPath(uri), 'utf8');
	}
}

/**
 * Synchronizes a remote file system to an in-memory file system
 *
 * TODO: Implement Disposable with Disposer
 */
export class FileSystemUpdater {

	/**
	 * Promise for a pending or fulfilled structure fetch
	 */
	private structureFetch?: Promise<void>;

	/**
	 * Map from URI to Promise of pending or fulfilled content fetch
	 */
	private fetches = new Map<string, Promise<void>>();

	/**
	 * Limits concurrent fetches to not fetch thousands of files in parallel
	 */
	private concurrencyLimit = new Semaphore(100);

	constructor(private remoteFs: FileSystem, private inMemoryFs: InMemoryFileSystem) {}

	/**
	 * Fetches the file content for the given URI and adds the content to the in-memory file system
	 *
	 * @param uri URI of the file to fetch
	 * @param childOf A parent span for tracing
	 */
	async fetch(uri: string, childOf = new Span()): Promise<void> {
		// Limit concurrent fetches
		const promise = this.concurrencyLimit.execute(async () => {
			try {
				const content = await this.remoteFs.getTextDocumentContent(uri);
				this.inMemoryFs.add(uri, content);
				this.inMemoryFs.getContent(uri);
			} catch (err) {
				this.fetches.delete(uri);
				throw err;
			}
		});
		this.fetches.set(uri, promise);
		return promise;
	}

	/**
	 * Returns a promise that is resolved when the given URI has been fetched (at least once) to the in-memory file system.
	 * This function cannot be cancelled because multiple callers get the result of the same operation.
	 *
	 * @param uri URI of the file to ensure
	 * @param childOf A parent span for tracing
	 */
	async ensure(uri: string, childOf = new Span()): Promise<void> {
		const span = childOf.tracer().startSpan('Ensure file content', { childOf });
		span.addTags({ uri });
		try {
			return await (this.fetches.get(uri) || this.fetch(uri, span));
		} catch (err) {
			span.setTag('error', true);
			span.log({ 'event': 'error', 'error.object': err, 'message': err.message, 'stack': err.stack });
			throw err;
		} finally {
			span.finish();
		}
	}

	/**
	 * Fetches the file/directory structure for the given directory from the remote file system and saves it in the in-memory file system
	 *
	 * @param childOf A parent span for tracing
	 */
	fetchStructure(childOf = new Span()): Promise<void> {
		const promise = (async () => {
			const span = childOf.tracer().startSpan('Fetch workspace structure', { childOf });
			try {
				const uris = await this.remoteFs.getWorkspaceFiles(undefined, span);
				for (const uri of uris) {
					this.inMemoryFs.add(uri);
				}
			} catch (err) {
				this.structureFetch = undefined;
				span.setTag('error', true);
				span.log({ 'event': 'error', 'error.object': err, 'message': err.message, 'stack': err.stack });
				throw err;
			} finally {
				span.finish();
			}
		})();
		this.structureFetch = promise;
		return promise;
	}

	/**
	 * Returns a promise that is resolved as soon as the file/directory structure for the given directory has been synced
	 * from the remote file system to the in-memory file system (at least once)
	 *
	 * @param childOf A parent span for tracing
	 */
	async ensureStructure(childOf = new Span()) {
		const span = childOf.tracer().startSpan('Ensure workspace structure', { childOf });
		try {
			return await (this.structureFetch || this.fetchStructure(span));
		} catch (err) {
			span.setTag('error', true);
			span.log({ 'event': 'error', 'error.object': err, 'message': err.message, 'stack': err.stack });
			throw err;
		} finally {
			span.finish();
		}
	}

	/**
	 * Invalidates the content fetch cache of a file.
	 * The next call to `ensure` will do a refetch.
	 *
	 * @param uri URI of the file that changed
	 */
	invalidate(uri: string): void {
		this.fetches.delete(uri);
	}

	/**
	 * Invalidates the structure fetch cache.
	 * The next call to `ensureStructure` will do a refetch.
	 */
	invalidateStructure(): void {
		this.structureFetch = undefined;
	}
}
{'fraction_non_alphanumeric': 0.11808918118089182, 'fraction_numerical': 0.0013008600130086, 'mean_word_length': 3.9053527118043245, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3562361', 'n_tokens_mistral': 4333, 'n_tokens_neox': 3943, 'n_words': 1460}