<commit_before>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import nls = require('vs/nls');
import * as path from 'path';
import { createWriteStream } from 'fs';
import { Readable } from 'stream';
import { nfcall, ninvoke } from 'vs/base/common/async';
import { mkdirp, rimraf } from 'vs/base/node/pfs';
import { Promise, TPromise } from 'vs/base/common/winjs.base';
import { open as openZip, Entry, ZipFile } from 'yauzl';

export interface IExtractOptions {
	overwrite?: boolean;

	/**
	 * Source path within the ZIP archive. Only the files contained in this
	 * path will be extracted.
	 */
	sourcePath?: string;
}

interface IOptions {
	sourcePathRegex: RegExp;
}

function modeFromEntry(entry: Entry) {
	let attr = entry.externalFileAttributes >> 16 || 33188;

	return [448 /* S_IRWXU */, 56 /* S_IRWXG */, 7 /* S_IRWXO */]
		.map(mask => attr & mask)
		.reduce((a, b) => a + b, attr & 61440 /* S_IFMT */);
}

function extractEntry(zipfile: ZipFile, entry: Entry, targetPath: string, options: IOptions): Promise {
	const fileName = entry.fileName.replace(options.sourcePathRegex, '');
	const dirName = path.dirname(fileName);
	const targetDirName = path.join(targetPath, dirName);
	const targetFileName = path.join(targetPath, fileName);
	const mode = modeFromEntry(entry);

	return ninvoke(zipfile, zipfile.openReadStream, entry)
		.then(ostream => mkdirp(targetDirName)
			.then(() => new Promise((c, e) => {
				let istream = createWriteStream(targetFileName, { mode });
				istream.once('finish', () => c(null));
				istream.once('error', e);
				ostream.once('error', e);
				ostream.pipe(istream);
			})));
}

function extractZip(zipfile: ZipFile, targetPath: string, options: IOptions): Promise {
	return new Promise((c, e) => {
		const promises: Promise[] = [];

		zipfile.once('error', e);
		zipfile.on('entry', (entry: Entry) => {
			if (!options.sourcePathRegex.test(entry.fileName)) {
				return;
			}

			promises.push(extractEntry(zipfile, entry, targetPath, options));
		});
		zipfile.once('close', () => Promise.join(promises).done(c, e));
	});
}

export function extract(zipPath: string, targetPath: string, options: IExtractOptions): Promise {
	const sourcePathRegex = new RegExp(options.sourcePath ? `^${ options.sourcePath }` : '');

	let promise = nfcall<ZipFile>(openZip, zipPath);

	if (options.overwrite) {
		promise = promise.then(zipfile => { rimraf(targetPath); return zipfile; });
	}

	return promise.then(zipfile => extractZip(zipfile, targetPath, { sourcePathRegex }));
}<commit_msg>make extract zip sequence instead of parallel<commit_after>/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import nls = require('vs/nls');
import * as path from 'path';
import { createWriteStream } from 'fs';
import { Readable } from 'stream';
import { nfcall, ninvoke, sequence } from 'vs/base/common/async';
import { mkdirp, rimraf } from 'vs/base/node/pfs';
import { Promise, TPromise } from 'vs/base/common/winjs.base';
import { open as openZip, Entry, ZipFile } from 'yauzl';

export interface IExtractOptions {
	overwrite?: boolean;

	/**
	 * Source path within the ZIP archive. Only the files contained in this
	 * path will be extracted.
	 */
	sourcePath?: string;
}

interface IOptions {
	sourcePathRegex: RegExp;
}

function modeFromEntry(entry: Entry) {
	let attr = entry.externalFileAttributes >> 16 || 33188;

	return [448 /* S_IRWXU */, 56 /* S_IRWXG */, 7 /* S_IRWXO */]
		.map(mask => attr & mask)
		.reduce((a, b) => a + b, attr & 61440 /* S_IFMT */);
}

function extractEntry(stream: Readable, fileName: string, mode: number, targetPath: string, options: IOptions): Promise {
	const dirName = path.dirname(fileName);
	const targetDirName = path.join(targetPath, dirName);
	const targetFileName = path.join(targetPath, fileName);

	return mkdirp(targetDirName).then(() => new Promise((c, e) => {
		let istream = createWriteStream(targetFileName, { mode });
		istream.once('finish', () => c(null));
		istream.once('error', e);
		stream.once('error', e);
		stream.pipe(istream);
	}));
}

function extractZip(zipfile: ZipFile, targetPath: string, options: IOptions): Promise {
	return new Promise((c, e) => {
		const promiseFactory: { (): Promise; }[] = [];

		zipfile.once('error', e);
		zipfile.once('close', () => sequence(promiseFactory).then(c, e));
		zipfile.on('entry', (entry: Entry) => {
			if (!options.sourcePathRegex.test(entry.fileName)) {
				return;
			}

			const stream = ninvoke(zipfile, zipfile.openReadStream, entry);
			const fileName = entry.fileName.replace(options.sourcePathRegex, '');
			const mode = modeFromEntry(entry);

			promiseFactory.push(() => stream.then(stream => extractEntry(stream, fileName, mode, targetPath, options)));
		});
	});
}

export function extract(zipPath: string, targetPath: string, options: IExtractOptions): Promise {
	const sourcePathRegex = new RegExp(options.sourcePath ? `^${ options.sourcePath }` : '');

	let promise = nfcall<ZipFile>(openZip, zipPath);

	if (options.overwrite) {
		promise = promise.then(zipfile => { rimraf(targetPath); return zipfile; });
	}

	return promise.then(zipfile => extractZip(zipfile, targetPath, { sourcePathRegex }));
}
{'fraction_non_alphanumeric': 0.2068426536992011, 'fraction_numerical': 0.006252170892671066, 'mean_word_length': 3.3365963855421685, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '626899', 'n_tokens_mistral': 1832, 'n_tokens_neox': 1717, 'n_words': 454}