<commit_before>///<reference path="../.d.ts"/>

import fs = require("fs");
import unzip = require("unzip");
import Future = require("fibers/future");
import path = require("path");
import util = require("util");
import rimraf = require("rimraf");
import helpers = require("./helpers");

export class FileSystem implements IFileSystem {
	private _stat = Future.wrap(fs.stat);
	private _readFile = Future.wrap(fs.readFile);
	private _writeFile = Future.wrap<void>(fs.writeFile);
	private _readdir = Future.wrap(fs.readdir);
	private _chmod = Future.wrap(fs.chmod);

//TODO: try 'archiver' module for zipping
	public zipFiles(zipFile: string, files: string[], zipPathCallback: (path: string) => string): IFuture<void> {
		//we are resolving it here instead of in the constructor, because config has dependency on file system and config shouldn't require logger
		var $logger = $injector.resolve("logger");
		var zipstream = require("zipstream");
		var zip = zipstream.createZip({ level: 9 });
		var outFile = fs.createWriteStream(zipFile);
		zip.pipe(outFile);

		var result = new Future<void>();
		outFile.on("error", (err) => result.throw(err));

		var fileIdx = -1;
		var zipCallback = () => {
			fileIdx++;
			if (fileIdx < files.length) {
				var file = files[fileIdx];

				var relativePath = zipPathCallback(file);
				relativePath = relativePath.replace(/\\/g, "/");
				$logger.trace("zipping as '%s' file '%s'", relativePath, file);

				zip.addFile(
					fs.createReadStream(file),
					{ name: relativePath },
					zipCallback);
			} else {
				outFile.on("finish", () => result.return());

				zip.finalize((bytesWritten) => {
					$logger.debug("zipstream: %d bytes written", bytesWritten);
					outFile.end();
				});
			}
		};
		zipCallback();

		return result;
	}

	public unzip(zipFile: string, destinationDir: string): IFuture<void> {
		if (helpers.isDarwin()) {
			var $childProcess = $injector.resolve("$childProcess");

			this.createDirectory(destinationDir).wait();
			var unzipProc = $childProcess.spawn('unzip', ['-u', zipFile, '-d', destinationDir],
				{ stdio: "ignore", detached: true });
			return this.futureFromEvent(unzipProc, "close");
		}
		else {
			return this.futureFromEvent(
				this.createReadStream(zipFile)
					.pipe(unzip.Extract({ path: destinationDir })), "close");
		}
	}

	public exists(path: string): IFuture<boolean> {
		var future = new Future<boolean>();
		fs.exists(path, (exists: boolean) => future.return(exists));
		return future;
	}

	public deleteFile(path: string): IFuture<void> {
		var future = new Future<void>();
		fs.unlink(path, (err) => {
			if (err && err.code !== "ENOENT") {  // ignore "file doesn't exist" error
				future.throw(err);
			} else {
				future.return();
			}
		})<commit_msg>Make sure code with wait()s work<commit_after>///<reference path="../.d.ts"/>

import fs = require("fs");
import unzip = require("unzip");
import Future = require("fibers/future");
import path = require("path");
import util = require("util");
import rimraf = require("rimraf");
import helpers = require("./helpers");

export class FileSystem implements IFileSystem {
	private _stat = Future.wrap(fs.stat);
	private _readFile = Future.wrap(fs.readFile);
	private _writeFile = Future.wrap<void>(fs.writeFile);
	private _readdir = Future.wrap(fs.readdir);
	private _chmod = Future.wrap(fs.chmod);

//TODO: try 'archiver' module for zipping
	public zipFiles(zipFile: string, files: string[], zipPathCallback: (path: string) => string): IFuture<void> {
		//we are resolving it here instead of in the constructor, because config has dependency on file system and config shouldn't require logger
		var $logger = $injector.resolve("logger");
		var zipstream = require("zipstream");
		var zip = zipstream.createZip({ level: 9 });
		var outFile = fs.createWriteStream(zipFile);
		zip.pipe(outFile);

		var result = new Future<void>();
		outFile.on("error", (err) => result.throw(err));

		var fileIdx = -1;
		var zipCallback = () => {
			fileIdx++;
			if (fileIdx < files.length) {
				var file = files[fileIdx];

				var relativePath = zipPathCallback(file);
				relativePath = relativePath.replace(/\\/g, "/");
				$logger.trace("zipping as '%s' file '%s'", relativePath, file);

				zip.addFile(
					fs.createReadStream(file),
					{ name: relativePath },
					zipCallback);
			} else {
				outFile.on("finish", () => result.return());

				zip.finalize((bytesWritten) => {
					$logger.debug("zipstream: %d bytes written", bytesWritten);
					outFile.end();
				});
			}
		};
		zipCallback();

		return result;
	}

	public unzip(zipFile: string, destinationDir: string): IFuture<void> {
		return (() => {
			if (helpers.isDarwin()) {
				var $childProcess = $injector.resolve("$childProcess");

				this.createDirectory(destinationDir).wait();
				var unzipProc = $childProcess.spawn('unzip', ['-u', zipFile, '-d', destinationDir],
					{ stdio: "ignore", detached: true });
				this.futureFromEvent(unzipProc, "close").wait();
			}
			else {
				this.futureFromEvent(
					this.createReadStream(zipFile)
						.pipe(unzip.Extract({ path: destinationDir })), "close").wait();
			}
		}).future<void>()();
	}

	public exists(path: string): IFuture<boolean> {
		var future = new Future<boolean>();
		fs.exists(path, (exists: boolean) => future.return(exists));
		return future;
	}

	public deleteFile(path: string): IFuture<void> {
		var future = new Future<void>();
		fs.unlink(path, (err) => {
			if (err && err.code !== "ENOENT") {  // ignore "file doesn't exist" error
				future.throw(err);
			} else {
				future.return();
			}
		})
{'fraction_non_alphanumeric': 0.16714387974230493, 'fraction_numerical': 0.0007158196134574087, 'mean_word_length': 4.044223826714801, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '453475', 'n_tokens_mistral': 2019, 'n_tokens_neox': 1760, 'n_words': 435}