<commit_before>/*
	MIT License http://www.opensource.org/licenses/mit-license.php
	Author <NAME> @sokra
*/
"use strict";

var EventEmitter = require("events").EventEmitter;
var async = require("neo-async");
var fs = require("graceful-fs");
var path = require("path");

var watcherManager = require("./watcherManager");

var FS_ACCURACY = 1000;


function withoutCase(str) {
	return str.toLowerCase();
}


function Watcher(directoryWatcher, filePath, startTime) {
	EventEmitter.call(this);
	this.directoryWatcher = directoryWatcher;
	this.path = filePath;
	this.startTime = startTime && +startTime;
	// TODO this.data seem to be only read, weird
	this.data = 0;
}

Watcher.prototype = Object.create(EventEmitter.prototype);
Watcher.prototype.constructor = Watcher;

Watcher.prototype.checkStartTime = function checkStartTime(mtime, initial) {
	if(typeof this.startTime !== "number") return !initial;
	var startTime = this.startTime;
	return startTime <= mtime;
};

Watcher.prototype.close = function close() {
	this.emit("closed");
};


function DirectoryWatcher(directoryPath, options) {
	EventEmitter.call(this);
	this.options = options;
	this.path = directoryPath;
	this.files = Object.create(null);
	this.directories = Object.create(null);
	this.initialScan = true;
	this.nestedWatching = false;
	this.initialScanRemoved = new Set();
	this.watchers = Object.create(null);
	this.parentWatcher = null;
	this.refs = 0;
	this._activeEvents = new Map();

	this.createWatcher();
	this.doInitialScan();
}
module.exports = DirectoryWatcher;

DirectoryWatcher.prototype = Object.create(EventEmitter.prototype);
DirectoryWatcher.prototype.constructor = DirectoryWatcher;

DirectoryWatcher.prototype.createWatcher = function createWatcher() {
	try {
		const options = this.options;
		var interval = typeof options.poll === "number" ? options.poll : undefined;
		// TODO options.poll
		// TODO options.ignored
		this.watcher = fs.watch(this.path);
		this.watcher.on("change", this.onWatchEvent.bind(this));
		//this.watcher.on("add", this.onFileAdded.bind(this));
		//this.watcher.on("addDir", this.onDirectoryAdded.bind(this));
		//this.watcher.on("change", this.onChange.bind(this));
		//this.watcher.on("unlink", this.onFileUnlinked.bind(this));
		//this.watcher.on("unlinkDir", this.onDirectoryUnlinked.bind(this));
		this.watcher.on("error", this.onWatcherError.bind(this));
	} catch(err) {
		this.onWatcherError(err);
	}
}

DirectoryWatcher.prototype.setMissing = function setMissing(itemPath, initial, type) {

	if(this.initialScan) {
		this.initialScanRemoved.add(directoryPath);
	}

	if(itemPath === this.path) { // Is this needed?
		console.log("YES it is needed");
		if(!initial && this.watchers[withoutCase(this.path)]) {
			this.watchers[withoutCase(this.path)].forEach(function(w) {
				w.emit("change", itemPath, w.data, initial ? "initial" : type);
			});
		}
		return
	}

	var oldDirectory = this.directories[itemPath];
	if(oldDirectory) {
		if(this.nestedWatching)
			this.directories[itemPath].close();
		delete this.directories[itemPath];
		if(!initial && this.watchers[withoutCase(this.path)]) {
			this.watchers[withoutCase(this.path)].forEach(function(w) {
				w.emit("change", itemPath, w.data, initial ? "initial" : type);
			});
		}
		if(this.watchers[withoutCase(itemPath) + "#directory"]) {
			this.watchers[withoutCase(itemPath) + "#directory"].forEach(function(w) {
				w.emit("change", itemPath, w.data, initial ? "initial" : type);
			});
		}
	}

	var oldFile = this.files[itemPath];
	if(oldFile) {
		var now = Date.now();
		this.files[itemPath] = [now, null];

		if(this.watchers[withoutCase(itemPath)]) {
			this.watchers[withoutCase(itemPath)].forEach(function(w) {
				w.emit("remove", type);
			});
		}
		if(this.watchers[withoutCase(this.path)]) {
			this.watchers[withoutCase(this.path)].forEach(function(w) {
				w.emit("change", itemPath, null, initial ? "initial" : type);
			});
		}
	}
};

DirectoryWatcher.prototype.setFileTime = function setFileTime(filePath, mtime, initial, type) {
	var now = Date.now();
	var old = this.files[filePath];

	this.files[filePath] = [initial ? Math.min(now, mtime) : now, mtime];

	// we add the fs accuracy to reach the maximum possible mtime
	mtime = mtime + FS_ACCURACY;

	if(!old) {
		if(this.watchers[withoutCase(filePath)]) {
			this.watchers[withoutCase(filePath)].forEach(function(w) {
				if(!initial || w.checkStartTime(mtime, initial)) {
					w.emit("change", mtime, initial ? "initial" : type);
				}
			});
		}
	} else if(!initial) {
		if(this.watchers[withoutCase(filePath)]) {
			this.watchers[withoutCase(filePath)].forEach(function(w) {
				w.emit("change", mtime, type);
			});
		}
	}
	if(this.watchers[withoutCase(this.path)]) {
		this.watchers[withoutCase(this.path)].forEach(function(w) {
			if(!initial || w.checkStartTime(mtime, initial)) {
				w.emit("change", filePath, mtime, initial ? "initial" : type);
			}
		});
	}
};

DirectoryWatcher.prototype.setDirectory = function setDirectory(directoryPath, initial, type) {
	if(directoryPath === this.path) {
		if(!initial && this.watchers[withoutCase(this.path)]) {
			this.watchers[withoutCase(this.path)].forEach(function(w) {
				w.emit("change", directoryPath, w.data, initial ? "initial" : type);
			});
		}
	} else {
		var old = this.directories[directoryPath];
		if(!old) {
			if(this.nestedWatching) {
				this.createNestedWatcher(directoryPath);
			} else {
				this.directories[directoryPath] = true;
			}
			if(!initial && this.watchers[withoutCase(this.path)]) {
				this.watchers[withoutCase(this.path)].forEach(function(w) {
					w.emit("change", directoryPath, w.data, initial ? "initial" : type);
				});
			}
			if(this.watchers[withoutCase(directoryPath) + "#directory"]) {
				this.watchers[withoutCase(directoryPath) + "#directory"].forEach(function(w) {
					w.emit("change", w.data, initial ? "initial" : type);
				});
			}
		}
	}
};

DirectoryWatcher.prototype.createNestedWatcher = function(directoryPath) {
	this.directories[directoryPath] = watcherManager.watchDirectory(directoryPath, this.options, 1);
	this.directories[directoryPath].on("change", function(filePath, mtime, type) {
		if(this.watchers[withoutCase(this.path)]) {
			this.watchers[withoutCase(this.path)].forEach(function(w) {
				if(w.checkStartTime(mtime, false)) {
					w.emit("change", filePath, mtime, type);
				}<commit_msg>remove extra check for initial<commit_after>/*
	MIT License http://www.opensource.org/licenses/mit-license.php
	Author <NAME> @sokra
*/
"use strict";

var EventEmitter = require("events").EventEmitter;
var async = require("neo-async");
var fs = require("graceful-fs");
var path = require("path");

var watcherManager = require("./watcherManager");

var FS_ACCURACY = 1000;


function withoutCase(str) {
	return str.toLowerCase();
}


function Watcher(directoryWatcher, filePath, startTime) {
	EventEmitter.call(this);
	this.directoryWatcher = directoryWatcher;
	this.path = filePath;
	this.startTime = startTime && +startTime;
	// TODO this.data seem to be only read, weird
	this.data = 0;
}

Watcher.prototype = Object.create(EventEmitter.prototype);
Watcher.prototype.constructor = Watcher;

Watcher.prototype.checkStartTime = function checkStartTime(mtime, initial) {
	if(typeof this.startTime !== "number") return !initial;
	var startTime = this.startTime;
	return startTime <= mtime;
};

Watcher.prototype.close = function close() {
	this.emit("closed");
};


function DirectoryWatcher(directoryPath, options) {
	EventEmitter.call(this);
	this.options = options;
	this.path = directoryPath;
	this.files = Object.create(null);
	this.directories = Object.create(null);
	this.initialScan = true;
	this.nestedWatching = false;
	this.initialScanRemoved = new Set();
	this.watchers = Object.create(null);
	this.parentWatcher = null;
	this.refs = 0;
	this._activeEvents = new Map();

	this.createWatcher();
	this.doInitialScan();
}
module.exports = DirectoryWatcher;

DirectoryWatcher.prototype = Object.create(EventEmitter.prototype);
DirectoryWatcher.prototype.constructor = DirectoryWatcher;

DirectoryWatcher.prototype.createWatcher = function createWatcher() {
	try {
		const options = this.options;
		var interval = typeof options.poll === "number" ? options.poll : undefined;
		// TODO options.poll
		// TODO options.ignored
		this.watcher = fs.watch(this.path);
		this.watcher.on("change", this.onWatchEvent.bind(this));
		//this.watcher.on("add", this.onFileAdded.bind(this));
		//this.watcher.on("addDir", this.onDirectoryAdded.bind(this));
		//this.watcher.on("change", this.onChange.bind(this));
		//this.watcher.on("unlink", this.onFileUnlinked.bind(this));
		//this.watcher.on("unlinkDir", this.onDirectoryUnlinked.bind(this));
		this.watcher.on("error", this.onWatcherError.bind(this));
	} catch(err) {
		this.onWatcherError(err);
	}
}

DirectoryWatcher.prototype.setMissing = function setMissing(itemPath, initial, type) {

	if(this.initialScan) {
		this.initialScanRemoved.add(directoryPath);
	}

	if(itemPath === this.path) { // Is this needed?
		console.log("YES it is needed");
		if(!initial && this.watchers[withoutCase(this.path)]) {
			this.watchers[withoutCase(this.path)].forEach(function(w) {
				w.emit("change", itemPath, w.data, type);
			});
		}
		return
	}

	var oldDirectory = this.directories[itemPath];
	if(oldDirectory) {
		if(this.nestedWatching)
			this.directories[itemPath].close();
		delete this.directories[itemPath];
		if(!initial && this.watchers[withoutCase(this.path)]) {
			this.watchers[withoutCase(this.path)].forEach(function(w) {
				w.emit("change", itemPath, w.data, type);
			});
		}
		if(this.watchers[withoutCase(itemPath) + "#directory"]) {
			this.watchers[withoutCase(itemPath) + "#directory"].forEach(function(w) {
				w.emit("change", itemPath, w.data, type);
			});
		}
	}

	var oldFile = this.files[itemPath];
	if(oldFile) {
		var now = Date.now();
		this.files[itemPath] = [now, null];

		if(this.watchers[withoutCase(itemPath)]) {
			this.watchers[withoutCase(itemPath)].forEach(function(w) {
				w.emit("remove", type);
			});
		}
		if(this.watchers[withoutCase(this.path)]) {
			this.watchers[withoutCase(this.path)].forEach(function(w) {
				w.emit("change", itemPath, null, type);
			});
		}
	}
};

DirectoryWatcher.prototype.setFileTime = function setFileTime(filePath, mtime, initial, type) {
	var now = Date.now();
	var old = this.files[filePath];

	this.files[filePath] = [initial ? Math.min(now, mtime) : now, mtime];

	// we add the fs accuracy to reach the maximum possible mtime
	mtime = mtime + FS_ACCURACY;

	if(!old) {
		if(this.watchers[withoutCase(filePath)]) {
			this.watchers[withoutCase(filePath)].forEach(function(w) {
				if(!initial || w.checkStartTime(mtime, initial)) {
					w.emit("change", mtime, type);
				}
			});
		}
	} else if(!initial) {
		if(this.watchers[withoutCase(filePath)]) {
			this.watchers[withoutCase(filePath)].forEach(function(w) {
				w.emit("change", mtime, type);
			});
		}
	}
	if(this.watchers[withoutCase(this.path)]) {
		this.watchers[withoutCase(this.path)].forEach(function(w) {
			if(!initial || w.checkStartTime(mtime, initial)) {
				w.emit("change", filePath, mtime, type);
			}
		});
	}
};

DirectoryWatcher.prototype.setDirectory = function setDirectory(directoryPath, initial, type) {
	if(directoryPath === this.path) {
		if(!initial && this.watchers[withoutCase(this.path)]) {
			this.watchers[withoutCase(this.path)].forEach(function(w) {
				w.emit("change", directoryPath, w.data, type);
			});
		}
	} else {
		var old = this.directories[directoryPath];
		if(!old) {
			if(this.nestedWatching) {
				this.createNestedWatcher(directoryPath);
			} else {
				this.directories[directoryPath] = true;
			}
			if(!initial && this.watchers[withoutCase(this.path)]) {
				this.watchers[withoutCase(this.path)].forEach(function(w) {
					w.emit("change", directoryPath, w.data, type);
				});
			}
			if(this.watchers[withoutCase(directoryPath) + "#directory"]) {
				this.watchers[withoutCase(directoryPath) + "#directory"].forEach(function(w) {
					w.emit("change", w.data, type);
				});
			}
		}
	}
};

DirectoryWatcher.prototype.createNestedWatcher = function(directoryPath) {
	this.directories[directoryPath] = watcherManager.watchDirectory(directoryPath, this.options, 1);
	this.directories[directoryPath].on("change", function(filePath, mtime, type) {
		if(this.watchers[withoutCase(this.path)]) {
			this.watchers[withoutCase(this.path)].forEach(function(w) {
				if(w.checkStartTime(mtime, false)) {
					w.emit("change", filePath, mtime, type);
				}
{'fraction_non_alphanumeric': 0.16386387184187456, 'fraction_numerical': 0.0011158045747987567, 'mean_word_length': 4.801202034211743, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3359015', 'n_tokens_mistral': 4621, 'n_tokens_neox': 4231, 'n_words': 678}