<commit_before>package hxd.fs;

#if (sys || nodejs)

@:allow(hxd.fs.LocalFileSystem)
@:allow(hxd.fs.FileConverter)
@:access(hxd.fs.LocalFileSystem)
class LocalEntry extends FileEntry {

	var fs : LocalFileSystem;
	var relPath : String;
	var file : String;
	var originalFile : String;

	function new(fs, name, relPath, file) {
		this.fs = fs;
		this.name = name;
		this.relPath = relPath;
		this.file = file;
	}

	override function getBytes() : haxe.io.Bytes {
		return sys.io.File.getBytes(file);
	}

	override function readBytes( out : haxe.io.Bytes, outPos : Int, pos : Int, len : Int ) : Int {
		var f = sys.io.File.read(file);
		f.seek(pos, SeekBegin);
		var tot = f.readBytes(out, outPos, len);
		f.close();
		return tot;
	}

	var isDirCached : Null<Bool>;
	override function get_isDirectory() : Bool {
		if( isDirCached != null ) return isDirCached;
		return isDirCached = sys.FileSystem.isDirectory(file);
	}

	override function load( ?onReady : Void -> Void ) : Void {
		#if macro
		onReady();
		#else
		if( onReady != null ) haxe.Timer.delay(onReady, 1);
		#end
	}

	override function loadBitmap( onLoaded : hxd.fs.LoadedBitmap -> Void ) : Void {
		#if js
		#if (multidriver && !macro)
		var engine = h3d.Engine.getCurrent(); // hide
		#end
		var image = new js.html.Image();
		image.onload = function(_) {
			#if (multidriver && !macro)
			if( engine.driver == null ) return;
			engine.setCurrent();
			#end
			onLoaded(new LoadedBitmap(image));
		};
		image.src = "file://"+file;
		#elseif (macro || dataOnly)
		throw "Not implemented";
		#else
		var bmp = new hxd.res.Image(this).toBitmap();
		onLoaded(new hxd.fs.LoadedBitmap(bmp));
		#end
	}

	override function get_path() {
		return relPath == null ? "<root>" : relPath;
	}

	override function exists( name : String ) {
		return fs.exists(relPath == null ? name : relPath + "/" + name);
	}

	override function get( name : String ) {
		return fs.get(relPath == null ? name : relPath + "/" + name);
	}

	override function get_size() {
		return sys.FileSystem.stat(file).size;
	}

	override function iterator() {
		var arr = new Array<FileEntry>();
		for( f in sys.FileSystem.readDirectory(file) ) {
			switch( f ) {
			case ".svn", ".git" if( sys.FileSystem.isDirectory(file+"/"+f) ):
				continue;
			case ".tmp" if( this == fs.root ):
				continue;
			default:
				arr.push(fs.open(relPath == null ? f : relPath + "/" + f,false));
			}
		}
		return new hxd.impl.ArrayIterator(arr);
	}

	var watchCallback : Void -> Void;
	#if (hl && (hl_ver >= version("1.12.0")))
	var watchHandle : hl.uv.Fs;
	var lastChanged : Float = 0;
	#else
	var watchTime : Float;
	#end

	static var WATCH_INDEX = 0;
	static var WATCH_LIST : Array<LocalEntry> = null;
	static var tmpDir : String = null;

	inline function getModifTime(){
		return sys.FileSystem.stat(originalFile != null ? originalFile : file).mtime.getTime();
	}

	#if hl
	#if (hl_ver >= version("1.12.0")) @:hlNative("std","file_is_locked") #end static function fileIsLocked( b : hl.Bytes ) { return false; }
	#end

	#if (!hl || hl_ver < version("1.12.0"))
	static function checkFiles() {
		var filesToCheck = Math.ceil(WATCH_LIST.length / 60);
		if( filesToCheck > LocalFileSystem.FILES_CHECK_MAX )
			filesToCheck = LocalFileSystem.FILES_CHECK_MAX;
		for( i in 0...filesToCheck )
			checkNext();
	}

	static function checkNext() {
		var w = WATCH_LIST[WATCH_INDEX++];
		if( w == null ) {
			WATCH_INDEX = 0;
			return;
		}
		var t = try w.getModifTime() catch( e : Dynamic ) return;
		if( t == w.watchTime ) return;

		#if (sys || nodejs)
		if( tmpDir == null ) {
			tmpDir = Sys.getEnv("TEMP");
			if( tmpDir == null ) tmpDir = Sys.getEnv("TMPDIR");
			if( tmpDir == null ) tmpDir = Sys.getEnv("TMP");
		}
		var lockFile = tmpDir+"/"+w.file.split("/").pop()+".lock";
		if( sys.FileSystem.exists(lockFile) ) return;
		if( !w.isDirectory )
		try {
			#if nodejs
			var cst = js.node.Fs.constants;
			var fid = js.node.Fs.openSync(w.file, cast (cst.O_RDONLY | cst.O_EXCL | 0x10000000));
			js.node.Fs.closeSync(fid);
			#elseif hl
			if( fileIsLocked(@:privateAccess Sys.getPath(w.file)) )
				return;
			#end
		}catch( e : Dynamic ) return;
		#end

		w.watchTime = t;
		w.watchCallback();
	}
	#end

	override function watch( onChanged : Null < Void -> Void > ) {
		if( onChanged == null ) {
			if( watchCallback != null ) {
				WATCH_LIST.remove(this);
				watchCallback = null;
				#if (hl && (hl_ver >= version("1.12.0")))
				watchHandle.close();
				watchHandle = null;
				#end
			}
			return;
		}
		if( watchCallback == null ) {
			if( WATCH_LIST == null ) {
				WATCH_LIST = [];
				#if (!macro && (!hl || (hl_ver < version("1.12.0"))))
				haxe.MainLoop.add(checkFiles);
				#end
			}
			var path = path;
			for( w in WATCH_LIST )
				if( w.path == path ) {
					#if (hl && (hl_ver >= version("1.12.0")))
					if(w.watchHandle != null) {
						w.watchHandle.close();
						w.watchHandle = null;
					}
					#end
					w.watchCallback = null;
					WATCH_LIST.remove(w);
				}
			WATCH_LIST.push(this);
		}
		#if (hl && (hl_ver >= version("1.12.0")))
		watchHandle = new hl.uv.Fs(originalFile, function(ev) {
			switch(ev) {
				case Change:
					// For any reason, uvfs send 2 events for a single change
					if(Sys.time() - lastChanged > 0.1 ) {
						lastChanged = Sys.time();
						haxe.Timer.delay(onChanged, 100);
					}
				case Rename:
			}
		});
		#else
		watchTime = getModifTime();
		#end
		watchCallback = function() { fs.convert.run(this); onChanged(); }
	}

}

class LocalFileSystem implements FileSystem {

	var root : FileEntry;
	var fileCache = new Map<String,{r:LocalEntry}>();
	public var baseDir(default,null) : String;
	public var convert(default,null) : FileConverter;
	static var isWindows = Sys.systemName() == "Windows";
	public static var FILES_CHECK_MAX = 5;<commit_msg>Fix calling watch several times on same file, prevent spam (#1054)<commit_after>package hxd.fs;

#if (sys || nodejs)

@:allow(hxd.fs.LocalFileSystem)
@:allow(hxd.fs.FileConverter)
@:access(hxd.fs.LocalFileSystem)
class LocalEntry extends FileEntry {

	var fs : LocalFileSystem;
	var relPath : String;
	var file : String;
	var originalFile : String;

	function new(fs, name, relPath, file) {
		this.fs = fs;
		this.name = name;
		this.relPath = relPath;
		this.file = file;
	}

	override function getBytes() : haxe.io.Bytes {
		return sys.io.File.getBytes(file);
	}

	override function readBytes( out : haxe.io.Bytes, outPos : Int, pos : Int, len : Int ) : Int {
		var f = sys.io.File.read(file);
		f.seek(pos, SeekBegin);
		var tot = f.readBytes(out, outPos, len);
		f.close();
		return tot;
	}

	var isDirCached : Null<Bool>;
	override function get_isDirectory() : Bool {
		if( isDirCached != null ) return isDirCached;
		return isDirCached = sys.FileSystem.isDirectory(file);
	}

	override function load( ?onReady : Void -> Void ) : Void {
		#if macro
		onReady();
		#else
		if( onReady != null ) haxe.Timer.delay(onReady, 1);
		#end
	}

	override function loadBitmap( onLoaded : hxd.fs.LoadedBitmap -> Void ) : Void {
		#if js
		#if (multidriver && !macro)
		var engine = h3d.Engine.getCurrent(); // hide
		#end
		var image = new js.html.Image();
		image.onload = function(_) {
			#if (multidriver && !macro)
			if( engine.driver == null ) return;
			engine.setCurrent();
			#end
			onLoaded(new LoadedBitmap(image));
		};
		image.src = "file://"+file;
		#elseif (macro || dataOnly)
		throw "Not implemented";
		#else
		var bmp = new hxd.res.Image(this).toBitmap();
		onLoaded(new hxd.fs.LoadedBitmap(bmp));
		#end
	}

	override function get_path() {
		return relPath == null ? "<root>" : relPath;
	}

	override function exists( name : String ) {
		return fs.exists(relPath == null ? name : relPath + "/" + name);
	}

	override function get( name : String ) {
		return fs.get(relPath == null ? name : relPath + "/" + name);
	}

	override function get_size() {
		return sys.FileSystem.stat(file).size;
	}

	override function iterator() {
		var arr = new Array<FileEntry>();
		for( f in sys.FileSystem.readDirectory(file) ) {
			switch( f ) {
			case ".svn", ".git" if( sys.FileSystem.isDirectory(file+"/"+f) ):
				continue;
			case ".tmp" if( this == fs.root ):
				continue;
			default:
				arr.push(fs.open(relPath == null ? f : relPath + "/" + f,false));
			}
		}
		return new hxd.impl.ArrayIterator(arr);
	}

	var watchCallback : Void -> Void;
	#if (hl && (hl_ver >= version("1.12.0")))
	var watchHandle : hl.uv.Fs;
	var lastChanged : Float = 0;
	var onChangedDelay : haxe.Timer;
	#else
	var watchTime : Float;
	#end

	static var WATCH_INDEX = 0;
	static var WATCH_LIST : Array<LocalEntry> = null;
	static var tmpDir : String = null;

	inline function getModifTime(){
		return sys.FileSystem.stat(originalFile != null ? originalFile : file).mtime.getTime();
	}

	#if hl
	#if (hl_ver >= version("1.12.0")) @:hlNative("std","file_is_locked") #end static function fileIsLocked( b : hl.Bytes ) { return false; }
	#end

	#if (!hl || hl_ver < version("1.12.0"))
	static function checkFiles() {
		var filesToCheck = Math.ceil(WATCH_LIST.length / 60);
		if( filesToCheck > LocalFileSystem.FILES_CHECK_MAX )
			filesToCheck = LocalFileSystem.FILES_CHECK_MAX;
		for( i in 0...filesToCheck )
			checkNext();
	}

	static function checkNext() {
		var w = WATCH_LIST[WATCH_INDEX++];
		if( w == null ) {
			WATCH_INDEX = 0;
			return;
		}
		var t = try w.getModifTime() catch( e : Dynamic ) return;
		if( t == w.watchTime ) return;

		#if (sys || nodejs)
		if( tmpDir == null ) {
			tmpDir = Sys.getEnv("TEMP");
			if( tmpDir == null ) tmpDir = Sys.getEnv("TMPDIR");
			if( tmpDir == null ) tmpDir = Sys.getEnv("TMP");
		}
		var lockFile = tmpDir+"/"+w.file.split("/").pop()+".lock";
		if( sys.FileSystem.exists(lockFile) ) return;
		if( !w.isDirectory )
		try {
			#if nodejs
			var cst = js.node.Fs.constants;
			var fid = js.node.Fs.openSync(w.file, cast (cst.O_RDONLY | cst.O_EXCL | 0x10000000));
			js.node.Fs.closeSync(fid);
			#elseif hl
			if( fileIsLocked(@:privateAccess Sys.getPath(w.file)) )
				return;
			#end
		}catch( e : Dynamic ) return;
		#end

		w.watchTime = t;
		w.watchCallback();
	}
	#end

	override function watch( onChanged : Null < Void -> Void > ) {
		if( onChanged == null ) {
			if( watchCallback != null ) {
				WATCH_LIST.remove(this);
				watchCallback = null;
				#if (hl && (hl_ver >= version("1.12.0")))
				watchHandle.close();
				watchHandle = null;
				#end
			}
			return;
		}
		if( watchCallback == null ) {
			if( WATCH_LIST == null ) {
				WATCH_LIST = [];
				#if (!macro && (!hl || (hl_ver < version("1.12.0"))))
				haxe.MainLoop.add(checkFiles);
				#end
			}
			var path = path;
			for( w in WATCH_LIST )
				if( w.path == path ) {
					#if (hl && (hl_ver >= version("1.12.0")))
					if(w.watchHandle != null) {
						w.watchHandle.close();
						w.watchHandle = null;
					}
					#end
					w.watchCallback = null;
					WATCH_LIST.remove(w);
				}
			WATCH_LIST.push(this);
		}
		#if (hl && (hl_ver >= version("1.12.0")))
		if(watchHandle != null)
			watchHandle.close();
		watchHandle = new hl.uv.Fs(originalFile, function(ev) {
			switch(ev) {
				case Change:
					lastChanged = Sys.time();
					if(onChangedDelay != null)
						onChangedDelay.stop();
					onChangedDelay = haxe.Timer.delay(onChanged, 10);
				case Rename:
			}
		});
		#else
		watchTime = getModifTime();
		#end
		watchCallback = function() { fs.convert.run(this); onChanged(); }
	}

}

class LocalFileSystem implements FileSystem {

	var root : FileEntry;
	var fileCache = new Map<String,{r:LocalEntry}>();
	public var baseDir(default,null) : String;
	public var convert(default,null) : FileConverter;
	static var isWindows = Sys.systemName() == "Windows";
	public static var FILES_CHECK_MAX = 5;
{'fraction_non_alphanumeric': 0.15846061950678386, 'fraction_numerical': 0.008874477344483317, 'mean_word_length': 3.2633684976355037, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1042368', 'n_tokens_mistral': 4920, 'n_tokens_neox': 4275, 'n_words': 1021}