<commit_before>/* jslint node: true */
'use strict';

//	ENiGMA½
let Config		= require('./config.js').config;

//	base/modules
let fs			= require('fs');
let _			= require('lodash');
let pty			= require('ptyw.js');

module.exports = class ArchiveUtil {
	
	constructor() {
		this.archivers = {};
		this.longestSignature = 0;
	}

	init() {
		//
		//	Load configuration
		//
		if(_.has(Config, 'archivers')) {
			Object.keys(Config.archivers).forEach(archKey => {
				const arch = Config.archivers[archKey];
				if(!_.isString(arch.sig) || 
					!_.isString(arch.compressCmd) ||
					!_.isString(arch.decompressCmd) ||
					!_.isArray(arch.compressArgs) ||
					!_.isArray(arch.decompressArgs))
				{
					//	:TODO: log warning
					return;
				}

				const archiver = {
					compressCmd		: arch.compressCmd,
					compressArgs	: arch.compressArgs,
					decompressCmd	: arch.decompressCmd,
					decompressArgs	: arch.decompressArgs,
					sig				: new Buffer(arch.sig, 'hex'),
					offset			: arch.offset || 0,
				};
				
				this.archivers[archKey] = archiver;
				
				if(archiver.offset + archiver.sig.length > this.longestSignature) {
					this.longestSignature = archiver.offset + archiver.sig.length;
				}
			});
		}
	}
	
	getArchiver(archType) {
		if(!archType) {
			return;
		}
		
		archType = archType.toLowerCase();
		return this.archivers[archType];
	}
	
	haveArchiver(archType) {
		return this.getArchiver(archType) ? true : false;
	}

	detectType(path, cb) {
		fs.open(path, 'r', (err, fd) => {
			if(err) {
				cb(err);
				return;
			}
			
			let buf = new Buffer(this.longestSignature);
			fs.read(fd, buf, 0, buf.length, 0, (err, bytesRead) => {
				if(err) {
					cb(err);
					return;
				}

				//	return first match
				const detected = _.findKey(this.archivers, arch => {
					const lenNeeded = arch.offset + arch.sig.length;
					
					if(buf.length < lenNeeded) {
						return false;
					}

					const comp = buf.slice(arch.offset, arch.offset + arch.sig.length);
					return (arch.sig.equals(comp));
				});

				cb(detected ? null : new Error('Unknown type'), detected);
			});			
		});
	}

	compressTo(archType, archivePath, files, cb) {
		const archiver = this.getArchiver(archType);
		
		if(!archiver) {
			return cb(new Error(`Unknown archive type: ${archType}`));
		}

		let args = _.clone(archiver.compressArgs);	//	don't muck with orig
		for(let i = 0; i < args.length; ++i) {
			args[i] = args[i].format({
				archivePath	: archivePath,
				fileList	: files.join(' '),
			});
		}

		let comp = pty.spawn(archiver.compressCmd, args, this.getPtyOpts());

		comp.once('exit', exitCode => {
			cb(exitCode ? new Error(`Compression failed with exit code: ${exitCode}`) : null);
		});
	}

	extractTo(archivePath, extractPath, archType, cb) {
		const archiver = this.getArchiver(archType);
		
		if(!archiver) {
			return cb(new Error(`Unknown archive type: ${archType}`));
		}
		
		let args = _.clone(archiver.decompressArgs);	//	don't muck with orig
		for(let i = 0; i < args.length; ++i) {
			args[i] = args[i].format({
				archivePath		: archivePath,
				extractPath		: extractPath,
			});
		}
		
		let comp = pty.spawn(archiver.decompressCmd, args, this.getPtyOpts());
		
		comp.once('exit', exitCode => {
			cb(exitCode ? new Error(`Decompression failed with exit code: ${exitCode}`) : null);
		});		
	}
	
	getPtyOpts() {
		return {
			//	:TODO: cwd
			name	: 'enigma-archiver',
			cols	: 80,
			rows	: 24,
			env		: process.env,	
		};
	}
}<commit_msg>Attempt to detect when an archiver does not exist (HACK!)<commit_after>/* jslint node: true */
'use strict';

//	ENiGMA½
const Config	= require('./config.js').config;

//	base/modules
const fs		= require('fs');
const _			= require('lodash');
const pty		= require('ptyw.js');

module.exports = class ArchiveUtil {
	
	constructor() {
		this.archivers = {};
		this.longestSignature = 0;
	}

	init() {
		//
		//	Load configuration
		//
		if(_.has(Config, 'archivers')) {
			Object.keys(Config.archivers).forEach(archKey => {
				const arch = Config.archivers[archKey];
				if(!_.isString(arch.sig) || 
					!_.isString(arch.compressCmd) ||
					!_.isString(arch.decompressCmd) ||
					!_.isArray(arch.compressArgs) ||
					!_.isArray(arch.decompressArgs))
				{
					//	:TODO: log warning
					return;
				}

				const archiver = {
					compressCmd		: arch.compressCmd,
					compressArgs	: arch.compressArgs,
					decompressCmd	: arch.decompressCmd,
					decompressArgs	: arch.decompressArgs,
					sig				: new Buffer(arch.sig, 'hex'),
					offset			: arch.offset || 0,
				};
				
				this.archivers[archKey] = archiver;
				
				if(archiver.offset + archiver.sig.length > this.longestSignature) {
					this.longestSignature = archiver.offset + archiver.sig.length;
				}
			});
		}
	}
	
	getArchiver(archType) {
		if(!archType) {
			return;
		}
		
		archType = archType.toLowerCase();
		return this.archivers[archType];
	}
	
	haveArchiver(archType) {
		return this.getArchiver(archType) ? true : false;
	}

	detectType(path, cb) {
		fs.open(path, 'r', (err, fd) => {
			if(err) {
				cb(err);
				return;
			}
			
			let buf = new Buffer(this.longestSignature);
			fs.read(fd, buf, 0, buf.length, 0, (err, bytesRead) => {
				if(err) {
					return cb(err);
				}

				//	return first match
				const detected = _.findKey(this.archivers, arch => {
					const lenNeeded = arch.offset + arch.sig.length;
					
					if(bytesRead < lenNeeded) {
						return false;
					}

					const comp = buf.slice(arch.offset, arch.offset + arch.sig.length);
					return (arch.sig.equals(comp));
				});

				cb(detected ? null : new Error('Unknown type'), detected);
			});			
		});
	}

	spawnHandler(comp, action, cb) {
		//	pty.js doesn't currently give us a error when things fail,
		//	so we have this horrible, horrible hack:
		let err;
		comp.once('data', d => {
			if(_.isString(d) && d.startsWith('execvp(3) failed.: No such file or directory')) {
				err = new Error(`${action} failed: ${d.trim()}`);
			}
		});
		
		comp.once('exit', exitCode => {
			if(exitCode) {
				return cb(new Error(`${action} failed with exit code: ${exitCode}`));
			}
			if(err) {
				return cb(err);
			}
			return cb(null);
		});	
	}

	compressTo(archType, archivePath, files, cb) {
		const archiver = this.getArchiver(archType);
		
		if(!archiver) {
			return cb(new Error(`Unknown archive type: ${archType}`));
		}

		let args = _.clone(archiver.compressArgs);	//	don't muck with orig
		for(let i = 0; i < args.length; ++i) {
			args[i] = args[i].format({
				archivePath	: archivePath,
				fileList	: files.join(' '),
			});
		}

		let comp = pty.spawn(archiver.compressCmd, args, this.getPtyOpts());

		return this.spawnHandler(comp, 'Compression', cb);
	}

	extractTo(archivePath, extractPath, archType, cb) {
		const archiver = this.getArchiver(archType);
		
		if(!archiver) {
			return cb(new Error(`Unknown archive type: ${archType}`));
		}
		
		let args = _.clone(archiver.decompressArgs);	//	don't muck with orig
		for(let i = 0; i < args.length; ++i) {
			args[i] = args[i].format({
				archivePath		: archivePath,
				extractPath		: extractPath,
			});
		}
		
		let comp = pty.spawn(archiver.decompressCmd, args, this.getPtyOpts());

		return this.spawnHandler(comp, 'Decompression', cb);
	}
	
	getPtyOpts() {
		return {
			//	:TODO: cwd
			name	: 'enigma-archiver',
			cols	: 80,
			rows	: 24,
			env		: process.env,	
		};
	}
};
{'fraction_non_alphanumeric': 0.1599025974025974, 'fraction_numerical': 0.002840909090909091, 'mean_word_length': 2.884918549658434, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5141734', 'n_tokens_mistral': 3239, 'n_tokens_neox': 2640, 'n_words': 532}