<commit_before>/* jslint node: true */

var fs             = require('fs');
var path           = require('path');
var util           = require('util');
var fm             = require('front-matter');
var sanitize       = require('sanitize-filename');
var es             = require('event-stream');
var cheerio        = require('cheerio');
var jade           = require('jade');
var wkhtmltopdf    = require('wkhtmltopdf');
var concatStream   = require('concat-stream');
var EventEmitter   = require('eventemitter2').EventEmitter2;

const VERBOSITY_WARNING = 2;
const VERBOSITY_DEBUG   = 5;

function Documark (basePath) {
	var config  = null;
	var content = null;
	var plugins = [];
	var debug   = 0;
	var file, basePath;

	function setBasePath (newPath) {
		var filename = 'document.jade';

		newPath = path.resolve(newPath || '.');

		if (fs.statSync(newPath).isFile()) {
			file = newPath;
		} else if (fs.existsSync(path.join(newPath, filename))) {
			file = path.join(newPath, filename);
		} else {
			throw new Error('No ' + filename + ' file found in ' + newPath);
		}

		basePath = path.dirname(file);
	}

	// Make Documark an event emitter
	EventEmitter.call(this);

	this.setMaxListeners(100);

	// Methods
	this.path = function (newPath) {
		if (typeof newPath !== 'undefined') {
			setBasePath(newPath);
		}

		return basePath;
	};

	this.file = function (newFile) {
		if (typeof newFile !== 'undefined') {
			setBasePath(newFile);
		}

		return file;
	};

	this.config = function () {
		if (config === null) {
			this.load();
		}

		config.pdf = (config.pdf || {});

		return config;
	};

	this.content = function () {
		if (config === null) {
			this.load();
		}

		return content;
	};

	this.plugins = function () {
		return plugins;
	};

	this.verbosity = function (level) {
		if (typeof level !== 'undefined') {
			debug = (parseInt(level, 10) || 0);
		}

		return debug;
	};

	this.isVerbosity = function (level) {
		return this.verbosity() >= level;
	};

	this.load = function () {
		var basePath = this.path();

		// Get file contents
		var rawContent = fs.readFileSync(this.file(), { encoding: 'utf-8' });

		// Get config and content
		var data = fm(rawContent);

		rawContent = null;
		config     = (data.attributes || {});
		content    = data.body;

		// Load config.json if no front matter is available
		if ( ! Object.keys(config).length) {
			var configFile = path.join(basePath, 'config.json');

			if (fs.existsSync(configFile)) {
				config = require(configFile);
			}
		}
	};

	this.outputFile = function () {
		var filename = (this.config().filename || 'Document.pdf');

		return path.join(this.path(), sanitize(filename));
	};

	this.applyPlugins = function ($, plugins, cb) {
		var self = this;
		var plugin;

		// Handle plugins one by one to allow adding plugins while iterating
		var nextPlugin = function (err) {
			if (err) {
				cb('Error in plugin ' +
					(plugin ? plugin.name || util.inspect(plugin) : '<unknown>') +
					': ' + err
				);
				return;
			}
			if ( ! plugins.length) {
				cb(null, $);
				return;
			}

			plugin = plugins.shift();

			if (typeof plugin !== 'function') {
				// Load relative to document path
				if (plugin.indexOf('./') === 0) {
					plugin = path.join(self.path(), plugin);
				}

				plugin = require(plugin);
			}
			if (self.isVerbosity(VERBOSITY_DEBUG)) {
				console.log('Processing plugin: ' + util.inspect(plugin.name || plugin));
			}

			plugin($, self, nextPlugin);
		}

		nextPlugin();
	};

	this.contentStream = function () {
		return es.through().pause().queue(this.content());
	};

	this.parserStream = function () {
		var outputFile = this.outputFile();
		var basePath   = this.path();
		var config     = this.config();

		return es.map(function (data, cb) {
			var options = {
				filename: outputFile,
				basedir: basePath
			};

			cb(null, jade.compile(data, options)(config));
		});
	};

	this.pluginStream = function () {
		var self = this;

		return es.through(function (data) {
			var $ = cheerio.load(data);

			// Determine initial plugins
			plugins = self.config().plugins;

			if ( ! Array.isArray(plugins) ) {
				plugins = [];
			}

			// Handle plugins
			self.applyPlugins($, plugins, function (err, $) {
				if (err) {
					throw new Error(err);
				}

				this.emit('data', $.html());
			}.bind(this));
		});
	};

	this.compileStream = function (config) {
		var self   = this;
		var stream = es.map(function (data, cb) {
			self.emit('pre-compile');

			wkhtmltopdf(data, self.config().pdf, function (code, signal) {
				if (code || signal) {
					throw new Error('Error generating PDF (code: ' +
						code + ', signal: ' + signal + ')!');
				}
			}).pipe(concatStream(function (data) {
				cb(null, data);
				stream.end();
			}));
		});

		return stream;
	};

	this.outputStream = function () {
		var self = this;

		return fs.createWriteStream(this.outputFile())
			.on('finish', function () {
				self.emit('post-compile');
			})
			;
	};

	this.compile = function () {
		this.load();

		var input = this.contentStream();
		var self  = this;

		input
			.pipe(this.parserStream())
			.pipe(this.pluginStream())
			.pipe(this.compileStream())
			.pipe(this.outputStream())
			;

		input.resume();
	};

	// Set path
	this.path(basePath);
}

util.inherits(Documark, EventEmitter);

module.exports = Documark;
<commit_msg>Simplify verbosity levels (warning=1, debug=3).<commit_after>/* jslint node: true */

var fs             = require('fs');
var path           = require('path');
var util           = require('util');
var fm             = require('front-matter');
var sanitize       = require('sanitize-filename');
var es             = require('event-stream');
var cheerio        = require('cheerio');
var jade           = require('jade');
var wkhtmltopdf    = require('wkhtmltopdf');
var concatStream   = require('concat-stream');
var EventEmitter   = require('eventemitter2').EventEmitter2;

const VERBOSITY_WARNING = 1;
const VERBOSITY_DEBUG   = 3;

function Documark (basePath) {
	var config  = null;
	var content = null;
	var plugins = [];
	var debug   = 0;
	var file, basePath;

	function setBasePath (newPath) {
		var filename = 'document.jade';

		newPath = path.resolve(newPath || '.');

		if (fs.statSync(newPath).isFile()) {
			file = newPath;
		} else if (fs.existsSync(path.join(newPath, filename))) {
			file = path.join(newPath, filename);
		} else {
			throw new Error('No ' + filename + ' file found in ' + newPath);
		}

		basePath = path.dirname(file);
	}

	// Make Documark an event emitter
	EventEmitter.call(this);

	this.setMaxListeners(100);

	// Methods
	this.path = function (newPath) {
		if (typeof newPath !== 'undefined') {
			setBasePath(newPath);
		}

		return basePath;
	};

	this.file = function (newFile) {
		if (typeof newFile !== 'undefined') {
			setBasePath(newFile);
		}

		return file;
	};

	this.config = function () {
		if (config === null) {
			this.load();
		}

		config.pdf = (config.pdf || {});

		return config;
	};

	this.content = function () {
		if (config === null) {
			this.load();
		}

		return content;
	};

	this.plugins = function () {
		return plugins;
	};

	this.verbosity = function (level) {
		if (typeof level !== 'undefined') {
			debug = (parseInt(level, 10) || 0);
		}

		return debug;
	};

	this.isVerbosity = function (level) {
		return this.verbosity() >= level;
	};

	this.load = function () {
		var basePath = this.path();

		// Get file contents
		var rawContent = fs.readFileSync(this.file(), { encoding: 'utf-8' });

		// Get config and content
		var data = fm(rawContent);

		rawContent = null;
		config     = (data.attributes || {});
		content    = data.body;

		// Load config.json if no front matter is available
		if ( ! Object.keys(config).length) {
			var configFile = path.join(basePath, 'config.json');

			if (fs.existsSync(configFile)) {
				config = require(configFile);
			}
		}
	};

	this.outputFile = function () {
		var filename = (this.config().filename || 'Document.pdf');

		return path.join(this.path(), sanitize(filename));
	};

	this.applyPlugins = function ($, plugins, cb) {
		var self = this;
		var plugin;

		// Handle plugins one by one to allow adding plugins while iterating
		var nextPlugin = function (err) {
			if (err) {
				cb('Error in plugin ' +
					(plugin ? plugin.name || util.inspect(plugin) : '<unknown>') +
					': ' + err
				);
				return;
			}
			if ( ! plugins.length) {
				cb(null, $);
				return;
			}

			plugin = plugins.shift();

			if (typeof plugin !== 'function') {
				// Load relative to document path
				if (plugin.indexOf('./') === 0) {
					plugin = path.join(self.path(), plugin);
				}

				plugin = require(plugin);
			}
			if (self.isVerbosity(VERBOSITY_DEBUG)) {
				console.log('Processing plugin: ' + util.inspect(plugin.name || plugin));
			}

			plugin($, self, nextPlugin);
		}

		nextPlugin();
	};

	this.contentStream = function () {
		return es.through().pause().queue(this.content());
	};

	this.parserStream = function () {
		var outputFile = this.outputFile();
		var basePath   = this.path();
		var config     = this.config();

		return es.map(function (data, cb) {
			var options = {
				filename: outputFile,
				basedir: basePath
			};

			cb(null, jade.compile(data, options)(config));
		});
	};

	this.pluginStream = function () {
		var self = this;

		return es.through(function (data) {
			var $ = cheerio.load(data);

			// Determine initial plugins
			plugins = self.config().plugins;

			if ( ! Array.isArray(plugins) ) {
				plugins = [];
			}

			// Handle plugins
			self.applyPlugins($, plugins, function (err, $) {
				if (err) {
					throw new Error(err);
				}

				this.emit('data', $.html());
			}.bind(this));
		});
	};

	this.compileStream = function (config) {
		var self   = this;
		var stream = es.map(function (data, cb) {
			self.emit('pre-compile');

			wkhtmltopdf(data, self.config().pdf, function (code, signal) {
				if (code || signal) {
					throw new Error('Error generating PDF (code: ' +
						code + ', signal: ' + signal + ')!');
				}
			}).pipe(concatStream(function (data) {
				cb(null, data);
				stream.end();
			}));
		});

		return stream;
	};

	this.outputStream = function () {
		var self = this;

		return fs.createWriteStream(this.outputFile())
			.on('finish', function () {
				self.emit('post-compile');
			})
			;
	};

	this.compile = function () {
		this.load();

		var input = this.contentStream();
		var self  = this;

		input
			.pipe(this.parserStream())
			.pipe(this.pluginStream())
			.pipe(this.compileStream())
			.pipe(this.outputStream())
			;

		input.resume();
	};

	// Set path
	this.path(basePath);
}

util.inherits(Documark, EventEmitter);

module.exports = Documark;

{'fraction_non_alphanumeric': 0.1651410424061274, 'fraction_numerical': 0.0026153558752101624, 'mean_word_length': 2.8991260014566644, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1149322', 'n_tokens_mistral': 4139, 'n_tokens_neox': 3524, 'n_words': 820}