<commit_before>#!/usr/bin/node

const dash = require('dashdash');
const debug = require('debug')('url-inspector');

const parser = dash.createParser({options: [
	{
		names: ['help', 'h'],
		type: 'bool',
		help: 'Print this help and exit.'
	},
	{
		names: ['all'],
		type: 'bool',
		help: 'enable all metadata'
	}
]});

let opts;
try {
	opts = parser.parse(process.argv);
} catch(e) {
	console.error(e.toString());
	opts = {help: true};
}

let url = opts._args.pop();

if (opts.help || !url) {
	const help = parser.help({includeEnv: true}).trimRight();
	console.log('usage: url-inspector [OPTIONS] <url>\n' + 'options:\n' + help);
	process.exit(0);
}

const inspector = require('..');

opts.file = true;
if (url.startsWith('./') || url.startsWith('/')) url = "file://" + url;

inspector(url, opts, function(err, meta) {
	if (err) {
		console.error(err);
		process.exit(1);
	} else {
		console.log(require('util').inspect(meta));
		process.exit(0);
	}<commit_msg>Add --providers <path> option<commit_after>#!/usr/bin/node

const dash = require('dashdash');
const debug = require('debug')('url-inspector');

const parser = dash.createParser({options: [
	{
		names: ['help', 'h'],
		type: 'bool',
		help: 'Print this help and exit.'
	},
	{
		names: ['all'],
		type: 'bool',
		help: 'enable all metadata'
	},
	{
		names: ['providers', 'p'],
		type: 'string',
		help: 'Path to js file exporting providers list'
	}
]});

let opts;
try {
	opts = parser.parse(process.argv);
} catch(e) {
	console.error(e.toString());
	opts = {help: true};
}

let url = opts._args.pop();

if (opts.help || !url) {
	const help = parser.help({includeEnv: true}).trimRight();
	console.log('usage: url-inspector [OPTIONS] <url>\n' + 'options:\n' + help);
	process.exit(0);
}

const inspector = require('..');

opts.file = true;
if (url.startsWith('./') || url.startsWith('/')) url = "file://" + url;

if (opts.providers) {
	opts.providers = require(opts.providers);
}

inspector(url, opts, function(err, meta) {
	if (err) {
		console.error(err);
		process.exit(1);
	} else {
		console.log(require('util').inspect(meta));
		process.exit(0);
	}
{'fraction_non_alphanumeric': 0.2195929957406531, 'fraction_numerical': 0.002839564600094652, 'mean_word_length': 4.143552311435523, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '726802', 'n_tokens_mistral': 821, 'n_tokens_neox': 790, 'n_words': 176}