<commit_before>#!/usr/bin/env node
/* jshint undef:true, unused:true, node:true */

try {
    var request = require("request");
    var toMarkdown = require("to-markdown").toMarkdown;
    var argv = require("minimist")(process.argv.slice(2));
    var meta = require("./package.json");
} catch (e) {
    console.error("Make sure to `npm install` before running this code.");
    console.error("Here's the error that happened during module import:", e, "\n\n");
    process.exit(1);
}


var verboseFlag = argv.v;
var verbose = function() {
    if (verboseFlag) {
        console.error.apply(console, arguments);
    }
};


var url = argv._[0];
if (!url) {
    console.error("%s %s", meta.name, meta.version);
    console.error("\nUsage:\n");
    console.error("\thtml-to-markdown -v [--xform=some-style] <url>\n");
    console.error("-v");
    console.error("\tverbose mode. non-content comments to stderr (content only ever to stdout)");
    console.error("\n--xform");
    console.error("\tnone (default)");
    console.error("\twikipedia");
    console.error("\n");
    process.exit(1);
}

// xform is the function, style is the argument string.
var xform;
var style = argv.xform;
if (style) {
    try {
        xform = require("./xforms/" + style);
        verbose("using xform style:", style);
    } catch(e) {
        console.error("Could not find xform style:", style);
        console.error("Exiting.");
        process.exit(1);
    }
} else {<commit_msg>Switching to ES6 is worth it for multiline strings. Happy panda.<commit_after>#!/usr/bin/env node
/* jshint undef:true, unused:true, node:true */

try {
    var request = require("request");
    var toMarkdown = require("to-markdown").toMarkdown;
    var argv = require("minimist")(process.argv.slice(2));
    var meta = require("./package.json");
} catch (e) {
    console.error("Make sure to `npm install` before running this code.");
    console.error("Here's the error that happened during module import:", e, "\n\n");
    process.exit(1);
}


var verboseFlag = argv.v;
var verbose = function() {
    if (verboseFlag) {
        console.error.apply(console, arguments);
    }
};


var url = argv._[0];
if (!url) {
    var usage = `\n${meta.name} ${meta.version}

Usage:

    html-to-markdown -v [--xform=some-style] <url>

-v
    verbose mode. non-content comments to stderr (content only ever to stdout)

--xform
    none (default)
    wikipedia
`;
    console.error(usage);
    process.exit(1);
}

// xform is the function, style is the argument string.
var xform;
var style = argv.xform;
if (style) {
    try {
        xform = require("./xforms/" + style);
        verbose("using xform style:", style);
    } catch(e) {
        console.error("Could not find xform style:", style);
        console.error("Exiting.");
        process.exit(1);
    }
} else {
{'fraction_non_alphanumeric': 0.1686746987951807, 'fraction_numerical': 0.0038979447200566974, 'mean_word_length': 3.0502152080344334, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4563340', 'n_tokens_mistral': 929, 'n_tokens_neox': 903, 'n_words': 244}