<commit_before>#!/usr/bin/env node

var net = require('net'),
    util = require('util'),
    cli = require('cli'),
    replServer = require('repl'),
    yrepl = require('../lib/repl');

require('colors');


cli.enable('status', 'help');
cli.parse({
    url: ['u', 'Import document from the url passed', 'url'],
    get: ['g', 'Add this script before starting the repl', 'path'],
    html: ['H', 'Import HTML string', 'string'],
    silent: ['s', 'Silent mode: on or off', 'string', 'on']
});


cli.main(function() {
    var opts = this.options;

    var Y = require('yui3');
    var YUI = Y.YUI;
    if (opts.silent == 'on') {
        Y = Y.silent();
    }
    Y = Y.useSync('yui-base');

    var start = function() {
        if (opts.get) {
            util.print('Fetching script:'.magenta + opts.get.yellow);
            var r = yrepl.startPrompt(Y, YUI);
            r.commands['.get'].action.call(r, opts.get);
        } else {
            yrepl.startPrompt(Y, YUI);
        }
    }
    
    if (opts.html) {
        var html = opts.html;
        util.print('Importing HTML into document'.magenta);
        Y.useSync('node');
        if (html.indexOf('<body') === -1) {
            //No Body, append to body
            Y.one('body').append(html);
        } else if (html.indexOf('<html') > -1) {
            //This is an HTML doc
            Y.one('doc').set('innerHTML', html);
        } else if (html.indexOf('<body') > -1) {
            //Body without HTML
            Y.one('body').set('outerHTML', html);
        }
        util.print(' [done]\n'.white);
        start();
    } else if (opts.url) {
        util.print('Fetching URL: '.magenta, opts.url.yellow);
        Y.useSync('node');
        Y.fetch(opts.url, function() {
            util.print(' [done]\n'.white);
            start();
        });
    } else {
        start();
    }

});
<commit_msg>Minor changes migrating from the old package.<commit_after>#!/usr/bin/env node

var net = require('net'),
    util = require('util'),
    cli = require('cli'),
    replServer = require('repl'),
    yrepl = require('../lib/repl');

require('colors');


cli.enable('status', 'help');
cli.parse({
    url: ['u', 'Import document from the url passed', 'url'],
    get: ['g', 'Add this script before starting the repl', 'path'],
    html: ['H', 'Import HTML string', 'string'],
    silent: ['s', 'Silent mode: on or off', 'string', 'on']
});


cli.main(function() {
    var opts = this.options;

    var Y = require('yui');
    var YUI = Y.YUI;
    if (opts.silent == 'on') {
        //Y = Y.silent();
    }
    Y = YUI();

    var start = function() {
        if (opts.get) {
            util.print('Fetching script:'.magenta + opts.get.yellow);
            var r = yrepl.startPrompt(Y, YUI);
            r.commands['.get'].action.call(r, opts.get);
        } else {
            yrepl.startPrompt(Y, YUI);
        }
    }
    
    if (opts.html) {
        var html = opts.html;
        util.print('Importing HTML into document'.magenta);
        Y.useSync('node');
        if (html.indexOf('<body') === -1) {
            //No Body, append to body
            Y.one('body').append(html);
        } else if (html.indexOf('<html') > -1) {
            //This is an HTML doc
            Y.one('doc').set('innerHTML', html);
        } else if (html.indexOf('<body') > -1) {
            //Body without HTML
            Y.one('body').set('outerHTML', html);
        }
        util.print(' [done]\n'.white);
        start();
    } else if (opts.url) {
        util.print('Fetching URL: '.magenta, opts.url.yellow);
        Y.useSync('node');
        Y.fetch(opts.url, function() {
            util.print(' [done]\n'.white);
            start();
        });
    } else {
        start();
    }

});

{'fraction_non_alphanumeric': 0.19371169730881962, 'fraction_numerical': 0.0018651745270450307, 'mean_word_length': 1.95125786163522, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4961265', 'n_tokens_mistral': 1234, 'n_tokens_neox': 1230, 'n_words': 290}