<commit_before>'use strict';

var fs = require('fs-extra'),
    url = require('url'),
    path = require('path'),
    request = require('request'),
    assert = require('assert');


var prototype = {

    fetch: function (uri, filename, callback) {
        var local, result = {};

        if (typeof filename === 'function') {
            callback = filename;
            filename = undefined;
        }

        if (!filename) {
            filename = url.parse(uri).pathname;
            filename = path.basename(filename);
            filename = filename || 'out.txt';
        }

        local = path.join(this.outdir, filename);
        result.name = local;
        if (fs.existsSync(local)) {
            fs.readFile(local, 'utf8', function (err, data) {
                result.data = data;
                result.isFromCache = true;
                callback(err, result);
            });
        } else {
            request(uri, function (err, resp, body) {
                err || console.log('%s -> %s', uri, local);
                try {
                    fs.outputFileSync(local, body);
                } catch (e) {
                    console.error('failed writing %s', local);
                }
                result.data = body;
                result.isFromCache = false;
                callback(err, result);
            });
        }
    }

};

exports.create = function (config) {
    assert.ok(config.outdir, 'A output dir must be provided.');

    if (!fs.ensureDirSync(config.outdir)) {
        fs.mkdirsSync(config.outdir);
    }

    return Object.create(prototype, {
        outdir: {
            value: config.outdir,
            enumerable: true,
            writable: false
        }
    });
};<commit_msg>Mask the auth section of the URL<commit_after>'use strict';

var fs = require('fs-extra'),
    url = require('url'),
    path = require('path'),
    request = require('request'),
    assert = require('assert'),
    urlUtil = require('url');


var prototype = {

    fetch: function (uri, filename, callback) {
        var local, result = {};

        if (typeof filename === 'function') {
            callback = filename;
            filename = undefined;
        }

        if (!filename) {
            filename = url.parse(uri).pathname;
            filename = path.basename(filename);
            filename = filename || 'out.txt';
        }

        local = path.join(this.outdir, filename);
        result.name = local;
        if (fs.existsSync(local)) {
            fs.readFile(local, 'utf8', function (err, data) {
                result.data = data;
                result.isFromCache = true;
                callback(err, result);
            });
        } else {
            request(uri, function (err, resp, body) {
                var urlObj = urlUtil.parse(uri),
                    maskedURI = uri.replace(urlObj.auth, "xxx");

                err || console.log('%s -> %s', maskedURI, local);
                try {
                    fs.outputFileSync(local, body);
                } catch (e) {
                    console.error('failed writing %s', local);
                }
                result.data = body;
                result.isFromCache = false;
                callback(err, result);
            });
        }
    }

};

exports.create = function (config) {
    assert.ok(config.outdir, 'A output dir must be provided.');

    if (!fs.ensureDirSync(config.outdir)) {
        fs.mkdirsSync(config.outdir);
    }

    return Object.create(prototype, {
        outdir: {
            value: config.outdir,
            enumerable: true,
            writable: false
        }
    });
};
{'fraction_non_alphanumeric': 0.13165112452002195, 'fraction_numerical': 0.0005485463521667581, 'mean_word_length': 1.5361613351877608, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7102021', 'n_tokens_mistral': 1012, 'n_tokens_neox': 969, 'n_words': 226}