<commit_before>var request = require('request');
var url = require('url');
var fs = require('fs');
var minimist = require('minimist');
var _ = require('underscore');
var yaml = require('js-yaml');
var colors = require('colors');
var Ftp = require('ftp');

process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0';
var argv = require('minimist')(process.argv.slice(2));

var downloadHTTP = function(address, test) {
    (test ? request.head : request.get)(address.availability, function(err, res) {
        if (err) {
            console.log(("ERROR\t" + err + " " + address.availability).red);
            return;
        }
        res.statusCode == 200 ?
            console.log(res.statusCode + " OK\t" + address.availability) :
            console.log((res.statusCode + " ERROR\t" + address.availability).red);
    }).setMaxListeners(20);
};

var downloadFTP = function(address, test) {
    var ftp = new Ftp();
    var opt = url.parse(address.availability);
    opt.user = (opt.auth || ':').split(':')[0];
    opt.password = (opt.auth || ':').split(':')[1];
    opt.connTimeout = 5000;
    ftp.on('ready', function() {
        ftp.get(opt.path, function(err, stream) {
            if (err) {
                console.log(("FTP\t" + err + " " + address.availability).red);
                ftp.destroy();
                return;
            }
            console.log("FTP OK\t" + address.availability);
            if (test) {
                ftp.destroy();
            }
        });
    });
    ftp.on('error', function(err) {
        console.log(("FTP\t" + err + " " + address.availability).red);
        ftp.destroy();
    });
    ftp.connect(opt);
};

var download = function(address, test) {
    var options = url.parse(address.availability);
    options.protocol == 'ftp:' ?
        downloadFTP(address, test) :
        downloadHTTP(address, test);
};

var downloadFile = function(file, test) {
    fs.readFile(file, 'utf8', function(err, data) {
        if (err) { <commit_msg>Update to new schema per https://github.com/osmlab/addresses/pull/9#issuecomment-35763049 .<commit_after>var request = require('request');
var url = require('url');
var fs = require('fs');
var minimist = require('minimist');
var _ = require('underscore');
var yaml = require('js-yaml');
var colors = require('colors');
var Ftp = require('ftp');

process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0';
var argv = require('minimist')(process.argv.slice(2));

var downloadHTTP = function(address, test) {
    (test ? request.head : request.get)(address.data, function(err, res) {
        if (err) {
            console.log(("ERROR\t" + err + " " + address.data).red);
            return;
        }
        res.statusCode == 200 ?
            console.log(res.statusCode + " OK\t" + address.data) :
            console.log((res.statusCode + " ERROR\t" + address.data).red);
    }).setMaxListeners(20);
};

var downloadFTP = function(address, test) {
    var ftp = new Ftp();
    var opt = url.parse(address.data);
    opt.user = (opt.auth || ':').split(':')[0];
    opt.password = (opt.auth || ':').split(':')[1];
    opt.connTimeout = 5000;
    ftp.on('ready', function() {
        ftp.get(opt.path, function(err, stream) {
            if (err) {
                console.log(("FTP\t" + err + " " + address.data).red);
                ftp.destroy();
                return;
            }
            console.log("FTP OK\t" + address.data);
            if (test) {
                ftp.destroy();
            }
        });
    });
    ftp.on('error', function(err) {
        console.log(("FTP\t" + err + " " + address.data).red);
        ftp.destroy();
    });
    ftp.connect(opt);
};

var download = function(address, test) {
    var options = url.parse(address.data);
    options.protocol == 'ftp:' ?
        downloadFTP(address, test) :
        downloadHTTP(address, test);
};

var downloadFile = function(file, test) {
    fs.readFile(file, 'utf8', function(err, data) {
        if (err) { 
{'fraction_non_alphanumeric': 0.18046441191317517, 'fraction_numerical': 0.00933871781928319, 'mean_word_length': 2.4581151832460733, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1307865', 'n_tokens_mistral': 1312, 'n_tokens_neox': 1252, 'n_words': 244}