<commit_before>/*jslint node: true */
/*jslint es5: true */
/*jslint nomen: true */

"use strict";

var fs = require('graceful-fs'),
    https = require('https'),
    csv = require('csv'),
    path = require('path'),
    moment = require('moment'),
    RUNTIME = moment().toJSON(),
    TYPES = ['boolean', 'date', 'time', 'datetime', 'integer', 'number', 'text'],
    TYPENDX = TYPES.reduce(function (s, t, i) { s[t] = i; return s; }, {}),
    BOOLEANREPS = ["true", "false", "0", "1"];

function isEmpty(obj) {
    return (obj === null || obj === undefined || (obj.hasOwnProperty('length') && obj.length === 0));
}

function contains(container, item) {
    return (container.indexOf(item) !== -1);
}


function testBoolean(v) {
    return contains(BOOLEANREPS, String(v).toLowerCase());
}
function testDateTime(v) {
    return (moment(v, [moment.ISO_8601], true).isValid());
}
function testTime(v) {
    return (moment(v, ['HH:mm:ssZZ'], true).isValid());
}
function testDate(v) {
    return (moment(v, ['YYYY-MM-DD'], true).isValid());
}
function testIntegerNumber(v) {
    var n = Number(v);
    return Math.floor(n) === n;
}
function testNumber(v) {
    var n = Number(v);
    return !isNaN(n);
}
function guessType(value) {
    // try bool
    if (testBoolean(value)) {
        return 'boolean';
    } //else try datetime
    // try date
    if (testDate(value)) {
        return 'date';
    } //else try datetime
    if (testTime(value)) {
        return 'time';
    } //else try datetime
    if (testDateTime(value)) {
        return 'datetime';
    } //else try number
    if (testNumber(value)) {
        // try int/long
        if (testIntegerNumber(value)) {
            return 'integer';
        }
        return 'number';
    } //else assume text
    return 'text';
}

function fallBackType(et, nt) {
    if (et === undefined) {
        return nt;
    }
    return TYPES[Math.max(TYPENDX[et], TYPENDX[nt])];
}

function SchemaDoc() {
    this.hideLevels = 1;
    this.count = {errors: 0, newfields: 0};
}

SchemaDoc.prototype.setHideLevels = function (num) {
    this.hideLevels = num;
};

SchemaDoc.prototype.setBase = function (newbase) {
    this.base = newbase;
    var me = this;

    //initialize refKeys after loading
    //force learnable if type is unknown
    this.refKeys = [];
    Object.keys(this.base).forEach(function (k) {
        var fd = me.base[k];
        if (!contains(TYPES, fd.type)) {
            fd.type = undefined;
            fd.learnable = true;
        }
        if (fd.type === "integer"  || fd.type === "number") {
            fd.min = Number(fd.min);
            fd.max = Number(fd.max);
        }
        if (fd.isReferenceToKey) {
            me.refKeys.push(fd.key);
            me.base[fd.isReferenceToKey].isReferenceable = true;
        }
    });
};

function handleJSONData(strm, cb) {
    var data = "", error = false;
    strm.on('data', function (chunk) { if (!error) { data += chunk; } })
        .on('end', function () { cb(error ? {} : JSON.parse(data)); })
        .on('error', function (err) { if (err) { error = true; console.error(err); } });
}

function handleCSVData(strm, cb) {
    var newbase = {}, error = false;

    function ifNotEmptyBool(testval, defval) {
        defval = defval || false;

        if (testval === "" || !testBoolean(testval)) {
            return defval;
        } // else
        return contains(["false", "0"], testval.toLowerCase()) ? false : true;
    }

    function ifNotEmpty(testval, nullval, setval) {
        setval = setval || testval;
        return testval === "" ? nullval : setval;
    }

    function mapcsv(data) {
        var key = data.key, fd = {key: key};

        fd.learnable = ifNotEmptyBool(data.learnable, false, !!data.learnable);
        fd.type = !contains(TYPES, data.type) ? undefined : data.type;
        fd.format = ifNotEmpty(data.format);
        fd.example = ifNotEmpty(data['example-value']);
        fd.min = ifNotEmpty(data['min-value']);
        fd.max = ifNotEmpty(data['max-value']);
        fd.minlen = ifNotEmpty(data['min-len']);
        fd.maxlen = ifNotEmpty(data['max-len']);
        fd.created = ifNotEmpty(data.created, RUNTIME);
        fd.updated = data.updated;
        fd.isReferenceToKey = ifNotEmpty(data.isReferenceToKey);
        fd.isReferenceable = ifNotEmptyBool(data.isReferenceable);
        fd.list = ifNotEmpty(data['value-list'], undefined, data['value-list'].split('|'));

        // don't load any of the errors.
        // don't load the counts

        newbase[key] = fd;
    }

    function onerr(err) { if (err) { error = true; console.error(err); } }

    strm.pipe(csv.parse({ delimiter: ',', columns: true }))
        .pipe(csv.transform(mapcsv, onerr))
        .on('end', function () { cb(error ? {} : newbase); })
        .on('error', onerr);
}

function handleData(type, strm, cb) {
    if (type === 'json' || type === 'application/json') {
        return handleJSONData(strm, cb);
    }
    if (type === 'csv' || type === 'text/csv') {
        return handleCSVData(strm, cb);
    }
    console.error("can't decide how to handle type == %s", type);
    return cb({});
}
<commit_msg>don't load the examples<commit_after>/*jslint node: true */
/*jslint es5: true */
/*jslint nomen: true */

"use strict";

var fs = require('graceful-fs'),
    https = require('https'),
    csv = require('csv'),
    path = require('path'),
    moment = require('moment'),
    RUNTIME = moment().toJSON(),
    TYPES = ['boolean', 'date', 'time', 'datetime', 'integer', 'number', 'text'],
    TYPENDX = TYPES.reduce(function (s, t, i) { s[t] = i; return s; }, {}),
    BOOLEANREPS = ["true", "false", "0", "1"];

function isEmpty(obj) {
    return (obj === null || obj === undefined || (obj.hasOwnProperty('length') && obj.length === 0));
}

function contains(container, item) {
    return (container.indexOf(item) !== -1);
}


function testBoolean(v) {
    return contains(BOOLEANREPS, String(v).toLowerCase());
}
function testDateTime(v) {
    return (moment(v, [moment.ISO_8601], true).isValid());
}
function testTime(v) {
    return (moment(v, ['HH:mm:ssZZ'], true).isValid());
}
function testDate(v) {
    return (moment(v, ['YYYY-MM-DD'], true).isValid());
}
function testIntegerNumber(v) {
    var n = Number(v);
    return Math.floor(n) === n;
}
function testNumber(v) {
    var n = Number(v);
    return !isNaN(n);
}
function guessType(value) {
    // try bool
    if (testBoolean(value)) {
        return 'boolean';
    } //else try datetime
    // try date
    if (testDate(value)) {
        return 'date';
    } //else try datetime
    if (testTime(value)) {
        return 'time';
    } //else try datetime
    if (testDateTime(value)) {
        return 'datetime';
    } //else try number
    if (testNumber(value)) {
        // try int/long
        if (testIntegerNumber(value)) {
            return 'integer';
        }
        return 'number';
    } //else assume text
    return 'text';
}

function fallBackType(et, nt) {
    if (et === undefined) {
        return nt;
    }
    return TYPES[Math.max(TYPENDX[et], TYPENDX[nt])];
}

function SchemaDoc() {
    this.hideLevels = 1;
    this.count = {errors: 0, newfields: 0};
}

SchemaDoc.prototype.setHideLevels = function (num) {
    this.hideLevels = num;
};

SchemaDoc.prototype.setBase = function (newbase) {
    this.base = newbase;
    var me = this;

    //initialize refKeys after loading
    //force learnable if type is unknown
    this.refKeys = [];
    Object.keys(this.base).forEach(function (k) {
        var fd = me.base[k];
        if (!contains(TYPES, fd.type)) {
            fd.type = undefined;
            fd.learnable = true;
        }
        if (fd.type === "integer"  || fd.type === "number") {
            fd.min = Number(fd.min);
            fd.max = Number(fd.max);
        }
        if (fd.isReferenceToKey) {
            me.refKeys.push(fd.key);
            me.base[fd.isReferenceToKey].isReferenceable = true;
        }
    });
};

function handleJSONData(strm, cb) {
    var data = "", error = false;
    strm.on('data', function (chunk) { if (!error) { data += chunk; } })
        .on('end', function () { cb(error ? {} : JSON.parse(data)); })
        .on('error', function (err) { if (err) { error = true; console.error(err); } });
}

function handleCSVData(strm, cb) {
    var newbase = {}, error = false;

    function ifNotEmptyBool(testval, defval) {
        defval = defval || false;

        if (testval === "" || !testBoolean(testval)) {
            return defval;
        } // else
        return contains(["false", "0"], testval.toLowerCase()) ? false : true;
    }

    function ifNotEmpty(testval, nullval, setval) {
        setval = setval || testval;
        return testval === "" ? nullval : setval;
    }

    function mapcsv(data) {
        var key = data.key, fd = {key: key};

        fd.learnable = ifNotEmptyBool(data.learnable, false, !!data.learnable);
        fd.type = !contains(TYPES, data.type) ? undefined : data.type;
        fd.format = ifNotEmpty(data.format);
        fd.min = ifNotEmpty(data['min-value']);
        fd.max = ifNotEmpty(data['max-value']);
        fd.minlen = ifNotEmpty(data['min-len']);
        fd.maxlen = ifNotEmpty(data['max-len']);
        fd.created = ifNotEmpty(data.created, RUNTIME);
        fd.updated = data.updated;
        fd.isReferenceToKey = ifNotEmpty(data.isReferenceToKey);
        fd.isReferenceable = ifNotEmptyBool(data.isReferenceable);
        fd.list = ifNotEmpty(data['value-list'], undefined, data['value-list'].split('|'));

        // don't load any of the errors.
        // don't load the counts
        // don't load the example

        newbase[key] = fd;
    }

    function onerr(err) { if (err) { error = true; console.error(err); } }

    strm.pipe(csv.parse({ delimiter: ',', columns: true }))
        .pipe(csv.transform(mapcsv, onerr))
        .on('end', function () { cb(error ? {} : newbase); })
        .on('error', onerr);
}

function handleData(type, strm, cb) {
    if (type === 'json' || type === 'application/json') {
        return handleJSONData(strm, cb);
    }
    if (type === 'csv' || type === 'text/csv') {
        return handleCSVData(strm, cb);
    }
    console.error("can't decide how to handle type == %s", type);
    return cb({});
}

{'fraction_non_alphanumeric': 0.17434402332361515, 'fraction_numerical': 0.0025267249757045676, 'mean_word_length': 2.585714285714286, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4958872', 'n_tokens_mistral': 3401, 'n_tokens_neox': 3374, 'n_words': 773}