<commit_before>
/*
 * DataColumn abstracts the functionality of each column
 * of a dataset. A column has a type (text|number|date).
 *
 * API:
 *
 * column.name() ... returns the name (string)
 * column.type() ... return column type (string)
 * column.length ... number of rows (number)
 * column.val(i) ... parsed value in row i
 * column.each(func) ... apply function to each value
 * column.raw() ... access raw, unparsed values
 *
 */
dw.column = function(name, rows, type) {

    function guessType() {

        if (_.every(rows, _.isNumber)) return dw.column.types.number();
        if (_.every(rows, _.isDate)) return dw.column.types.date();
        // guessing column type by counting parsing errors
        // for every known type
        var types = [
                dw.column.types.date(rows.slice(0, 20)),
                dw.column.types.number(rows.slice(0, 20)),
                dw.column.types.text()
            ],
            type,
            k = rows.length,
            tolerance = 0.1; // allowing 10% mis-parsed values

        _.each(rows, function(val) {
            _.each(types, function(t) {
                t.parse(val);
            });
        });
        _.every(types, function(t) {
            if (t.errors() / k < tolerance) type = t;
            return !type;
        });
        return type;
    }

    type = type ? dw.column.types[type](rows.slice(0, 50)) : guessType();

    var range,
        total,
        origRows = rows.slice(0);

    // public interface
    var column = {
        // column label
        name: function() {
            if (arguments.length) {
                name = arguments[0];
                return column;
            }
            return name;
        },
        // number of rows
        length: rows.length,
        // column.val(i) .. returns ith row of the col, parsed
        val: function(i) {
            return type.parse(rows[i]);
        },
        // each
        each: function(f) {
            for (i=0; i<rows.length; i++) {
                f(column.val(i), i);
            }
        },
        // access to raw values
        raw: function() { return rows; },
        // column type
        type: function() { return type.name(); },
        // [min,max] range
        range: function() {
            if (!type.toNum) return false;
            if (!range) {
                range = [Number.MAX_VALUE, -Number.MAX_VALUE];
                column.each(function(v) {<commit_msg>added comments. type(true) returns type as object<commit_after>
/*
 * DataColumn abstracts the functionality of each column
 * of a dataset. A column has a type (text|number|date).
 *
 * API:
 *
 * column.name() ... returns the name (string)
 * column.type() ... return column type (string)
 * column.length ... number of rows (number)
 * column.val(i) ... parsed value in row i
 * column.each(func) ... apply function to each value
 * column.raw() ... access raw, unparsed values
 *
 */
dw.column = function(name, rows, type) {

    function guessType() {

        if (_.every(rows, _.isNumber)) return dw.column.types.number();
        if (_.every(rows, _.isDate)) return dw.column.types.date();
        // guessing column type by counting parsing errors
        // for every known type
        var types = [
                dw.column.types.date(rows.slice(0, 20)),
                dw.column.types.number(rows.slice(0, 20)),
                dw.column.types.text()
            ],
            type,
            k = rows.length,
            tolerance = 0.1; // allowing 10% mis-parsed values

        _.each(rows, function(val) {
            _.each(types, function(t) {
                t.parse(val);
            });
        });
        _.every(types, function(t) {
            if (t.errors() / k < tolerance) type = t;
            return !type;
        });
        return type;
    }

    type = type ? dw.column.types[type](rows.slice(0, 50)) : guessType();

    var range,
        total,
        origRows = rows.slice(0);

    // public interface
    var column = {
        // column label
        name: function() {
            if (arguments.length) {
                name = arguments[0];
                return column;
            }
            return name;
        },

        /**
         * number of rows
         */
        length: rows.length,

        /**
         * returns ith row of the col, parsed
         *
         * @param i
         * @param unfiltered  if set to true, precedent calls of filterRows are ignored
         */
        val: function(i, unfiltered) {
            var r = unfiltered ? origRows : rows;
            if (i < 0) i += r.length;
            return type.parse(r[i]);
        },

        /**
         * apply function to each value
         */
        each: function(f) {
            for (i=0; i<rows.length; i++) {
                f(column.val(i), i);
            }
        },

        // access to raw values
        raw: function() { return rows; },
        // column type
        type: function(o) { return o ? type : type.name(); },
        // [min,max] range
        range: function() {
            if (!type.toNum) return false;
            if (!range) {
                range = [Number.MAX_VALUE, -Number.MAX_VALUE];
                column.each(function(v) {
{'fraction_non_alphanumeric': 0.13495533168599125, 'fraction_numerical': 0.006272571754419312, 'mean_word_length': 1.4950213371266003, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2448780', 'n_tokens_mistral': 1522, 'n_tokens_neox': 1466, 'n_words': 416}