<commit_before>// Identifier class

var databank = require('databank'),
    DatabankObject = databank.DatabankObject;

var Identifier = DatabankObject.subClass("identifier");

Identifier.schema = {pkey: "id",
                     fields: ["person", "type", "created", "modified"],
                     indices: ["person"]
                    };
Identifier.beforeCreate = function(props, callback) {
    // Timestamp
    props.modified = props.created = Date.now();
    callback(null, props);
};

Identifier.beforeUpdate = function(props, callback) {
    // Can't overwrite these properties
    if (props.id) {
        delete props.id;
    }
    if (props.created) {
        delete props.created;
    }
    // Timestamp
    props.modified = Date.now();
    callback(null, props);
};

Identifier.beforeSave = function(callback) {
    this.modified = Date.now();
    callback(null);
};

module.exports = Identifier;

<commit_msg>start canonicalizing identifiers<commit_after>// Identifier class

var url = require('url'),
    databank = require('databank'),
    DatabankObject = databank.DatabankObject;

var Identifier = DatabankObject.subClass("identifier");

Identifier.schema = {pkey: "id",
                     fields: ["person", "type", "created", "modified"],
                     indices: ["person"]
                    };

Identifier.beforeCreate = function(props, callback) {
    props.id = Identifier.canonical(props.id);
    // Timestamp
    props.modified = props.created = Date.now();
    callback(null, props);
};

Identifier.beforeUpdate = function(props, callback) {
    // Can't overwrite these properties
    if (props.id) {
        delete props.id;
    }
    if (props.created) {
        delete props.created;
    }
    // Timestamp
    props.modified = Date.now();
    callback(null, props);
};

Identifier.beforeSave = function(callback) {
    this.modified = Date.now();
    callback(null);
};

// To the extent possible, makes uris canonical

Identifier.canonical = function(uri) {

    var parts;

    if (uri.match(/^[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}$/i)) {
        // It's email-ish
        return uri.toLowerCase();

    } else {

        parts = url.parse(uri);

        if ((parts.protocol === "http:" && parts.port === '80') ||
            (parts.protocol === "https:" && parts.port === '443')) {
            delete parts.port;
            parts.host = parts.hostname;
        }

        // XXX: special case for Twitter URLs

        return url.format(parts);
    }
};

module.exports = Identifier;

{'fraction_non_alphanumeric': 0.1515988945913936, 'fraction_numerical': 0.0043426766679826295, 'mean_word_length': 2.466484268125855, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2593903', 'n_tokens_mistral': 765, 'n_tokens_neox': 738, 'n_words': 161}