<commit_before>// determine the ASN of the connecting IP

var dns = require('dns');
var providers = [];

exports.register = function () {
    var config = this.config.get('connect.asn.ini');
    if (config.main.providers) {
        providers = config.main.providers.split(/[\s,;]+/);
    }
    else {
        providers.push('origin.asn.cymru.com');
        // broken due to TXT handling bug in node.js
        // providers.push('asn.routeviews.org');
        // TODO: test after 3/1/2014 and see if bug is fixed
    }
};

exports.hook_lookup_rdns = function (next, connection) {
    var plugin = this;
    var ip = connection.remote_ip;
    var pending = 0;

    for (var i=0; i < providers.length; i++) {
        var zone = providers[i];
        // connection.logdebug(plugin, "zone: " + zone);

        var query = ip.split('.').reverse().join('.') + '.' + zone;
        // connection.logdebug(plugin, "query: " + query);

        pending++;
        dns.resolveTxt(query, function (err, addrs) {
            pending--;
            if (err) {
                connection.logerror(plugin, "error: " + err + ' running: ' + query);
            }
            else {
                for (var i=0; i < addrs.length; i++) {
                    connection.logdebug(plugin, zone + " answer: " + addrs[i]);
                    if (zone === 'origin.asn.cymru.com') {
                        plugin.parse_cymru(addrs[i], connection);
                    }
                    else if (zone === 'asn.routeviews.org') {
                        plugin.parse_routeviews(addrs[i], connection);
                    }
                    else {
                        connection.logerror(plugin, "unrecognized ASN provider: " + zone);
                    }
                }
            }
            if (pending === 0) return next();
        });
    }
    if (pending === 0) return next();
};

exports.parse_routeviews = function (str, connection) {
    var plugin = this;
    var r = str.split(/ /);
    if (r.length !== 3) {
        connection.logerror(plugin, "result length not 3: " + r.length + ' string="' + str + '"');
        return null;
    }
    connection.loginfo(plugin, 'routeviews: asn=' + r[0] + ' net=' + r[1]);
    if (!connection.notes.asn) connection.notes.asn = {};
    connection.notes.asn['routeviews'] = { asn: r[0], net: r[1] };
    return r[0];
};

exports.parse_cymru = function (str, connection) {
    var plugin = this;
    var r = str.split(/\s+\|\s+/);
    // 172.16.31.10.origin.asn.cymru.com. 14350 IN TXT "40431 | 192.168.127.12/21 | US | arin | 2007-03-02"
    // handle this: cymru: result length not 5: 4 string="10290 | 192.168.127.12/24 | US | arin |"
    if (r.length < 4) {
        connection.logerror(plugin, "cymru: bad result length " + r.length + ' string="' + str + '"');
        return null;
    }
    connection.loginfo(plugin, 'cymru: asn=' + r[0] + ' net=' + r[1] + ' country=' + r[2] +
                               ' authority=' + r[3] + (r[4] ? ' date=' + r[4] : ''));
    if (!connection.notes.asn) connection.notes.asn = {};
    connection.notes.asn['cymru'] = { asn: r[0], net: r[1], country: r[2], authority: r[3], date: r[4] };
    return r[0];
};<commit_msg>connect.asn: skip rfc1918 addresses, tidy output<commit_after>// determine the ASN of the connecting IP

var dns = require('dns');
var is_rfc1918 = require('./net_utils').is_rfc1918;

var providers = [];

exports.register = function () {
    var config = this.config.get('connect.asn.ini');
    if (config.main.providers) {
        providers = config.main.providers.split(/[\s,;]+/);
    }
    else {
        providers.push('origin.asn.cymru.com');
        // broken due to TXT handling bug in node.js
        // providers.push('asn.routeviews.org');
        // TODO: test after 3/1/2014 and see if bug is fixed
    }
};

exports.hook_lookup_rdns = function (next, connection) {
    var plugin = this;
    var ip = connection.remote_ip;
    if (is_rfc1918(ip)) return next();
    var pending = 0;

    for (var i=0; i < providers.length; i++) {
        var zone = providers[i];
        // connection.logdebug(plugin, "zone: " + zone);

        var query = ip.split('.').reverse().join('.') + '.' + zone;
        // connection.logdebug(plugin, "query: " + query);

        pending++;
        dns.resolveTxt(query, function (err, addrs) {
            pending--;
            if (err) {
                connection.logerror(plugin, "error: " + err + ' running: ' + query);
            }
            else {
                for (var i=0; i < addrs.length; i++) {
                    connection.logdebug(plugin, zone + " answer: " + addrs[i]);
                    if (zone === 'origin.asn.cymru.com') {
                        plugin.parse_cymru(addrs[i], connection);
                    }
                    else if (zone === 'asn.routeviews.org') {
                        plugin.parse_routeviews(addrs[i], connection);
                    }
                    else {
                        connection.logerror(plugin, "unrecognized ASN provider: " + zone);
                    }
                }
            }
            if (pending === 0) return next();
        });
    }
    if (pending === 0) return next();
};

exports.parse_routeviews = function (str, connection) {
    var plugin = this;
    var r = str.split(/ /);
    if (r.length !== 3) {
        connection.logerror(plugin, "result length not 3: " + r.length + ' string="' + str + '"');
        return null;
    }
    connection.loginfo(plugin, 'routeviews: asn=' + r[0] + ' net=' + r[1]);
    if (!connection.notes.asn) connection.notes.asn = {};
    connection.notes.asn['routeviews'] = { asn: r[0], net: r[1] };
    return r[0];
};

exports.parse_cymru = function (str, connection) {
    var plugin = this;
    var r = str.split(/\s+\|\s*/);
    // 172.16.31.10.origin.asn.cymru.com. 14350 IN TXT "40431 | 192.168.127.12/21 | US | arin | 2007-03-02"
    // handle this: cymru: result length not 5: 4 string="10290 | 192.168.127.12/24 | US | arin |"
    if (r.length < 4) {
        connection.logerror(plugin, "cymru: bad result length " + r.length + ' string="' + str + '"');
        return null;
    }
    connection.loginfo(plugin, 'cymru: asn=' + r[0] + ' net=' + r[1] + ' country=' + r[2] +
                               ' authority=' + r[3] + (r[4] ? ' date=' + r[4] : ''));
    if (!connection.notes.asn) connection.notes.asn = {};
    connection.notes.asn['cymru'] = { asn: r[0], net: r[1], country: r[2], authority: r[3], date: r[4] };
    return r[0];
};
{'fraction_non_alphanumeric': 0.17261448920202174, 'fraction_numerical': 0.03032623678970746, 'mean_word_length': 1.9295648272768058, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4083270', 'n_tokens_mistral': 2297, 'n_tokens_neox': 2110, 'n_words': 488}