<commit_before>// relme.js
//
// get the rel-me links from an HTML page
//
// Copyright 2012, StatusNet Inc. and others
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

var async = require("async"),
    HTML5 = require("html5"),
    http = require("http"),
    https = require("https"),
    urlParse = require("url").parse;

var RelMe = {
    TypeError: function(type) {
        this.name    = "RelMe.TypeError";
        this.type    = type;
        this.message = "Bad type: " + type;
    },
    HTTPError: function(code) {
        this.name    = "RelMe.HTTPError";
        this.code    = code;
        this.message = "Bad HTTP status code: " + code;
    },
    ProtocolError: function(protocol) {
        this.name     = "RelMe.ProtocolError";
        this.protocol = protocol;
        this.message  = "Unrecognized protocol: " + protocol;
    },
    getLinks: function(url, callback) {

        var parts = urlParse(url),
            options = {
                hostname: parts.hostname,
                port: parseInt(parts.port, 10) || ((parts.protocol == 'https:') ? 443: 80),
                path: parts.path,
                method: 'GET',
                headers: {
                    'User-Agent': 'OpenFollow/0.1.0 (http://openfollow.net/)',
                    'Connection': 'close'
                }
            },
            mod = (parts.protocol == 'http:') ? http : 
                (parts.protocol == 'https:') ? https :
                null;

        if (!mod) {
            callback(new RelMe.ProtocolError(parts.protocol), null);
            return;
        }

        // XXX: use a cache, you big jerk!

        mod.get(options, function(res) {
            var body = "";

            if (res.statusCode !== 200) {
                callback(new RelMe.HTTPError(res.statusCode), null);
                return;
            }

            if (!res.headers['content-type'] || 
                res.headers['content-type'].substr(0, 9) !== 'text/html') {
                callback(new RelMe.TypeError(res.headers['content-type']), null);
                return;
            }

            res.on("data", function(chunk) {
                body = body + chunk;
            });

            res.on("error", function(err) {
                callback(err, null);
            });

            res.on("close", function() {
                callback(new Error("Connection closed prematurely"), null);
            });

            res.on("end", function() {

                var tz, links = [];

                tz = new HTML5.Tokenizer(body);

                tz.on("token", function(tok) {
                    var href = null, isRelMe = false, i, attr;
                    switch (tok.type) {
                    case 'StartTag':
                        if (tok.name === 'a') {
                            for (i = 0; i < tok.data.length; i++) {
                                attr = tok.data[i];
                                if (attr.nodeName === 'href') {
                                    href = attr.nodeValue;
                                } else if (attr.nodeName === 'rel' && 
                                           attr.nodeValue.split(/\s+/).indexOf('me') !== -1) {
                                    isRelMe = true;
                                }
                            }
                            if (isRelMe && href) {
                                links.push(href);
                            }
                        }
                        break;
                    case 'EOF':
                        callback(null, links);
                        return;
                    }
                });

                tz.tokenize();
            });
        }).on('error', function(err) {
            console.log("Got to error");
            callback(err, null);
        });
    }
};

// Some housekeeping for our custom classes

RelMe.TypeError.prototype = new Error;
RelMe.TypeError.prototype.constructor = RelMe.TypeError;

RelMe.HTTPError.prototype = new Error;
RelMe.HTTPError.prototype.constructor = RelMe.HTTPError;

RelMe.ProtocolError.prototype = new Error;
RelMe.ProtocolError.prototype.constructor = RelMe.ProtocolError;

module.exports = RelMe;<commit_msg>Uniquify the links found in a page<commit_after>// relme.js
//
// get the rel-me links from an HTML page
//
// Copyright 2012, StatusNet Inc. and others
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

var async = require("async"),
    HTML5 = require("html5"),
    http = require("http"),
    https = require("https"),
    urlParse = require("url").parse;

var RelMe = {
    TypeError: function(type) {
        this.name    = "RelMe.TypeError";
        this.type    = type;
        this.message = "Bad type: " + type;
    },
    HTTPError: function(code) {
        this.name    = "RelMe.HTTPError";
        this.code    = code;
        this.message = "Bad HTTP status code: " + code;
    },
    ProtocolError: function(protocol) {
        this.name     = "RelMe.ProtocolError";
        this.protocol = protocol;
        this.message  = "Unrecognized protocol: " + protocol;
    },
    getLinks: function(url, callback) {

        var parts = urlParse(url),
            options = {
                hostname: parts.hostname,
                port: parseInt(parts.port, 10) || ((parts.protocol == 'https:') ? 443: 80),
                path: parts.path,
                method: 'GET',
                headers: {
                    'User-Agent': 'OpenFollow/0.1.0 (http://openfollow.net/)',
                    'Connection': 'close'
                }
            },
            mod = (parts.protocol == 'http:') ? http : 
                (parts.protocol == 'https:') ? https :
                null;

        if (!mod) {
            callback(new RelMe.ProtocolError(parts.protocol), null);
            return;
        }

        // XXX: use a cache, you big jerk!

        mod.get(options, function(res) {
            var body = "";

            if (res.statusCode !== 200) {
                callback(new RelMe.HTTPError(res.statusCode), null);
                return;
            }

            if (!res.headers['content-type'] || 
                res.headers['content-type'].substr(0, 9) !== 'text/html') {
                callback(new RelMe.TypeError(res.headers['content-type']), null);
                return;
            }

            res.on("data", function(chunk) {
                body = body + chunk;
            });

            res.on("error", function(err) {
                callback(err, null);
            });

            res.on("close", function() {
                callback(new Error("Connection closed prematurely"), null);
            });

            res.on("end", function() {

                var tz, links = {};

                tz = new HTML5.Tokenizer(body);

                tz.on("token", function(tok) {
                    var href = null, isRelMe = false, i, attr, prop, res = [];
                    switch (tok.type) {
                    case 'StartTag':
                        if (tok.name === 'a') {
                            for (i = 0; i < tok.data.length; i++) {
                                attr = tok.data[i];
                                if (attr.nodeName === 'href') {
                                    href = attr.nodeValue;
                                } else if (attr.nodeName === 'rel' && 
                                           attr.nodeValue.split(/\s+/).indexOf('me') !== -1) {
                                    isRelMe = true;
                                }
                            }
                            if (isRelMe && href) {
                                links[href] = 1;
                            }
                        }
                        break;
                    case 'EOF':
                        for (prop in links) {
                            if (links.hasOwnProperty(prop)) {
                                res.push(prop);
                            }
                        }
                        callback(null, res);
                        return;
                    }
                });

                tz.tokenize();
            });
        }).on('error', function(err) {
            console.log("Got to error");
            callback(err, null);
        });
    }
};

// Some housekeeping for our custom classes

RelMe.TypeError.prototype = new Error;
RelMe.TypeError.prototype.constructor = RelMe.TypeError;

RelMe.HTTPError.prototype = new Error;
RelMe.HTTPError.prototype.constructor = RelMe.HTTPError;

RelMe.ProtocolError.prototype = new Error;
RelMe.ProtocolError.prototype.constructor = RelMe.ProtocolError;

module.exports = RelMe;
{'fraction_non_alphanumeric': 0.12414930913590431, 'fraction_numerical': 0.005877500515570221, 'mean_word_length': 1.3131409492010493, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '706052', 'n_tokens_mistral': 2593, 'n_tokens_neox': 2452, 'n_words': 631}