<commit_before>var cheerio = require("cheerio");
var request = require("request");

var getContentTypeForURL = function(url, callback) {
  var opts = {
    url: url,
    timeout: 10 * 1000,
    method: "HEAD"
  };

  request(opts, function(err, res, body) {
    if (!err) {
      callback(res.headers["content-type"]);
    } else {
      callback(null);
    }
  });
};

var getTitleForURL = function(url, callback) {
  console.log("Scraping:", url);
  getContentTypeForURL(url, function(contentType) {
    if (contentType.indexOf("text/html") === 0) {
      request(url, function(err, res, body) {
        if (!err) {
          var $ = cheerio.load(body);
          var title = $("title").text();
          if (title && title.length > 0) {
            title = title.replace(/[\n|\r]/g, "").trim();
            callback(title);
          }
        }
      });
    }
  });
};

module.exports = function(bot, config) {
  bot.hear(/(https?:\/\/[^\s]+)/g, function(msg, reply) {
    var urls = msg.match;
    urls.forEach(function(url, i) {
      getTitleForURL(url, function(title) {
        if (urls.length === 1) {
          reply(title);
        } else {
          reply("[" + i + "]: " + title);
        }
      });
    });<commit_msg>Improved stability<commit_after>var cheerio = require("cheerio");
var request = require("request");

var getContentTypeForURL = function(url, callback) {
  var opts = {
    url: url,
    timeout: 10 * 1000,
    method: "HEAD"
  };

  request(opts, function(err, res, body) {
    if (!err) {
      callback(res.headers["content-type"]);
    } else {
      callback(null);
    }
  });
};

var getTitleForURL = function(url, callback) {
  console.log("Scraping:", url);
  getContentTypeForURL(url, function(contentType) {
    if (contentType && contentType.indexOf("text/html") === 0) {
      request(url, function(err, res, body) {
        if (!err) {
          var $ = cheerio.load(body);
          var title = $("title").text();
          if (title && title.length > 0) {
            title = title.replace(/[\n|\r]/g, "").trim();
            callback(title);
          }
        }
      });
    }
  });
};

module.exports = function(bot, config) {
  bot.hear(/(https?:\/\/[^\s]+)/g, function(msg, reply) {
    var urls = msg.match;
    urls.forEach(function(url, i) {
      getTitleForURL(url, function(title) {
        if (urls.length === 1) {
          reply(title);
        } else {
          reply("[" + i + "]: " + title);
        }
      });
    });
{'fraction_non_alphanumeric': 0.18795180722891566, 'fraction_numerical': 0.007228915662650603, 'mean_word_length': 2.251958224543081, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '735841', 'n_tokens_mistral': 845, 'n_tokens_neox': 823, 'n_words': 160}