<commit_before>'use strict';
var fs = require('fs');
var misp = require('npm-metainspector');

/* module built by sempiternum */
module.exports.description = "Catch metadata from linked URLs";

module.exports.main = function(client){
  client.addListener('message#', function(message) {
    console.log(message);
	
	
	
	
  });
};<commit_msg>inspect URLs and show the title<commit_after>'use strict';
/* module built by github.com/sempiternum */

const MetaInspector = require('node-metainspector');

module.exports.description = "Show metadata from URLs in messages";

module.exports.main = function(irc_client){
  irc_client.addListener('message#', function(nick,to,text,message) {
    if(text.indexOf('http://') + text.indexOf('https://') !== -2){
      const urls = text.match(/http(s)?:\/\/[^ ]+/ig);
      console.log('seen ' + urls.length + ' urls');
      if(urls.length > 3) {
        //too many URLs, ignoring them
        return;
      }
      for(let u of urls){
        console.log('analyzing URL ' + u);
        let client = new MetaInspector(u, { timeout: 5000, limit: 1024*1024*1 });
        client.on("fetch", function(){
          console.log('fetched ' + u + ' -- ' + client);
          irc_client.say(to, client.title + ' - ' + u);
        });

        client.on("error", function(err){
          console.log('error: ', err)
          irc_client.say(to, err);
        });
        client.fetch();
      }
    }
  });
};

{'fraction_non_alphanumeric': 0.17454798331015298, 'fraction_numerical': 0.01043115438108484, 'mean_word_length': 2.816976127320955, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1648268', 'n_tokens_mistral': 481, 'n_tokens_neox': 447, 'n_words': 96}