<commit_before>/*
 * This module prints ActivityPub Note and Person objects based on a URL. Its
 * primary use is to print information about permalinks on Mastodon instances,
 * hence the module name, but could theoretically be used on any endpoint that
 * supports those ActivityPub object types.
 *
 * Because of the focus on Mastodon instances (on a sufficiently recent
 * version), assumptions are made about how to operate. This module is not
 * expected to be a fully-fledged ActivityPub client. Issues with handling URLs
 * on Mastodon 1.6.0 and later are considered bugs. Other issues will be
 * handled on a case-by-case basis.
 *
 * Mastodon instances can be on any hostname. Instance autodiscovery is
 * somewhat irresponsible as it would cause a request to be made every time a
 * URL is posted to a channel.
 *
 * Instances are thus discovered from other instances organically. A small list
 * of instances is included in default_instances.json, which is used in a
 * default configuration. When a toot is fetched, this bot also fetches
 * /api/v1/timelines/public and learns new instances from that.
 *
 * In case an instance isn't already known, users can share a toot URL with the
 * !masto command, which will force the bot to do an API lookup.
 *
 * Bot administrators can block hostnames from being used as instances by
 * adding them to the "blocked" key in this module's config.json.
 */

var Cacheman = require('cacheman')
  , contentType = require('content-type')
  , execSync = require('child_process').execSync
  , htmlToTextMod = require('html-to-text')
  , robotsParser = require('robots-parser')
  , robotsTxtCache = new Cacheman()
  , sprintf = require('extsprintf').sprintf
  , util = require('util')
  , urlMod = require('url');

/* variables defined in module.exports.init */
var bot = null
  , log = null
  , moduleConfig = null
  , request = null;

function robotRequest(options, fn) {
  var url = options.url;
  var userAgent = moduleConfig != null && moduleConfig.userAgent || 'EuIrcBot';
  var robotsTxtUrl = urlMod.resolve(url, '/robots.txt');

  var handle = function (robotsTxt) {
    robots = robotsParser(robotsTxtUrl, robotsTxt);
    /* `User-agent: *` rules are generally overzealous. We're not a
     * search engine spider. */
    delete robots._rules['*'];

    if (robots.isAllowed(url, userAgent)) {
      console.log(options);
      request(options, fn);
    } else {
      fn(new Error(util.format('URL access disallowed by %s: %s', robotsTxtUrl, url)));
    }
  };

  robotsTxtCache.get(robotsTxtUrl, function (err, robotsTxt) {<commit_msg>mastodon: remove extraneous console.log<commit_after>/*
 * This module prints ActivityPub Note and Person objects based on a URL. Its
 * primary use is to print information about permalinks on Mastodon instances,
 * hence the module name, but could theoretically be used on any endpoint that
 * supports those ActivityPub object types.
 *
 * Because of the focus on Mastodon instances (on a sufficiently recent
 * version), assumptions are made about how to operate. This module is not
 * expected to be a fully-fledged ActivityPub client. Issues with handling URLs
 * on Mastodon 1.6.0 and later are considered bugs. Other issues will be
 * handled on a case-by-case basis.
 *
 * Mastodon instances can be on any hostname. Instance autodiscovery is
 * somewhat irresponsible as it would cause a request to be made every time a
 * URL is posted to a channel.
 *
 * Instances are thus discovered from other instances organically. A small list
 * of instances is included in default_instances.json, which is used in a
 * default configuration. When a toot is fetched, this bot also fetches
 * /api/v1/timelines/public and learns new instances from that.
 *
 * In case an instance isn't already known, users can share a toot URL with the
 * !masto command, which will force the bot to do an API lookup.
 *
 * Bot administrators can block hostnames from being used as instances by
 * adding them to the "blocked" key in this module's config.json.
 */

var Cacheman = require('cacheman')
  , contentType = require('content-type')
  , execSync = require('child_process').execSync
  , htmlToTextMod = require('html-to-text')
  , robotsParser = require('robots-parser')
  , robotsTxtCache = new Cacheman()
  , sprintf = require('extsprintf').sprintf
  , util = require('util')
  , urlMod = require('url');

/* variables defined in module.exports.init */
var bot = null
  , log = null
  , moduleConfig = null
  , request = null;

function robotRequest(options, fn) {
  var url = options.url;
  var userAgent = moduleConfig != null && moduleConfig.userAgent || 'EuIrcBot';
  var robotsTxtUrl = urlMod.resolve(url, '/robots.txt');

  var handle = function (robotsTxt) {
    robots = robotsParser(robotsTxtUrl, robotsTxt);
    /* `User-agent: *` rules are generally overzealous. We're not a
     * search engine spider. */
    delete robots._rules['*'];

    if (robots.isAllowed(url, userAgent)) {
      request(options, fn);
    } else {
      fn(new Error(util.format('URL access disallowed by %s: %s', robotsTxtUrl, url)));
    }
  };

  robotsTxtCache.get(robotsTxtUrl, function (err, robotsTxt) {
{'fraction_non_alphanumeric': 0.09262832883056735, 'fraction_numerical': 0.0015438054805094559, 'mean_word_length': 3.8484565014031804, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '782007', 'n_tokens_mistral': 1513, 'n_tokens_neox': 1469, 'n_words': 604}