<commit_before>/**
 * Wappalyzer v5
 *
 * Created by <NAME> <<EMAIL>>
 *
 * License: GPLv3 http://www.gnu.org/licenses/gpl-3.0.txt
 */

'use strict';

const validation = {
  hostname: /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/,
  hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/
};

class Wappalyzer {
  constructor() {
    this.apps = {};
    this.categories = {};
    this.driver = {};

    this.detected = {};
    this.hostnameCache = {};
    this.adCache = [];

    this.config = {
      websiteURL: 'https://www.wappalyzer.com/',
      twitterURL: 'https://twitter.com/Wappalyzer',
      githubURL: 'https://github.com/AliasIO/Wappalyzer',
    };
  }

  /**
   * Log messages to console
   */
  log(message, source, type) {
    this.driver.log(message, source || '', type || 'debug');
  }

  analyze(hostname, url, data, context) {
    var apps = {};

    // Remove hash from URL
    data.url = url = url.split('#')[0];

    if ( typeof data.html !== 'string' ) {
      data.html = '';
    }

    if ( this.detected[url] === undefined ) {
      this.detected[url] = {};
    }

    Object.keys(this.apps).forEach(appName => {
      apps[appName] = this.detected[url] && this.detected[url][appName] ? this.detected[url][appName] : new Application(appName, this.apps[appName]);

      var app = apps[appName];

      if ( url ) {
        this.analyzeUrl(app, url);
      }

      if ( data.html ) {
        this.analyzeHtml(app, data.html);
        this.analyzeMeta(app, data.html);
      }

      if ( data.scripts ) {
        this.analyzeScripts(app, data.scripts);
      }

      if ( data.headers ) {
        this.analyzeHeaders(app, data.headers);
      }

      if ( data.env ) {
        this.analyzeEnv(app, data.env);
      }

      if ( data.robotsTxt ) {
        this.analyzeRobotsTxt(app, data.robotsTxt);
      }
    })

    Object.keys(apps).forEach(appName => {
      var app = apps[appName];

      if ( !app.detected || !app.getConfidence() ) {
        delete apps[app.name];
      }
    });

    this.resolveExcludes(apps);
    this.resolveImplies(apps, url);

    this.cacheDetectedApps(apps, url);
    this.trackDetectedApps(apps, url, hostname, data.html);

    if ( Object.keys(apps).length ) {
      this.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url, 'core');
    }

    this.driver.displayApps(this.detected[url], context);
  }

  /**
   * Cache detected ads
   */
  cacheDetectedAds(ad) {
    this.adCache.push(ad);
  }

  /**
   *
   */
  robotsTxtAllows(url) {
    return new Promise((resolve, reject) => {
      var parsed = this.parseUrl(url);

      this.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
        .then(robotsTxt => {
          robotsTxt.forEach(disallow => parsed.pathname.indexOf(disallow) === 0 && reject());<commit_msg>Dont fetch robots.txt on non HTTP requests<commit_after>/**
 * Wappalyzer v5
 *
 * Created by <NAME> <<EMAIL>>
 *
 * License: GPLv3 http://www.gnu.org/licenses/gpl-3.0.txt
 */

'use strict';

const validation = {
  hostname: /(www.)?((.+?)\.(([a-z]{2,3}\.)?[a-z]{2,6}))$/,
  hostnameBlacklist: /((local|dev(elopment)?|stag(e|ing)?|test(ing)?|demo(shop)?|admin|google|cache)\.|\/admin|\.local)/
};

class Wappalyzer {
  constructor() {
    this.apps = {};
    this.categories = {};
    this.driver = {};

    this.detected = {};
    this.hostnameCache = {};
    this.adCache = [];

    this.config = {
      websiteURL: 'https://www.wappalyzer.com/',
      twitterURL: 'https://twitter.com/Wappalyzer',
      githubURL: 'https://github.com/AliasIO/Wappalyzer',
    };
  }

  /**
   * Log messages to console
   */
  log(message, source, type) {
    this.driver.log(message, source || '', type || 'debug');
  }

  analyze(hostname, url, data, context) {
    var apps = {};

    // Remove hash from URL
    data.url = url = url.split('#')[0];

    if ( typeof data.html !== 'string' ) {
      data.html = '';
    }

    if ( this.detected[url] === undefined ) {
      this.detected[url] = {};
    }

    Object.keys(this.apps).forEach(appName => {
      apps[appName] = this.detected[url] && this.detected[url][appName] ? this.detected[url][appName] : new Application(appName, this.apps[appName]);

      var app = apps[appName];

      if ( url ) {
        this.analyzeUrl(app, url);
      }

      if ( data.html ) {
        this.analyzeHtml(app, data.html);
        this.analyzeMeta(app, data.html);
      }

      if ( data.scripts ) {
        this.analyzeScripts(app, data.scripts);
      }

      if ( data.headers ) {
        this.analyzeHeaders(app, data.headers);
      }

      if ( data.env ) {
        this.analyzeEnv(app, data.env);
      }

      if ( data.robotsTxt ) {
        this.analyzeRobotsTxt(app, data.robotsTxt);
      }
    })

    Object.keys(apps).forEach(appName => {
      var app = apps[appName];

      if ( !app.detected || !app.getConfidence() ) {
        delete apps[app.name];
      }
    });

    this.resolveExcludes(apps);
    this.resolveImplies(apps, url);

    this.cacheDetectedApps(apps, url);
    this.trackDetectedApps(apps, url, hostname, data.html);

    if ( Object.keys(apps).length ) {
      this.log(Object.keys(apps).length + ' apps detected: ' + Object.keys(apps).join(', ') + ' on ' + url, 'core');
    }

    this.driver.displayApps(this.detected[url], context);
  }

  /**
   * Cache detected ads
   */
  cacheDetectedAds(ad) {
    this.adCache.push(ad);
  }

  /**
   *
   */
  robotsTxtAllows(url) {
    return new Promise((resolve, reject) => {
      var parsed = this.parseUrl(url);

      if ( parsed.protocol !== 'http:' && parsed.protocol !== 'https:' ) {
        reject();
      }

      this.driver.getRobotsTxt(parsed.host, parsed.protocol === 'https:')
        .then(robotsTxt => {
          robotsTxt.forEach(disallow => parsed.pathname.indexOf(disallow) === 0 && reject());
{'fraction_non_alphanumeric': 0.18698503447116194, 'fraction_numerical': 0.0033630401883302506, 'mean_word_length': 2.4906103286384975, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1501091', 'n_tokens_mistral': 2146, 'n_tokens_neox': 2085, 'n_words': 319}