<commit_before>'use strict';

const Wappalyzer = require('./wappalyzer');
const request = require('request');
const url = require('url');
const fs = require('fs');
const Browser = require('zombie');

const json = JSON.parse(fs.readFileSync(__dirname + '/apps.json'));

class Driver {
  constructor(pageUrl, options) {
    this.options = Object.assign({}, {
      debug: false,
      delay: 500,
      maxDepth: 3,
      maxUrls: 10,
      maxWait: 1000,
      recursive: false,
      requestTimeout: 3000,
      userAgent: 'Mozilla/5.0 (compatible; Wappalyzer)',
    }, options || {});

    this.options.debug = Boolean(this.options.debug);
    this.options.recursive = Boolean(this.options.recursive);
    this.options.delay = this.options.recursive ? parseInt(this.options.delay, 10) : 0;
    this.options.maxDepth = parseInt(this.options.maxDepth, 10);
    this.options.maxUrls = parseInt(this.options.maxUrls, 10);
    this.options.maxWait = parseInt(this.options.maxWait, 10);

    this.origPageUrl = url.parse(pageUrl);
    this.analyzedPageUrls = [];
    this.apps = [];
    this.meta = {};

    this.wappalyzer = new Wappalyzer();

    this.wappalyzer.apps = json.apps;
    this.wappalyzer.categories = json.categories;

    this.wappalyzer.parseJsPatterns();

    this.wappalyzer.driver.log = (message, source, type) => this.log(message, source, type);
    this.wappalyzer.driver.displayApps = (detected, meta, context) => this.displayApps(detected, meta, context);
  }

  analyze() {
    this.time = {
      start: new Date().getTime(),
      last: new Date().getTime(),
    }

    return this.crawl(this.origPageUrl);
  }

  log(message, source, type) {
    this.options.debug && console.log('[wappalyzer ' + type + ']', '[' + source + ']', message);
  }

  displayApps(detected, meta) {
    this.timer('displayApps');

    this.meta = meta;

    Object.keys(detected).forEach(appName => {
      const app = detected[appName];

      var categories = [];

      app.props.cats.forEach(id => {
        var category = {};

        category[id] = json.categories[id].name;

        categories.push(category)
      });

      if ( !this.apps.some(detectedApp => detectedApp.name === app.name) ) {
        this.apps.push({
          name: app.name,
          confidence: app.confidenceTotal.toString(),
          version: app.version,
          icon: app.props.icon || 'default.svg',
          website: app.props.website,
          categories
        });
      }
    });
  }

  fetch(pageUrl, index, depth) {
    this.timer('fetch');

    return new Promise(resolve => {
      // Return when the URL is a duplicate or maxUrls has been reached
      if ( this.analyzedPageUrls.indexOf(pageUrl.href) !== -1 || this.analyzedPageUrls.length >= this.options.maxUrls ) {
        return resolve();
      }

      this.analyzedPageUrls.push(pageUrl.href);

      this.wappalyzer.log('depth: ' + depth + '; delay: ' + ( this.options.delay * index ) + 'ms; url: ' + pageUrl.href, 'driver');

      const browser = new Browser({
        silent: true,
        userAgent: this.options.userAgent,
        waitDuration: this.options.maxWait + 'ms',
      });

      this.sleep(this.options.delay * index)
        .then(() => {
          this.timer('browser.visit start');

          browser.visit(pageUrl.href, this.options.requestTimeout, error => {
            this.timer('browser.visit end');

            if ( !browser.resources['0'] || !browser.resources['0'].response ) {
              this.wappalyzer.log('No response from server', 'browser', 'error');

              return resolve();
            }

            browser.wait(this.options.maxWait)
              .catch(error => this.wappalyzer.log(error.message, 'browser', 'error'))
              .finally(() => {
                this.timer('browser.wait end');

                const headers = this.getHeaders(browser);
                const html    = this.getHtml(browser);
                const scripts = this.getScripts(browser);
                const js      = this.getJs(browser);

                this.wappalyzer.analyze(pageUrl.hostname, pageUrl.href, {
                  headers,
                  html,
                  js,
                  scripts
                });

                const links = browser.body.getElementsByTagName('a');

                resolve(links);
              });
          });
        });
    });
  }

  getHeaders(browser) {
    const headers = {};

    browser.resources['0'].response.headers._headers.forEach(header => {
      if ( !headers[header[0]] ){
        headers[header[0]] = [];
      }

      headers[header[0]].push(header[1]);
    });

    return headers;
  }<commit_msg>Replace promise with callback in browser.wait<commit_after>'use strict';

const Wappalyzer = require('./wappalyzer');
const request = require('request');
const url = require('url');
const fs = require('fs');
const Browser = require('zombie');

const json = JSON.parse(fs.readFileSync(__dirname + '/apps.json'));

class Driver {
  constructor(pageUrl, options) {
    this.options = Object.assign({}, {
      debug: false,
      delay: 500,
      maxDepth: 3,
      maxUrls: 10,
      maxWait: 1000,
      recursive: false,
      requestTimeout: 3000,
      userAgent: 'Mozilla/5.0 (compatible; Wappalyzer)',
    }, options || {});

    this.options.debug = Boolean(this.options.debug);
    this.options.delay = this.options.recursive ? parseInt(this.options.delay, 10) : 0;
    this.options.maxDepth = parseInt(this.options.maxDepth, 10);
    this.options.maxUrls = parseInt(this.options.maxUrls, 10);
    this.options.maxWait = parseInt(this.options.maxWait, 10);
    this.options.recursive = Boolean(this.options.recursive);
    this.options.requestTimeout = parseInt(this.options.requestTimeout, 10);

    this.origPageUrl = url.parse(pageUrl);
    this.analyzedPageUrls = [];
    this.apps = [];
    this.meta = {};

    this.wappalyzer = new Wappalyzer();

    this.wappalyzer.apps = json.apps;
    this.wappalyzer.categories = json.categories;

    this.wappalyzer.parseJsPatterns();

    this.wappalyzer.driver.log = (message, source, type) => this.log(message, source, type);
    this.wappalyzer.driver.displayApps = (detected, meta, context) => this.displayApps(detected, meta, context);
  }

  analyze() {
    this.time = {
      start: new Date().getTime(),
      last: new Date().getTime(),
    }

    return this.crawl(this.origPageUrl);
  }

  log(message, source, type) {
    this.options.debug && console.log('[wappalyzer ' + type + ']', '[' + source + ']', message);
  }

  displayApps(detected, meta) {
    this.timer('displayApps');

    this.meta = meta;

    Object.keys(detected).forEach(appName => {
      const app = detected[appName];

      var categories = [];

      app.props.cats.forEach(id => {
        var category = {};

        category[id] = json.categories[id].name;

        categories.push(category)
      });

      if ( !this.apps.some(detectedApp => detectedApp.name === app.name) ) {
        this.apps.push({
          name: app.name,
          confidence: app.confidenceTotal.toString(),
          version: app.version,
          icon: app.props.icon || 'default.svg',
          website: app.props.website,
          categories
        });
      }
    });
  }

  fetch(pageUrl, index, depth) {
    this.timer('fetch');

    return new Promise(resolve => {
      // Return when the URL is a duplicate or maxUrls has been reached
      if ( this.analyzedPageUrls.indexOf(pageUrl.href) !== -1 || this.analyzedPageUrls.length >= this.options.maxUrls ) {
        return resolve();
      }

      this.analyzedPageUrls.push(pageUrl.href);

      this.wappalyzer.log('depth: ' + depth + '; delay: ' + ( this.options.delay * index ) + 'ms; url: ' + pageUrl.href, 'driver');

      const browser = new Browser({
        silent: true,
        userAgent: this.options.userAgent,
        waitDuration: this.options.maxWait,
      });

      this.sleep(this.options.delay * index)
        .then(() => {
          this.timer('browser.visit start');

          browser.visit(pageUrl.href, this.options.requestTimeout, error => {
            this.timer('browser.visit end');

            if ( !browser.resources['0'] || !browser.resources['0'].response ) {
              this.wappalyzer.log('No response from server', 'browser', 'error');

              return resolve();
            }

            browser.wait(this.options.maxWait, () => {
              this.timer('browser.wait end');

              const headers = this.getHeaders(browser);
              const html    = this.getHtml(browser);
              const scripts = this.getScripts(browser);
              const js      = this.getJs(browser);

              this.wappalyzer.analyze(pageUrl.hostname, pageUrl.href, {
                headers,
                html,
                js,
                scripts
              });

              const links = browser.body.getElementsByTagName('a');

              resolve(links);
            });
          });
        });
    });
  }

  getHeaders(browser) {
    const headers = {};

    browser.resources['0'].response.headers._headers.forEach(header => {
      if ( !headers[header[0]] ){
        headers[header[0]] = [];
      }

      headers[header[0]].push(header[1]);
    });

    return headers;
  }
{'fraction_non_alphanumeric': 0.1497584541062802, 'fraction_numerical': 0.007300053676865271, 'mean_word_length': 2.3141230878690857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4634728', 'n_tokens_mistral': 2905, 'n_tokens_neox': 2765, 'n_words': 504}