<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys
import PyV8
import urllib
from urlparse import urlparse

try:
    import json
except ImportError:
    import simplejson as json


class Wappalyzer(object):

    def __init__(self, url):
        self.file_dir = os.path.dirname(__file__)

        f = open(os.path.join(self.file_dir, '../../share/apps.json'))
        data = json.loads(f.read())
        f.close()

        self.categories = data['categories']
        self.apps = data['apps']
        self.url = url

    def analyze(self):
        ctxt = PyV8.JSContext()
        ctxt.enter()

        f1 = open(os.path.join(self.file_dir, '../php/js/wappalyzer.js'))
        f2 = open(os.path.join(self.file_dir, '../php/js/driver.js'))
        ctxt.eval(f1.read())
        ctxt.eval(f2.read())
        f1.close()
        f2.close()

        host = urlparse(self.url).hostname
        html = urllib.urlopen(self.url).read()

        data = {'host': host, 'url': self.url, 'html': html, 'headers': {}}
        apps = json.dumps(self.apps)
        categories = json.dumps(self.categories)
        return ctxt.eval("w.apps = %s; w.categories = %s; w.driver.data = %s; w.driver.init();" % (apps, categories, json.dumps(data)))

if __name__ == '__main__':
    try:
        w = Wappalyzer(sys.argv[1])
        print w.analyze()
    except IndexError:
        print ('Usage: python %s <url>' % sys.argv[0])<commit_msg>Fix path to wappalyzer.js; use requests instead of urllib<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys
import PyV8
import requests
from urlparse import urlparse

try:
    import json
except ImportError:
    import simplejson as json


class Wappalyzer(object):

    def __init__(self, url):
        self.file_dir = os.path.dirname(__file__)

        f = open(os.path.join(self.file_dir, '../../share/apps.json'))
        data = json.loads(f.read())
        f.close()

        self.categories = data['categories']
        self.apps = data['apps']
        self.url = url

    def analyze(self):
        ctxt = PyV8.JSContext()
        ctxt.enter()

        f1 = open(os.path.join(self.file_dir, '../../share/js/wappalyzer.js'))
        f2 = open(os.path.join(self.file_dir, '../php/js/driver.js'))
        ctxt.eval(f1.read())
        ctxt.eval(f2.read())
        f1.close()
        f2.close()

        host = urlparse(self.url).hostname
        html = requests.get(self.url).text

        data = {'host': host, 'url': self.url, 'html': html, 'headers': {}}
        apps = json.dumps(self.apps)
        categories = json.dumps(self.categories)
        return ctxt.eval("w.apps = %s; w.categories = %s; w.driver.data = %s; w.driver.init();" % (apps, categories, json.dumps(data)))

if __name__ == '__main__':
    try:
        w = Wappalyzer(sys.argv[1])
        print w.analyze()
    except IndexError:
        print ('Usage: python %s <url>' % sys.argv[0])
{'fraction_non_alphanumeric': 0.16467780429594273, 'fraction_numerical': 0.007500852369587453, 'mean_word_length': 2.3685419058553387, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4484311', 'n_tokens_mistral': 1017, 'n_tokens_neox': 1005, 'n_words': 211}