<commit_before>#-------------------------------------------------------------------------------
# Name:         sfp_builtwith
# Purpose:      Query builtwith.com using their API.
#
# Author:      <NAME> <<EMAIL>>
#
# Created:     10/08/2017
# Copyright:   (c) <NAME>
# Licence:     GPL
#-------------------------------------------------------------------------------

import sys
import json
import time
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_builtwith(SpiderFootPlugin):
    """BuiltWith:Footprint,Investigate,Passive:Search Engines:apikey:Query BuiltWith.com's Domain API for information about your target's web technology stack, e-mail addresses and more."""


    # Default options
    opts = { 
        "api_key": "",
        "maxage": 30
    }

    # Option descriptions
    optdescs = {
        "api_key": "Builtwith.com Domain API key.",
        "maxage": "The maximum age of the data returned, in days, in order to be considered valid."
    }

    # Be sure to completely clear any class variables in setup()
    # or you run the risk of data persisting between scan runs.

    results = dict()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = dict()

        # Clear / reset any other class member variables here
        # or you risk them persisting between threads.

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return [ "DOMAIN_NAME" ]

    # What events this module produces
    def producedEvents(self):
        return [ "INTERNET_NAME", "EMAILADDR", "HUMAN_NAME", 
                 "WEB_TECHNOLOGY", "PHONE_NUMBER" ]

    def query(self, t):
        ret = None

        url = "https://api.builtwith.com/v11/api.json?LOOKUP=" + t + "&KEY=" + self.opts['api_key']

        res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], 
            useragent="SpiderFoot")

        if res['code'] == "404":
            return None

        if not res['content']:
            return None

        try:
            ret = json.loads(res['content'])['Results'][0]
        except Exception as e:
            self.sf.error("Error processing JSON response from builtwith.com: " + str(e), False)
            return None

        return ret

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)

       # Don't look up stuff twice
        if self.results.has_key(eventData):
            self.sf.debug("Skipping " + eventData + " as already mapped.")
            return None
        else:
            self.results[eventData] = True

        data = self.query(eventData)
        if data == None:
            return None

        if "Meta" in data:
            if data['Meta'].get("Names", []):
                for nb in data['Meta']['Names']:
                    e = SpiderFootEvent("HUMAN_NAME", nb['Name'], 
                                        self.__name__, event)
                    self.notifyListeners(e)
                    if nb.get('Email', None):
                        e = SpiderFootEvent("EMAILADDR", nb['Email'], 
                                            self.__name__, event)
                        self.notifyListeners(e)

            if data['Meta'].get("Emails", []):
                for email in data['Meta']['Emails']:
                    e = SpiderFootEvent("EMAILADDR", email,
                                        self.__name__, event)
                    self.notifyListeners(e)
<commit_msg>Check if API key was set<commit_after>#-------------------------------------------------------------------------------
# Name:         sfp_builtwith
# Purpose:      Query builtwith.com using their API.
#
# Author:      <NAME> <<EMAIL>>
#
# Created:     10/08/2017
# Copyright:   (c) <NAME>
# Licence:     GPL
#-------------------------------------------------------------------------------

import sys
import json
import time
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_builtwith(SpiderFootPlugin):
    """BuiltWith:Footprint,Investigate,Passive:Search Engines:apikey:Query BuiltWith.com's Domain API for information about your target's web technology stack, e-mail addresses and more."""


    # Default options
    opts = { 
        "api_key": "",
        "maxage": 30
    }

    # Option descriptions
    optdescs = {
        "api_key": "Builtwith.com Domain API key.",
        "maxage": "The maximum age of the data returned, in days, in order to be considered valid."
    }

    # Be sure to completely clear any class variables in setup()
    # or you run the risk of data persisting between scan runs.

    results = dict()
    errorState = False

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = dict()

        # Clear / reset any other class member variables here
        # or you risk them persisting between threads.

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return [ "DOMAIN_NAME" ]

    # What events this module produces
    def producedEvents(self):
        return [ "INTERNET_NAME", "EMAILADDR", "HUMAN_NAME", 
                 "WEB_TECHNOLOGY", "PHONE_NUMBER" ]

    def query(self, t):
        ret = None

        url = "https://api.builtwith.com/v11/api.json?LOOKUP=" + t + "&KEY=" + self.opts['api_key']

        res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], 
            useragent="SpiderFoot")

        if res['code'] == "404":
            return None

        if not res['content']:
            return None

        try:
            ret = json.loads(res['content'])['Results'][0]
        except Exception as e:
            self.sf.error("Error processing JSON response from builtwith.com: " + str(e), False)
            return None

        return ret

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        if self.opts['api_key'] == "":
            self.sf.error("You enabled sfp_builtwith but did not set an API key!", False)
            self.errorState = True
            return None

       # Don't look up stuff twice
        if self.results.has_key(eventData):
            self.sf.debug("Skipping " + eventData + " as already mapped.")
            return None
        else:
            self.results[eventData] = True

        data = self.query(eventData)
        if data == None:
            return None

        if "Meta" in data:
            if data['Meta'].get("Names", []):
                for nb in data['Meta']['Names']:
                    e = SpiderFootEvent("HUMAN_NAME", nb['Name'], 
                                        self.__name__, event)
                    self.notifyListeners(e)
                    if nb.get('Email', None):
                        e = SpiderFootEvent("EMAILADDR", nb['Email'], 
                                            self.__name__, event)
                        self.notifyListeners(e)

            if data['Meta'].get("Emails", []):
                for email in data['Meta']['Emails']:
                    e = SpiderFootEvent("EMAILADDR", email,
                                        self.__name__, event)
                    self.notifyListeners(e)

{'fraction_non_alphanumeric': 0.14571169067081874, 'fraction_numerical': 0.004152069547164915, 'mean_word_length': 1.6801112656467316, 'pattern_counts': {'":': 11, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4519630', 'n_tokens_mistral': 2107, 'n_tokens_neox': 1999, 'n_words': 588}