<commit_before>#-------------------------------------------------------------------------------
# Name:         sfp_psbdmp
# Purpose:      Query psbdmp.com for potentially hacked e-mail addresses.
#
# Author:      <NAME> <<EMAIL>>
#
# Created:     21/11/2016
# Copyright:   (c) <NAME>
# Licence:     GPL
#-------------------------------------------------------------------------------

import sys
import json
import time
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_psbdmp(SpiderFootPlugin):
    """Psbdmp.com:Footprint,Investigate,Passive:Search Engines::Check psbdmp.com (PasteBin Dump) for potentially hacked e-mails and domains."""

    # Default options
    opts = { 
    }

    # Option descriptions
    optdescs = {
    }

    # Be sure to completely clear any class variables in setup()
    # or you run the risk of data persisting between scan runs.

    results = dict()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = dict()

        # Clear / reset any other class member variables here
        # or you risk them persisting between threads.

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        ret = ["EMAILADDR", "DOMAIN_NAME"]

        return ret

    # What events this module produces
    def producedEvents(self):
        ret = ["LEAKSITE_URL"]

        return ret

    def query(self, qry):
        ret = None

        if "@" in qry:
            url = "http://psbdmp.com/api/search/email/" + qry
        else:
            url = "http://psbdmp.com/api/search/domain/" + qry

        # https://haveibeenpsbdmp.com/API/v2#RateLimiting
        time.sleep(1)
        res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], 
            useragent="SpiderFoot")

        try:
            ret = json.loads(res['content'])
        except Exception as e:
            self.sf.error("Error processing JSON response from psbdmp.com: " + str(e), False)
            return None
        <commit_msg>Don't report errors for 403.<commit_after>#-------------------------------------------------------------------------------
# Name:         sfp_psbdmp
# Purpose:      Query psbdmp.com for potentially hacked e-mail addresses.
#
# Author:      <NAME> <<EMAIL>>
#
# Created:     21/11/2016
# Copyright:   (c) <NAME>
# Licence:     GPL
#-------------------------------------------------------------------------------

import sys
import json
import time
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_psbdmp(SpiderFootPlugin):
    """Psbdmp.com:Footprint,Investigate,Passive:Search Engines::Check psbdmp.com (PasteBin Dump) for potentially hacked e-mails and domains."""

    # Default options
    opts = { 
    }

    # Option descriptions
    optdescs = {
    }

    # Be sure to completely clear any class variables in setup()
    # or you run the risk of data persisting between scan runs.

    results = dict()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = dict()

        # Clear / reset any other class member variables here
        # or you risk them persisting between threads.

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        ret = ["EMAILADDR", "DOMAIN_NAME"]

        return ret

    # What events this module produces
    def producedEvents(self):
        ret = ["LEAKSITE_URL"]

        return ret

    def query(self, qry):
        ret = None

        if "@" in qry:
            url = "http://psbdmp.com/api/search/email/" + qry
        else:
            url = "http://psbdmp.com/api/search/domain/" + qry

        res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], 
            useragent="SpiderFoot")

        if res['code'] == 403:
            self.sf.info("Unable to fetch data from psbdmp.com right now.")
            return None

        try:
            ret = json.loads(res['content'])
        except Exception as e:
            self.sf.error("Error processing JSON response from psbdmp.com: " + str(e), False)
            return None
        
{'fraction_non_alphanumeric': 0.17130821436929558, 'fraction_numerical': 0.005616662766206412, 'mean_word_length': 1.7898172323759791, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '881158', 'n_tokens_mistral': 1244, 'n_tokens_neox': 1172, 'n_words': 353}