<commit_before>#!/usr/bin/env python

'''
    Author : <NAME> (@WebBreacher)
    Description : Takes each username from the web_accounts_list.json file and performs the lookup to see if the
                  discovery determinator is still valid

    TODO - 
        1 - Make it so the script will toggle validity factor per entry and write to output file
        2 - Make it so the script will append comment to the entry and output to file
        3 - Colorize output
        4 - Make a stub file shows last time sites were checked and problems.

    ISSUES -
        1 - Had an issue with SSL handshakes and this script. Had to do the following to get it working
            [From https://github.com/kennethreitz/requests/issues/2022]
            # sudo apt-get install libffi-dev
            # pip install pyOpenSSL ndg-httpsclient pyasn1
'''
import requests
import json
import os
import random
import datetime
import string
import signal
import sys

###################
# Variables && Functions
###################
# Set HTTP Header info.
headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0'}
overall_results = {}

def signal_handler(signal, frame):
    print('[!!!] You pressed Ctrl+C. Exitting script.')
    FinalOutput()
    sys.exit(0)

def web_call(url):
    try:
        # Make web request for that URL, timeout in X secs and don't verify SSL/TLS certs
        r = requests.get(url, headers=headers, timeout=30, verify=False)
    except requests.exceptions.Timeout:
        r = '     [!] ERROR: CONNECTION TIME OUT. Try increasing the timeout delay.'
        return
    except requests.exceptions.TooManyRedirects:
        r = '     [!] ERROR: TOO MANY REDIRECTS. Try changing the URL.'
        return
    except requests.exceptions.RequestException as e:
        r = '     [!] ERROR: CRITICAL ERROR. %s' % e
        return
    else:
        return r

def FinalOutput():
    if len(overall_results) > 0:
        print '------------'
        print 'The following previously "valid" sites had errors:'
        for site, results in overall_results.iteritems():
            print '     %s --> %s' % (site, results)
    else:
        print ":) No problems with the JSON file were found."

###################
# Main
###################
signal.signal(signal.SIGINT, signal_handler)
# Read in the JSON file
with open('web_accounts_list.json') as data_file:    
    data = json.load(data_file)
print '[-] %s sites found in file.' % len(data['sites'])

for site in data['sites'] :
    code_match, string_match = False, False
    # Examine the current validity of the entry
    if site['valid'] == False:
        print '[!] Skipping %s - Marked as not valid.' % site['name']
        continue
    if site['known_accounts'][0] == False:
        print '[!] Skipping %s - No valid user names to test.' % site['name']
        continue

    # Perform initial lookup
    # Pull the first user from known_accounts and replace the {account} with it
    url = site['check_uri'].replace("{account}", site['known_accounts'][0])
    print '[-] Looking up %s' % url
    r = web_call(url)
    if isinstance(r, str):
        # We got an error on the web call
        print r
        continue

    # Analyze the responses against what they should be
    if r.status_code == int(site['account_existence_code']):
        code_match = True
    else:
        code_match = False
    if r.text.find(site['account_existence_string']) > 0:
        string_match = True
    else:
        string_match = False
    
    if code_match == True and string_match == True:
        #print '     [+] Response code and Search Strings match expected.'
        # Generate a random string to use in place of known_accounts
        not_there_string = ''.join(random.choice(string.ascii_lowercase + string.ascii_uppercase + string.digits) for x in range(20))
        url_fp = site['check_uri'].replace("{account}", not_there_string)
        r_fp = web_call(url_fp)
        if isinstance(r_fp, str):
            # If this is a string then web got an error
            print r_fp
            continue

        if r_fp.status_code == int(site['account_existence_code']):
            code_match = True
        else:
            code_match = False
        if r_fp.text.find(site['account_existence_string']) > 0:
            string_match = True
        else:
            string_match = False
        if code_match == True and string_match == True:
            print '     [-] Code: %s; String: %s' % (code_match, string_match)
            print '     [!] ERROR: FALSE POSITIVE DETECTED. Response code and Search Strings match expected.'
            #TODO set site['valid'] = False
            overall_results[site['name']] = 'False Positive'
        else:
            #print '     [+] Passed false positives test.'
            pass
    elif code_match == True and string_match == False:
        #TODO set site['valid'] = False
        print '     [!] ERROR: BAD DETECTION STRING. "%s" was not found on resulting page.' % site['account_existence_string']
        overall_results[site['name']] = 'Bad detection string.'
    elif code_match == False and string_match == True:
        #TODO set site['valid'] = False
        print '     [!] ERROR: BAD DETECTION RESPONSE CODE. HTTP Response code different than expected.'
        overall_results[site['name']] = 'Bad detection code. Expected: %s; Received: %s.' % (str(r.status_code), site['account_existence_code'])
    else:
        #TODO set site['valid'] = False
        print '     [!] ERROR: BAD CODE AND STRING. Neither the HTTP response code or detection string worked.'
        overall_results[site['name']] = 'Bad detection code and string. Expected Code: %s; Received Code: %s.' % (str(r.status_code), site['account_existence_code'])

FinalOutput()
<commit_msg>Need to fix openssl error<commit_after>#!/usr/bin/env python

'''
    Author : <NAME> (@WebBreacher)
    Description : Takes each username from the web_accounts_list.json file and performs the lookup to see if the
                  discovery determinator is still valid

    TODO - 
        1 - Make it so the script will toggle validity factor per entry and write to output file
        2 - Make it so the script will append comment to the entry and output to file
        3 - Colorize output
        4 - Make a stub file shows last time sites were checked and problems.

    ISSUES -
        1 - Had an issue with SSL handshakes and this script. Had to do the following to get it working
            [From https://github.com/kennethreitz/requests/issues/2022]
            # sudo apt-get install libffi-dev
            # pip install pyOpenSSL ndg-httpsclient pyasn1
'''
import requests
import json
import os
import random
import datetime
import string
import signal
import sys

###################
# Variables && Functions
###################
# Set HTTP Header info.
headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0'}
overall_results = {}

def signal_handler(signal, frame):
    print('[!!!] You pressed Ctrl+C. Exitting script.')
    FinalOutput()
    sys.exit(0)

def web_call(url):
    try:
        # Make web request for that URL, timeout in X secs and don't verify SSL/TLS certs
        r = requests.get(url, headers=headers, timeout=30, verify=False)
    except requests.exceptions.Timeout:
        return '     [!] ERROR: CONNECTION TIME OUT. Try increasing the timeout delay.'
    except requests.exceptions.TooManyRedirects:
        return '     [!] ERROR: TOO MANY REDIRECTS. Try changing the URL.'
    except requests.exceptions.RequestException as e:
        return '     [!] ERROR: CRITICAL ERROR. %s' % e
    else:
        return r

def FinalOutput():
    if len(overall_results) > 0:
        print '------------'
        print 'The following previously "valid" sites had errors:'
        for site, results in overall_results.iteritems():
            print '     %s --> %s' % (site, results)
    else:
        print ":) No problems with the JSON file were found."

###################
# Main
###################
signal.signal(signal.SIGINT, signal_handler)
# Read in the JSON file
with open('web_accounts_list.json') as data_file:    
    data = json.load(data_file)
print '[-] %s sites found in file.' % len(data['sites'])

for site in data['sites'] :
    code_match, string_match = False, False
    # Examine the current validity of the entry
    if site['valid'] == False:
        print '[!] Skipping %s - Marked as not valid.' % site['name']
        continue
    if site['known_accounts'][0] == False:
        print '[!] Skipping %s - No valid user names to test.' % site['name']
        continue

    # Perform initial lookup
    # Pull the first user from known_accounts and replace the {account} with it
    url = site['check_uri'].replace("{account}", site['known_accounts'][0])
    print '[-] Looking up %s' % url
    r = web_call(url)
    if isinstance(r, str):
        # We got an error on the web call
        print r
        continue

    # Analyze the responses against what they should be
    if r.status_code == int(site['account_existence_code']):
        code_match = True
    else:
        code_match = False
    if r.text.find(site['account_existence_string']) > 0:
        string_match = True
    else:
        string_match = False
    
    if code_match == True and string_match == True:
        #print '     [+] Response code and Search Strings match expected.'
        # Generate a random string to use in place of known_accounts
        not_there_string = ''.join(random.choice(string.ascii_lowercase + string.ascii_uppercase + string.digits) for x in range(20))
        url_fp = site['check_uri'].replace("{account}", not_there_string)
        r_fp = web_call(url_fp)
        if isinstance(r_fp, str):
            # If this is a string then web got an error
            print r_fp
            continue

        if r_fp.status_code == int(site['account_existence_code']):
            code_match = True
        else:
            code_match = False
        if r_fp.text.find(site['account_existence_string']) > 0:
            string_match = True
        else:
            string_match = False
        if code_match == True and string_match == True:
            print '     [-] Code: %s; String: %s' % (code_match, string_match)
            print '     [!] ERROR: FALSE POSITIVE DETECTED. Response code and Search Strings match expected.'
            #TODO set site['valid'] = False
            overall_results[site['name']] = 'False Positive'
        else:
            #print '     [+] Passed false positives test.'
            pass
    elif code_match == True and string_match == False:
        #TODO set site['valid'] = False
        print '     [!] ERROR: BAD DETECTION STRING. "%s" was not found on resulting page.' % site['account_existence_string']
        overall_results[site['name']] = 'Bad detection string.'
    elif code_match == False and string_match == True:
        #TODO set site['valid'] = False
        print '     [!] ERROR: BAD DETECTION RESPONSE CODE. HTTP Response code different than expected.'
        overall_results[site['name']] = 'Bad detection code. Expected: %s; Received: %s.' % (str(r.status_code), site['account_existence_code'])
    else:
        #TODO set site['valid'] = False
        print '     [!] ERROR: BAD CODE AND STRING. Neither the HTTP response code or detection string worked.'
        overall_results[site['name']] = 'Bad detection code and string. Expected Code: %s; Received Code: %s.' % (str(r.status_code), site['account_existence_code'])

FinalOutput()

{'fraction_non_alphanumeric': 0.1134020618556701, 'fraction_numerical': 0.006202330064537759, 'mean_word_length': 2.3936291240045504, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1657393', 'n_tokens_mistral': 3599, 'n_tokens_neox': 3356, 'n_words': 1151}