<commit_before>#
# pyfuzz v0.2 By <NAME> ( www.AyoobAli.com )
#
import http.client
import sys
import os
import getopt
from optparse import OptionParser
import string
import signal

def signal_handler(signal, frame):
	print("\nScan stopped by user.")
	sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)

def main():
	
	
    parser = OptionParser(usage="%prog -u http://example.com/en/ -l sharepoint.txt", version="%prog 0.1")
    parser.add_option("-u", "--url",   dest="targetURL", help="Target URL to scan")
    parser.add_option("-l", "--list",  dest="listFile",  help="List of paths to scan")
    parser.add_option("-r", "--redirect", action="store_true", dest="showRedirect", help="Show redirect codes (3xx)")
    (options, args) = parser.parse_args()

    if options.listFile == None or options.targetURL == None:
        parser.print_help()
        sys.exit()

    if not os.path.isfile(options.listFile):
        print("Error: File (", options.listFile, ") doesn't exist.")
        sys.exit()

    if options.targetURL[-1] != "/":
        options.targetURL += "/"

    targetPro = ""

    if options.targetURL[:5].lower() == 'https':
        targetDomain = options.targetURL[8:].split("/",1)[0].lower()
        targetPath = "/" + options.targetURL[8:].split("/",1)[1]
        connection = http.client.HTTPSConnection(targetDomain)
        targetPro = "https://"
        print("Target: ", targetPro+targetDomain, "(over HTTPS)")
        print("Path: ", targetPath)
    elif options.targetURL[:5].lower() == 'http:':
        targetDomain = options.targetURL[7:].split("/",1)[0].lower()
        targetPath = "/"+options.targetURL[7:].split("/",1)[1]
        connection = http.client.HTTPConnection(targetDomain)
        targetPro = "http://"
        print("Target set: ", targetDomain)
        print("Path: ", targetPath)
    else:
        targetDomain = options.targetURL.split("/",1)[0].lower()
        targetPath = "/"+options.targetURL.split("/",1)[1]
        connection = http.client.HTTPConnection(targetDomain)
        targetPro = "http://"
        print("Target set: ", targetDomain)
        print("Path: ", targetPath)

    connection.request("HEAD", targetPath+"randomhy27dtwjwysg.txt")
    res = connection.getresponse()

    if res.status == 200:
        print("NOTE: Looks like the server is returning code 200 for all requests, there might be lots of false positive links.")

    if res.status >= 300 and res.status < 400 and options.showRedirect != None:
        print("NOTE: Looks like the server is returning code", res.status, "for all requests, there might be lots of false positive links. try to scan without the option -r")

    tpData = res.read()

    with open(options.listFile) as lFile:
        pathList = lFile.readlines()
    totalURLs = len(pathList)
    print ("Scanning (",totalURLs,") files...")
    countFound = 0
    countAll = 0
    strLine = ""
    for pathLine in pathList:
        countAll = countAll + 1
        pathLine = pathLine.strip("\n")
        pathLine = pathLine.strip("\r")

        if pathLine != "":
            if pathLine[:1] == "/":
                pathLine = pathLine[1:]
            print (' ' * len(strLine), "\r", end="")
            strLine = "Checking ["+str(countAll)+"/"+str(totalURLs)+"] "+targetPath+pathLine
            print (strLine,"\r", end="")
            connection.request("GET", targetPath+pathLine)
            res = connection.getresponse()

            if res.status == 200:
                print (' ' * len(strLine), "\r", end="")
                print("Code", res.status,":",targetPro+targetDomain+targetPath+pathLine)
                countFound += 1

            if options.showRedirect != None:
                if res.status >= 300 and res.status < 400:
                    print (' ' * len(strLine), "\r", end="")
                    print("Code", res.status,":",targetPro+targetDomain+targetPath+pathLine, "(",res.getheader("location"),")")
                    countFound += 1

            tpData = res.read()

    connection.close()
    print (' ' * len(strLine), "\r", end="")
    print ( "Total Pages found:",countFound )<commit_msg>Fixed SSL Error<commit_after>#
# pyfuzz v0.3 By <NAME> ( www.AyoobAli.com )
#
import http.client
import sys
import os
import getopt
from optparse import OptionParser
import string
import signal
import ssl

def signal_handler(signal, frame):
	print("\nScan stopped by user.")
	sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)

def main():
	
	
    parser = OptionParser(usage="%prog -u http://example.com/en/ -l sharepoint.txt", version="%prog 0.1")
    parser.add_option("-u", "--url",   dest="targetURL", help="Target URL to scan")
    parser.add_option("-l", "--list",  dest="listFile",  help="List of paths to scan")
    parser.add_option("-r", "--redirect", action="store_true", dest="showRedirect", help="Show redirect codes (3xx)")
    (options, args) = parser.parse_args()

    if options.listFile == None or options.targetURL == None:
        parser.print_help()
        sys.exit()

    if not os.path.isfile(options.listFile):
        print("Error: File (", options.listFile, ") doesn't exist.")
        sys.exit()

    if options.targetURL[-1] != "/":
        options.targetURL += "/"

    targetPro = ""

    if options.targetURL[:5].lower() == 'https':
        targetDomain = options.targetURL[8:].split("/",1)[0].lower()
        targetPath = "/" + options.targetURL[8:].split("/",1)[1]
        connection = http.client.HTTPSConnection(targetDomain, timeout=20, context=ssl._create_unverified_context())
        targetPro = "https://"
        print("Target: ", targetPro+targetDomain, "(over HTTPS)")
        print("Path: ", targetPath)
    elif options.targetURL[:5].lower() == 'http:':
        targetDomain = options.targetURL[7:].split("/",1)[0].lower()
        targetPath = "/"+options.targetURL[7:].split("/",1)[1]
        connection = http.client.HTTPConnection(targetDomain)
        targetPro = "http://"
        print("Target set: ", targetDomain)
        print("Path: ", targetPath)
    else:
        targetDomain = options.targetURL.split("/",1)[0].lower()
        targetPath = "/"+options.targetURL.split("/",1)[1]
        connection = http.client.HTTPConnection(targetDomain)
        targetPro = "http://"
        print("Target set: ", targetDomain)
        print("Path: ", targetPath)

    connection.request("HEAD", targetPath+"randomhy27dtwjwysg.txt")
    res = connection.getresponse()

    if res.status == 200:
        print("NOTE: Looks like the server is returning code 200 for all requests, there might be lots of false positive links.")

    if res.status >= 300 and res.status < 400 and options.showRedirect != None:
        print("NOTE: Looks like the server is returning code", res.status, "for all requests, there might be lots of false positive links. try to scan without the option -r")

    tpData = res.read()

    with open(options.listFile) as lFile:
        pathList = lFile.readlines()
    totalURLs = len(pathList)
    print ("Scanning (",totalURLs,") files...")
    countFound = 0
    countAll = 0
    strLine = ""
    for pathLine in pathList:
        countAll = countAll + 1
        pathLine = pathLine.strip("\n")
        pathLine = pathLine.strip("\r")

        if pathLine != "":
            if pathLine[:1] == "/":
                pathLine = pathLine[1:]
            print (' ' * len(strLine), "\r", end="")
            strLine = "Checking ["+str(countAll)+"/"+str(totalURLs)+"] "+targetPath+pathLine
            print (strLine,"\r", end="")
            connection.request("HEAD", targetPath+pathLine)
            res = connection.getresponse()

            if res.status == 200:
                print (' ' * len(strLine), "\r", end="")
                print("Code", res.status,":",targetPro+targetDomain+targetPath+pathLine)
                countFound += 1

            if options.showRedirect != None:
                if res.status >= 300 and res.status < 400:
                    print (' ' * len(strLine), "\r", end="")
                    print("Code", res.status,":",targetPro+targetDomain+targetPath+pathLine, "(",res.getheader("location"),")")
                    countFound += 1

            tpData = res.read()

    connection.close()
    print (' ' * len(strLine), "\r", end="")
    print ( "Total Pages found:",countFound )
{'fraction_non_alphanumeric': 0.16206150888995674, 'fraction_numerical': 0.013455069678039404, 'mean_word_length': 2.7065894924309886, 'pattern_counts': {'":': 10, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1000968', 'n_tokens_mistral': 2546, 'n_tokens_neox': 2428, 'n_words': 586}