<commit_before>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2011 sqlmap developers (http://www.sqlmap.org/)
See the file 'doc/COPYING' for copying permission
"""

import random
import re
import time

from lib.core.agent import agent
from lib.core.common import average
from lib.core.common import Backend
from lib.core.common import clearConsoleLine
from lib.core.common import dataToStdout
from lib.core.common import extractRegexResult
from lib.core.common import getUnicode
from lib.core.common import listToStrValue
from lib.core.common import popValue
from lib.core.common import pushValue
from lib.core.common import randomInt
from lib.core.common import randomStr
from lib.core.common import removeReflectiveValues
from lib.core.common import singleTimeLogMessage
from lib.core.common import singleTimeWarnMessage
from lib.core.common import stdev
from lib.core.common import wasLastRequestDBMSError
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.data import queries
from lib.core.enums import DBMS
from lib.core.enums import PAYLOAD
from lib.core.settings import FROM_TABLE
from lib.core.settings import UNION_MIN_RESPONSE_CHARS
from lib.core.settings import UNION_STDEV_COEFF
from lib.core.settings import MIN_RATIO
from lib.core.settings import MAX_RATIO
from lib.core.settings import MIN_STATISTICAL_RANGE
from lib.core.settings import MIN_UNION_RESPONSES
from lib.core.settings import ORDER_BY_STEP
from lib.core.unescaper import unescaper
from lib.parse.html import htmlParser
from lib.request.comparison import comparison
from lib.request.connect import Connect as Request

def __findUnionCharCount(comment, place, parameter, value, prefix, suffix, where=PAYLOAD.WHERE.ORIGINAL):
    """
    Finds number of columns affected by UNION based injection
    """
    retVal = None

    def __orderByTechnique():
        def __orderByTest(cols):
            query = agent.prefixQuery("ORDER BY %d" % cols, prefix=prefix)
            query = agent.suffixQuery(query, suffix=suffix, comment=comment)
            payload = agent.payload(newValue=query, place=place, parameter=parameter, where=where)
            page, _ = Request.queryPage(payload, place=place, content=True, raise404=False)
            return not re.search(r"((warning|error)[^\n]*order)|(order by)", page or "", re.I)

        if __orderByTest(1) and not __orderByTest(randomInt()):
            infoMsg = "ORDER BY technique seems to be usable. "
            infoMsg += "This should reduce the time needed "
            infoMsg += "to find the right number "
            infoMsg += "of query columns. Automatically extending the "
            infoMsg += "range for UNION query injection technique"
            singleTimeLogMessage(infoMsg)

            lowCols, highCols = 1, ORDER_BY_STEP
            found = None
            while not found:
                if __orderByTest(highCols):
                    lowCols = highCols
                    highCols += ORDER_BY_STEP
                else:
                    while not found:
                        mid = highCols - (highCols - lowCols) / 2
                        if __orderByTest(mid):
                            lowCols = mid
                        else:
                            highCols = mid
                        if (highCols - lowCols) < 2:
                            found = lowCols

            return found

    pushValue(kb.errorIsNone)
    items, ratios = [], []
    kb.errorIsNone = False
    lowerCount, upperCount = conf.uColsStart, conf.uColsStop

    if lowerCount == 1:
        found = kb.orderByColumns or __orderByTechnique()
        if found:
            kb.orderByColumns = found
            infoMsg = "target url appears to have %d columns in query" % found
            singleTimeLogMessage(infoMsg)
            return found

    if abs(upperCount - lowerCount) < MIN_UNION_RESPONSES:
        upperCount = lowerCount + MIN_UNION_RESPONSES

    min_, max_ = MAX_RATIO, MIN_RATIO
    pages = {}

    for count in range(lowerCount, upperCount+1):
        query = agent.forgeInbandQuery('', -1, count, comment, prefix, suffix, kb.uChar)
        payload = agent.payload(place=place, parameter=parameter, newValue=query, where=where)
        page, headers = Request.queryPage(payload, place=place, content=True, raise404=False)
        if kb.uChar:
            pages[count] = page
        else:
            ratio = comparison(page, headers, getRatioValue=True) or MIN_RATIO
            ratios.append(ratio)
            min_, max_ = min(min_, ratio), max(max_, ratio)
            items.append((count, ratio))

    if kb.uChar:
        for regex in (kb.uChar, r'>\s*%s\s*<' % kb.uChar):
            contains = [(count, re.search(regex, page, re.IGNORECASE) is not None) for count, page in pages.items()]
            if len(filter(lambda x: x[1], contains)) == 1:
                retVal = filter(lambda x: x[1], contains)[0][0]
                break

    else:
        ratios.pop(ratios.index(min_))
        ratios.pop(ratios.index(max_))

        minItem, maxItem = None, None

        for item in items:
            if item[1] == min_:
                minItem = item<commit_msg>minor update<commit_after>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2011 sqlmap developers (http://www.sqlmap.org/)
See the file 'doc/COPYING' for copying permission
"""

import random
import re
import time

from lib.core.agent import agent
from lib.core.common import average
from lib.core.common import Backend
from lib.core.common import clearConsoleLine
from lib.core.common import dataToStdout
from lib.core.common import extractRegexResult
from lib.core.common import getUnicode
from lib.core.common import listToStrValue
from lib.core.common import popValue
from lib.core.common import pushValue
from lib.core.common import randomInt
from lib.core.common import randomStr
from lib.core.common import removeReflectiveValues
from lib.core.common import singleTimeLogMessage
from lib.core.common import singleTimeWarnMessage
from lib.core.common import stdev
from lib.core.common import wasLastRequestDBMSError
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.data import queries
from lib.core.enums import DBMS
from lib.core.enums import PAYLOAD
from lib.core.settings import FROM_TABLE
from lib.core.settings import UNION_MIN_RESPONSE_CHARS
from lib.core.settings import UNION_STDEV_COEFF
from lib.core.settings import MIN_RATIO
from lib.core.settings import MAX_RATIO
from lib.core.settings import MIN_STATISTICAL_RANGE
from lib.core.settings import MIN_UNION_RESPONSES
from lib.core.settings import ORDER_BY_STEP
from lib.core.unescaper import unescaper
from lib.parse.html import htmlParser
from lib.request.comparison import comparison
from lib.request.connect import Connect as Request

def __findUnionCharCount(comment, place, parameter, value, prefix, suffix, where=PAYLOAD.WHERE.ORIGINAL):
    """
    Finds number of columns affected by UNION based injection
    """
    retVal = None

    def __orderByTechnique():
        def __orderByTest(cols):
            query = agent.prefixQuery("ORDER BY %d" % cols, prefix=prefix)
            query = agent.suffixQuery(query, suffix=suffix, comment=comment)
            payload = agent.payload(newValue=query, place=place, parameter=parameter, where=where)
            page, _ = Request.queryPage(payload, place=place, content=True, raise404=False)
            return not re.search(r"((warning|error)[^\n]*order)|(order by)", page or "", re.I)

        if __orderByTest(1) and not __orderByTest(randomInt()):
            infoMsg = "ORDER BY technique seems to be usable. "
            infoMsg += "This should reduce the time needed "
            infoMsg += "to find the right number "
            infoMsg += "of query columns. Automatically extending the "
            infoMsg += "range for UNION query injection technique"
            singleTimeLogMessage(infoMsg)

            lowCols, highCols = 1, ORDER_BY_STEP
            found = None
            while not found:
                if __orderByTest(highCols):
                    lowCols = highCols
                    highCols += ORDER_BY_STEP
                else:
                    while not found:
                        mid = highCols - (highCols - lowCols) / 2
                        if __orderByTest(mid):
                            lowCols = mid
                        else:
                            highCols = mid
                        if (highCols - lowCols) < 2:
                            found = lowCols

            return found

    pushValue(kb.errorIsNone)
    items, ratios = [], []
    kb.errorIsNone = False
    lowerCount, upperCount = conf.uColsStart, conf.uColsStop

    if lowerCount == 1:
        found = kb.orderByColumns or __orderByTechnique()
        if found:
            kb.orderByColumns = found
            infoMsg = "target url appears to have %d columns in query" % found
            singleTimeLogMessage(infoMsg)
            return found

    if abs(upperCount - lowerCount) < MIN_UNION_RESPONSES:
        upperCount = lowerCount + MIN_UNION_RESPONSES

    min_, max_ = MAX_RATIO, MIN_RATIO
    pages = {}

    for count in range(lowerCount, upperCount+1):
        query = agent.forgeInbandQuery('', -1, count, comment, prefix, suffix, kb.uChar)
        payload = agent.payload(place=place, parameter=parameter, newValue=query, where=where)
        page, headers = Request.queryPage(payload, place=place, content=True, raise404=False)
        if kb.uChar:
            pages[count] = page
        ratio = comparison(page, headers, getRatioValue=True) or MIN_RATIO
        ratios.append(ratio)
        min_, max_ = min(min_, ratio), max(max_, ratio)
        items.append((count, ratio))

    if kb.uChar:
        for regex in (kb.uChar, r'>\s*%s\s*<' % kb.uChar):
            contains = [(count, re.search(regex, page, re.IGNORECASE) is not None) for count, page in pages.items()]
            if len(filter(lambda x: x[1], contains)) == 1:
                retVal = filter(lambda x: x[1], contains)[0][0]
                break

    if not retVal:
        ratios.pop(ratios.index(min_))
        ratios.pop(ratios.index(max_))

        minItem, maxItem = None, None

        for item in items:
            if item[1] == min_:
                minItem = item
{'fraction_non_alphanumeric': 0.08583815028901734, 'fraction_numerical': 0.005202312138728324, 'mean_word_length': 2.5710354317165462, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1978334', 'n_tokens_mistral': 3070, 'n_tokens_neox': 2940, 'n_words': 867}