<commit_before>from __future__ import print_function
from cement.core import handler, controller
from copy import deepcopy
from common import ScanningMethod, StandardOutput, JsonOutput, \
        VersionsFile, RequestsLogger
from common import template, enum_list, dict_combine, base_url, file_len
from common.exceptions import FileEmptyException
from common.output import ProgressBar
from common.http import BlockAll
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from os.path import dirname
from requests import Session
from functools import partial
import common
import common.functions as f
import hashlib
import os
import re
import requests
import signal
import sys
import traceback

try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

global shutdown
shutdown = False
def handle_interrupt(signal, stack):
    print("\nShutting down...")
    global shutdown
    shutdown = True

signal.signal(signal.SIGINT, handle_interrupt)

try:
    requests.packages.urllib3.disable_warnings()
except:
    pass

class BasePluginInternal(controller.CementBaseController):
    DEFAULT_UA = 'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)'
    not_found_url = "misc/test/error/404/ispresent.html"
    NUMBER_DEFAULT = 'number_default'
    NUMBER_THEMES_DEFAULT = 350
    NUMBER_PLUGINS_DEFAULT = 1000
    SPLIT_PATTERN = re.compile('[ \t]+')

    out = None
    session = None
    vf = None

    class Meta:
        label = 'baseplugin'
        stacked_on = 'scan'

        argument_formatter = common.SmartFormatter

        epilog = template('help_epilog.mustache')

    def _path(self, path, pwd):
        if path.startswith('/'):
            return path
        else:
            return pwd + "/" + path

    def _threads(self, pargs):
        threads = pargs.threads
        if pargs.threads_identify:
            threads_identify = pargs.threads_identify
        else:
            threads_identify = threads

        if pargs.threads_scan:
            threads_scan = pargs.threads_scan
        else:
            threads_scan = threads

        if pargs.threads_enumerate:
            threads_enumerate = pargs.threads_enumerate
        else:
            threads_enumerate = threads

        return threads, threads_identify, threads_scan, threads_enumerate

    def _options(self, pargs):
        pwd = self.app.config.get('general', 'pwd')
        if pargs.url_file != None:
            url_file = self._path(pargs.url_file, pwd)
        else:
            url = pargs.url

        enumerate = pargs.enumerate
        verb = pargs.verb
        method = pargs.method
        output = pargs.output
        timeout = pargs.timeout
        timeout_host = pargs.timeout_host
        debug_requests = pargs.debug_requests
        follow_redirects = pargs.follow_redirects
        plugins_base_url = pargs.plugins_base_url
        themes_base_url = pargs.themes_base_url
        debug = pargs.debug
        number = pargs.number if not pargs.number == 'all' else 100000
        if pargs.error_log:
            error_log = self._path(pargs.error_log, pwd)
        else:
            error_log = '-'

        headers = {}
        if pargs.host:
            headers = {'Host': pargs.host}

        threads, threads_identify, threads_scan, threads_enumerate = self._threads(pargs)

        if pargs.massscan_override:
            threads = 10
            threads_identify = 100
            threads_scan = 100<commit_msg>Update base_plugin_internal.py<commit_after>from __future__ import print_function
from cement.core import handler, controller
from copy import deepcopy
from common import ScanningMethod, StandardOutput, JsonOutput, \
        VersionsFile, RequestsLogger
from common import template, enum_list, dict_combine, base_url, file_len
from common.exceptions import FileEmptyException
from common.output import ProgressBar
from common.http import BlockAll
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from os.path import dirname
from requests import Session
from functools import partial
import common
import common.functions as f
import hashlib
import os
import re
import requests
import signal
import sys
import traceback

try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

global shutdown
shutdown = False
def handle_interrupt(signal, stack):
    print("\nShutting down...")
    global shutdown
    shutdown = True

signal.signal(signal.SIGINT, handle_interrupt)

try:
    requests.packages.urllib3.disable_warnings()
except:
    pass

class BasePluginInternal(controller.CementBaseController):
    DEFAULT_UA = 'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)'
    not_found_url = "misc/test/error/404/ispresent.html"
    NUMBER_DEFAULT = 'number_default'
    NUMBER_THEMES_DEFAULT = 350
    NUMBER_PLUGINS_DEFAULT = 1000
    SPLIT_PATTERN = re.compile('[ \t]+')

    out = None
    session = None
    vf = None

    class Meta:
        label = 'baseplugin'
        stacked_on = 'scan'

        argument_formatter = common.SmartFormatter

        epilog = template('help_epilog.mustache')

    def _path(self, path, pwd):
        if path.startswith('/'):
            return path
        else:
            return pwd + "/" + path

    def _threads(self, pargs):
        threads = pargs.threads
        if pargs.threads_identify:
            threads_identify = pargs.threads_identify
        else:
            threads_identify = threads

        if pargs.threads_scan:
            threads_scan = pargs.threads_scan
        else:
            threads_scan = threads

        if pargs.threads_enumerate:
            threads_enumerate = pargs.threads_enumerate
        else:
            threads_enumerate = threads

        return threads, threads_identify, threads_scan, threads_enumerate

    def _options(self, pargs):
        pwd = self.app.config.get('general', 'pwd')
        if pargs.url_file != None:
            url_file = self._path(pargs.url_file, pwd)
        else:
            url = pargs.url

        enumerate = pargs.enumerate
        verb = pargs.verb
        method = pargs.method
        output = pargs.output
        timeout = pargs.timeout
        timeout_host = pargs.timeout_host
        debug_requests = pargs.debug_requests
        follow_redirects = pargs.follow_redirects
        plugins_base_url = pargs.plugins_base_url
        themes_base_url = pargs.themes_base_url
        debug = pargs.debug
        number = pargs.number if not pargs.number == 'all' else 100000
        if pargs.error_log:
            error_log = self._path(pargs.error_log, pwd)
        else:
            error_log = '-'

        headers = {}
        if pargs.host:
            headers = {'Host': pargs.host}

        threads, threads_identify, threads_scan, threads_enumerate = self._threads(pargs)

        if pargs.massscan_override:
            threads = 10
            threads_identify = 500
            threads_scan = 500
{'fraction_non_alphanumeric': 0.07139803334758443, 'fraction_numerical': 0.008265640587145504, 'mean_word_length': 2.6343863283272917, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7580928', 'n_tokens_mistral': 2084, 'n_tokens_neox': 1991, 'n_words': 518}