<commit_before># -*- coding: utf-8 -*-
"""
This is part of WebScout software
Docs EN: http://hack4sec.pro/wiki/index.php/WebScout_en
Docs RU: http://hack4sec.pro/wiki/index.php/WebScout
License: MIT
Copyright (c) <NAME> <http://anton-kuzmin.ru> (ru) <http://anton-kuzmin.pro> (en)

Thread class for DnsBrute* modules
"""
import time
import Queue
import re
import threading
import requests

import dns.query
import dns.message

from classes.Registry import Registry

class DnsBruteThread(threading.Thread):
    """ Thread class for DnsBrute* modules """
    done = False

    def __init__(self, queue, domain, proto, msymbol, ignore_ip, dns_srv, delay, http_nf_re, result, counter):
        threading.Thread.__init__(self)
        self.queue = queue
        self.domain = domain
        self.proto = proto
        self.dns_srv = dns_srv
        self.counter = counter
        self.msymbol = msymbol
        self.result = result
        self.delay = int(delay)
        self.done = False
        self.logger = Registry().get('logger')
        self.ignore_ip = ignore_ip
        self.http_nf_re = re.compile(http_nf_re) if len(http_nf_re) else None

    def run(self):
        """ Run thread """
        ip_re = re.compile(r"(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})")
        ns_resp_re = re.compile(r";ANSWER\s(?P<data>(.|\s)*)\s;AUTHORITY", re.M)

        req_func = getattr(dns.query, self.proto.lower())

        while True:
            host = None
            if self.delay:
                time.sleep(self.delay)
            try:
                host = self.queue.get()
                self.counter.up()
                check_name = self.domain.replace(self.msymbol, host)
                query = dns.message.make_query(check_name, 'A')
                result = req_func(query, self.dns_srv, timeout=5)
                response = ns_resp_re.search(result.to_text())
                if response is not None:
                    for ip in ip_re.findall(response.group('data')):
                        if not len(self.ignore_ip) or ip != self.ignore_ip:
                            if self.http_nf_re is not None:
                                resp = requests.get(
                                    "http://{0}/".format(ip),
                                    headers={'Host': check_name},
                                    allow_redirects=False)
                                if not self.http_nf_re.findall(resp.text):
                                    self.result.append({'name': check_name, 'ip': ip, 'dns': self.dns_srv})
                                else:
                                    resp = requests.get(
                                        "https://{0}/".format(ip),
                                        headers={'Host': check_name},
                                        allow_redirects=False,
                                        verify=False)
                                    if not self.http_nf_re.findall(resp.text):
                                        self.result.append({'name': check_name, 'ip': ip, 'dns': self.dns_srv})
                            else:
                                self.result.append({'name': check_name, 'ip': ip, 'dns': self.dns_srv})
                        break

                if len(self.result) >= int(Registry().get('config')['main']['positive_limit_stop']):
                    Registry().set('positive_limit_stop', True)

            except Queue.Empty:
                self.done = True
                break<commit_msg>Work by Http class, not clean requests module<commit_after># -*- coding: utf-8 -*-
"""
This is part of WebScout software
Docs EN: http://hack4sec.pro/wiki/index.php/WebScout_en
Docs RU: http://hack4sec.pro/wiki/index.php/WebScout
License: MIT
Copyright (c) <NAME> <http://anton-kuzmin.ru> (ru) <http://anton-kuzmin.pro> (en)

Thread class for DnsBrute* modules
"""
import time
import Queue
import re
import threading
import requests

import dns.query
import dns.message

from classes.Registry import Registry

class DnsBruteThread(threading.Thread):
    """ Thread class for DnsBrute* modules """
    done = False

    def __init__(self, queue, domain, proto, msymbol, ignore_ip, dns_srv, delay, http_nf_re, result, counter):
        threading.Thread.__init__(self)
        self.queue = queue
        self.domain = domain
        self.proto = proto
        self.dns_srv = dns_srv
        self.counter = counter
        self.msymbol = msymbol
        self.result = result
        self.delay = int(delay)
        self.done = False
        self.logger = Registry().get('logger')
        self.ignore_ip = ignore_ip
        self.http_nf_re = re.compile(http_nf_re) if len(http_nf_re) else None

    def run(self):
        """ Run thread """
        ip_re = re.compile(r"(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})")
        ns_resp_re = re.compile(r";ANSWER\s(?P<data>(.|\s)*)\s;AUTHORITY", re.M)

        req_func = getattr(dns.query, self.proto.lower())

        while True:
            host = None
            if self.delay:
                time.sleep(self.delay)
            try:
                host = self.queue.get()
                self.counter.up()
                check_name = self.domain.replace(self.msymbol, host)
                query = dns.message.make_query(check_name, 'A')
                result = req_func(query, self.dns_srv, timeout=5)
                response = ns_resp_re.search(result.to_text())
                if response is not None:
                    for ip in ip_re.findall(response.group('data')):
                        if not len(self.ignore_ip) or ip != self.ignore_ip:
                            if self.http_nf_re is not None:
                                resp = Registry().get('http').get(
                                    "http://{0}/".format(ip),
                                    headers={'Host': check_name},
                                    allow_redirects=False)
                                if not self.http_nf_re.findall(resp.text):
                                    self.result.append({'name': check_name, 'ip': ip, 'dns': self.dns_srv})
                                else:
                                    resp = Registry().get('http').get(
                                        "https://{0}/".format(ip),
                                        headers={'Host': check_name},
                                        allow_redirects=False,
                                        verify=False)
                                    if not self.http_nf_re.findall(resp.text):
                                        self.result.append({'name': check_name, 'ip': ip, 'dns': self.dns_srv})
                            else:
                                self.result.append({'name': check_name, 'ip': ip, 'dns': self.dns_srv})
                        break

                if len(self.result) >= int(Registry().get('config')['main']['positive_limit_stop']):
                    Registry().set('positive_limit_stop', True)

            except Queue.Empty:
                self.done = True
                break
{'fraction_non_alphanumeric': 0.12089932126696833, 'fraction_numerical': 0.003959276018099547, 'mean_word_length': 1.4559027777777778, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '951548', 'n_tokens_mistral': 2055, 'n_tokens_neox': 1978, 'n_words': 410}