<commit_before>from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
import threading
import time
import os
import shutil
import tempfile
from copy import deepcopy

import grab

# The port on which the fake http server listens requests
FAKE_SERVER_PORT = 9876

# Simple URL which could be used in tests
BASE_URL = 'http://localhost:%d' % FAKE_SERVER_PORT

# This global objects is used by Fake HTTP Server
# It return content of HTML variable for any GET request
RESPONSE = {}
RESPONSE_ONCE = {}
RESPONSE_ONCE_HEADERS = []

# Fake HTTP Server saves request details
# into global REQUEST variable
REQUEST = {'get': None, 'post': None, 'headers': None,
           'path': None}

SLEEP = {'get': 0, 'post': 0}

TEST_DIR = os.path.dirname(os.path.realpath(__file__))

# Global variable which is used in all tests to build
# Grab instance with specific transport layer
GRAB_TRANSPORT = None

# TODO: wrap all RESPONSE* things into class

def reset_env():
    RESPONSE.update({'get': '', 'post': '', 'cookies': None,
                     'once_code': None, 'get_callback': None})
    RESPONSE_ONCE.update({'get': None, 'post': None})
    for x in xrange(len(RESPONSE_ONCE_HEADERS)):
        RESPONSE_ONCE_HEADERS.pop()

# Initial env configuration
reset_env()

def prepare_test_environment():
    global TMP_DIR, TMP_FILE

    TMP_DIR = tempfile.mkdtemp()
    TMP_FILE = os.path.join(TMP_DIR, '__temp')


def clear_test_environment():
    remove_directory(TMP_DIR)


def remove_directory(path):
    for root, dirs, files in os.walk(path):
        for fname in files:
            os.unlink(os.path.join(root, fname))
        for _dir in dirs:
            shutil.rmtree(os.path.join(root, _dir))


class FakeServerThread(threading.Thread):
    def __init__(self, port=FAKE_SERVER_PORT, *args, **kwargs):
        super(FakeServerThread, self).__init__(*args, **kwargs)
        self.daemon = True
        self.listen_port = port

    def start(self):
        super(FakeServerThread, self).start()
        time.sleep(0.1)

    def run(self):
        # TODO: reset REQUEST before each get/post request
        class RequestHandlerClass(BaseHTTPRequestHandler):
            def do_GET(self):
                """
                Process GET request.

                Reponse body contains content from ``RESPONSE['get']``
                """

                print '<-- (GET)'
                time.sleep(SLEEP['get'])

                REQUEST['headers'] = self.headers
                print "RH:", REQUEST['headers']
                REQUEST['path'] = self.path<commit_msg>Fix sleep issue in debug web server used in tests<commit_after>from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
import threading
import time
import os
import shutil
import tempfile
from copy import deepcopy

import grab

# The port on which the fake http server listens requests
FAKE_SERVER_PORT = 9876

# Simple URL which could be used in tests
BASE_URL = 'http://localhost:%d' % FAKE_SERVER_PORT

# This global objects is used by Fake HTTP Server
# It return content of HTML variable for any GET request
RESPONSE = {}
RESPONSE_ONCE = {}
RESPONSE_ONCE_HEADERS = []

# Fake HTTP Server saves request details
# into global REQUEST variable
REQUEST = {'get': None, 'post': None, 'headers': None,
           'path': None}

SLEEP = {'get': 0, 'post': 0}

TEST_DIR = os.path.dirname(os.path.realpath(__file__))

# Global variable which is used in all tests to build
# Grab instance with specific transport layer
GRAB_TRANSPORT = None

# TODO: wrap all RESPONSE* things into class

def reset_env():
    RESPONSE.update({'get': '', 'post': '', 'cookies': None,
                     'once_code': None, 'get_callback': None})
    RESPONSE_ONCE.update({'get': None, 'post': None})
    for x in xrange(len(RESPONSE_ONCE_HEADERS)):
        RESPONSE_ONCE_HEADERS.pop()

# Initial env configuration
reset_env()

def prepare_test_environment():
    global TMP_DIR, TMP_FILE

    TMP_DIR = tempfile.mkdtemp()
    TMP_FILE = os.path.join(TMP_DIR, '__temp')


def clear_test_environment():
    remove_directory(TMP_DIR)


def remove_directory(path):
    for root, dirs, files in os.walk(path):
        for fname in files:
            os.unlink(os.path.join(root, fname))
        for _dir in dirs:
            shutil.rmtree(os.path.join(root, _dir))


class FakeServerThread(threading.Thread):
    def __init__(self, port=FAKE_SERVER_PORT, *args, **kwargs):
        super(FakeServerThread, self).__init__(*args, **kwargs)
        self.daemon = True
        self.listen_port = port
        SLEEP['get'] = 0
        SLEEP['post'] = 0

    def start(self):
        super(FakeServerThread, self).start()
        time.sleep(0.1)

    def run(self):
        # TODO: reset REQUEST before each get/post request
        class RequestHandlerClass(BaseHTTPRequestHandler):
            def do_GET(self):
                """
                Process GET request.

                Reponse body contains content from ``RESPONSE['get']``
                """

                print '<-- (GET)'
                time.sleep(SLEEP['get'])

                REQUEST['headers'] = self.headers
                print "RH:", REQUEST['headers']
                REQUEST['path'] = self.path
{'fraction_non_alphanumeric': 0.1076834642651849, 'fraction_numerical': 0.0034489365778884846, 'mean_word_length': 2.9130434782608696, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3893540', 'n_tokens_mistral': 1646, 'n_tokens_neox': 1585, 'n_words': 460}