<commit_before>import logging
from time import sleep
import datetime
from multiprocessing import Process, Event
from Queue import Empty

import requests

from inspector import Inspector
from block_cookie_policy import BlockCookiePolicy

class HTTPWorker(Process):
    def __init__(self, name, headers, request_queue, url_filter, url_builder, result_queue, pause_time=0,
                 allow_cookies=True):
        Process.__init__(self, name=name)
        self.__request_queue = request_queue
        self.__url_filter = url_filter
        self.__url_builder = url_builder
        self.__pause_time = pause_time
        self.__exit = Event()
        self.__killed = Event()
        self.__result_queue = result_queue
        self.inspector = Inspector()
        self.__session = requests.Session()
        self.__session.headers = headers
        if not allow_cookies:
            self.__session.cookies.set_policy(BlockCookiePolicy())

    def __request(self, data):
        url = self.__url_builder.build(data)
        try:
            start_request = datetime.datetime.now()
            response = self.__session.get(url)
            end_request = datetime.datetime.now()
            elapsed_time = end_request - start_request
            print response.cookies
        except requests.RequestException as e:<commit_msg>remove debug message<commit_after>import logging
from time import sleep
import datetime
from multiprocessing import Process, Event
from Queue import Empty

import requests

from inspector import Inspector
from block_cookie_policy import BlockCookiePolicy

class HTTPWorker(Process):
    def __init__(self, name, headers, request_queue, url_filter, url_builder, result_queue, pause_time=0,
                 allow_cookies=True):
        Process.__init__(self, name=name)
        self.__request_queue = request_queue
        self.__url_filter = url_filter
        self.__url_builder = url_builder
        self.__pause_time = pause_time
        self.__exit = Event()
        self.__killed = Event()
        self.__result_queue = result_queue
        self.inspector = Inspector()
        self.__session = requests.Session()
        self.__session.headers = headers
        if not allow_cookies:
            self.__session.cookies.set_policy(BlockCookiePolicy())

    def __request(self, data):
        url = self.__url_builder.build(data)
        try:
            start_request = datetime.datetime.now()
            response = self.__session.get(url)
            end_request = datetime.datetime.now()
            elapsed_time = end_request - start_request
        except requests.RequestException as e:
{'fraction_non_alphanumeric': 0.07055682684973302, 'fraction_numerical': 0.0007627765064836003, 'mean_word_length': 2.823615160349854, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7022929', 'n_tokens_mistral': 686, 'n_tokens_neox': 673, 'n_words': 171}