<commit_before>import cStringIO, gzip, httplib, socket, sys, threading, time, urllib, urlparse
import Queue

CONNECTION_TIMEOUT = 30
EMPTY_QUEUE_WAIT = 5
STOP_CRAWLE = False

class Handler(object):
    """An _abstract_ class for handling what urls to retrieve and how to
    parse and save them. The functions of this class need to be designed in
    such a way so that they are threadsafe as multiple threads will have
    access to the same instance.
    """

    def preProcess(self, requestResponse):
        """PreProcess is called directly before making the reqeust. Any of the
        request parameters can be modified here.

        Setting the responseURL to None will cause the request to be dropped.
        This is useful for testing if a redirect link should be followed.
        """
        return
    
    def process(self, requestResponse, queue):
        """Process is called after the request has been made. It needs to be
        implemented by a subclass.

        Keyword Arguments:
        requestResponse -- the request response object
        queue -- the handler to the queue class
        """
        raise NotImplementedError(' '.join(('Handler.process must be defined',
                                            'in a subclass')))

class RequestResponse(object):
    """This class is a container for information pertaining to requests and
    responses.

    Attributes:
    	redirects	- None if request should not redirect, otherwise a
			  number > 0 to indicate how many redirects to support.
    """

    def __init__(self, url, headers=None, method='GET', params=None,
                 redirects=10):
        self.requestHeaders = headers
        self.requestURL = url
        self.requestMethod = method
        self.requestParams = params
        self.redirects = redirects

        self.responseStatus = None
        self.responseURL = url
        self.responseHeaders = None
        self.responseBody = None
        self.responseTime = None

        self.errorMsg = None
        self.errorObject = None


class HTTPConnectionQueue(object):
    """This class handles the queue of sockets for a particular address.

    This essentially is a queue of socket objects which also adds a transparent
    field to each connection object which is the requestCount. When the
    requestCount exceeds the REQUEST_LIMIT the connection is automatically
    reset.
    """
    REQUEST_LIMIT = None

    def __init__(self, address, encrypted=False):
        """Constructs a HTTPConnectionQueueobject.

        Keyword Arguments:
        address -- The address for which this object maps to.
        """
        self.address = address
        self.encrypted = encrypted
        self.queue = Queue.Queue(0)

    def getConnection(self):
        """Return a HTTP(S)Connection object for the appropriate address.
        
        First try to return the object from the queue, however if the queue
        is empty create a new socket object to return.

        Dynamically add new field to HTTPConnection called requestCount to
        keep track of the number of requests made with the specific connection.
        """
        try:
            connection = self.queue.get(block=False)
            """Reset the connection if exceeds request limit"""
            if self.REQUEST_LIMIT and \
                    connection.requestCount >= self.REQUEST_LIMIT:
                connection.close()
                if self.encrypted:
                    connection = httplib.HTTPSConnection(*self.address)
                else:
                    connection = httplib.HTTPConnection(*self.address)
                connection.requestCount = 0
        except Queue.Empty:
            if self.encrypted:
                connection = httplib.HTTPSConnection(*self.address)
            else:
                connection = httplib.HTTPConnection(*self.address)
            connection.requestCount = 0
        return connection

    def putConnection(self, connection):
        """Put the HTTPConnection object back on the queue."""
        connection.requestCount += 1
        self.queue.put(connection)


class HTTPConnectionControl(object):
    """This class handles HTTPConnectionQueues by storing a queue in a
    dictionary with the address as the index to the dictionary. Additionally
    this class handles resetting the connection when it reaches a specified
    request limit."""
    
    socket.setdefaulttimeout(CONNECTION_TIMEOUT)

    def __init__(self, handler):
        """Constructs the HTTPConnection Control object. These objects are to
        be shared between each thread.

        Keyword Arguments:
        handler -- The Handler class for checking if a url is valid.
        """
        
        self.connectionQueues = {}
        self.lock = threading.Lock()
        self.handler = handler

    def request(self, reqRes):
        """Handles the request to the server.
        """
        if STOP_CRAWLE:
            reqRes.errorMsg = 'CRAWL-E Stopped'
            return

        self.handler.preProcess(reqRes)
        if reqRes.responseURL == None:
            reqRes.errorMsg = 'Aborted in preProcess'
            return

        u = urlparse.urlparse(reqRes.responseURL)
        if u.scheme not in ['http', 'https'] or u.netloc == '':
            reqRes.errorMsg = 'Invalid URL'
            return

        try:
            address = socket.gethostbyname(u.hostname), u.port
            encrypted = u.scheme == 'https'<commit_msg>Updated crawle-e to simulate socket error when retreiving an opendns redirect address.<commit_after>import cStringIO, gzip, httplib, socket, sys, threading, time, urllib, urlparse
import Queue

CONNECTION_TIMEOUT = 30
EMPTY_QUEUE_WAIT = 5
STOP_CRAWLE = False

class Handler(object):
    """An _abstract_ class for handling what urls to retrieve and how to
    parse and save them. The functions of this class need to be designed in
    such a way so that they are threadsafe as multiple threads will have
    access to the same instance.
    """

    def preProcess(self, requestResponse):
        """PreProcess is called directly before making the reqeust. Any of the
        request parameters can be modified here.

        Setting the responseURL to None will cause the request to be dropped.
        This is useful for testing if a redirect link should be followed.
        """
        return
    
    def process(self, requestResponse, queue):
        """Process is called after the request has been made. It needs to be
        implemented by a subclass.

        Keyword Arguments:
        requestResponse -- the request response object
        queue -- the handler to the queue class
        """
        raise NotImplementedError(' '.join(('Handler.process must be defined',
                                            'in a subclass')))

class RequestResponse(object):
    """This class is a container for information pertaining to requests and
    responses.

    Attributes:
    	redirects	- None if request should not redirect, otherwise a
			  number > 0 to indicate how many redirects to support.
    """

    def __init__(self, url, headers=None, method='GET', params=None,
                 redirects=10):
        self.requestHeaders = headers
        self.requestURL = url
        self.requestMethod = method
        self.requestParams = params
        self.redirects = redirects

        self.responseStatus = None
        self.responseURL = url
        self.responseHeaders = None
        self.responseBody = None
        self.responseTime = None

        self.errorMsg = None
        self.errorObject = None


class HTTPConnectionQueue(object):
    """This class handles the queue of sockets for a particular address.

    This essentially is a queue of socket objects which also adds a transparent
    field to each connection object which is the requestCount. When the
    requestCount exceeds the REQUEST_LIMIT the connection is automatically
    reset.
    """
    REQUEST_LIMIT = None

    def __init__(self, address, encrypted=False):
        """Constructs a HTTPConnectionQueueobject.

        Keyword Arguments:
        address -- The address for which this object maps to.
        """
        self.address = address
        self.encrypted = encrypted
        self.queue = Queue.Queue(0)

    def getConnection(self):
        """Return a HTTP(S)Connection object for the appropriate address.
        
        First try to return the object from the queue, however if the queue
        is empty create a new socket object to return.

        Dynamically add new field to HTTPConnection called requestCount to
        keep track of the number of requests made with the specific connection.
        """
        try:
            connection = self.queue.get(block=False)
            """Reset the connection if exceeds request limit"""
            if self.REQUEST_LIMIT and \
                    connection.requestCount >= self.REQUEST_LIMIT:
                connection.close()
                if self.encrypted:
                    connection = httplib.HTTPSConnection(*self.address)
                else:
                    connection = httplib.HTTPConnection(*self.address)
                connection.requestCount = 0
        except Queue.Empty:
            if self.encrypted:
                connection = httplib.HTTPSConnection(*self.address)
            else:
                connection = httplib.HTTPConnection(*self.address)
            connection.requestCount = 0
        return connection

    def putConnection(self, connection):
        """Put the HTTPConnection object back on the queue."""
        connection.requestCount += 1
        self.queue.put(connection)


class HTTPConnectionControl(object):
    """This class handles HTTPConnectionQueues by storing a queue in a
    dictionary with the address as the index to the dictionary. Additionally
    this class handles resetting the connection when it reaches a specified
    request limit."""
    
    socket.setdefaulttimeout(CONNECTION_TIMEOUT)

    def __init__(self, handler):
        """Constructs the HTTPConnection Control object. These objects are to
        be shared between each thread.

        Keyword Arguments:
        handler -- The Handler class for checking if a url is valid.
        """
        
        self.connectionQueues = {}
        self.lock = threading.Lock()
        self.handler = handler

    def request(self, reqRes):
        """Handles the request to the server.
        """
        if STOP_CRAWLE:
            reqRes.errorMsg = 'CRAWL-E Stopped'
            return

        self.handler.preProcess(reqRes)
        if reqRes.responseURL == None:
            reqRes.errorMsg = 'Aborted in preProcess'
            return

        u = urlparse.urlparse(reqRes.responseURL)
        if u.scheme not in ['http', 'https'] or u.netloc == '':
            reqRes.errorMsg = 'Invalid URL'
            return

        try:
            address = socket.gethostbyname(u.hostname), u.port
            if address == ('192.168.3.11', None):
                # Simulate failure on OPEN DNS
                msg = 'No address associated with hostname'
                reqRes.errorMsg = 'Socket Error'
                reqRes.errorObject = Exception(None, msg)
                return
            encrypted = u.scheme == 'https'
{'fraction_non_alphanumeric': 0.06656045317755355, 'fraction_numerical': 0.002566825986900336, 'mean_word_length': 2.17923466516601, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4952392', 'n_tokens_mistral': 2794, 'n_tokens_neox': 2686, 'n_words': 1054}