<commit_before>import re
import urlparse
import logging
import datetime
import time

from twisted.web import client
import twisted.web.error
from twisted.internet import defer, task, reactor
import twisted.internet.error

from parse import HtmlParser, ParseError

class HTTPFetcher(client.HTTPPageGetter):
    elapsed_time = datetime.timedelta()
    
    def connectionMade(self):
        r = client.HTTPPageGetter.connectionMade(self)
        self._start_time = time.clock()
        return r
    
    def handleResponse(self, result):
        #FIXME: why does this get called before connectionLost?
        end_time = time.clock()
        self.elapsed_time = datetime.timedelta(seconds=end_time-self._start_time)
        self.factory.gotElapsedTime(self.elapsed_time)
        return client.HTTPPageGetter.handleResponse(self, result)
    
    def connectionLost(self, reason):
        if not self.elapsed_time:
            end_time = time.clock()
            self.elapsed_time = datetime.timedelta(seconds=end_time-self._start_time)
            self.factory.gotElapsedTime(self.elapsed_time)
        return client.HTTPPageGetter.connectionLost(self, reason)

class HTTPFetcherFactory(client.HTTPClientFactory):
    protocol = HTTPFetcher
    
    def __init__(self, url, headers=None):
        client.HTTPClientFactory.__init__(self, url, headers=headers, agent="Katipo/0.1~a1 (+http://rob.coup.net.nz/)", timeout=30)
        self.status = None
        self.elapsed_time = datetime.timedelta()
        self.deferred.addErrback(self._check_http_error)
        self.deferred.addCallback(
            lambda data: (data, int(self.status), self.response_headers, self.elapsed_time))
    
    def _check_http_error(self, failure):
        # attempt to turn non-200 errors into valid pages
        failure.trap(twisted.web.error.Error)
        
        if failure.value.status is not None:
            return failure.value.response
        else:
            return failure
    
    def gotElapsedTime(self, elapsed_time):
        self.elapsed_time = elapsed_time
    
    @staticmethod
    def getPage(url, contextFactory=None, *args, **kwargs):
        scheme, host, port, path = client._parse(url)
        factory = HTTPFetcherFactory(url, *args, **kwargs)
        if scheme == 'https':
            from twisted.internet import ssl
            if contextFactory is None:<commit_msg>Update the User-Agent string to point to the Bot wiki page<commit_after>import re
import urlparse
import logging
import datetime
import time

from twisted.web import client
import twisted.web.error
from twisted.internet import defer, task, reactor
import twisted.internet.error

from parse import HtmlParser, ParseError

class HTTPFetcher(client.HTTPPageGetter):
    elapsed_time = datetime.timedelta()
    
    def connectionMade(self):
        r = client.HTTPPageGetter.connectionMade(self)
        self._start_time = time.clock()
        return r
    
    def handleResponse(self, result):
        #FIXME: why does this get called before connectionLost?
        end_time = time.clock()
        self.elapsed_time = datetime.timedelta(seconds=end_time-self._start_time)
        self.factory.gotElapsedTime(self.elapsed_time)
        return client.HTTPPageGetter.handleResponse(self, result)
    
    def connectionLost(self, reason):
        if not self.elapsed_time:
            end_time = time.clock()
            self.elapsed_time = datetime.timedelta(seconds=end_time-self._start_time)
            self.factory.gotElapsedTime(self.elapsed_time)
        return client.HTTPPageGetter.connectionLost(self, reason)

class HTTPFetcherFactory(client.HTTPClientFactory):
    protocol = HTTPFetcher
    
    def __init__(self, url, headers=None):
        client.HTTPClientFactory.__init__(self, url, headers=headers, agent="Katipo/0.1~a1 (+http://code.google.com/p/katipo/wiki/Bot)", timeout=30)
        self.status = None
        self.elapsed_time = datetime.timedelta()
        self.deferred.addErrback(self._check_http_error)
        self.deferred.addCallback(
            lambda data: (data, int(self.status), self.response_headers, self.elapsed_time))
    
    def _check_http_error(self, failure):
        # attempt to turn non-200 errors into valid pages
        failure.trap(twisted.web.error.Error)
        
        if failure.value.status is not None:
            return failure.value.response
        else:
            return failure
    
    def gotElapsedTime(self, elapsed_time):
        self.elapsed_time = elapsed_time
    
    @staticmethod
    def getPage(url, contextFactory=None, *args, **kwargs):
        scheme, host, port, path = client._parse(url)
        factory = HTTPFetcherFactory(url, *args, **kwargs)
        if scheme == 'https':
            from twisted.internet import ssl
            if contextFactory is None:
{'fraction_non_alphanumeric': 0.08817427385892117, 'fraction_numerical': 0.0033195020746887966, 'mean_word_length': 2.958128078817734, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4177085', 'n_tokens_mistral': 1403, 'n_tokens_neox': 1406, 'n_words': 315}