<commit_before>
from urlparse import urlparse
import socket
import ssl

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
import gzip


class HttpRequestor(object):
    
    def __init__(self, session, url, logger):
    
        self.session = session
        self.url = url
        self.logger = logger
        
        self.socket = None
        self.socket_file = None
        
        self.status_line = ''
        self.response_code = 0
        self.raw_headers = ''
        self.headers = {}
        
        self.raw_response = ''
    
    def start_request(self, sock):
        self.socket = sock
        
        hostname, port, raw_request = self.request_info()
        self.logger.info("Opening socket to %s:%s..." % (hostname, port))

        try:
            self.socket.connect((hostname, port))
        except socket.gaierror:
            self.logger.error("unknown host: %s" % hostname)
            self.session.http_error = 1 # TODO: make this some kind of enum
            self.session.save()
            return False
        except socket.error, msg:
            self.logger.error("socket error")
            self.logger.error(msg)
            self.session.http_error = 2 # TODO: make this some kind of enum
            self.session.save()
            return False

        self.socket.sendall(raw_request)
        return True
    
    def request_info(self):
                
        if not self.url:
            self.url = self.session.http_url
        
        parsed_url = urlparse(self.url)
        hostname = parsed_url.hostname
        path = parsed_url.path or '/'
        
        if parsed_url.scheme == 'http':
            default_port = 80
        elif parsed_url.scheme == 'https':
            self.logger.warning("Request is over HTTPS! (maybe not yet implemented)")
            if self.socket:
                self.socket = ssl.wrap_socket(self.socket)
            default_port = 443
            
        port = parsed_url.port or default_port
                
        raw_request = self.session.get_raw_request(self.url)
        <commit_msg>Removed a log message that doesn't apply anymore.<commit_after>
from urlparse import urlparse
import socket
import ssl

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
import gzip


class HttpRequestor(object):
    
    def __init__(self, session, url, logger):
    
        self.session = session
        self.url = url
        self.logger = logger
        
        self.socket = None
        self.socket_file = None
        
        self.status_line = ''
        self.response_code = 0
        self.raw_headers = ''
        self.headers = {}
        
        self.raw_response = ''
    
    def start_request(self, sock):
        self.socket = sock
        
        hostname, port, raw_request = self.request_info()
        self.logger.info("Opening socket to %s:%s..." % (hostname, port))

        try:
            self.socket.connect((hostname, port))
        except socket.gaierror:
            self.logger.error("unknown host: %s" % hostname)
            self.session.http_error = 1 # TODO: make this some kind of enum
            self.session.save()
            return False
        except socket.error, msg:
            self.logger.error("socket error")
            self.logger.error(msg)
            self.session.http_error = 2 # TODO: make this some kind of enum
            self.session.save()
            return False

        self.socket.sendall(raw_request)
        return True
    
    def request_info(self):
                
        if not self.url:
            self.url = self.session.http_url
        
        parsed_url = urlparse(self.url)
        hostname = parsed_url.hostname
        path = parsed_url.path or '/'
        
        if parsed_url.scheme == 'http':
            default_port = 80
        elif parsed_url.scheme == 'https':
            if self.socket:
                self.socket = ssl.wrap_socket(self.socket)
            default_port = 443
            
        port = parsed_url.port or default_port
                
        raw_request = self.session.get_raw_request(self.url)
        
{'fraction_non_alphanumeric': 0.08568019093078759, 'fraction_numerical': 0.0038186157517899762, 'mean_word_length': 1.6643356643356644, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4714802', 'n_tokens_mistral': 1166, 'n_tokens_neox': 1148, 'n_words': 287}