<commit_before>from poster.encode import multipart_encode
from poster.streaminghttp import register_openers
import urllib2

class do_request:
############################################################################################
    def __init__(self, server_base_url):
        "Configure the servers base URL"
        self.server_base_url = server_base_url

############################################################################################
    def request_raw(self, url, data):
        "Send a HTTP request to the configured server and return the result object"
        datagen, headers = multipart_encode(data)
        request = urllib2.Request(self.server_base_url + url, datagen, headers)
        result = urllib2.urlopen(request)
        return result

############################################################################################
    def request(self, url, data):
        "Send a HTTP request to the configured server and return the result body"
        result = self.request_raw(url, data)
        return result.read()


############################################################################################
    def request_full(self, url, data, gen = False):
        "Send a HTTP request to the configured server and return the result body and headers"
        result = self.request_raw(url, data)

        if gen == False:
            return (result.read(), result.info())

        def writer(path):
            with open(d[path], 'w') as f:
                while True:
                    Chunk = result.read(1000 * 1000)
                    if not chunk: break
                    f.write(Chunk)

        return (writer, result.info())

        
<commit_msg>fixed another bad variable name in refactored pull code<commit_after>from poster.encode import multipart_encode
from poster.streaminghttp import register_openers
import urllib2

class do_request:
############################################################################################
    def __init__(self, server_base_url):
        "Configure the servers base URL"
        self.server_base_url = server_base_url

############################################################################################
    def request_raw(self, url, data):
        "Send a HTTP request to the configured server and return the result object"
        datagen, headers = multipart_encode(data)
        request = urllib2.Request(self.server_base_url + url, datagen, headers)
        result = urllib2.urlopen(request)
        return result

############################################################################################
    def request(self, url, data):
        "Send a HTTP request to the configured server and return the result body"
        result = self.request_raw(url, data)
        return result.read()


############################################################################################
    def request_full(self, url, data, gen = False):
        "Send a HTTP request to the configured server and return the result body and headers"
        result = self.request_raw(url, data)

        if gen == False:
            return (result.read(), result.info())

        def writer(path):
            with open(path, 'w') as f:
                while True:
                    Chunk = result.read(1000 * 1000)
                    if not Chunk: break
                    f.write(Chunk)

        return (writer, result.info())

        

{'fraction_non_alphanumeric': 0.2733854619171735, 'fraction_numerical': 0.006371271358239212, 'mean_word_length': 3.207064555420219, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1739911', 'n_tokens_mistral': 822, 'n_tokens_neox': 771, 'n_words': 254}