<commit_before>import urllib.request

from collection_pipelines.core import CollectionPipelineProcessor


def http_get(url):
    return urllib.request.urlopen(url).read()


class http(CollectionPipelineProcessor):
    def __init__(self, url):
        self._set_url(url)
        self.source(self.make_generator)

    def make_generator(self):
        self.receiver.send(http_get(self.url))

    def _set_url(self, url):
        if not url.startswith('http://'):
            url = 'http://' + url

        self.url = url
<commit_msg>Update http GET function to return string rather than bytes<commit_after>import urllib.request

from collection_pipelines.core import CollectionPipelineProcessor


def http_get(url):
    """
    Returns:
        str: response body.
    """
    return urllib.request.urlopen(url).read().decode('utf-8')


class http(CollectionPipelineProcessor):
    def __init__(self, url):
        self._set_url(url)
        self.source(self.make_generator)

    def make_generator(self):
        self.receiver.send(http_get(self.url))

    def _set_url(self, url):
        if not url.startswith('http://'):
            url = 'http://' + url

        self.url = url

{'fraction_non_alphanumeric': 0.11600338696020322, 'fraction_numerical': 0.000846740050804403, 'mean_word_length': 3.0341296928327646, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3599296', 'n_tokens_mistral': 369, 'n_tokens_neox': 356, 'n_words': 75}