<commit_before>import os
from io import BytesIO
import zipfile
import requests


class InMemoryZip(object):
    """
    Adopted from -
    http://stackoverflow.com/questions/2463770/python-in-memory-zip-library
    """
    def __init__(self):
        # Create the in-memory file-like object
        self.in_memory_zip = BytesIO()

    def append(self, filename_in_zip, file_contents):
        '''Appends a file with name filename_in_zip and contents of
        file_contents to the in-memory zip.'''
        # Get a handle to the in-memory zip in append mode
        zf = zipfile.ZipFile(self.in_memory_zip, "a", zipfile.ZIP_DEFLATED, False)

        # Write the file to the in-memory zip
        zf.writestr(filename_in_zip, file_contents)

        # Mark the files as having been created on Windows so that
        # Unix permissions are not inferred as 0000
        for zfile in zf.filelist:
            zfile.create_system = 0

        return self

    def read(self):
        '''Returns a byte array with the contents of the in-memory zip.'''
        self.in_memory_zip.seek(0)
        return self.in_memory_zip.read()

    def writetofile(self, filename):
        '''Writes the in-memory zip to a file.'''
        f = file(filename, "w")
        f.write(self.read())
        f.close()



def zip_dir(dir_path):
    """
    Gets a dir path and returns a bytearray with the directory's zipped content
    :param dir_path:
    :return:
    """
    imz = InMemoryZip()

    for root, dirs, files in os.walk(dir_path):
        for file in files:
            with open(os.path.join(root, file), 'rb') as f:
                imz.append(file, f.read())

    return imz.read()<commit_msg>-- Added InMemory zip support -- added post requests support<commit_after>import os
from io import BytesIO
import zipfile
import requests


class InMemoryZip(object):
    """
    Adopted from -
    http://stackoverflow.com/questions/2463770/python-in-memory-zip-library
    """
    def __init__(self):
        # Create the in-memory file-like object
        self.in_memory_zip = BytesIO()

    def append(self, filename_in_zip, file_contents):
        '''Appends a file with name filename_in_zip and contents of
        file_contents to the in-memory zip.'''
        # Get a handle to the in-memory zip in append mode
        zf = zipfile.ZipFile(self.in_memory_zip, "a", zipfile.ZIP_DEFLATED, False)

        # Write the file to the in-memory zip
        zf.writestr(filename_in_zip, file_contents)

        # Mark the files as having been created on Windows so that
        # Unix permissions are not inferred as 0000
        for zfile in zf.filelist:
            zfile.create_system = 0

        return self

    def read(self):
        '''Returns a byte array with the contents of the in-memory zip.'''
        self.in_memory_zip.seek(0)
        return self.in_memory_zip.read()

    def writetofile(self, filename):
        '''Writes the in-memory zip to a file.'''
        f = file(filename, "w")
        f.write(self.read())
        f.close()



def zip_dir(dir_path):
    """
    Gets a dir path and returns a bytearray with the directory's zipped content
    :param dir_path:
    :return:
    """
    imz = InMemoryZip()

    for root, dirs, files in os.walk(dir_path):
        for file in files:
            with open(os.path.join(root, file), 'rb') as f:
                imz.append(os.path.join(root, file), f.read())

    return imz.read()
{'fraction_non_alphanumeric': 0.0975254730713246, 'fraction_numerical': 0.007569141193595342, 'mean_word_length': 2.4121151936444885, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '270918', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1042, 'n_words': 334}