<commit_before>"""
Streaming Multipart Encoded Files
=================================

**NOTE: This is temporary until we can fork python-requests and/or urllib3 to
        provide the same functionality.
"""
import os
import fnmatch
from urllib import quote
from uuid import uuid4
from cStringIO import StringIO

from .utils import guess_mimetype


CRLF = '\r\n'

class MultipartWriter(object):

    def __init__(self, buf, headers={}, subtype='mixed', boundary=None):
        self.buf = buf
        if boundary is None:
            boundary = self._make_boundary()
        self.boundary = boundary
        headers['Content-Type'] = 'multipart/%s; boundary="%s"' % (
            subtype, self.boundary
        )
        self.headers = headers

    def open(self, **kwargs):
        self.buf.write('--')
        self.buf.write(self.boundary)
        self.buf.write(CRLF)
        return MultipartWriter(self.buf, **kwargs)

    def add(self, fn, content):
        self.buf.write('--')
        self.buf.write(self.boundary)
        self.buf.write(CRLF)

        headers = {'Content-Type': guess_mimetype(fn)}
        headers.update(content_disposition_header(fn))

        self._write_headers(headers)
        if content:
            self.buf.write(content)
            self.buf.write(CRLF)

    def close(self):
        self.buf.write('--')
        self.buf.write(self.boundary)
        self.buf.write('--')
        self.buf.write(CRLF)

    def _make_boundary(self):
        return uuid4().hex

    def write_headers(self):
        self._write_headers(self.headers)

    def _write_headers(self, headers):
        if headers:
            for name in sorted(headers.keys()):
                self.buf.write(name)
                self.buf.write(': ')
                self.buf.write(headers[name])
                self.buf.write(CRLF)
        self.buf.write(CRLF)


def content_disposition_header(fn, disptype='file'):
    disposition = '%s; filename="%s"' % (disptype, quote(fn, safe=''))
    return {'Content-Disposition': disposition}


def recursive(dirname, fnpattern='*'):
    """
    Python-requests can supports chunked output by passing a generator to
    the [data] keyword arg in the request api.  This should be a chunked
    generator... eventually.

    TODO: transform this into a generator for chunked file output
    """
    
    # this should really be a io.BufferedWriter or something
    buf = StringIO()


    def walk(dirname, part):
        subpart = part.open(headers=content_disposition_header(dirname))
        subpart.write_headers()

        ls = os.listdir(dirname)
        files = filter(lambda p: os.path.isfile(os.path.join(dirname, p)), ls)
        dirs  = filter(lambda p: os.path.isdir(os.path.join(dirname, p)), ls)
        <commit_msg>minor<commit_after>"""
Streaming Multipart Encoded Files
=================================

**NOTE: This is temporary until we can fork python-requests and/or urllib3 to
        provide the same functionality.
"""
import os
import fnmatch
from urllib import quote
from uuid import uuid4
from cStringIO import StringIO

from .utils import guess_mimetype


CRLF = '\r\n'

class MultipartWriter(object):

    def __init__(self, buf, headers={}, subtype='mixed', boundary=None):
        self.buf = buf
        
        if boundary is None:
            boundary = self._make_boundary()
        self.boundary = boundary
        
        headers['Content-Type'] = 'multipart/%s; boundary="%s"' % (
            subtype,
            self.boundary
        )
        self.headers = headers

    def open(self, **kwargs):
        self.buf.write('--')
        self.buf.write(self.boundary)
        self.buf.write(CRLF)
        return MultipartWriter(self.buf, **kwargs)

    def add(self, fn, content):
        self.buf.write('--')
        self.buf.write(self.boundary)
        self.buf.write(CRLF)

        headers = {'Content-Type': guess_mimetype(fn)}
        headers.update(content_disposition_header(fn))

        self._write_headers(headers)
        if content:
            self.buf.write(content)
            self.buf.write(CRLF)

    def close(self):
        self.buf.write('--')
        self.buf.write(self.boundary)
        self.buf.write('--')
        self.buf.write(CRLF)

    def _make_boundary(self):
        return uuid4().hex

    def write_headers(self):
        self._write_headers(self.headers)

    def _write_headers(self, headers):
        if headers:
            for name in sorted(headers.keys()):
                self.buf.write(name)
                self.buf.write(': ')
                self.buf.write(headers[name])
                self.buf.write(CRLF)
        self.buf.write(CRLF)


def content_disposition_header(fn, disptype='file'):
    disposition = '%s; filename="%s"' % (
        disptype,
        quote(fn, safe='')
    )
    return {'Content-Disposition': disposition}


def recursive(dirname, fnpattern='*'):
    """
    Python-requests can supports chunked output by passing a generator to
    the [data] keyword arg in the request api.  This should be a chunked
    generator... eventually.

    TODO: transform this into a generator for chunked file output
    """
    
    # this should really be a io.BufferedWriter or something
    buf = StringIO()


    def walk(dirname, part):
        subpart = part.open(headers=content_disposition_header(dirname))
        subpart.write_headers()

        ls = os.listdir(dirname)
        files = filter(lambda p: os.path.isfile(os.path.join(dirname, p)), ls)
        dirs  = filter(lambda p: os.path.isdir(os.path.join(dirname, p)), ls)
        
{'fraction_non_alphanumeric': 0.12970261554998208, 'fraction_numerical': 0.0010748835542816195, 'mean_word_length': 2.423053341508277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2186347', 'n_tokens_mistral': 1711, 'n_tokens_neox': 1625, 'n_words': 400}