<commit_before>from io import BytesIO
from numpy import random
from requests_toolbelt import MultipartEncoder
from streaming_form_data.parser import StreamingFormDataParser
from streaming_form_data.targets import NullTarget, BaseTarget
from time import time


class DummyTarget(BaseTarget):
    def __init__(self, print_report, gather_data):
        self.print_report = print_report
        self.gather_data = gather_data
        self.result = bytearray()

    def start(self):
        if self.print_report:
            print('DummyTarget: start')

    def data_received(self, chunk):
        if self.print_report:
            print('DummyTarget: data_received:', len(chunk), 'bytes')
        if self.gather_data:
            self.result += chunk

    def finish(self):
        if self.print_report:
            print('DummyTarget: finish')

    def get_result(self):
        return self.result


def fill_bytes_random_fast(size):
    random.seed(42)
    return random.bytes(size)


def main():
    print('Prepare data...')
    begin_time = time()

    kibibyte = 1024
    mebibyte = kibibyte * kibibyte
    filedata_size = 40 * mebibyte

    filedata = fill_bytes_random_fast(filedata_size)
    # print('data sample:', filedata[0:100])

    with BytesIO(filedata) as fd:
        content_type = 'binary/octet-stream'

        encoder = MultipartEncoder(fields={
            'file': ('file', fd, content_type)
        })
        headers = {'Content-Type': encoder.content_type}<commit_msg>speed up target implementation in speedtest.py; add unlucky data set to speedtest.py<commit_after>from io import BytesIO
from numpy import random
from requests_toolbelt import MultipartEncoder
from streaming_form_data.parser import StreamingFormDataParser
from streaming_form_data.targets import NullTarget, BaseTarget
from time import time


class DummyTarget(BaseTarget):
    def __init__(self, print_report, gather_data):
        self._print_report = print_report
        self._gather_data = gather_data
        self._values = []

    def start(self):
        if self._print_report:
            print('DummyTarget: start')

    def data_received(self, chunk):
        if self._print_report:
            print('DummyTarget: data_received:', len(chunk), 'bytes')
        if self._gather_data:
            self._values.append(chunk)

    def finish(self):
        if self._print_report:
            print('DummyTarget: finish')

    def get_result(self):
        return b''.join(self._values)


def fill_bytes_random(size):
    random.seed(42)
    return random.bytes(size)


def main():
    print('Prepare data...')
    begin_time = time()

    kibibyte = 1024
    mebibyte = kibibyte * kibibyte
    filedata_size = 40 * mebibyte

    filedata = fill_bytes_random(filedata_size)

    with BytesIO(filedata) as fd:
        content_type = 'binary/octet-stream'

        encoder = MultipartEncoder(fields={
            'file': ('file', fd, content_type)
        })
        headers = {'Content-Type': encoder.content_type}
{'fraction_non_alphanumeric': 0.09278350515463918, 'fraction_numerical': 0.006651147322913203, 'mean_word_length': 3.032171581769437, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6916156', 'n_tokens_mistral': 944, 'n_tokens_neox': 887, 'n_words': 215}