<commit_before># pylint: skip-file
import os
import sys
import numpy as np
from numba import njit
import PIL.Image
import scipy.ndimage
from time import time

sys.path.append("..")
from dataset import DatasetIndex, Dataset, ImagesBatch, action, inbatch_parallel, any_action_failed


# Example of custom Batch class which defines some actions
class MyImages(ImagesBatch):
    @action
    @inbatch_parallel(init='indices', post='assemble')
    def load(self, ix, src, fmt=None):
        if fmt == 'PIL':
            return PIL.Image.fromarray(src[ix])
        else:
            return src[ix]


if __name__ == "__main__":
    # number of items in the dataset
    K = 1000
    S = 128

    # Fill-in dataset with sample data
    def gen_data():
        ix = np.arange(K)
        data = np.random.randint(0, 255, size=K*S*S).reshape(K, S, S)
        dsindex = DatasetIndex(ix)
        ds = Dataset(index=dsindex, batch_class=MyImages)
        return ds, data


    # Create datasets
    print("Generating...")
    ds_data, data = gen_data()

    res = ds_data.p.load(data).convert_to_PIL(dst='images').resize((384, 384))
    #res = ds_data.p.load(data).resize((384, 384), method='PIL')

    print("Start...")
    t = time()
    res.run(K//10, n_epochs=1)
    print("End", time() - t)<commit_msg>Removed dst from convert_to_PIL call<commit_after># pylint: skip-file
import os
import sys
import numpy as np
from numba import njit
import PIL.Image
import scipy.ndimage
from time import time

sys.path.append("..")
from dataset import DatasetIndex, Dataset, ImagesBatch, action, inbatch_parallel, any_action_failed


# Example of custom Batch class which defines some actions
class MyImages(ImagesBatch):
    @action
    @inbatch_parallel(init='indices', post='assemble')
    def load(self, ix, src, fmt=None):
        if fmt == 'PIL':
            return PIL.Image.fromarray(src[ix])
        else:
            return src[ix]


if __name__ == "__main__":
    # number of items in the dataset
    K = 1000
    S = 128

    # Fill-in dataset with sample data
    def gen_data():
        ix = np.arange(K)
        data = np.random.randint(0, 255, size=K*S*S).reshape(K, S, S)
        dsindex = DatasetIndex(ix)
        ds = Dataset(index=dsindex, batch_class=MyImages)
        return ds, data


    # Create datasets
    print("Generating...")
    ds_data, data = gen_data()

    res = ds_data.p.load(data).convert_to_PIL('images').resize((384, 384))
    #res = ds_data.p.load(data).resize((384, 384), method='PIL')

    print("Start...")
    t = time()
    res.run(K//10, n_epochs=1)
    print("End", time() - t)
{'fraction_non_alphanumeric': 0.12956555171088044, 'fraction_numerical': 0.019992310649750097, 'mean_word_length': 3.00924499229584, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5677246', 'n_tokens_mistral': 929, 'n_tokens_neox': 869, 'n_words': 237}