<commit_before>import pmi
from mpi4py import MPI
pmi.exec_('import test_pmi_mod')
import numpy as np
import h5py


class MyTestLocal(object):
    def __init__(self, filename, n):
        self.filename = filename
        self.f = h5py.File(
            self.filename, 'w',
            driver='mpio', comm=MPI.COMM_WORLD)
        self.n = n
        self.data = np.random.random(n)

    def show(self):
        print MPI.COMM_WORLD.rank, self.data

    def fill(self):
        self.f.create_dataset('ds1',
                              dtype=self.data.dtype,
                              shape=(MPI.COMM_WORLD.size*self.n,))
        idx_0 = MPI.COMM_WORLD.rank*self.n
        idx_1 = idx_0+self.n
        self.f['ds1'][idx_0:idx_1] = self.data

    def close(self):
        self.f.close()


class MyTest(object):
    __metaclass__ = pmi.Proxy
    pmiproxydefs = {<commit_msg>Add chunking.<commit_after>import pmi
from mpi4py import MPI
pmi.exec_('import test_pmi_mod')
import numpy as np
import h5py


class MyTestLocal(object):
    def __init__(self, filename, n):
        self.filename = filename
        self.f = h5py.File(
            self.filename, 'w',
            driver='mpio', comm=MPI.COMM_WORLD)
        self.n = n
        self.data = np.random.random(n)

    def show(self):
        print MPI.COMM_WORLD.rank, self.data

    def fill(self):
        self.f.create_dataset('ds1',
                              dtype=self.data.dtype,
                              shape=(MPI.COMM_WORLD.size*self.n,),
                              chunks=(self.n,))
        idx_0 = MPI.COMM_WORLD.rank*self.n
        idx_1 = idx_0+self.n
        self.f['ds1'][idx_0:idx_1] = self.data

    def close(self):
        self.f.close()


class MyTest(object):
    __metaclass__ = pmi.Proxy
    pmiproxydefs = {
{'fraction_non_alphanumeric': 0.11872909698996656, 'fraction_numerical': 0.011148272017837236, 'mean_word_length': 1.928221859706362, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1824141', 'n_tokens_mistral': 639, 'n_tokens_neox': 619, 'n_words': 114}