<commit_before>from MFDataset import Dataset
import netCDF4_classic, numpy
from numpy.random import seed, randint
from numpy.testing import assert_array_equal
import tempfile, unittest, os

nx=100; ydim=5; zdim=10
nfiles = 10
ninc = nx/nfiles
files = [tempfile.mktemp(".nc") for nfile in range(nfiles)]
data = randint(0,10,size=(nx,ydim,zdim))

class VariablesTestCase(unittest.TestCase):

    def setUp(self):
        self.files = files
        for nfile,file in enumerate(self.files):
            if nfile == 0:
                f = netCDF4_classic.Dataset(file,'w',format='NETCDF3_CLASSIC')
            else:
                f = netCDF4_classic.Dataset(file,'w')
            f.createDimension('x',None)
            f.createDimension('y',ydim)
            f.createDimension('z',zdim)
            f.history = 'created today'
            x = f.createVariable('x','i',('x',))
            x.units = 'zlotnys'
            dat = f.createVariable('data','i',('x','y','z',))
            dat.name = 'phony data' 
            nx1 = nfile*ninc; nx2 = ninc*(nfile+1)
            x[0:ninc] = numpy.arange(nfile*ninc,ninc*(nfile+1))
            dat[0:ninc] = data[nx1:nx2]
            f.close()

    def tearDown(self):
        # Remove the temporary files
        for file in self.files:
            os.remove(file)

    def runTest(self):
        """testing multi-file dataset access"""
        f = Dataset(self.files)
        assert_array_equal(numpy.arange(0,nx),f.variables['x'][:])
        datin = f.variables['data'][4:-4:4,3:5,2:8]
        assert_array_equal(datin,data[4:-4:4,3:5,2:8])
        f.close()

if __name__ == '__main__':
    unittest.main()<commit_msg>fixed bug in attribute check<commit_after>from MFDataset import Dataset
import netCDF4_classic, numpy
from numpy.random import seed, randint
from numpy.testing import assert_array_equal
import tempfile, unittest, os

nx=100; ydim=5; zdim=10
nfiles = 10
ninc = nx/nfiles
files = [tempfile.mktemp(".nc") for nfile in range(nfiles)]
data = randint(0,10,size=(nx,ydim,zdim))

class VariablesTestCase(unittest.TestCase):

    def setUp(self):
        self.files = files
        for nfile,file in enumerate(self.files):
            if nfile == 0:
                f = netCDF4_classic.Dataset(file,'w',format='NETCDF3_CLASSIC')
            else:
                f = netCDF4_classic.Dataset(file,'w')
            f.createDimension('x',None)
            f.createDimension('y',ydim)
            f.createDimension('z',zdim)
            f.history = 'created today'
            x = f.createVariable('x','i',('x',))
            x.units = 'zlotnys'
            dat = f.createVariable('data','i',('x','y','z',))
            dat.name = 'phony data' 
            nx1 = nfile*ninc; nx2 = ninc*(nfile+1)
            x[0:ninc] = numpy.arange(nfile*ninc,ninc*(nfile+1))
            dat[0:ninc] = data[nx1:nx2]
            f.close()

    def tearDown(self):
        # Remove the temporary files
        for file in self.files:
            os.remove(file)

    def runTest(self):
        """testing multi-file dataset access"""
        f = Dataset(self.files)
        assert f.history == 'created today'
        assert_array_equal(numpy.arange(0,nx),f.variables['x'][:])
        datin = f.variables['data'][4:-4:4,3:5,2:8]
        assert f.variables['data'].name == 'phony data'
        assert_array_equal(datin,data[4:-4:4,3:5,2:8])
        f.close()

if __name__ == '__main__':
    unittest.main()
{'fraction_non_alphanumeric': 0.1482668220215555, 'fraction_numerical': 0.022720652490533063, 'mean_word_length': 2.5184426229508197, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '218731', 'n_tokens_mistral': 1203, 'n_tokens_neox': 1182, 'n_words': 221}