<commit_before>from parcels import (FieldSet, ScipyParticle, JITParticle, Variable, ErrorCode)
from parcels.particlefile import _set_calendar
from parcels.tools.converters import _get_cftime_calendars, _get_cftime_datetimes
from parcels import ParticleSetSOA, ParticleFileSOA, KernelSOA  # noqa
from parcels import ParticleSetAOS, ParticleFileAOS, KernelAOS  # noqa
import numpy as np
import pytest
import os
from netCDF4 import Dataset
import cftime
import random as py_random
import xarray as xr

pset_modes = ['soa', 'aos']
ptype = {'scipy': ScipyParticle, 'jit': JITParticle}
pset_type = {'soa': {'pset': ParticleSetSOA, 'pfile': ParticleFileSOA, 'kernel': KernelSOA},
             'aos': {'pset': ParticleSetAOS, 'pfile': ParticleFileAOS, 'kernel': KernelAOS}}


def fieldset(xdim=40, ydim=100):
    U = np.zeros((ydim, xdim), dtype=np.float32)
    V = np.zeros((ydim, xdim), dtype=np.float32)
    lon = np.linspace(0, 1, xdim, dtype=np.float32)
    lat = np.linspace(-60, 60, ydim, dtype=np.float32)
    depth = np.zeros(1, dtype=np.float32)
    data = {'U': np.array(U, dtype=np.float32), 'V': np.array(V, dtype=np.float32)}
    dimensions = {'lat': lat, 'lon': lon, 'depth': depth}
    return FieldSet.from_data(data, dimensions)


@pytest.fixture(name="fieldset")
def fieldset_ficture(xdim=40, ydim=100):
    return fieldset(xdim=xdim, ydim=ydim)


def close_and_compare_netcdffiles(filepath, ofile, assystemcall=False):
    if assystemcall:
        os.system('parcels_convert_npydir_to_netcdf %s' % ofile.tempwritedir_base)
    else:
        import parcels.scripts.convert_npydir_to_netcdf as convert
        convert.convert_npydir_to_netcdf(ofile.tempwritedir_base, pfile_class=ofile.__class__)

    engine = 'zarr' if 'zarr' in str(filepath) else 'netcdf4'
    ncfile1 = xr.open_dataset(filepath, engine=engine)

    ofile.name = filepath + 'b.nc'
    ofile.export()

    if engine == 'zarr':
        assert os.path.getsize(filepath) < os.path.getsize(ofile.name)
    else:
        assert os.path.getsize(filepath) == os.path.getsize(ofile.name)

    ncfile2 = xr.open_dataset(filepath + 'b.nc')
    for v in ncfile2.keys():
        if v == 'time':
            assert np.all(ncfile1[v].values-ncfile2[v].values < np.timedelta64(1, 's'), where=np.isfinite(ncfile1[v].values))
        else:
            assert np.allclose(ncfile1[v].values, ncfile2[v].values, equal_nan=True)

    for a in ncfile2.attrs:
        if a != 'parcels_version':
            assert getattr(ncfile1, a) == getattr(ncfile2, a)

    ncfile2.close()
    return ncfile1

<commit_msg>Small improvements to zarr unit tests<commit_after>from parcels import (FieldSet, ScipyParticle, JITParticle, Variable, ErrorCode)
from parcels.particlefile import _set_calendar
from parcels.tools.converters import _get_cftime_calendars, _get_cftime_datetimes
from parcels import ParticleSetSOA, ParticleFileSOA, KernelSOA  # noqa
from parcels import ParticleSetAOS, ParticleFileAOS, KernelAOS  # noqa
import numpy as np
import pytest
import os
from netCDF4 import Dataset
import cftime
import random as py_random
import xarray as xr

pset_modes = ['soa', 'aos']
ptype = {'scipy': ScipyParticle, 'jit': JITParticle}
pset_type = {'soa': {'pset': ParticleSetSOA, 'pfile': ParticleFileSOA, 'kernel': KernelSOA},
             'aos': {'pset': ParticleSetAOS, 'pfile': ParticleFileAOS, 'kernel': KernelAOS}}


def fieldset(xdim=40, ydim=100):
    U = np.zeros((ydim, xdim), dtype=np.float32)
    V = np.zeros((ydim, xdim), dtype=np.float32)
    lon = np.linspace(0, 1, xdim, dtype=np.float32)
    lat = np.linspace(-60, 60, ydim, dtype=np.float32)
    depth = np.zeros(1, dtype=np.float32)
    data = {'U': np.array(U, dtype=np.float32), 'V': np.array(V, dtype=np.float32)}
    dimensions = {'lat': lat, 'lon': lon, 'depth': depth}
    return FieldSet.from_data(data, dimensions)


@pytest.fixture(name="fieldset")
def fieldset_ficture(xdim=40, ydim=100):
    return fieldset(xdim=xdim, ydim=ydim)


def close_and_compare_netcdffiles(filepath, ofile, assystemcall=False):
    if assystemcall:
        os.system('parcels_convert_npydir_to_netcdf %s' % ofile.tempwritedir_base)
    else:
        import parcels.scripts.convert_npydir_to_netcdf as convert
        convert.convert_npydir_to_netcdf(ofile.tempwritedir_base, pfile_class=ofile.__class__)

    engine = 'zarr' if 'zarr' in str(filepath) else 'netcdf4'
    ncfile1 = xr.open_dataset(filepath, engine=engine)

    ofile.name = filepath + 'b.nc'
    ofile.export()

    if engine == 'zarr':
        assert os.path.getsize(filepath) < os.path.getsize(ofile.name)  # zarr expected to be smaller filesize
    else:
        assert os.path.getsize(filepath) == os.path.getsize(ofile.name)

    ncfile2 = xr.open_dataset(filepath + 'b.nc')
    for v in ncfile2.keys():
        if v == 'time':
            assert np.allclose(ncfile1[v].values, ncfile2[v].values, atol=np.timedelta64(1, 's'), equal_nan=True)
        else:
            assert np.allclose(ncfile1[v].values, ncfile2[v].values, equal_nan=True)

    for a in ncfile2.attrs:
        if a != 'parcels_version':
            assert getattr(ncfile1, a) == getattr(ncfile2, a)

    ncfile2.close()
    return ncfile1


{'fraction_non_alphanumeric': 0.1273639521420301, 'fraction_numerical': 0.018718641451177152, 'mean_word_length': 4.136769078295342, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '93870', 'n_tokens_mistral': 2021, 'n_tokens_neox': 1951, 'n_words': 412}