<jupyter_start><jupyter_text>Explicitly open a Dask cluster,<jupyter_code>cluster = dask.distributed.LocalCluster(
    n_workers=4, processes=True,
    local_directory=os.getenv('TMPDIR'),
    ip='0.0.0.0',
)
client = dask.distributed.Client(cluster)
client<jupyter_output><empty_output><jupyter_text>Specify file paths,<jupyter_code>source_path='/gxfs_work1/geomar/smomw355/model_data/ocean-only/VIKING20X.L46-KFS003/nemo/output'
example_file='1_VIKING20X.L46-KFS003_1m_19580101_19581231_grid_T.nc'
target_path='/gxfs_work1/geomar/smomw260/github/NEMO-to-cloud'
target_prefix='1_VIKING20X.L46-KFS003_1m_grid_T'<jupyter_output><empty_output><jupyter_text>Clean-up target file system location,<jupyter_code>!rm -r "$target_path/$target_prefix".nc "$target_path/$target_prefix".zarr
!ls "$target_path"<jupyter_output><empty_output><jupyter_text>Check original netcdf format details,<jupyter_code>!ncdump -s -h "$source_path/$example_file"<jupyter_output>netcdf \1_VIKING20X.L46-KFS003_1m_19580101_19581231_grid_T {
dimensions:
	y = 2499 ;
	x = 2404 ;
	deptht = 46 ;
	axis_nbounds = 2 ;
	time_counter = UNLIMITED ; // (12 currently)
variables:
	float nav_lat(y, x) ;
		nav_lat:standard_name = "latitude" ;
		nav_lat:long_name = "Latitude" ;
		nav_lat:units = "degrees_north" ;
		nav_lat:_Storage = "chunked" ;
		nav_lat:_ChunkSizes = 2499, 2404 ;
		nav_lat:_Endianness = "little" ;
	float nav_lon(y, x) ;
		nav_lon:standard_name = "longitude" ;
		nav_lon:long_name = "Longitude" ;
		nav_lon:units = "degrees_east" ;
		nav_lon:_Storage = "chunked" ;
		nav_lon:_ChunkSizes = 2499, 2404 ;
		nav_lon:_Endianness = "little" ;
	float deptht(deptht) ;
		deptht:name = "deptht" ;
		deptht:long_name = "Vertical T levels" ;
		deptht:units = "m" ;
		deptht:positive = "down" ;
		deptht:bounds = "deptht_bounds" ;
		deptht:_Storage = "contiguous" ;
		deptht:_Endianness = "little" ;
	float deptht_bounds(deptht, axis_nbounds) ;
		deptht_bounds:units = "m" ;
		deptht[...]<jupyter_text>Open example file,<jupyter_code>original_data = xarray.open_dataset(source_path+'/'+example_file)[['votemper', 'vosaline']]
original_data.encoding
original_data.votemper.encoding
original_data.vosaline.encoding
%time original_data.votemper.isel(deptht=0).mean('time_counter').plot()
%time original_data.vosaline.isel(deptht=0).mean('time_counter').plot()<jupyter_output>CPU times: user 1.04 s, sys: 307 ms, total: 1.35 s
Wall time: 2.3 s<jupyter_text>Specify a target chunk size,<jupyter_code># this one fits nicely to the original chunks and is very speedy...
original_data_rechunked = original_data.chunk({
    'time_counter': 1,
    'deptht': None, 'y': None, 'x': None,
})
# this one might already be useful...
original_data_rechunked_not_used_here = original_data.chunk({
    'time_counter': 2,
    'deptht': None, # full size
    'y': 750, 'x': 750,
})<jupyter_output><empty_output><jupyter_text>Check that chunk size is compatible with Dask workers,<jupyter_code>def print_chunk_size(da, dtype=numpy.float32):
    target_chunk_size=numpy.prod(
        [max(c) for c in list(da.chunks)]
    )*numpy.zeros(1, dtype=dtype).nbytes/1e6 # in MB
    print(f"{target_chunk_size} MB")
original_data_rechunked.vosaline.dtype
print_chunk_size(original_data_rechunked.vosaline, numpy.float32)
original_data_rechunked_not_used_here.vosaline.dtype
print_chunk_size(original_data_rechunked_not_used_here.vosaline, numpy.float32)
worker_mem = cluster.worker_spec[0]['options']['memory_limit']/1e9 # in GB
worker_cpus = cluster.worker_spec[0]['options']['nthreads']
this_is_several_times = 10
max_chunk_size = (worker_mem/worker_cpus)/this_is_several_times*1e3
print(f"{max_chunk_size} MB")<jupyter_output>536.8709120000001 MB<jupyter_text>Write to disk with an explicit variable encoding,<jupyter_code>original_data_rechunked.encoding = {}
%%time
original_data_rechunked.to_zarr(
    store=target_path+'/'+target_prefix+'.zarr', mode='w',
    encoding={
        'votemper': {'dtype': 'float32'},
        'vosaline': {'dtype': 'float32'}
    },
)<jupyter_output>CPU times: user 368 ms, sys: 68.5 ms, total: 437 ms
Wall time: 9.51 s<jupyter_text>Open Zarr store,<jupyter_code>zarr_store = xarray.open_zarr(
    store=target_path+'/'+target_prefix+'.zarr'
)
zarr_store.encoding
zarr_store.votemper.encoding
zarr_store.vosaline.encoding
%time zarr_store.votemper.isel(deptht=0).mean('time_counter').plot()
%time zarr_store.vosaline.isel(deptht=0).mean('time_counter').plot()<jupyter_output>CPU times: user 657 ms, sys: 109 ms, total: 766 ms
Wall time: 1.88 s<jupyter_text>Save as netcdf again,<jupyter_code>%time zarr_store.to_netcdf(target_path+'/'+target_prefix+'.nc')
!ncdump -s -h "$target_path/$target_prefix".nc<jupyter_output>netcdf \1_VIKING20X.L46-KFS003_1m_grid_T {
dimensions:
	deptht = 46 ;
	y = 2499 ;
	x = 2404 ;
	time_counter = 12 ;
variables:
	float deptht(deptht) ;
		deptht:_FillValue = NaNf ;
		deptht:bounds = "deptht_bounds" ;
		deptht:long_name = "Vertical T levels" ;
		deptht:name = "deptht" ;
		deptht:positive = "down" ;
		deptht:units = "m" ;
		deptht:_Storage = "contiguous" ;
		deptht:_Endianness = "little" ;
	float nav_lat(y, x) ;
		nav_lat:_FillValue = NaNf ;
		nav_lat:long_name = "Latitude" ;
		nav_lat:standard_name = "latitude" ;
		nav_lat:units = "degrees_north" ;
		nav_lat:_Storage = "contiguous" ;
		nav_lat:_Endianness = "little" ;
	float nav_lon(y, x) ;
		nav_lon:_FillValue = NaNf ;
		nav_lon:long_name = "Longitude" ;
		nav_lon:standard_name = "longitude" ;
		nav_lon:units = "degrees_east" ;
		nav_lon:_Storage = "contiguous" ;
		nav_lon:_Endianness = "little" ;
	double time_centered(time_counter) ;
		time_centered:_FillValue = NaN ;
		time_centered:bounds = "time_centered_bounds" ;
		[...]<jupyter_text>From netcdf again,<jupyter_code>netcdf_data = xarray.open_dataset(target_path+'/'+target_prefix+'.nc')
netcdf_data.encoding
netcdf_data.votemper.encoding
netcdf_data.vosaline.encoding
%time netcdf_data.votemper.isel(deptht=0).mean('time_counter').plot()
%time netcdf_data.vosaline.isel(deptht=0).mean('time_counter').plot()<jupyter_output>CPU times: user 842 ms, sys: 230 ms, total: 1.07 s
Wall time: 1.12 s<jupyter_text>Python environment,<jupyter_code>pip list
!mamba list --explicit<jupyter_output>__    __    __    __
                 /  \  /  \  /  \  /  \
                /    \/    \/    \/    \
███████████████/  /██/  /██/  /██/  /████████████████████████
              /  / \   / \   / \   / \  \____
             /  /   \_/   \_/   \_/   \    o \__,
            / _/                       \_____/  `
            |/
        ███╗   ███╗ █████╗ ███╗   ███╗██████╗  █████╗
        ████╗ ████║██╔══██╗████╗ ████║██╔══██╗██╔══██╗
        ██╔████╔██║███████║██╔████╔██║██████╔╝███████║
        ██║╚██╔╝██║██╔══██║██║╚██╔╝██║██╔══██╗██╔══██║
        ██║ ╚═╝ ██║██║  ██║██║ ╚═╝ ██║██████╔╝██║  ██║
        ╚═╝     ╚═╝╚═╝  ╚═╝╚═╝     ╚═╝╚═════╝ ╚═╝  ╚═╝

        mamba (0.9.1) supported by @QuantStack

        GitHub:  https://github.com/mamba-org/mamba
        Twitter: https://twitter.com/QuantStack

█████████████████████████████████████████████████████████████

# This file may be used to create an environment using:
# $ conda create --name <env> --file <this file>
# platfor[...]
{'fraction_non_alphanumeric': 0.17799082951229678, 'fraction_numerical': 0.03195776017785188, 'mean_word_length': 4.654359780047133, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '492119', 'license': 'MIT', 'n_tokens_mistral': 3258, 'n_tokens_neox': 3025, 'n_words': 437, 'path': 'exploratory/21-04-01_netcdf-zarr-conversion-tests.ipynb', 'repo_name': 'geomar-od/NEMO-cloud-project', 'size': '7197'}