<jupyter_start><jupyter_text>**Conclusion:** * It is always slower when writing the whole block of data compared to writing data in a loop timestep by timesteps* This is also true when specifying the chunksize to [1, 900, 900] (which is also seems to be the default when not specifying the chunksize...)* This is also true when not using compression (which speeds up things considerably)<jupyter_code>%load_ext autoreload
%autoreload 2
import xarray as xr
import radolan_to_netcdf as rtn
import numpy as np
import netCDF4
fn = '/tmp/test.nc'
test_data = np.random.randn(100, 900, 900)<jupyter_output><empty_output><jupyter_text>Write time step by time step<jupyter_code>rtn.create_empty_netcdf(fn=fn, product_name='RW')
%%time

with netCDF4.Dataset(fn, 'a') as nc_fh: 
    for i in range(test_data.shape[0]):
        nc_fh['rainfall_amount'][i, :, :] = test_data[i, :, :]<jupyter_output>CPU times: user 35.9 s, sys: 394 ms, total: 36.3 s
Wall time: 9.23 s<jupyter_text>Write whole block of data at once<jupyter_code>rtn.create_empty_netcdf(fn=fn, product_name='RW')
%%time

with netCDF4.Dataset(fn, 'a') as nc_fh: 
    nc_fh['rainfall_amount'][0:100, :, :] = test_data[0:100, :, :]
ls -lh /tmp/test.nc<jupyter_output>-rw-r--r--  1 chwala-c  wheel    79M  7 Mai 09:42 /tmp/test.nc<jupyter_text>Try if it is faster with defined chunksize<jupyter_code>metadata_per_timestamp = {
    'maxrange': {
        'variable_parameters': {
            'datatype': 'i2',
            'dimensions': ('time'),
        },
        'attributes': {
            'units': 'km',
        },
    },
    'radarlocations': {
        'variable_parameters': {
            'datatype': str,
            'dimensions': ('time'),
        },
        'attributes': {
            'long_name': 'List of radar locations available at time stamp'
        },
    },
}

config = {
        'variables': {
            'rainfall_amount': {
                'variable_parameters': {
                    'datatype': 'i2',
                    'dimensions': ('time', 'y', 'x'),
                    'chunksizes': [1, 900, 900],
                    'fill_value': -9999,
                    'zlib': True,
                    'complevel': 5,
                },
                'attributes': {
                    'long_name': 'Hourly rainfall',
                    'standard_name': 'rainfall_amount',
                    'units': 'kg',
                    'scale_factor': 0.1,
                    'add_offset': 0,
                    'coordinates': 'longitudes latitudes',
                    'grid_mapping': 'RADOLAN_grid',
                },
            },
        },
        'metadata_per_timestamp': metadata_per_timestamp,
        'metadata_fixed': {
            'n_lats': 900,
            'n_lons': 900,
        }
}
rtn.create_empty_netcdf(fn=fn, product_config_dict=config, product_name='RW')
%%time

with netCDF4.Dataset(fn, 'a') as nc_fh: 
    for i in range(test_data.shape[0]):
        nc_fh['rainfall_amount'][i, :, :] = test_data[i, :, :]
ls -lh /tmp/test.nc
rtn.create_empty_netcdf(fn=fn, product_config_dict=config, product_name='RW')
%%time

with netCDF4.Dataset(fn, 'a') as nc_fh: 
    nc_fh['rainfall_amount'][0:100, :, :] = test_data[0:100, :, :]  
ls -lh /tmp/test.nc<jupyter_output>-rw-r--r--  1 chwala-c  wheel    79M  7 Mai 09:43 /tmp/test.nc<jupyter_text>Try without compression<jupyter_code>metadata_per_timestamp = {
    'maxrange': {
        'variable_parameters': {
            'datatype': 'i2',
            'dimensions': ('time'),
        },
        'attributes': {
            'units': 'km',
        },
    },
    'radarlocations': {
        'variable_parameters': {
            'datatype': str,
            'dimensions': ('time'),
        },
        'attributes': {
            'long_name': 'List of radar locations available at time stamp'
        },
    },
}

config = {
        'variables': {
            'rainfall_amount': {
                'variable_parameters': {
                    'datatype': 'i2',
                    'dimensions': ('time', 'y', 'x'),
                    'fill_value': -9999,
                    'zlib': False,
                    'complevel': 0,
                },
                'attributes': {
                    'long_name': 'Hourly rainfall',
                    'standard_name': 'rainfall_amount',
                    'units': 'kg',
                    'scale_factor': 0.1,
                    'add_offset': 0,
                    'coordinates': 'longitudes latitudes',
                    'grid_mapping': 'RADOLAN_grid',
                },
            },
        },
        'metadata_per_timestamp': metadata_per_timestamp,
        'metadata_fixed': {
            'n_lats': 900,
            'n_lons': 900,
        }
}
rtn.create_empty_netcdf(fn=fn, product_config_dict=config, product_name='RW')
%%time

with netCDF4.Dataset(fn, 'a') as nc_fh: 
    for i in range(test_data.shape[0]):
        nc_fh['rainfall_amount'][i, :, :] = test_data[i, :, :]
ls -lh /tmp/test.nc
rtn.create_empty_netcdf(fn=fn, product_config_dict=config, product_name='RW')
%%time

with netCDF4.Dataset(fn, 'a') as nc_fh: 
    nc_fh['rainfall_amount'][0:100, :, :] = test_data[0:100, :, :]<jupyter_output>CPU times: user 1.74 s, sys: 1.05 s, total: 2.78 s
Wall time: 1.05 s<jupyter_text>Trying different cache sizesC documentationhttps://www.unidata.ucar.edu/software/netcdf/docs/group__variables.htmlga2788cbfc6880ec70c304292af2bc7546Python documentaitonhttp://unidata.github.io/netcdf4-python/netCDF4/index.htmlnetCDF4.Variable.set_var_chunk_cache<jupyter_code>metadata_per_timestamp = {
    'maxrange': {
        'variable_parameters': {
            'datatype': 'i2',
            'dimensions': ('time'),
        },
        'attributes': {
            'units': 'km',
        },
    },
    'radarlocations': {
        'variable_parameters': {
            'datatype': str,
            'dimensions': ('time'),
        },
        'attributes': {
            'long_name': 'List of radar locations available at time stamp'
        },
    },
}

config = {
        'variables': {
            'rainfall_amount': {
                'variable_parameters': {
                    'datatype': 'i2',
                    'dimensions': ('time', 'y', 'x'),
                    'chunksizes': [1, 900, 900],
                    'fill_value': -9999,
                    'zlib': True,
                    'complevel': 5,
                },
                'attributes': {
                    'long_name': 'Hourly rainfall',
                    'standard_name': 'rainfall_amount',
                    'units': 'kg',
                    'scale_factor': 0.1,
                    'add_offset': 0,
                    'coordinates': 'longitudes latitudes',
                    'grid_mapping': 'RADOLAN_grid',
                },
            },
        },
        'metadata_per_timestamp': metadata_per_timestamp,
        'metadata_fixed': {
            'n_lats': 900,
            'n_lons': 900,
        }
}

rtn.create_empty_netcdf(fn=fn, product_config_dict=config, product_name='RW')
%%time

with netCDF4.Dataset(fn, 'a') as nc_fh: 
    nc_fh['rainfall_amount'].set_var_chunk_cache(size=100e6, nelems=1, preemption=1)
    nc_fh['rainfall_amount'][0:100, :, :] = test_data[0:100, :, :]<jupyter_output>CPU times: user 10.6 s, sys: 1.04 s, total: 11.6 s
Wall time: 9.83 s
{'fraction_non_alphanumeric': 0.14223314223314223, 'fraction_numerical': 0.027846027846027847, 'mean_word_length': 1.989392084863321, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '410285', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2606, 'n_tokens_neox': 2386, 'n_words': 472, 'path': 'notebooks/misc/test performance of NetCDF batch write.ipynb', 'repo_name': 'plvoit/radolan_to_netcdf', 'size': '7326'}