<commit_before>from numpy.random.mtrand import uniform
from netCDF4_classic import Dataset
import os

ndim = 100000
array = uniform(size=(ndim,))

def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=False,chunking='seq'):
    file = Dataset(filename,'w')
    file.createDimension('n', ndim)
    foo = file.createVariable('data', dtype,('n'),zlib=zlib,least_significant_digit=least_significant_digit,shuffle=shuffle,chunking=chunking)
    foo[:] = data
    file.close()
    file = Dataset(filename)
    data = file.variables['data'][:]
    print data[0:4]

# uncompressed.
filename = 'uncompressed.nc'
least_significant_digit = None
zlib = False
write_netcdf(filename,zlib,least_significant_digit,array)
print 'size of uncompressed.nc (no compression) = ',os.stat('uncompressed.nc').st_size
<commit_msg>add complevel keyword<commit_after>from numpy.random.mtrand import uniform
from netCDF4_classic import Dataset
import os

ndim = 100000
array = uniform(size=(ndim,))

def write_netcdf(filename,zlib,least_significant_digit,data,dtype='f8',shuffle=False,chunking='seq',complevel=6):
    file = Dataset(filename,'w')
    file.createDimension('n', ndim)
    foo = file.createVariable('data', dtype,('n'),zlib=zlib,least_significant_digit=least_significant_digit,shuffle=shuffle,chunking=chunking,complevel=complevel)
    foo[:] = data
    file.close()
    file = Dataset(filename)
    data = file.variables['data'][:]
    print data[0:4]

# uncompressed.
filename = 'uncompressed.nc'
least_significant_digit = None
zlib = False
write_netcdf(filename,zlib,least_significant_digit,array)
print 'size of uncompressed.nc (no compression) = ',os.stat('uncompressed.nc').st_size

{'fraction_non_alphanumeric': 0.13191990577149587, 'fraction_numerical': 0.012367491166077738, 'mean_word_length': 6.902325581395349, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2466725', 'n_tokens_mistral': 628, 'n_tokens_neox': 572, 'n_words': 96}