<jupyter_start><jupyter_text>NOTE: no error handling implemented Befeore running install netCDF4 module (https://unidata.github.io/netcdf4-python/netCDF4/index.html)  Modify the following section settings your local save path and the id of the folder to elaborate from https://github.com/Swiss-Polar-Institute/frictionless-data-packages<jupyter_code>### VARIABLE TO  MODIFY ###
id="10.5281_zenodo.3250980"
basesavepath="c:\\SPI\\"
from datapackage import Package
from tableschema import Table
import datetime
import os
import urllib.request
from netCDF4 import Dataset
import re
from datetime import datetime, timedelta
from netCDF4 import num2date, date2num, stringtochar
import numpy as np
def CreateNetCDF(savepath_NetCDF,package,resourceData):
    #create netcdf
    print("create netcdf")
    netcdf = Dataset(savepath_NetCDF, "w", format="NETCDF4")

    #create dimension
    print("create dimension")
    row = netcdf.createDimension("row", None) ##unlimited dimension

    #create global attribute
    print("create global attribute")
    netcdf.cdm_data_type="Other"
    netcdf.Conventions="COARDS, CF-1.6, ACDD-1.3"
    netcdf.creator_type = "institution"  
    #for key in package.descriptor:
    #    netcdf.netcdf(key,package.descriptor[key])

    if 'x_spi_platform_code' in package.descriptor:
        netcdf.platform_code=package.descriptor['x_spi_platform_code']
    if 'x_spi_platform_name' in package.descriptor:
        netcdf.platform_name=package.descriptor['x_spi_platform_name']
    if 'x_spi_type' in package.descriptor:
        netcdf.type=package.descriptor['x_spi_type']
    if 'x_spi_citation' in package.descriptor:
        netcdf.citation=package.descriptor['x_spi_citation']


    if 'keywords' in package.descriptor:
        netcdf.keywords=','.join(package.descriptor['keywords'])
    if 'contributors' in package.descriptor:
        if 'title' in package.descriptor['contributors'][0]:
            netcdf.creator_name = package.descriptor['contributors'][0]['title']
        if 'path' in package.descriptor['contributors'][0]:
            netcdf.creator_url = package.descriptor['contributors'][0]['path']
        if 'organisation' in package.descriptor['contributors'][0]:
            netcdf.institution = package.descriptor['contributors'][0]['organisation']
    if 'homepage' in package.descriptor:
        netcdf.DOI=package.descriptor['homepage']
        netcdf.infoUrl = package.descriptor['homepage']
    if 'licenses' in package.descriptor:    
        if 'name' in package.descriptor['licenses'][0]:    
            netcdf.license = package.descriptor['licenses'][0]['name']
    if 'description' in package.descriptor:
        netcdf.summary = package.descriptor['description']
    if 'contributors' in package.descriptor:
        netcdf.contributors=str(package.descriptor['contributors'])

    #create variables
    print("create variables")
    netcdf_variables={}
    time = netcdf.createVariable("time","f8",("row",))
    time._CoordinateAxisType = "Time";
    time.axis = "T";
    time.long_name = "Time";
    timestandard_name = "time";
    time.time_origin = "01-JAN-1970 00:00:00";
    time.units = "seconds since 1970-01-01T00:00:00Z"; 
    for field in resourceData.schema.fields:
        fieldName=field.name.replace(" ","")
        fieldName=re.sub(r'\[.*\]','',fieldName)
        fieldtype=""
        fieldsize=""
        if field.type=="string":
            fieldtype="S1"        
            maxFieldSize=0
            for row in resourceData.iter(keyed = True):
                if len(row[field.name])>maxFieldSize:
                    maxFieldSize=len(row[field.name]) 
            fieldsize=str(maxFieldSize)
        elif field.type=="integer":
            fieldtype="i4"
        elif field.type=="time":
            fieldtype="S1"
            fieldsize="4"
        elif field.type=="datetime":  ##is the time variable
            fieldtype=""
            netcdf_variables['time']=field.name
        elif field.type=="number":
            fieldtype="f8"

        if fieldtype=="":
            continue
        if fieldsize=="":
            netcdf_variables[field.name]=fieldName
            netcdf_variable=netcdf.createVariable(fieldName,fieldtype,("row",))
        else:
            netcdf.createDimension(fieldName + "_strlen", int(fieldsize))
            netcdf_variables[field.name]=fieldName
            netcdf_variable=netcdf.createVariable(fieldName,fieldtype,("row",fieldName + "_strlen",))

        netcdf_variable.description=field.descriptor['description']
        if 'unit' in field.descriptor:
            netcdf_variable.unit=field.descriptor['unit']
        if 'x_spi_cf_standard_name' in field.descriptor:
            netcdf_variable.standard_name=field.descriptor['x_spi_cf_standard_name']
        if 'x_spi_cf_unit' in field.descriptor:
            netcdf_variable.unit=field.descriptor['x_spi_cf_unit']
        if 'x_spi_cf_attribute' in field.descriptor:
            netcdf_variable.attribute=field.descriptor['x_spi_cf_attribute']
        netcdf_variable.spi_original_name=field.name
        #netcdf_variables[field.name]=newField

    #netcdf.close()

    print("fill values")    
    #netcdf = Dataset(savepath_NetCDF, "a", format="NETCDF4")
    values={}
    for row in resourceData.iter(keyed = True):
        datetime_value=row[netcdf_variables['time']]
        datetime_value_numeric=date2num([datetime_value],units="seconds since 1970-01-01T00:00:00Z",calendar="gregorian")[0]
        if not bool(values) :
            values['time']=[datetime_value_numeric]
        else:
            values['time']=np.append(values['time'],datetime_value_numeric)
        #netcdf.variables['time'][:]=values    
        for field in resourceData.schema.fields:
            if netcdf_variables['time'] != field.name:
                if netcdf.variables[netcdf_variables[field.name]].dtype == "S1":
                    #size=str(netcdf.dimensions[netcdf_variables[field.name] +'_strlen'].size)
                    #sizeFormat="{:<" + size + "}"
                    #value=sizeFormat.format(row[field.name])
                    value=row[field.name]
                    if netcdf_variables[field.name] in values:
                        values[netcdf_variables[field.name]]=np.append(values[netcdf_variables[field.name]],value)
                    else:
                        values[netcdf_variables[field.name]]=[value]                                    
                else:
                    if netcdf_variables[field.name] in values:
                        values[netcdf_variables[field.name]]=np.append(values[netcdf_variables[field.name]],row[field.name])
                    else:
                        values[netcdf_variables[field.name]]=[row[field.name]]

    print("populate netcdf variables")
    for field in values:         
        if field=='time':
            netcdf.variables['time'][:]=values[field]
        elif netcdf.variables[field].dtype == "S1":
            size=str(netcdf.dimensions[field +'_strlen'].size)
            datain=np.array(values[field],dtype='S' + size)
            netcdf.variables[field][:] = stringtochar(datain) 
        else:
            netcdf.variables[field][:]=values[field]
    netcdf.close() 
    print("END")
datapackage_link="https://raw.githubusercontent.com/Swiss-Polar-Institute/frictionless-data-packages/dev/" + id + "/datapackage.json"
tableschema_link="https://raw.githubusercontent.com/Swiss-Polar-Institute/frictionless-data-packages/dev/" + id + "/tableschema.json"
netcdf_name=id +".nc"
savepath=os.path.join(basesavepath, id)
savepath_datapackage=os.path.join(savepath, "datapackage.json")
savepath_tableschema=os.path.join(savepath, "tableschema.json")
savepath_NetCDF=os.path.join(savepath, netcdf_name)
if not os.path.exists(savepath):
    os.mkdir(savepath)
if os.path.exists(savepath_datapackage):
    os.remove(savepath_datapackage)
urllib.request.urlretrieve(datapackage_link, savepath_datapackage)
if os.path.exists(savepath_tableschema):
    os.remove(savepath_tableschema)
urllib.request.urlretrieve(tableschema_link, savepath_tableschema)
if os.path.exists(savepath_NetCDF):
    os.remove(savepath_NetCDF)
package = Package(savepath_datapackage)
print (package.descriptor['name'])
dataIsValid=False
for resource in package.resources:
    print(f'Resource: {resource.name} ')
    if resource.tabular is False:               
        continue
    if resource.valid is False:
        print(f'Tableschema: {resource.name} is NOT valid!')
        print(resource.errors)
        continue
    resource.read()
    dataIsValid=True
    print(f'Tableschema: {resource.name} is valid!')
    resourceData=package.get_resource(resource.name) 
    print("create netcdf for " + ace_chromium_isotope_concentration)
    CreateNetCDF(savepath_NetCDF,package,resourceData)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12282289074869938, 'fraction_numerical': 0.009613209681067632, 'mean_word_length': 3.2089481199428844, 'pattern_counts': {'":': 9, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '156424', 'license': 'CC0-1.0', 'n_tokens_mistral': 2792, 'n_tokens_neox': 2716, 'n_words': 429, 'path': 'frictionless-data-packages.ipynb', 'repo_name': 'Swiss-Polar-Institute/frictionless-data-packages', 'size': '8842'}