<jupyter_start><jupyter_text>Demo Script for ERDDAP transformationsTake Mooring Timeseries data and grid to 1hr so parameter(time,depth) - which is 1hr, 1m traditionally for EcoFOCI.  Do not interpolate in depth.  Use ERDDAP as datasourceTake CTD Collection of casts and grid ? (is this useful - not really)**designed with akutan in mind**<jupyter_code>from erddapy import ERDDAP
import pandas as pd
import numpy as np
import xarray as xa
server_url = 'http://akutan.pmel.noaa.gov:8080/erddap'<jupyter_output><empty_output><jupyter_text>Loop through all datasets<jupyter_code>e = ERDDAP(server=server_url)
df = pd.read_csv(e.get_search_url(response='csv', search_for='datasets_Mooring AND final'))
print(f"{df['Dataset ID'].count()} datasets to be looped through")
print("Names:", df['Dataset ID'].values)
from requests.exceptions import HTTPError
#build datasets.xml 
import os

try:
    os.remove("erddap_gridded_from_tabledap.xml")
except OSError:
    pass

f=open("erddap_gridded_from_tabledap.xml", "a+")

output_path = '/home/akutan/bell/in_and_outbox/erddap_generated/gridded_data_from_tabledap/'
for dataset_id in sorted(df['Dataset ID'].values):
    if (not 'itae' in dataset_id) and (not 'gridded' in dataset_id) and (not 'Met' in dataset_id):
        print(f'Working on {dataset_id}')
        #read and import dataset
        #print(dataset_id)
        try:
            d = ERDDAP(server=server_url,
                protocol='tabledap',
                response='csv'
            )
            d.dataset_id=dataset_id
        except HTTPError:
            print('Failed to generate url {}'.format(dataset_id))

        try:
            df_m = d.to_pandas(
                        index_col='time (UTC)',
                        parse_dates=True,
                        skiprows=(1,)  # units information can be dropped.
                        )
            df_m.sort_index(inplace=True)
            df_m.columns = [x[1].split()[0] for x in enumerate(df_m.columns)]

            #-9999 and 1e35 are missing values... but erddap doesn't catch the -9999 yet 
            #and some 1e35 are off a bit
            df_m.loc[(df_m.depth == -9999)] = np.nan # only in pressure field

        except:
            print(f"something failed in data download {dataset_id}")
            pass

        try:
            dfr = df_m.groupby('depth').resample('1H').mean()

            dfr.index.names = ['depth','date']
            xfr = dfr.drop('depth',axis=1).to_xarray()
            xfr['date'] = xfr.date.astype('datetime64[ns]')

            xfr.to_netcdf(dataset_id+'.nc',encoding={'time':{'units':'hours since 1970-01-01'}})        

            upperID = dataset_id.split('_')[-2].upper()
            timestart = str(xfr.date.min().values).replace('00.000000000','00Z')

            #write out xml file
            f.write(f"""
            <dataset type="EDDGridFromNcFiles" datasetID="1hr_gridded_{dataset_id}" active="true">
                <reloadEveryNMinutes>10080</reloadEveryNMinutes>
                <updateEveryNMillis>10000</updateEveryNMillis>
                <fileDir>{output_path}</fileDir>
                <fileNameRegex>{dataset_id}.nc</fileNameRegex>
                <recursive>false</recursive>
                <pathRegex>.*</pathRegex>
                <metadataFrom>last</metadataFrom>
                <matchAxisNDigits>20</matchAxisNDigits>
                <fileTableInMemory>false</fileTableInMemory>
                <accessibleViaFiles>false</accessibleViaFiles>
                <!-- sourceAttributes>
                </sourceAttributes -->
                <addAttributes>
                    <att name="cdm_data_type">Grid</att>
                    <att name="Conventions">COARDS, CF-1.6, ACDD-1.3</att>
                    <att name="infoUrl">https://pmel.noaa.gov/</att>
                    <att name="institution">NOAA/PMEL - EcoFOCI</att>
                    <att name="keywords">gridded, active, available, chemistry, chlorophyll, Chlorophyll_Fluorescence, color, concentration, concentration_of_chlorophyll_in_sea_water, data, date, density, depth, dissolved, dissolved o2, downwelling, downwelling_photosynthetic_photon_radiance_in_sea_water, earth, Earth Science &gt; Oceans &gt; Ocean Chemistry &gt; Chlorophyll, Earth Science &gt; Oceans &gt; Ocean Optics &gt; Photosynthetically Active Radiation, Earth Science &gt; Oceans &gt; Ocean Optics &gt; Radiance, Earth Science &gt; Oceans &gt; Salinity/Density &gt; Salinity, latitude, local, longitude, number, O2, ocean, ocean color, oceans, optical, optical properties, optics, oxygen, Oxygen_Concentration, Oxygen_Saturation, PAR, photon, photosynthetic, photosynthetically, practical, pressure, properties, radiance, radiation, salinity, saturation, science, sea, sea_water_practical_salinity, seawater, serial, Serial_Number, source, statistics, temperature, water</att>
                    <att name="keywords_vocabulary">GCMD Science Keywords</att>
                    <att name="license">[standard]</att>
                    <att name="standard_name_vocabulary">CF Standard Name Table v55</att>
                    <att name="summary">QC'd (final) mooring data from {upperID}.  This dataset has been gridded to 1hr resolution (with pressure left at discrete depths).  It uses only datasets from the associated erddap tabular {dataset_id}.  A depth field exisists for each known instrument depth, even if the parameter wasn't measured at that depth. </att>
                    <att name="title">Gridded Mooring Data from {upperID} - final</att>
                </addAttributes>
                <axisVariable>
                    <sourceName>depth</sourceName>
                    <destinationName>depth</destinationName>
                    <!-- sourceAttributes>
                        <att name="_FillValue" type="double">NaN</att>
                    </sourceAttributes -->
                    <addAttributes>
                        <att name="ioos_category">Location</att>
                        <att name="long_name">Depth</att>
                        <att name="standard_name">depth</att>
                        <att name="units">m</att>
                    </addAttributes>
                </axisVariable>
                <axisVariable>
                    <sourceName>date</sourceName>
                    <destinationName>time</destinationName>
                    <!-- sourceAttributes>
                        <att name="calendar">proleptic_gregorian</att>
                        <att name="units">hours since 1970-01-01 00:00:00</att>
                    </sourceAttributes -->
                    <addAttributes>
                        <att name="ioos_category">Time</att>
                        <att name="long_name">Date</att>
                        <att name="source_name">date</att>
                        <att name="standard_name">time</att>
                        <att name="units">hours since 1970-01-01 00:00:00</att>
                    </addAttributes>
                </axisVariable>
                <dataVariable>
                    <sourceName>latitude</sourceName>
                    <destinationName>latitude</destinationName>
                    <dataType>double</dataType>
                    <!-- sourceAttributes>
                        <att name="_FillValue" type="double">NaN</att>
                    </sourceAttributes -->
                    <addAttributes>
                        <att name="colorBarMaximum" type="double">90.0</att>
                        <att name="colorBarMinimum" type="double">-90.0</att>
                        <att name="ioos_category">Location</att>
                        <att name="long_name">Latitude</att>
                        <att name="standard_name">latitude</att>
                        <att name="units">degrees_north</att>
                    </addAttributes>
                </dataVariable>
                <dataVariable>
                    <sourceName>longitude</sourceName>
                    <destinationName>longitude</destinationName>
                    <dataType>double</dataType>
                    <!-- sourceAttributes>
                        <att name="_FillValue" type="double">NaN</att>
                    </sourceAttributes -->
                    <addAttributes>
                        <att name="colorBarMaximum" type="double">180.0</att>
                        <att name="colorBarMinimum" type="double">-180.0</att>
                        <att name="ioos_category">Location</att>
                        <att name="long_name">Longitude</att>
                        <att name="standard_name">longitude</att>
                        <att name="units">degrees_east</att>
                    </addAttributes>
                </dataVariable>
                <dataVariable>
                    <sourceName>temperature</sourceName>
                    <destinationName>temperature</destinationName>
                    <dataType>double</dataType>
                    <!-- sourceAttributes>
                        <att name="_FillValue" type="double">NaN</att>
                    </sourceAttributes -->
                    <addAttributes>
                       <att name="missing_value" type="float">-9999.0</att>
                       <att name="_FillValue" type="float">1.0e35</att>
                        <att name="epic_key">T_20</att>
                        <att name="long_name">Sea temperature in-situ ITS-90 scale</att>
                        <att name="standard_name">sea_water_temperature</att>
                        <att name="units">degree_C</att>
                        <att name="colorBarMaximum" type="double">20.0</att>
                        <att name="colorBarMinimum" type="double">-2.0</att>
                        <att name="ioos_category">Temperature</att>
                        <att name="standard_name">sea_water_temperature</att>
                    </addAttributes>
                </dataVariable>
                <dataVariable>
                    <sourceName>pressure</sourceName>
                    <destinationName>pressure</destinationName>
                    <dataType>double</dataType>
                    <!-- sourceAttributes>
                        <att name="_FillValue" type="double">NaN</att>
                    </sourceAttributes -->
                    <addAttributes>
                       <att name="missing_value" type="float">-9999.0</att>
                       <att name="_FillValue" type="float">1.0e35</att>
                        <att name="ioos_category">Sea Level</att>
                        <att name="long_name">Sea water pressure, equals 0 at sea-level</att>
                        <att name="standard_name">sea_water_pressure_due_to_sea_water</att>
                        <att name="units">dbar</att>  
                    </addAttributes>
                </dataVariable>
                <dataVariable>
                    <sourceName>salinity</sourceName>
                    <destinationName>salinity</destinationName>
                    <dataType>double</dataType>
                    <!-- sourceAttributes>
                        <att name="_FillValue" type="double">NaN</att>
                    </sourceAttributes -->
                    <addAttributes>
                        <att name="missing_value" type="float">-9999.0</att>
                        <att name="_FillValue" type="float">1.0e35</att>
                        <att name="colorBarMaximum" type="double">37.0</att>
                        <att name="colorBarMinimum" type="double">32.0</att>
                        <att name="ioos_category">Salinity</att>
                        <att name="long_name">Sea Water Practical Salinity</att>
                        <att name="standard_name">sea_water_practical_salinity</att>
                        <att name="units">PSU</att>
                    </addAttributes>
                </dataVariable>
                <dataVariable>
                    <sourceName>PAR</sourceName>
                    <destinationName>PAR</destinationName>
                    <dataType>double</dataType>
                    <!-- sourceAttributes>
                        <att name="_FillValue" type="double">NaN</att>
                    </sourceAttributes -->
                    <addAttributes>
                        <att name="missing_value" type="float">-9999.0</att>
                        <att name="_FillValue" type="float">1.0e35</att>
                        <att name="colorBarMaximum" type="double">70.0</att>
                        <att name="colorBarMinimum" type="double">0.0</att>
                        <att name="ioos_category">Optical Properties</att>
                        <att name="long_name">Downwelling Photosynthetic Photon Radiance In Sea Water</att>
                        <att name="standard_name">downwelling_photosynthetic_photon_radiance_in_sea_water</att>
                    </addAttributes>
                </dataVariable>
                <dataVariable>
                    <sourceName>Chlorophyll_Fluorescence</sourceName>
                    <destinationName>Chlorophyll_Fluorescence</destinationName>
                    <dataType>double</dataType>
                    <!-- sourceAttributes>
                        <att name="_FillValue" type="double">NaN</att>
                    </sourceAttributes -->
                    <addAttributes>
                        <att name="missing_value" type="float">-9999.0</att>
                        <att name="_FillValue" type="float">1.0e35</att>
                        <att name="colorBarMaximum" type="double">30.0</att>
                        <att name="colorBarMinimum" type="double">0.03</att>
                        <att name="colorBarScale">Log</att>
                        <att name="ioos_category">Ocean Color</att>
                        <att name="long_name">Concentration Of Chlorophyll In Sea Water</att>
                        <att name="standard_name">concentration_of_chlorophyll_in_sea_water</att>
                    </addAttributes>
                </dataVariable>
                <dataVariable>
                    <sourceName>Oxygen_Saturation</sourceName>
                    <destinationName>Oxygen_Saturation</destinationName>
                    <dataType>double</dataType>
                    <!-- sourceAttributes>
                        <att name="_FillValue" type="double">NaN</att>
                    </sourceAttributes -->
                    <addAttributes>
                       <att name="missing_value" type="float">-9999.0</att>
                       <att name="_FillValue" type="float">1.0e35</att>
                        <att name="colorBarMaximum" type="double">100.0</att>
                        <att name="colorBarMinimum" type="double">0.0</att>
                        <att name="ioos_category">Dissolved O2</att>
                        <att name="units">percent</att>
                        <att name="standard_name">oxygen_saturation_over_air</att>
                    </addAttributes>
                </dataVariable>
                <dataVariable>
                    <sourceName>Oxygen_Concentration</sourceName>
                    <destinationName>Oxygen_Concentration</destinationName>
                    <dataType>double</dataType>
                    <!-- sourceAttributes>
                        <att name="_FillValue" type="double">NaN</att>
                    </sourceAttributes -->
                    <addAttributes>
                       <att name="missing_value" type="float">-9999.0</att>
                       <att name="_FillValue" type="float">1.0e35</att>
                        <att name="ioos_category">Dissolved O2</att>
                        <att name="units">&#xb5;mole/kg</att>
                        <att name="standard_name">volume_oxygen_in_solution_volume_of_sea_water</att>
                    </addAttributes>
                </dataVariable>
                <dataVariable>
                    <sourceName>Serial_Number</sourceName>
                    <destinationName>Serial_Number</destinationName>
                    <dataType>double</dataType>
                    <!-- sourceAttributes>
                        <att name="_FillValue" type="double">NaN</att>
                    </sourceAttributes -->
                    <addAttributes>
                        <att name="colorBarMaximum" type="double">100.0</att>
                        <att name="colorBarMinimum" type="double">0.0</att>
                        <att name="ioos_category">Statistics</att>
                        <att name="long_name">Serial Number</att>
                    </addAttributes>
                </dataVariable>
            </dataset>
            """)
        except:
            print('Something failed in gridding this set')
print("Done")<jupyter_output>Working on datasets_Mooring_00bs2a_final
Working on datasets_Mooring_00bs2w_final
Working on datasets_Mooring_00bs3w_final
Working on datasets_Mooring_00bs4s_final
Working on datasets_Mooring_00bs4w_final
Working on datasets_Mooring_00bs6a_final
Working on datasets_Mooring_00bsm2a_final
Working on datasets_Mooring_00bsp2s_final
Working on datasets_Mooring_00bsp3s_final
Working on datasets_Mooring_00bsp4s_final
Working on datasets_Mooring_00cb1a_final
Working on datasets_Mooring_00kc1a_final
Working on datasets_Mooring_00kc2a_final
Working on datasets_Mooring_00pa1a_final
Working on datasets_Mooring_01bs2c_final
Working on datasets_Mooring_01bs4a_final
Working on datasets_Mooring_01bs4b_final
Working on datasets_Mooring_01bsm2a_final
Working on datasets_Mooring_01bsp2b_final
Working on datasets_Mooring_01bsp2s_final
Working on datasets_Mooring_01gbm3a_final
Working on datasets_Mooring_01gbm3b_final
Working on datasets_Mooring_02bs2c_final
Working on datasets_Mooring_02bs4a_final
Working[...]
{'fraction_non_alphanumeric': 0.10910927765617107, 'fraction_numerical': 0.01655722063198069, 'mean_word_length': 1.7901659880989664, 'pattern_counts': {'":': 0, '<': 372, '<?xml version=': 0, '>': 372, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '58957', 'license': 'MIT', 'n_tokens_mistral': 5019, 'n_tokens_neox': 4536, 'n_words': 848, 'path': 'EcoFOCI_Moorings/ERDDAP_Automated_Tools/ERDDAPTable2Grid_nc.ipynb', 'repo_name': 'NOAA-PMEL/EcoFOCI_Jupyter_Notebooks', 'size': '17817'}