<jupyter_start><jupyter_text>Retrieval of Bottom Temps from ERDDAP Dataset for distribution<jupyter_code>import datetime
from erddapy import ERDDAP
import pandas as pd
pd.options.mode.chained_assignment = None
import numpy as np

import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import cmocean

#using xarray for bathymetry data read
import xarray as xa
#search and boundary parameters

server_url = 'http://akutan.pmel.noaa.gov:8080/erddap/'

deployments = []

constraints = {
    'time>=': '2010-01-01T00:00:00Z',
    'time<=': datetime.datetime.utcnow().strftime('%Y-%m-%dT00:00:00Z'),
    'depth>=': 55,
}

variables = ['latitude, longitude, temperature, time, depth, timeseries_id']

e = ERDDAP(server=server_url)
df_1 = pd.read_csv(e.get_search_url(response='csv', search_for='Mooring bs2 -gridded final -CTD'))
e = ERDDAP(server=server_url)
df_2 = pd.read_csv(e.get_search_url(response='csv', search_for='Mooring bsm2 -gridded final -CTD -Meteorological'))
df = pd.concat([df_1,df_2])


print(sorted(df['Dataset ID'].values))

from requests.exceptions import HTTPError

dfs = {}
for dataset_id in sorted(df['Dataset ID'].values):
    print(dataset_id)
    try:
        d = ERDDAP(server=server_url,
            protocol='tabledap',
            response='csv'
        )
        d.dataset_id=dataset_id
        d.variables = variables
        d.constraints = constraints
    except HTTPError:
        print('Failed to generate url {}'.format(dataset_id))

    try:
        df_m = d.to_pandas(
                    index_col='time (UTC)',
                    parse_dates=True,
                    skiprows=(1,)  # units information can be dropped.
                    )
        df_m.sort_index(inplace=True)
        df_m.columns = [x[1].split()[0] for x in enumerate(df_m.columns)]
        
        #-9999 and 1e35 are missing values... but erddap doesn't catch the -9999 yet 
        #and some 1e35 are off a bit
        df_m['depth'][(df_m.depth == -9999)] = np.nan # only in pressure field
        dfs.update({dataset_id:df_m})
    except:
        pass

merged_dataset = pd.DataFrame()
for dataset in dfs:
    dfg = dfs[dataset].dropna().groupby('timeseries_id')
    print(list(dfs[dataset].dropna().groupby('timeseries_id').groups.keys()))
    name =  list(dfs[dataset].dropna().groupby('timeseries_id').groups.keys())[-1]
    if (not 'wcp' in name) and (not 'sg' in name):
        print(name)
        merged_dataset = pd.concat([merged_dataset,pd.DataFrame(dfs[dataset].dropna().groupby('timeseries_id').get_group(name))])
    else:
        name =  list(dfs[dataset].dropna().groupby('timeseries_id').groups.keys())[-2]
        merged_dataset = pd.concat([merged_dataset,pd.DataFrame(dfs[dataset].dropna().groupby('timeseries_id').get_group(name))])

#append 2019-2021
df_new = pd.read_csv('/Users/bell/Downloads/19bs2c_s39_0060m.csv',parse_dates=True,index_col='time')
merged_dataset.temperature.plot()
merged_dataset.resample('1D').mean().temperature.plot()
df_new.resample('1D').mean().temperature.plot()
merged_dataset.to_csv('M5_BottomTemp.csv')
merged_dataset.resample('1H').mean().to_csv('M5_BottomTemp_hourly.csv')
tmp = merged_dataset.resample('1H').mean()
tmp[tmp.index.hour == 12].drop(['depth','latitude','longitude'],axis=1).to_csv('10ckp2a_BottomTemp_12utc.csv')
merged_dataset.index = merged_dataset.index.tz_localize(None)
pd.concat([merged_dataset.resample('1D').mean().temperature,df_new.resample('1D').mean().temperature]).to_csv('M2_BottomTemp_daily.csv')

df_new2=pd.read_csv('M2_BottomTemp_daily.csv',parse_dates=True,index_col='Unnamed: 0')
df_new2.resample('1w').mean().temperature.plot(figsize=(16,4))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15050258082042922, 'fraction_numerical': 0.02635153490899212, 'mean_word_length': 4.237553342816501, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '231346', 'license': 'MIT', 'n_tokens_mistral': 1353, 'n_tokens_neox': 1283, 'n_words': 213, 'path': 'EcoFOCI_Moorings/BottomTemp_ERDDAP_Retrieval.ipynb', 'repo_name': 'shaunwbell/ipythonnb', 'size': '3681'}