<jupyter_start><jupyter_text>Process ISCCP files__I need to fix the time format and interpolate to the CAM 1deg grid__  Add paths to run locally or off Cheyenne or Casper (what is the difference between these two again?)<jupyter_code>import sys
# Add common resources folder to path
sys.path.append('/glade/u/home/jonahshaw/Scripts/git_repos/CESM2_analysis')
sys.path.append('/glade/u/home/jonahshaw/Scripts/git_repos/CESM2_analysis/Common/')
# sys.path.append("/home/jonahks/git_repos/netcdf_analysis/Common/")

from imports import (
    pd, np, xr, mpl, plt, sns, os, 
    datetime, sys, crt, gridspec,
    ccrs, metrics, Iterable, cmaps
    )

from functions import (
    masked_average, add_weights, sp_map,
    season_mean, get_dpm, leap_year, share_ylims,
    to_png
    )

from cloud_metric import Cloud_Metric
from collections import deque
%matplotlib inline
from interp_functions import *
# In the CAM output, the ISCCP total is named "CLDTOT_ISCCP"<jupyter_output><empty_output><jupyter_text>Load CAM data for interpolation<jupyter_code>case_dir = '/glade/scratch/jonahshaw/archive/Kay_COSP_2012/'

cam4metric = Cloud_Metric(case_dir)
cam4metric.add_case('cam4_1deg_release_amip',label='CAM4')<jupyter_output>Loading GOCCP data...done.
Loading CALIOP SLFs...Could not load cloudtop CALIOP slfs from caliop_olimpia/ct_slf_olimpia/cloudtop_slfs.nc
Loading CERES-EBAF fluxes...done.
Loading ISCCP cloud total...Processed timeseries directory found for cam4_1deg_release_amip. Will load data as required.
cam4_1deg_release_amip load successfully.<jupyter_text>Plot to trigger variable loading<jupyter_code>cam4metric.plot1D('CLDTOT_ISCCP');<jupyter_output>Could not find variable in GOCCP or CERES-EBAF datasets.<jupyter_text>____<jupyter_code>cam4da = cam4metric.get_case('cam4_1deg_release_amip').case_da<jupyter_output><empty_output><jupyter_text>Load ISCCP data and fix the time format<jupyter_code>ls /glade/work/jenkay/obs/COSP/ISCCP/
isccp_clt = xr.open_dataset('/glade/work/jenkay/obs/COSP/ISCCP/cltisccp_198307-200806.nc',decode_times=False)

isccp_clt['time'].attrs['units'] = isccp_clt['time'].attrs['units'] + '-01' # a little hacky

isccp_clt['time'].attrs['calendar'] = '360_day'

isccp_clt2 = xr.decode_cf(isccp_clt)<jupyter_output><empty_output><jupyter_text>Interpolate to the CAM grid<jupyter_code>isccp_interpolated,_ = interp_like2D(isccp_clt2,cam4da['CLDTOT_ISCCP'])<jupyter_output><empty_output><jupyter_text>Copy over dataset-wide attributes<jupyter_code>attrs = isccp_clt.attrs
# print(attrs['Comments'])

datestamp = datetime.today().strftime("%Y/%m/%d")
attrs['Comments'] = attrs['Comments'] + '. Interpolated to 1.25x1 deg CESM grid and time format updated by Jonah Shaw on %s.' % datestamp
# print(attrs['Comments'])
isccp_interpolated = isccp_interpolated.assign_attrs(isccp_clt.attrs)
isccp_interpolated.attrs<jupyter_output><empty_output><jupyter_text>Write python-ready file to Jonah obs directory<jupyter_code>save_dir = '/glade/work/jonahshaw/obs/ISCCP/'
filename = 'cltisccp_198307-200806.nc'
dest_path = "%s/%s" % (save_dir,filename)
dest_path
isccp_interpolated.to_netcdf(dest_path)<jupyter_output><empty_output><jupyter_text>Streamline with loops<jupyter_code>og_dir = '/glade/work/jenkay/obs/COSP/ISCCP/'
save_dir = '/glade/work/jonahshaw/obs/ISCCP/'
files = os.listdir('/glade/work/jenkay/obs/COSP/ISCCP/')

for i in files:
    filepath = "%s/%s" % (og_dir,i)
    outpath = "%s/%s" % (save_dir,i)
    print(filepath)
    
    ds = xr.open_dataset(filepath,decode_times=False)

    ds['time'].attrs['units'] = ds['time'].attrs['units'] + '-01' # a little hacky

    ds['time'].attrs['calendar'] = '360_day'

    ds2 = xr.decode_cf(ds)
    
    ds_interpd,_ = interp_like2D(ds2,cam4da['CLDTOT_ISCCP'])  
    
    # Copy attributes over
    attrs = ds.attrs
    datestamp = datetime.today().strftime("%Y/%m/%d")
    attrs['Comments'] = attrs['Comments'] + '. Interpolated to 1.25x1 deg CESM grid and time format updated by <NAME> on %s.' % datestamp
    # print(attrs['Comments'])
    ds_interpd = ds_interpd.assign_attrs(attrs)
    
    ds_interpd.to_netcdf(outpath)
    
#     break<jupyter_output>/glade/work/jenkay/obs/COSP/ISCCP//ctpisccp_198307-200806.nc
/glade/work/jenkay/obs/COSP/ISCCP//cttisccp_unweighted_198307-200806.nc
/glade/work/jenkay/obs/COSP/ISCCP//cttisccp_198307-200806.nc
/glade/work/jenkay/obs/COSP/ISCCP//albisccp_unweighted_198307-200806.nc
/glade/work/jenkay/obs/COSP/ISCCP//albisccp_198307-200806.nc
/glade/work/jenkay/obs/COSP/ISCCP//ctpisccp_unweighted_198307-200806.nc
/glade/work/jenkay/obs/COSP/ISCCP//clisccp_198307-200806.nc
/glade/work/jenkay/obs/COSP/ISCCP//cltisccp_198307-200806.nc<jupyter_text>Old Work  Find the files<jupyter_code>testy = isccp_interpolated.groupby('time.month').mean('time')
out = testy.mean('month')
isccpobs = out['cltisccp'] #.plot()
cam4isccp = cam4da['CLDTOT_ISCCP'].groupby('time.month').mean('time').mean('month')#.plot()
(isccpobs - cam4isccp).plot()
os.chdir('/glade/work/jenkay/obs/COSP')
ls
isccp_clt = xr.open_dataset('ISCCP/cltisccp_198307-200806.nc',decode_times=False)

isccp_clt['time'].attrs['units'] = isccp_clt['time'].attrs['units'] + '-01' # a little hacky

isccp_clt['time'].attrs['calendar'] = '360_day'

isccp_clt2 = xr.decode_cf(isccp_clt)
isccp_clt2
isccp_clt['time'].attrs['calendar']<jupyter_output><empty_output><jupyter_text>Time fix:https://rabernat.github.io/research_computing_2018/xarray-tips-and-tricks.html<jupyter_code>def fix_calendar(ds, timevar='T'):
    if ds[timevar].attrs['calendar'] == '360':
        ds[timevar].attrs['calendar'] = '360_day'
    return ds

isccp_clt = fix_calendar(isccp_clt,timevar='time')
isccp_clt = xr.decode_cf(isccp_clt)
isccp_clt
cftime.num2date(0, 'months since 1960-01-01', '360_day')
cftime.num2date(0, 'months since 1983-06', '360_day')
isccp_clt.attrs
isccp_clt = isccp_clt.assign_attrs({'calendar': '360_day'})
xr.decode_cf(isccp_clt,decode_times='360')
isccp_clt['time'].attrs['calendar'] = '360_day'
xr.decode_cf(isccp_clt)
isccp_clt['time'].attrs['units'] = 'months since 1983-06-01'
isccp_clt['time'].attrs['units']
xr.decode_cf(isccp_clt)
isccp_clt = xr.open_dataset('ISCCP/cltisccp_198307-200806.nc',decode_times=True)
isccp_clt = xr.open_dataset('ISCCP/cltisccp_198307-200806.nc',decode_times=False)
t150 = isccp_clt['cltisccp'].isel(time=150)
t150.plot()
attrs = {"units": "hours since 2000-01-01"}

ds = xr.Dataset({"time": ("time", [0, 1, 2, 3], attrs)})

out = xr.decode_cf(ds)
out<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14494537621172487, 'fraction_numerical': 0.044006770272349595, 'mean_word_length': 6.135016465422613, 'pattern_counts': {'":': 2, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '179606', 'license': 'MIT', 'n_tokens_mistral': 2888, 'n_tokens_neox': 2657, 'n_words': 377, 'path': 'CESM2_COSP/isccp_proc.ipynb', 'repo_name': 'jshaw35/CESM2_analysis', 'size': '6503'}