<jupyter_start><jupyter_text>Getting 1pct nbp / nep / fgco2 / tas data from esgf<jupyter_code>def get_annual_CMIP6_data_esgf(activity, table, variable, experiment, institution, source, member):
    # eg activity='CMIP', table='Amon', variable='tas', experiment='historical', institution="NCAR", source="CESM2", member="r10i1p1f1"
    
    try:
    
        result = esgf_search(activity_id=activity, table_id=table, variable_id=variable, experiment_id=experiment,institution_id=institution, source_id=source, member_id=member)

        if not result:
            print('No results for this request')
            return None

        # select results with only the latest datestamp:
        latest = sorted([x.split('/')[15] for x in result])[-1]
        result = [x for x in result if x.split('/')[15]==latest]

        # remove duplicate results
        result_1 = []

        for item in result:
            if item.split('/')[-1] in [x.split('/')[-1] for x in result_1]:
                continue
            else:
                result_1 += [item]

        ds = xr.open_mfdataset(result_1, combine='by_coords')

        dims = list(ds[variable].dims)
        dims.remove('time')

        lat_dim_name = [x for x in dims if 'lat' in x][0]

        weights = np.cos(np.deg2rad(ds[variable][lat_dim_name]))
        weights.name = "weights"

        ta_timeseries = ds[variable].weighted(weights).mean(dims)

        return ta_timeseries.groupby('time.year').mean('time').to_pandas().rename(institution+'_'+source+'_'+member+'_'+variable+'_'+experiment)
    
    except:
        print('retrieval failed')
        return None
pd.Series(['v20190306','v20200429']).sort_values()
search_results = esgf_search(table_id='Lmon', variable_id='nbp',experiment_id='1pctCO2-bgc')
search_df = pd.DataFrame([x.split('/')[-10:-1] for x in search_results],columns=['activity','institution','source','experiment','member','table','variable','grid','date'])
chosen_indices = []
for i,row in search_df.iterrows():
    duplicates = search_df.query("activity==\'"+row.loc['activity']+"\' & table==\'"+row.loc['table']+"\' & variable==\'"+row.loc['variable']+"\' & experiment==\'"+row.loc['experiment']+"\' & institution==\'"+row.loc['institution']+"\' & source==\'"+row.loc['source']+"\' & member==\'"+row.loc['member']+"\' & grid==\'"+row.loc['grid']+"\'")
    duplicate_dates = duplicates.loc[:,'date'].sort_values()
    latest_date = duplicate_dates.iloc[-1]
    if row.loc['date'] == latest_date:
        chosen_indices += [i]
    else:
        continue
search_df = search_df.loc[chosen_indices].drop_duplicates()
experiments = ['1pctCO2','1pctCO2-bgc','1pctCO2-rad']
variables = ['nep','nbp','fgco2','fco2nat','tas']
tables = dict(zip(variables,['Emon','Lmon','Omon','Amon','Amon']))
search_dfs = []
for experiment in experiments:
    for variable in variables:
        search_results = esgf_search(table_id=tables[variable], variable_id=variable,experiment_id=experiment)
        search_df = pd.DataFrame([x.split('/')[-10:-1] for x in search_results],columns=['activity','institution','source','experiment','member','table','variable','grid','date'])
        chosen_indices = []
        for i,row in search_df.iterrows():
            duplicates = search_df.query("activity==\'"+row.loc['activity']+"\' & table==\'"+row.loc['table']+"\' & variable==\'"+row.loc['variable']+"\' & experiment==\'"+row.loc['experiment']+"\' & institution==\'"+row.loc['institution']+"\' & source==\'"+row.loc['source']+"\' & member==\'"+row.loc['member']+"\' & grid==\'"+row.loc['grid']+"\'")
            duplicate_dates = duplicates.loc[:,'date'].sort_values()
            latest_date = duplicate_dates.iloc[-1]
            if row.loc['date'] == latest_date:
                chosen_indices += [i]
            else:
                continue
        search_df = search_df.loc[chosen_indices].drop_duplicates()
        search_dfs += [search_df]
search_df = pd.concat(search_dfs,axis=0)
# filter search results by date
chosen_models = search_df.loc[(search_df.experiment.isin(['1pctCO2-bgc','1pctCO2-rad']))&(search_df.variable.isin(['fco2nat','fgco2']))].source.unique()
search_df.loc[(search_df.experiment.isin(['1pctCO2-bgc','1pctCO2-rad']))].source.unique()
# get area weights for each model
def get_model_area(source,area_var='areacella'):
    
    try:
    
        files_area = esgf_search(variable_id=area_var, source_id=source)

        ds_area = xr.open_dataset(files_area[0])

        return ds_area
    
    except:
        print('retrieval failed')
        return None
model_areacella = {}
for model in chosen_models:
    print('getting '+model+' areacella')
    model_areacella[model] = get_model_areacella(model)
model_areacello = {}
for model in chosen_models:
    print('getting '+model+' areacello')
    model_areacello[model] = get_model_area(model,'areacello')
## switches:
## if gr available, get that
hist_info = gs_stores.loc[(gs_stores.experiment_id=='historical')&(gs_stores.variable_id=='tas')&(gs_stores.table_id=='Amon')]
hist_data = []

for index,row in hist_info.iterrows():
    hist_data += [get_annual_CMIP6_data_gstore(row.loc['activity_id'], row.loc['table_id'], row.loc['variable_id'], row.loc['experiment_id'], row.loc['institution_id'], row.loc['source_id'], row.loc['member_id'])]
pd.concat(hist_data,axis=1).to_csv('./cmip6_data/historical_tas.csv')
def get_annual_CMIP6_data_gstore_nbp(activity, table, variable, experiment, institution, source, member):
    # eg activity='CMIP', table='Amon', variable='tas', experiment='historical', institution="NCAR", source="CESM2", member="r10i1p1f1"
    
    area_query = gs_stores.query("variable_id=='areacella' & source_id==\'"+source+"\'")
    
    if area_query.empty:
        files_area = esgf_search(variable_id='areacella', activity_id=activity, institution_id=institution, source_id=source)
        if not files_area:
            print('No areacella for this request')
            return None
        ds_area = xr.open_dataset(files_area[0])
    else:
        ds_area = xr.open_zarr(gcs.get_mapper(area_query.zstore.values[0]), consolidated=True)
    
    query = gs_stores.query("activity_id==\'"+activity+"\' & table_id==\'"+table+"\' & variable_id==\'"+variable+"\' & experiment_id==\'"+experiment+"\' & institution_id==\'"+institution+"\' & source_id==\'"+source+"\' & member_id==\'"+member+"\'")
    
    if query.empty:
        print('No results for this request')
        return None

    # create a mutable-mapping-style interface to the store
    mapper = gcs.get_mapper(query.zstore.values[0])

    # open it using xarray and zarr
    ds = xr.open_zarr(mapper, consolidated=True)
    
    if source=='E3SM-1-1' and variable=='tas' and experiment=='piControl':
        ds = xr.open_mfdataset(esgf_search(activity_id=activity, table_id=table, variable_id=variable, experiment_id=experiment, institution_id=institution, source_id=source, member_id=member)[7:],combine='by_coords')
    
    coords = list(ds[variable].coords.keys())
    if 'latitude' in coords:
        dims = ['latitude','longitude']
        _dims = ['lat','lon']
    else:
        dims = ['lat','lon']
        _dims = ['latitude','longitude']
        
    if not dims[0] in list(ds_area['areacella'].coords.keys()):
        ds_area = ds_area.rename(dict(zip(_dims,dims)))
    
    total_area = ds_area.areacella.sum(dim=dims)
    ta_timeseries = (ds[variable] * ds_area.areacella).sum(dim=dims)
    
    return ta_timeseries.groupby('time.year').mean('time').to_pandas().rename(institution+'_'+source+'_'+member+'_'+experiment)
nbp_cmip6=gs_stores.loc[(gs_stores.variable_id=='nbp')&(gs_stores.table_id=='Lmon')]

# nbp_data = []

# for index,row in nbp_cmip6.iterrows():
#     nbp_data += [get_annual_CMIP6_data_gstore_nbp(row.loc['activity_id'], row.loc['table_id'], row.loc['variable_id'], row.loc['experiment_id'], row.loc['institution_id'], row.loc['source_id'], row.loc['member_id'])]
pd.concat(nbp_data,axis=1).to_csv('./cmip6_data/nbp.csv')
def get_annual_CMIP6_data_gstore_fgco2(activity, table, variable, experiment, institution, source, member):
    # eg activity='CMIP', table='Amon', variable='tas', experiment='historical', institution="NCAR", source="CESM2", member="r10i1p1f1"
    
    area_query = gs_stores.query("variable_id=='areacello' & source_id==\'"+source+"\'")

    if area_query.empty:
        files_area = esgf_search(variable_id='areacello', activity_id=activity, institution_id=institution, source_id=source)
        if not files_area:
            print('No areacello for this request')
            return None
        ds_area = xr.open_dataset(files_area[0])
    else:
        ds_area = xr.open_zarr(gcs.get_mapper(area_query.zstore.values[0]), consolidated=True)

    query = gs_stores.query("activity_id==\'"+activity+"\' & table_id==\'"+table+"\' & variable_id==\'"+variable+"\' & experiment_id==\'"+experiment+"\' & institution_id==\'"+institution+"\' & source_id==\'"+source+"\' & member_id==\'"+member+"\'")

    if query.empty:
        print('No results for this request')
        return None

    # create a mutable-mapping-style interface to the store
    mapper = gcs.get_mapper(query.zstore.values[0])

    # open it using xarray and zarr
    ds = xr.open_zarr(mapper, consolidated=True)

    if source=='E3SM-1-1' and variable=='tas' and experiment=='piControl':
        ds = xr.open_mfdataset(esgf_search(activity_id=activity, table_id=table, variable_id=variable, experiment_id=experiment, institution_id=institution, source_id=source, member_id=member)[7:],combine='by_coords')

    dims = list(ds[variable].dims)
    dims.remove('time')
#     total_area = ds_area.areacello.sum(dim=dims)

    ta_timeseries = (ds[variable] * ds_area.areacello).sum(dim=dims)

    return ta_timeseries.groupby('time.year').mean('time').to_pandas().rename(institution+'_'+source+'_'+member+'_'+experiment)
gs_stores['isme'] = gs_stores['ism']+'_'+gs_stores['experiment_id']
fgco2_cmip6=gs_stores.loc[(gs_stores.variable_id=='fgco2')&(gs_stores.table_id=='Omon')&(gs_stores.grid_label=='gn')&(gs_stores.isme.isin(nbp_cmip6.isme))]

fgco2_data = []

for index,row in fgco2_cmip6.loc[237786:].iterrows():
    fgco2_data += [get_annual_CMIP6_data_gstore_fgco2(row.loc['activity_id'], row.loc['table_id'], row.loc['variable_id'], row.loc['experiment_id'], row.loc['institution_id'], row.loc['source_id'], row.loc['member_id'])]
activity, table, variable, experiment, institution, source, member = row.loc['activity_id'], row.loc['table_id'], row.loc['variable_id'], row.loc['experiment_id'], row.loc['institution_id'], row.loc['source_id'], row.loc['member_id']

area_query = gs_stores.query("variable_id=='areacello' & source_id==\'"+source+"\'")

if area_query.empty:
    files_area = esgf_search(variable_id='areacello', activity_id=activity, institution_id=institution, source_id=source)
    if not files_area:
        print('No areacello for this request')
#         return None
    ds_area = xr.open_dataset(files_area[0])
else:
    ds_area = xr.open_zarr(gcs.get_mapper(area_query.zstore.values[0]), consolidated=True)

query = gs_stores.query("activity_id==\'"+activity+"\' & table_id==\'"+table+"\' & variable_id==\'"+variable+"\' & experiment_id==\'"+experiment+"\' & institution_id==\'"+institution+"\' & source_id==\'"+source+"\' & member_id==\'"+member+"\'")

if query.empty:
    print('No results for this request')
#     return None

# create a mutable-mapping-style interface to the store
mapper = gcs.get_mapper(query.zstore.values[0])

# open it using xarray and zarr
ds = xr.open_zarr(mapper, consolidated=True)

if source=='E3SM-1-1' and variable=='tas' and experiment=='piControl':
    ds = xr.open_mfdataset(esgf_search(activity_id=activity, table_id=table, variable_id=variable, experiment_id=experiment, institution_id=institution, source_id=source, member_id=member)[7:],combine='by_coords')

dims = list(ds[variable].dims)
dims.remove('time')

total_area = ds_area.areacello.sum(dim=dims)
ta_timeseries = (ds[variable] * ds_area.areacello).sum(dim=dims)
pd.concat(fgco2_data,axis=1).to_csv('./cmip6_data/fgco2.csv')
abrupt_4x_ism = gs_stores.loc[(gs_stores.experiment_id=='abrupt-4xCO2')&(gs_stores.variable_id.isin(['tas','rlut','rsdt','rsut']))]
abrupt_4x_ism = list(set([x for x in abrupt_4x_ism.ism if abrupt_4x_ism.loc[abrupt_4x_ism.ism==x].shape[0]>=4]))
onepct_ism = gs_stores.loc[(gs_stores.experiment_id=='1pctCO2')&(gs_stores.variable_id.isin(['tas','rlut','rsdt','rsut']))]
onepct_ism = list(set([x for x in onepct_ism.ism if onepct_ism.loc[onepct_ism.ism==x].shape[0]>=4]))
piControl_ism = gs_stores.loc[(gs_stores.experiment_id=='piControl')&(gs_stores.variable_id=='tas')]
piControl_ism = list(set(piControl_ism.ism))
areacella_s_gs = [x.split('_')[1] for x in list(set(gs_stores.loc[(gs_stores.variable_id=='areacella')].ism))]
areacella_list = esgf_search(activity_id='CMIP', variable_id='areacella')
areacella_list_nodupl = []

for item in areacella_list:
    if item.split('/')[-1] in [x.split('/')[-1] for x in areacella_list_nodupl]:
        continue
    else:
        areacella_list_nodupl += [item]
    
areacella_ism_list = list(set([x.split('/')[8]+'_'+x.split('/')[9]+'_'+x.split('/')[11] for x in areacella_list_nodupl]))
areacella_s_esgf = list(set([x.split('_')[1] for x in areacella_ism_list]))
areacella_s_all = list(set(areacella_s_gs).union(areacella_s_esgf))
abrupt_4x_ism_areacella_exist = [x for x in abrupt_4x_ism if x.split('_')[1] in areacella_s_all]
onepct_ism_areacella_exist = [x for x in onepct_ism if x.split('_')[1] in areacella_s_all]
piControl_ism_areacella_exist = [x for x in piControl_ism if x.split('_')[1] in areacella_s_all]
def get_cmip6_data_gs(ism,var,exp):
    print('getting '+ism+' '+var)
    ism_split = ism.split('_')
    _out = get_annual_CMIP6_data_gstore('CMIP', 'Amon', var, exp, ism_split[0], ism_split[1], ism_split[2])
    print('got '+ism)
    return _out
onepct_tas_df_list = []
for ism in onepct_ism_areacella_exist:
    onepct_tas_df_list += [get_cmip6_data_gs(ism,'tas','1pctCO2')]

onepct_rlut_df_list = []
for ism in onepct_ism_areacella_exist:
    onepct_rlut_df_list += [get_cmip6_data_gs(ism,'rlut','1pctCO2')]
    
onepct_rsut_df_list = []
for ism in onepct_ism_areacella_exist:
    onepct_rsut_df_list += [get_cmip6_data_gs(ism,'rsut','1pctCO2')]
    
onepct_rsdt_df_list = []
for ism in onepct_ism_areacella_exist:
    onepct_rsdt_df_list += [get_cmip6_data_gs(ism,'rsdt','1pctCO2')]
def get_annual_CMIP6_data_gstore_co2mass(activity, table, variable, experiment, institution, source, member):
    # eg activity='CMIP', table='Amon', variable='tas', experiment='historical', institution="NCAR", source="CESM2", member="r10i1p1f1"
    
    query = gs_stores.query("activity_id==\'"+activity+"\' & table_id==\'"+table+"\' & variable_id==\'"+variable+"\' & experiment_id==\'"+experiment+"\' & institution_id==\'"+institution+"\' & source_id==\'"+source+"\' & member_id==\'"+member+"\'")
    
    if query.empty:
        print('No results for this request')
        return None

    # create a mutable-mapping-style interface to the store
    mapper = gcs.get_mapper(query.zstore.values[0])

    # open it using xarray and zarr
    ds = xr.open_zarr(mapper, consolidated=True)
    
    return ds[variable].groupby('time.year').mean('time').to_pandas().rename(institution+'_'+source+'_'+member)

onepct_co2mass_ism = gs_stores.loc[(gs_stores.experiment_id=='1pctCO2')&(gs_stores.variable_id=='co2mass')].ism

def get_cmip6_data_gs_co2mass(ism,var,exp):
    print('getting '+ism+' '+var)
    ism_split = ism.split('_')
    _out = get_annual_CMIP6_data_gstore_co2mass('CMIP', 'Amon', var, exp, ism_split[0], ism_split[1], ism_split[2])
    print('got '+ism)
    return _out

onepct_co2mass_df_list = []
for ism in onepct_co2mass_ism:
    onepct_co2mass_df_list += [get_cmip6_data_gs_co2mass(ism,'co2mass','1pctCO2')]
    
onepct_co2_esgf_results = esgf_search(activity_id='CMIP', table_id='Amon', variable_id='co2mass', experiment_id='1pctCO2')
onepct_co2_esgf_ism = [x.split('/')[8]+'_'+x.split('/')[9]+'_'+x.split('/')[11] for x in onepct_co2_esgf_results]
onepct_co2_esgf_ism = [x for x in onepct_co2_esgf_ism if not x in list(onepct_co2mass_ism)]

def get_annual_CMIP6_data_esgf_co2mass(activity, table, variable, experiment, institution, source, member):
    # eg activity='CMIP', table='Amon', variable='tas', experiment='historical', institution="NCAR", source="CESM2", member="r10i1p1f1"
    
    result = esgf_search(activity_id=activity, table_id=table, variable_id=variable, experiment_id=experiment,institution_id=institution, source_id=source, member_id=member)
    
    if not result:
        print('No results for this request')
        return None
    
    # select results with only the latest datestamp:
    latest = sorted([x.split('/')[15] for x in result])[-1]
    result = [x for x in result if x.split('/')[15]==latest]
    
    # remove duplicate results
    result_1 = []

    for item in result:
        if item.split('/')[-1] in [x.split('/')[-1] for x in result_1]:
            continue
        else:
            result_1 += [item]

    ds = xr.open_mfdataset(result_1, combine='by_coords')
    
    return ds[variable].groupby('time.year').mean('time').to_pandas().rename(institution+'_'+source+'_'+member)

for x in onepct_co2_esgf_ism:
    onepct_co2mass_df_list += [get_annual_CMIP6_data_esgf_co2mass('CMIP','Amon','co2mass','1pctCO2',x.split('_')[0],x.split('_')[1],x.split('_')[2])]
def get_annual_CMIP6_data_gstore_co2(activity, table, variable, experiment, institution, source, member):
    # eg activity='CMIP', table='Amon', variable='tas', experiment='historical', institution="NCAR", source="CESM2", member="r10i1p1f1"
    
    query = gs_stores.query("activity_id==\'"+activity+"\' & table_id==\'"+table+"\' & variable_id==\'"+variable+"\' & experiment_id==\'"+experiment+"\' & institution_id==\'"+institution+"\' & source_id==\'"+source+"\' & member_id==\'"+member+"\'")
    
    if query.empty:
        print('No results for this request')
        return None

    # create a mutable-mapping-style interface to the store
    mapper = gcs.get_mapper(query.zstore.values[0])

    # open it using xarray and zarr
    ds = xr.open_zarr(mapper, consolidated=True)
    
    area_query = gs_stores.query("variable_id=='areacella' & source_id==\'"+source+"\'")
    
    if area_query.empty:
        files_area = esgf_search(variable_id='areacella', activity_id=activity, institution_id=institution, source_id=source)
        if not files_area:
            print('No areacella for this request')
            return None
        ds_area = xr.open_dataset(files_area[0])
    else:
        ds_area = xr.open_zarr(gcs.get_mapper(area_query.zstore.values[0]), consolidated=True)
    
    coords = list(ds[variable].coords.keys())
    if 'lat' in coords:
        dims = ['lat','lon']
    else:
        dims = ['latitude','longitude']
    
    total_area = ds_area.areacella.sum(dim=dims)
    ta_timeseries = (ds[variable].isel(plev=0) * ds_area.areacella).sum(dim=dims) / total_area
    
    return ta_timeseries.groupby('time.year').mean('time').to_pandas().rename(institution+'_'+source+'_'+member)

onepct_co2_ism = gs_stores.loc[(gs_stores.experiment_id=='1pctCO2')&(gs_stores.variable_id=='co2')].ism

def get_cmip6_data_gs_co2(ism,var,exp):
    print('getting '+ism+' '+var)
    ism_split = ism.split('_')
    _out = get_annual_CMIP6_data_gstore_co2('CMIP', 'Amon', var, exp, ism_split[0], ism_split[1], ism_split[2])
    print('got '+ism)
    return _out

onepct_co2_ism_areacella_exist = [x for x in onepct_co2_ism if x.split('_')[1] in areacella_s_all]

onepct_co2_df_list = []
for ism in onepct_co2_ism_areacella_exist:
    onepct_co2_df_list += [get_cmip6_data_gs_co2(ism,'co2','1pctCO2')]
def get_annual_CMIP6_data_esgf_co2(activity, table, variable, experiment, institution, source, member):
    # eg activity='CMIP', table='Amon', variable='tas', experiment='historical', institution="NCAR", source="CESM2", member="r10i1p1f1"
    
    result = esgf_search(activity_id=activity, table_id=table, variable_id=variable, experiment_id=experiment,institution_id=institution, source_id=source, member_id=member)
    
    if not result:
        print('No results for this request')
        return None
    
    # select results with only the latest datestamp:
    latest = sorted([x.split('/')[15] for x in result])[-1]
    result = [x for x in result if x.split('/')[15]==latest]
    
    # remove duplicate results
    result_1 = []

    for item in result:
        if item.split('/')[-1] in [x.split('/')[-1] for x in result_1]:
            continue
        else:
            result_1 += [item]

    ds = xr.open_mfdataset(result_1, combine='by_coords')
    
    files_area = esgf_search(variable_id='areacella', activity_id=activity, institution_id=institution, source_id=source)
    
    if not files_area:
        print('No areacella for this request')
        return None
    
    ds_area = xr.open_dataset(files_area[0])
    
    coords = list(ds[variable].coords.keys())
    if 'lat' in coords:
        dims = ['lat','lon']
    else:
        dims = ['latitude','longitude']
    
    total_area = ds_area.areacella.sum(dim=dims)
    ta_timeseries = (ds[variable].isel(plev=0) * ds_area.areacella).sum(dim=dims) / total_area
    
    return ta_timeseries.groupby('time.year').mean('time').to_pandas().rename(institution+'_'+source+'_'+member)
onepct_co2_esgf_results = esgf_search(activity_id='CMIP', table_id='Amon', variable_id='co2', experiment_id='1pctCO2')
onepct_co2_esgf_ism = [x.split('/')[8]+'_'+x.split('/')[9]+'_'+x.split('/')[11] for x in onepct_co2_esgf_results]
for ism in [x for x in set(onepct_co2_esgf_ism) if not x in onepct_co2_ism_areacella_exist]:
    print('getting '+ism)
    onepct_co2_df_list += [get_annual_CMIP6_data_esgf_co2('CMIP', 'Amon', 'co2', '1pctCO2', ism.split('_')[0], ism.split('_')[1], ism.split('_')[2])]
pd.concat(onepct_co2_df_list,axis=1).to_csv('./cmip6_data/onepct_co2.csv')
# piControl_df_list = []
for ism in piControl_ism_areacella_exist:
    piControl_df_list += [get_cmip6_data_gs(ism,'tas','piControl')]
ESM3_picontrol = xr.open_mfdataset(esgf_search(activity_id='CMIP', table_id='Amon', variable_id='tas', experiment_id='piControl', institution_id='E3SM-Project', source_id='E3SM-1-0', member_id='r1i1p1f1'),combine='by_coords')
# ds_area = xr.open_dataset(files_area[0])

# coords = list(ds[variable].coords.keys())
# if 'lat' in coords:
#     dims = ['lat','lon']
# else:
#     dims = ['latitude','longitude']

# total_area = ds_area.areacella.sum(dim=dims)
# ta_timeseries = (ds[variable].isel(plev=0) * ds_area.areacella).sum(dim=dims) / total_area

# return ta_timeseries.groupby('time.year').mean('time').to_pandas().rename(institution+'_'+source+'_'+member)

ESM3_area = xr.open_dataset(esgf_search(activity_id='CMIP', variable_id='areacella',source_id='E3SM-1-0')[-1])
dims=['lat','lon']
total_area = ESM3_area.areacella.sum(dim=dims)
ta_timeseries = (ESM3_picontrol['tas'] * ESM3_area.areacella).sum(dim=dims) / total_area
ESM3_picontrol_tas = ta_timeseries.groupby('time.year').mean('time').to_pandas().rename('E3SM-Project_E3SM-1-0_r1i1p1f1')
ESM3_picontrol_tas.plot()
piControl_rlut_df_list = []
for ism in piControl_ism_areacella_exist:
    piControl_rlut_df_list += [get_cmip6_data_gs(ism,'rlut','piControl')]
piControl_rsut_df_list = []
for ism in piControl_ism_areacella_exist:
    piControl_rsut_df_list += [get_cmip6_data_gs(ism,'rsut','piControl')]
piControl_rsdt_df_list = []
for ism in piControl_ism_areacella_exist:
    piControl_rsdt_df_list += [get_cmip6_data_gs(ism,'rsdt','piControl')]
abrutp4x_tas_df_list = []
for ism in abrupt_4x_ism_areacella_exist:
    abrutp4x_tas_df_list += [get_cmip6_data_gs(ism,'tas','abrupt-4xCO2')]
abrutp4x_rlut_df_list = []
for ism in abrupt_4x_ism_areacella_exist:
    abrutp4x_rlut_df_list += [get_cmip6_data_gs(ism,'rlut','abrupt-4xCO2')]
abrutp4x_rsut_df_list = []
for ism in abrupt_4x_ism_areacella_exist:
    abrutp4x_rsut_df_list += [get_cmip6_data_gs(ism,'rsut','abrupt-4xCO2')]
abrutp4x_rsdt_df_list = []
for ism in abrupt_4x_ism_areacella_exist:
    abrutp4x_rsdt_df_list += [get_cmip6_data_gs(ism,'rsdt','abrupt-4xCO2')]
abrutp4x_rsdt_df = pd.concat(abrutp4x_rsdt_df_list,axis=1)
abrutp4x_rsut_df = pd.concat(abrutp4x_rsut_df_list,axis=1)
abrutp4x_rlut_df = pd.concat(abrutp4x_rlut_df_list,axis=1)
abrutp4x_tas_df = pd.concat(abrutp4x_tas_df_list,axis=1)
piControl_tas_df = pd.concat(piControl_df_list,axis=1)
piControl_rsdt_df = pd.concat(piControl_rsdt_df_list,axis=1)
piControl_rsut_df = pd.concat(piControl_rsut_df_list,axis=1)
piControl_rlut_df = pd.concat(piControl_rlut_df_list,axis=1)
onepct_tas_df = pd.concat(onepct_tas_df_list,axis=1)
onepct_rlut_df = pd.concat(onepct_rlut_df_list,axis=1)
onepct_rsut_df = pd.concat(onepct_rsut_df_list,axis=1)
onepct_rsdt_df = pd.concat(onepct_rsdt_df_list,axis=1)
onepct_co2mass_df = pd.concat(onepct_co2mass_df_list,axis=1)

# onepct_co2mass_df.to_csv('./cmip6_data/onepct_co2mass.csv')
# onepct_tas_df.to_csv('./cmip6_data/onepct_tas.csv')
# onepct_rlut_df.to_csv('./cmip6_data/onepct_rlut.csv')
# onepct_rsut_df.to_csv('./cmip6_data/onepct_rsut.csv')
# onepct_rsdt_df.to_csv('./cmip6_data/onepct_rsdt.csv')
# piControl_rlut_df.to_csv('./cmip6_data/piControl_rlut.csv')
# piControl_rsut_df.to_csv('./cmip6_data/piControl_rsut.csv')
# piControl_rsdt_df.to_csv('./cmip6_data/piControl_rsdt.csv')
# abrutp4x_rsdt_df.to_csv('./cmip6_data/abrupt-4xCO2_rsdt.csv')
# abrutp4x_rsut_df.to_csv('./cmip6_data/abrupt-4xCO2_rsut.csv')
# abrutp4x_rlut_df.to_csv('./cmip6_data/abrupt-4xCO2_rlut.csv')
# abrutp4x_tas_df.to_csv('./cmip6_data/abrupt-4xCO2_tas.csv')
# piControl_tas_df.to_csv('./cmip6_data/piControl_tas.csv')
def get_annual_CMIP6_data_esgf(activity, table, variable, experiment, institution, source, member):
    # eg activity='CMIP', table='Amon', variable='tas', experiment='historical', institution="NCAR", source="CESM2", member="r10i1p1f1"
    
    result = esgf_search(activity_id=activity, table_id=table, variable_id=variable, experiment_id=experiment,institution_id=institution, source_id=source, member_id=member)
    
    if not result:
        print('No results for this request')
        return None
    
    # select results with only the latest datestamp:
    latest = sorted([x.split('/')[15] for x in result])[-1]
    result = [x for x in result if x.split('/')[15]==latest]
    
    # remove duplicate results
    result_1 = []

    for item in result:
        if item.split('/')[-1] in [x.split('/')[-1] for x in result_1]:
            continue
        else:
            result_1 += [item]

    ds = xr.open_mfdataset(result_1, combine='by_coords')
    
    files_area = esgf_search(variable_id='areacella', activity_id=activity, institution_id=institution, source_id=source)
    
    if not files_area:
        print('No areacella for this request')
        return None
    
    ds_area = xr.open_dataset(files_area[0])
    
    coords = list(ds[variable].coords.keys())
    if 'lat' in coords:
        dims = ['lat','lon']
    else:
        dims = ['latitude','longitude']
    
    total_area = ds_area.areacella.sum(dim=dims)
    ta_timeseries = (ds[variable] * ds_area.areacella).sum(dim=dims) / total_area
    
    return ta_timeseries.groupby('time.year').mean('time').to_pandas().rename(institution+'_'+source+'_'+member)
def get_annual_CMIP6_data_esgf_multivar(activity, table, variables, experiment, institution, source, member):
    # eg activity='CMIP', table='Amon', variable='tas', experiment='historical', institution="NCAR", source="CESM2", member="r10i1p1f1"
    
    result = esgf_search(activity_id=activity, table_id=table, experiment_id=experiment,institution_id=institution, source_id=source, member_id=member)
    
    result = [x for x in result if x.split('/')[13] in variables]
    
    if not result:
        print('No results for this request')
        return None
    
    # select results with only the latest datestamp:
#     latest = sorted([x.split('/')[15] for x in result])[-1]
#     result = [x for x in result if x.split('/')[15]==latest]
    
    # remove duplicate results
    result_1 = []

    for item in result:
        if item.split('/')[-1] in [x.split('/')[-1] for x in result_1]:
            continue
        else:
            result_1 += [item]

    ds = xr.open_mfdataset(result_1, combine='by_coords')
    
    files_area = esgf_search(variable_id='areacella', activity_id=activity, institution_id=institution, source_id=source)
    
    if not files_area:
        print('No areacella for this request')
        return None
    
    ds_area = xr.open_dataset(files_area[0])
    
    coords = list(ds[variables].coords.keys())
    if 'lat' in coords:
        dims = ['lat','lon']
    else:
        dims = ['latitude','longitude']
    
    total_area = ds_area.areacella.sum(dim=dims)
    ta_timeseries = (ds[variables] * ds_area.areacella).sum(dim=dims) / total_area
    
    _out = ta_timeseries.groupby('time.year').mean('time').to_dataframe()[variables]
    
    return pd.concat([_out],axis=1,keys=[institution+'_'+source+'_'+member])
piControl_list = esgf_search(activity_id='CMIP', table_id='Amon', variable_id='tas', experiment_id='piControl')
piControl_list_nodupl = []

for item in piControl_list:
    if item.split('/')[-1] in [x.split('/')[-1] for x in piControl_list_nodupl]:
        continue
    else:
        piControl_list_nodupl += [item]
        
abrupt4x_list = esgf_search(activity_id='CMIP', table_id='Amon', variable_id='tas', experiment_id='abrupt-4xCO2')
abrupt4x_list_nodupl = []

for item in abrupt4x_list:
    if item.split('/')[-1] in [x.split('/')[-1] for x in abrupt4x_list_nodupl]:
        continue
    else:
        abrupt4x_list_nodupl += [item]
        
areacella_list = esgf_search(activity_id='CMIP', variable_id='areacella')
areacella_list_nodupl = []

for item in areacella_list:
    if item.split('/')[-1] in [x.split('/')[-1] for x in areacella_list_nodupl]:
        continue
    else:
        areacella_list_nodupl += [item]
        
abrupt4x_ism_list = list(set([x.split('/')[8]+'_'+x.split('/')[9]+'_'+x.split('/')[11] for x in abrupt4x_list_nodupl]))
piControl_ism_list = list(set([x.split('/')[8]+'_'+x.split('/')[9]+'_'+x.split('/')[11] for x in piControl_list_nodupl]))
areacella_ism_list = list(set([x.split('/')[8]+'_'+x.split('/')[9]+'_'+x.split('/')[11] for x in areacella_list_nodupl]))
areacella_s_list = list(set([x.split('_')[1] for x in areacella_ism_list]))

piControl_ism_areacella_exist = [x for x in piControl_ism_list if x.split('_')[1] in areacella_s_list]
abrupt4x_ism_areacella_exist = [x for x in abrupt4x_ism_list if x.split('_')[1] in areacella_s_list]
abrupt4x_tas_df = pd.read_csv('./cmip6_data/abrupt-4xCO2_tas.csv',index_col=0)
esgf_abrupt4x_list = [x for x in abrupt4x_ism_areacella_exist if not x in abrupt4x_tas_df.columns]

piControl_tas_df = pd.read_csv('./cmip6_data/piControl_tas.csv',index_col=0)
esgf_piControl_list = [x for x in piControl_ism_areacella_exist if not x in piControl_tas_df.columns]
def get_CMIP6_data(ism,exp='abrupt-4xCO2',var='tas',multivar=False):
    ism_split = ism.split('_')
    if multivar:
        _out = get_annual_CMIP6_data_esgf_multivar('CMIP', 'Amon', var, exp, ism_split[0], ism_split[1], ism_split[2])
    else:
        _out = get_annual_CMIP6_data_esgf('CMIP', 'Amon', var, exp, ism_split[0], ism_split[1], ism_split[2])
    print(ism+' complete')
    return _out
# abrupt4x_df_list_esgf = []
for x in esgf_abrupt4x_list:
    abrupt4x_df_list_esgf += [get_CMIP6_data(x,'abrupt-4xCO2',['tas','rlut','rsut','rsdt'],True)]
# P1=multiprocessing.Pool(processes=8)
# abrupt4xCO2_df = P1.starmap(get_CMIP6_data,[(x,'abrupt-4xCO2',['tas','rlut','rsut','rsdt'],True) for x in abrupt4x_ism_areacella_exist])
# P1.close
piControl_df_list_esgf = []
for x in piControl_ism_areacella_exist:
    piControl_df_list_esgf += [get_CMIP6_data(x,'piControl','tas')]
# P1=multiprocessing.Pool(processes=8)
# piControl_df = P1.starmap(get_CMIP6_data,[(x,'piControl','tas',False) for x in piControl_ism_areacella_exist])
# P1.close<jupyter_output>NASA-GISS_GISS-E2-1-G_r1i1p1f2 complete
NASA-GISS_GISS-E2-1-G_r1i1p1f3 complete
{'fraction_non_alphanumeric': 0.1640457802177352, 'fraction_numerical': 0.017400204708290685, 'mean_word_length': 4.99182308121167, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '44174', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 12494, 'n_tokens_neox': 12282, 'n_words': 1804, 'path': 'GIR/tools/get_cmip6_data.ipynb', 'repo_name': 'njleach/GIR', 'size': '32241'}