<jupyter_start><jupyter_text>Land and Ocean Difference<jupyter_code>import numpy as np
import pandas as pd
import xarray as xr
from tqdm import tqdm
import gc
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
import matplotlib.ticker as mticker
from util import *
import geopandas as gpd
from geopandas import GeoDataFrame as gdf

def open_chi(path):  
    ds=(xr.open_dataset(path)*100)
    ds=ds.assign_coords(lon=(((ds.lon + 180) % 360) - 180))
    ds=ds.reindex(lon=sorted(ds.lon))  
    return ds
year = "2011"
method_ls = ["MAM4","ML"]
chi_ls = ["chi_b","chi_c","chi_h"]

file_path = {}
file_path["MAM4"] = "/data/keeling/a/zzheng25/d/mam4_paper_data/chi_only/mam4_chi/"
file_path["ML"] = "/data/keeling/a/zzheng25/d/mam4_paper_data/chi_only/ml_chi/"
file_path["diff"] = "/data/keeling/a/zzheng25/d/mam4_paper_data/chi_only/mam4_minus_ml_chi/"
file_path["diff_abs"] = "/data/keeling/a/zzheng25/d/mam4_paper_data/chi_only/mam4_minus_ml_chi/"
mask_path = "/data/keeling/a/zzheng25/d/mam4_paper_data/chi_only/mask/"

per_ls = [
    "bc_a1_per","bc_a4_per",
    "dst_a1_per","dst_a2_per",
    "ncl_a1_per","ncl_a2_per",
    "pom_a1_per","pom_a4_per",
    "so4_a1_per","so4_a2_per",
    "soa_a1_per","soa_a2_per"
    ]

comp = open_nc("/data/keeling/a/zzheng25/d/mam4_paper_data/chi_only/comp_analysis/"+str(year)+"_year_comp.nc")\
       .to_dataframe()[per_ls].reset_index()<jupyter_output><empty_output><jupyter_text>load data<jupyter_code>da={}
for chi in tqdm(chi_ls):
    da[chi]={}
    for method in method_ls:
        if method=="diff_abs":
            da_temp = open_chi(file_path[method]+str(year)+"_"+chi+"_mean_abs.nc")[chi]
            mask = open_chi(mask_path+str(year)+"_"+chi+".nc")["mask"]
            da[chi][method] = da_temp.where(mask)
        
        else:
            da_temp = open_chi(file_path[method]+str(year)+"_"+chi+"_mean.nc")[chi]
            mask = open_chi(mask_path+str(year)+"_"+chi+".nc")["mask"]
            da[chi][method] = da_temp.where(mask)
        
        del da_temp, mask
        gc.collect()<jupyter_output>100%|██████████| 3/3 [00:02<00:00,  1.13it/s]<jupyter_text>Workflow<jupyter_code># get ocean and land data
def get_land_ocean(method, da):
    chi_b=da["chi_b"][method].to_dataframe().reset_index()
    chi_c=da["chi_c"][method].to_dataframe().reset_index()
    chi_h=da["chi_h"][method].to_dataframe().reset_index()

    merge_1=chi_b.merge(chi_c,on=["lat","lon"],how="outer")
    df=merge_1.merge(chi_h,on=["lat","lon"],how="outer")
    
    world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))
    points= gdf(df.reset_index(), geometry=gpd.points_from_xy(df.reset_index().lon, df.reset_index().lat))
    points.crs="EPSG:4326"
    jps = gpd.sjoin(world, points, how='right', op='contains')
    vari_ls_new = ["lat","lon","chi_b","chi_c","chi_h"]
    land = jps[~jps["index_left"].isnull()][vari_ls_new]
    ocean = jps[jps["index_left"].isnull()][vari_ls_new]
    
    land_new = land.set_index(["lat","lon"]).stack()\
    .reset_index(name="mixing_state_index")\
    .rename(columns={'level_2':'mixing_state_type'})
    print("land")
    display(land_new.groupby(["mixing_state_type"])["mixing_state_index"].describe()["mean"])

    ocean_new = ocean.set_index(["lat","lon"]).stack()\
    .reset_index(name="mixing_state_index")\
    .rename(columns={'level_2':'mixing_state_type'})
    print("ocean")
    display(ocean_new.groupby(["mixing_state_type"])["mixing_state_index"].describe()["mean"])
    print("\n")
    
    return land_new, ocean_new

# comp analysis for land or ocean
def comp_analysis(df, chi, lat_min=-90, lat_max=90, lon_min=-180, lon_max=180, comp=comp):
    df_temp = df[(df["lat"]>=lat_min) & (df["lat"]<=lat_max) &
                 (df["lon"]>=lon_min) & (df["lon"]<=lon_max) & 
                 (df["mixing_state_type"]==chi)]
#     print(df_temp.shape)
    df_temp_comp=df_temp.merge(comp, on=["lat","lon"], how="inner")
#     print(df_temp_comp.shape)
    display(df_temp_comp.describe().transpose()["mean"])
print("MAM4")
method="MAM4"
l_mam4, o_mam4 = get_land_ocean(method, da)

print("ML")
method="ML"
l_ml, o_ml = get_land_ocean(method, da)<jupyter_output>MAM4
land<jupyter_text>chi_opt1  MAM4<jupyter_code>chi = "chi_b"; lat_min = 45; lat_max = 60
comp_analysis(o_mam4, chi, lat_min, lat_max)
chi = "chi_b"; lat_min = -90; lat_max = -66.5
comp_analysis(l_mam4, chi, lat_min, lat_max)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18235161363123448, 'fraction_numerical': 0.023696682464454975, 'mean_word_length': 4.877984084880636, 'pattern_counts': {'":': 1, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '55901', 'license': 'MIT', 'n_tokens_mistral': 1906, 'n_tokens_neox': 1849, 'n_words': 221, 'path': '3_analysis/chi_o_land_ocean_comp_analysis.ipynb', 'repo_name': 'zzheng93/code_ms_ml_mam4', 'size': '4431'}