<jupyter_start><jupyter_text>Retrieve sunlight data<jupyter_code># !pip install rioxarray -q
import xarray as xr
import pandas as pd
import geopandas as gpd
import numpy as np
import matplotlib.pyplot as plt
# import rioxarray as rx
import scipy.stats as ss 
ds = xr.open_dataset('../adaptor.mars.internal-1628422647.5552244-10052-10-6bc58ad9-e69f-43a1-8834-014dc0739c00.nc')
ds
fig, axs = plt.subplots(2,3, figsize=(15,8))

for i, var in enumerate(ds.keys()):
    da = ds[var]
    axs.flatten()[i].set_title(da.long_name)
    da.mean('time').plot(ax=axs.flatten()[i])
    
plt.tight_layout()
nox = xr.open_dataset('../data/nox/coco_hourly_study_region.nc')
ds = ds.sel(longitude=slice(nox.longitude.min(), nox.longitude.max()), latitude=slice(nox.latitude.max(), nox.latitude.min()))
nox = nox.interp(latitude=ds.latitude, longitude=ds.longitude)
merged = xr.merge([ds, nox])
df = merged.to_dataframe()
df = df.dropna()
# df = df.iloc[:2]
df.head()
corr = df.corr().iloc[:-1,-1:]
corr.style.background_gradient(cmap='coolwarm')
plt.scatter((df.uvb), np.log(df.EMISSIONS_2019), alpha=0.1)
ss.pearsonr((df.uvb), df.EMISSIONS_2019)[0]<jupyter_output><empty_output><jupyter_text>___<jupyter_code>def spatial_corr(da, variables, time_lag=0):
    '''Return mean spatial correlation coefficient for two variables in Dataset.
       Specify two variables to calculate corr coef. between. '''
    
    assert len(variables) == 2, 'Enter two variables.'
     
    rs = []
    for t in da.time.values:
        da_ = da.sel(time=t)
#         sd = da_.std()[variables[1]].values
        
        
        df_ = da_.to_dataframe()[variables]
        df_ = df_.dropna()
        
        unique = df_.nunique().loc[variables].values
        
        if (unique > 1).sum() == len(variables):
            if len(df_) < 2:
                continue
            else:
                r = ss.pearsonr(df_[variables[0]], df_[variables[1]])[0]
                rs.append((t, r))
        else:
            continue
            
    return pd.DataFrame(rs).rename(columns={0:'time', 1:'r'}).set_index('time')
d = spatial_corr(xr.merge([ds, nox]), variables=['EMISSIONS_2019', 'tsrc'])
d.groupby(d.index.hour).mean().plot()<jupyter_output><empty_output><jupyter_text>___<jupyter_code>xr.corr(merged['tsrc'], merged['EMISSIONS_2019'], dim='time').plot()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16390658174097664, 'fraction_numerical': 0.03524416135881104, 'mean_word_length': 3.5570599613152805, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '366890', 'license': 'MIT', 'n_tokens_mistral': 945, 'n_tokens_neox': 870, 'n_words': 149, 'path': 'exploratory_data_analysis/radiation_data.ipynb', 'repo_name': 'vidurmithal/malepom-test', 'size': '2355'}