<jupyter_start><jupyter_text>Create the environment<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
%cd /content/drive/My Drive/ESoWC
import pandas as pd
import xarray as xr

import numpy as np
import pandas as pd
from sklearn import preprocessing
import seaborn as sns

#Our class
from create_dataset.make_dataset import CustomDataset
fn_land = 'Data/land_cover_data.nc'
fn_weather = 'Data/05_2019_weather_and_CO_for_model.nc'
fn_conc = 'Data/totalcolConcentretations_featured.nc'
fn_traffic = 'Data/emissions_traffic_hourly_merged.nc'<jupyter_output><empty_output><jupyter_text>Load datasets Land<jupyter_code>land_instance = CustomDataset(fn_land)
land_instance.get_dataset()
land_instance.resample("1H")
land_fixed = land_instance.get_dataset()
land_fixed = land_fixed.drop_vars('NO emissions') #They are already in the weather dataset
land_fixed = land_fixed.transpose('latitude','longitude','time')  
land_fixed<jupyter_output><empty_output><jupyter_text>Weather<jupyter_code>weather = xr.open_dataset(fn_weather)
weather
#This variable is too much correlated with the tcw
weather_fixed = weather.drop_vars('tcwv')

weather_fixed = weather_fixed.transpose('latitude','longitude','time')  
weather_fixed<jupyter_output><empty_output><jupyter_text>Conc<jupyter_code>conc_fidex = xr.open_dataset(fn_conc)
conc_fidex<jupyter_output><empty_output><jupyter_text>Traffic<jupyter_code>traffic_instance = CustomDataset(fn_traffic)
traffic_ds= traffic_instance.get_dataset()
traffic_ds
traffic_ds=traffic_ds.drop_vars('emissions')
lat_bins = np.arange(43,51.25,0.25)
lon_bins = np.arange(4,12.25,0.25)
traffic_ds = traffic_ds.sortby(['latitude','longitude','hour'])  
traffic_ds = traffic_ds.interp(latitude=lat_bins, longitude=lon_bins, method="linear")
days = np.arange(1,32,1)
traffic_ds=traffic_ds.expand_dims({'Days':days})
traffic_ds
trafic_df = traffic_ds.to_dataframe()
trafic_df = trafic_df.reset_index()
trafic_df['time'] = (pd.to_datetime(trafic_df['Days']-1,errors='ignore',
                             unit='d',origin='2019-05') +
                    pd.to_timedelta(trafic_df['hour'], unit='h'))
trafic_df=trafic_df.drop(columns=['Days', 'hour'])
trafic_df = trafic_df.set_index(['latitude','longitude','time'])
trafic_df.head()
traffic_fixed = trafic_df.to_xarray()
traffic_fixed = traffic_fixed.transpose('latitude','longitude','time')  
traffic_fixed
traffic_fixed.isel(time=[15]).traffic.plot()<jupyter_output><empty_output><jupyter_text>Merge<jupyter_code>tot_dataset = weather_fixed.merge(land_fixed)
tot_dataset = tot_dataset.merge(conc_fidex)
tot_dataset = tot_dataset.merge(traffic_fixed)

tot_dataset<jupyter_output><empty_output><jupyter_text>Check<jupyter_code>weather_fixed.to_dataframe().isnull().sum()
land_fixed.to_dataframe().isnull().sum()
conc_fidex.to_dataframe().isnull().sum()
traffic_fixed.to_dataframe().isnull().sum()
tot_dataset.to_dataframe().isnull().sum()
tot_dataset.isel(time=[12]).EMISSIONS_2019.plot()
tot_dataset.isel(time=[12]).u10.plot()
tot_dataset.isel(time=[15]).height.plot()
tot_dataset.isel(time=[12]).NO_tc.plot()
tot_dataset.isel(time=[15]).traffic.plot()<jupyter_output><empty_output><jupyter_text>Save the dataset<jupyter_code>tot_dataset.to_netcdf('Data/05_2019_dataset_complete_for_model_CO.nc', 'w', 'NETCDF4')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13650030066145522, 'fraction_numerical': 0.01803968731208659, 'mean_word_length': 9.36448598130841, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '7429', 'license': 'MIT', 'n_tokens_mistral': 1292, 'n_tokens_neox': 1246, 'n_words': 144, 'path': 'notebooks/exploratory/05_2019_CO_dataset_complete.ipynb', 'repo_name': 'esowc/MaLePoM', 'size': '3326'}