<jupyter_start><jupyter_text>Create the environment<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
%cd /content/drive/My Drive/ESoWC
import pandas as pd
import xarray as xr

import numpy as np
import pandas as pd
from sklearn import preprocessing
import seaborn as sns

#Our class
from create_dataset.make_dataset import CustomDataset
fn_weather_05_19 = 'Data/05_2019_weather_and_CO.nc'
fn_weather_07_19 = 'Data/07_2019_weather_and_CO.nc'
fn_weather_05_20 = 'Data/05_2020_weather_and_CO.nc'
fn_weather_06_20 = 'Data/06_2020_weather_and_CO.nc'
fn_land = 'Data/land_cover_data.nc'
fn_conc = 'Data/totalcolConcentretations_featured.nc'
fn_traffic = 'Data/emissions_traffic_hourly_merged.nc'<jupyter_output><empty_output><jupyter_text>Land<jupyter_code>land_instance = CustomDataset(fn_land)
land_instance.resample("1H")
land_fixed = land_instance.get_dataset()
land_fixed = land_fixed.drop_vars('NO emissions') #They are already in the weather dataset
land_fixed = land_fixed.transpose('latitude','longitude','time')  
land_fixed<jupyter_output>Opening dataset at :  Data/land_cover_data.nc
Done!<jupyter_text>Conc<jupyter_code>conc_fidex = xr.open_dataset(fn_conc)
conc_fidex<jupyter_output><empty_output><jupyter_text>Traffic<jupyter_code>traffic_instance = CustomDataset(fn_traffic)
traffic_ds= traffic_instance.get_dataset()
traffic_ds=traffic_ds.drop_vars('emissions')
lat_bins = np.arange(43,51.25,0.25)
lon_bins = np.arange(4,12.25,0.25)
traffic_ds = traffic_ds.sortby(['latitude','longitude','hour'])  
traffic_ds = traffic_ds.interp(latitude=lat_bins, longitude=lon_bins, method="linear")
days = np.arange(1,32,1)
traffic_ds=traffic_ds.expand_dims({'Days':days})
trafic_df = traffic_ds.to_dataframe()
trafic_df = trafic_df.reset_index()
trafic_df['time'] = (pd.to_datetime(trafic_df['Days']-1,errors='ignore',
                             unit='d',origin='2019-05') +
                    pd.to_timedelta(trafic_df['hour'], unit='h'))
trafic_df=trafic_df.drop(columns=['Days', 'hour'])
trafic_df = trafic_df.set_index(['latitude','longitude','time'])
traffic_fixed = trafic_df.to_xarray()
traffic_fixed = traffic_fixed.transpose('latitude','longitude','time')  
traffic_fixed<jupyter_output>Opening dataset at :  Data/emissions_traffic_hourly_merged.nc
Done!<jupyter_text>05_2019<jupyter_code>tot_dataset_05_19 = xr.open_dataset('Data/05_2019_dataset_complete_for_model_CO.nc')
tot_dataset_05_19 = tot_dataset_05_19.rename({'EMISSIONS_2019':'EMISSIONS'})
tot_dataset_05_19<jupyter_output><empty_output><jupyter_text>07_2019 Land<jupyter_code>land_fixed_07_19 = land_fixed

land_df = land_fixed_07_19.to_dataframe()
land_df = land_df.reset_index()
land_df.time = land_df.time + pd.DateOffset(months=2)
land_df = land_df.set_index(['latitude','longitude','time'])
land_fixed_07_19 = land_df.to_xarray()
land_fixed_07_19<jupyter_output><empty_output><jupyter_text>Weather<jupyter_code>weather_07_19 = xr.open_dataset('Data/07_2019_weather_and_CO_for_model.nc')
#This variable is too much correlated with the tcw
weather_fixed_07_19 = weather_07_19.drop_vars('tcwv')
weather_fixed_07_19 = weather_fixed_07_19.rename({'EMISSIONS_2019':'EMISSIONS'})
weather_fixed_07_19 = weather_fixed_07_19.transpose('latitude','longitude','time')  
weather_fixed_07_19<jupyter_output><empty_output><jupyter_text>Conc<jupyter_code>conc_fidex_07_19 = conc_fidex

conc_df = conc_fidex_07_19.to_dataframe()
conc_df = conc_df.reset_index()
conc_df.time = conc_df.time + pd.DateOffset(months=2)
conc_df = conc_df.set_index(['latitude','longitude','time'])
conc_fidex_07_19 = conc_df.to_xarray()
conc_fidex_07_19<jupyter_output><empty_output><jupyter_text>Traffic<jupyter_code>traffic_fixed_07_19 = traffic_fixed

traffic_df = traffic_fixed_07_19.to_dataframe()
traffic_df = traffic_df.reset_index()
traffic_df.time = traffic_df.time + pd.DateOffset(months=2)
traffic_df = traffic_df.set_index(['latitude','longitude','time'])
traffic_fixed_07_19 = traffic_df.to_xarray()
traffic_fixed_07_19<jupyter_output><empty_output><jupyter_text>Merge<jupyter_code>tot_dataset_07_19 = weather_fixed_07_19.merge(land_fixed_07_19)
tot_dataset_07_19 = tot_dataset_07_19.merge(conc_fidex_07_19)
tot_dataset_07_19 = tot_dataset_07_19.merge(traffic_fixed_07_19)

tot_dataset_07_19<jupyter_output><empty_output><jupyter_text>05_2020 Land<jupyter_code>land_fixed_05_20 = land_fixed

land_df = land_fixed_05_20.to_dataframe()
land_df = land_df.reset_index()
land_df.time = land_df.time + pd.DateOffset(years=1)
land_df = land_df.set_index(['latitude','longitude','time'])
land_fixed_05_20 = land_df.to_xarray()
land_fixed_05_20<jupyter_output><empty_output><jupyter_text>Weather<jupyter_code>weather_05_20 = xr.open_dataset('Data/05_2020_weather_and_CO_for_model.nc')
#This variable is too much correlated with the tcw
weather_fixed_05_20 = weather_05_20.drop_vars('tcwv')
weather_fixed_05_20 = weather_fixed_05_20.rename({'EMISSIONS_2020':'EMISSIONS'})
weather_fixed_05_20 = weather_fixed_05_20.transpose('latitude','longitude','time')  
weather_fixed_05_20<jupyter_output><empty_output><jupyter_text>Conc<jupyter_code>conc_fidex_05_20 = conc_fidex

conc_df = conc_fidex_05_20.to_dataframe()
conc_df = conc_df.reset_index()
conc_df.time = conc_df.time + pd.DateOffset(years=1)
conc_df = conc_df.set_index(['latitude','longitude','time'])
conc_fidex_05_20 = conc_df.to_xarray()
conc_fidex_05_20<jupyter_output><empty_output><jupyter_text>Traffic<jupyter_code>traffic_fixed_05_20 = traffic_fixed

traffic_df = traffic_fixed_05_20.to_dataframe()
traffic_df = traffic_df.reset_index()
traffic_df.time = traffic_df.time + pd.DateOffset(years=1)
traffic_df = traffic_df.set_index(['latitude','longitude','time'])
traffic_fixed_05_20 = traffic_df.to_xarray()
traffic_fixed_05_20<jupyter_output><empty_output><jupyter_text>Merge<jupyter_code>tot_dataset_05_20 = weather_fixed_05_20.merge(land_fixed_05_20)
tot_dataset_05_20 = tot_dataset_05_20.merge(conc_fidex_05_20)
tot_dataset_05_20 = tot_dataset_05_20.merge(traffic_fixed_05_20)

tot_dataset_05_20<jupyter_output><empty_output><jupyter_text>Merge<jupyter_code>tot_dataset = tot_dataset_05_19.merge(tot_dataset_07_19)
tot_dataset = tot_dataset.merge(tot_dataset_05_20)

tot_dataset<jupyter_output><empty_output><jupyter_text>Save<jupyter_code>tot_dataset.to_netcdf('Data/3_months_dataset_complete_for_model_CO.nc', 'w', 'NETCDF4')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11516194648724769, 'fraction_numerical': 0.06149272414332655, 'mean_word_length': 10.97003745318352, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '47946', 'license': 'MIT', 'n_tokens_mistral': 2807, 'n_tokens_neox': 2569, 'n_words': 252, 'path': 'notebooks/exploratory/3_months_CO_dataset_complete.ipynb', 'repo_name': 'esowc/MaLePoM', 'size': '6391'}