<jupyter_start><jupyter_text>Create the environment<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
%cd /content/drive/My Drive/ESoWC
import pandas as pd
import xarray as xr
import numpy as np
import random

from sklearn import linear_model
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import ExtraTreesRegressor

from sklearn.feature_selection import VarianceThreshold

from sklearn.decomposition import PCA

from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score

from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split

from xgboost import XGBRegressor
from sklearn.metrics import mean_absolute_error

import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Load dataset and clean<jupyter_code>dataset = xr.open_dataset('Data/3_months_dataset_complete_for_model_CO.nc')
dataset
df = dataset.to_dataframe()
df = df.dropna()
df.head()
df.shape<jupyter_output><empty_output><jupyter_text>Managing dataframe Now we create the feature days and hours<jupyter_code>df_flat = df
df_flat['days'] = df_flat.index.get_level_values("time").day 
df_flat['hours'] = df_flat.index.get_level_values("time").hour
df_flat = df_flat.reset_index()
df_flat = df_flat.drop(["time"], axis=1)
df_flat.head()<jupyter_output><empty_output><jupyter_text>Split the dataframe in features and target<jupyter_code>X = df_flat.drop(columns = ['EMISSIONS'])
y = df_flat['EMISSIONS']

input_variables = X.columns
target_variable = 'EMISSIONS'

seed = 1234<jupyter_output><empty_output><jupyter_text>Normalizing<jupyter_code>scaler = MinMaxScaler()
df_sc = scaler.fit_transform(df_flat)

df_norm = pd.DataFrame(df_sc, columns=df_flat.columns)
df_norm.head()<jupyter_output><empty_output><jupyter_text>Baseline performance: Linear regression with 10 fold cross validation<jupyter_code>kfolds = KFold(10,shuffle=True,random_state=seed)

model = linear_model.LinearRegression()
scores = cross_val_score(model, df_norm[input_variables], df_norm[target_variable], cv=kfolds)

print("R2 Mean %.3f StdDev %.3f"%(scores.mean(),scores.std()))
scores = cross_val_score(model, df_norm[input_variables], df_norm[target_variable], cv=kfolds, scoring='neg_mean_squared_error')

print("MSE %.3f StdDev %.3f"%(abs(scores.mean()),scores.std()))<jupyter_output>MSE 0.000 StdDev 0.000<jupyter_text>Random Forest feature importance<jupyter_code>forest = ExtraTreesRegressor(n_estimators=100, random_state=0)
forest.fit(df_norm[input_variables], df_norm[target_variable])
importances = forest.feature_importances_
importances
importances = forest.feature_importances_
std = np.std([tree.feature_importances_ for tree in forest.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

# Print the feature ranking
print("Feature ranking:")

for f in range(df_norm[input_variables].shape[1]):
    print("%2d. feature %2d %7s (%.3f)" % (f + 1, indices[f], df_norm[input_variables].columns[indices[f]], importances[indices[f]]))

# Plot the feature importances of the forest
plt.figure(figsize=(9, 6))
plt.title("Feature importances")
plt.bar(range(df_norm[input_variables].shape[1]), importances[indices],
       color="r", yerr=std[indices], align="center")
plt.xticks(range(df_norm[input_variables].shape[1]), df_norm[input_variables].columns[indices],rotation=90)
plt.xlim([-1, df_norm[input_variables].shape[1]])
plt.show()<jupyter_output>Feature ranking:
 1. feature 33    days (0.055)
 2. feature 19   built (0.050)
 3. feature 11  sp_hum (0.047)
 4. feature 18  height (0.047)
 5. feature 32 traffic (0.044)
 6. feature  1 longitude (0.040)
 7. feature 34   hours (0.038)
 8. feature 15 tmp_shift_1 (0.037)
 9. feature 10     tmp (0.037)
10. feature  9      sp (0.034)
11. feature  0 latitude (0.033)
12. feature 30 CO_tc_add_trend (0.033)
13. feature 17 rel_hum_shift_8 (0.032)
14. feature  7     cvl (0.031)
15. feature  6     tcc (0.029)
16. feature 16 tot_wind_shift_12 (0.029)
17. feature 13     tcw (0.029)
18. feature 24   CO_tc (0.026)
19. feature  8     cvh (0.026)
20. feature  3     v10 (0.026)
21. feature 27 CO2_tc_add_trend (0.023)
22. feature 12 rel_hum (0.023)
23. feature 31 O3_tc_add_trend (0.023)
24. feature  2     u10 (0.022)
25. feature  4     hcc (0.021)
26. feature 29 NO2_tc_add_trend (0.021)
27. feature 28 CH4_tc_add_trend (0.019)
28. feature 14 tot_wind (0.018)
29. feature 25   O3_tc (0.018)
30. feature  5 [...]
{'fraction_non_alphanumeric': 0.11892497200447928, 'fraction_numerical': 0.061142217245240764, 'mean_word_length': 5.8919753086419755, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '550610', 'license': 'MIT', 'n_tokens_mistral': 1795, 'n_tokens_neox': 1613, 'n_words': 371, 'path': 'notebooks/model/3_months_CO_baseline_model.ipynb', 'repo_name': 'esowc/MaLePoM', 'size': '4465'}