<jupyter_start><jupyter_text>Urban Air Pollution Challenge:The objective of this challenge is to predict PM2.5 particulate matter concentration in the air every day for each city. PM2.5 refers to atmospheric particulate matter that have a diameter of less than 2.5 micrometers and is one of the most harmful air pollutants. PM2.5 is a common measure of air quality that normally requires ground-based sensors to measure. The data covers the last three months, spanning hundreds of cities across the globe.<jupyter_code>import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn.preprocessing import LabelEncoder
import shap
from sklearn.model_selection import KFold, train_test_split
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error
import lightgbm as lgbm
import xgboost as xgb
import warnings
warnings.filterwarnings('ignore')
# Utility error function
def rmse(y,x):
    return np.sqrt(mean_squared_error(x,y))
# Load the data
train_df = pd.read_csv('./raw_data/Train.csv')
test = pd.read_csv('./raw_data/Test.csv')
sub = pd.read_csv('./raw_data/SampleSubmission.csv')
train_df.head(3)
def check_missing_data(data: pd.DataFrame) -> pd.DataFrame:
  """Checks a given dataframe for missing values and
  types of the data features.
  """
  total = data.isna().sum()
  percent = (data.isna().sum()/data.isna().count()*100)
  tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
  types = []
  for col in data.columns:
      dtype = str(data[col].dtype)
      types.append(dtype)
  tt['Types'] = types
  return(np.transpose(tt))

check_missing_data(train_df)
train_df.describe()
# 75% of target observations in the train set fall within 80.
# This means that there exist outliers since the max value of the target feature
# is 815 as seen from train.describe().
print(f'with outliers: {len(train_df)}')
train = train_df[train_df['target'] <= 500]
print(f'Without outliers: {len(train)}')
# Dropping target related features in the train set
drop_cols = ['target_min', 'target_max', 'target_variance', 'target_count']

train.drop(drop_cols, axis=1, inplace=True)
encoder = LabelEncoder()
train['Place_ID'] = encoder.fit_transform(train.Place_ID).astype('int32')
test['Place_ID'] = encoder.fit_transform(test.Place_ID).astype('int32')
train['day'] = pd.to_datetime(train['Date']).dt.day.astype('int32')
train['month'] = pd.to_datetime(train['Date']).dt.month.astype('int32')

test['day'] = pd.to_datetime(test['Date']).dt.day.astype('int32')
test['month'] = pd.to_datetime(test['Date']).dt.month.astype('int32')
# Correlation analysis
corr = train.corr()
# barhistogram of correlations to the target variable
(corr
     .target
     .drop("target") # can't compare the variable under study to itself
     .sort_values(ascending=False)
     .plot
     .barh(figsize=(9,6)))
plt.title("correlation bar_hist")
drop = ['target', 'Date', 'Place_ID X Date', 'Place_ID', 'L3_AER_AI_sensor_altitude']

y = train['target']
X = train.drop(drop, axis=1)

ids = test['Place_ID X Date']
test = test.drop(drop[1:], axis=1)

X = X.fillna(value=X.median()).astype('float32')
test = test.fillna(value=test.median()).astype('float32')

# LGBM model
lgb_params = {
    'metric' : 'rmse',
    'boosting': 'gbdt',
    'learning_rate': 0.025, #0.025 (32.3533)
    'max_depth': 11, 
    'num_leaves': 80,
    'objective': 'regression',
    #'subsample': 0.9,
    'feature_fraction': 0.5,
    'bagging_fraction': 0.5,
    #'lambda_l2': 0.2,
    'max_bin': 1000 }

# split data into train and validation sets
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)

#train_data = lgbm.Dataset(X_train, label=y_train)
#test_data = lgbm.Dataset(X_val, label=y_val)

#lgb_model = lgbm.train(lgb_params, train_data, valid_sets=[train_data, test_data],
#                       num_boost_round=9000, early_stopping_rounds=100
#                      ) #0.03lr

#lgb_df = lgbm.Dataset(X, y)
#lgb_model = lgbm.train(lgb_params, lgb_df, num_boost_round=5000)
# LGBM with 8-fold CV
folds = KFold(n_splits=8, shuffle=False, random_state=42)

scores = []
for n_fold, (trn_idx, val_idx) in enumerate(folds.split(X, y)):
    trn_x, trn_y = X.iloc[trn_idx], y.iloc[trn_idx]
    val_x, val_y = X.iloc[val_idx], y.iloc[val_idx]
    
    tr_data = lgbm.Dataset(trn_x, label=trn_y)
    val_data = lgbm.Dataset(val_x, label=val_y)

    lgb_model = lgbm.train(lgb_params, tr_data, valid_sets=[tr_data, val_data],
                           num_boost_round=5000, early_stopping_rounds=100
                          ) 
    root_mse = rmse(val_y, lgb_model.predict(val_x))
    scores.append(root_mse)
    print(root_mse)

print("Average score in 5-fold CV:", np.mean(scores))
feature_importances = pd.DataFrame(lgb_model.feature_importance(),
                                   index = trn_x.columns,
                                   columns=['importance']).sort_values('importance',ascending=False)
feature_importances
%time shap_values = shap.TreeExplainer(lgb_model).shap_values(X_val)
shap.summary_plot(shap_values, X_val)<jupyter_output><empty_output><jupyter_text>The shap summary plot above shows the 30 most important features.For each feature a distribution is plotted on how the train samples influence the model outcome. The more red the dots, the higher the feature value, the more blue the lower the feature value.<jupyter_code># making predictions
predictions = lgb_model.predict(test, num_iteration=lgb_model.best_iteration)
sub['Place_ID X Date'] = ids
sub['target'] = predictions
sub.head()
lgbm_score = lgb_model.best_score['valid_1']['rmse']
lgbm_score
#%mkdir ./submissions
sub.to_csv(f'./submissions/sub_lgb{np.round(lgbm_score, 4)}', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12810773480662985, 'fraction_numerical': 0.019164364640883978, 'mean_word_length': 4.684985279685967, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '182784', 'license': 'MIT', 'n_tokens_mistral': 2116, 'n_tokens_neox': 2000, 'n_words': 486, 'path': 'urban_air_pollution/model.ipynb', 'repo_name': 'steph-en-m/competitive-programming', 'size': '5792'}