<jupyter_start><jupyter_text>**_DSL Project_**         Predict impact of air quality on mortality rates(Kaggle Competition)https://www.kaggle.com/c/predict-impact-of-air-quality-on-death-rates/overview    **Group Number - 2** <NAME>   &emsp;19BM6JP15         <NAME>  &emsp;19BM6JP24     <NAME> &emsp;19BM6JP25   TO DO- ANOVA on variables per region- XGB Regressor- Best depth via cost pruning for RFR- Graphs for regulartization   Libraries<jupyter_code>#!pip install impyute
#!pip freeze > requirements.txt
import pandas as pd
import numpy as np
from sklearn.linear_model import LinearRegression
import xgboost as xgb
from sklearn import metrics
import matplotlib.pyplot as plt
import seaborn as sns
from impyute.imputation.cs import mice
from sklearn.model_selection import train_test_split
from sklearn import linear_model
from sklearn.linear_model import LinearRegression, Lasso, Ridge
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import cross_val_score
from sklearn.metrics import  make_scorer, mean_absolute_error, r2_score, median_absolute_error, mean_squared_error
import math

import missingno as msno

import plotly.express as px
import plotly.graph_objects as go

from statsmodels.tsa.seasonal import seasonal_decompose
from dateutil.parser import parse

import gc
import random
import os

import scipy.stats as sps 
#plt.style.available
plt.style.use('fivethirtyeight')<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def seed_everything(SEED=1729):
    random.seed(SEED)
    os.environ['PYTHONHASHSEED'] = str(SEED)
    np.random.seed(SEED)
    print('done')

seed_everything()
def reduce_mem_usage(df, verbose=True):
    '''
    Reduces the usage of memory by a dataframe
    '''
    start_mem = df.memory_usage().sum() / 2**20
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64'] #defining a list of numeric types
    #start_mem = df.memory_usage().sum() / 2**20   #converting bytes into MB
    for col in df.columns: # iterating over each column of dataframe
        col_type = df[col].dtypes # extracting the data type of a single column
        if col_type in numerics: # checking whether the data type of column is a numeric datatype or not
            c_min = df[col].min()
            c_max = df[col].max()
          
            if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                df[col] = df[col].astype('float16')
            elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                df[col] = df[col].astype('float32')
            else:
                df[col] = df[col].astype('float64')    
    end_mem = df.memory_usage().sum() / 2**20
    if verbose: print('Mem. usage decreased to {:5.5f} Mb ({:.2f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))
    return df
def reg_metrics (y_true,y_predicted):
  mse = mean_squared_error(y_true,y_predicted)
  print("MSE = " + str(mse))
  print("RMSE =" + str(math.sqrt(mse)))
  print("Mean Absolute Error =" + str(mean_absolute_error(y_true,y_predicted)))
  print('Median Absolute Error  = ' + str(median_absolute_error(y_true, y_predicted)))
  print('R^2                    = ' + str(r2_score(y_true, y_predicted)))
  del mse
  gc.collect()
def resid_plot(y_true, y_predicted):
    '''
    1. Plots residual plots by taking the y_true and y_predicted as input.
    2. Can be used with any regression method be it linear, ridge, lasso, boosting-based, etc.
    '''
    temp1 = np.array(y_true).reshape(-1,1)
    temp2 = np.array(y_predicted).reshape(-1,1)

    resids = np.subtract(temp1,temp2) #calculating residuals

    sns.regplot(x=y_predicted, y=resids, fit_reg=True)

    plt.title("Residuals vs. Predicted Values")
    plt.xlabel("Predicted Values")
    plt.ylabel("Residuals")

    plt.show()
    plt.close()
    del temp1, temp2
    gc.collect()
def resids_hist(y_true,y_predicted):
  '''
  Plots histogram of residuals
  '''

  temp1 = np.array(y_true).reshape(-1,1)
  temp2 = np.array(y_predicted).reshape(-1,1)

  resids = np.subtract(temp1,temp2) 

  sns.distplot(resids)

  plt.title("Histogram of Residual Values")
  plt.xlabel("Residuals")
  plt.ylabel("Count")
  del temp1, temp2
  gc.collect()
def resid_qq(y_true, y_predicted):
    resids = np.subtract(np.array(y_true).reshape(-1,1),  np.array(y_predicted).reshape(-1,1))
    sps.probplot(resids.flatten(), plot = plt)
    plt.title('Q-Q Plot of residuals')
    plt.xlabel('Quantiles of standard Normal distribution')
    plt.ylabel('Quantiles of residuals')
    plt.show()<jupyter_output><empty_output><jupyter_text>Regularization<jupyter_code>def plot_regularization(l, train_RMSE, test_RMSE, coefs, min_idx, title):   
    plt.plot(l, test_RMSE, color = 'green', label = 'Test RMSE')
    plt.plot(l, train_RMSE, label = 'Train RMSE')    
    plt.axvline(min_idx, color = 'black', linestyle = '--')
    plt.legend()
    plt.xlabel('Regularization parameter')
    plt.ylabel('Root Mean Square Error')
    plt.title(title)
    plt.show()
    
    plt.plot(l, coefs)
    plt.axvline(min_idx, color = 'black', linestyle = '--')
    plt.title('Model coefficient values \n vs. regularizaton parameter')
    plt.xlabel('Regularization parameter')
    plt.ylabel('Model coefficient value')
    plt.show()
def test_regularization_l2(x_train, y_train, x_test, y_test, l2):
    train_RMSE = []
    test_RMSE = []
    coefs = []
    for reg in l2:
        lin_mod = Ridge(alpha = reg)
        lin_mod.fit(x_train, y_train)
        coefs.append(lin_mod.coef_)
        y_score_train = lin_mod.predict(x_train)
        train_RMSE.append(mean_squared_error(y_train, y_score_train)**0.5)
        y_score = lin_mod.predict(x_test)
        test_RMSE.append(mean_squared_error(y_test, y_score)**0.5)
    min_idx = np.argmin(test_RMSE)
    min_l2 = l2[min_idx]
    min_RMSE = test_RMSE[min_idx] 
    
    title = 'Train and test root mean square error \n vs. regularization parameter'
    plot_regularization(l2, train_RMSE, test_RMSE, coefs, min_l2, title)
    return min_l2, min_RMSE
def ridge(X_train,y_train, X_test, y_test):
  l2 = [x for x in range(0,50)]
  out_l2 = test_regularization_l2(X_train, y_train, X_test, y_test, l2)
  print(out_l2)
  lin_mod_l2 = Ridge(alpha = out_l2[0])
  lin_mod_l2.fit(X_train, y_train)
  y_score_l2 = lin_mod_l2.predict(X_test)
  print()
  reg_metrics(y_test, y_score_l2)
  resids_hist(np.array(y_test), np.array(y_score_l2))  
  resid_qq(np.array(y_test), np.array(y_score_l2)) 
  resid_plot(np.array(y_test), np.array(y_score_l2))
def test_regularization_l1(x_train, y_train, x_test, y_test, l1):
    train_RMSE = []
    test_RMSE = []
    coefs = []
    for reg in l1:
        lin_mod = Lasso(alpha = reg)
        lin_mod.fit(x_train, y_train)
        coefs.append(lin_mod.coef_)
        y_score_train = lin_mod.predict(x_train)
        train_RMSE.append(mean_squared_error(y_train, y_score_train)**0.5)
        y_score = lin_mod.predict(x_test)
        test_RMSE.append(mean_squared_error(y_test, y_score)**0.5)
    min_idx = np.argmin(test_RMSE)
    min_l1 = l1[min_idx]
    min_RMSE = test_RMSE[min_idx]
    
    title = 'Train and test root mean square error \n vs. regularization parameter'
    plot_regularization(l1, train_RMSE, test_RMSE, coefs, min_l1, title)
    return min_l1, min_RMSE
def lasso(X_train,y_train,X_test,y_test):
    l1 = [x/5000 for x in range(1,50)]
    out_l1 = test_regularization_l1(X_train, y_train, X_test, y_test, l1)
    print(out_l1)
    lin_mod_l1 = Lasso(alpha = out_l1[0])
    lin_mod_l1.fit(X_train, y_train)
    y_score_l1 = lin_mod_l1.predict(X_test)
    print()
    reg_metrics(y_test, y_score_l1) 
    resids_hist(np.array(y_test), np.array(y_score_l1))
    resid_qq(np.array(y_test), np.array(y_score_l1)) 
    resid_plot(np.array(y_test), np.array(y_score_l1))<jupyter_output><empty_output><jupyter_text>Data import<jupyter_code>train = pd.read_csv('/content/train.csv', parse_dates=['date'], index_col='Id')
test = pd.read_csv('/content/test.csv', parse_dates=['date'], index_col='Id')
region=pd.read_csv('/content/regions.csv')
train.head()<jupyter_output><empty_output><jupyter_text>Exploratory data analysis<jupyter_code>#train = reduce_mem_usage(train)
#test = reduce_mem_usage(test)
region
# Checking that the date column is monotonic
for r in train['region'].unique():
  print(train[train['region']==r]['date'].is_monotonic)
del r
train.info()
train.describe()
test.info()
test.describe()
region.info()<jupyter_output><empty_output><jupyter_text>Missing Values<jupyter_code>train.isna().sum()
msno.bar(train)
msno.bar(test)

# No missing values
temp = train.set_index('date')

for r in train['region'].unique():
  t = temp[temp['region']==r]
  print(msno.matrix(t, freq = 'A'))

del temp, t
gc.collect()
msno.heatmap(train)

# https://github.com/ResidentMario/missingno

# correlation heatmap measures nullity correlation: how strongly the presence or absence of one variable affects the presence of another
# Variables that are always full or always empty have no meaningful correlation, and so are silently removed from the visualization—in this case for instance the datetime and injury number columns, which are completely filled, are not included.
for r in train['region'].unique():
  t = train[train['region']==r]
  print(msno.heatmap(t))

del t
gc.collect()
msno.dendrogram(train)
for r in train['region'].unique():
  t = train[train['region']==r]
  print(msno.dendrogram(t))

del t
gc.collect()<jupyter_output><empty_output><jupyter_text>**_Missing values present in 4 columns, O3 ,PM10, PM25, NO2_**  Merging Region names with the rest of the data<jupyter_code>train1 = train.merge(region, left_on='region', right_on='Code')
#train1.drop(columns = ['region'],inplace=True)
test1 = test.merge(region, left_on='region', right_on='Code')
#test1.drop(columns = ['region'],inplace=True)<jupyter_output><empty_output><jupyter_text>univariate analysis<jupyter_code># Visualizing mortality_rate
def mr(df):
  plt.subplot(121)
  sns.distplot(df['mortality_rate']);

  plt.subplot(122)
  df['mortality_rate'].plot.box(figsize=(16,5))

  plt.show()

mr(train)
# Visualizing O3
def o3(df):
    plt.subplot(121)
    O3_1 = train['O3'].dropna()
    sns.distplot(O3_1);

    plt.subplot(122)
    O3_1.plot.box(figsize=(16,5))

    plt.show()
    plt.close()

o3(train)
# Visualizing PM10
def pm10(df):
    plt.subplot(121)
    PM10_1 = train['PM10'].dropna()
    sns.distplot(PM10_1);

    plt.subplot(122)
    PM10_1.plot.box(figsize=(16,5))

    plt.show()

pm10(train)
# Visualizing PM25

def pm25(df):
    plt.subplot(121)
    PM25_1 = train['PM25'].dropna()
    sns.distplot(PM25_1);

    plt.subplot(122)
    PM25_1.plot.box(figsize=(16,5))

    plt.show()

pm25(train)
# Visualizing NO2
def no2(df):
    plt.subplot(121)
    NO2_1 = train['NO2'].dropna()
    sns.distplot(NO2_1);

    plt.subplot(122)
    NO2_1.plot.box(figsize=(16,5))

    plt.show()
  
no2(train)
# Visualizing T2M

def t2m(df):
    plt.subplot(121)
    sns.distplot(train['T2M']);

    plt.subplot(122)
    train['T2M'].plot.box(figsize=(16,5))

    plt.show()

t2m(train)
# Visualizing categorical features
plt.subplot(231)
train['region'].value_counts(normalize=True).plot.bar(figsize=(20,10), title= 'region')
print('mortality rate across regions')
for r in train['region'].unique():
  t = train[train['region']==r]
  mr(t)
for r in train['region'].unique():
  t = train[train['region']==r]
  print(o3(t))
for r in train['region'].unique():
  t = train[train['region']==r]
  print(no2(t))
for r in train['region'].unique():
  t = train[train['region']==r]
  print(pm10(t))
for r in train['region'].unique():
  t = train[train['region']==r]
  print(pm25(t))
for r in train['region'].unique():
  t = train[train['region']==r]
  t2m(t)
#train1.groupby("region")["Region"].unique()
train1.head()<jupyter_output><empty_output><jupyter_text>Transformations  Bivariate analysis - Region Wise  variation with date - time series plot   For region 1 - North East<jupyter_code>train_r1 = train1[train1["Region"]=="North East"]
fig1 = px.line(train_r1, x='date', y='mortality_rate')
fig1.show()
fig2 = go.Figure([go.Scatter(x=train_r1['date'], y=train_r1['mortality_rate'])])
#fig2.show()

fig2.update_layout(title_text='Time Series with Rangeslider',
                  xaxis_rangeslider_visible=True)
fig3 = go.Figure([go.Scatter(x=train_r1['date'], y=train_r1["O3"])])
#fig2.show()

fig3.update_layout(title_text='Time Series with Rangeslider',
                  xaxis_rangeslider_visible=True)
fig2 = go.Figure([go.Scatter(x=train_r1['date'], y=train_r1["PM10"])])
#fig2.show()

fig2.update_layout(title_text='Time Series with Rangeslider',
                  xaxis_rangeslider_visible=True)
fig4 = go.Figure([go.Scatter(x=train_r1['date'], y=train_r1["PM25"])])
#fig2.show()

fig4.update_layout(title_text='Time Series with Rangeslider',
                  xaxis_rangeslider_visible=True)
fig5 = go.Figure([go.Scatter(x=train_r1['date'], y=train_r1["NO2"])])
#fig2.show()

fig5.update_layout(title_text='Time Series with Rangeslider',
                  xaxis_rangeslider_visible=True)<jupyter_output><empty_output><jupyter_text>scatter plot of PM25 with other varibles<jupyter_code>plt.scatter(x=train_r1['mortality_rate'], y=train_r1["PM25"])
plt.scatter(x=train_r1['PM10'], y=train_r1["PM25"])
plt.scatter(x=train_r1['O3'], y=train_r1["PM25"])
plt.scatter(x=train_r1['NO2'], y=train_r1["PM25"])
plt.scatter(x=train_r1['T2M'], y=train_r1["PM25"])<jupyter_output><empty_output><jupyter_text>scatter plot of NO2 with other variables<jupyter_code>plt.scatter(x=train_r1['mortality_rate'], y=train_r1["NO2"])
plt.scatter(x=train_r1['O3'], y=train_r1["NO2"])
plt.scatter(x=train_r1['PM10'], y=train_r1["NO2"])
plt.scatter(x=train_r1['PM25'], y=train_r1["NO2"])
plt.scatter(x=train_r1['T2M'], y=train_r1["NO2"])<jupyter_output><empty_output><jupyter_text>Missing value imputation using MICE   For Region1 - North East<jupyter_code>train_r1.head()
train_r1.isna().sum()
train_r1.drop(columns=["region","date", "Code", "Region"], inplace=True)
from impyute.imputation.cs import mice

# start the MICE training
train_r1_imp=mice(train_r1.values)
train_r1_imp = pd.DataFrame(train_r1_imp, columns = ["mortality_rate", "O3", "PM10", "PM25","NO2","T2M"])
train_r1_imp.isna().sum()
# Visualizing NO2
# plt.figure(1)
plt.subplot(121)
NO2_1 = train_r1_imp['NO2'].dropna()
sns.distplot(NO2_1);

plt.subplot(122)
NO2_1.plot.box(figsize=(16,5))

plt.show()


plt.subplot(121)
NO2_1 = train_r1['NO2'].dropna()
sns.distplot(NO2_1);

plt.subplot(122)
NO2_1.plot.box(figsize=(16,5))

plt.show()
plt.scatter(x=train_r1_imp['O3'], y=train_r1_imp["NO2"])
plt.scatter(x=train_r1['O3'], y=train_r1["NO2"])
plt.scatter(x=train_r1_imp['PM10'], y=train_r1_imp["NO2"])
plt.scatter(x=train_r1['PM10'], y=train_r1["NO2"])
# Visualizing NO2
# plt.figure(1)
plt.subplot(121)
NO2_1_imp = train_r1_imp['PM25'].dropna()
NO2_1 = train_r1['PM25'].dropna()
#sns.distplot(NO2_1_imp)
#sns.distplot(NO2_1)

sns.distplot(NO2_1_imp, hist=False, rug=True)
sns.distplot(NO2_1, hist=False, rug=True)

plt.scatter(x=train_r1_imp['PM10'], y=train_r1_imp["PM25"])
plt.scatter(x=train_r1['PM10'], y=train_r1["PM25"])
train_r1_imp['PM25'].describe(), train_r1['PM25'].describe()
## changing negative values to min value
train_r1_imp['PM25'] = np.where(train_r1_imp['PM25']<=0, 0.904000, train_r1_imp['PM25'] )
train_r1_imp['NO2'].describe(), train_r1['NO2'].describe()
## changing negative values to min value
train_r1_imp['NO2'] = np.where(train_r1_imp['NO2']<=0, 1.104000, train_r1_imp['NO2'] )<jupyter_output><empty_output><jupyter_text>Since we are getting good enough results region wise we can impute PM25 and NO2 on the basis of regions  Region wise imputation<jupyter_code>train1.head()
regions = list(region["Region"].values)
train_imp = pd.DataFrame()

for reg in regions:
    
    train_r = train1[train1["Region"] == reg].reset_index(drop=True)
    train_r_imp = mice(train_r.drop(["region","date", "Code", "Region"], axis = 1).values)
    train_r_imp = pd.DataFrame(train_r_imp, columns = ["mortality_rate", "O3", "PM10", "PM25","NO2","T2M"])
    
    ## changing negative values to min value
    train_r_imp['PM25'] = np.where(train_r_imp['PM25']<=0, min(train_r['PM25']), train_r_imp['PM25'] )
    
    ## changing negative values to min value
    train_r_imp['NO2'] = np.where(train_r_imp['NO2']<=0, min(train_r['NO2']), train_r_imp['NO2'] )
    
    print("\n\n")
    print("train_r = before", train_r.isna().sum())
    
    train_r['PM25'] = train_r_imp['PM25']
    train_r['NO2'] = train_r_imp['NO2']
    train_r['O3'] = train_r_imp['O3']
    train_r['PM10'] = train_r_imp['PM10']
    
    print("train_r_imp = \n", train_r_imp.isna().sum())
    print("train_r = \n", train_r.isna().sum())
    print("----------------------------------------")
    
    train_imp = train_imp.append(train_r)
    
print("train_imp = ", train_imp.isna().sum(), '\n')
    
    
pd.set_option('display.max_rows', None)
#train_imp
                                    
train_imp.head()
temp_r_imp = mice(train_imp.drop(columns=['region','date','Code','Region']).values)
temp_r_imp = pd.DataFrame(temp_r_imp, columns = ["mortality_rate", "O3", "PM10", "PM25","NO2","T2M"])

train_r_imp_full = train_imp.copy()
train_r_imp_full["NO2"] = temp_r_imp["NO2"]
train_r_imp_full["PM25"] = temp_r_imp["PM25"]
train_r_imp_full["O3"] = temp_r_imp["O3"]
train_r_imp_full["PM10"] = temp_r_imp["PM10"]

train_r_imp_full.isna().sum()  #### FINAL DATAFRAME WITH IMPUTED VALUES REGION WISE
train_r_imp_full.shape, train1.shape
train_r_imp_full.head()
train_r_imp_full['NO2'].describe()
train1['NO2'].describe()
train_r_imp_full['PM25'].describe()
train1['PM25'].describe()
plt.subplot(121)
NO2_1= train1['PM25'].dropna()
NO2_1_imp  = train_r_imp_full['PM25'].dropna()

sns.distplot(NO2_1_imp, hist=False, rug=True)
sns.distplot(NO2_1, hist=False, rug=True)<jupyter_output><empty_output><jupyter_text>Region wise imputation is fine but at an overall level it does not suits well..  Impution on full data - MICE<jupyter_code># start the MICE training
temp_imp = mice(train1.drop(["region","date", "Code", "Region"], axis = 1).values)
temp_imp = pd.DataFrame(temp_imp, columns = ["mortality_rate", "O3", "PM10", "PM25","NO2","T2M"])

train_full_imp = train1.copy()
train_full_imp["NO2"] = temp_imp["NO2"]
train_full_imp["PM25"] = temp_imp["PM25"]
train_full_imp["O3"] = temp_imp["O3"]
train_full_imp["PM10"] = temp_imp["PM10"]
train_full_imp.isna().sum() ### FINAL DATAFRAME WITH IMPUTATION ON FULL DATA
plt.subplot(121)
NO2_1= train1['PM25'].dropna()
NO2_1_imp  = train_full_imp['PM25'].dropna()

sns.distplot(NO2_1_imp, hist=False, rug=True)
sns.distplot(NO2_1, hist=False, rug=True)<jupyter_output><empty_output><jupyter_text>Dropping Missing Values  Feature Engineering one hot encoding for regions<jupyter_code>dummy1= pd.get_dummies(train['region'])
train_f = train.join(dummy1)

dummy2= pd.get_dummies(test['region'])
test_f = test.join(dummy2)
train_f.shape,test_f.shape<jupyter_output><empty_output><jupyter_text>doy (day of year) and year<jupyter_code>train_f['year'] = train_f['date'].map(lambda x: x.year)
test_f['year'] = test_f['date'].map(lambda x: x.year)

train_f['year^2'] = train_f['year']**2
test_f['year^2'] = test_f['year']**2

train_f['doy'] = train_f['date'].map(lambda x: x.dayofyear)
test_f['doy'] = test_f['date'].map(lambda x: x.dayofyear)

train_f['doy^2'] = train_f['doy']**2
test_f['doy^2'] = test_f['doy']**2<jupyter_output><empty_output><jupyter_text>Log Transformation<jupyter_code>train_f['NO2_log']=np.log(train_f['NO2'])
train_f['PM10_log']=np.log(train_f['PM10'])
train_f['PM25_log']=np.log(train_f['PM25'])

test_f['NO2_log']=np.log(test_f['NO2'])
test_f['PM10_log']=np.log(test_f['PM10'])
test_f['PM25_log']=np.log(test_f['PM25'])
#train_f.info(),test_f.info()
total_f=total=pd.concat([train_f,test_f],sort=False)
total_f['T2M_rolling'] = total_f.groupby('region')['T2M'].transform(lambda x: x.rolling(30).mean())
total_f.shape<jupyter_output><empty_output><jupyter_text>Modelling<jupyter_code>train_f=total_f[:18403].dropna()
test_f=total_f[18403:].drop(['mortality_rate'],axis=1)
train_f.shape,test_f.shape
X_train, X_test, y_train, y_test = train_test_split(train_f.drop(['mortality_rate'],axis=1), train_f['mortality_rate'], test_size=0.2)
RMSE= make_scorer(metrics.mean_squared_error)
X=train_f.drop(['mortality_rate'],axis=1)
y=train_f['mortality_rate']<jupyter_output><empty_output><jupyter_text>Predictor Set 1<jupyter_code>predictors=['O3','PM10','PM25','NO2','T2M']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>CV Model<jupyter_code>models = [
    LinearRegression(),
    linear_model.Lasso(alpha=0.0098),
    Ridge(alpha=49),
    RandomForestRegressor(max_depth=2, random_state=0),
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>Predictor Set 2<jupyter_code>predictors=['O3','PM10','PM25','NO2_log','T2M']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)
models = [
    LinearRegression(),
    linear_model.Lasso(alpha=0.002),
    Ridge(alpha=25),
    RandomForestRegressor(max_depth=2, random_state=0),
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>-LinearRegression      0.2453372337810475 -Lasso                 0.25363080541258476  -Ridge                 0.24533682390299894 -RandomForestRegressor 0.253308657930002    -LB socre-LinearRegression  0.   -Lasso             0. -Ridge             0. -RandomForestRegressor 0.   Predictor Set 3<jupyter_code>predictors=['O3','PM10','PM25_log','NO2_log','T2M']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)
models = [
    LinearRegression(),
    Lasso(alpha=0.0014),
    Ridge(alpha=49),
    RandomForestRegressor(max_depth=2)
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>-LinearRegression      0.24551812699677633 -Lasso                 0.25364131894103525  -Ridge                 0.2455161592022304     -RandomForestRegressor 0.253308657930002    -LB socre-LinearRegression  0. -Lasso             0.  -Ridge             0.  -RandomForestRegressor 0.   Predictor Set 4<jupyter_code>predictors=['O3','PM10_log','PM25_log','NO2_log','T2M']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)
models = [
    LinearRegression(),
    Lasso(alpha=0.0006),
    Ridge(alpha=49),
    RandomForestRegressor(max_depth=2, random_state=0),
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>-LinearRegression      0.24552977824727984 -Lasso                 0.2536717498662236   -Ridge                 0.2455272255973945      -RandomForestRegressor 0.253308657930002    -LB socre-LinearRegression  0. -Lasso             0. -Ridge              0.  -RandomForestRegressor 0.   Predictor Set 5<jupyter_code>predictors=['O3','PM10_log','PM25_log','NO2_log','T2M_rolling']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)
models = [
    LinearRegression(),
    Lasso(alpha=0.0014),
    Ridge(alpha=49),
    RandomForestRegressor(max_depth=2, random_state=0),
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>-LinearRegression      0.22796036576314532 -Lasso                 0.24235147415525443   -Ridge                 0.2279576652270535  -RandomForestRegressor 0.2460673252607887  -LB score-LinearRegression  0.  -Lasso             0.-Ridge              0.  -RandomForestRegressor 0.   Predictor Set 6<jupyter_code>predictors=['O3','PM10_log','PM25_log','NO2_log','T2M_rolling','year','year^2','doy','doy^2']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)
models = [
    LinearRegression(),
    Lasso(alpha=0.008),
    Ridge(alpha=1.0),
    RandomForestRegressor(max_depth=2),
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>-LinearRegression      0.25691874212490495  -Lasso                 0.24943533679256355  -Ridge                 0.23111601199135615     -RandomForestRegressor 0.2435420386323374   -LB score-LinearRegression  0.  -Lasso             0.-Ridge              0.  -RandomForestRegressor 0.   Predictor Set 7<jupyter_code>predictors=['T2M_rolling', 'year', 'year^2', 'doy', 'doy^2','E12000001', 'E12000002', 'E12000003', 'E12000004','E12000005', 'E12000006', 'E12000007', 'E12000008']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)
models = [
    LinearRegression(),
    Lasso(alpha=0.006),
    Ridge(alpha=1.0),
    RandomForestRegressor(max_depth=2, random_state=0),
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>Best Predictors<jupyter_code>predictors1=['NO2_log', 'year', 'year^2', 'doy', 'doy^2','PM10_log']
predictors2=['T2M_rolling', 'year', 'year^2', 'doy', 'doy^2','E12000001', 'E12000002', 'E12000003', 'E12000004','E12000005', 'E12000006', 'E12000007', 'E12000008']
predictors3=['E12000001', 'E12000002', 'E12000003', 'E12000004','E12000005', 'E12000006', 'E12000007', 'E12000008']
pred = [
    predictors1,
    predictors2,
    predictors3,
]
for preds in pred:
    #model_name = model.__class__.__name__
    
    accuracies = cross_val_score(LinearRegression(),X[preds], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>Final Algo - mixed LR Notation:X0 - training predictorsy0 - training targetX1 - test predictorsy1 - predictions<jupyter_code>a = LinearRegression()
b = LinearRegression()
c = LinearRegression()
ab = LinearRegression()

y0 = train_f['mortality_rate']



#modela
a_predictors = ['NO2_log', 'year', 'year^2', 'doy', 'doy^2','PM10_log']
a_X0 = train_f[a_predictors]
a.fit(a_X0, y0)



#modelb
b_predictors = [
        'T2M_rolling', 'year', 'year^2', 'doy', 'doy^2',
        'E12000001', 'E12000002', 'E12000003', 'E12000004',
        'E12000005', 'E12000006', 'E12000007', 'E12000008'
        ]
b_X0 = train_f[b_predictors]
b.fit(b_X0, y0)



#modelc
c_predictors = [
        'E12000001', 'E12000002', 'E12000003', 'E12000004',
        'E12000005', 'E12000006', 'E12000007', 'E12000008'
        ]
c_X0 = train_f[c_predictors]
c.fit(c_X0, y0)


region_effect_reduction = 0.85


# Model AB
# Produces the final solution. Combines A and "reduced B" by using their predictions as features for a linear regression.

ab_X0 = pd.DataFrame({
    'A': a.predict(a_X0),
    'reduced B': b.predict(b_X0) - (c.predict(c_X0) * region_effect_reduction)
    })

ab.fit(ab_X0, y0)

# predict

a_X1 = test_f[a_predictors]
b_X1 = test_f[b_predictors]
c_X1 = test_f[c_predictors]

ab_X1 = pd.DataFrame({'A': a.predict(a_X1),'reduced B': b.predict(b_X1) - (c.predict(c_X1) * region_effect_reduction)})

ab_y1 = ab.predict(ab_X1)



predictions = pd.DataFrame(index=test_f.index)
#predictions['mortality_rate'] =  a.predict(a_X1)
predictions['mortality_rate'] = ab_y1
predictions.to_csv('final_0.0.csv')<jupyter_output><empty_output><jupyter_text>Resultsbest LB socre for -(region_effect_reduction = 0.85)private score - 0.28948public score - 0.28784benchmark -private score - 0.29022public socre - 0.29259<jupyter_code><jupyter_output><empty_output><jupyter_text>MICE Imputation on Full Data  Feature Engineering<jupyter_code>train_full_imp.head()
test1.head()<jupyter_output><empty_output><jupyter_text>one hot encoding for regions<jupyter_code>dummy1= pd.get_dummies(train_full_imp['region'])
train_f = train_full_imp.join(dummy1)
train_f.drop(columns='Region',inplace=True)

dummy2= pd.get_dummies(test['region'])
test_f = test.join(dummy2)
train_f.shape,test_f.shape<jupyter_output><empty_output><jupyter_text>doy (day of year) and year<jupyter_code>train_f['year'] = train_f['date'].map(lambda x: x.year)
test_f['year'] = test_f['date'].map(lambda x: x.year)

train_f['year^2'] = train_f['year']**2
test_f['year^2'] = test_f['year']**2

train_f['doy'] = train_f['date'].map(lambda x: x.dayofyear)
test_f['doy'] = test_f['date'].map(lambda x: x.dayofyear)

train_f['doy^2'] = train_f['doy']**2
test_f['doy^2'] = test_f['doy']**2
def f(x):
  if (x<=0):
    return(np.nan)
  else:
    return(x)
train_f['NO2']=train_f['NO2'].apply(lambda x: f(x))
train_f['PM25']=train_f['PM25'].apply(lambda x: f(x))
train_f.isna().sum()
train_f.ffill(axis=0, inplace=True)<jupyter_output><empty_output><jupyter_text>Log Transformation<jupyter_code>train_f['NO2_log']=np.log(train_f['NO2'])
train_f['PM10_log']=np.log(train_f['PM10'])
train_f['PM25_log']=np.log(train_f['PM25'])

test_f['NO2_log']=np.log(test_f['NO2'])
test_f['PM10_log']=np.log(test_f['PM10'])
test_f['PM25_log']=np.log(test_f['PM25'])
#train_f.info(),test_f.info()
total_f=total=pd.concat([train_f,test_f],sort=False)
total_f['T2M_rolling'] = total_f.groupby('region')['T2M'].transform(lambda x: x.rolling(30).mean())
total_f.shape
total_f.isna().sum()<jupyter_output><empty_output><jupyter_text>Modelling<jupyter_code>train_f=total_f[:18403].dropna()
test_f=total_f[18403:].drop(['mortality_rate'],axis=1)
train_f.shape,test_f.shape
X_train, X_test, y_train, y_test = train_test_split(train_f.drop(['mortality_rate'],axis=1), train_f['mortality_rate'], test_size=0.2)
RMSE= make_scorer(metrics.mean_squared_error)
X=train_f.drop(['mortality_rate'],axis=1)
y=train_f['mortality_rate']<jupyter_output><empty_output><jupyter_text>Predictor Set 1<jupyter_code>predictors=['O3','PM10','PM25','NO2','T2M']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>CV Model<jupyter_code>models = [
    LinearRegression(),
    linear_model.Lasso(alpha=0.002),
    Ridge(alpha=49),
    RandomForestRegressor(max_depth=2),
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>Predictor Set 2<jupyter_code>predictors=['O3','PM10','PM25','NO2_log','T2M']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)
models = [
    LinearRegression(),
    linear_model.Lasso(alpha=0.002),
    Ridge(alpha=25),
    RandomForestRegressor(max_depth=2, random_state=0),
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>-LinearRegression      0.2453372337810475 -Lasso                 0.25363080541258476  -Ridge                 0.24533682390299894 -RandomForestRegressor 0.253308657930002    -LB socre-LinearRegression  0.   -Lasso             0. -Ridge             0. -RandomForestRegressor 0.   Predictor Set 3<jupyter_code>predictors=['O3','PM10','PM25_log','NO2_log','T2M']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)
models = [
    LinearRegression(),
    Lasso(alpha=0.0014),
    Ridge(alpha=49),
    RandomForestRegressor(max_depth=2)
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>-LinearRegression      0.24551812699677633 -Lasso                 0.25364131894103525  -Ridge                 0.2455161592022304     -RandomForestRegressor 0.253308657930002    -LB socre-LinearRegression  0. -Lasso             0.  -Ridge             0.  -RandomForestRegressor 0.   Predictor Set 4<jupyter_code>predictors=['O3','PM10_log','PM25_log','NO2_log','T2M']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)
models = [
    LinearRegression(),
    Lasso(alpha=0.0006),
    Ridge(alpha=49),
    RandomForestRegressor(max_depth=2, random_state=0),
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>-LinearRegression      0.24552977824727984 -Lasso                 0.2536717498662236   -Ridge                 0.2455272255973945      -RandomForestRegressor 0.253308657930002    -LB socre-LinearRegression  0. -Lasso             0. -Ridge              0.  -RandomForestRegressor 0.   Predictor Set 5<jupyter_code>predictors=['O3','PM10_log','PM25_log','NO2_log','T2M_rolling']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)
models = [
    LinearRegression(),
    Lasso(alpha=0.0014),
    Ridge(alpha=49),
    RandomForestRegressor(max_depth=2, random_state=0),
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>-LinearRegression      0.22796036576314532 -Lasso                 0.24235147415525443   -Ridge                 0.2279576652270535  -RandomForestRegressor 0.2460673252607887  -LB score-LinearRegression  0.  -Lasso             0.-Ridge              0.  -RandomForestRegressor 0.   Predictor Set 6<jupyter_code>predictors=['O3','PM10_log','PM25_log','NO2_log','T2M_rolling','year','year^2','doy','doy^2']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)
models = [
    LinearRegression(),
    Lasso(alpha=0.008),
    Ridge(alpha=1.0),
    RandomForestRegressor(max_depth=2),
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>-LinearRegression      0.25691874212490495  -Lasso                 0.24943533679256355  -Ridge                 0.23111601199135615     -RandomForestRegressor 0.2435420386323374   -LB score-LinearRegression  0.  -Lasso             0.-Ridge              0.  -RandomForestRegressor 0.   Predictor Set 7<jupyter_code>predictors=['T2M_rolling', 'year', 'year^2', 'doy', 'doy^2','E12000001', 'E12000002', 'E12000003', 'E12000004','E12000005', 'E12000006', 'E12000007', 'E12000008']<jupyter_output><empty_output><jupyter_text>Regularized Linear Regression without CV  Ridge Regression<jupyter_code>ridge(X_train[predictors],y_train,X_test[predictors],y_test)<jupyter_output><empty_output><jupyter_text>Lasso Regression<jupyter_code>lasso(X_train[predictors],y_train,X_test[predictors],y_test)
models = [
    LinearRegression(),
    Lasso(alpha=0.006),
    Ridge(alpha=1.0),
    RandomForestRegressor(max_depth=2, random_state=0),
]
CV = 5
for model in models:
    model_name = model.__class__.__name__
    
    accuracies = cross_val_score(model,X[predictors], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>Best Predictors<jupyter_code>predictors1=['NO2_log', 'year', 'year^2', 'doy', 'doy^2','PM10_log']
predictors2=['T2M_rolling', 'year', 'year^2', 'doy', 'doy^2','E12000001', 'E12000002', 'E12000003', 'E12000004','E12000005', 'E12000006', 'E12000007', 'E12000008']
predictors3=['E12000001', 'E12000002', 'E12000003', 'E12000004','E12000005', 'E12000006', 'E12000007', 'E12000008']
pred = [
    predictors1,
    predictors2,
    predictors3,
]
for preds in pred:
    #model_name = model.__class__.__name__
    
    accuracies = cross_val_score(LinearRegression(),X[preds], y,scoring=RMSE, cv=CV)
    
    print(model_name,"RMSE",np.sqrt(accuracies),"\n")
    print("average RMSE :", np.sqrt(accuracies).mean(),"\n","\n")<jupyter_output><empty_output><jupyter_text>Final Algo - mixed LR Notation:X0 - training predictorsy0 - training targetX1 - test predictorsy1 - predictions<jupyter_code>a = LinearRegression()
b = LinearRegression()
c = LinearRegression()
ab = LinearRegression()

y0 = train_f['mortality_rate']



#modela
a_predictors = ['NO2_log', 'year', 'year^2', 'doy', 'doy^2','PM10_log']
a_X0 = train_f[a_predictors]
a.fit(a_X0, y0)



#modelb
b_predictors = [
        'T2M_rolling', 'year', 'year^2', 'doy', 'doy^2',
        'E12000001', 'E12000002', 'E12000003', 'E12000004',
        'E12000005', 'E12000006', 'E12000007', 'E12000008'
        ]
b_X0 = train_f[b_predictors]
b.fit(b_X0, y0)



#modelc
c_predictors = [
        'E12000001', 'E12000002', 'E12000003', 'E12000004',
        'E12000005', 'E12000006', 'E12000007', 'E12000008'
        ]
c_X0 = train_f[c_predictors]
c.fit(c_X0, y0)


region_effect_reduction = 0.85


# Model AB
# Produces the final solution. Combines A and "reduced B" by using their predictions as features for a linear regression.

ab_X0 = pd.DataFrame({
    'A': a.predict(a_X0),
    'reduced B': b.predict(b_X0) - (c.predict(c_X0) * region_effect_reduction)
    })

ab.fit(ab_X0, y0)

# predict

a_X1 = test_f[a_predictors]
b_X1 = test_f[b_predictors]
c_X1 = test_f[c_predictors]

ab_X1 = pd.DataFrame({'A': a.predict(a_X1),'reduced B': b.predict(b_X1) - (c.predict(c_X1) * region_effect_reduction)})

ab_y1 = ab.predict(ab_X1)



predictions = pd.DataFrame(index=test_f.index)
#predictions['mortality_rate'] =  a.predict(a_X1)
predictions['mortality_rate'] = ab_y1
predictions.index.name = 'Id'
predictions.to_csv('final_mice.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15438941115944063, 'fraction_numerical': 0.05872035919741827, 'mean_word_length': 5.4792424242424245, 'pattern_counts': {'":': 0, '<': 299, '<?xml version=': 0, '>': 295, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '72', 'dir': 'jupyter-structured-clean-dedup', 'id': '432936', 'license': 'MIT', 'n_tokens_mistral': 18269, 'n_tokens_neox': 16694, 'n_words': 2413, 'path': 'assets/DSL_pb_V3.1.ipynb', 'repo_name': 'VHKA1729/air-pollution', 'size': '42794'}