<jupyter_start><jupyter_text>Functions<jupyter_code>def find_correlation(data, threshold=0.9):
    corr_mat = data.corr()
    corr_mat.loc[:, :] = np.tril(corr_mat, k=-1)
    already_in = set()
    result = []
    for col in corr_mat:
        perfect_corr = corr_mat[col][abs(corr_mat[col])> threshold].index.tolist()
        if perfect_corr and col not in already_in:
            already_in.update(set(perfect_corr))
            perfect_corr.append(col)
            result.append(perfect_corr)
    select_nested = [f[1:] for f in result]
    select_flat = [i for j in select_nested for i in j]
    return select_flat
def generate_stat(X_stat):
    for names in X_stat.drop(columns=['city','year','weekofyear','week_start_date']):
        X_stat['month_avg_'+names] = X_stat[names].rolling(3 ,min_periods=1).mean()
        X_stat['month_min_'+names] = X_stat[names].rolling(3 ,min_periods=1).min()
        X_stat['month_max_'+names] = X_stat[names].rolling(3 ,min_periods=1).max()
        X_stat['lag_3_'+names] = X_stat[names].shift(2)
    return X_stat
def PP_X(X1):
    X1 = X1[(X1['city']=='sj')]
    le = preprocessing.LabelEncoder()
    X1['city'] = le.fit_transform(X1['city'])
    X1['week_start_date'] = pd.to_datetime(X1['week_start_date'])
    X1['ndvi_s'] = X1['ndvi_se'] + X1['ndvi_sw']
    X1['ndvi_n'] = X1["ndvi_ne"] + X1['ndvi_nw']
    X1 = generate_stat(X1)
#     X1['month'] = pd.DatetimeIndex(X1['week_start_date']).month
    X1 = X1.fillna(method = 'bfill')

    X1 = X1.drop(
        columns=[
            'weekofyear', 
            'week_start_date',
            'city',
            'year'
        ]
    )
    
    return X1
def PP_Y(Y1):
    Y1 = Y1[(Y1['city']=='sj')]
    return Y1<jupyter_output><empty_output><jupyter_text>Read and clean data<jupyter_code>X = pd.read_csv("dengue_features_train.csv")
Y = pd.read_csv("dengue_labels_train.csv")
X_New = pd.read_csv("dengue_features_test.csv")
Y_New = pd.read_csv('submission_format.csv')
X , X_New = PP_X(X) , PP_X(X_New)
Y , Y_New = PP_Y(Y) , PP_Y(Y_New)<jupyter_output><empty_output><jupyter_text>Principal Componant Analysis<jupyter_code>pca = PCA(n_components=10)
X2 = pd.DataFrame(pca.fit_transform(X))
X = pd.concat([X,X2],axis=1)
X_New2 = pd.DataFrame(pca.transform(X_New))
X_New = pd.concat([X_New,X_New2],axis=1)<jupyter_output><empty_output><jupyter_text>Drop Highly Correlated Data<jupyter_code>columns_to_drop = find_correlation(X_New , 0.9)
X = X.drop(columns=columns_to_drop)
X_New = X_New.drop(columns=columns_to_drop)<jupyter_output><empty_output><jupyter_text>Rmove Outliers<jupyter_code>X , Y = X.loc[Y['total_cases'] <150] , Y.loc[Y['total_cases'] <150]
Xrf = X
Yrf = Y['total_cases'].values.ravel()
X_testrf = X_New<jupyter_output><empty_output><jupyter_text>Random Split<jupyter_code>X_train, X_test, Y_train, Y_test = train_test_split(Xrf,Yrf ,test_size=0.2)<jupyter_output><empty_output><jupyter_text>Linear Split<jupyter_code>X_train, X_test, Y_train, Y_test = Xrf[:int(len(X)*0.7)], Xrf[int(len(X)*0.7):] , Yrf[:int(len(Y)*0.7)] , Yrf[int(len(X)*0.7):]
rf = RandomForestRegressor()
model1 = rf.fit(X_train,Y_train)
predict = rf.predict(X_test)                                              
print(mean_absolute_error(Y_test, predict.round(0).astype(int)))
predict_RF = model1.predict(X_testrf)
# mode12 = rf.fit(Xrf,Yrf)<jupyter_output>16.093406593406595<jupyter_text>Variable Importance<jupyter_code>plt.figure(figsize=(20,15))
importance = rf.feature_importances_
feat_importances_act = pd.Series(importance, index=X_train.columns)
feat_importances = feat_importances_act.nlargest(20)
feat_importances.plot(kind='barh')<jupyter_output><empty_output><jupyter_text>linear regresion model<jupyter_code>reg = linear_model.Ridge (alpha = 0.5)
reg.fit(X_train[feat_importances_act.nlargest(4).keys()],Y_train)
predict2 = reg.predict(X_test[feat_importances_act.nlargest(4).keys()])     
print(mean_absolute_error(Y_test, [max(x,0) for x in predict2.round(0).astype(int)]))  
predict_lnr = reg.predict(X_testrf[feat_importances_act.nlargest(4).keys()])
predict_lnr= [max(x,0) for x in predict_lnr]
Y_New['total_cases'] = np.array(predict_lnr, dtype=np.float32).round(0)
Y_New['total_cases'] = Y_New['total_cases'].astype(int)
Y_New.to_csv('submission_sj.csv')
X_crr = X.copy(deep=True)
X_crr['total_cases'] = Y['total_cases']
corr = X_crr.corr()
corr.total_cases.drop('total_cases').sort_values(ascending=False).plot.barh()
plt.figure(figsize=(10, 6))
sns.heatmap(corr, 
        xticklabels=corr.columns,
        yticklabels=corr.columns)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15929009640666084, 'fraction_numerical': 0.024101665205959685, 'mean_word_length': 5.340277777777778, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '522551', 'license': 'MIT', 'n_tokens_mistral': 1925, 'n_tokens_neox': 1857, 'n_words': 228, 'path': 'DengAI_sj_PP.ipynb', 'repo_name': 'JMIsham/UOMCSE_SMTT', 'size': '4564'}