<jupyter_start><jupyter_text>Load in all projects from project panel, and merge with MDG panel data<jupyter_code>ipf_feature_cols = [
    'origcommamt',
    'lnorigcommamt',
    'cpia_approval',
    'region',
    'fundingsource', 
    'fcsatappfy',
    'uppermiddle_income_appfy',
    'origprojlength',
    'prepttl_exp',
    'lngdp_ppp',
    'iegrate' # not strictly "at approval" but important to understand relative importance, so including
]
during_project_features = [
    'cpia_change', 
    'gdpgrowthav', 
    'iegrate', 
    'iegsat', 
    'actprojlength',
    'changeprepttlisr12',
    'oneh_ttl_turnover',
    'twoh_ttl_turnover',
]
country_panel, approv_df, review_df = project_util.load_projects_with_embeddings(
    ipf_feature_cols=ipf_feature_cols,
    during_project_features=during_project_features
)
icr_embeds = pd.read_pickle("../data/transformed_data/icr_report_embeddings.pkl")
review_df = review_df.merge(icr_embeds[['project', 'umap_1', 'umap_2']], left_on='id', right_on='project')
review_df = review_df.rename(columns={ 'umap_1': 'icr_embed_x_umap', 'umap_2': 'icr_embed_y_umap' })
review_df.plot.scatter(x='icr_embed_x_umap', y='icr_embed_y_umap')
plt.show()<jupyter_output><empty_output><jupyter_text>Primary data complete, now extract the features we will work with<jupyter_code>basic_features = [
     'id', 
     'regionname', 
     'project_name', 
     'pdo', 
     'boardapprovaldate', 
     'closingdate', 
     'closingyear', 
     'project_country', 
     'panel_country',
]

loan_features = [
    'financing_type',
    'financing_instr',
    'sector_percentage', # = percentage in primary sector
    'number_sectors',
    'practice_type_code',
    'n_practices',
    'gp_percentage',
    'hhi'
] 

sector_features = [
    'sector1', 'sector2', 'sector3', 'sector4', 'sector5', 'theme1', 'theme2'
]

cat_features = [
    'impagency',
    'cons_serv_reqd_ind',
    'supplementprojectflg',
    'prodlinetext',
    'parent_sector_name'
]

# embed_features = embed_cols
embed_features = [
    'pdo_embed_x_umap',
    'pdo_embed_y_umap',
    'dli_embed_x_umap',
    'dli_embed_y_umap'
]

review_embeds = [
    'icr_embed_x_umap',
    'icr_embed_y_umap'
]
common_features = basic_features + loan_features + sector_features + cat_features + embed_features
approv_features = [col for col in ipf_feature_cols if col not in common_features] + common_features
review_features = [col for col in during_project_features if col not in common_features] + review_embeds + common_features
approv_data = project_util.assemble_input_df(approv_df, approv_features, country_panel)
review_data = project_util.assemble_input_df(review_df, review_features, country_panel)<jupyter_output><empty_output><jupyter_text>Sector coding, running models @ project start<jupyter_code># remove these (if still present) as they have trivial importance and are expanding the feature space without gain
trivial_imp_features = [
    'cons_serv_reqd_ind', 
    'impagency',
    'prodlinetext',
    'supplementprojectflg'
]<jupyter_output><empty_output><jupyter_text>Approval data, default CLF + REG<jupyter_code>approv_feature_cols = loan_features + cat_features + ['pdo_length'] + embed_features + ipf_feature_cols
approv_feature_cols = [col for col in approv_feature_cols if col not in trivial_imp_features]

approv_input_df, approv_health_results, approv_edu_results = project_util.construct_residual_df(
    data=approv_data,
    project_feature_cols=approv_feature_cols
)
random_seed = 42
clf_default = RandomForestClassifier(min_samples_leaf=5)

# getting overfitting badly, so start pruning
cols_to_exclude = [
    "uppermiddle_income_appfy", # seems to not be adding much vs ln gdp per cap
    "focused_project", # shown above to not matter much
    "scattered_project", # as above,
    "gp", # has not entered significantly
    "pdo_length",
    "lngdp_ppp", # since already in controls
    "gp_percentage",
    "practice_type_code",
    "parent_sector_name" # worried this is leaking label
]

bin_scores, reg_scores, summed_imp, models = project_util.run_residual_reg(
    consolidated_df=approv_input_df,
    probe_feature_cols=approv_feature_cols,
    clf=clf_default, 
    columns_to_drop=cols_to_exclude
)
approv_input_df.residual_target_above_threshold.value_counts()
print("ROC AUC: ", bin_scores["roc_auc"])
print("Reg train R2: ", reg_scores["r2_train"])

plot_imp = { col: imp for col, imp in summed_imp.items() if imp > 0 }
plot_imp["pdo_embed"] = plot_imp.pop("pdo_embed_x_umap") + plot_imp.pop("pdo_embed_y_umap")
plot_imp["dli_embed"] = plot_imp.pop("dli_embed_x_umap") + plot_imp.pop("dli_embed_y_umap")
plot_imp = project_util.sort_imp(plot_imp)

plt.figure(figsize=(10, 10))
plt.barh(y=list(plot_imp.keys()), width=list(plot_imp.values()))
plt.show()
# take a look at the different financing instruments
approv_df.financing_instr.value_counts()
# slight violence to terminology here as we are doing 1-by-1 exclusion not cumulative
do_drop_one = False

if do_drop_one:
    cols_to_ablate = [col for col in approv_feature_cols if col not in cols_to_exclude and "embed" not in col]
    cols_to_ablate.append(["pdo_embed_x_umap", "pdo_embed_y_umap"])
    cols_to_ablate.append(["dli_embed_x_umap", "dli_embed_y_umap"])

    approv_drop_one = project_util.conduct_drop_one(
        clf=clf_default,
        data=approv_input_df,
        all_feature_cols=approv_feature_cols,
        cols_to_ablate=cols_to_ablate, 
        cols_to_exclude=cols_to_exclude,
        ref_bin_scores=bin_scores,
        ref_reg_scores=reg_scores,
        feature_imp=plot_imp,
        random_seed=random_seed
    )
    
    approv_drop_one.sort_values(by=['penalty_score'], ascending=False)
X = approv_input_df.drop(columns=["residual_target", f'residual_target_above_threshold'])
X = project_util.drop_agg_cols(X, cols_to_exclude)
reg_est = models[0]
bin_est = models[1]
reg_pred = reg_est.predict(X.drop(columns=['id']))
bin_est = reg_est.predict(X.drop(columns=['id']))
approv_input_df['reg_predict'] = reg_pred
approv_input_df['pos_proba'] = reg_pred
plot_df = approv_data.merge(approv_input_df[['id', 'reg_predict', 'pos_proba', 'residual_target']])
region_df = plot_df.groupby(['region']).agg(avg_residual_prediction=('reg_predict', 'mean'), avg_pos=('pos_proba', 'mean'))
region_df.sort_values(by=['avg_residual_prediction'], ascending=False)
sector_df = plot_df.groupby(['parent_sector_name']).agg(
    count=('id', 'count'), avg_residual_prediction=('reg_predict', 'mean'), avg_pos=('pos_proba', 'mean'))
sector_df[sector_df['count'] > 1000].sort_values(by=['avg_residual_prediction'], ascending=False)
plot_df['norm_iegrate'] = (plot_df['iegrate'] - plot_df['iegrate'].mean()) / plot_df['iegrate'].std()

# show normalized ratings, just for sense of variance
plot_df.plot.scatter(x='norm_iegrate', y='residual_target')
plt.show()<jupyter_output><empty_output><jupyter_text>Now for at review time<jupyter_code>review_feature_cols = loan_features + cat_features + ['pdo_length'] + review_embeds + during_project_features
review_feature_cols = [col for col in review_feature_cols if col not in trivial_imp_features]

review_input_df, review_health_results, review_edu_results = project_util.construct_residual_df(
    data=review_data,
    project_feature_cols=review_feature_cols
)
review_bin_scores, review_reg_scores, review_summed_imp, review_models = project_util.run_residual_reg(
    consolidated_df=review_input_df,
    probe_feature_cols=review_feature_cols,
    clf=clf_default, 
    columns_to_drop=cols_to_exclude + ['gdpgrowthav']
)
review_bin_scores
review_summed_imp
print("Review ROC AUC: ", review_bin_scores["roc_auc"])
print("Review reg train R2: ", review_reg_scores["r2_train"])

review_plot_imp = { col: imp for col, imp in review_summed_imp.items() if imp > 0 }
review_plot_imp["icr_embed"] = review_plot_imp.pop("icr_embed_x_umap") + review_plot_imp.pop("icr_embed_y_umap")
review_plot_imp = project_util.sort_imp(review_plot_imp)

plt.figure(figsize=(10, 10))
plt.barh(y=list(review_plot_imp.keys()), width=list(review_plot_imp.values()))
plt.show()<jupyter_output>Review ROC AUC:  0.9772050865800865
Review reg train R2:  0.9318417063237451<jupyter_text>Conduct cross-validation<jupyter_code>from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.linear_model import Lasso
binary_grid_search = {
    "logistic": dict(clf=Lasso(), params={}),
    "random_forest": dict(clf=RandomForestClassifier(random_state=42), params={
        "min_samples_leaf": [1, 5, 10],
        "max_depth": [10, None]
    }),
    "svc": dict(clf=SVC(probability=True), params={
        "C": [1, 10],
        "kernel": ["rbf", "sigmoid"]
    }),
    "xgb": dict(clf=XGBClassifier(), params={
        "learning_rate": [0.01, 0.1, 0.3],
        "min_child_weight": [1, 3, 6]
    })
}
X_cv = approv_input_df.drop(columns=['id', "residual_target", f'residual_target_above_threshold'])

y_reg = approv_input_df["residual_target"]
y_class = approv_input_df['residual_target_above_threshold']    
y_class.value_counts()
approv_input_df.residual_target_above_threshold.value_counts()
reload(project_util)
X_cv = project_util.drop_agg_cols(X_cv, cols_to_exclude)
X_cv.columns
Xcv_train, Xcv_val, ycv_train, ycv_val = train_test_split(X_cv, y_class, test_size=0.2, stratify=y_class)
clf_tester = RandomForestClassifier(min_samples_leaf=5)
clf_tester.fit(Xcv_train, ycv_train)
print("Score: ", clf_tester.score(Xcv_val, ycv_val))
gsearch_1 = GridSearchCV(
    estimator=RandomForestClassifier(), param_grid=[{"min_samples_leaf": [5]}], verbose=3.5, 
    cv=StratifiedKFold(shuffle=True)
)
gsearch_1.fit(X_cv, y_class)
#                          (min_samples_leaf=5)
model_results = {}

for model_type in binary_grid_search:
    print("Running CV on: ", model_type)
    cv_clf = GridSearchCV(
        estimator=binary_grid_search[model_type]["clf"], 
        param_grid=binary_grid_search[model_type]["params"],
        return_train_score=True,
        verbose=1.5,
        cv=StratifiedKFold(shuffle=True)
    )
    cv_clf.fit(X_cv, y_class)
    print("Best score: ", cv_clf.best_score_)
    model_results[model_type] = cv_clf
best_scores = { model_type: model_results[model_type].best_score_ for model_type in binary_grid_search }
best_scores
model_results["random_forest"].best_params_
X_cv_rev = review_input_df.drop(columns=['id', "residual_target", f'residual_target_above_threshold'])

y_reg_rev = review_input_df["residual_target"]
y_class_rev = review_input_df['residual_target_above_threshold']   
X_cv_rev = project_util.drop_agg_cols(X_cv_rev, cols_to_exclude)
review_model_results = {}

for model_type in binary_grid_search:
    print("Running CV on: ", model_type)
    rev_cv_clf = GridSearchCV(
        estimator=binary_grid_search[model_type]["clf"], 
        param_grid=binary_grid_search[model_type]["params"],
        return_train_score=True,
        verbose=1.5,
        cv=StratifiedKFold(shuffle=True)
    )
    rev_cv_clf.fit(X_cv_rev, y_class_rev)
    print("Best score: ", rev_cv_clf.best_score_)
    review_model_results[model_type] = rev_cv_clf
best_scores_rev = { model_type: review_model_results[model_type].best_score_ for model_type in binary_grid_search }
best_scores_rev<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11518976091014133, 'fraction_numerical': 0.009510265754155187, 'mean_word_length': 5.924307692307693, 'pattern_counts': {'":': 11, '<': 24, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '545242', 'license': 'MIT', 'n_tokens_mistral': 4299, 'n_tokens_neox': 4111, 'n_words': 659, 'path': 'notebooks/project_level_predictions.ipynb', 'repo_name': 'luke-grassroot/aid-outcomes-ml', 'size': '11251'}