<jupyter_start><jupyter_text>Reading in the blocked grouped table<jupyter_code>fbg = pd.read_csv("final_blocked_grouped.csv")
del fbg['Unnamed: 0']
fbg.head()<jupyter_output><empty_output><jupyter_text>Extracting days_active<jupyter_code>da_1 = fbg.loc[:,['rev_user_text','days_active']]
da_1 = da_1.drop_duplicates()
da_1.head(20)<jupyter_output><empty_output><jupyter_text>Removing ipb_address and keyword to add it later<jupyter_code>#blk = fbg.loc[:,['ipb_address','keyword']]
#blk = blk.drop_duplicates()
#blk.head(20)
#blk.to_csv('blocked_users.csv')
#del fbg['ipb_address']
#del fbg['keyword']
#fbg.head()<jupyter_output><empty_output><jupyter_text>Reading in the non-blocked grouped table<jupyter_code>fnbg = pd.read_csv("final_nonblocked_grouped.csv")
del fnbg['Unnamed: 0']
fnbg.head()
da_2 = fnbg.loc[:,['rev_user_text','days_active']]
da_2 = da_2.drop_duplicates()
da_2.head(20)<jupyter_output><empty_output><jupyter_text>Adding da_1 and da_2<jupyter_code>da_frames = [da_1,da_2]
da_all = pd.concat(da_frames)
da_all.to_csv('days_active.csv')<jupyter_output><empty_output><jupyter_text>Checking overlap with Textual data<jupyter_code>b_list = fbg['rev_user_text']
nb_list = fnbg['rev_user_text']
type(b_list)<jupyter_output><empty_output><jupyter_text>Removing Duplicates<jupyter_code>b_list_u = b_list.unique()
nb_list_u = nb_list.unique()
b_list_u
type(b_list_u)
nb_list_u<jupyter_output><empty_output><jupyter_text>Converting back to series<jupyter_code>#my_list = map(lambda x: x[0], b_list_u)
b_ser = pd.Series(b_list_u)
b_ser
#my_list = map(lambda x: x[0], nb_list_u)
nb_ser = pd.Series(nb_list_u)
nb_ser
df_blockuserlist = pd.read_csv( 'blockuserlist.txt', sep = '\t')
df_blockuserlist
blist = b_ser.to_frame(name='0')
blist
rev_all = pd.merge(blist,df_blockuserlist,how="inner",on=["0"])
rev_all<jupyter_output><empty_output><jupyter_text>Pivoting the grouped tables for feature extraction  (A) Pivotting the non blocked grouped table<jupyter_code>p1 = fnbg.pivot(index='rev_user_text', columns='diff_week')
p1.head(50)<jupyter_output><empty_output><jupyter_text>Resetting the multiple column levels to individual column for each week<jupyter_code>p1.columns = ['rev_id_1','rev_id_2','rev_id_3','rev_id_4','rev_id_5','rev_id_6','rev_id_7','rev_id_8',
              'rev_minor_edit_1','rev_minor_edit_2','rev_minor_edit_3','rev_minor_edit_4','rev_minor_edit_5',
              'rev_minor_edit_6','rev_minor_edit_7','rev_minor_edit_8',
              'rev_deleted_1','rev_deleted_2','rev_deleted_3','rev_deleted_4','rev_deleted_5','rev_deleted_6',
              'rev_deleted_7','rev_deleted_8',
              'rev_len_1','rev_len_2','rev_len_3','rev_len_4','rev_len_5','rev_len_6','rev_len_7','rev_len_8',
              'days_active_1','days_active_2','days_active_3','days_active_4','days_active_5','days_active_6',
              'days_active_7','days_active_8',
              'rev_minor_edit_n_1','rev_minor_edit_n_2','rev_minor_edit_n_3','rev_minor_edit_n_4','rev_minor_edit_n_5',
              'rev_minor_edit_n_6','rev_minor_edit_n_7','rev_minor_edit_n_8',
              'rev_deleted_n_1','rev_deleted_n_2','rev_deleted_n_3','rev_deleted_n_4','rev_deleted_n_5',
              'rev_deleted_n_6','rev_deleted_n_7','rev_deleted_n_8']
p1.head(50)<jupyter_output><empty_output><jupyter_text>Reseting the index and reverting rev_user_text to a column<jupyter_code>p1.reset_index(inplace=True)
p1.head(50)<jupyter_output><empty_output><jupyter_text>Deleting days active columns except one(not needed for each week)<jupyter_code>del p1['days_active_2']
del p1['days_active_3']
del p1['days_active_4']
del p1['days_active_5']
del p1['days_active_6']
del p1['days_active_7']
del p1['days_active_8']
p1 = p1.rename(index=str,columns={"days_active_1":"days_active"})
p1.head()
p1['days_active'].head()
p1['blocked'] = 0
p1.head()
p1.to_csv('Nonblocked_grouped_pivoted.csv')<jupyter_output><empty_output><jupyter_text>(B) Pivotting the blocked grouped table<jupyter_code>p2 = fbg.pivot(index='rev_user_text', columns='diff_week')
p2.head(50)<jupyter_output><empty_output><jupyter_text>Resetting the multiple level columns to weekly columns<jupyter_code>p2.columns = ['rev_id_1','rev_id_2','rev_id_3','rev_id_4','rev_id_5','rev_id_6','rev_id_7','rev_id_8',
              'rev_minor_edit_1','rev_minor_edit_2','rev_minor_edit_3','rev_minor_edit_4','rev_minor_edit_5',
              'rev_minor_edit_6','rev_minor_edit_7','rev_minor_edit_8',
              'rev_deleted_1','rev_deleted_2','rev_deleted_3','rev_deleted_4','rev_deleted_5','rev_deleted_6',
              'rev_deleted_7','rev_deleted_8',
              'rev_len_1','rev_len_2','rev_len_3','rev_len_4','rev_len_5','rev_len_6','rev_len_7','rev_len_8',
              'days_active_1','days_active_2','days_active_3','days_active_4','days_active_5','days_active_6',
              'days_active_7','days_active_8',
              'rev_minor_edit_n_1','rev_minor_edit_n_2','rev_minor_edit_n_3','rev_minor_edit_n_4','rev_minor_edit_n_5',
              'rev_minor_edit_n_6','rev_minor_edit_n_7','rev_minor_edit_n_8',
              'rev_deleted_n_1','rev_deleted_n_2','rev_deleted_n_3','rev_deleted_n_4','rev_deleted_n_5',
              'rev_deleted_n_6','rev_deleted_n_7','rev_deleted_n_8']
p2.head(50)<jupyter_output><empty_output><jupyter_text>Reverting the index to bring out rev_user_text as a column<jupyter_code>p2.reset_index(inplace=True)
p2.head(50)<jupyter_output><empty_output><jupyter_text>Deleting the days_active which are not needed<jupyter_code>del p2['days_active_2']
del p2['days_active_3']
del p2['days_active_4']
del p2['days_active_5']
del p2['days_active_6']
del p2['days_active_7']
del p2['days_active_8']
p2 = p2.rename(index=str,columns={"days_active_1":"days_active"})
p2.head()
p2['blocked'] = 1
p2.head()
p2.to_csv('Blocked_grouped_pivoted.csv')<jupyter_output><empty_output><jupyter_text>Concatenating the blocked and non blocked pivot data<jupyter_code>p1 = pd.read_csv("Nonblocked_grouped_pivoted.csv")
del p1['Unnamed: 0']
p1.head()
p2 = pd.read_csv("Blocked_grouped_pivoted.csv")
del p2['Unnamed: 0']
p2.head()
da_all = pd.read_csv("days_active.csv")
del da_all['Unnamed: 0']
da_all.head()
#blk = pd.read_csv("blocked_users.csv")
#del blk['Unnamed: 0']
#blk.head()
frames = [p1,p2]
p_all = pd.concat(frames)
del p_all['days_active']
p_all = pd.merge(p_all,da_all,how="left",left_on=["rev_user_text"],right_on=["rev_user_text"])
p_all.head(10)
#p_all = pd.merge(p_all,blk,how="left",left_on=["rev_user_text"],right_on=["ipb_address"])
#p_all.head(10)
#p_all.tail(10)
#del p_all['ipb_address']
#p_all.sample(10)
#p_all['blocked'].value_counts()
#p_all['keyword'].value_counts()
p_all.to_csv('all_users_pivoted_n.csv')<jupyter_output><empty_output><jupyter_text>Reading in the all_users_pivoted_n<jupyter_code>p_all = pd.read_csv("all_users_pivoted_n.csv")
del p_all['Unnamed: 0']
p_all.head()
p_all.isna().sum()
p_all[p_all['days_active'].isnull()]
p_all['days_active'].value_counts()<jupyter_output><empty_output><jupyter_text>Building initial Classification Model Filling missing NaN values with zero<jupyter_code>p_all = p_all.fillna(0)
p_all.head()
p_all.isna().sum()
p_all.head()
pd.options.display.max_columns = None
display(p_all[p_all['rev_deleted_1']==1785.0])<jupyter_output><empty_output><jupyter_text>Dividing p_all into Train and Validate sets<jupyter_code>#p_all['is_train'] = np.random.uniform(0, 1, len(p_all)) <= .75
#Train, Validate = p_all[p_all['is_train']==True], p_all[p_all['is_train']==False]
p_all.to_csv('final_dataset.csv')<jupyter_output><empty_output><jupyter_text>Resume from here >>><jupyter_code>p_all = pd.read_csv("final_dataset.csv")
del p_all['Unnamed: 0']
p_all.head()
p_all.count()<jupyter_output><empty_output><jupyter_text>Removing the base rev_minor_edits and  rev_deleted columns<jupyter_code>p_all = p_all.drop(p_all.ix[:,'rev_minor_edit_1':'rev_deleted_8'].head(0).columns, axis=1)
p_all.count()<jupyter_output><empty_output><jupyter_text>Deleting is_train as we need to balance the classes before separating into train and validate<jupyter_code>del p_all['is_train']<jupyter_output><empty_output><jupyter_text>Class Balancing<jupyter_code>p_all['blocked'].value_counts()<jupyter_output><empty_output><jupyter_text>Separating the classes temporarily<jupyter_code>p_all_0 = p_all[p_all['blocked']==0]
p_all_1 = p_all[p_all['blocked']==1]
p_all_0.shape
p_all_1.shape<jupyter_output><empty_output><jupyter_text>Randomly sampling 98000 rows from p_all_0<jupyter_code>p_all_0 = p_all_0.ix[np.random.choice(p_all_0.index,98000,replace=False)]
p_all_0.shape
p_all_0.nunique()
p_all_0.head()<jupyter_output><empty_output><jupyter_text>Joining p_all_0 and p_all_1 and separating into train and validate sets<jupyter_code>frames = [p_all_0,p_all_1]
p_all = pd.concat(frames)
p_all['blocked'].value_counts()
p_all['is_train'] = np.random.uniform(0, 1, len(p_all)) <= .75
Train = p_all[p_all['is_train']==True]
Validate = p_all[p_all['is_train']==False]
Train['blocked'].value_counts()[1]/Train['blocked'].value_counts()[0]
Validate['blocked'].value_counts()[1]/Validate['blocked'].value_counts()[0]<jupyter_output><empty_output><jupyter_text>Removing the rev_user_text column from Train, Validate<jupyter_code>del Train['rev_user_text']
del Validate['rev_user_text']
del Train['is_train']
del Validate['is_train']
Train.head()
Validate.head()<jupyter_output><empty_output><jupyter_text>Logistic regression<jupyter_code>x_train = Train.loc[:, Train.columns != 'blocked']
y_train = Train["blocked"]
x_validate = Validate.loc[:, Validate.columns != 'blocked']
y_validate = Validate["blocked"]<jupyter_output><empty_output><jupyter_text>Fitting a model using train data<jupyter_code>logreg = LogisticRegression()
logreg.fit(x_train,y_train)<jupyter_output><empty_output><jupyter_text>Predicting using model on test data<jupyter_code>y_pred = logreg.predict(x_validate)
y_pred
print('Accuracy of logistic regression classifier on test set: {:.2f}'.format(logreg.score(x_validate,y_validate)))<jupyter_output>Accuracy of logistic regression classifier on test set: 0.67<jupyter_text>Confusion Matrix<jupyter_code>type(y_validate)
type(y_pred)
from sklearn.metrics import confusion_matrix
confusion_matrix = confusion_matrix(y_validate,y_pred)
print(confusion_matrix)<jupyter_output>[[20315  4032]
 [12280 12302]]<jupyter_text>Precision, Recall, F-measure and Support<jupyter_code>from sklearn.metrics import classification_report
print(classification_report(y_validate,y_pred))<jupyter_output>precision    recall  f1-score   support

          0       0.62      0.83      0.71     24347
          1       0.75      0.50      0.60     24582

avg / total       0.69      0.67      0.66     48929<jupyter_text>AUC and ROC curve<jupyter_code>from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve
logit_roc_auc = roc_auc_score(y_validate,logreg.predict(x_validate))
logit_roc_auc
fpr, tpr, thresholds = roc_curve(y_validate, logreg.predict_proba(x_validate)[:,1])
plt.figure(figsize=(9,9))
plt.plot(fpr, tpr, label='Logistic Regression (area = %0.2f)' % logit_roc_auc)
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic')
plt.legend(loc="lower right")
plt.show()<jupyter_output><empty_output><jupyter_text>Random Forest Fitting model on train data<jupyter_code>from sklearn.ensemble import RandomForestClassifier
clf=RandomForestClassifier(n_estimators=100)
clf.fit(x_train,y_train)<jupyter_output><empty_output><jupyter_text>Predicting using the model on test data<jupyter_code>y_pred_rf=clf.predict(x_validate)
y_pred_rf
print("Accuracy of Random Forest classifier on test set:",metrics.accuracy_score(y_validate,y_pred_rf))<jupyter_output>Accuracy of Random Forest classifier on test set: 0.6559913343824726<jupyter_text>Confusion matrix<jupyter_code>type(y_validate)
type(y_pred_rf)
from sklearn.metrics import confusion_matrix
confusion_matrix_rf = confusion_matrix(y_validate,y_pred_rf)
print(confusion_matrix_rf)<jupyter_output>[[15758  8589]
 [ 8243 16339]]<jupyter_text>Precision, Recall, F-measure and Support<jupyter_code>from sklearn.metrics import classification_report
print(classification_report(y_validate,y_pred_rf))<jupyter_output>precision    recall  f1-score   support

          0       0.66      0.65      0.65     24347
          1       0.66      0.66      0.66     24582

avg / total       0.66      0.66      0.66     48929<jupyter_text>AUC and ROC curve<jupyter_code>from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve
logit_roc_auc_rf = roc_auc_score(y_validate,clf.predict(x_validate))
logit_roc_auc_rf
fpr, tpr, thresholds = roc_curve(y_validate,logreg.predict_proba(x_validate)[:,1])
fpr_rf,tpr_rf, thresholds_rf = roc_curve(y_validate,clf.predict_proba(x_validate)[:,1])
plt.figure(figsize=(9,9))
plt.plot(fpr, tpr, 'b', label='Logistic Regression (area = %0.2f)' % logit_roc_auc)
plt.plot(fpr_rf, tpr_rf, 'g' ,label='Random Forest (area = %0.2f)' % logit_roc_auc_rf)
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic')
plt.legend(loc="lower right")
plt.show()<jupyter_output><empty_output><jupyter_text>Checking feature importance using Random Forest<jupyter_code>x_validate.columns
feature_imp = pd.Series(clf.feature_importances_,index=x_validate.columns).sort_values(ascending=False)
feature_imp<jupyter_output><empty_output><jupyter_text>Building new Logistic regression and Random Forest models with just the most important features<jupyter_code>x_train_imp = x_train[['rev_len_1','rev_id_1','rev_minor_edit_n_1','days_active','rev_len_2','rev_id_2',
                       'rev_deleted_n_1']]
x_train_imp.head()
x_train_imp = x_train[['rev_len_1','rev_id_1','rev_minor_edit_n_1','days_active','rev_len_2','rev_id_2',
                       'rev_deleted_n_1']]
x_train_imp.head()
x_validate_imp = x_validate[['rev_len_1','rev_id_1','rev_minor_edit_n_1','days_active','rev_len_2','rev_id_2',
                       'rev_deleted_n_1']]
x_validate_imp.head()<jupyter_output><empty_output><jupyter_text>New Logistic Regression model<jupyter_code>logreg_new = LogisticRegression()
logreg_new.fit(x_train_imp,y_train)
y_pred_lr_new = logreg_new.predict(x_validate_imp)
print('Accuracy of logistic regression classifier on test set: {:.2f}'.format(logreg_new.score(x_validate_imp,
                                                                                               y_validate)))
from sklearn.metrics import confusion_matrix
confusion_matrix = confusion_matrix(y_validate,y_pred_lr_new)
print(confusion_matrix)
from sklearn.metrics import classification_report
print(classification_report(y_validate,y_pred_lr_new))
from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve
logit_roc_auc = roc_auc_score(y_validate,logreg_new.predict(x_validate_imp))
logit_roc_auc<jupyter_output><empty_output><jupyter_text>Overall performance is same or slightly worse for Logistic regression with these features  New Random Forest model<jupyter_code>from sklearn.ensemble import RandomForestClassifier
clf_new=RandomForestClassifier(n_estimators=100)
clf_new.fit(x_train_imp,y_train)
y_pred_rf_new=clf_new.predict(x_validate_imp)
print("Accuracy of Random Forest classifier on test set:",metrics.accuracy_score(y_validate,y_pred_rf_new))
from sklearn.metrics import confusion_matrix
confusion_matrix_rf = confusion_matrix(y_validate,y_pred_rf_new)
print(confusion_matrix_rf)
from sklearn.metrics import classification_report
print(classification_report(y_validate,y_pred_rf_new))
from sklearn.metrics import roc_auc_score
from sklearn.metrics import roc_curve
logit_roc_auc_rf = roc_auc_score(y_validate,clf_new.predict(x_validate_imp))
logit_roc_auc_rf
fpr, tpr, thresholds = roc_curve(y_validate,logreg_new.predict_proba(x_validate_imp)[:,1])
fpr_rf,tpr_rf, thresholds_rf = roc_curve(y_validate,clf_new.predict_proba(x_validate_imp)[:,1])
plt.figure(figsize=(9,9))
plt.plot(fpr, tpr, 'b', label='Logistic Regression (area = %0.2f)' % logit_roc_auc)
plt.plot(fpr_rf, tpr_rf, 'g' ,label='Random Forest (area = %0.2f)' % logit_roc_auc_rf)
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic')
plt.legend(loc="lower right")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13518730771550944, 'fraction_numerical': 0.030765518489473367, 'mean_word_length': 7.879485806106053, 'pattern_counts': {'":': 2, '<': 169, '<?xml version=': 0, '>': 170, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '43', 'dir': 'jupyter-structured-clean-dedup', 'id': '523449', 'license': 'MIT', 'n_tokens_mistral': 6794, 'n_tokens_neox': 6554, 'n_words': 851, 'path': 'Misc/as3uj_feature_extraction_model_building.ipynb', 'repo_name': 'Arnab777as3uj/Wikimedia-Trust-Safety', 'size': '16577'}