<jupyter_start><jupyter_text>MODEL SELECTION & TRAINING<jupyter_code>fakes_preds = []
gens_preds = []
sorted_id_list = np.sort(data_f['user_id'].unique())
dev_val_user_ids = sorted_id_list[300:]
validation_user_ids = dev_val_user_ids[0:50]
dev_user_ids = dev_val_user_ids[50:]

for iteration in np.arange(0,10):

    train_idx, test_idx = train_test_split(np.arange(1,25), train_size=0.5, test_size=0.5)
    dev_df = data_f.loc[data_f['user_id'].isin(dev_user_ids)]
    dev_vf = visual_f.loc[dev_df.index]
    val_df = data_f.loc[data_f['user_id'].isin(validation_user_ids)]
    val_vf = visual_f.loc[val_df.index]

    dev_df_gen = dev_df.loc[dev_df['fakeness']==0]
    dev_df_fake = dev_df.loc[dev_df['fakeness']==1]
    dev_df_gen_12 = dev_df_gen.loc[dev_df_gen['sig_id'].isin(train_idx)]
    dev_df_valid_12 = dev_df_gen.loc[dev_df_gen['sig_id'].isin(test_idx)]

    val_df_gen = val_df.loc[val_df['fakeness']==0]
    val_df_fake = val_df.loc[val_df['fakeness']==1]
    val_df_gen_12 = val_df_gen.loc[val_df_gen['sig_id'].isin(train_idx)]
    val_df_valid_gen_12 = val_df_gen.loc[val_df_gen['sig_id'].isin(test_idx)]

    for user_id in tqdm.tqdm(validation_user_ids, ascii=True):
        clf = MLPClassifier(hidden_layer_sizes=(50,25))
        y_train = (pd.concat([val_df_gen_12.loc[val_df_gen_12['user_id']==user_id],dev_df_gen_12.loc[dev_df_gen_12['user_id']!=user_id]]))['user_id']==user_id
        X_train = visual_f.loc[y_train.index]  
        clf.fit(X_train, y_train)
        y_valid_fakes = val_df_fake.loc[(val_df_fake['user_id']==user_id)]
        X_valid_f = visual_f.loc[y_valid_fakes.index]
        fakes_preds.append(clf.predict_proba(X_valid_f)[:,1])
        y_valid_gens = val_df_valid_gen_12.loc[val_df_valid_gen_12['user_id']==user_id]
        X_valid_g = visual_f.loc[y_valid_gens.index]
        gens_preds.append(clf.predict_proba(X_valid_g)[:,1])<jupyter_output>100%|##################################################################################| 50/50 [02:43<00:00,  3.17s/it]
100%|##################################################################################| 50/50 [02:51<00:00,  3.01s/it]
100%|##################################################################################| 50/50 [02:59<00:00,  3.89s/it]
100%|##################################################################################| 50/50 [02:52<00:00,  3.19s/it]
100%|##################################################################################| 50/50 [03:07<00:00,  3.65s/it]
100%|##################################################################################| 50/50 [03:15<00:00,  3.94s/it]
100%|##################################################################################| 50/50 [03:10<00:00,  4.11s/it]
100%|##################################################################################| 50/50 [03:17<00:00,  3.95s/it]
100%|###################################[...]<jupyter_text>GLOBAL THRESHOLD SELECTION<jupyter_code>flat_fakes_preds = np.expand_dims(np.array([item for sublist in fakes_preds for item in sublist]),axis=1)
flat_gens_preds = np.expand_dims(np.array([item for sublist in gens_preds for item in sublist]),axis=1)
all_preds = np.vstack((flat_fakes_preds,flat_gens_preds))
all_labels = np.vstack((np.zeros((flat_fakes_preds.shape[0],1)),np.ones((flat_gens_preds.shape[0],1))))

fpr,tpr,threshold = roc_curve(all_labels,all_preds)
fnr = 1 - tpr
EER = fpr[np.nanargmin(np.absolute((fnr - fpr)))]
eer_th = threshold[np.nanargmin(np.absolute((fnr - fpr)))]
print('EER_glob : ', EER*100,'\nEER_Threshold_glob : ', eer_th)
glob_th = eer_th
assert len(fakes_preds)==len(gens_preds)
EER_accum=0
for idx,val in enumerate(fakes_preds):
    user_fakes_preds = np.expand_dims(np.array(fakes_preds[idx]),axis=1)
    user_gens_preds = np.expand_dims(np.array(gens_preds[idx]),axis=1)
    all_user_preds = np.vstack((user_fakes_preds,user_gens_preds))
    all_user_labels = np.vstack((np.zeros((user_fakes_preds.shape[0],1)),np.ones((user_gens_preds.shape[0],1)))) 
    fpr,tpr,threshold = roc_curve(all_user_labels,all_user_preds)
    fnr = 1 - tpr
    EER = fpr[np.nanargmin(np.absolute((fnr - fpr)))]
    EER_accum += EER
    
print('EER_user : ', (EER_accum*100)/len(fakes_preds)) 
print(glob_th)<jupyter_output>0.7003721283330443<jupyter_text>TRAIN AND TEST ON THE EXPLOITATION SET<jupyter_code>test_gens_preds = []
test_fakes_preds = []
exp_user_ids = sorted_id_list[:300]
train_idx, test_idx = train_test_split(np.arange(1,25), train_size=0.5)
exp_df = data_f.loc[data_f['user_id'].isin(exp_user_ids)]
exp_vf = visual_f.loc[exp_df.index]
exp_df_gen = exp_df.loc[exp_df['fakeness']==0]
exp_df_fake = exp_df.loc[exp_df['fakeness']==1]
exp_df_fake_10 = exp_df_fake.loc[exp_df_fake['sig_id'].isin(choice(np.arange(1,31),10,replace=False))]
exp_df_gen_12 = exp_df_gen.loc[exp_df_gen['sig_id'].isin(train_idx)]
exp_df_valid_gen_12 = exp_df_gen.loc[exp_df_gen['sig_id'].isin(test_idx)]

dev_val_df = data_f.loc[data_f['user_id'].isin(dev_val_user_ids)]
dev_val_vf = visual_f.loc[dev_val_df.index]
dev_val_df_gen = dev_val_df.loc[dev_val_df['fakeness']==0]
dev_val_df_fake = dev_val_df.loc[dev_val_df['fakeness']==1]
dev_val_df_valid_gen_14 = dev_val_df_gen.loc[dev_val_df_gen['sig_id'].isin(choice(np.arange(1,25),14,replace=False))]
for user_id in tqdm.tqdm(exp_user_ids, ascii=True):
    clf = SVC(C=1,gamma='scale',class_weight='balanced', probability=False, kernel=user_kernel)
    y_train = (pd.concat([exp_df_gen_12.loc[exp_df_gen_12['user_id']==user_id],dev_val_df_valid_gen_14.loc[dev_val_df_valid_gen_14['user_id']!=user_id]]))['user_id']==user_id
    X_train = visual_f.loc[y_train.index]  
    clf.fit(X_train, y_train)
    y_valid_fakes = exp_df_fake_10.loc[(exp_df_fake_10['user_id']==user_id)]
    X_valid_f = visual_f.loc[y_valid_fakes.index]
    test_fakes_preds.append(clf.decision_function(X_valid_f))
    y_valid_gens = exp_df_valid_gen_12.loc[exp_df_valid_gen_12['user_id']==user_id]
    X_valid_g = visual_f.loc[y_valid_gens.index]
    test_gens_preds.append(clf.decision_function(X_valid_g))
flat_test_fakes_preds = np.expand_dims(np.array([item for sublist in test_fakes_preds for item in sublist]),axis=1)
flat_test_gens_preds = np.expand_dims(np.array([item for sublist in test_gens_preds for item in sublist]),axis=1)
print("____At the EER threshold decided on the Validation set____")
print("FRR : ",(1-len(flat_test_gens_preds[flat_test_gens_preds>=glob_th])/len(flat_test_gens_preds))*100)
print("FARskilled : ",(1-len(flat_test_fakes_preds[flat_test_fakes_preds<glob_th])/len(flat_test_fakes_preds))*100)
all_test_preds = np.vstack((flat_test_fakes_preds,flat_test_gens_preds))
all_test_labels = np.vstack((np.zeros((flat_test_fakes_preds.shape[0],1)),np.ones((flat_test_gens_preds.shape[0],1))))

fpr,tpr,threshold = roc_curve(all_test_labels,all_test_preds)
fnr = 1 - tpr
EER = fpr[np.nanargmin(np.absolute((fnr - fpr)))]
eer_th = threshold[np.nanargmin(np.absolute((fnr - fpr)))]
print('EER_glob for test set: ', EER*100,'\nEER_Threshold_glob for test set: ', eer_th)
assert len(test_fakes_preds)==len(test_gens_preds)
EER_accum=0
for idx,val in enumerate(test_fakes_preds):
    user_test_fakes_preds = np.expand_dims(np.array(test_fakes_preds[idx]),axis=1)
    user_test_gens_preds = np.expand_dims(np.array(test_gens_preds[idx]),axis=1)
    all_user_test_preds = np.vstack((user_test_fakes_preds,user_test_gens_preds))
    all_user_test_labels = np.vstack((np.zeros((user_test_fakes_preds.shape[0],1)),np.ones((user_test_gens_preds.shape[0],1)))) 
    fpr,tpr,threshold = roc_curve(all_user_test_labels,all_user_test_preds)
    fnr = 1 - tpr
    EER = fpr[np.nanargmin(np.absolute((fnr - fpr)))]
    EER_accum += EER
    
print('EER_user for test set : ', (EER_accum*100)/len(test_fakes_preds))<jupyter_output>EER_user for test set :  2.7999999999999985
{'fraction_non_alphanumeric': 0.2292439554816426, 'fraction_numerical': 0.041703978508379175, 'mean_word_length': 7.794150731158605, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '93268', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 3435, 'n_tokens_neox': 3235, 'n_words': 322, 'path': 'pp_10fold_nn_signet_orig_split.ipynb', 'repo_name': 'gonultasbu/sigver_bmg', 'size': '7817'}