<jupyter_start><jupyter_text>1. Import data<jupyter_code>[data_raw, protein_raw, activities_raw, 
 data_extend, protein_extend, activities_extend] = pickle.load(open('NR.pkl', 'rb'))
#mean, std = data_raw.mean(), data_raw.std() 
cont_names = data_raw.columns
cat_names = []
dep_var = 'Activity'
data_raw['Activity'] = [int(i<1) for i in activities_raw]
data_extend['Activity'] = [int(i<1) for i in activities_extend]
data_raw.shape, data_extend.shape
list(data_raw['Activity']).count(0), list(data_raw['Activity']).count(1)<jupyter_output><empty_output><jupyter_text>2. Split dataset<jupyter_code>sss = StratifiedShuffleSplit(n_splits=1, test_size=0.4, random_state=GLOBAL_SEED)
group_labels = np.array([i.split('_')[0]+'_'+str(data_raw.loc[i, 'Activity']) for i in data_raw.index])
for train_idx, valid_idx in sss.split(data_raw, group_labels):pass
data_train = data_raw.iloc[train_idx, :]
data_test = data_raw.iloc[valid_idx, :]
data_train.shape, data_test.shape, data_extend.shape<jupyter_output><empty_output><jupyter_text>3. Hyperparameter selection<jupyter_code>def get_dl_AUC(data, train_idx, valid_idx, data_test, cont_names, dep_var, layern=2, celln=100, ps=0.5):
    procs = []
    data_tabular = (TabularList.from_df(data, cat_names=[], cont_names=cont_names, procs=procs)
                           .split_by_idx(valid_idx=valid_idx)
                           .label_from_df(cols=dep_var)
                           .add_test(TabularList.from_df(data_test, cat_names=[], cont_names=cont_names, procs=procs))
                           .databunch())
    learn = tabular_learner(data_tabular, layers=[celln for i in range(layern)], metrics=accuracy, ps=ps, silent=True)
    _ = learn.fit_one_cycle(50)
    ###
    preds, _ = learn.get_preds(ds_type=DatasetType.Valid)
    _, pred_class = preds.max(1)
    pred_prob = preds[:, 1]
    valid_auc = roc_auc_score(data.iloc[valid_idx, :]['Activity'], pred_prob)
    ###
    preds, _ = learn.get_preds(ds_type=DatasetType.Test)
    _, pred_class = preds.max(1)
    pred_prob = preds[:, 1]
    test_auc = roc_auc_score(data_test['Activity'], pred_prob)
    return valid_auc, test_auc
sss = StratifiedKFold(n_splits=5, shuffle=True, random_state=GLOBAL_SEED)
group_labels = np.array([i.split('_')[0]+'_'+str(data_train.loc[i, 'Activity']) for i in data_train.index])
folds = list(sss.split(data_train, group_labels))
#result = pd.read_csv('Rawdata_DL_Parameters.csv', index_col=0)
#result = [list(i) for i in result.values]
result = []
#for layern in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]:
for layern in [6, 7, 8, 9, 10]:
    for celln in [100, 300, 500]:
        for ps in [0, 0.1, 0.3, 0.5]:
            #train_aucs = []
            valid_aucs = []
            test_aucs = []
            for fold in range(5):
                train_idx, valid_idx = folds[fold]
                valid_auc, test_auc = get_dl_AUC(data_train, train_idx, valid_idx, data_test, cont_names, 
                                                            dep_var, layern, celln, ps)
                result.append([fold, layern, celln, ps, valid_auc, test_auc])
                #train_aucs.append(train_auc)
                valid_aucs.append(valid_auc)
                test_aucs.append(test_auc)
            result.append(['Mean', layern, celln, ps, 
                           np.array(valid_aucs).mean(), np.array(test_aucs).mean()])
            print(layern, celln, ps, np.array(valid_aucs).mean(), np.array(test_aucs).mean())
result = pd.DataFrame(result, columns=['Fold', 'Layer', 'Cell', 'PS', 'Valid_AUC', 'Test_AUC'])
result.to_csv('Rawdata_DL_Parameters.csv')
result = pd.read_csv('Rawdata_DL_Parameters.csv', index_col=0)
result.loc[result['Fold']=='Mean', :].sort_values('Valid_AUC', ascending=False).head(2)<jupyter_output><empty_output><jupyter_text>4. Oversampling prevents over fitting<jupyter_code>def oversampling(data_train, N=3000, sigma=0.1, test_size=0.4):
    sss = StratifiedShuffleSplit(n_splits=1, test_size=test_size, random_state=0) #random_state=0
    protein_train = np.array([i.split('_')[0] for i in data_train.index])
    for train_idx, valid_idx in sss.split(data_train, protein_train):pass
    # sampling
    sample_data = []
    sample_label = []
    sample_proteins = []
    proteins = np.array([i.split('_')[0] for i in data_train.iloc[train_idx, :].index])
    for p in set(proteins):
        temp = data_train.iloc[train_idx, :].loc[proteins==p, :]
        if N-sum(proteins==p)<=0:
            continue
        for i in range(N-sum(proteins==p)):
            rrow = list(temp.iloc[random.choice(range(len(temp.index))), :])
            sample_data.append(rrow)
            sample_label.append(int(rrow[-1]))
            sample_proteins.append(p)
    if len(sample_label) == 0:
        return data_train, train_idx, valid_idx
    sample_data = pd.DataFrame(norm.rvs(np.array(sample_data), scale=sigma), columns=data_train.columns)
    sample_data.index = [sample_proteins[i]+'_OS_'+str(i) for i in range(len(sample_proteins))]
    sample_data[dep_var] = sample_label
    os_data = pd.DataFrame(data_train)
    os_data = os_data.append(sample_data)
    os_train_idx = list(set(range(len(os_data.index)))-set(valid_idx))
    return os_data, os_train_idx, valid_idx

def oversampling_dl_AUC(data, data_test, cont_names, dep_var, N, sigma, test_size=0.2, layern=2, celln=100, ps=0.5):
    os_data, train_idx, valid_idx = oversampling(data, N, sigma, test_size)
    train_auc, test_auc = get_dl_AUC(os_data, train_idx, valid_idx, data_test, cont_names, dep_var, layern, celln, ps)
    return train_auc, test_auc

def oversamping_dl_result(data, data_test, cont_names, dep_var, layern=2, celln=100, ps=0.5):
    result = []
    sigma = 0.1
    test_size = np.arange(0.1, 0.51, 0.05)
    N = np.arange(1000, 5001, 500)
    test_size, N = np.meshgrid(test_size, N)
    train_aucs = np.zeros(N.shape)
    test_aucs = np.zeros(N.shape)
    for i in range(N.shape[0]):
        for j in range(N.shape[1]):
            train_auc, test_auc = oversampling_dl_AUC(data, data_test, cont_names, dep_var, N[i][j], sigma, test_size[i][j], layern, celln, ps)
            train_aucs[i][j] = train_auc
            test_aucs[i][j] = test_auc
            print(N[i][j], round(1-test_size[i][j], 2), round(train_auc, 3), round(test_auc, 3))
            result.append([N[i][j], round(1-test_size[i][j], 2), train_auc, test_auc])
    result = pd.DataFrame(result, columns=['N', 'Train size', 'Test AUC', 'Extend AUC'])
    return N, (1-test_size), train_aucs, test_aucs, result
layern=5; celln=300; ps=0.5
N, train_size, test_aucs, extend_aucs, result = oversamping_dl_result(data_raw, data_extend, cont_names, dep_var, 
                                                                     layern=layern, celln=celln, ps=ps)
pickle.dump([N, train_size, test_aucs, extend_aucs, result], open('NR_surface_DL_layer5_cell300_ps0.5.pkl', 'wb'))
N, train_size, test_aucs, extend_aucs, result = pickle.load(open('NR_surface_DL_layer5_cell300_ps0.5.pkl', 'rb'))
result.sort_values('Extend AUC', ascending=False).head(2)<jupyter_output><empty_output><jupyter_text>5. Result<jupyter_code>from mpl_toolkits.mplot3d import Axes3D  # noqa: F401 unused import
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter
import seaborn as sns
import matplotlib.pyplot as plt
#plt.style.available
plt.style.use("seaborn-paper")

fig = plt.figure(1, (8, 5), dpi=300)
ax = fig.gca(projection='3d')
# Plot the surface.
surf = ax.plot_surface(N, train_size, test_aucs, cmap='viridis', #cm.coolwarm
                       linewidth=0, antialiased=False)
_ = ax.set_zlim(0.86, 0.90)
_ = ax.set_xlabel('Oversampling')
_ = ax.set_ylabel('The Ratio of Training Set')
_ = ax.set_zlabel('AUC')
_ = fig.colorbar(surf, shrink=0.5, aspect=5)
_ = plt.show()
fig = plt.figure(1, (8, 5), dpi=300)
ax = fig.gca(projection='3d')
# Plot the surface.
surf = ax.plot_surface(N, train_size, extend_aucs, cmap='viridis', #cm.coolwarm
                       linewidth=0, antialiased=False)
_ = ax.set_zlim(0.62, 0.69)
_ = ax.set_xlabel('Oversampling')
_ = ax.set_ylabel('The Ratio of Training Set')
_ = ax.set_zlabel('AUC')
_ = fig.colorbar(surf, shrink=0.5, aspect=5)
_ = plt.show()<jupyter_output><empty_output><jupyter_text>6. Final model<jupyter_code>def oversampling_raw(data, N=3000, sigma=0.1):
    # sampling
    sample_data = []
    sample_label = []
    sample_proteins = []
    proteins = np.array([i.split('_')[0] for i in data.index])
    for p in set(proteins):
        temp = data.loc[proteins==p, :]
        if N-sum(proteins==p)<=0:
            continue
        for i in range(N-sum(proteins==p)):
            rrow = list(temp.iloc[random.choice(range(len(temp.index))), :])
            sample_data.append(rrow)
            sample_label.append(int(rrow[-1]))
            sample_proteins.append(p)
    if len(sample_label) == 0:
        return data
    sample_data = pd.DataFrame(norm.rvs(np.array(sample_data), scale=sigma), columns=data.columns)
    sample_data.index = [sample_proteins[i]+'_OS_'+str(i) for i in range(len(sample_proteins))]
    sample_data[dep_var] = sample_label
    os_data = pd.DataFrame(data)
    os_data = os_data.append(sample_data)
    return os_data
result.sort_values('Extend AUC', ascending=False).head(2)
N = 4000
layern=5; celln=300; ps=0.5
data = data_raw.copy()
data = pd.concat([data_raw, data_extend], axis=0, sort=False)
data = oversampling_raw(data, N)

Valid_set = random.sample(list(data.index), 1000)
Valid_set = data.loc[Valid_set, :].copy()
Valid_set.index = ['Valid_'+i for i in Valid_set.index]
valid_idx = list(range(data.shape[0], data.shape[0]+Valid_set.shape[0]))
data = pd.concat([data, Valid_set], axis=0, sort=False)
procs = []
data_tabular = (TabularList.from_df(data, cat_names=[], cont_names=cont_names, procs=procs)
                       .split_by_idx(valid_idx=valid_idx)
                       .label_from_df(cols=dep_var)
                       .add_test(TabularList.from_df(data_extend, cat_names=[], cont_names=cont_names, procs=procs))
                       .databunch())
learn = tabular_learner(data_tabular, layers=[celln for i in range(layern)], metrics=accuracy, ps=ps, silent=False)
_ = learn.fit_one_cycle(50)
###
preds, _ = learn.get_preds(ds_type=DatasetType.Valid)
_, pred_class = preds.max(1)
pred_prob = preds[:, 1]
roc_auc_score(data.iloc[valid_idx, :]['Activity'], pred_prob)
###
preds, _ = learn.get_preds(ds_type=DatasetType.Test)
_, pred_class = preds.max(1)
pred_prob = preds[:, 1]
roc_auc_score(data_extend['Activity'], pred_prob)
pickle.dump([data, data_extend, valid_idx, cont_names, dep_var], 
            open('NR_DL_N4000_layer5_cell300_ps0.5.pkl', 'wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14651227939116632, 'fraction_numerical': 0.021383882715472965, 'mean_word_length': 3.935483870967742, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '434595', 'license': 'MIT', 'n_tokens_mistral': 4155, 'n_tokens_neox': 3992, 'n_words': 692, 'path': 'Notebooks/3. DeepLearing_modeling.ipynb', 'repo_name': 'ddhmed/pisnrs', 'size': '10709'}