<jupyter_start><jupyter_text>\*\*\*needs cleaning***<jupyter_code>import pandas as pd
import numpy as np
import sys 
import os
import itertools
import time
import random

#import utils
sys.path.insert(0, '../utils/')
from utils_preprocess_v3 import *
from utils_modeling_v9 import *
from utils_plots_v2 import *

#sklearn
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

start_time = time.time()
data = pd.read_csv('../data/datasets_processed/OpenPBTA_data_mean.csv', index_col='Unnamed: 0', dtype = 'unicode')
data = data.T.reset_index().rename(columns = {'index' : 'node'})
response = pd.read_csv('../data/datasets_processed/OpenPBTA_response.csv', index_col='Kids_First_Biospecimen_ID')
interactome = pd.read_csv('../data/interactomes/inbiomap_processed.txt', sep = '\t')
# get nodes from data and graph
data_nodes = data['node'].tolist()
interactome_nodes = list(set(np.concatenate((interactome['node1'], interactome['node2']))))

# organize data
organize = Preprocessing()
save_location = '../data/reduced_interactomes/reduced_interactome_OpenPBTA.txt'
organize.transform(data_nodes, interactome_nodes, interactome, data, save_location, load_graph = True)
# extract info from preprocessing
X = organize.sorted_X.T.values
y = response.values.reshape(-1,1)
L_norm = organize.L_norm
L = organize.L

g = organize.g
num_to_node = organize.num_to_node

# split for training
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42)

# scaling X
scaler_X = StandardScaler()
X_train = scaler_X.fit_transform(X_train)
X_test = scaler_X.transform(X_test)

# scalying y
scaler_y = StandardScaler()
y_train = scaler_y.fit_transform(y_train).reshape(-1)
y_test = scaler_y.transform(y_test).reshape(-1)
val_1, vec_1 = scipy.linalg.eigh(L_norm.toarray())
val_zeroed = val_1 - min(val_1) + 1e-8
L_rebuild = vec_1.dot(np.diag(val_zeroed)).dot(np.linalg.inv(vec_1))
X_train_lower = np.linalg.cholesky(L_rebuild)
X_train_lower.dot(X_train_lower.T).sum()
L_norm.sum()
np.save('L_half.npy', X_train_lower)<jupyter_output><empty_output><jupyter_text>Lasso + LapRidge<jupyter_code># hyperparameters
alpha1_list = np.logspace(-1,0,15)
alpha2_list = np.logspace(-1,2,15)
threshold_list = np.logspace(-3,-1,10)
max_features = 10
alpha_pairs = list(itertools.product(alpha1_list, alpha2_list))
def loss_fn(X,Y, L, alpha1, alpha2, beta):
    return 0.5/(len(X)) * cp.norm2(cp.matmul(X, beta) - Y)**2 + \
           alpha1 * cp.norm1(beta) + \
           alpha2 * cp.sum(cp.quad_form(beta,L)) 

def run(pair, X_train, y_train, L_norm):
    beta = cp.Variable(X_train.shape[1])
    alpha1 = cp.Parameter(nonneg=True)
    alpha2 = cp.Parameter(nonneg=True)
    alpha1.value = pair[0]
    alpha2.value = pair[1]
    
    problem = cp.Problem(cp.Minimize(loss_fn(X_train, y_train, L_norm, alpha1, alpha2, beta )))
    problem.solve(solver=cp.SCS, verbose=True, max_iters=50000)
    
    np.save('openpbta/' + str(pair) + '.npy', beta.value)

    return beta.value
betas = Parallel(n_jobs=8, verbose=10)(delayed(run)(alpha_pairs[i],
                                                       X_train,
                                                       y_train,
                                                       L_norm) for i in range(len(alpha_pairs)))
# load betas
beta_order = [str(i) + '.npy' for i in alpha_pairs]
betas = [np.load('openpbta/' + i) for i in beta_order]
feats = [getFeatures(None, i, threshold=0.001, max_features=10) for i in betas]
regr = LinearRegression()
scores = [getScoring(regr, X_train, y_train, X_test, y_test, i, None) for i in feats]
train_scores = [i[0] for i in scores]
test_scores = [i[1] for i in scores]
gridsearch_results = pd.DataFrame(np.array(test_scores), columns = ['Test MSE'])
getGridsearchPlot(gridsearch_results, alpha1_list, alpha2_list, save_location = None)
np.where(test_scores == min(test_scores))
min(test_scores)
getTranslatedNodes(feats[17], betas[17][feats[17]], num_to_node, g, )<jupyter_output><empty_output><jupyter_text>MCP + LapRidge<jupyter_code># define training params
alpha1_list = np.logspace(-3,-2,15)
alpha2_list = np.logspace(-1,2,15)
threshold_list = np.logspace(-3,-1,10)
max_features = 10
alpha_list_pairs = list(itertools.product(alpha1_list, alpha2_list))
results = {}
feats_list = []
betas = []
for i in alpha2_list:
    X_train_new = np.vstack((X_train, np.sqrt(i)*X_train_lower))
    y_train_new = np.concatenate((y_train, np.zeros(len(X_train_lower))))
    s = pycasso.Solver(X_train_new, y_train_new, lambdas=alpha1_list, penalty = 'mcp')
    s.train()
    beta = s.coef()['beta']
    
    betas += [i for i in beta]
    
    feats = [getFeatures(None, i, threshold=0.001, max_features=10) for i in beta]
    feats_list += feats
    print([len(i) for i in feats])
    
    regr = LinearRegression()
    scores = [getScoring(regr, X_train, y_train, X_test, y_test, i, None) for i in feats]
    
    results[i] = scores
train_scores = []
test_scores = []
for k,v in results.items():
    train_scores += [i[0] for i in v]
    test_scores += [i[1] for i in v]
gridsearch_results = pd.DataFrame(np.array(test_scores), columns = ['Test MSE'])
getGridsearchPlot(gridsearch_results, alpha1_list, alpha2_list, save_location = None)
min(test_scores)
np.where(test_scores == min(test_scores))
getTranslatedNodes(feats_list[75], betas[75][feats_list[75]], num_to_node, g)<jupyter_output><empty_output><jupyter_text>SCAD + LapRidge<jupyter_code># define training params
alpha1_list = np.logspace(-3,-2,15)
alpha2_list = np.logspace(-1,2,15)
threshold_list = np.logspace(-3,-1,10)
max_features = 10
alpha_list_pairs = list(itertools.product(alpha1_list, alpha2_list))
results = {}
feats_list = []
betas = []
for i in alpha2_list:
    X_train_new = np.vstack((X_train, np.sqrt(i)*X_train_lower))
    y_train_new = np.concatenate((y_train, np.zeros(len(X_train_lower))))
    s = pycasso.Solver(X_train_new, y_train_new, lambdas=alpha1_list, penalty = 'scad')
    s.train()
    beta = s.coef()['beta']
    
    betas += [i for i in beta]
    
    feats = [getFeatures(None, i, threshold=0.001, max_features=10) for i in beta]
    feats_list += feats
    print([len(i) for i in feats])
    
    regr = LinearRegression()
    scores = [getScoring(regr, X_train, y_train, X_test, y_test, i, None) for i in feats]
    
    results[i] = scores
train_scores = []
test_scores = []
for k,v in results.items():
    train_scores += [i[0] for i in v]
    test_scores += [i[1] for i in v]
gridsearch_results = pd.DataFrame(np.array(test_scores), columns = ['Test MSE'])
getGridsearchPlot(gridsearch_results, alpha1_list, alpha2_list, save_location = None)
min(test_scores)
np.where(test_scores == min(test_scores))
getTranslatedNodes(feats_list[201], betas[201][feats_list[201]], num_to_node, g)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13750544504138232, 'fraction_numerical': 0.023086975461013504, 'mean_word_length': 4.659819227608875, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '14870', 'license': 'MIT', 'n_tokens_mistral': 2698, 'n_tokens_neox': 2603, 'n_words': 504, 'path': 'notebooks/OpenPBTA_LapRidge.ipynb', 'repo_name': 'microsoft/normalized_trend_filtering', 'size': '6887'}