<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Feb 24 09:33:58 2015

@author: <EMAIL>
"""

# -*- coding: utf-8 -*-
"""
Created on Mon Feb 23 09:21:51 2015

@author: <EMAIL>
"""

import os
import numpy as np
import nibabel as nib
from fetch_data import fetch_adni_petmr, fetch_adni_masks,\
                       set_features_base_dir, set_cache_base_dir
import matplotlib.pyplot as plt

from nilearn.decoding import SpaceNetClassifier
from sklearn.cross_validation import StratifiedShuffleSplit
from sklearn.linear_model import LogisticRegression
from sklearn.datasets.base import Bunch
from scipy import sparse

### 2D array to 4D nifti
def array_to_niis(data, mask_img):
    data_ = np.zeros(data.shape[:1] + mask_img.shape)
    data_[:, mask_img.get_data().astype(np.bool)] = data
    data_ = np.transpose(data_, axes=(1,2,3,0))
    return nib.Nifti1Image(data_, mask_img.get_affine())

### main loop
# (train, test, g1_feat, g2_feat, mask_img)
def train_and_test(train, test, g1_feat, g2_feat,
                   mask_img, w_pet, alpha_, lambda_):
    
    x_train_stacked = []
    x_test_stacked = []
    lgr_coeffs = []
    coeffs = []
    for k in range(1):#g1_feat.shape[2]):
        x = np.concatenate((g1_feat[..., k], g2_feat[..., k]), axis=0)
        xtrain = x[train]
        y_train = y[train]
        xtest = x[test]
        y_test = y[test]

        xtrain_tilde = sparse.vstack([sparse.csr_matrix(xtrain, dtype=np.float64),
                                      np.sqrt(alpha_)*sparse.eye(w_pet.shape[1],
                                      dtype=np.float64)])
        
        y_train_tilde = sparse.vstack([sparse.csr_matrix(np.tile(y_train,(1,1)).T),
                                       sparse.csr_matrix(w_pet.T)])


        xtest_tilde = sparse.vstack([sparse.csr_matrix(xtest, dtype=np.float64),
                                      np.sqrt(alpha_)*sparse.eye(w_pet.shape[1],
                                      dtype=np.float64)])
        
        y_test_tilde = sparse.vstack([sparse.csr_matrix(np.tile(y_test,(1,1)).T),
                                       sparse.csr_matrix(w_pet.T)])


        # spacenet
        spnc = SpaceNetClassifier(penalty='smooth-lasso', eps=1e-1,
                                  mask=mask_img, n_jobs=20, memory=CACHE_DIR,
                                  screening_percentile=100)
                 
        #xtrain_img = array_to_niis(xtrain_tilde, mask_img)
        #xtest_img = array_to_niis(xtest_tilde, mask_img)
        spnc.fit(xtrain_tilde, y_train)
        x_train_stacked.append(spnc.predict(xtrain_tilde))
        x_test_stacked.append(spnc.predict(xtest_tilde))
        coeffs.append(spnc.coef_)

    # stacking
    x_train_ = np.asarray(x_train_stacked).T
    x_test_ = np.asarray(x_test_stacked).T
    lgr = LogisticRegression()
    lgr.fit(x_train_, y_train_tilde)
    lgr_coeffs = lgr.coef_
    acc = lgr.score(x_test_,  y_test_tilde)

    return Bunch(accuracy=acc, coeffs=coeffs, lgr_coeffs=lgr_coeffs)
    
### set paths
CACHE_DIR = set_cache_base_dir()
FIG_DIR = os.path.join(CACHE_DIR, 'figures', 'petmr')
FEAT_DIR = set_features_base_dir()
FMRI_DIR = os.path.join(FEAT_DIR, 'smooth_preproc', 'fmri_subjects_68seeds')

### Load masks
masks = fetch_adni_masks()
mask_img = nib.load(masks['mask_petmr'])

### load fMRI features
dataset = fetch_adni_petmr()
fmri = dataset['func']
subj_list = dataset['subjects']
dx_group = np.array(dataset['dx_group'])
idx = {}
for g in ['AD', 'LMCI', 'EMCI', 'Normal']:
    idx[g] = np.where(dx_group == g)
X = []
for i in np.arange(len(subj_list)):
    X.append(np.load(os.path.join(FMRI_DIR, subj_list[i]+'.npz'))['corr'])
X = np.array(X, copy=False)

### load PET prior
pet_model_path = os.path.join(FEAT_DIR, 'pet_models',
                              'ad_mci_svm_coeffs_pet_diff.npz')
model = np.load(pet_model_path)['svm_coeffs']
w_pet = np.array(model)

### prepare data
g1_feat = X[idx['AD'][0]]
idx_ = np.hstack((idx['LMCI'][0], idx['EMCI'][0]))
g2_feat = X[idx_]
y = np.ones(len(idx['AD'][0]) + len(idx_))
y[len(y) - len(g2_feat):] = 0


### prepare shuffle split
n_iter = 1
sss = StratifiedShuffleSplit(y, n_iter=n_iter, test_size=.2,
                             random_state=np.random.seed(42))

alpha_ = .1
lambda_ = .7

### spacenet !
p = []
for train, test in sss:
    p.append(train_and_test(train, test, g1_feat, g2_feat, mask_img, w_pet, alpha_, lambda_))

"""
from joblib import Parallel, delayed
p = Parallel(n_jobs=40, verbose=5)(delayed(train_and_test)\
(train, test, g1_feat, g2_feat, mask_img, wprior, lambda_) for train, test in sss)
"""<commit_msg>spacenet prior<commit_after># -*- coding: utf-8 -*-
"""
Spacenet using the prior

Created on Wed Feb 11 15:54:46 2015

@author: <EMAIL>
"""

import os
import numpy as np
import nibabel as nib
from fetch_data import set_features_base_dir, fetch_adni_petmr,\
                        set_cache_base_dir, fetch_adni_masks, set_group_indices
from sklearn.cross_validation import StratifiedShuffleSplit
import matplotlib.pyplot as plt
from nilearn.decoding import SpaceNetRegressor
from sklearn.linear_model import LogisticRegression
from sklearn.datasets.base import Bunch


### 2D array to 4D nifti
def array_to_niis(data, mask_img):
    data_ = np.zeros(data.shape[:1] + mask_img.shape)
    data_[:, mask_img.get_data().astype(np.bool)] = data
    data_ = np.transpose(data_, axes=(1,2,3,0))
    return nib.Nifti1Image(data_, mask_img.get_affine())

def train_and_test(train, test, g1_feat, g2_feat, mask_img, w_pet,
                   alpha_, lambda_):
    
    x_train_stacked = []
    x_test_stacked = []
    lgr_coeffs = []
    coeffs = []
    for k in range(g1_feat.shape[2]):

        x = np.concatenate((g1_feat[..., k], g2_feat[..., k]), axis=0)
        xtrain, y_train = x[train], y[train]
        xtrain_img = array_to_niis(xtrain, mask_img)
        xtest, y_test = x[test], y[test]
        xtest_img = array_to_niis(xtest, mask_img)
        
        spnr = SpaceNetRegressor(penalty='smooth-lasso', w_prior=w_pet,
                                 mask=mask_img, alpha_=alpha_,
                                 lambda_=lambda_, max_iter=100, cv=4)
        spnr.fit(xtrain_img, y_train)
        x_train_stacked.append(spnr.predict(xtrain_img))
        x_test_stacked.append(spnr.predict(xtest_img))
        coeffs.append(spnr.coef_)

    x_train_ = np.asarray(x_train_stacked).T
    x_test_ = np.asarray(x_test_stacked).T
    lgr = LogisticRegression()
    lgr.fit(x_train_, y_train)
    lgr_coeffs = lgr.coef_
    acc = lgr.score(x_test_, y_test)

    return Bunch(accuracy=acc, coeffs=coeffs, lgr_coeffs=lgr_coeffs)


### set paths
CACHE_DIR = set_cache_base_dir()
FIG_DIR = os.path.join(CACHE_DIR, 'figures', 'petmr')
FEAT_DIR = set_features_base_dir()
FMRI_DIR = os.path.join(FEAT_DIR, 'smooth_preproc', 'fmri_subjects_68seeds')


### load fMRI features
masks = fetch_adni_masks()
mask_img = nib.load(masks['mask_petmr'])
dataset = fetch_adni_petmr()
fmri = dataset['func']
subj_list = dataset['subjects']
dx_group = np.array(dataset['dx_group'])
idx = set_group_indices(dx_group)    
X = []
for i in np.arange(len(subj_list)):
    X.append(np.load(os.path.join(FMRI_DIR, subj_list[i]+'.npz'))['corr'])
X = np.array(X, copy=False)

### load PET a priori
pet_model_path = os.path.join(FEAT_DIR, 'pet_models',
                              'svm_coeffs_pet_diff.npz')
model = np.load(pet_model_path)['svm_coeffs']
w_pet = np.array(model)
w_pet = w_pet[0,:]
#w_pet = w_pet/np.max(w_pet)

### prepare data
g1_feat = X[idx['AD'][0]]
#idx_ = idx['Normal'][0]
idx_ = np.hstack((idx['LMCI'][0], idx['EMCI'][0]))
g2_feat = X[idx_]
y = np.ones(len(idx['AD'][0]) + len(idx_))
y[len(y) - len(g2_feat):] = 0


n_iter = 100
sss = StratifiedShuffleSplit(y, n_iter=n_iter, test_size=.2,
                             random_state=np.random.seed(42))

lambda_ = 3.7
alpha_ = .5
"""
for train, test in sss:
    p = train_and_test(train, test, g1_feat, g2_feat,
                       mask_img, w_pet, alpha_, lambda_)
"""
from joblib import Parallel, delayed
p = Parallel(n_jobs=20, verbose=5)(delayed(train_and_test)\
(train, test, g1_feat, g2_feat, mask_img) for train, test in sss)

np.savez_compressed('spacenet_prior_sl_'+str(n_iter),data=p)
{'fraction_non_alphanumeric': 0.13779718583212033, 'fraction_numerical': 0.017952450266860747, 'mean_word_length': 3.2087799897907097, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6857080', 'n_tokens_mistral': 3280, 'n_tokens_neox': 3158, 'n_words': 562}