<jupyter_start><jupyter_text>Beta ExtractionReplicating LSA regression using scikit-learn just to make sure they're okay.<jupyter_code>%matplotlib inline
%load_ext autoreload
%autoreload 2
import json
import sys
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

if sys.platform == 'darwin':
    cfg = os.path.join("/Users", "njchiang", "CloudStation", "Grad",
                       "Research", "montilab-ucla", "analogy", "config", "project.json")
    plat = "osx"

elif sys.platform == "linux":
    import platform
    if platform.linux_distribution()[0] == "debian":
        cfg = os.path.join("/home", "njchiang", "data", "CloudStation", "Grad",
                           "Research", "montilab-ucla", "analogy", "config", "project.json")
        plat = "linux"
    else:
        cfg = os.path.join("/u", "project", "monti", "Analysis", "Analogy",
                           "code", "analogy", "config", "project.json")
        plat = "hoff"
else:
    cfg = os.path.join("D:\\", "CloudStation", "Grad",
                       "Research", "montilab-ucla", "analogy", "config", "project.json")
    plat = "win"

with open(cfg, "r") as f:
    projectSettings = json.load(f)

paths = projectSettings["filepaths"]["{}Paths".format(plat)]
sys.path.append(paths["github"])
sys.path.append(paths["code"])
from fmri.beta_extract import create_lss_from_lsa, load_aggregated_data, calc_scores
from fmri.analogy_utils import analysisSettings, contrastSettings, order, \
    pu, pa, pv, compile_models, rsa, save_rois, load_rois

from sklearn.linear_model import LinearRegression
import statsmodels.api as sm 
from sklearn.externals.joblib import Parallel, delayed
x = np.random.rand(100, 10)
y = np.random.rand(100, 2)
model = sm.OLS(y[:, 0], x)
results = model.fit()
scores = Parallel(n_jobs=-2)(delayed(calc_scores)(y[:, v], x) for v in range(y.shape[1]))
# Run LSS extraction!
maskname = "grayMatter-bin_mask"
for sub in projectSettings["subjects"]:
    mask = pu.load_img(
        os.path.join(paths["root"], "derivatives", sub, "masks",
                     "{}.nii.gz".format(maskname)), logger=None)
    fmri_data, design = load_aggregated_data(paths, sub, maskname)
    betas = np.zeros([144, fmri_data.shape[1]])
    tstats = np.zeros([144, fmri_data.shape[1]])
    for i, wordpair in enumerate(order.ABTag[::2]):
        print(wordpair)
        regressors = create_lss_from_lsa(design, wordpair, tt="AB")
#         scores = Parallel(n_jobs=-2)(delayed(calc_scores)(fmri_data[:, v], regressors) for v in range(fmri_data.shape[1]))
#         scores = np.array(scores)
#         betas[i] = scores[:, 0]
#         tstats[i] = scores[:, 1]
        betas[i] = LinearRegression().fit(regressors, fmri_data).coef_[:, 0]
    pu.unmask_img(betas, mask).to_filename(
        os.path.join(paths["root"], "derivatives", sub, "betas",
                    "{}_task-analogy_betas-cope-LSS-condensed.nii.gz".format(sub)))
#     pu.unmask_img(tstats, mask).to_filename(
#         os.path.join(paths["root"], "derivatives", sub, "betas",
#                     "{}_task-analogy_betas-tstat-LSS-condensed.nii.gz".format(sub)))<jupyter_output>Reading file from: D:\fmri\Analogy\derivatives\sub-01\masks\grayMatter-bin_mask.nii.gz
Reading file from: D:\fmri\Analogy\derivatives\sub-01\masks\grayMatter-bin_mask.nii.gz
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\func\sub-01_task-analogy_run-01_events.csv
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\mc\run-01\prefiltered_func_data_mcf_final.par
Applying mask
Creating design matrix at 2018-10-17 07:42:47.348655
Ended at 2018-10-17 07:42:47.507657
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\func\sub-01_task-analogy_run-02_events.csv
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\mc\run-02\prefiltered_func_data_mcf_final.par
Applying mask
Creating design matrix at 2018-10-17 07:42:58.912630
Ended at 2018-10-17 07:42:59.028615
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\func\sub-01_task-analogy_run-03_events.csv
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\mc\run-03\prefiltered_func_data_mcf_fin[...]<jupyter_text>Scratch<jupyter_code>maskname = "grayMatter-bin_mask"
mask = pu.load_img(
        os.path.join(paths["root"], "derivatives", "sub-01", "masks",
                     "{}.nii.gz".format(maskname)), logger=None)
pu.unmask_img(betas, mask).to_filename(
    os.path.join(paths["root"], "derivatives", "sub-01", "betas",
                "sub-01_task-analogy_betas-LSS-condensed.nii.gz"))
# sort by columns and take every 2...3?
for tag in test:
    if ":" in tag:
        trialtype = tag.split("_")[1]
        if trialtype == "AB":
            this_tag = tag.split("_")[0].split("::")[0]
        elif trialtype == "CD":
            this_tag = tag.split("_")[0].split("::")[1]
        else: 
            this_tag = tag.split("_")[0]


n_trials = 108
test = des.matrix
i = []
nuisance = test[:, n_trials:]
trial_regs = test[:, :n_trials]
if sum(i) > 1:
    trial = trial_regs[:, i]
    trial_nuisance = np.sum(np.delete(trial_regs, i, axis=1), axis=1, keepdims=True)
else:
    trial = np.zeros([trial_regs.shape[0], 1])
    trial_nuisance = np.sum(trial_regs, axis=1, keepdims=True)

this_design = np.hstack([trial, trial_nuisance, nuisance])
this_design
labels[labels.ABTag == "Hello Bloop"].index
test[:, i:i+1]
frametimes = np.arange(0, fmri_data.shape[0], tr)
cond_ids = [i for i, r in test_l.iterrows()]

onsets = labels["Onset"]
durations = labels["Duration"]
design_kwargs = {"drift_model": "blank", "add_regs": motion.values}

LSA_des = pa.make_designmat(frametimes, cond_ids, onsets, durations,
                            design_kwargs=design_kwargs, logger=logger)
for sub, runs in projectSettings["subjects"].items():
    [regress_betas(paths, sub, r, maskname="template-bin_mask", write=True) for r in runs]
masks_dict = {"ba10": "dfc-left-BA10-bin_mask",
              "lofc": "aal-LOFC-bin_mask",
              "ba46": "dfc-left-BA46-bin_mask",
              "lpitg": "anat-left-pITG-bin_mask",
              "laitg": "anat-left-aITG-bin_mask",
              "lastg": "anat-left-aSTG-bin_mask",
              "lifg": "anat-ba44-45_mask",
              "lips": "anat-LIPS_mask",
              "v1": "aal-V1-bin_mask",
              "csf": "csf-bin_mask"}
save_rois(masks_dict, t="LSA")
save_rois(masks_dict, t="cope-LSS")
save_rois(masks_dict, t="pymvpa")<jupyter_output>Writing sub-01 betas
centering image
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\func\sub-01_task-analogy_run-01_events.csv
centering image
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\func\sub-01_task-analogy_run-02_events.csv
centering image
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\func\sub-01_task-analogy_run-03_events.csv
centering image
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\func\sub-01_task-analogy_run-04_events.csv
centering image
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\func\sub-01_task-analogy_run-05_events.csv
centering image
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\func\sub-01_task-analogy_run-06_events.csv
centering image
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\func\sub-01_task-analogy_run-07_events.csv
centering image
Loading label file from: D:\fmri\Analogy\derivatives\sub-01\func\sub-01_task-analogy_run-08_events.csv
Concatenating 8 images
Read[...]
{'fraction_non_alphanumeric': 0.15450643776824036, 'fraction_numerical': 0.03125, 'mean_word_length': 3.7709532949456173, 'pattern_counts': {'":': 19, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '492361', 'license': 'MIT', 'n_tokens_mistral': 2909, 'n_tokens_neox': 2730, 'n_words': 508, 'path': 'notebooks/BetaExtract.ipynb', 'repo_name': 'njchiang/analogy-fmri', 'size': '7456'}