<commit_before># <NAME> and <NAME> 2015
# Some code taken from nilearn searchlight implementation: https://github.com/nilearn/nilearn/blob/master/nilearn/decoding/searchlight.py

import os

import time
import sys
import warnings
from distutils.version import LooseVersion
import random

import cPickle 
import numpy as np
import matplotlib.pyplot as plt
from nilearn import datasets
from nilearn import plotting

import nibabel as nib

import sklearn
from sklearn import neighbors
from sklearn.externals.joblib import Parallel, delayed, cpu_count
from sklearn import svm
from sklearn.cross_validation import cross_val_score
from sklearn.base import BaseEstimator
from sklearn import neighbors
from sklearn.svm import SVR

from nilearn import masking
from nilearn.input_data import NiftiMasker

from nltools.analysis import Predict
import glob

########## HELPER FUNCTIONS ############

# # parallelizable searchlight function (called within different cores)
# def parallel_searchlight(X, y, estimator, A, n_jobs=-1,verbose=0):
#     group_iter = GroupIterator(A.shape[0], n_jobs)
#     scores = Parallel(n_jobs=n_jobs, verbose=verbose)(
#         delayed(_group_iter_search_light)(
#             A.rows[list_i],
#             estimator, X, y,
#             thread_id + 1, A.shape[0], verbose)
#         for thread_id, list_i in enumerate(group_iter))
#     return np.concatenate(scores)


# #code that is executed on each of the cores (core searchlight functionality)
# def _group_iter_search_light(list_rows, estimator, X, y,
#                              scoring, thread_id, total, verbose=0):
#     par_scores = np.zeros(len(list_rows))
#     t0 = time.time()
#     for i, row in enumerate(list_rows):
#         kwargs = dict()
#         if not LooseVersion(sklearn.__version__) < LooseVersion('0.15'):
#             kwargs['scoring'] = scoring
#         elif scoring is not None:
#             warnings.warn('Scikit-learn version is too old. '
#                           'scoring argument ignored', stacklevel=2)
#         #import random
#         #par_scores[i] = random.random() #np.mean(cross_val_score(estimator, X[:, row],y, n_jobs=1,**kwargs))
        
        
#         ######## REGRESSION HAPPENS HERE ########
#         #svr = Predict(X[:, row], y, algorithm='svr', cv_dict = {'kfolds':5}, **{'kernel':"linear"})
#         #svr.predict(save_images=False, save_output=False).rmse
        
#         #svr_lin = SVR(kernel='linear', C=1e3)
#         #y_lin = svr_lin.fit(X, y).predict(X)
        
#         import random
#         par_scores[i] = random.random()
        
#         ########
        
        
#         if verbose > 0:
#             # One can't print less than each 10 iterations
#             step = 11 - min(verbose, 10)
#             if (i % step == 0):
#                 # If there is only one job, progress information is fixed
#                 if total == len(list_rows):
#                     crlf = "\r"
#                 else:
#                     crlf = "\n"
#                 percent = float(i) / len(list_rows)
#                 percent = round(percent * 100, 2)
#                 dt = time.time() - t0
#                 # We use a max to avoid a division by zero
#                 remaining = (100. - percent) / max(0.01, percent) * dt
#                 sys.stderr.write(
#                     "Job #%d, processed %d/%d voxels "
#                     "(%0.2f%%, %i seconds remaining)%s"
#                     % (thread_id, i, len(list_rows), percent, remaining, crlf))
#     return par_scores

# # iterator that helps with parallelization process
# class GroupIterator(object):
#     """Group iterator
#     Provides group of features for search_light loop
#     that may be used with Parallel.
#     Parameters
#     ----------
#     n_features : int
#         Total number of features
#     n_jobs : int, optional
#         The number of CPUs to use to do the computation. -1 means
#         'all CPUs'. Defaut is 1
#     """
#     def __init__(self, n_features, n_jobs=1):
#         self.n_features = n_features
#         if n_jobs == -1:
#             n_jobs = cpu_count()
#         self.n_jobs = n_jobs

#     def __iter__(self):
#         split = np.array_split(np.arange(self.n_features), self.n_jobs)
#         for list_i in split:
#             yield list_i
    
########## CLASS DEFINITIONS ############

class Searchlight:
    def __init__(self, brain_mask=None, process_mask=None, radius=4): #no scoring param
        self.resource_folder = os.path.join(os.getcwd(),'resources')
        self.outfolder = os.path.join(os.getcwd(),'outfolder')
        
        if type(brain_mask) is str:
            brain_mask = nib.load(brain_mask)
        elif brain_mask is None:
            brain_mask = nib.load(os.path.join(self.resource_folder,'MNI152_T1_2mm_brain_mask.nii.gz'))
        elif type(brain_mask) is not nib.nifti1.Nifti1Image:
            print(brain_mask)
            print(type(brain_mask))
            raise ValueError("brain_mask is not a nibabel instance")
        self.brain_mask = brain_mask
        
        if type(process_mask) is str:
            process_mask = nib.load(process_mask)
        elif process_mask is None:
            process_mask = nib.load(os.path.join(self.resource_folder,"FSL_RIns_thr0.nii.gz"))
        elif type(brain_mask) is not nib.nifti1.Nifti1Image:
            print(process_mask)
            print(type(process_mask))
            raise ValueError("process_mask is not a nibabel instance")
        self.process_mask = process_mask
            
        self.radius = radius
        self.nifti_masker = NiftiMasker(mask_img=self.brain_mask)
        
    def predict(self, core_i, n_cores, params): #CHANGE NAME
        
        (bdata, A, self.nifti_masker, subject_id, algorithm, cv_dict, output_dir, kwargs) = params
        
        print("getting data")
        if isinstance(bdata, str):<commit_msg>change one default filename<commit_after># <NAME> and <NAME> 2015
# Some code taken from nilearn searchlight implementation: https://github.com/nilearn/nilearn/blob/master/nilearn/decoding/searchlight.py

import os

import time
import sys
import warnings
from distutils.version import LooseVersion
import random

import cPickle 
import numpy as np
import matplotlib.pyplot as plt
from nilearn import datasets
from nilearn import plotting

import nibabel as nib

import sklearn
from sklearn import neighbors
from sklearn.externals.joblib import Parallel, delayed, cpu_count
from sklearn import svm
from sklearn.cross_validation import cross_val_score
from sklearn.base import BaseEstimator
from sklearn import neighbors
from sklearn.svm import SVR

from nilearn import masking
from nilearn.input_data import NiftiMasker

from nltools.analysis import Predict
import glob

########## HELPER FUNCTIONS ############

# # parallelizable searchlight function (called within different cores)
# def parallel_searchlight(X, y, estimator, A, n_jobs=-1,verbose=0):
#     group_iter = GroupIterator(A.shape[0], n_jobs)
#     scores = Parallel(n_jobs=n_jobs, verbose=verbose)(
#         delayed(_group_iter_search_light)(
#             A.rows[list_i],
#             estimator, X, y,
#             thread_id + 1, A.shape[0], verbose)
#         for thread_id, list_i in enumerate(group_iter))
#     return np.concatenate(scores)


# #code that is executed on each of the cores (core searchlight functionality)
# def _group_iter_search_light(list_rows, estimator, X, y,
#                              scoring, thread_id, total, verbose=0):
#     par_scores = np.zeros(len(list_rows))
#     t0 = time.time()
#     for i, row in enumerate(list_rows):
#         kwargs = dict()
#         if not LooseVersion(sklearn.__version__) < LooseVersion('0.15'):
#             kwargs['scoring'] = scoring
#         elif scoring is not None:
#             warnings.warn('Scikit-learn version is too old. '
#                           'scoring argument ignored', stacklevel=2)
#         #import random
#         #par_scores[i] = random.random() #np.mean(cross_val_score(estimator, X[:, row],y, n_jobs=1,**kwargs))
        
        
#         ######## REGRESSION HAPPENS HERE ########
#         #svr = Predict(X[:, row], y, algorithm='svr', cv_dict = {'kfolds':5}, **{'kernel':"linear"})
#         #svr.predict(save_images=False, save_output=False).rmse
        
#         #svr_lin = SVR(kernel='linear', C=1e3)
#         #y_lin = svr_lin.fit(X, y).predict(X)
        
#         import random
#         par_scores[i] = random.random()
        
#         ########
        
        
#         if verbose > 0:
#             # One can't print less than each 10 iterations
#             step = 11 - min(verbose, 10)
#             if (i % step == 0):
#                 # If there is only one job, progress information is fixed
#                 if total == len(list_rows):
#                     crlf = "\r"
#                 else:
#                     crlf = "\n"
#                 percent = float(i) / len(list_rows)
#                 percent = round(percent * 100, 2)
#                 dt = time.time() - t0
#                 # We use a max to avoid a division by zero
#                 remaining = (100. - percent) / max(0.01, percent) * dt
#                 sys.stderr.write(
#                     "Job #%d, processed %d/%d voxels "
#                     "(%0.2f%%, %i seconds remaining)%s"
#                     % (thread_id, i, len(list_rows), percent, remaining, crlf))
#     return par_scores

# # iterator that helps with parallelization process
# class GroupIterator(object):
#     """Group iterator
#     Provides group of features for search_light loop
#     that may be used with Parallel.
#     Parameters
#     ----------
#     n_features : int
#         Total number of features
#     n_jobs : int, optional
#         The number of CPUs to use to do the computation. -1 means
#         'all CPUs'. Defaut is 1
#     """
#     def __init__(self, n_features, n_jobs=1):
#         self.n_features = n_features
#         if n_jobs == -1:
#             n_jobs = cpu_count()
#         self.n_jobs = n_jobs

#     def __iter__(self):
#         split = np.array_split(np.arange(self.n_features), self.n_jobs)
#         for list_i in split:
#             yield list_i
    
########## CLASS DEFINITIONS ############

class Searchlight:
    def __init__(self, brain_mask=None, process_mask=None, radius=4): #no scoring param
        self.resource_folder = os.path.join(os.getcwd(),'resources')
        self.outfolder = os.path.join(os.getcwd(),'outfolder')
        
        if type(brain_mask) is str:
            brain_mask = nib.load(brain_mask)
        elif brain_mask is None:
            brain_mask = nib.load(os.path.join(self.resource_folder,'MNI152_T1_2mm_brain_mask_dil.nii.gz'))
        elif type(brain_mask) is not nib.nifti1.Nifti1Image:
            print(brain_mask)
            print(type(brain_mask))
            raise ValueError("brain_mask is not a nibabel instance")
        self.brain_mask = brain_mask
        
        if type(process_mask) is str:
            process_mask = nib.load(process_mask)
        elif process_mask is None:
            process_mask = nib.load(os.path.join(self.resource_folder,"FSL_RIns_thr0.nii.gz"))
        elif type(brain_mask) is not nib.nifti1.Nifti1Image:
            print(process_mask)
            print(type(process_mask))
            raise ValueError("process_mask is not a nibabel instance")
        self.process_mask = process_mask
            
        self.radius = radius
        self.nifti_masker = NiftiMasker(mask_img=self.brain_mask)
        
    def predict(self, core_i, n_cores, params): #CHANGE NAME
        
        (bdata, A, self.nifti_masker, subject_id, algorithm, cv_dict, output_dir, kwargs) = params
        
        print("getting data")
        if isinstance(bdata, str):
{'fraction_non_alphanumeric': 0.11338635979015062, 'fraction_numerical': 0.009307835505161618, 'mean_word_length': 2.3143578238923164, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6575384', 'n_tokens_mistral': 3775, 'n_tokens_neox': 3591, 'n_words': 936}