<commit_before>from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
import os
import nrrd
from kam_interface.matrices import generate_voxel_matrices
from kam_interface.utilities import *
from scipy.io import savemat

class MCC_fix(MouseConnectivityCache):
     def get_structure_mask(self, structure_id, file_name=None,
                            annotation_file_name=None):
        file_name = self.get_cache_path(file_name, self.STRUCTURE_MASK_KEY,
                                        structure_id)
        if os.path.exists(file_name):
            return nrrd.read(file_name)
        else:
            ont = self.get_ontology()
            structure_ids = ont.get_descendant_ids([structure_id])
            annotation, _ = self.get_annotation_volume(annotation_file_name)
            mask = self.make_structure_mask(structure_ids, annotation)
            
            if self.cache:
                self.safe_mkdir(os.path.dirname(file_name))
                nrrd.write(file_name, mask)

            return mask, None

# setup the run
param_fn='run_setup.py'
with open(param_fn) as f:
    code = compile(f.read(), param_fn, 'exec')
    exec(code)

mcc = MCC_fix(manifest_file=os.path.join(data_dir,'manifest.json'),
              resolution=resolution)
ontology = mcc.get_ontology()
sources = ontology[source_acronyms]
targets = ontology[target_acronyms]

if experiments_fn is not None:
    LIMS_id_list=unpickle(experiments_fn)
else:
    LIMS_id_list=None

experiment_dict=generate_voxel_matrices(mcc, sources, targets,
                                        LIMS_id_list=LIMS_id_list,
                                        min_voxels_per_injection=min_vox,
                                        laplacian=laplacian,
                                        verbose=True,
                                        source_shell=source_shell,
                                        source_coverage=source_coverage,
                                        fit_gaussian=fit_gaussian,
                                        cre=cre)
experiment_dict['source_acro']=np.array(source_acronyms,dtype=np.object)
experiment_dict['source_ids']=np.array(sources.id)
experiment_dict['target_acro']=np.array(target_acronyms,dtype=np.object)
experiment_dict['target_ids']=np.array(targets.id)

try:
    os.mkdir(save_dir)
except OSError:
    pass
save_file_name=os.path.join(save_dir,save_stem + '.mat')
savemat(save_file_name,experiment_dict,oned_as='column',do_compression=True)
if save_mtx:
    # only save X, Y, Lx, Ly
    from scipy.io import mmwrite
    X=experiment_dict['experiment_source_matrix'].T
    Y_ipsi=experiment_dict['experiment_target_matrix_ipsi'].T
    Y_contra=experiment_dict['experiment_target_matrix_contra'].T
    Lx=experiment_dict['Lx'].T
    Ly_ipsi=experiment_dict['Ly_ipsi'].T
    Ly_contra=experiment_dict['Ly_contra'].T
    Omega=experiment_dict['Omega'].T
    h5write(os.path.join(save_dir,save_stem+'_X.h5'),X)
    h5write(os.path.join(save_dir,save_stem+'_Y_ipsi.h5'),Y_ipsi)
    h5write(os.path.join(save_dir,save_stem+'_Y_contra.h5'),Y_contra)
    Lx_fn=absjoin(save_dir,save_stem+'_Lx.mtx')
    Ly_ipsi_fn=absjoin(save_dir,save_stem+'_Ly_ipsi.mtx')
    Ly_contra_fn=absjoin(save_dir,save_stem+'_Ly_contra.mtx')
    mmwrite(Lx_fn,Lx)
    mmwrite(Ly_ipsi_fn,Ly_ipsi)
    mmwrite(Ly_contra_fn,Ly_contra)
    mmwrite(os.path.join(save_dir,save_stem+'_Omega.h5'),Omega)
    h5write(os.path.join(save_dir,save_stem+'_W0_ipsi.h5'),
            np.zeros((Y_ipsi.shape[0],X.shape[0])))
    h5write(os.path.join(save_dir,save_stem+'_W0_contra.h5'),
            np.zeros((Y_contra.shape[0],X.shape[0])))
    if cross_val_matrices:
        from sklearn import cross_validation
        fid=open(cmdfile,'w')
        n_inj=X.shape[1]
        # Sets up nested outer/inner cross-validation. The inner loop is for
        # model selection (validation), the outer for testing.
        if cross_val=='LOO':
            outer_sets=cross_validation.LeaveOneOut(n_inj)
        else:
            outer_sets=cross_validation.KFold(n_inj,n_folds=cross_val)
        for i,(train,test) in enumerate(outer_sets):
            X_train=X[:,train]
            X_test=X[:,test]
            Y_train_ipsi=Y_ipsi[:,train]
            Y_test_ipsi=Y_ipsi[:,test]
            Omega_train = Omega[:,train]
            Omega_test = Omega[:,test]
            Y_train_contra=Y_contra[:,train]
            Y_test_contra=Y_contra[:,test]<commit_msg>fixed dealing with sparse matrices<commit_after>from allensdk.core.mouse_connectivity_cache import MouseConnectivityCache
import os
import nrrd
from kam_interface.matrices import generate_voxel_matrices
from kam_interface.utilities import *
from scipy.io import savemat

class MCC_fix(MouseConnectivityCache):
     def get_structure_mask(self, structure_id, file_name=None,
                            annotation_file_name=None):
        file_name = self.get_cache_path(file_name, self.STRUCTURE_MASK_KEY,
                                        structure_id)
        if os.path.exists(file_name):
            return nrrd.read(file_name)
        else:
            ont = self.get_ontology()
            structure_ids = ont.get_descendant_ids([structure_id])
            annotation, _ = self.get_annotation_volume(annotation_file_name)
            mask = self.make_structure_mask(structure_ids, annotation)
            
            if self.cache:
                self.safe_mkdir(os.path.dirname(file_name))
                nrrd.write(file_name, mask)

            return mask, None

# setup the run
param_fn='run_setup.py'
with open(param_fn) as f:
    code = compile(f.read(), param_fn, 'exec')
    exec(code)

mcc = MCC_fix(manifest_file=os.path.join(data_dir,'manifest.json'),
              resolution=resolution)
ontology = mcc.get_ontology()
sources = ontology[source_acronyms]
targets = ontology[target_acronyms]

if experiments_fn is not None:
    LIMS_id_list=unpickle(experiments_fn)
else:
    LIMS_id_list=None

experiment_dict=generate_voxel_matrices(mcc, sources, targets,
                                        LIMS_id_list=LIMS_id_list,
                                        min_voxels_per_injection=min_vox,
                                        laplacian=laplacian,
                                        verbose=True,
                                        source_shell=source_shell,
                                        source_coverage=source_coverage,
                                        fit_gaussian=fit_gaussian,
                                        cre=cre)
experiment_dict['source_acro']=np.array(source_acronyms,dtype=np.object)
experiment_dict['source_ids']=np.array(sources.id)
experiment_dict['target_acro']=np.array(target_acronyms,dtype=np.object)
experiment_dict['target_ids']=np.array(targets.id)

try:
    os.mkdir(save_dir)
except OSError:
    pass
save_file_name=os.path.join(save_dir,save_stem + '.mat')
savemat(save_file_name,experiment_dict,oned_as='column',do_compression=True)
if save_mtx:
    # only save X, Y, Lx, Ly
    from scipy.io import mmwrite
    X=experiment_dict['experiment_source_matrix'].T
    Y_ipsi=experiment_dict['experiment_target_matrix_ipsi'].T
    Y_contra=experiment_dict['experiment_target_matrix_contra'].T
    Lx=experiment_dict['Lx'].T
    Ly_ipsi=experiment_dict['Ly_ipsi'].T
    Ly_contra=experiment_dict['Ly_contra'].T
    Omega=experiment_dict['Omega'].T
    h5write(os.path.join(save_dir,save_stem+'_X.h5'),X)
    h5write(os.path.join(save_dir,save_stem+'_Y_ipsi.h5'),Y_ipsi)
    h5write(os.path.join(save_dir,save_stem+'_Y_contra.h5'),Y_contra)
    Lx_fn=absjoin(save_dir,save_stem+'_Lx.mtx')
    Ly_ipsi_fn=absjoin(save_dir,save_stem+'_Ly_ipsi.mtx')
    Ly_contra_fn=absjoin(save_dir,save_stem+'_Ly_contra.mtx')
    mmwrite(Lx_fn,Lx)
    mmwrite(Ly_ipsi_fn,Ly_ipsi)
    mmwrite(Ly_contra_fn,Ly_contra)
    mmwrite(os.path.join(save_dir,save_stem+'_Omega.h5'),Omega)
    # h5write(os.path.join(save_dir,save_stem+'_W0_ipsi.h5'),
    #         np.zeros((Y_ipsi.shape[0],X.shape[0])))
    # h5write(os.path.join(save_dir,save_stem+'_W0_contra.h5'),
    #         np.zeros((Y_contra.shape[0],X.shape[0])))
    if cross_val_matrices:
        from sklearn import cross_validation
        fid=open(cmdfile,'w')
        n_inj=X.shape[1]
        # Sets up nested outer/inner cross-validation. The inner loop is for
        # model selection (validation), the outer for testing.
        if cross_val=='LOO':
            outer_sets=cross_validation.LeaveOneOut(n_inj)
        else:
            outer_sets=cross_validation.KFold(n_inj,n_folds=cross_val)
        for i,(train,test) in enumerate(outer_sets):
            X_train=X[:,train]
            X_test=X[:,test]
            Y_train_ipsi=Y_ipsi[:,train]
            Y_test_ipsi=Y_ipsi[:,test]
            Omega_train = Omega[:,train]
            Omega_test = Omega[:,test]
            Y_train_contra=Y_contra[:,train]
            Y_test_contra=Y_contra[:,test]
{'fraction_non_alphanumeric': 0.10434588314154843, 'fraction_numerical': 0.00402189699474919, 'mean_word_length': 2.8240068346860316, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6316622', 'n_tokens_mistral': 3101, 'n_tokens_neox': 3027, 'n_words': 367}