<commit_before>"""
Export neuroimaging results created by neuroimaging software packages 
(FSL, AFNI, ...) following NIDM-Results specification.

Specification: http://nidm.nidash.org/specs/nidm-results.html

@author: <NAME> <<EMAIL>>
@copyright: University of Warwick 2013-2014
"""

from prov.model import ProvBundle, ProvDocument
import os
import datetime
from objects.constants import *
from objects.modelfitting import *
from objects.contrast import *
from objects.inference import *
import uuid 

class NIDMExporter():
    """ 
    Generic class to parse a result directory to extract the pieces of 
    information to be stored in NIDM-Results and to generate a NIDM-Results 
    export.
    """
    
    def parse(self):
        """ 
        Parse a result directory to extract the pieces information to be 
        stored in NIDM-Results. 
        """   
        # Methods: find_software, find_model_fitting, find_contrasts and
        # find_inferences should be defined in the children classes and return
        # a list of NIDM Objects as specified in the objects module

        # Object of type Software describing the neuroimaging software package 
        # used for the analysis
        self.software = self._find_software()

        # List of objects of type ModelFitting describing the model fitting 
        # step in NIDM-Results (main activity: Model Parameters Estimation)
        self.model_fittings = self._find_model_fitting()

        # Dictionary of (key, value) pairs where where key is a tuple 
        # containing the identifier of a ModelParametersEstimation object and a 
        # tuple of identifiers of ParameterEstimateMap objects and value is an 
        # object of type Contrast describing the contrast estimation step in 
        # NIDM-Results (main activity: Contrast Estimation)
        self.contrasts = self._find_contrasts()

        # Inference activity and entities
        # Dictionary of (key, value) pairs where key is the identifier of a 
        # ContrastEstimation object and value is an object of type Inference
        # describing the inference step in NIDM-Results (main activity: 
        # Inference)
        self.inferences = self._find_inferences()
                
        # Initialise prov document
        self.doc = ProvDocument();
        self._add_namespaces()

    def export(self):
        """ 
        Generate a NIDM-Results export. 
        """  
        if not os.path.isdir(self.export_dir):
            os.mkdir(self.export_dir)

        # Initialise main bundle
        self._create_bundle(self.version)
        self.bundle.update(self.software.export())

        # Add model fitting steps
        for model_fitting in self.model_fittings:
            self.bundle.update(model_fitting.export())
            self.bundle.wasAssociatedWith(model_fitting.activity.id, 
                self.software.id)

        # Add contrast estimation steps
        for (model_fitting_id, pe_ids), contrasts in self.contrasts.items():
            model_fitting = self._get_model_fitting(model_fitting_id)
            for contrast in contrasts:
                self.bundle.update(contrast.export())
                self.bundle.used(contrast.estimation.id, 
                    model_fitting.rms_map.id)
                self.bundle.used(contrast.estimation.id, 
                    model_fitting.mask_map.id)
                self.bundle.wasAssociatedWith(contrast.estimation.id, 
                    self.software.id)
                for pe_id in pe_ids:
                    self.bundle.used(contrast.estimation.id, pe_id)

        # Add inference steps
        for contrast_id, inferences in self.inferences.items():
            contrast = self._get_contrast(contrast_id)
            for inference in inferences:
                self.bundle.update(inference.export())
                if contrast.z_stat_map:
                    used_id = contrast.z_stat_map.id
                else:
                    used_id = contrast.stat_map.id
                self.bundle.used(inference.inference_act.id, used_id)
                self.bundle.wasAssociatedWith(inference.inference_act.id, self.software.id)

        # Write-out prov file
        self.save_prov_to_files()

        return self.export_dir

    def _get_model_fitting(self, mf_id):
        """ 
        Retreive model fitting with identifier 'mf_id' from the list of model 
        fitting objects stored in self.model_fittings
        """   
        for model_fitting in self.model_fittings:
            if model_fitting.activity.id == mf_id:
                return model_fitting
        raise Exception("Model fitting activity with id: "+str(mf_id)+\
            " not found.")

    def _get_contrast(self, con_id):
        """ 
        Retreive contrast with identifier 'con_id' from the list of contrast 
        objects stored in self.contrasts
        """   
        for contrasts in self.contrasts.values():
            for contrast in contrasts:
                if contrast.estimation.id == con_id:
                    return contrast
        raise Exception("Contrast activity with id: "+str(con_id)+\
            " not found.")

    def _add_namespaces(self):
        """ 
        Add namespaces to NIDM document. 
        """
        self.doc.add_namespace(NIDM)
        self.doc.add_namespace(NIIRI)
        self.doc.add_namespace(CRYPTO)
        self.doc.add_namespace(DCT)

    def _create_bundle(self, version):
        """ 
        Initialise NIDM-Results bundle.
        """
        software_lc = self.software.name.lower()
        software_uc = self.software.name.upper()

        bundle_id = NIIRI[str(uuid.uuid4())]
        self.bundle = ProvBundle(identifier=bundle_id)

        self.doc.entity(bundle_id, 
            other_attributes=( (PROV['type'], PROV['Bundle'],), 
                               (PROV['label'],software_uc+" Results" ),
                               (NIDM['objectModel'],
                                            NIDM[software_uc+'Results']),
                               (NIDM['version'], version))
            )

        self.doc.wasGeneratedBy(NIIRI[software_lc+'_results_id'], 
            time=str(datetime.datetime.now().time()))

    def _get_model_parameters_estimations(self, error_model):
        """
        Infer model estimation method from the 'error_model'. Return an object
        of type ModelParametersEstimation.
        """
        if error_model.dependance == INDEPEDENT_CORR:
            if error_model.variance_homo:
                estimation_method = ESTIMATION_OLS
            else:
                estimation_method = ESTIMATION_WLS
        else:
            estimation_method = ESTIMATION_GLS

        mpe = ModelParametersEstimation(estimation_method, self.software.id)

        return mpe

    def save_prov_to_files(self, showattributes=False):
        """
        Write-out provn serialisation to nidm.provn.
        """
        self.doc.add_bundle(self.bundle)
        provn_fid = open(os.path.join(self.export_dir, 'nidm.provn'), 'w');
        provn_fid.write(self.doc.get_provn(4))<commit_msg>Convert provn to ttl (using the java ProvToolbox)<commit_after>"""
Export neuroimaging results created by neuroimaging software packages 
(FSL, AFNI, ...) following NIDM-Results specification.

Specification: http://nidm.nidash.org/specs/nidm-results.html

@author: <NAME> <<EMAIL>>
@copyright: University of Warwick 2013-2014
"""

from prov.model import ProvBundle, ProvDocument
import os
import datetime
from objects.constants import *
from objects.modelfitting import *
from objects.contrast import *
from objects.inference import *
import uuid 
from subprocess import call

class NIDMExporter():
    """ 
    Generic class to parse a result directory to extract the pieces of 
    information to be stored in NIDM-Results and to generate a NIDM-Results 
    export.
    """
    
    def parse(self):
        """ 
        Parse a result directory to extract the pieces information to be 
        stored in NIDM-Results. 
        """   
        # Methods: find_software, find_model_fitting, find_contrasts and
        # find_inferences should be defined in the children classes and return
        # a list of NIDM Objects as specified in the objects module

        # Object of type Software describing the neuroimaging software package 
        # used for the analysis
        self.software = self._find_software()

        # List of objects of type ModelFitting describing the model fitting 
        # step in NIDM-Results (main activity: Model Parameters Estimation)
        self.model_fittings = self._find_model_fitting()

        # Dictionary of (key, value) pairs where where key is a tuple 
        # containing the identifier of a ModelParametersEstimation object and a 
        # tuple of identifiers of ParameterEstimateMap objects and value is an 
        # object of type Contrast describing the contrast estimation step in 
        # NIDM-Results (main activity: Contrast Estimation)
        self.contrasts = self._find_contrasts()

        # Inference activity and entities
        # Dictionary of (key, value) pairs where key is the identifier of a 
        # ContrastEstimation object and value is an object of type Inference
        # describing the inference step in NIDM-Results (main activity: 
        # Inference)
        self.inferences = self._find_inferences()
                
        # Initialise prov document
        self.doc = ProvDocument();
        self._add_namespaces()

    def export(self):
        """ 
        Generate a NIDM-Results export. 
        """  
        if not os.path.isdir(self.export_dir):
            os.mkdir(self.export_dir)

        # Initialise main bundle
        self._create_bundle(self.version)
        self.bundle.update(self.software.export())

        # Add model fitting steps
        for model_fitting in self.model_fittings:
            self.bundle.update(model_fitting.export())
            self.bundle.wasAssociatedWith(model_fitting.activity.id, 
                self.software.id)

        # Add contrast estimation steps
        for (model_fitting_id, pe_ids), contrasts in self.contrasts.items():
            model_fitting = self._get_model_fitting(model_fitting_id)
            for contrast in contrasts:
                self.bundle.update(contrast.export())
                self.bundle.used(contrast.estimation.id, 
                    model_fitting.rms_map.id)
                self.bundle.used(contrast.estimation.id, 
                    model_fitting.mask_map.id)
                self.bundle.wasAssociatedWith(contrast.estimation.id, 
                    self.software.id)
                for pe_id in pe_ids:
                    self.bundle.used(contrast.estimation.id, pe_id)

        # Add inference steps
        for contrast_id, inferences in self.inferences.items():
            contrast = self._get_contrast(contrast_id)
            for inference in inferences:
                self.bundle.update(inference.export())
                if contrast.z_stat_map:
                    used_id = contrast.z_stat_map.id
                else:
                    used_id = contrast.stat_map.id
                self.bundle.used(inference.inference_act.id, used_id)
                self.bundle.wasAssociatedWith(inference.inference_act.id, self.software.id)

        # Write-out prov file
        self.save_prov_to_files()

        return self.export_dir

    def _get_model_fitting(self, mf_id):
        """ 
        Retreive model fitting with identifier 'mf_id' from the list of model 
        fitting objects stored in self.model_fittings
        """   
        for model_fitting in self.model_fittings:
            if model_fitting.activity.id == mf_id:
                return model_fitting
        raise Exception("Model fitting activity with id: "+str(mf_id)+\
            " not found.")

    def _get_contrast(self, con_id):
        """ 
        Retreive contrast with identifier 'con_id' from the list of contrast 
        objects stored in self.contrasts
        """   
        for contrasts in self.contrasts.values():
            for contrast in contrasts:
                if contrast.estimation.id == con_id:
                    return contrast
        raise Exception("Contrast activity with id: "+str(con_id)+\
            " not found.")

    def _add_namespaces(self):
        """ 
        Add namespaces to NIDM document. 
        """
        self.doc.add_namespace(NIDM)
        self.doc.add_namespace(NIIRI)
        self.doc.add_namespace(CRYPTO)
        self.doc.add_namespace(DCT)

    def _create_bundle(self, version):
        """ 
        Initialise NIDM-Results bundle.
        """
        software_lc = self.software.name.lower()
        software_uc = self.software.name.upper()

        bundle_id = NIIRI[str(uuid.uuid4())]
        self.bundle = ProvBundle(identifier=bundle_id)

        self.doc.entity(bundle_id, 
            other_attributes=( (PROV['type'], PROV['Bundle'],), 
                               (PROV['label'],software_uc+" Results" ),
                               (NIDM['objectModel'],
                                            NIDM[software_uc+'Results']),
                               (NIDM['version'], version))
            )

        self.doc.wasGeneratedBy(NIIRI[software_lc+'_results_id'], 
            time=str(datetime.datetime.now().time()))

    def _get_model_parameters_estimations(self, error_model):
        """
        Infer model estimation method from the 'error_model'. Return an object
        of type ModelParametersEstimation.
        """
        if error_model.dependance == INDEPEDENT_CORR:
            if error_model.variance_homo:
                estimation_method = ESTIMATION_OLS
            else:
                estimation_method = ESTIMATION_WLS
        else:
            estimation_method = ESTIMATION_GLS

        mpe = ModelParametersEstimation(estimation_method, self.software.id)

        return mpe

    def save_prov_to_files(self, showattributes=False):
        """
        Write-out provn serialisation to nidm.provn.
        """
        self.doc.add_bundle(self.bundle)
        provn_file = os.path.join(self.export_dir, 'nidm.provn')
        provn_fid = open(provn_file, 'w');
        # FIXME None
        provn_fid.write(self.doc.get_provn(4).replace("None", "-"))

        ttl_file = provn_file.replace(".provn", ".ttl")
        print "provconvert -infile "+provn_file+" -outfile "+ttl_file
        call("provconvert -infile "+provn_file+" -outfile "+ttl_file, shell=True)
{'fraction_non_alphanumeric': 0.08257070116287071, 'fraction_numerical': 0.0013761783527145117, 'mean_word_length': 2.128955866523143, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3996362', 'n_tokens_mistral': 4126, 'n_tokens_neox': 3837, 'n_words': 1003}