<commit_before># TmLibrary - TissueMAPS library for distibuted image analysis routines.
# Copyright (C) 2016  <NAME>, University of Zurich and <NAME>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import re
import logging
import imp
import collections
import importlib
import traceback
import numpy as np
from cStringIO import StringIO


from tmlib.workflow.jterator.utils import determine_language
from tmlib.workflow.jterator import handles as hdls
from tmlib.errors import PipelineRunError

logger = logging.getLogger(__name__)


class CaptureOutput(dict):
    '''Class for capturing standard output and error and storing the strings
    in dictionary.

    Examples
    --------
    with CaptureOutput() as output:
        foo()

    Warning
    -------
    Using this approach screws up debugger break points.
    '''
    def __enter__(self):
        self._stdout = sys.stdout
        self._stderr = sys.stderr
        sys.stdout = self._stringio_out = StringIO()
        sys.stderr = self._stringio_err = StringIO()
        return self

    def __exit__(self, *args):
        sys.stdout = self._stdout
        sys.stderr = self._stderr
        output = self._stringio_out.getvalue()
        error = self._stringio_err.getvalue()
        self.update({'stdout': output, 'stderr': error})


class ImageAnalysisModule(object):

    '''Class for a Jterator module, the building block of an image analysis
    pipeline.
    '''

    def __init__(self, name, source_file, handles):
        '''
        Parameters
        ----------
        name: str
            name of the module
        source_file: str
            name or path to program file that should be executed
        handles: tmlib.workflow.jterator.description.HandleDescriptions
            description of module input/output as provided
        '''
        self.name = name
        self.source_file = source_file
        self.handles = handles
        self.outputs = dict()
        self.persistent_store = dict()

    def build_figure_filename(self, figures_dir, job_id):
        '''Builds name of figure file into which module will write figure
        output of the current job.

        Parameters
        ----------
        figures_dir: str
            path to directory for figure output
        job_id: int
            one-based job index

        Returns
        -------
        str
            absolute path to the figure file
        '''
        return os.path.join(figures_dir, '%s_%.5d.json' % (self.name, job_id))

    @property
    def keyword_arguments(self):
        '''dict: name and value of each input handle as key-value pairs'''
        kwargs = collections.OrderedDict()
        for handle in self.handles.input:
            kwargs[handle.name] = handle.value
        return kwargs

    @property
    def language(self):
        '''str: language of the module (e.g. "Python")'''
        return determine_language(self.source_file)

    def _exec_m_module(self, engine):
        module_name = os.path.splitext(os.path.basename(self.source_file))[0]
        logger.debug(
            'import module "%s" from source file: %s',
            module_name, self.source_file
        )
        logger.debug(
            'add module source file to Matlab path: "%s"', self.source_file
        )
        engine.eval(
            'addpath(\'{0}\');'.format(os.path.dirname(self.source_file))
        )
        engine.eval('version = {0}.VERSION'.format(module_name))
        function_call_format_string = '[{outputs}] = {name}.main({inputs});'
        # NOTE: Matlab doesn't add imported classes to the workspace. It access
        # the "VERSION" property, we need to assign it to a variable first.
        version = engine.get('version')
        if version != self.handles.version:
            raise PipelineRunError(<commit_msg>Note that `_exec_m_module()` has a bug if using multiple-file modules.<commit_after># TmLibrary - TissueMAPS library for distibuted image analysis routines.
# Copyright (C) 2016  <NAME>, University of Zurich and <NAME>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import re
import logging
import imp
import collections
import importlib
import traceback
import numpy as np
from cStringIO import StringIO


from tmlib.workflow.jterator.utils import determine_language
from tmlib.workflow.jterator import handles as hdls
from tmlib.errors import PipelineRunError

logger = logging.getLogger(__name__)


class CaptureOutput(dict):
    '''Class for capturing standard output and error and storing the strings
    in dictionary.

    Examples
    --------
    with CaptureOutput() as output:
        foo()

    Warning
    -------
    Using this approach screws up debugger break points.
    '''
    def __enter__(self):
        self._stdout = sys.stdout
        self._stderr = sys.stderr
        sys.stdout = self._stringio_out = StringIO()
        sys.stderr = self._stringio_err = StringIO()
        return self

    def __exit__(self, *args):
        sys.stdout = self._stdout
        sys.stderr = self._stderr
        output = self._stringio_out.getvalue()
        error = self._stringio_err.getvalue()
        self.update({'stdout': output, 'stderr': error})


class ImageAnalysisModule(object):

    '''Class for a Jterator module, the building block of an image analysis
    pipeline.
    '''

    def __init__(self, name, source_file, handles):
        '''
        Parameters
        ----------
        name: str
            name of the module
        source_file: str
            name or path to program file that should be executed
        handles: tmlib.workflow.jterator.description.HandleDescriptions
            description of module input/output as provided
        '''
        self.name = name
        self.source_file = source_file
        self.handles = handles
        self.outputs = dict()
        self.persistent_store = dict()

    def build_figure_filename(self, figures_dir, job_id):
        '''Builds name of figure file into which module will write figure
        output of the current job.

        Parameters
        ----------
        figures_dir: str
            path to directory for figure output
        job_id: int
            one-based job index

        Returns
        -------
        str
            absolute path to the figure file
        '''
        return os.path.join(figures_dir, '%s_%.5d.json' % (self.name, job_id))

    @property
    def keyword_arguments(self):
        '''dict: name and value of each input handle as key-value pairs'''
        kwargs = collections.OrderedDict()
        for handle in self.handles.input:
            kwargs[handle.name] = handle.value
        return kwargs

    @property
    def language(self):
        '''str: language of the module (e.g. "Python")'''
        return determine_language(self.source_file)

    def _exec_m_module(self, engine):
        module_name = os.path.splitext(os.path.basename(self.source_file))[0]
        logger.debug(
            'import module "%s" from source file: %s',
            module_name, self.source_file
        )
        # FIXME: this inserts the wrong directory if the MATLAB module
        # has the form `+directory` or `@directory` -- let's allow
        # this for the moment since the module discovery code only
        # deals with single-file modules, but it needs to be revisited
        source_dir = os.path.dirname(self.source_file)
        logger.debug(
            'adding module source directory `%s` to MATLAB path ...',
            source_dir
        )
        engine.eval(
            "addpath('{0}');".format(source_dir)
        )
        engine.eval('version = {0}.VERSION'.format(module_name))
        function_call_format_string = '[{outputs}] = {name}.main({inputs});'
        # NOTE: Matlab doesn't add imported classes to the workspace. It access
        # the "VERSION" property, we need to assign it to a variable first.
        version = engine.get('version')
        if version != self.handles.version:
            raise PipelineRunError(
{'fraction_non_alphanumeric': 0.08951351351351351, 'fraction_numerical': 0.0019459459459459458, 'mean_word_length': 2.4544436146377895, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3290897', 'n_tokens_mistral': 2487, 'n_tokens_neox': 2426, 'n_words': 890}