<commit_before>import os
import re
import logging
import numpy as np
from .stats import OnlineStatistics
from ..writers import DatasetWriter
from ..readers import NumpyImageReader
from ..api import ClusterRoutines

logger = logging.getLogger(__name__)


class IllumstatsGenerator(ClusterRoutines):

    '''
    Class for calculating illumination statistics.
    '''

    def __init__(self, experiment, prog_name, verbosity):
        '''
        Initialize an instance of class IllumstatsGenerator.

        Parameters
        ----------
        experiment: tmlib.experiment.Experiment
            configured experiment object
        prog_name: str
            name of the corresponding program (command line interface)
        verbosity: int
            logging level
        '''
        super(IllumstatsGenerator, self).__init__(
                experiment, prog_name, verbosity)
        self.experiment = experiment
        self.prog_name = prog_name
        self.verbosity = verbosity

    @property
    def stats_file_format_string(self):
        '''
        Returns
        -------
        image_file_format_string: str
            format string that specifies how the names of the statistics files
            should be formatted
        '''
        return self.experiment.plates[0].cycles[0].STATS_FILE_FORMAT

    def create_job_descriptions(self, args):
        '''
        Create job descriptions for parallel computing.

        Parameters
        ----------
        args: tmlib.corilla.args.CorillaInitArgs
            program-specific arguments

        Returns
        -------
        Dict[str, List[dict] or dict]
            job descriptions
        '''
        job_descriptions = dict()
        job_descriptions['run'] = list()
        count = 0
        for plate in self.experiment.plates:
            for cycle in plate.cycles:
                md = cycle.image_metadata

                # Group image files per channel
                channels = np.unique(md.channel_ix)
                img_batches = list()
                for c in channels:
                    files = md[(md.channel_ix == c)].name
                    img_batches.append(files)

                for i, batch in enumerate(img_batches):
                    count += 1
                    job_descriptions['run'].append({
                        'id': count,
                        'inputs': {
                            'image_files': [
                                os.path.join(cycle.image_dir, f) for f in batch
                            ]
                        },
                        'outputs': {
                            'stats_file':
                                os.path.join(
                                    cycle.stats_dir,
                                    self.stats_file_format_string.format(
                                        channel_ix=channels[i])
                                )
                        },
                        'channel': channels[i],
                        'cycle': cycle.index
                    })
        return job_descriptions

    def run_job(self, batch):
        '''
        Calculate online statistics and write results to a HDF5 file.

        Parameters
        ----------
        batch: dict
            job_descriptions element
        '''
        image_files = batch['inputs']['image_files']
        logger.info('calculate illumination statistics')
        with NumpyImageReader() as reader:
            img = reader.read(image_files[0])
            stats = OnlineStatistics(image_dimensions=img.shape)
            for f in image_files:
                logger.debug('update statistics for image: %s',
                             os.path.basename(f))
                img = reader.read(f)
                stats.update(img)
        stats_file = batch['outputs']['stats_file']
        logger.info('write calculated statistics to file')<commit_msg>Apply new rule for "outputs" in job description to corilla<commit_after>import os
import re
import logging
import numpy as np
from .stats import OnlineStatistics
from ..writers import DatasetWriter
from ..readers import NumpyImageReader
from ..api import ClusterRoutines

logger = logging.getLogger(__name__)


class IllumstatsGenerator(ClusterRoutines):

    '''
    Class for calculating illumination statistics.
    '''

    def __init__(self, experiment, prog_name, verbosity):
        '''
        Initialize an instance of class IllumstatsGenerator.

        Parameters
        ----------
        experiment: tmlib.experiment.Experiment
            configured experiment object
        prog_name: str
            name of the corresponding program (command line interface)
        verbosity: int
            logging level
        '''
        super(IllumstatsGenerator, self).__init__(
                experiment, prog_name, verbosity)
        self.experiment = experiment
        self.prog_name = prog_name
        self.verbosity = verbosity

    @property
    def stats_file_format_string(self):
        '''
        Returns
        -------
        image_file_format_string: str
            format string that specifies how the names of the statistics files
            should be formatted
        '''
        return self.experiment.plates[0].cycles[0].STATS_FILE_FORMAT

    def create_job_descriptions(self, args):
        '''
        Create job descriptions for parallel computing.

        Parameters
        ----------
        args: tmlib.corilla.args.CorillaInitArgs
            program-specific arguments

        Returns
        -------
        Dict[str, List[dict] or dict]
            job descriptions
        '''
        job_descriptions = dict()
        job_descriptions['run'] = list()
        count = 0
        for plate in self.experiment.plates:
            for cycle in plate.cycles:
                md = cycle.image_metadata

                # Group image files per channel
                channels = np.unique(md.channel_ix)
                img_batches = list()
                for c in channels:
                    files = md[(md.channel_ix == c)].name
                    img_batches.append(files)

                for i, batch in enumerate(img_batches):
                    count += 1
                    job_descriptions['run'].append({
                        'id': count,
                        'inputs': {
                            'image_files': [
                                os.path.join(cycle.image_dir, f) for f in batch
                            ]
                        },
                        'outputs': {
                            'stats_files': [
                                os.path.join(
                                    cycle.stats_dir,
                                    self.stats_file_format_string.format(
                                        channel_ix=channels[i])
                                )
                            ]
                        },
                        'channel': channels[i],
                        'cycle': cycle.index
                    })
        return job_descriptions

    def run_job(self, batch):
        '''
        Calculate online statistics and write results to a HDF5 file.

        Parameters
        ----------
        batch: dict
            job_descriptions element
        '''
        image_files = batch['inputs']['image_files']
        logger.info('calculate illumination statistics')
        with NumpyImageReader() as reader:
            img = reader.read(image_files[0])
            stats = OnlineStatistics(image_dimensions=img.shape)
            for f in image_files:
                logger.debug('update statistics for image: %s',
                             os.path.basename(f))
                img = reader.read(f)
                stats.update(img)
        stats_file = batch['outputs']['stats_files'][0]
        logger.info('write calculated statistics to file')
{'fraction_non_alphanumeric': 0.08268015170670039, 'fraction_numerical': 0.0016434892541087231, 'mean_word_length': 1.3600835322195703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '739340', 'n_tokens_mistral': 1906, 'n_tokens_neox': 1786, 'n_words': 476}