<commit_before>"""
Authors: <NAME>, <NAME>, <NAME>

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

This module contains the ProcessUnit class.

"""

import os,logging

from cwsl.configuration import configuration
from cwsl.utils import utils
from cwsl.core.argument_creator import ArgumentCreator
from cwsl.core.file_creator import FileCreator
from cwsl.core.constraint import Constraint
from cwsl.core.scheduler import SimpleExecManager

module_logger = logging.getLogger('cwsl.core.process_unit')


class ProcessUnit(object):
    """ This class sets up the execution of an operation
    performed on a DataSet.

    This class takes in a list of input DataSets, an output
    pattern to write files to, the shell command that needs to be
    run and any output_constraints that need to be added to the
    output.

    It sets up the ArgumentCreator classes required to
    run a particular, single processing operation for all of
    the possible combinations in the input and output DataSets.

    This could be done using a VisTrails module directly, but doing it like
    this allows us to separate the GUI presentation from the execution.

    """

    def __init__(self, 
                 inputlist, 
                 output_pattern, 
                 shell_command,
                 extra_constraints=set([]), 
                 map_dict={},
                 cons_keywords={}, 
                 positional_args=[],
                 execution_options={}):

        """ The class takes in a DataSet object, constraints to change and
        the path to an executable. It has an self.execute() method that
        returns a FileCreator to be used as input to the next module.

        Extra constraints to be applied to the output are given by
        extra_constraints.

        map_dict is a dictionary linking constraint names in the input DataSets
        to new constraints in the output. e.g.
        if map_dict = {"model": "obs-model"} then the "model" constraint in the
        input is renamed to be the "obs-model" Constraint in the output.

        """

        if extra_constraints:
            module_logger.debug("extra_constraints are: {0}"
                                .format(extra_constraints))

        self.inputlist = inputlist
        self.cons_keywords = cons_keywords
        self.positional_args = positional_args
        self.execution_options = execution_options

        # The initial Constraints for the output are built from the
        # output file pattern.
        self.pattern_constraints = FileCreator.constraints_from_pattern(output_pattern)

        # Fill the empty output constraints from the input DataSets.
        fixed_constraints = self.fill_empty_constraints(extra_constraints)
        final_constraints = self.apply_mappings(map_dict, fixed_constraints)

        module_logger.debug("Final output constraints are: {0}".format(final_constraints))

        # Make a file_creator from the new, fixed constraints.
        self.file_creator = FileCreator(output_pattern, final_constraints)

        self.shell_command = shell_command

    def fill_empty_constraints(self, extra_constraints):

        out_cons_names = [cons.key for cons in extra_constraints]
        # These are constraints that exist in the pattern but not in
        # the extra_constraints - they must be filled from the input
        # if they exist there.
        unfilled_constraints = set([cons for cons in self.pattern_constraints
                                    if cons.key not in out_cons_names])

        empty_given_constraints = set([cons for cons in extra_constraints
                                       if not cons.values])

        missing_constraints = unfilled_constraints.union(empty_given_constraints)
        module_logger.debug("Constraints that need to be filled from the input are: {0}".
                            format(missing_constraints))

        module_logger.debug("Before filling, extra_constraints is: {0}"
                            .format(extra_constraints))
<commit_msg>Initial implementation of empty constraint errors.<commit_after>"""
Authors: <NAME>, <NAME>, <NAME>

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

This module contains the ProcessUnit class.

"""

import os,logging

from cwsl.configuration import configuration
from cwsl.utils import utils
from cwsl.core.argument_creator import ArgumentCreator
from cwsl.core.file_creator import FileCreator
from cwsl.core.constraint import Constraint
from cwsl.core.scheduler import SimpleExecManager

module_logger = logging.getLogger('cwsl.core.process_unit')


class ProcessUnit(object):
    """ This class sets up the execution of an operation
    performed on a DataSet.

    This class takes in a list of input DataSets, an output
    pattern to write files to, the shell command that needs to be
    run and any output_constraints that need to be added to the
    output.

    It sets up the ArgumentCreator classes required to
    run a particular, single processing operation for all of
    the possible combinations in the input and output DataSets.

    This could be done using a VisTrails module directly, but doing it like
    this allows us to separate the GUI presentation from the execution.

    """

    def __init__(self, 
                 inputlist, 
                 output_pattern, 
                 shell_command,
                 extra_constraints=set([]), 
                 map_dict={},
                 cons_keywords={}, 
                 positional_args=[],
                 execution_options={}):

        """ The class takes in a DataSet object, constraints to change and
        the path to an executable. It has an self.execute() method that
        returns a FileCreator to be used as input to the next module.

        Extra constraints to be applied to the output are given by
        extra_constraints.

        map_dict is a dictionary linking constraint names in the input DataSets
        to new constraints in the output. e.g.
        if map_dict = {"model": "obs-model"} then the "model" constraint in the
        input is renamed to be the "obs-model" Constraint in the output.

        """

        if extra_constraints:
            module_logger.debug("extra_constraints are: {0}"
                                .format(extra_constraints))

        self.inputlist = inputlist
        self.cons_keywords = cons_keywords
        self.positional_args = positional_args
        self.execution_options = execution_options

        # The initial Constraints for the output are built from the
        # output file pattern.
        self.pattern_constraints = FileCreator.constraints_from_pattern(output_pattern)

        # Fill the empty output constraints from the input DataSets.
        fixed_constraints = self.fill_empty_constraints(extra_constraints)
        final_constraints = self.apply_mappings(map_dict, fixed_constraints)

        module_logger.debug("Final output constraints are: {0}".format(final_constraints))

        # Make a file_creator from the new, fixed constraints.
        self.file_creator = FileCreator(output_pattern, final_constraints)

        self.shell_command = shell_command

    def fill_empty_constraints(self, extra_constraints):

        out_cons_names = []
        for cons in extra_constraints:
            if not cons.values:
                raise EmptyOverwriteError("Constraint on {} is empty - can not overwrite existing constraint"
                                          .format(cons.key))
            else:
                out_cons_names.append(cons.key)

        # These are constraints that exist in the pattern but not in
        # the extra_constraints - they must be filled from the input
        # if they exist there.
        unfilled_constraints = set([cons for cons in self.pattern_constraints
                                    if cons.key not in out_cons_names])

        empty_given_constraints = set([cons for cons in extra_constraints
                                       if not cons.values])

        missing_constraints = unfilled_constraints.union(empty_given_constraints)
        module_logger.debug("Constraints that need to be filled from the input are: {0}".
                            format(missing_constraints))

        module_logger.debug("Before filling, extra_constraints is: {0}"
                            .format(extra_constraints))

{'fraction_non_alphanumeric': 0.0556386159760786, 'fraction_numerical': 0.0025630072618539087, 'mean_word_length': 2.636893203883495, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5363115', 'n_tokens_mistral': 2370, 'n_tokens_neox': 2268, 'n_words': 918}