<commit_before>"""
Authors: <NAME>, <NAME>

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Contains the ArgumentCreator class.

"""

import itertools
import logging

from cwsl.core.constraint import Constraint

module_logger = logging.getLogger('cwsl.core.argument_creator')



class ArgumentCreator(object):
    '''
    The ArgumentCreator class compares (one or more) input DataSets with
    (one or more) output FileCreators.

    There are different cases to consider -

    1. The ArgumentCreator adds/removes no attributes.
        In this case, there is a one-to-one mapping between input and output.

        Here the ArgumentCreator should return the a tuple (inputfile, outputfile),
        one output file (from the file creator) for each file
        in the input DataSet.

    2. The output has more attributes than the input.

        Here each input file will create many output files.

        In this case, it returns a mapping like
        (inputfile1, outputfile1), (inputfile1, outputfile2) ...
        up to (inputfile1, outputfileN), where N is the number of
        allowed values of the added attribute.

        It will do this once for every file in the input Dataset.

    3. The ArgumentCreator removes one or more attributes.

        Here many input files will be combined to create a single outputfile.

        For this case, the looper will return something like
        ([inputfile1, inputfile2, ..., inputfileN], outputfile)

    There are also mixed cases in which these operations are combined.

    '''

    def __init__(self, input_datasets, output_file_creators,
                 map_dict={}):

        ''' The class takes in a list of DataSet objects for its input and
        a list of FileCreator objects for output.

        Optionally a map_dict can be passed in which maps a constraint in the
        input with one from the output.

        The FileCreator is associated with a DataSet to check
        if files already exist.

        '''

        self.inputs = input_datasets
        self.output = output_file_creators

        self.map_dict = map_dict

        # Put the inputs/outputs into lists if they are not iterable.
        if not hasattr(self.inputs, '__iter__'):
            self.input = [self.inputs]
        if not hasattr(self.output, '__iter__'):
            self.output = [self.output]

        in_constraints = set.union(*[in_set.constraints
                                     for in_set in self.inputs])
        out_constraints = set.union(*[out_set.constraints
                                      for out_set in self.output])
        module_logger.debug('All input constraints are: ' + str(in_constraints))
        module_logger.debug('All output constraints are: ' + str(out_constraints))

        # Now apply the check constraints function to map
        # empty output constraints to the corresponding constraint in the input
        # and ensure that constraints are not repeated.
        self.input_cons, self.output_cons = self.check_constraints(in_constraints,
                                                                   out_constraints)
        
        # Find the shared attributes between the input and output.
        self.shared_constraints = self.input_cons.intersection(self.output_cons)

        # Find constraints that exist in the input but not the output - these
        # will be added to the returned combination dictionary for use in positional
        # or keyword arguments.
        self.input_only = self.input_cons.difference(self.output_cons)
        # The cons.values set is turned into a list so we can just get one value.
        self.input_only_dict = {cons.key: list(cons.values)[0] for cons in self.input_only}

        # Output only constraints are effectively shared constraints and must be added to
        # the valid combinations and the shared_constraints set.
        in_keys = [cons.key for cons in self.input_cons]
        self.output_only = [cons for cons in self.output_cons
                            if cons.key not in in_keys]
        
        self.shared_constraints = self.shared_constraints.union(self.output_only)

        module_logger.debug("Shared input and output constraints are: {0}".
                            format(self.shared_constraints))

        self.shared_keys = set([cons.key for cons in self.shared_constraints])

    def check_constraints(self, in_cons, out_cons):
        """ This method checks and cleans up any problems with input and
            output constraints in a logical manner. """

        # Check for repeated constraints.
        # If a constraint appears multiple times, add it to
        # the 'repeated_keys' list.
        new_ins = set()
        key_list = [cons.key for cons in in_cons]

        key_counter = {}
        for key in key_list:
            try:
                key_counter[key] += 1
            except KeyError:
                key_counter[key] = 1

        repeated_keys = []
        for cons in in_cons:
            if key_counter[cons.key] == 1:
                new_ins.add(cons)
            else:
                repeated_keys.append(cons.key)

        # Check that there are no empty constraints in the input - just
        # to make sure!
        for in_constraint in in_cons:
            if in_constraint.values == set():
                module_logger.error("Constraint {0} has no values!".format(in_constraint))
            assert(in_constraint.values != set())

        if repeated_keys:
            module_logger.debug("Repeated keys are: {0}".format(repeated_keys))

        # For repeated output keys, just get the shortest list.
        for key in set(repeated_keys):
            multi_cons = [cons for cons in in_cons
                          if cons.key == key]
            new_vals = set.intersection(*[set(con.values
                                              for con in multi_cons)])
            new_constraint = Constraint(key, new_vals)
            new_ins.add(new_constraint)

        # Now fix up empty output constraints.
        new_outs = set()
        for out_constraint in out_cons:
            if out_constraint.values:
                new_outs.add(out_constraint)
            else:
                # If it is empty, get the matching constraint
                # from the input.
                new_cons = [in_con for in_con in new_ins
                            if in_con.key == out_constraint.key]
                assert(len(new_cons) == 1)
                if not new_cons:
                    module_logger.error("Output constraint {0} has no values!"
                                        .format(out_constraints))
                new_outs.add(*new_cons)

        return new_ins, new_outs

    def __iter__(self):
        # Sets up the combinations in the argument_creator for looping.

        valids = [ds.valid_combinations for ds in self.inputs]<commit_msg>Removed check for repeated constraints<commit_after>"""
Authors: <NAME>, <NAME>

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Contains the ArgumentCreator class.

"""

import itertools
import logging

from cwsl.core.constraint import Constraint

module_logger = logging.getLogger('cwsl.core.argument_creator')



class ArgumentCreator(object):
    '''
    The ArgumentCreator class compares (one or more) input DataSets with
    (one or more) output FileCreators.

    There are different cases to consider -

    1. The ArgumentCreator adds/removes no attributes.
        In this case, there is a one-to-one mapping between input and output.

        Here the ArgumentCreator should return the a tuple (inputfile, outputfile),
        one output file (from the file creator) for each file
        in the input DataSet.

    2. The output has more attributes than the input.

        Here each input file will create many output files.

        In this case, it returns a mapping like
        (inputfile1, outputfile1), (inputfile1, outputfile2) ...
        up to (inputfile1, outputfileN), where N is the number of
        allowed values of the added attribute.

        It will do this once for every file in the input Dataset.

    3. The ArgumentCreator removes one or more attributes.

        Here many input files will be combined to create a single outputfile.

        For this case, the looper will return something like
        ([inputfile1, inputfile2, ..., inputfileN], outputfile)

    There are also mixed cases in which these operations are combined.

    '''

    def __init__(self, input_datasets, output_file_creators,
                 map_dict={}):

        ''' The class takes in a list of DataSet objects for its input and
        a list of FileCreator objects for output.

        Optionally a map_dict can be passed in which maps a constraint in the
        input with one from the output.

        The FileCreator is associated with a DataSet to check
        if files already exist.

        '''

        self.inputs = input_datasets
        self.output = output_file_creators

        self.map_dict = map_dict

        # Put the inputs/outputs into lists if they are not iterable.
        if not hasattr(self.inputs, '__iter__'):
            self.input = [self.inputs]
        if not hasattr(self.output, '__iter__'):
            self.output = [self.output]

        in_constraints = set.union(*[in_set.constraints
                                     for in_set in self.inputs])
        out_constraints = set.union(*[out_set.constraints
                                      for out_set in self.output])
        module_logger.debug('All input constraints are: ' + str(in_constraints))
        module_logger.debug('All output constraints are: ' + str(out_constraints))

        # Now apply the check constraints function to map
        # empty output constraints to the corresponding constraint in the input
        # and ensure that constraints are not repeated.
        self.input_cons, self.output_cons = self.check_constraints(in_constraints,
                                                                   out_constraints)
        
        # Find the shared attributes between the input and output.
        self.shared_constraints = self.input_cons.intersection(self.output_cons)

        # Find constraints that exist in the input but not the output - these
        # will be added to the returned combination dictionary for use in positional
        # or keyword arguments.
        self.input_only = self.input_cons.difference(self.output_cons)

        # The cons.values set is turned into a list so we can just get one value.
        module_logger.debug("self.input_only: {}".format(self.input_only))
        self.input_only_dict = {cons.key: list(cons.values)[0] for cons in self.input_only}

        # Output only constraints are effectively shared constraints and must be added to
        # the valid combinations and the shared_constraints set.
        in_keys = [cons.key for cons in self.input_cons]
        self.output_only = [cons for cons in self.output_cons
                            if cons.key not in in_keys]
        
        self.shared_constraints = self.shared_constraints.union(self.output_only)

        module_logger.debug("Shared input and output constraints are: {0}".
                            format(self.shared_constraints))

        self.shared_keys = set([cons.key for cons in self.shared_constraints])

    def check_constraints(self, in_cons, out_cons):
        """ This method checks and cleans up any problems with input and
            output constraints in a logical manner. """

        # Check that there are no empty constraints in the input - just
        # to make sure!
        for in_constraint in in_cons:
            if in_constraint.values == set():
                module_logger.error("Constraint {0} has no values!".format(in_constraint))
            assert(in_constraint.values != set())

        # Now fix up empty output constraints.
        new_outs = set()
        for out_constraint in out_cons:
            if out_constraint.values:
                new_outs.add(out_constraint)
            else:
                # If it is empty, get the matching constraint
                # from the input.
                new_cons = [in_con for in_con in new_ins
                            if in_con.key == out_constraint.key]
                assert(len(new_cons) == 1)
                if not new_cons:
                    module_logger.error("Output constraint {0} has no values!"
                                        .format(out_constraints))
                new_outs.add(*new_cons)

        return in_cons, new_outs

    def __iter__(self):
        # Sets up the combinations in the argument_creator for looping.

        valids = [ds.valid_combinations for ds in self.inputs]
{'fraction_non_alphanumeric': 0.06377402446126966, 'fraction_numerical': 0.0036400698893418754, 'mean_word_length': 2.2043386983904827, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '97116', 'n_tokens_mistral': 3545, 'n_tokens_neox': 3406, 'n_words': 1358}