<commit_before>"""
Authors: <NAME>, <NAME>

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Contains the FileCreator class.

"""

import re
import itertools
import logging
module_logger = logging.getLogger('cwsl.core.file_creator')
import os.path

from cwsl.core.constraint import Constraint
from cwsl.core.metafile import MetaFiles


class FileCreator(object):
    '''
    This class is an object that creates the output MockClimateFiles
    objects, given an output pattern and a set of Constraints.

    We cannot use the actual ClimateFiles objects because these
    require ForeignKey relationships with other tables which
    may not exist.

    A FileCreator has a 'output_pattern' attribute which defines
    what the filename of any created output files should be.

    The ArgumentCreator class compares these possible created files
    and throws away any that cannot be created as they do not have
    matching files in the input DataSet.

    The output pattern has a particular syntax. The pattern is given as a
    string with attribute names surrounded by % signs.

    eg:

    "/projects/ua6/CAWCR_CVC_processed/%variable%_%modeling_realm%_%model%.nc"

    This class will raise an error if it is instantiated with a pattern with
    empty constraints - it does not make sense to have a file creator that
    has 'empty' or 'all' constraints - they must be in canonical form.

    It will also raise an error if instantiated with 'extra' constraints
    that do not match those from its output pattern.

    '''

    def __init__(self, output_pattern, extra_constraints):
        ''' This constructor sets up the DataSet for this file creator from the
        pattern of its output and 'extra' constraints that set the value of
        its attributes.

        '''

        self.output_pattern = output_pattern

        # Make the constraints from the output pattern.
        self.constraints = FileCreator.constraints_from_pattern(output_pattern)

        # Add the extra constraints to the self.constraints, strip out any that
        # are not part of the output pattern.
        self.merge_constraints(extra_constraints)

        # This object must create files, so after merging all constraints must
        # be in canonical form.
        for constraint in self.constraints:
            if not constraint.values:
                module_logger.error("Constraint {0} is empty - should be in canonical form!"
                                    .format(constraint))
                print constraint
                raise EmptyConstraintError

        # Set up the subset types (the attribute names present in the DataSet.
        self.subset_types = [cons.key for cons in self.constraints]
        self.huge_iterator = itertools.product(*[cons.values
                                                 for cons in self.constraints])

        # A set to hold all the valid combinations of attributes.
        self.valid_combinations = set()
        # One to hold the valid_hashes
        self.valid_hashes = set()

    def get_files(self, att_dict, check=False, update=True, map_dict={}):
        """ This method returns all possible MockClimateFiles from the
        FileCreator that match an input attribute dictionary.

        If check is True, then we check that the hash of the for the
        file is in the - 'valid_hashes' hash list. This is used when using
        the FileCreator as an input, we only want to give files that
        actually exists.

        """

        for mapping in map_dict:
            map_val = att_dict.pop(mapping)
            att_dict[map_dict[mapping]] = map_val

        # Get the keys of the input dictionary.
        search_keys = [att for att in att_dict.keys()]

        to_loop = []
        existing_values = []
        for key in self.subset_types:
            # If a key is not in the att_dict, grab the existing constraint.
            if key not in search_keys:
                existing_cons = self.get_constraint(key)
                to_loop.append((existing_cons.key, existing_cons.values))
                existing_values.append(existing_cons.values)
                module_logger.debug("Adding {0} to to_loop: constraint from output"
                                    .format((existing_cons.key, existing_cons.values)))
                assert(type(existing_cons.values == set))
            else:
                new_cons = Constraint(key, [att_dict[key]])
                to_loop.append((new_cons.key, new_cons.values))
                module_logger.debug("Adding {0} to to_loop: constraint from input"
                                    .format((new_cons.key, new_cons.values)))

        module_logger.debug("to_loop is: {0}".format(to_loop))

        keys = [cons[0] for cons in to_loop]
        values = [cons[1] for cons in to_loop]

        module_logger.debug("Keys to search for are: {0}".format(keys))
        module_logger.debug("Building itertools product for values: {0}"
                            .format(values))
        new_iter = itertools.product(*values)

        outfiles = []
        for combination in new_iter:
            new_file = self.climate_file_from_combination(keys, combination,
                                                          check, update)
            if new_file:
                outfiles.append(new_file)

        return outfiles

    @property
    def files(self):
        """ This property returns all the files that can be possibly
        returned by this file_creator.

        This can make some massive lists! Shouldn't really be
        used in looping.

        """
        for combination in self.huge_iterator:
            # Create a set of constraints for this combination.
            yield self.climate_file_from_combination(self.subset_types, combination,
                                                     check=False, update=False)

    def get_constraint(self, attribute_name):
        """ Get a particular constraint by name."""

        for constraint in self.constraints:
            if constraint.key == attribute_name:
                return constraint

        # If it can't be found, return None.
        return None

    def merge_constraints(self, new_constraints):
        """ This function adds the constraint values to the constraints from
        a pattern.

        This surely can be written in a more concise way.

        """

        existing_cons_names = [cons.key for cons in self.constraints]

        # Now add the constraints - only if they are in the pattern!
        for cons in new_constraints:
            if cons.key in existing_cons_names:
                self.constraints.add(cons)

        attribute_names = [cons.key for cons in self.constraints]

        repeated_atts = []
        for name in attribute_names:
            if attribute_names.count(name) > 1:
                repeated_atts.append(name)

        to_remove = [cons for cons in self.constraints
                     if cons.key in repeated_atts]

        new_cons_dict = {}
        for cons in to_remove:
            new_cons_dict[cons.key] = set([])

        module_logger.debug("Constraints to remove: {0}"
                            .format(to_remove))

        for cons in to_remove:<commit_msg>Removed inaccurate docstring.<commit_after>"""
Authors: <NAME>, <NAME>

Copyright 2014 CSIRO

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Contains the FileCreator class.

"""

import re
import itertools
import logging
import os.path

from cwsl.core.constraint import Constraint
from cwsl.core.metafile import MetaFiles

module_logger = logging.getLogger('cwsl.core.file_creator')



class FileCreator(object):
    '''
    This class is an object that creates the output MockClimateFiles
    objects, given an output pattern and a set of Constraints.

    A FileCreator has a 'output_pattern' attribute which defines
    what the filename of any created output files should be.

    The ArgumentCreator class compares these possible created files
    and throws away any that cannot be created as they do not have
    matching files in the input DataSet.

    The output pattern has a particular syntax. The pattern is given as a
    string with attribute names surrounded by % signs.

    eg:

    "/projects/ua6/CAWCR_CVC_processed/%variable%_%modeling_realm%_%model%.nc"

    This class will raise an error if it is instantiated with a pattern with
    empty constraints - it does not make sense to have a file creator that
    has 'empty' or 'all' constraints - they must be in canonical form.

    It will also raise an error if instantiated with 'extra' constraints
    that do not match those from its output pattern.

    '''

    def __init__(self, output_pattern, extra_constraints):
        ''' This constructor sets up the DataSet for this file creator from the
        pattern of its output and 'extra' constraints that set the value of
        its attributes.

        '''

        module_logger.debug("Building FileCreator with pattern: {0} and extra_constraints: {1}"
                            .format(output_pattern, extra_constraints))

        self.output_pattern = output_pattern

        # Make the constraints from the output pattern.
        self.constraints = FileCreator.constraints_from_pattern(output_pattern)

        # Add the extra constraints to the self.constraints, strip out any that
        # are not part of the output pattern.
        self.merge_constraints(extra_constraints)

        # This object must create files, so after merging all constraints must
        # be in canonical form.
        for constraint in self.constraints:
            if not constraint.values:
                module_logger.error("Constraint {0} is empty - should be in canonical form!"
                                    .format(constraint))
                raise EmptyConstraintError

        # Set up the subset types (the attribute names present in the DataSet.
        self.subset_types = [cons.key for cons in self.constraints]
        self.huge_iterator = itertools.product(*[cons.values
                                                 for cons in self.constraints])

        # A set to hold all the valid combinations of attributes.
        self.valid_combinations = set()
        # One to hold the valid_hashes
        self.valid_hashes = set()

        module_logger.debug("After init, self.constraints: {}"
                            .format(self.constraints))

    def get_files(self, att_dict, check=False, update=True, map_dict={}):
        """ This method returns all possible MockClimateFiles from the
        FileCreator that match an input attribute dictionary.

        If check is True, then we check that the hash of the for the
        file is in the - 'valid_hashes' hash list. This is used when using
        the FileCreator as an input, we only want to give files that
        actually exists.

        """

        module_logger.debug("Search attribute dict is: {}".format(att_dict))
        module_logger.debug("Subset types are: {}".format(self.subset_types))
        module_logger.debug("Before getting constraint, all constraints are: {}"
                            .format(self.constraints))
        
        for mapping in map_dict:
            map_val = att_dict.pop(mapping)
            att_dict[map_dict[mapping]] = map_val

        # Get the keys of the input dictionary.
        search_keys = [att for att in att_dict.keys()]

        to_loop = []
        existing_values = []
        for key in self.subset_types:
            # If a key is not in the att_dict, grab the existing constraint.
            if key not in search_keys:
                existing_cons = self.get_constraint(key)
                to_loop.append((existing_cons.key, existing_cons.values))
                existing_values.append(existing_cons.values)
                module_logger.debug("Adding {0} to to_loop: constraint from output"
                                    .format((existing_cons.key, existing_cons.values)))
                assert(type(existing_cons.values == set))
            else:
                new_cons = Constraint(key, [att_dict[key]])
                to_loop.append((new_cons.key, new_cons.values))
                module_logger.debug("Adding {0} to to_loop: constraint from input"
                                    .format((new_cons.key, new_cons.values)))

        module_logger.debug("to_loop is: {0}".format(to_loop))

        keys = [cons[0] for cons in to_loop]
        values = [cons[1] for cons in to_loop]

        module_logger.debug("Keys to search for are: {0}".format(keys))
        module_logger.debug("Building itertools product for values: {0}"
                            .format(values))
        new_iter = itertools.product(*values)

        outfiles = []
        for combination in new_iter:
            new_file = self.climate_file_from_combination(keys, combination,
                                                          check, update)
            if new_file:
                outfiles.append(new_file)

        return outfiles

    @property
    def files(self):
        """ This property returns all the files that can be possibly
        returned by this file_creator.

        This can make some massive lists! Shouldn't really be
        used in looping.

        """
        for combination in self.huge_iterator:
            # Create a set of constraints for this combination.
            yield self.climate_file_from_combination(self.subset_types, combination,
                                                     check=False, update=False)

    def get_constraint(self, attribute_name):
        """ Get a particular constraint by name."""

        for constraint in self.constraints:
            if constraint.key == attribute_name:
                module_logger.debug("Returning constraint: {}"
                                    .format(constraint))
                return constraint

        # If it can't be found, return None.
        return None

    def merge_constraints(self, new_constraints):
        """ This function adds the constraint values to the constraints from
        a pattern.

        """

        existing_cons_names = [cons.key for cons in self.constraints]

        # Now add the constraints - only if they are in the pattern!
        for cons in new_constraints:
            if cons.key in existing_cons_names:
                self.constraints.add(cons)

        attribute_names = [cons.key for cons in self.constraints]

        repeated_atts = []
        for name in attribute_names:
            if attribute_names.count(name) > 1:
                repeated_atts.append(name)

        to_remove = [cons for cons in self.constraints
                     if cons.key in repeated_atts]

        new_cons_dict = {}
        for cons in to_remove:
            new_cons_dict[cons.key] = set([])

        module_logger.debug("Constraints to remove: {0}"
                            .format(to_remove))

        for cons in to_remove:
{'fraction_non_alphanumeric': 0.06603122844422149, 'fraction_numerical': 0.0025083087728099327, 'mean_word_length': 2.2127316680096696, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '87010', 'n_tokens_mistral': 4088, 'n_tokens_neox': 3922, 'n_words': 1522}