<commit_before>from .util import DotDict, recursively_convert_to_json_serializable

import json
import inspect
import copy
from functools import wraps
import traceback
import warnings

import pandas as pd
from collections import defaultdict

from ..version import __version__

class DataSet(object):

    def __init__(self, *args, **kwargs):
        super(DataSet, self).__init__(*args, **kwargs)
        self.initialize_expectations()

    @classmethod
    def expectation(cls, method_arg_names):
        """
        The core expectation decorator, this method takes a single parameter which it uses to build and save the
        expectation config to the DataSet object. The parameter defines an ordered list of the positional arguments to
        be used by the method implementing the expectation.

        Note that intermediate decorators that call the core @expectation decorator will most likely need to pass their
        decorated methods' signature up to the expectation decorator. For example, the MetaPandasDataSet column_map_expectation
        decorator relies on the DataSet expectation decorator, but will pass through the signature from the implementing method.

        When decorated with @expectation, a method will:
            1. Build and update the expectation config.
            2. Handle the "include_config" boolean parameter, which allows a caller to retrieve the generated
                configuration immediately after running the expectation.
            3. Handle the "catch_exceptions" parameter, which allows a caller to catch any exception and report an
                aggregate trace, useful for validation.
            4. Handle the "output_format" parameter, and pass it down to the implementing method if its signature expects it.
                By handing down the output_format, methods implementing expectations can optionally provide additional output formats
                specific to the use cases that they handle.
        """
        def outer_wrapper(func):
            @wraps(func)
            def wrapper(self, *args, **kwargs):

                #Get the name of the method
                method_name = func.__name__

                # Combine all arguments into a single new "kwargs"
                all_args = dict(zip(method_arg_names, args))
                all_args.update(kwargs)

                #Unpack display parameters; remove them from all_args if appropriate
                if "include_config" in kwargs:
                    include_config = kwargs["include_config"]
                    del all_args["include_config"]
                else:
                    include_config = self.default_expectation_args["include_config"]

                if "catch_exceptions" in kwargs:
                    catch_exceptions = kwargs["catch_exceptions"]
                    del all_args["catch_exceptions"]
                else:
                    catch_exceptions = self.default_expectation_args["catch_exceptions"]

                if "output_format" in kwargs:
                    output_format = kwargs["output_format"]
                else:
                    output_format = self.default_expectation_args["output_format"]

                if "meta" in kwargs:
                    #Ensure that meta is JSON serializable.
                    #FIXXME: If it's not, should we try to fix it? Allow users to override and proceed anyway?
                    json.dumps(kwargs["meta"])

                    meta = kwargs["meta"]
                    del all_args["meta"]
                else:
                    meta = None

                if "meta_notes" in kwargs:
                    meta = { "notes": kwargs["meta_notes"] }
                    del all_args["meta_notes"]

                # This intends to get the signature of the inner wrapper, if there is one.
                if "output_format" in inspect.getargspec(func)[0][1:]:
                    all_args["output_format"] = output_format
                else:
                    if "output_format" in all_args:
                        del all_args["output_format"]

                all_args = recursively_convert_to_json_serializable(all_args)
                #Ensure that all_args is JSON serializable.
                #FIXXME: If it's not, should we try to fix it? Allow users to override and proceed anyway?
                json.dumps(all_args)
                expectation_args = copy.deepcopy(all_args)

                #Construct the expectation_config object
                expectation_config = DotDict({
                    "expectation_type": method_name,
                    "kwargs": expectation_args
                })

                if meta is not None:
                    expectation_config["meta"] = meta

                raised_exception = False
                exception_traceback = None

                #Finally, execute the expectation method itself
                try:
                    return_obj = func(self, **expectation_args)

                except Exception as err:
                    if catch_exceptions:
                        raised_exception = True
                        exception_traceback = traceback.format_exc()

                        if output_format != "BOOLEAN_ONLY":
                            return_obj = {
                                "success": False
                            }
                        else:
                            return_obj = False
                    else:
                        raise(err)

                #Add a "success" object to the config
                if output_format == "BOOLEAN_ONLY":
                    expectation_config["success_on_last_run"] = return_obj
                else:
                    expectation_config["success_on_last_run"] = return_obj["success"]

                #Append the expectation to the config.
                self.append_expectation(expectation_config)

                if output_format != 'BOOLEAN_ONLY':

                    if include_config:
                        return_obj["expectation_type"] = expectation_config["expectation_type"]
                        return_obj["expectation_kwargs"] = copy.deepcopy(dict(expectation_config["kwargs"]))

                    if catch_exceptions:
                        return_obj["raised_exception"] = raised_exception
                        return_obj["exception_traceback"] = exception_traceback

                return return_obj

            # wrapper.__name__ = func.__name__
            # wrapper.__doc__ = func.__doc__
            return wrapper

        return outer_wrapper

    @classmethod
    def column_map_expectation(cls, func):
        raise NotImplementedError

    @classmethod
    def column_aggregate_expectation(cls, func):
        raise NotImplementedError

    def initialize_expectations(self, config=None, name=None):
        if config != None:
            #!!! Should validate the incoming config with jsonschema here

            # Copy the original so that we don't overwrite it by accident
            self._expectations_config = DotDict(copy.deepcopy(config))

        else:
            self._expectations_config = DotDict({
                "dataset_name" : name,
                "meta": {
                    "great_expectations.__version__": __version__
                },
                "expectations" : []
            })

        self.default_expectation_args = {
            "include_config" : False,
            "catch_exceptions" : False,
            "output_format" : 'BASIC',
        }

    def append_expectation(self, expectation_config):
        expectation_type = expectation_config['expectation_type']

        #Drop existing expectations with the same expectation_type.
        #For column_expectations, append_expectation should only replace expectations
        # where the expectation_type AND the column match
        #!!! This is good default behavior, but
        #!!!    it needs to be documented, and
        #!!!    we need to provide syntax to override it.

        if 'column' in expectation_config['kwargs']:
            column = expectation_config['kwargs']['column']

            self._expectations_config.expectations = [f for f in filter(
                lambda exp: (exp['expectation_type'] != expectation_type) or ('column' in exp['kwargs'] and exp['kwargs']['column'] != column),
                self._expectations_config.expectations
            )]
        else:<commit_msg>Move JSON checking to append_expectation<commit_after>from .util import DotDict, recursively_convert_to_json_serializable

import json
import inspect
import copy
from functools import wraps
import traceback
import warnings

import pandas as pd
from collections import defaultdict

from ..version import __version__

class DataSet(object):

    def __init__(self, *args, **kwargs):
        super(DataSet, self).__init__(*args, **kwargs)
        self.initialize_expectations()

    @classmethod
    def expectation(cls, method_arg_names):
        """
        The core expectation decorator, this method takes a single parameter which it uses to build and save the
        expectation config to the DataSet object. The parameter defines an ordered list of the positional arguments to
        be used by the method implementing the expectation.

        Note that intermediate decorators that call the core @expectation decorator will most likely need to pass their
        decorated methods' signature up to the expectation decorator. For example, the MetaPandasDataSet column_map_expectation
        decorator relies on the DataSet expectation decorator, but will pass through the signature from the implementing method.

        When decorated with @expectation, a method will:
            1. Build and update the expectation config.
            2. Handle the "include_config" boolean parameter, which allows a caller to retrieve the generated
                configuration immediately after running the expectation.
            3. Handle the "catch_exceptions" parameter, which allows a caller to catch any exception and report an
                aggregate trace, useful for validation.
            4. Handle the "output_format" parameter, and pass it down to the implementing method if its signature expects it.
                By handing down the output_format, methods implementing expectations can optionally provide additional output formats
                specific to the use cases that they handle.
        """
        def outer_wrapper(func):
            @wraps(func)
            def wrapper(self, *args, **kwargs):

                #Get the name of the method
                method_name = func.__name__

                # Combine all arguments into a single new "kwargs"
                all_args = dict(zip(method_arg_names, args))
                all_args.update(kwargs)

                #Unpack display parameters; remove them from all_args if appropriate
                if "include_config" in kwargs:
                    include_config = kwargs["include_config"]
                    del all_args["include_config"]
                else:
                    include_config = self.default_expectation_args["include_config"]

                if "catch_exceptions" in kwargs:
                    catch_exceptions = kwargs["catch_exceptions"]
                    del all_args["catch_exceptions"]
                else:
                    catch_exceptions = self.default_expectation_args["catch_exceptions"]

                if "output_format" in kwargs:
                    output_format = kwargs["output_format"]
                else:
                    output_format = self.default_expectation_args["output_format"]

                if "meta" in kwargs:
                    meta = kwargs["meta"]
                    del all_args["meta"]
                else:
                    meta = None

                if "meta_notes" in kwargs:
                    meta = { "notes": kwargs["meta_notes"] }
                    del all_args["meta_notes"]

                # This intends to get the signature of the inner wrapper, if there is one.
                if "output_format" in inspect.getargspec(func)[0][1:]:
                    all_args["output_format"] = output_format
                else:
                    if "output_format" in all_args:
                        del all_args["output_format"]

                all_args = recursively_convert_to_json_serializable(all_args)
                expectation_args = copy.deepcopy(all_args)

                #Construct the expectation_config object
                expectation_config = DotDict({
                    "expectation_type": method_name,
                    "kwargs": expectation_args
                })

                if meta is not None:
                    expectation_config["meta"] = meta

                raised_exception = False
                exception_traceback = None

                #Finally, execute the expectation method itself
                try:
                    return_obj = func(self, **expectation_args)

                except Exception as err:
                    if catch_exceptions:
                        raised_exception = True
                        exception_traceback = traceback.format_exc()

                        if output_format != "BOOLEAN_ONLY":
                            return_obj = {
                                "success": False
                            }
                        else:
                            return_obj = False
                    else:
                        raise(err)

                #Add a "success" object to the config
                if output_format == "BOOLEAN_ONLY":
                    expectation_config["success_on_last_run"] = return_obj
                else:
                    expectation_config["success_on_last_run"] = return_obj["success"]

                #Append the expectation to the config.
                self.append_expectation(expectation_config)

                if output_format != 'BOOLEAN_ONLY':

                    if include_config:
                        return_obj["expectation_type"] = expectation_config["expectation_type"]
                        return_obj["expectation_kwargs"] = copy.deepcopy(dict(expectation_config["kwargs"]))

                    if catch_exceptions:
                        return_obj["raised_exception"] = raised_exception
                        return_obj["exception_traceback"] = exception_traceback

                return return_obj

            # wrapper.__name__ = func.__name__
            # wrapper.__doc__ = func.__doc__
            return wrapper

        return outer_wrapper

    @classmethod
    def column_map_expectation(cls, func):
        raise NotImplementedError

    @classmethod
    def column_aggregate_expectation(cls, func):
        raise NotImplementedError

    def initialize_expectations(self, config=None, name=None):
        if config != None:
            #!!! Should validate the incoming config with jsonschema here

            # Copy the original so that we don't overwrite it by accident
            self._expectations_config = DotDict(copy.deepcopy(config))

        else:
            self._expectations_config = DotDict({
                "dataset_name" : name,
                "meta": {
                    "great_expectations.__version__": __version__
                },
                "expectations" : []
            })

        self.default_expectation_args = {
            "include_config" : False,
            "catch_exceptions" : False,
            "output_format" : 'BASIC',
        }

    def append_expectation(self, expectation_config):
        expectation_type = expectation_config['expectation_type']

        #Test to ensure the new expectation is serializable.
        #FIXME: If it's not, are we sure we want to raise an error?
        #FIXME: Should we allow users to override the error?
        #FIXME: Should we try to convert the object using something like recursively_convert_to_json_serializable?
        json.dumps(expectation_config)

        #Drop existing expectations with the same expectation_type.
        #For column_expectations, append_expectation should only replace expectations
        # where the expectation_type AND the column match
        #!!! This is good default behavior, but
        #!!!    it needs to be documented, and
        #!!!    we need to provide syntax to override it.

        if 'column' in expectation_config['kwargs']:
            column = expectation_config['kwargs']['column']

            self._expectations_config.expectations = [f for f in filter(
                lambda exp: (exp['expectation_type'] != expectation_type) or ('column' in exp['kwargs'] and exp['kwargs']['column'] != column),
                self._expectations_config.expectations
            )]
        else:
{'fraction_non_alphanumeric': 0.06331294173546356, 'fraction_numerical': 0.0007127160420502465, 'mean_word_length': 1.8299159663865545, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '228050', 'n_tokens_mistral': 4008, 'n_tokens_neox': 3770, 'n_words': 1242}