<commit_before>"""Methods for creating saliency maps."""

import pickle
import numpy
from keras import backend as K
from gewittergefahr.gg_io import storm_tracking_io as tracking_io
from gewittergefahr.gg_utils import monte_carlo
from gewittergefahr.gg_utils import file_system_utils
from gewittergefahr.gg_utils import error_checking
from gewittergefahr.deep_learning import model_interpretation

TOLERANCE = 1e-6
DEFAULT_IDEAL_ACTIVATION = 2.

INPUT_MATRICES_KEY = 'list_of_input_matrices'
SALIENCY_MATRICES_KEY = 'list_of_saliency_matrices'

FULL_IDS_KEY = tracking_io.FULL_IDS_KEY
STORM_TIMES_KEY = tracking_io.STORM_TIMES_KEY
MODEL_FILE_KEY = model_interpretation.MODEL_FILE_KEY
COMPONENT_TYPE_KEY = 'component_type_string'
TARGET_CLASS_KEY = 'target_class'
LAYER_NAME_KEY = 'layer_name'
IDEAL_ACTIVATION_KEY = 'ideal_activation'
NEURON_INDICES_KEY = 'neuron_indices'
CHANNEL_INDEX_KEY = 'channel_index'
SOUNDING_PRESSURES_KEY = 'sounding_pressure_matrix_pascals'

STANDARD_FILE_KEYS = [
    INPUT_MATRICES_KEY, SALIENCY_MATRICES_KEY, FULL_IDS_KEY, STORM_TIMES_KEY,
    MODEL_FILE_KEY, COMPONENT_TYPE_KEY, TARGET_CLASS_KEY, LAYER_NAME_KEY,
    IDEAL_ACTIVATION_KEY, NEURON_INDICES_KEY, CHANNEL_INDEX_KEY,
    SOUNDING_PRESSURES_KEY
]

MEAN_INPUT_MATRICES_KEY = model_interpretation.MEAN_INPUT_MATRICES_KEY
MEAN_SALIENCY_MATRICES_KEY = 'list_of_mean_saliency_matrices'
THRESHOLD_COUNTS_KEY = 'threshold_count_matrix'
STANDARD_FILE_NAME_KEY = 'standard_saliency_file_name'
PMM_METADATA_KEY = 'pmm_metadata_dict'
MONTE_CARLO_DICT_KEY = 'monte_carlo_dict'

PMM_FILE_KEYS = [
    MEAN_INPUT_MATRICES_KEY, MEAN_SALIENCY_MATRICES_KEY, THRESHOLD_COUNTS_KEY,
    MODEL_FILE_KEY, STANDARD_FILE_NAME_KEY, PMM_METADATA_KEY,
    MONTE_CARLO_DICT_KEY
]


def _do_saliency_calculations(
        model_object, loss_tensor, list_of_input_matrices):
    """Does saliency calculations.

    T = number of input tensors to the model
    E = number of examples (storm objects)

    :param model_object: Instance of `keras.models.Model`.
    :param loss_tensor: Keras tensor defining the loss function.
    :param list_of_input_matrices: length-T list of numpy arrays, comprising one
        or more examples (storm objects).  list_of_input_matrices[i] must have
        the same dimensions as the [i]th input tensor to the model.
    :return: list_of_saliency_matrices: length-T list of numpy arrays,
        comprising the saliency map for each example.
        list_of_saliency_matrices[i] has the same dimensions as
        list_of_input_matrices[i] and defines the "saliency" of each value x,
        which is the gradient of the loss function with respect to x.
    """

    if isinstance(model_object.input, list):
        list_of_input_tensors = model_object.input
    else:
        list_of_input_tensors = [model_object.input]

    list_of_gradient_tensors = K.gradients(loss_tensor, list_of_input_tensors)
    num_input_tensors = len(list_of_input_tensors)
    for i in range(num_input_tensors):
        list_of_gradient_tensors[i] /= K.maximum(
            K.std(list_of_gradient_tensors[i]), K.epsilon())

    inputs_to_gradients_function = K.function(
        list_of_input_tensors + [K.learning_phase()], list_of_gradient_tensors)

    # list_of_saliency_matrices = None
    # num_examples = list_of_input_matrices[0].shape[0]
    #
    # for i in range(num_examples):
    #     these_input_matrices = [a[[i], ...] for a in list_of_input_matrices]
    #     these_saliency_matrices = inputs_to_gradients_function(
    #         these_input_matrices + [0])
    #
    #     if list_of_saliency_matrices is None:
    #         list_of_saliency_matrices = these_saliency_matrices + []
    #     else:
    #         for i in range(num_input_tensors):
    #             list_of_saliency_matrices[i] = numpy.concatenate(
    #                 (list_of_saliency_matrices[i], these_saliency_matrices[i]),
    #                 axis=0)

    list_of_saliency_matrices = inputs_to_gradients_function(
        list_of_input_matrices + [0])

    for i in range(num_input_tensors):<commit_msg>Made style tweaks to saliency_maps.py.<commit_after>"""Methods for creating saliency maps."""

import pickle
import numpy
from keras import backend as K
from gewittergefahr.gg_io import storm_tracking_io as tracking_io
from gewittergefahr.gg_utils import monte_carlo
from gewittergefahr.gg_utils import file_system_utils
from gewittergefahr.gg_utils import error_checking
from gewittergefahr.deep_learning import model_interpretation

TOLERANCE = 1e-6
DEFAULT_IDEAL_ACTIVATION = 2.

INPUT_MATRICES_KEY = 'list_of_input_matrices'
SALIENCY_MATRICES_KEY = 'list_of_saliency_matrices'

FULL_IDS_KEY = tracking_io.FULL_IDS_KEY
STORM_TIMES_KEY = tracking_io.STORM_TIMES_KEY
MODEL_FILE_KEY = model_interpretation.MODEL_FILE_KEY
COMPONENT_TYPE_KEY = 'component_type_string'
TARGET_CLASS_KEY = 'target_class'
LAYER_NAME_KEY = 'layer_name'
IDEAL_ACTIVATION_KEY = 'ideal_activation'
NEURON_INDICES_KEY = 'neuron_indices'
CHANNEL_INDEX_KEY = 'channel_index'
SOUNDING_PRESSURES_KEY = 'sounding_pressure_matrix_pascals'

STANDARD_FILE_KEYS = [
    INPUT_MATRICES_KEY, SALIENCY_MATRICES_KEY, FULL_IDS_KEY, STORM_TIMES_KEY,
    MODEL_FILE_KEY, COMPONENT_TYPE_KEY, TARGET_CLASS_KEY, LAYER_NAME_KEY,
    IDEAL_ACTIVATION_KEY, NEURON_INDICES_KEY, CHANNEL_INDEX_KEY,
    SOUNDING_PRESSURES_KEY
]

MEAN_INPUT_MATRICES_KEY = model_interpretation.MEAN_INPUT_MATRICES_KEY
MEAN_SALIENCY_MATRICES_KEY = 'list_of_mean_saliency_matrices'
THRESHOLD_COUNTS_KEY = 'threshold_count_matrix'
STANDARD_FILE_NAME_KEY = 'standard_saliency_file_name'
PMM_METADATA_KEY = 'pmm_metadata_dict'
MONTE_CARLO_DICT_KEY = 'monte_carlo_dict'

PMM_FILE_KEYS = [
    MEAN_INPUT_MATRICES_KEY, MEAN_SALIENCY_MATRICES_KEY, THRESHOLD_COUNTS_KEY,
    MODEL_FILE_KEY, STANDARD_FILE_NAME_KEY, PMM_METADATA_KEY,
    MONTE_CARLO_DICT_KEY
]


def _do_saliency_calculations(
        model_object, loss_tensor, list_of_input_matrices):
    """Does saliency calculations.

    T = number of input tensors to the model
    E = number of examples (storm objects)

    :param model_object: Instance of `keras.models.Model`.
    :param loss_tensor: Keras tensor defining the loss function.
    :param list_of_input_matrices: length-T list of numpy arrays, comprising one
        or more examples (storm objects).  list_of_input_matrices[i] must have
        the same dimensions as the [i]th input tensor to the model.
    :return: list_of_saliency_matrices: length-T list of numpy arrays,
        comprising the saliency map for each example.
        list_of_saliency_matrices[i] has the same dimensions as
        list_of_input_matrices[i] and defines the "saliency" of each value x,
        which is the gradient of the loss function with respect to x.
    """

    if isinstance(model_object.input, list):
        list_of_input_tensors = model_object.input
    else:
        list_of_input_tensors = [model_object.input]

    list_of_gradient_tensors = K.gradients(loss_tensor, list_of_input_tensors)
    num_input_tensors = len(list_of_input_tensors)

    for i in range(num_input_tensors):
        list_of_gradient_tensors[i] /= K.maximum(
            K.std(list_of_gradient_tensors[i]), K.epsilon()
        )

    inputs_to_gradients_function = K.function(
        list_of_input_tensors + [K.learning_phase()], list_of_gradient_tensors
    )

    # list_of_saliency_matrices = None
    # num_examples = list_of_input_matrices[0].shape[0]
    #
    # for i in range(num_examples):
    #     these_input_matrices = [a[[i], ...] for a in list_of_input_matrices]
    #     these_saliency_matrices = inputs_to_gradients_function(
    #         these_input_matrices + [0])
    #
    #     if list_of_saliency_matrices is None:
    #         list_of_saliency_matrices = these_saliency_matrices + []
    #     else:
    #         for i in range(num_input_tensors):
    #             list_of_saliency_matrices[i] = numpy.concatenate(
    #                 (list_of_saliency_matrices[i], these_saliency_matrices[i]),
    #                 axis=0)

    list_of_saliency_matrices = inputs_to_gradients_function(
        list_of_input_matrices + [0]
    )

    for i in range(num_input_tensors):
{'fraction_non_alphanumeric': 0.06589383770591824, 'fraction_numerical': 0.0019524100061012813, 'mean_word_length': 4.496981891348089, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6951205', 'n_tokens_mistral': 3004, 'n_tokens_neox': 2882, 'n_words': 543}