<commit_before># -*- coding: utf-8 -*-

"""
Supporting functions for Bishop
"""

__author__ = "<NAME>"
__license__ = "MIT"

import ConfigParser
import os
import pickle
import pkg_resources
from PosteriorContainer import *
from Observer import *
from Map import *
from Agent import *


def CompareInferences(ContainerA, ContainerB, decimals=1):
    """
    This function takes two PosteriorContainer objects,
    finds common variables (terrains or objects), and it
    computes the likelihood that the underlying value is
    the same. This likelihood is computed by binning samples
    so the function requires a level of granularity.

    Args:
        ContainerA (PosteriorContainer)
        ContainerB (PosteriorContainer)
        decimals (float): How many decimals to keep in samples?
    """
    # Find which dimensions are shared in common.
    SharedCosts = list(
        set(ContainerA.CostNames).intersection(ContainerB.CostNames))
    SharedRewards = list(
        set(ContainerA.ObjectNames).intersection(ContainerB.ObjectNames))
    # Print which things we're not using.
    sys.stdout.write("Terrain that cannot be compared: ")
    for Cost in ContainerA.CostNames:
        if Cost not in SharedCosts:
            sys.stdout.write(str(Cost) + " ")
    sys.stdout.write("\n")
    sys.stdout.write("Objects that cannot be compared: ")
    for Object in ContainerA.ObjectNames:
        if Object not in SharedRewards:
            sys.stdout.write(str(Object) + " ")
    sys.stdout.write("\n")
    # Get aligned sampled indices.
    # Now, the samples from SharedCost[i] can be accessed using
    # ContainerA.CostSamples(ContainerA_CostIndices[i])
    ContainerA_CostIndices = []
    ContainerB_CostIndices = []
    for Cost in SharedCosts:
        ContainerA_CostIndices.append(ContainerA.CostNames.index(Cost))
        ContainerB_CostIndices.append(ContainerB.CostNames.index(Cost))
    ContainerA_RewardIndices = []
    ContainerB_RewardIndices = []
    for Reward in SharedRewards:
        ContainerA_RewardIndices.append(ContainerA.ObjectNames.index(Reward))
        ContainerB_RewardIndices.append(ContainerB.ObjectNames.index(Reward))
    # For each shared cost, get the vectors, round them, and compute the
    # probability
    for i in range(len(SharedCosts)):
        Cost = SharedCosts[i]
        ContainerASamples = np.round(ContainerA.CostSamples[:, ContainerA_CostIndices[i]], decimals)
        ContainerBSamples = np.round(ContainerB.CostSamples[:, ContainerB_CostIndices[i]], decimals)
        # Create dictionaries where you add the probabilities.
        # Get the domain.
        # But first get rid of the ugliens np.round returns!
        ContainerASamples = ContainerASamples.tolist()
        ContainerASamples = [ContainerASamples[j][0] for j in range(len(ContainerASamples))]
        ContainerBSamples = ContainerBSamples.tolist()
        ContainerBSamples = [ContainerBSamples[j][0] for j in range(len(ContainerBSamples))]
        Domain = list(np.unique(list(np.unique(ContainerASamples)) +
                                list(np.unique(ContainerBSamples))))
        Probabilities_ContainerA = {}
        Probabilities_ContainerB = {}
        for j in range(len(Domain)):
            Probabilities_ContainerA[Domain[j]] = 0
            Probabilities_ContainerB[Domain[j]] = 0
        # Now loop over samples from containers and populate the dictionaries
        # with the likelihoods.
        for index in range(len(ContainerASamples)):
            Probabilities_ContainerA[ContainerASamples[
                index]] += np.exp(ContainerA.LogLikelihoods[index])
        for index in range(len(ContainerBSamples)):
            Probabilities_ContainerB[ContainerBSamples[
                index]] += np.exp(ContainerB.LogLikelihoods[index])
        # Note: Likelihoods are already normalized in the posterior container
        # Compute the probability that they're the same value by iterating over the domain, multiplying the
        # probabilities, and adding them.
        SameProb = 0
        for Sample in Domain:
            SameProb += Probabilities_ContainerA[Sample] * \
                Probabilities_ContainerB[Sample]
        sys.stdout.write("Probability that " + Cost + " is the same: " + str(SameProb) + "\n")
    for i in range(len(SharedRewards)):
        Reward = SharedRewards[i]
        ContainerASamples = np.round(ContainerA.RewardSamples[:, ContainerA_RewardIndices[i]], decimals)
        ContainerBSamples = np.round(ContainerB.RewardSamples[:, ContainerB_RewardIndices[i]], decimals)
        # Create dictionaries where you add the probabilities.
        # Get the domain.
        # But first get rid of the ugliens np.round returns!
        ContainerASamples = ContainerASamples.tolist()
        ContainerASamples = [ContainerASamples[j][0] for j in range(len(ContainerASamples))]
        ContainerBSamples = ContainerBSamples.tolist()
        ContainerBSamples = [ContainerBSamples[j][0] for j in range(len(ContainerBSamples))]
        Domain = list(np.unique(list(np.unique(ContainerASamples)) +
                                list(np.unique(ContainerBSamples))))
        Probabilities_ContainerA = {}
        Probabilities_ContainerB = {}
        for j in range(len(Domain)):
            Probabilities_ContainerA[Domain[j]] = 0
            Probabilities_ContainerB[Domain[j]] = 0
        # Now loop over samples from containers and populate the dictionaries
        # with the likelihoods.
        for index in range(len(ContainerASamples)):
            Probabilities_ContainerA[ContainerASamples[
                index]] += np.exp(ContainerA.LogLikelihoods[index])
        for index in range(len(ContainerBSamples)):
            Probabilities_ContainerB[ContainerBSamples[
                index]] += np.exp(ContainerB.LogLikelihoods[index])
        SameProb = 0
        for Sample in Domain:
            SameProb += Probabilities_ContainerA[Sample] * \
                Probabilities_ContainerB[Sample]
        sys.stdout.write("Probability that " + Reward + " is the same: " + str(SameProb) + "\n")


def SaveSamples(Container, Name):
    """
    Save object as a pickle file.

    Args:
        Container (PosteriorContainer): PosteriorContainer object
        Name (string): Filename. Function adds ".p" extension if it's not provided
    """
    try:
        if Name[-2:] != ".p":
            Name = Name + ".p"
        pickle.dump(Container, open(Name, "wb"))
    except Exception as error:
        print(error)


def LoadSamples(FileName):
    """
    Load samples from a pickle file

    Args:<commit_msg>New methods for comparing inferences<commit_after># -*- coding: utf-8 -*-

"""
Supporting functions for Bishop
"""

__author__ = "<NAME>"
__license__ = "MIT"

import ConfigParser
import os
import pickle
import pkg_resources
import copy
from PosteriorContainer import *
from Observer import *
from Map import *
from Agent import *
from DictionaryNavigation import *


def CompareInferences(ContainerA, ContainerB, decimals=1, csv=0, ContAName=None, ContBName=None):
    """
    This function takes two PosteriorContainer objects,
    finds common variables (terrains or objects), and it
    computes the likelihood that the underlying value is
    the same. This likelihood is computed by binning samples
    so the function requires a level of granularity.

    Args:
        ContainerA (PosteriorContainer)
        ContainerB (PosteriorContainer)
        decimals (float): How many decimals to keep in samples?
        ContAName (string): Container A's name
        ContBName (string): Container B's name
        csv (boolean): Should output be printed in a csv-friendly format?
    """
    # Find which dimensions are shared in common.
    SharedCosts = list(
        set(ContainerA.CostNames).intersection(ContainerB.CostNames))
    SharedRewards = list(
        set(ContainerA.ObjectNames).intersection(ContainerB.ObjectNames))
    # Print which things we're not using.
    if not csv:
        sys.stdout.write("Terrain that cannot be compared: ")
        for Cost in ContainerA.CostNames:
            if Cost not in SharedCosts:
                sys.stdout.write(str(Cost) + " ")
        sys.stdout.write("\n")
        sys.stdout.write("Objects that cannot be compared: ")
        for Object in ContainerA.ObjectNames:
            if Object not in SharedRewards:
                sys.stdout.write(str(Object) + " ")
        sys.stdout.write("\n")
    # Get aligned sampled indices.
    # Now, the samples from SharedCost[i] can be accessed using
    # ContainerA.CostSamples(ContainerA_CostIndices[i])
    ContainerA_CostIndices = []
    ContainerB_CostIndices = []
    for Cost in SharedCosts:
        ContainerA_CostIndices.append(ContainerA.CostNames.index(Cost))
        ContainerB_CostIndices.append(ContainerB.CostNames.index(Cost))
    ContainerA_RewardIndices = []
    ContainerB_RewardIndices = []
    for Reward in SharedRewards:
        ContainerA_RewardIndices.append(ContainerA.ObjectNames.index(Reward))
        ContainerB_RewardIndices.append(ContainerB.ObjectNames.index(Reward))
    # For each shared cost, get the vectors, round them, and compute the
    # probability
    if csv:
        if ContAName is not None or ContBName is not None:
            sys.stdout.write("EventA,EventB,")
        sys.stdout.write("decimals,")
        for Cost in SharedCosts:
            sys.stdout.write(str(Cost) + ",")
        for Reward in SharedRewards:
            if Reward == SharedRewards[-1]:
                sys.stdout.write(str(Reward) + "\n")
            else:
                sys.stdout.write(str(Reward) + ",")
        if ContAName is not None or ContBName is not None:
            sys.stdout.write(str(ContAName) + "," + str(ContBName) + ",")
        sys.stdout.write(str(decimals) + ",")
    for i in range(len(SharedCosts)):
        Cost = SharedCosts[i]
        ContainerASamples = np.round(ContainerA.CostSamples[
                                     :, ContainerA_CostIndices[i]], decimals)
        ContainerBSamples = np.round(ContainerB.CostSamples[
                                     :, ContainerB_CostIndices[i]], decimals)
        # Create dictionaries where you add the probabilities.
        # Get the domain.
        # But first get rid of the ugliens np.round returns!
        ContainerASamples = ContainerASamples.tolist()
        ContainerASamples = [ContainerASamples[j][0]
                             for j in range(len(ContainerASamples))]
        ContainerBSamples = ContainerBSamples.tolist()
        ContainerBSamples = [ContainerBSamples[j][0]
                             for j in range(len(ContainerBSamples))]
        Domain = list(np.unique(list(np.unique(ContainerASamples)) +
                                list(np.unique(ContainerBSamples))))
        Probabilities_ContainerA = {}
        Probabilities_ContainerB = {}
        for j in range(len(Domain)):
            Probabilities_ContainerA[Domain[j]] = 0
            Probabilities_ContainerB[Domain[j]] = 0
        # Now loop over samples from containers and populate the dictionaries
        # with the likelihoods.
        for index in range(len(ContainerASamples)):
            Probabilities_ContainerA[ContainerASamples[
                index]] += np.exp(ContainerA.LogLikelihoods[index])
        for index in range(len(ContainerBSamples)):
            Probabilities_ContainerB[ContainerBSamples[
                index]] += np.exp(ContainerB.LogLikelihoods[index])
        # Note: Likelihoods are already normalized in the posterior container
        # Compute the probability that they're the same value by iterating over the domain, multiplying the
        # probabilities, and adding them.
        SameProb = 0
        for Sample in Domain:
            SameProb += Probabilities_ContainerA[Sample] * \
                Probabilities_ContainerB[Sample]
        if csv:
            sys.stdout.write(str(SameProb) + ",")
        else:
            sys.stdout.write("Probability that " + Cost +
                             " is the same: " + str(SameProb) + "\n")
    for i in range(len(SharedRewards)):
        Reward = SharedRewards[i]
        ContainerASamples = np.round(ContainerA.RewardSamples[
                                     :, ContainerA_RewardIndices[i]], decimals)
        ContainerBSamples = np.round(ContainerB.RewardSamples[
                                     :, ContainerB_RewardIndices[i]], decimals)
        # Create dictionaries where you add the probabilities.
        # Get the domain.
        # But first get rid of the ugliens np.round returns!
        ContainerASamples = ContainerASamples.tolist()
        ContainerASamples = [ContainerASamples[j][0]
                             for j in range(len(ContainerASamples))]
        ContainerBSamples = ContainerBSamples.tolist()
        ContainerBSamples = [ContainerBSamples[j][0]
                             for j in range(len(ContainerBSamples))]
        Domain = list(np.unique(list(np.unique(ContainerASamples)) +
                                list(np.unique(ContainerBSamples))))
        Probabilities_ContainerA = {}
        Probabilities_ContainerB = {}
        for j in range(len(Domain)):
            Probabilities_ContainerA[Domain[j]] = 0
            Probabilities_ContainerB[Domain[j]] = 0
        # Now loop over samples from containers and populate the dictionaries
        # with the likelihoods.
        for index in range(len(ContainerASamples)):
            Probabilities_ContainerA[ContainerASamples[
                index]] += np.exp(ContainerA.LogLikelihoods[index])
        for index in range(len(ContainerBSamples)):
            Probabilities_ContainerB[ContainerBSamples[
                index]] += np.exp(ContainerB.LogLikelihoods[index])
        SameProb = 0
        for Sample in Domain:
            SameProb += Probabilities_ContainerA[Sample] * \
                Probabilities_ContainerB[Sample]
        if csv:
            if i == (len(SharedRewards) - 1):
                sys.stdout.write(str(SameProb) + "\n")
            else:
                sys.stdout.write(str(SameProb) + ",")
        else:
            sys.stdout.write("Probability that " + Reward +
                             " is the same: " + str(SameProb) + "\n")


def ProbabilityOfNoChange(ContA, ContB, TestVariable, Conditioning, decimals=1):
    """
    Compute the probability that a random variable has the same value in two events,
    conditioned on one or more random variables being the same. For instance, you can compute
    the probability that a terrain's cost is different, conditioned on the rewards being the same.

    Args:
        ContA (PosteriorContainer): PosteriorContainer object
        ContB (PosteriorContainer): PosteriorContainer object
        TestVariable (string): Random variable to test. Must exist in both containers.
        Conditioning (list of strings): Random variable names to fix across events. Must exist in both containers.
        decimals (int): Decimals to cut off from samples.
    """
    ContainerA = copy.deepcopy(ContA)
    ContainerB = copy.deepcopy(ContB)
    if (ContainerA.CostNames != ContainerB.CostNames) or (ContainerA.ObjectNames != ContainerB.ObjectNames):
        sys.stdout.write(
            "Error: ProbabilityOfChange only supports containers with matched objects and terrains.")
        return None
    # First check that test variable and conditioning exist in both Containers.
    Test = [(x in (ContainerA.CostNames + ContainerA.ObjectNames))
            for x in ([TestVariable] + Conditioning)]
    if sum(Test) != len(Test):
        sys.stdout.write(
            "Error: TestVariable and/or Conditioning variables do not exist on both PosteriorContainer objects. AUXILIARYFUNCTIONS-001")
        return None
    # Round all the samples!
    ContainerA.CostSamples = np.round(ContainerA.CostSamples, decimals)
    ContainerA.RewardSamples = np.round(ContainerA.RewardSamples, decimals)
    ContainerB.CostSamples = np.round(ContainerB.CostSamples, decimals)
    ContainerB.RewardSamples = np.round(ContainerB.RewardSamples, decimals)
    # Loop through samples and get the probabilities of each conditioning
    # variable matching.
    ConditioningProbabilities = {}
    for IndexA in range(ContainerA.Samples):
        for IndexB in range(ContainerB.Samples):
            # Check if sample pair matches conditioning.
            Hit = True
            for ConditioningVar in Conditioning:
                if ConditioningVar in ContainerA.CostNames:
                    if ContainerA.CostSamples[IndexA, ContainerA.CostNames.index(ConditioningVar)] != ContainerB.CostSamples[IndexB, ContainerB.CostNames.index(ConditioningVar)]:
                        Hit = False
                        break
                else:
                    if ContainerA.RewardSamples[IndexA, ContainerA.ObjectNames.index(ConditioningVar)] != ContainerB.RewardSamples[IndexB, ContainerB.ObjectNames.index(ConditioningVar)]:
                        Hit = False
                        break
            if Hit:
                # If we found a sample pair where the conditioning is true,
                # then save the sample.
                # Loop over the Conditiniong variables and add the
                # probabilities.
                SampleHit = []
                for ConditioningVar in Conditioning:
                    if ConditioningVar in ContainerA.CostNames:
                        # We could also get the sample from ContainerB since
                        # they're matched.
                        SampleHit.append(ContainerA.CostSamples[
                                         IndexA, ContainerA.CostNames.index(ConditioningVar)])
                    else:
                        SampleHit.append(ContainerA.RewardSamples[
                                         IndexA, ContainerA.ObjectNames.index(ConditioningVar)])
                Prob = np.exp(ContainerA.LogLikelihoods[
                              IndexA]) * np.exp(ContainerB.LogLikelihoods[IndexB])
                # Save if TestVariable matches
                if TestVariable in ContainerA.CostNames:
                    TestMatch = (ContainerA.CostSamples[IndexA, ContainerA.CostNames.index(
                        TestVariable)] == ContainerB.CostSamples[IndexB, ContainerB.CostNames.index(TestVariable)])
                else:
                    TestMatch = (ContainerA.RewardSamples[IndexA, ContainerA.ObjectNames.index(
                        TestVariable)] == ContainerB.RewardSamples[IndexB, ContainerB.ObjectNames.index(TestVariable)])
                # Now save the results.
                # First dictionary entry is whether the test sample matches.
                # Check if dictionary already has entries. Otherwise build a
                # dictionary inside.
                if TestMatch not in ConditioningProbabilities:
                    ConditioningProbabilities[TestMatch] = {}
                # The next N dictionary depths come from the ConditioningVar
                # variables. For each, try to go in and build dictionaries when
                # necessary.
                CurrentDict = ConditioningProbabilities[TestMatch]
                for SampleIndex in range(len(SampleHit) - 1):
                    Sample = SampleHit[SampleIndex]
                    if Sample not in CurrentDict:
                        CurrentDict[Sample] = {}
                    CurrentDict = CurrentDict[Sample]
                # By now, CurrentDict has the final dictionary. Now check if
                # the final sample has a value already.
                if SampleHit[-1] in CurrentDict:
                    CurrentDict[SampleHit[-1]] += Prob
                else:
                    CurrentDict[SampleHit[-1]] = Prob
    # return ConditioningProbabilities
    # Now that you have the dictionary, compute the probabilities.
    # \sum_{x} p(TestVariable match | Conditioning = x)p( Conditioning = x).
    # Every branch in the dictionary is a value of x.
    # But we need the normalizing constant.
    NormalizingConstant = RecursiveDictionaryExtraction(
        ConditioningProbabilities)
    print NormalizingConstant
    print ConditioningProbabilities
    # Normalize the tree
    ConditioningProbabilities = NormalizeDictionary(ConditioningProbabilities, NormalizingConstant)
    # Get indices for all entires:
    SucessDictionaryEntries = BuildKeyList(ConditioningProbabilities[True])
    FailDictionaryEntries = BuildKeyList(ConditioningProbabilities[False])
    # each line is a sample.
    # Iterate over Success samples
    print ConditioningProbabilities
    FullProb = 0
    for IndexPath in SucessDictionaryEntries:
        # Compute (p(Conditioning = IndexPath))
        if IndexPath in FailDictionaryEntries:
            p_fail = RetrieveValue(ConditioningProbabilities[False], IndexPath)
        else:
            p_fail = 0
        p_success = RetrieveValue(ConditioningProbabilities[True], IndexPath)
        # Normalize
        p_IndexPath = p_success + p_fail
        if p_IndexPath > 0:
            p_Match = p_success*1.0 / (p_IndexPath)
            FullProb += p_Match * p_IndexPath
    return FullProb


def SaveSamples(Container, Name):
    """
    Save object as a pickle file.

    Args:
        Container (PosteriorContainer): PosteriorContainer object
        Name (string): Filename. Function adds ".p" extension if it's not provided
    """
    try:
        if Name[-2:] != ".p":
            Name = Name + ".p"
        pickle.dump(Container, open(Name, "wb"))
    except Exception as error:
        print(error)


def LoadSamples(FileName):
    """
    Load samples from a pickle file

    Args:
{'fraction_non_alphanumeric': 0.08285991409172787, 'fraction_numerical': 0.0019398642095053346, 'mean_word_length': 2.45989133908597, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2105925', 'n_tokens_mistral': 5819, 'n_tokens_neox': 5480, 'n_words': 1541}