<commit_before># Copyright 2016 <NAME>, alexggmatthews, <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Optional

import numpy as np
import tensorflow as tf

from gpflow.kernels import Kernel
from .model import MeanAndVariance, GPModel, Data
from .. import likelihoods
from ..config import default_float, default_jitter
from ..covariances.dispatch import Kuf, Kuu
from ..inducing_variables import InducingPoints
from ..mean_functions import Zero, MeanFunction


class SGPRUpperMixin(GPModel):
    """
    Upper bound for the GP regression marginal likelihood.
    It is implemented here as a Mixin class which works with SGPR and GPRFITC.
    Note that the same inducing points are used for calculating the upper bound,
    as are used for computing the likelihood approximation. This may not lead to
    the best upper bound. The upper bound can be tightened by optimising Z, just
    as just like the lower bound. This is especially important in FITC, as FITC
    is known to produce poor inducing point locations. An optimisable upper bound
    can be found in https://github.com/markvdw/gp_upper.

    The key reference is

    ::

      @misc{titsias_2014,
        title={Variational Inference for Gaussian and Determinantal Point Processes},
        url={http://www2.aueb.gr/users/mtitsias/papers/titsiasNipsVar14.pdf},
        publisher={Workshop on Advances in Variational Inference (NIPS 2014)},
        author={Titsias, <NAME>.},
        year={2014},
        month={Dec}
      }
    """

    def upper_bound(self):
        x_data, y_data = self.data
        num_data = tf.cast(tf.shape(y_data)[0], default_float())

        Kdiag = self.kernel(x_data)
        kuu = Kuu(self.inducing_variables, self.kernel, jitter=default_jitter())
        kuf = Kuf(self.inducing_variables, self.kernel, x_data)

        L = tf.linalg.cholesky(kuu)
        LB = tf.linalg.cholesky(kuu + self.likelihood.variance ** -1.0 *
                                tf.linalg.matmul(kuf, kuf, transpose_b=True))

        Linvkuf = tf.linalg.triangular_solve(L, kuf, lower=True)
        # Using the Trace bound, from Titsias' presentation
        # c = tf.reduce_sum(Kdiag) - tf.reduce_sum(Linvkuf ** 2.0)
        Kff = self.kernel(x_data)
        Qff = tf.linalg.matmul(kuf, Linvkuf, transpose_a=True)

        # Alternative bound on max eigenval:
        c = tf.reduce_max(tf.reduce_sum(tf.abs(Kff - Qff), 0))
        corrected_noise = self.likelihood.variance + c

        const = -0.5 * num_data * tf.math.log(
            2 * np.pi * self.likelihood.variance)
        logdet = tf.reduce_sum(tf.math.log(
            tf.linalg.diag_part(L))) - tf.reduce_sum(
            tf.math.log(tf.linalg.diag_part(LB)))

        LC = tf.linalg.cholesky(kuu + corrected_noise ** -1.0 *
                                tf.linalg.matmul(kuf, kuf, transpose_b=True))
        v = tf.linalg.triangular_solve(LC,
                                       corrected_noise ** -1.0 *
                                       tf.linalg.matmul(kuf, y_data),
                                       lower=True)
        quad = -0.5 * corrected_noise ** -1.0 * tf.reduce_sum(
            y_data ** 2.0) + 0.5 * tf.reduce_sum(v ** 2.0)

        return const + logdet + quad


class SGPR(SGPRUpperMixin):
    """
    Sparse Variational GP regression. The key reference is

    ::

      @inproceedings{titsias2009variational,
        title={Variational learning of inducing variables in
               sparse Gaussian processes},
        author={<NAME>},
        booktitle={International Conference on
                   Artificial Intelligence and Statistics},
        pages={567--574},
        year={2009}
      }



    """

    def __init__(self,
                 data: Data,
                 kernel: Kernel,
                 mean_function: Optional[MeanFunction] = None,
                 inducing_variables: Optional[InducingPoints] = None
                 ):
        """
        X is a data matrix, size [N, D]
        Y is a data matrix, size [N, R]
        Z is a matrix of pseudo inputs, size [M, D]
        kernel, mean_function are appropriate GPflow objects

        This method only works with a Gaussian likelihood.
        """
        likelihood = likelihoods.Gaussian()
        x_data, y_data = data
        num_latent = y_data.shape[-1]
        super().__init__(kernel, likelihood, mean_function, num_latent)
        self.data = data
        self.num_data = x_data.shape[0]

        if not isinstance(inducing_variables, InducingPoints):
            inducing_variables = InducingPoints(inducing_variables)
        self.inducing_variables = inducing_variables

    def log_likelihood(self):
        """
        Construct a tensorflow function to compute the bound on the marginal
        likelihood. For a derivation of the terms in here, see the associated
        SGPR notebook.
        """
        x_data, y_data = self.data
        num_inducing = len(self.inducing_variables)
        num_data = tf.cast(tf.shape(y_data)[0], default_float())
        output_dim = tf.cast(tf.shape(y_data)[1], default_float())

        err = y_data - self.mean_function(x_data)
        Kdiag = self.kernel(x_data)<commit_msg>Method equivalence and VGP, SGPR and SGPFITC fixed<commit_after># Copyright 2016 <NAME>, alexggmatthews, <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Optional

import numpy as np
import tensorflow as tf

from gpflow.kernels import Kernel
from .model import MeanAndVariance, GPModel, Data
from .. import likelihoods
from ..config import default_float, default_jitter
from ..covariances.dispatch import Kuf, Kuu
from ..inducing_variables import InducingPoints
from ..mean_functions import Zero, MeanFunction


class SGPRUpperMixin(GPModel):
    """
    Upper bound for the GP regression marginal likelihood.
    It is implemented here as a Mixin class which works with SGPR and GPRFITC.
    Note that the same inducing points are used for calculating the upper bound,
    as are used for computing the likelihood approximation. This may not lead to
    the best upper bound. The upper bound can be tightened by optimising Z, just
    as just like the lower bound. This is especially important in FITC, as FITC
    is known to produce poor inducing point locations. An optimisable upper bound
    can be found in https://github.com/markvdw/gp_upper.

    The key reference is

    ::

      @misc{titsias_2014,
        title={Variational Inference for Gaussian and Determinantal Point Processes},
        url={http://www2.aueb.gr/users/mtitsias/papers/titsiasNipsVar14.pdf},
        publisher={Workshop on Advances in Variational Inference (NIPS 2014)},
        author={Titsias, <NAME>.},
        year={2014},
        month={Dec}
      }
    """

    def upper_bound(self):
        x_data, y_data = self.data
        num_data = tf.cast(tf.shape(y_data)[0], default_float())

        Kdiag = self.kernel(x_data)
        kuu = Kuu(self.inducing_variables, self.kernel, jitter=default_jitter())
        kuf = Kuf(self.inducing_variables, self.kernel, x_data)

        L = tf.linalg.cholesky(kuu)
        LB = tf.linalg.cholesky(kuu + self.likelihood.variance ** -1.0 *
                                tf.linalg.matmul(kuf, kuf, transpose_b=True))

        Linvkuf = tf.linalg.triangular_solve(L, kuf, lower=True)
        # Using the Trace bound, from Titsias' presentation
        # c = tf.reduce_sum(Kdiag) - tf.reduce_sum(Linvkuf ** 2.0)
        Kff = self.kernel(x_data)
        Qff = tf.linalg.matmul(kuf, Linvkuf, transpose_a=True)

        # Alternative bound on max eigenval:
        c = tf.reduce_max(tf.reduce_sum(tf.abs(Kff - Qff), 0))
        corrected_noise = self.likelihood.variance + c

        const = -0.5 * num_data * tf.math.log(
            2 * np.pi * self.likelihood.variance)
        logdet = tf.reduce_sum(tf.math.log(
            tf.linalg.diag_part(L))) - tf.reduce_sum(
            tf.math.log(tf.linalg.diag_part(LB)))

        LC = tf.linalg.cholesky(kuu + corrected_noise ** -1.0 *
                                tf.linalg.matmul(kuf, kuf, transpose_b=True))
        v = tf.linalg.triangular_solve(LC,
                                       corrected_noise ** -1.0 *
                                       tf.linalg.matmul(kuf, y_data),
                                       lower=True)
        quad = -0.5 * corrected_noise ** -1.0 * tf.reduce_sum(
            y_data ** 2.0) + 0.5 * tf.reduce_sum(v ** 2.0)

        return const + logdet + quad


class SGPR(SGPRUpperMixin):
    """
    Sparse Variational GP regression. The key reference is

    ::

      @inproceedings{titsias2009variational,
        title={Variational learning of inducing variables in
               sparse Gaussian processes},
        author={Titsias, <NAME>},
        booktitle={International Conference on
                   Artificial Intelligence and Statistics},
        pages={567--574},
        year={2009}
      }



    """

    def __init__(self,
                 data: Data,
                 kernel: Kernel,
                 mean_function: Optional[MeanFunction] = None,
                 inducing_variables: Optional[InducingPoints] = None,
                 num_latent: Optional[int] = None
                 ):
        """
        X is a data matrix, size [N, D]
        Y is a data matrix, size [N, R]
        Z is a matrix of pseudo inputs, size [M, D]
        kernel, mean_function are appropriate GPflow objects

        This method only works with a Gaussian likelihood.
        """
        likelihood = likelihoods.Gaussian()
        x_data, y_data = data
        num_latent = y_data.shape[-1] if num_latent is None else num_latent
        super().__init__(kernel, likelihood, mean_function, num_latent)
        self.data = data
        self.num_data = x_data.shape[0]

        if not isinstance(inducing_variables, InducingPoints):
            inducing_variables = InducingPoints(inducing_variables)
        self.inducing_variables = inducing_variables

    def log_likelihood(self):
        """
        Construct a tensorflow function to compute the bound on the marginal
        likelihood. For a derivation of the terms in here, see the associated
        SGPR notebook.
        """
        x_data, y_data = self.data
        num_inducing = len(self.inducing_variables)
        num_data = tf.cast(tf.shape(y_data)[0], default_float())
        output_dim = tf.cast(tf.shape(y_data)[1], default_float())

        err = y_data - self.mean_function(x_data)
        Kdiag = self.kernel(x_data)
{'fraction_non_alphanumeric': 0.08975804353481918, 'fraction_numerical': 0.011100511664209523, 'mean_word_length': 2.3997641509433962, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5134315', 'n_tokens_mistral': 3551, 'n_tokens_neox': 3356, 'n_words': 1010}