Repeated posterior evaluation (and backward) with same base_samples raises second backward issue
I am encountering an second backward error when doing repeated posterior evaluations with the same set of base_samples. If I use `detach_test_caches(True)` and `fast_pred_var()`, there is no issue. However, when I do not use `fast_pred_var()` and/or set `detach_test_caches(False)`, then the second backward issue is raised.

Repro:
[gpytorch_base_samples_caching_backward_issue.ipynb.txt](https://github.com/cornellius-gp/gpytorch/files/2937770/gpytorch_base_samples_caching_backward_issue.ipynb.txt)

diff --git a/gpytorch/lazy/lazy_tensor.py b/gpytorch/lazy/lazy_tensor.py
index f9aed6b0..9a0eaa30 100644
--- a/gpytorch/lazy/lazy_tensor.py
+++ b/gpytorch/lazy/lazy_tensor.py
@@ -995,7 +995,11 @@ class LazyTensor(object):
 
     @property
     def requires_grad(self):
-        return any(arg.requires_grad for arg in tuple(self._args) + tuple(self._kwargs.values()))
+        return any(
+            arg.requires_grad
+            for arg in tuple(self._args) + tuple(self._kwargs.values())
+            if hasattr(arg, "requires_grad")
+        )
 
     @requires_grad.setter
     def requires_grad(self, val):
diff --git a/gpytorch/models/exact_prediction_strategies.py b/gpytorch/models/exact_prediction_strategies.py
index 6366073c..2eb61e96 100644
--- a/gpytorch/models/exact_prediction_strategies.py
+++ b/gpytorch/models/exact_prediction_strategies.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import torch
+import contextlib
 
 from .. import settings
 from ..distributions import MultivariateNormal
@@ -315,9 +316,18 @@ class DefaultPredictionStrategy(object):
             return ZeroLazyTensor(*test_test_covar.size())
 
         if settings.fast_pred_var.off():
-            train_train_covar = self.likelihood(
-                MultivariateNormal(torch.zeros(1), self.train_train_covar), self.train_inputs
-            ).lazy_covariance_matrix
+            if settings.detach_test_caches.on():
+                if self.train_train_covar.requires_grad:
+                    self.train_train_covar = self.train_train_covar.detach()
+                lik_context = torch.no_grad
+            else:
+                lik_context = contextlib.suppress
+
+            with lik_context():
+                train_train_covar = self.likelihood(
+                    MultivariateNormal(torch.zeros(1), self.train_train_covar), self.train_inputs
+                ).lazy_covariance_matrix
+
             test_train_covar = test_train_covar.evaluate()
             train_test_covar = test_train_covar.transpose(-1, -2)
             covar_correction_rhs = train_train_covar.inv_matmul(train_test_covar).mul(-1)
['gpytorch/lazy/lazy_tensor.py', 'gpytorch/models/exact_prediction_strategies.py']