<jupyter_start><jupyter_text>ReLU LayersWe can write a ReLU layer $z = \max(Wx+b, 0)$ as theconvex optimization problem\begin{equation}\begin{array}{ll}\mbox{minimize} & \|z-\tilde Wx - b\|_2^2 \\[.2cm]\mbox{subject to} & z \geq 0, \\& \tilde W = W,\end{array}\label{eq:prob}\end{equation}with variables $z$ and $\tilde W$,and parameters $W$, $b$, and $x$.(Note that we have added an extra variable $\tilde W$ sothat the problem is DPP.)We can embed this problem into a TensorFlow Keras `Layer` and use itas a layer in a sequential neural network.We note that this example is purely illustrative;one can implement a ReLU layer much more efficientlyby directly performing the matrix multiplication, vector addition,and then taking the positive part.<jupyter_code>from cvxpylayers.tensorflow import CvxpyLayer
import cvxpy as cp
import tensorflow as tf
from tensorflow.keras import layers

tf.keras.backend.set_floatx('float64')
class ReluLayer(layers.Layer):
    def __init__(self, input_dim, output_dim):
        super(ReluLayer, self).__init__()
        self.W = tf.Variable(1e-3 * tf.random.normal((output_dim, input_dim), dtype=tf.float64))
        self.b = tf.Variable(1e-3 * tf.random.normal((output_dim,),  dtype=tf.float64))
        z = cp.Variable(output_dim)
        Wtilde = cp.Variable((output_dim, input_dim))
        W = cp.Parameter((output_dim, input_dim))
        b = cp.Parameter(output_dim)
        x = cp.Parameter(input_dim)
        problem = cp.Problem(cp.Minimize(
            cp.sum_squares(z - Wtilde @ x - b)), [z >= 0, Wtilde == W])
        self.cvxpy_layer = CvxpyLayer(problem, [W, b, x], [z])

    def call(self, x):
        if tf.rank(x) == 2:
            # when x is batched, repeat W and b 
            batch_size = x.shape[0]
            return self.cvxpy_layer(
                tf.stack([self.W for _ in tf.range(batch_size)]),
                tf.stack([self.b for _ in tf.range(batch_size)]), x)[0]
        else:
            return self.layer(self.W, self.b, x)[0]<jupyter_output><empty_output><jupyter_text>We generate synthetic data and create a network of two `ReluLayer`s followed by a linear layer.<jupyter_code>tf.random.set_seed(0)
model = tf.keras.Sequential([
    ReluLayer(20, 20),
    ReluLayer(20, 20),
    tf.keras.layers.Dense(1, input_shape=(20,), dtype=tf.float64)
])
X = tf.random.normal((300, 20), dtype=tf.float64)
Y = tf.random.normal((300, 1), dtype=tf.float64)<jupyter_output><empty_output><jupyter_text>Now we can optimize the parameters inside the network using, for example, the ADAM optimizer.The code below solves 15000 convex optimization problems and calls backward 15000 times.<jupyter_code>opt = tf.keras.optimizers.Adam(1e-2)
for _ in range(25):
    with tf.GradientTape() as tape:
        loss = (1 / X.shape[0]) * tf.math.reduce_sum((Y - model(X))**2)
        print(loss)
    grads = tape.gradient(loss, model.variables)
    opt.apply_gradients(zip(grads, model.variables))<jupyter_output>tf.Tensor(1.1182059444179566, shape=(), dtype=float64)
tf.Tensor(1.1135983945033263, shape=(), dtype=float64)
tf.Tensor(1.1057307058385368, shape=(), dtype=float64)
tf.Tensor(1.096769654269908, shape=(), dtype=float64)
tf.Tensor(1.084459023932493, shape=(), dtype=float64)
tf.Tensor(1.0690279563783676, shape=(), dtype=float64)
tf.Tensor(1.0512619022795688, shape=(), dtype=float64)
tf.Tensor(1.0322302881090637, shape=(), dtype=float64)
tf.Tensor(1.0125819721396856, shape=(), dtype=float64)
tf.Tensor(0.9913663215521406, shape=(), dtype=float64)
tf.Tensor(0.9653676807090706, shape=(), dtype=float64)
tf.Tensor(0.9346387377331861, shape=(), dtype=float64)
tf.Tensor(0.901583417900799, shape=(), dtype=float64)
tf.Tensor(0.8696330587760338, shape=(), dtype=float64)
tf.Tensor(0.8358727832552605, shape=(), dtype=float64)
tf.Tensor(0.8012632687553815, shape=(), dtype=float64)
tf.Tensor(0.7656949960892767, shape=(), dtype=float64)
tf.Tensor(0.732059435220954, shape=(), dtype=float64)
tf.Tensor(0.70[...]
{'fraction_non_alphanumeric': 0.14433509967196567, 'fraction_numerical': 0.10219530658591976, 'mean_word_length': 4.378561736770692, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '142331', 'license': 'Apache-2.0', 'n_tokens_mistral': 1645, 'n_tokens_neox': 1402, 'n_words': 319, 'path': 'examples/tf/ReLU Layers.ipynb', 'repo_name': 'spenrich/cvxpylayers', 'size': '3963'}