<jupyter_start><jupyter_text>PyTorch Linear Regression<jupyter_code>model = torch.nn.Sequential(
    torch.nn.Linear(in_features=1, out_features=1, bias=True)
)
print(model)
model[0].weight
## Run the Model
optim = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)

inputs = torch.autograd.Variable(torch.Tensor(x), requires_grad=False)
targets = torch.autograd.Variable(torch.Tensor(y), requires_grad=False)
losses = []
for epoch in range(nb_epochs):
    
    # Run Forward Propogation
    outputs = model(inputs)
    
    # Calculate the Loss
    loss_val = ((targets - outputs) ** 2).mean()/2
    losses.append(loss_val.data[0])
    
    # Zero out the gradient
    optim.zero_grad()
    
    # Do Backward Prop on loss w.r.t. paramaters
    loss_val.backward()
    
    # Update the gradients
    optim.step()
y_est = model(inputs)
fig, ax = plt.subplots(1, 2, figsize=(10, 6))
ax[0].scatter(x, y)
ax[0].plot(x, y_est.data.numpy(), "r-")
ax[1].plot(np.arange(nb_epochs), losses);<jupyter_output><empty_output><jupyter_text>TensorFlow<jupyter_code>import tensorflow as tf
## Define the Graph
g = tf.Graph()

with g.as_default():
    inputs = tf.placeholder(tf.float32, shape=[None, 1], name="inputs")
    targets = tf.placeholder(tf.float32, shape=[None, 1], name="targets")
    
    outputs = tf.layers.dense(inputs, 1)
    loss = tf.reduce_mean((targets - outputs) ** 2)
    train_op = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(loss)
## Run the graph
nb_epochs = 200
with tf.Session(graph=g) as sess:
    sess.run(tf.global_variables_initializer())
    
    losses = []
    for epoch in range(nb_epochs):
        _, loss_val = sess.run([train_op, loss], feed_dict={inputs: x, targets: y})
        losses.append(loss_val)
        
    y_est = sess.run(outputs, feed_dict={inputs: x, targets: y})
fig, ax = plt.subplots(1, 2, figsize=(10, 6))
ax[0].scatter(x, y)
ax[0].plot(x, y_est, "r-")
ax[1].plot(np.arange(nb_epochs), losses);
## Classification Problems
## Logistic Regression
from IPython.display import Image
IPython.display.display_jpeg("https://michelleful.github.io/PyCon2017/images/binary_classification.png")
centers = np.array([[0, 0]] * 100 + [[1, 1]] * 100)
np.random.seed(42)
data = np.random.normal(0, 0.2, (200, 2)) + centers
y = np.array([[0]] * 100 + [[1]] * 100)

plt.scatter(data[:,0], data[:,1], c=labels, cmap=plt.cm.RdYlBu)
plt.colorbar();<jupyter_output><empty_output><jupyter_text>Logistic Regression - Model* Take a weighted sum of the features and add a bias term to get the logit.* Convert the logit to a probability via the logistic-sigmoid function. ** Sigmoid Function **$$f(x) = \frac{e^x}{1+e^x}$$  Logistic Regression - Keras ** Define the Model **<jupyter_code>model = keras.models.Sequential([
    keras.layers.Dense(1, input_shape=[2], name="Logit"),
    keras.layers.Activation("sigmoid", name="Sigmoid")
])

model.summary()<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
Logit (Dense)                (None, 1)                 3         
_________________________________________________________________
Sigmoid (Activation)         (None, 1)                 0         
=================================================================
Total params: 3
Trainable params: 3
Non-trainable params: 0
_________________________________________________________________<jupyter_text>** Run the Model **<jupyter_code>model.compile("adam", loss="binary_crossentropy")
hist = model.fit(data, y, epochs=300, verbose=0)
plt.plot(hist.history["loss"])
for layer in model.layers:
    if layer.name == "Logit":
        (w0, w1), bias = layer.get_weights() # list of numpy arrays
        w0, w1, bias = w0[0], w1[0], bias[0]
        print("w0: {:.2f}, w1: {:.2f}, bias: {:.2f}".format(w0, w1, bias))
bias
w0, w1, bias
pred_labels = model.predict(data)

plt.scatter(data[:,0], data[:,1], c=pred_labels, cmap=plt.cm.RdYlBu,
            vmin=0.0, vmax=1)
xx = np.linspace(-1, 2, 100)

yy = -w0 / w1 * xx - bias / w1
plt.plot(xx, yy, 'k')
plt.axis((-1,2,-1,2))
plt.colorbar();<jupyter_output><empty_output><jupyter_text>TensorFlow<jupyter_code>g = tf.Graph()

with g.as_default():
    x_data = tf.placeholder(tf.float32, [None, 2], name="features")
    y_label = tf.placeholder(tf.float32, [None, 1], name="labels")

    W = tf.Variable(tf.truncated_normal([2, 1]), name="weights")
    b = tf.Variable(tf.zeros([1]), name="biases")
    logits = tf.matmul(x_data, W) + b

    loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, 
                                                                  labels=y_label))
    train_op = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(loss)

    predicted = tf.cast(tf.nn.sigmoid(logits) > 0.5, np.float32)
    accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, y_label), np.float32))
nb_epochs = 200
with tf.Session(graph=g) as sess:
    sess.run(tf.global_variables_initializer())
    
    for epoch in range(nb_epochs):
        sess.run(train_op, feed_dict={x_data: data, y_label: y})

    pred_labels = sess.run(tf.nn.sigmoid(logits), feed_dict={x_data: data, y_label: y})
    ww, bb = sess.run([W, b])
plt.scatter(data[:,0], data[:,1], c=pred_labels, cmap=plt.cm.RdYlBu,
            vmin=0.0, vmax=1.0)
xx = np.linspace(-1, 2, 100)
yy = -ww[0] / ww[1] * xx - bb / ww[1]
plt.plot(xx, yy, 'k')
plt.axis((-1,2,-1,2))
plt.colorbar();
centers = np.array([[0, 0]] * 100 + [[1, 1]] * 100
                   + [[0, 1]] * 100 + [[1, 0]] * 100)
np.random.seed(42)
data = np.random.normal(0, 0.2, (400, 2)) + centers
labels = np.array([[0]] * 200 + [[1]] * 200)

plt.scatter(data[:,0], data[:,1], c=labels, cmap=plt.cm.RdYlBu)
plt.colorbar();
## Simple function of X
X = Variable(torch.rand(10, 1))
X
y = Variable(5 * X.data + torch.rand(10))
y
plt.scatter(X.data.numpy(), y.data.numpy())
plt.xlabel("X")
plt.ylabel("Y")
plt.title("Y is a linear function of X");
torch.mm(X, w)
w = Variable(torch.ones(1, 1), requires_grad=True)

def plot_torch_linear_xy(X, y, y_est=None):
    plt.scatter(X.data.numpy(), y.data.numpy())
    if y_est is not None:
        plt.plot(X.data.numpy(), y_est.data.numpy(), 'r--')
    # y_est = w
    

y_est = torch.mm(X, w)
plot_torch_linear_xy(X, y, y_est)
w_delta
torch.z
# dy/dw = val of x

lr = 0.01

y_est = torch.mm(X, w)

loss = ((y - y_est) ** 2).mean()

# loss.grad.zero()
# Backpropogate to compute gradients w.r.t. loss
loss.backward(retain_variables=True)

# Update the parameters
w = w - lr * w.grad
print(w.grad)

plot_torch_linear_xy(X, y, y_est)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18486518695069268, 'fraction_numerical': 0.03158051541784597, 'mean_word_length': 3.882909090909091, 'pattern_counts': {'":': 1, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '435434', 'license': 'MIT', 'n_tokens_mistral': 2669, 'n_tokens_neox': 2520, 'n_words': 514, 'path': 'feed-forward/Logistic-Regression-NN.ipynb', 'repo_name': 'sampathweb/compare-pytorch-keras-tf', 'size': '6713'}