<jupyter_start><jupyter_text>This requires you to write a 2D GAN game. I let you to get into the topic yourself, whitout any explonations from my side. You can watch lecture, seminar, read papers and tutorials (fun, fun, fun).  Homework I want you to implement a simple 2D GAN game. The kind of animation, I want to see is like in [this video](https://www.youtube.com/watch?v=KeJINHjyzOU) at 15:30 or in [here](https://habrahabr.ru/post/275429/) but in 2D. You can google, search code at github, whatever, but the network should be based on Theano. Basically you will need to come up with true distribution $P$, say mixture of gaussians (surprise me), sample some data from it. Visualize it as a heatmap. To visualize $G$ density you can fix $N$ noise vectors $\{z_i\} \quad i=1,\dots, N$ and draw a circle for each $G(z_i)$. It is also funny to visualize discriminator as a vector field (can be done with `plt.arrow`, `plt.quiver plo). Look how it should be in the middle of [this page](http://www.inference.vc/an-alternative-update-rule-for-generative-adversarial-networks/).Please, make sure your code works if 'Run All' is pressed and it draws some animation.Good luck!<jupyter_code>import matplotlib.pyplot as plt
import numpy as np
import lasagne
import theano
import theano.tensor as T
from lasagne.nonlinearities import rectify, sigmoid, linear, tanh, leaky_rectify, sigmoid
from scipy.stats import norm
import seaborn as sns
# generator
G_input = T.matrix('Gx')
G_l1 = lasagne.layers.InputLayer((None, 1), G_input)
G_l2 = lasagne.layers.DenseLayer(G_l1, 10, nonlinearity=rectify)
G_l3 = lasagne.layers.DenseLayer(G_l2, 10, nonlinearity=rectify)
G_l4 = lasagne.layers.DenseLayer(G_l3, 1, nonlinearity=sigmoid)
G = G_l4

G_out = lasagne.layers.get_output(G)
# discriminators
D1_input = T.matrix('D1x')
D1_target = T.matrix('D1y')
D1_l1 = lasagne.layers.InputLayer((None, 1), D1_input)
D1_l2 = lasagne.layers.DenseLayer(D1_l1, 10, nonlinearity=tanh)
D1_l3 = lasagne.layers.DenseLayer(D1_l2, 10, nonlinearity=tanh)
D1_l4 = lasagne.layers.DenseLayer(D1_l3, 1, nonlinearity=sigmoid)
D1 = D1_l4

D2_l1 = lasagne.layers.InputLayer((None, 1), G_out)
D2_l2 = lasagne.layers.DenseLayer(D2_l1, 10, nonlinearity=tanh, W=D1_l2.W, b=D1_l2.b)
D2_l3 = lasagne.layers.DenseLayer(D2_l2, 10, nonlinearity=tanh, W=D1_l3.W, b=D1_l3.b)
D2_l4 = lasagne.layers.DenseLayer(D2_l3, 1, nonlinearity=sigmoid, W=D1_l4.W, b=D1_l4.b)
D2 = D2_l4

D1_out = lasagne.layers.get_output(D1)
D2_out = lasagne.layers.get_output(D2)
# output functions
discriminate = theano.function([D1_input], D1_out)
generate = theano.function([G_input], G_out)

G_obj = (T.log(D2_out)).mean()
D_obj = (T.log(D1_out) + T.log(1 - D2_out)).mean()

# parameters update and training
G_params = lasagne.layers.get_all_params(G, trainable=True)
G_lr = theano.shared(np.array(0.01, dtype=theano.config.floatX))
G_updates = lasagne.updates.nesterov_momentum(1 - G_obj, G_params, learning_rate=G_lr, momentum=0.6)
G_train = theano.function([G_input], G_obj, updates=G_updates)

D_params = lasagne.layers.get_all_params(D1, trainable=True)
D_lr = theano.shared(np.array(0.1, dtype=theano.config.floatX))
D_updates = lasagne.updates.nesterov_momentum(1 - D_obj, D_params, learning_rate=D_lr, momentum=0.6)
D_train = theano.function([G_input, D1_input], D_obj, updates=D_updates)
# generate dataset
mu1 = 0.3
sigma1 = 0.05

mu2 = 0.6
sigma2 = 0.1

M1 = 150
M2 = 150
k = 100

dataset = [np.float32(np.concatenate((np.random.normal(mu1, sigma1, M1),
                                      np.random.normal(mu2, sigma2, M2))))[np.random.permutation(M1 + M2)].reshape((M1 + M2, 1))
           for _ in range(k)]

x_min, x_max = np.min(dataset) - 1., np.max(dataset) + 1.


def sample_noise(sz):
    return (np.float32(np.linspace(x_min, x_max, sz) + np.random.random(sz) * 0.05))[np.random.permutation(sz)].reshape(sz, 1)

    
def plot_state(gen, discr, batch_size=1000, fname=None):
    plt.xlim((x_min, x_max))
    xs = np.linspace(x_min, x_max, batch_size, dtype=np.float32)
    
    ds = discriminate(xs.reshape(batch_size, 1))
    gs = generate(sample_noise(1000))
    
    plt.hist(np.array(dataset).reshape(-1), bins=40, normed=True, label="Prior distribution", alpha=0.7)
    
    plt.plot(xs, ds, label="Discriminator")
    plt.hist(gs, bins=30, normed=True, label="Generator", alpha=0.7)
    
    plt.plot(xs, [0.5] * xs.shape[0], label="0.5")
    plt.legend()
    
    if fname:
        plt.savefig(fname)
        plt.cla()
    else:
        plt.show()
    
plot_state(generate, discriminate)
!rm -rf ani_1d/
!mkdir -p ani_1d/
epochs = 150
histd, histg = np.zeros(epochs), np.zeros(epochs)

plt.ion()

for i in range(epochs):
    plot_state(generate, discriminate, fname="ani_1d/{:04d}.jpg".format(i))
    
    for x in dataset:
        n_size = x.shape[0]
        z = sample_noise(n_size)  # sample m-batch from noise prior
        histd[i] = D_train(z, x)
        
    
    z = sample_noise(n_size)
    histg[i] = G_train(z)
    
    
    if (i + 1) % 50 == 0:
        G_lr *= 0.999
        D_lr *= 0.999
        print(i + 1)
        plot_state(generate, discriminate, fname=None)
plot_state(generate, discriminate)
plt.plot(range(epochs), histd, label='discriminate_loss')
plt.plot(range(epochs), 1 - histg, label='generate_loss')
plt.legend()
plt.show()
%%time

!convert -delay 5 10 -loop 0 ani_1d/*.jpg "1d_animation.gif"
from IPython.display import Image
Image(url='1d_animation.gif')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13386978785662035, 'fraction_numerical': 0.03913679590343819, 'mean_word_length': 4.22848948374761, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '106422', 'license': 'MIT', 'n_tokens_mistral': 2275, 'n_tokens_neox': 2148, 'n_words': 501, 'path': 'deeplearning-shad/hw7/HW_GAN_1D.ipynb', 'repo_name': 'menshikh-iv/hw', 'size': '5468'}