<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Setup\n",
    "\n",
    "# img_directory = '/Users/rwilliams/Desktop/celeba/training'\n",
    "img_directory = '/home/ec2-user/training-data/img_align_celeba'\n",
    "model_save_path = '/home/ec2-user/tf-checkpoints/vaegan-celeba/checkpoint.ckpt'\n",
    "outputs_directory = '/home/ec2-user/outputs/vaegan-celeba'\n",
    "log_directory = '/home/ec2-user/tf-logs/vaegan-celeba'\n",
    "\n",
    "batch_size = 64\n",
    "training_set_size = 512\n",
    "img_size = 64\n",
    "learning_rate = 0.0003\n",
    "zsize = 128\n",
    "\n",
    "# weights similarity loss term for decoder loss\n",
    "# loss_gamma = 1e-2\n",
    "# trying higher gamma\n",
    "loss_gamma = 1e-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Jupyter imports\n",
    "\n",
    "# import matplotlib.pyplot as plt\n",
    "# %matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy as sp\n",
    "import os\n",
    "from utils import imshow, resize_crop, load_img, pixels01, pixels11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/.conda/envs/keras/lib/python3.6/site-packages/skimage/transform/_warps.py:84: UserWarning: The default mode, 'constant', will be changed to 'reflect' in skimage 0.15.\n",
      "  warn(\"The default mode, 'constant', will be changed to 'reflect' in \"\n"
     ]
    }
   ],
   "source": [
    "# load training data\n",
    "training = np.array([resize_crop(load_img(i+1, img_directory), (img_size, img_size)) for i in range(training_set_size)])\n",
    "# rescale each pixel to [-1, 1]. Supposed to help with GANs\n",
    "training = pixels11(training)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# create models\n",
    "\n",
    "import tensorflow as tf\n",
    "from autoencoder import Autoencoder\n",
    "from discriminator import Discriminator\n",
    "tf.reset_default_graph()\n",
    "tf.set_random_seed(42.0)\n",
    "\n",
    "# input images feed\n",
    "X = tf.placeholder(tf.float32, [None, img_size, img_size, 3])\n",
    "\n",
    "# for feeding random draws of z (latent variable)\n",
    "Z = tf.placeholder(tf.float32, [None, zsize])\n",
    "\n",
    "# flags to pass to networks to set batch normalization layers\n",
    "# as trainable or not\n",
    "encoder_batch_trainable = tf.placeholder(tf.bool)\n",
    "decoder_batch_trainable = tf.placeholder(tf.bool)\n",
    "disc_batch_trainable = tf.placeholder(tf.bool)\n",
    "\n",
    "# encoder, decoder that will be connected to a discriminator\n",
    "vae = Autoencoder(img_shape=(img_size, img_size, 3), zsize=zsize)\n",
    "encoder = vae.encoder(X, encoder_batch_trainable)\n",
    "decoder = vae.decoder(encoder, decoder_batch_trainable)\n",
    "\n",
    "# a second decoder for decoding samplings of z\n",
    "decoder_z_obj = Autoencoder(img_shape=(img_size, img_size, 3), zsize=zsize)\n",
    "decoder_z = decoder_z_obj.decoder(Z, decoder_batch_trainable, reuse=True)\n",
    "\n",
    "# discriminator attached to vae output\n",
    "disc_vae_obj = Discriminator(img_shape=(img_size, img_size, 3))\n",
    "disc_vae_obj.disc(decoder, disc_batch_trainable)\n",
    "disc_vae_logits = disc_vae_obj.logits\n",
    "\n",
    "# discriminator attached to X input\n",
    "# shares weights with other discriminator\n",
    "disc_x_obj = Discriminator(img_shape=(img_size, img_size, 3))\n",
    "disc_x_obj.disc(X, disc_batch_trainable, reuse=True)\n",
    "disc_x_logits = disc_x_obj.logits\n",
    "\n",
    "# discriminator attached to random Zs passed through decoder\n",
    "# shares weights with other discriminator\n",
    "disc_z_obj = Discriminator(img_shape=(img_size, img_size, 3))\n",
    "disc_z_obj.disc(decoder_z, disc_batch_trainable, reuse=True)\n",
    "disc_z_logits = disc_z_obj.logits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loss functions and optimizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set up loss functions and training_ops\n",
    "\n",
    "# latent loss used for training encoder\n",
    "latent_loss = vae.latent_loss()\n",
    "\n",
    "# loss that uses decoder to determine similarity between\n",
    "# actual input images and output images from the vae\n",
    "similarity_xentropy = tf.nn.sigmoid_cross_entropy_with_logits(\n",
    "    labels=disc_x_obj.similarity, \n",
    "    logits=disc_vae_obj.similarity)\n",
    "similarity_loss = tf.reduce_mean(similarity_xentropy)\n",
    "\n",
    "# losses for the discriminator's output. Labels are real: 0, fake: 1.\n",
    "# cross entropy with 1 labels, since training prob that image is fake\n",
    "disc_vae_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(\n",<commit_msg>similarity loss is now mse<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Setup\n",
    "\n",
    "# img_directory = '/Users/rwilliams/Desktop/celeba/training'\n",
    "img_directory = '/home/ec2-user/training-data/img_align_celeba'\n",
    "model_save_path = '/home/ec2-user/tf-checkpoints/vaegan-celeba/checkpoint.ckpt'\n",
    "outputs_directory = '/home/ec2-user/outputs/vaegan-celeba'\n",
    "log_directory = '/home/ec2-user/tf-logs/vaegan-celeba'\n",
    "\n",
    "batch_size = 64\n",
    "training_set_size = 512\n",
    "img_size = 64\n",
    "learning_rate = 0.0003\n",
    "zsize = 128\n",
    "\n",
    "# weights similarity loss term for decoder loss\n",
    "# loss_gamma = 1e-2\n",
    "# trying higher gamma\n",
    "loss_gamma = 1e-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Jupyter imports\n",
    "\n",
    "# import matplotlib.pyplot as plt\n",
    "# %matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy as sp\n",
    "import os\n",
    "from utils import imshow, resize_crop, load_img, pixels01, pixels11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/.conda/envs/keras/lib/python3.6/site-packages/skimage/transform/_warps.py:84: UserWarning: The default mode, 'constant', will be changed to 'reflect' in skimage 0.15.\n",
      "  warn(\"The default mode, 'constant', will be changed to 'reflect' in \"\n"
     ]
    }
   ],
   "source": [
    "# load training data\n",
    "training = np.array([resize_crop(load_img(i+1, img_directory), (img_size, img_size)) for i in range(training_set_size)])\n",
    "# rescale each pixel to [-1, 1]. Supposed to help with GANs\n",
    "training = pixels11(training)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# create models\n",
    "\n",
    "import tensorflow as tf\n",
    "from autoencoder import Autoencoder\n",
    "from discriminator import Discriminator\n",
    "tf.reset_default_graph()\n",
    "tf.set_random_seed(42.0)\n",
    "\n",
    "# input images feed\n",
    "X = tf.placeholder(tf.float32, [None, img_size, img_size, 3])\n",
    "\n",
    "# for feeding random draws of z (latent variable)\n",
    "Z = tf.placeholder(tf.float32, [None, zsize])\n",
    "\n",
    "# flags to pass to networks to set batch normalization layers\n",
    "# as trainable or not\n",
    "encoder_batch_trainable = tf.placeholder(tf.bool)\n",
    "decoder_batch_trainable = tf.placeholder(tf.bool)\n",
    "disc_batch_trainable = tf.placeholder(tf.bool)\n",
    "\n",
    "# encoder, decoder that will be connected to a discriminator\n",
    "vae = Autoencoder(img_shape=(img_size, img_size, 3), zsize=zsize)\n",
    "encoder = vae.encoder(X, encoder_batch_trainable)\n",
    "decoder = vae.decoder(encoder, decoder_batch_trainable)\n",
    "\n",
    "# a second decoder for decoding samplings of z\n",
    "decoder_z_obj = Autoencoder(img_shape=(img_size, img_size, 3), zsize=zsize)\n",
    "decoder_z = decoder_z_obj.decoder(Z, decoder_batch_trainable, reuse=True)\n",
    "\n",
    "# discriminator attached to vae output\n",
    "disc_vae_obj = Discriminator(img_shape=(img_size, img_size, 3))\n",
    "disc_vae_obj.disc(decoder, disc_batch_trainable)\n",
    "disc_vae_logits = disc_vae_obj.logits\n",
    "\n",
    "# discriminator attached to X input\n",
    "# shares weights with other discriminator\n",
    "disc_x_obj = Discriminator(img_shape=(img_size, img_size, 3))\n",
    "disc_x_obj.disc(X, disc_batch_trainable, reuse=True)\n",
    "disc_x_logits = disc_x_obj.logits\n",
    "\n",
    "# discriminator attached to random Zs passed through decoder\n",
    "# shares weights with other discriminator\n",
    "disc_z_obj = Discriminator(img_shape=(img_size, img_size, 3))\n",
    "disc_z_obj.disc(decoder_z, disc_batch_trainable, reuse=True)\n",
    "disc_z_logits = disc_z_obj.logits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loss functions and optimizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# set up loss functions and training_ops\n",
    "\n",
    "# latent loss used for training encoder\n",
    "latent_loss = vae.latent_loss()\n",
    "\n",
    "# loss that uses decoder to determine similarity between\n",
    "# actual input images and output images from the vae\n",
    "similarity_loss = tf.reduce_mean(\n",
    "    tf.square(disc_x_obj.similarity - disc_vae_obj.similarity))\n",
    "\n",
    "# losses for the discriminator's output. Labels are real: 0, fake: 1.\n",
    "# cross entropy with 1 labels, since training prob that image is fake\n",
    "disc_vae_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(\n",
{'fraction_non_alphanumeric': 0.1712372741447308, 'fraction_numerical': 0.011373016600935522, 'mean_word_length': 3.1303030303030304, 'pattern_counts': {'":': 91, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7533037', 'n_tokens_mistral': 4195, 'n_tokens_neox': 4015, 'n_words': 824}