title deep dream fchollethttps twitter comfchollet date created 20160113 last modified 20200502 description generating deep dreams with keras accelerator gpu introduction deep dream is an imagefiltering technique which consists of taking an image classification model and running gradient ascent over an input image to try to maximize the activations of specific layers and sometimes specific units in specific layers for this input it produces hallucinationlike visuals it was first introduced by alexander mordvintsev from google in july 2015 process load the original image define a number of processing scales octaves from smallest to largest resize the original image to the smallest scale for every scale starting with the smallest i e current one run gradient ascent upscale image to the next scale reinject the detail that was lost at upscaling time stop when we are back to the original size to obtain the detail lost during upscaling we simply take the original image shrink it down upscale it and compare the result to the resized original image setup these are the names of the layers for which we try to maximize activation as well as their weight in the final loss we try to maximize you can tweak these setting to obtain new visual effects playing with these hyperparameters will also allow you to achieve new effects this is our base image let s set up some image preprocessingdeprocessing utilities util function to open resize and format pictures into appropriate arrays util function to convert a numpy array into a valid image undo inception v3 preprocessing convert to uint8 and clip to the valid range 0 255 compute the deep dream loss first build a feature extraction model to retrieve the activations of our target layers given an input image build an inceptionv3 model loaded with pretrained imagenet weights get the symbolic outputs of each key layer we gave them unique names set up a model that returns the activation values for every target layer as a dict the actual loss computation is very simple initialize the loss we avoid border artifacts by only involving nonborder pixels in the loss set up the gradient ascent loop for one octave compute gradients normalize gradients run the training loop iterating over different octaves display the result you can use the trained model hosted on hugging face hubhttps huggingface cokerasiodeepdream and try the demo on hugging face spaceshttps huggingface cospaceskerasiodeepdream introduction deep dream is an image filtering technique which consists of taking an image classification model and running gradient ascent over an input image to try to maximize the activations of specific layers and sometimes specific units in specific layers for this input it produces hallucination like visuals it was first introduced by alexander mordvintsev from google in july 2015 process load the original image define a number of processing scales octaves from smallest to largest resize the original image to the smallest scale for every scale starting with the smallest i e current one run gradient ascent upscale image to the next scale reinject the detail that was lost at upscaling time stop when we are back to the original size to obtain the detail lost during upscaling we simply take the original image shrink it down upscale it and compare the result to the resized original image setup these are the names of the layers for which we try to maximize activation as well as their weight in the final loss we try to maximize you can tweak these setting to obtain new visual effects playing with these hyperparameters will also allow you to achieve new effects gradient ascent step size number of scales at which to run gradient ascent size ratio between scales number of ascent steps per scale this is our base image let s set up some image preprocessing deprocessing utilities util function to open resize and format pictures into appropriate arrays util function to convert a numpy array into a valid image undo inception v3 preprocessing convert to uint8 and clip to the valid range 0 255 compute the deep dream loss first build a feature extraction model to retrieve the activations of our target layers given an input image build an inceptionv3 model loaded with pre trained imagenet weights get the symbolic outputs of each key layer we gave them unique names set up a model that returns the activation values for every target layer as a dict the actual loss computation is very simple initialize the loss we avoid border artifacts by only involving non border pixels in the loss set up the gradient ascent loop for one octave compute gradients normalize gradients run the training loop iterating over different octaves make a copy display the result you can use the trained model hosted on hugging face hub https huggingface co keras io deep dream and try the demo on hugging face spaces https huggingface co spaces keras io deep dream
import numpy as np
import tensorflow as tf
import keras
from keras.applications import inception_v3

base_image_path = keras.utils.get_file(
    "sky.jpg", "https://i.imgur.com/aGBdQyK.jpg"
)
result_prefix = "sky_dream"






layer_settings = {
    "mixed4": 1.0,
    "mixed5": 1.5,
    "mixed6": 2.0,
    "mixed7": 2.5,
}


step = 0.01  
num_octave = 3  
octave_scale = 1.4  
iterations = 20  
max_loss = 15.0



from IPython.display import Image, display

display(Image(base_image_path))




def preprocess_image(image_path):
    
    
    img = keras.utils.load_img(image_path)
    img = keras.utils.img_to_array(img)
    img = np.expand_dims(img, axis=0)
    img = inception_v3.preprocess_input(img)
    return img


def deprocess_image(x):
    
    x = x.reshape((x.shape[1], x.shape[2], 3))
    
    x /= 2.0
    x += 0.5
    x *= 255.0
    
    x = np.clip(x, 0, 255).astype("uint8")
    return x





model = inception_v3.InceptionV3(weights="imagenet", include_top=False)


outputs_dict = dict(
    [
        (layer.name, layer.output)
        for layer in [model.get_layer(name) for name in layer_settings.keys()]
    ]
)



feature_extractor = keras.Model(inputs=model.inputs, outputs=outputs_dict)




def compute_loss(input_image):
    features = feature_extractor(input_image)
    
    loss = tf.zeros(shape=())
    for name in features.keys():
        coeff = layer_settings[name]
        activation = features[name]
        
        scaling = tf.reduce_prod(tf.cast(tf.shape(activation), "float32"))
        loss += (
            coeff
            * tf.reduce_sum(tf.square(activation[:, 2:-2, 2:-2, :]))
            / scaling
        )
    return loss





@tf.function
def gradient_ascent_step(img, learning_rate):
    with tf.GradientTape() as tape:
        tape.watch(img)
        loss = compute_loss(img)
    
    grads = tape.gradient(loss, img)
    
    grads /= tf.maximum(tf.reduce_mean(tf.abs(grads)), 1e-6)
    img += learning_rate * grads
    return loss, img


def gradient_ascent_loop(img, iterations, learning_rate, max_loss=None):
    for i in range(iterations):
        loss, img = gradient_ascent_step(img, learning_rate)
        if max_loss is not None and loss > max_loss:
            break
        print("... Loss value at step %d: %.2f" % (i, loss))
    return img




original_img = preprocess_image(base_image_path)
original_shape = original_img.shape[1:3]

successive_shapes = [original_shape]
for i in range(1, num_octave):
    shape = tuple([int(dim / (octave_scale**i)) for dim in original_shape])
    successive_shapes.append(shape)
successive_shapes = successive_shapes[::-1]
shrunk_original_img = tf.image.resize(original_img, successive_shapes[0])

img = tf.identity(original_img)  
for i, shape in enumerate(successive_shapes):
    print("Processing octave %d with shape %s" % (i, shape))
    img = tf.image.resize(img, shape)
    img = gradient_ascent_loop(
        img, iterations=iterations, learning_rate=step, max_loss=max_loss
    )
    upscaled_shrunk_original_img = tf.image.resize(shrunk_original_img, shape)
    same_size_original = tf.image.resize(original_img, shape)
    lost_detail = same_size_original - upscaled_shrunk_original_img

    img += lost_detail
    shrunk_original_img = tf.image.resize(original_img, shape)

keras.utils.save_img(result_prefix + ".png", deprocess_image(img.numpy()))



display(Image(result_prefix + ".png"))