<jupyter_start><jupyter_text>Code for plotting the keras model graphUncomment -> run -> comment out -> restart runtime<jupyter_code># %%capture
# !pip install pydot
# !apt-get install graphviz
# !sed -i 's/def _check_pydot()/def _check__pydot()/g' /usr/local/lib/python3.6/dist-packages/keras/utils/vis_utils.py
# !sed -i 's/_check_pydot()/#_check_pydot()/g' /usr/local/lib/python3.6/dist-packages/keras/utils/vis_utils.py<jupyter_output><empty_output><jupyter_text>[Cheatsheet](https://drive.google.com/open?id=1lNyhk2vvWh-YRApwfPBBQS2NBjQBTWRo)  **Disclaimer**:These functions were developed for teaching reasons.The main idea is to keep the code as simple and clean as possible for someone to get the structure of each model easily.For more configurable implementations of these models please refer to other available github repositories.  Imports<jupyter_code>%%capture
import keras
import keras.backend as K
from keras.models import Model
from keras.layers import Input, Dense, Conv2D, MaxPool2D, BatchNormalization, AvgPool2D, DepthwiseConv2D
from keras.layers import Flatten, concatenate, Dropout, ReLU, GlobalAvgPool2D, add, Lambda

from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot

from time import time
import numpy as np<jupyter_output><empty_output><jupyter_text>AlexNet<jupyter_code>def alexnet(input_shape, n_classes):
  input = Input(input_shape)
  
  # actually batch normalization didn't exist back then
  # they used LRN (Local Response Normalization) for regularization
  x = Conv2D(96, 11, strides=4, padding='same', activation='relu')(input)
  x = BatchNormalization()(x)
  x = MaxPool2D(3, strides=2)(x)
  
  x = Conv2D(256, 5, padding='same', activation='relu')(x)
  x = BatchNormalization()(x)
  x = MaxPool2D(3, strides=2)(x)
  
  x = Conv2D(384, 3, strides=1, padding='same', activation='relu')(x)
  
  x = Conv2D(384, 3, strides=1, padding='same', activation='relu')(x)
  
  x = Conv2D(256, 3, strides=1, padding='same', activation='relu')(x)
  x = BatchNormalization()(x)
  x = MaxPool2D(3, strides=2)(x)
  
  x = Flatten()(x)
  x = Dense(4096, activation='relu')(x)
  x = Dense(4096, activation='relu')(x)
  
  output = Dense(n_classes, activation='softmax')(x)
  
  model = Model(input, output)
  return model<jupyter_output><empty_output><jupyter_text>VGG<jupyter_code>def vgg(input_shape, n_classes):
  input = Input(input_shape)
  
  x = Conv2D(64, 3, padding='same', activation='relu')(input)
  x = Conv2D(64, 3, padding='same', activation='relu')(x)
  x = MaxPool2D(2, strides=2)(x)
  
  x = Conv2D(128, 3, padding='same', activation='relu')(x)
  x = Conv2D(128, 3, padding='same', activation='relu')(x)
  x = MaxPool2D(2, strides=2)(x)
  
  x = Conv2D(256, 3, padding='same', activation='relu')(x)
  x = Conv2D(256, 3, padding='same', activation='relu')(x)
  x = Conv2D(256, 3, padding='same', activation='relu')(x)
  x = MaxPool2D(2, strides=2)(x)
  
  x = Conv2D(512, 3, padding='same', activation='relu')(x)
  x = Conv2D(512, 3, padding='same', activation='relu')(x)
  x = Conv2D(512, 3, padding='same', activation='relu')(x)
  x = MaxPool2D(2, strides=2)(x)
  
  x = Conv2D(512, 3, padding='same', activation='relu')(x)
  x = Conv2D(512, 3, padding='same', activation='relu')(x)
  x = Conv2D(512, 3, padding='same', activation='relu')(x)
  x = MaxPool2D(2, strides=2)(x)
  
  x = Flatten()(x)
  x = Dense(4096, activation='relu')(x)
  x = Dense(4096, activation='relu')(x)
  output = Dense(n_classes, activation='softmax')(x)
  
  model = Model(input, output)
  return model<jupyter_output><empty_output><jupyter_text>Inception<jupyter_code>def googlenet(input_shape, n_classes):
  
  def incpeption_block(x, f):
    t1 = Conv2D(f[0], 1, activation='relu')(x)
    
    t2 = Conv2D(f[1], 1, activation='relu')(x)
    t2 = Conv2D(f[2], 3, padding='same', activation='relu')(t2)
    
    t3 = Conv2D(f[3], 1, activation='relu')(x)
    t3 = Conv2D(f[4], 5, padding='same', activation='relu')(t3)
    
    t4 = MaxPool2D(3, 1, padding='same')(x)
    t4 = Conv2D(f[5], 1, activation='relu')(t4)
    
    output = concatenate([t1, t2, t3, t4])
    return output
  
  
  input = Input(input_shape)
  
  x = Conv2D(64, 7, strides=2, padding='same', activation='relu')(input)
  x = MaxPool2D(3, strides=2, padding='same')(x)
  
  x = Conv2D(64, 1, activation='relu')(x)
  x = Conv2D(192, 3, padding='same', activation='relu')(x)
  x = MaxPool2D(3, strides=2)(x)
  
  x = incpeption_block(x, [64, 96, 128, 16, 32, 32])
  x = incpeption_block(x, [128, 128, 192, 32, 96, 64])
  x = MaxPool2D(3, strides=2, padding='same')(x)
  
  x = incpeption_block(x, [192, 96, 208, 16, 48, 64])
  x = incpeption_block(x, [160, 112, 224, 24, 64, 64])
  x = incpeption_block(x, [128, 128, 256, 24, 64, 64])
  x = incpeption_block(x, [112, 144, 288, 32, 64, 64])
  x = incpeption_block(x, [256, 160, 32, 32, 128, 128])
  x = MaxPool2D(3, strides=2, padding='same')(x)

  x = incpeption_block(x, [256, 160, 320, 32, 128, 128])
  x = incpeption_block(x, [384, 192, 384, 48, 128, 128])
  
  x = AvgPool2D(7, strides=1)(x)
  x = Dropout(0.4)(x)
  
  x = Flatten()(x)
  output = Dense(n_classes, activation='softmax')(x)
  
  model = Model(input, output)
  return model<jupyter_output><empty_output><jupyter_text>MobileNet<jupyter_code>def mobilenet(input_shape, n_classes):
  
  def conv_bn_rl(x, f, k=1, s=1, p='same'):
    x = Conv2D(f, k, strides=s, padding=p)(x)
    x = BatchNormalization()(x)
    x = ReLU()(x)
    return x
  
  
  def mobilenet_block(x, f, s=1):
    x = DepthwiseConv2D(3, s, padding='same')(x)
    x = BatchNormalization()(x)
    x = ReLU()(x)
    
    x = conv_bn_rl(x, f)
    return x
  
  
  input = Input(input_shape)
  
  x = conv_bn_rl(input, 32, 3, 2)
  x = mobilenet_block(x, 64)
  
  x = mobilenet_block(x, 128, 2)
  x = mobilenet_block(x, 128)
  
  x = mobilenet_block(x, 256, 2)
  x = mobilenet_block(x, 256)
  
  x = mobilenet_block(x, 512, 2)
  for i in range(5):
    x = mobilenet_block(x, 512)
  
  x = mobilenet_block(x, 1024, 2)
  x = mobilenet_block(x, 1024)
  
  x = AvgPool2D(7)(x)
  
  x = Flatten()(x)
  output = Dense(n_classes, activation='softmax')(x)
  
  model = Model(input, output)
  return model<jupyter_output><empty_output><jupyter_text>ShuffleNet<jupyter_code>def shufflenet(input_shape, n_classes, g=8):
  channels = 384, 769, 1536
  repetitions = 3, 7, 3
  
  def ch_shuffle(x, g):
    _, w, h, ch = K.int_shape(x)
    ch_g = ch // g
    
    def shuffle_op(x):
      x = K.reshape(x, [-1, w, h, ch_g, g])
      x = K.permute_dimensions(x, [0, 1, 2, 4, 3])
      x = K.reshape(x, [-1, w, h, ch])
      return x
    
    x = Lambda(shuffle_op)(x)
    return x
    
  
  def gconv(tensor, ch, g):
    _, _, _, in_ch = K.int_shape(tensor)
    ch_g = in_ch // g
    out_ch = ch // g
    group = []
    
    for i in range(g):
#       x = tensor[:, :, :, i*ch_g:(i+1)*ch_g]
      x = Lambda(lambda x: x[:, :, :, i*ch_g: (i+1)*ch_g])(tensor)
      x = Conv2D(out_ch, 1)(x)
      group.append(x)
    
    x = concatenate(group)
    return x
  
  
  def shufflenet_block(tensor, ch, s, g):
    x = gconv(tensor, ch // 4, g)
    x = BatchNormalization()(x)
    x = ReLU()(x)
    x = ch_shuffle(x, g)
    x = DepthwiseConv2D(3, strides=s, padding='same')(x)
    x = BatchNormalization()(x)
    x = gconv(x, ch if s==1 else ch-K.int_shape(tensor)[-1], g)
    x = BatchNormalization()(x)
    
    if s == 1:
      x = add([tensor, x])
    else:
      avg = AvgPool2D(3, strides=2, padding='same')(tensor)
      x = concatenate([avg, x])
    
    output = ReLU()(x)
    return output
  
  
  def stage(x, ch, r, g):
    x = shufflenet_block(x, ch, 2, g)
    
    for i in range(r):
      x = shufflenet_block(x, ch, 1, g)
      
    return x
  
  input = Input(input_shape)
  
  x = Conv2D(24, 3, strides=2, padding='same')(input)
  x = BatchNormalization()(x)
  x = ReLU()(x)
  x = MaxPool2D(3, strides=2, padding='same')(x)
  
  for ch, r in zip(channels, repetitions):
    x = stage(x, ch, r, g)
    
  x = GlobalAvgPool2D()(x)
  output = Dense(n_classes, activation='softmax')(x)
  
  model = Model(input, output)
  return model<jupyter_output><empty_output><jupyter_text>ResNet<jupyter_code>def resnet(input_shape, n_classes):
  
  def conv_bn_rl(x, f, k=1, s=1, p='same'):
    x = Conv2D(f, k, strides=s, padding=p)(x)
    x = BatchNormalization()(x)
    x = ReLU()(x)
    return x
  
  
  def conv_block(tensor, f1, f2, s):
    x = conv_bn_rl(tensor, f1, s=s)
    x = conv_bn_rl(x, f1, 3)
    x = Conv2D(f2, 1)(x)
    x = BatchNormalization()(x)
    
    shortcut = Conv2D(f2, 1, strides=s, padding='same')(tensor)
    shortcut = BatchNormalization()(shortcut)
    
    x = add([shortcut, x])
    output = ReLU()(x)
    
    return output
  
  
  def identity_block(tensor, f1, f2):
    x = conv_bn_rl(tensor, f1)
    x = conv_bn_rl(x, f1, 3)
    x = Conv2D(f2, 1)(x)
    x = BatchNormalization()(x)
    
    x = add([tensor, x])
    output = ReLU()(x)
    
    return output
    
  
  
  def resnet_block(x, f1, f2, r, s=2):
    x = conv_block(x, f1, f2, s)
    
    for _ in range(r-1):
      x = identity_block(x, f1, f2)
    
    return x
  
  
  input = Input(input_shape)
  
  x = conv_bn_rl(input, 64, 7, 2)
  x = MaxPool2D(3, strides=2, padding='same')(x)
  
  x = resnet_block(x, 64, 256, 3, 1)
  x = resnet_block(x, 128, 512, 4)
  x = resnet_block(x, 256, 1024, 6)
  x = resnet_block(x, 512, 2048, 3)
  
  x = AvgPool2D(7, strides=1)(x)
  x = Flatten()(x)
  
  output = Dense(n_classes, activation='softmax')(x)
  model = Model(input, output)
  
  return model<jupyter_output><empty_output><jupyter_text>DenseNet<jupyter_code>def densenet(input_shape, n_classes, k=32):
  repetitions = 6, 12, 48, 32
  
  def bn_rl_conv(x, f, k=1, s=1, p='same'):
    x = BatchNormalization()(x)
    x = ReLU()(x)
    x = Conv2D(f, k, strides=s, padding=p)(x)
    return x
  
  
  def dense_block(tensor, k):
    x = bn_rl_conv(tensor, 4*k)
    x = bn_rl_conv(x, k, 3)
    output = concatenate([tensor, x])
    return output
  
  
  def transition(x):
    x = bn_rl_conv(x, K.int_shape(x)[-1] // 2)
    x = AvgPool2D(2, strides=2, padding='same')(x)
    return x
  
  
  def dense_blocks(x, r, k):
    for _ in range(r):
      x = dense_block(x, k)
    return x
  
  
  input = Input(input_shape)
  
  x = Conv2D(64, 7, strides=2, padding='same')(input)
  x = MaxPool2D(3, strides=2, padding='same')(x)
  
  for r in repetitions:
    d = dense_blocks(x, r, k)
    x = transition(d)
  
  x = AvgPool2D(7, strides=1)(d)
  x = Flatten()(x)
  
  output = Dense(n_classes, activation='softmax')(x)
  model = Model(input, output)
  
  return model<jupyter_output><empty_output><jupyter_text>Print and plot the model<jupyter_code>input_shape = 224, 224, 3
n_classes = 1000

K.clear_session()
model = alexnet(input_shape, n_classes)
model.summary()
SVG(model_to_dot(model).create(prog='dot', format='svg'))<jupyter_output><empty_output><jupyter_text>Calculate inference time<jupyter_code>repetitions = 10
input = np.random.randn(1, *input_shape)

output = model.predict(input)
start = time()
for _ in range(repetitions):
  output = model.predict(input)
  
print((time() - start) / repetitions)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16376462184123583, 'fraction_numerical': 0.04920082150191982, 'mean_word_length': 3.743752647183397, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 48, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '661504', 'license': 'MIT', 'n_tokens_mistral': 5142, 'n_tokens_neox': 4708, 'n_words': 1011, 'path': 'ConvNets.ipynb', 'repo_name': 'zehrahayirci/DL-workshop-series', 'size': '11199'}