<jupyter_start><jupyter_text>Copyright 2020 Google LLC.Licensed under the Apache License, Version 2.0 (the "License");<jupyter_code># Download necessary libraries.
%%bash 
test -d rigl || git clone https://github.com/google-research/rigl rigl_repo && mv rigl_repo/rigl ./ 
test -d gresearch || git clone https://github.com/google-research/google-research google_research<jupyter_output><empty_output><jupyter_text>Parameter and FLOPs Counting for MobileNetv1<jupyter_code>import numpy as np
import tensorflow as tf
from google_research.micronet_challenge import counting
from rigl import sparse_utils
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
tf.compat.v1.reset_default_graph()
model=tf.keras.applications.MobileNet(input_shape=(224,224,3), weights=None)
model(tf.ones((2,224,224,3)))
masked_layers = []
dw_layers = []
for layer in model.layers:
  if isinstance(layer, (tf.keras.layers.Conv2D, tf.keras.layers.Dense, tf.keras.layers.DepthwiseConv2D)):     
    masked_layers.append(layer)
    if 'conv_dw' in layer.name:
      dw_layers.append(layer)
    # print(layer.name, sparse_utils._get_kernel(layer).shape)
PARAM_SIZE=32
import functools

get_stats = functools.partial(
    sparse_utils.get_stats, first_layer_name='conv1',
    last_layer_name='conv_preds', param_size=PARAM_SIZE)

def print_stats(masked_layers, default_sparsity=0.8, method='erdos_renyi',
                custom_sparsities=None, is_debug=False, width=1.):
  print('Method: %s, Sparsity: %f' % (method, default_sparsity))
  total_flops, total_param_bits, sparsity = get_stats(
      masked_layers, default_sparsity=default_sparsity, method=method,
      custom_sparsities=custom_sparsities, is_debug=is_debug, width=width)
  print('Total Flops: %.3f MFlops' % (total_flops/1e6))
  print('Total Size: %.3f Mbytes' % (total_param_bits/8e6))
  print('Real Sparsity: %.3f' % (sparsity))<jupyter_output><empty_output><jupyter_text>Printing sparse network stats<jupyter_code>c_sparsities = {'%s/depthwise_kernel:0' % l.name: 0. for l in dw_layers}
c_sparsities_uniform = c_sparsities.copy()
c_sparsities_uniform['conv1/kernel:0'] = 0.
# c_sparsities_uniform['conv_preds/kernel:0'] = 0.
# First layer has sparsity 0 by default.
print_stats(masked_layers, 0.75, 'erdos_renyi_kernel', c_sparsities, is_debug=False)
print_stats(masked_layers, 0.75, 'random', c_sparsities_uniform, is_debug=False)
print_stats(masked_layers, 0, 'random', is_debug=False)
print_stats(masked_layers, 0.85, 'erdos_renyi_kernel', c_sparsities, is_debug=False)
print_stats(masked_layers, 0.85, 'random', c_sparsities_uniform, is_debug=False)
print_stats(masked_layers, 0.9, 'erdos_renyi_kernel', c_sparsities, is_debug=False)
print_stats(masked_layers, 0.9, 'random', c_sparsities_uniform, is_debug=False)
print_stats(masked_layers, 0.95, 'erdos_renyi_kernel', c_sparsities, is_debug=False)
print_stats(masked_layers, 0.95, 'random', c_sparsities_uniform, is_debug=False)<jupyter_output>Method: erdos_renyi_kernel, Sparsity: 0.950000
Total Flops: 205.281 MFlops
Total Size: 1.560 Mbytes
Real Sparsity: 0.940
Method: random, Sparsity: 0.950000
Total Flops: 114.563 MFlops
Total Size: 1.563 Mbytes
Real Sparsity: 0.940<jupyter_text>Finding the width Multiplier for small dense model<jupyter_code>_, sparse_bits, _ = get_stats(masked_layers, 0.75, 'erdos_renyi_kernel', {'conv1/kernel:0':0.})
_, bits, _ = get_stats(masked_layers, 0., 'erdos_renyi_kernel', width=0.47)
print(sparse_bits/bits)
print_stats(masked_layers, 0., 'erdos_renyi_kernel', is_debug=False, width=0.47)
print_stats(masked_layers, 0.75, 'erdos_renyi_kernel', {'conv1/kernel:0':0.}, is_debug=False, width=1)
_, sparse_bits, _ = get_stats(masked_layers, 0.85, 'erdos_renyi_kernel', {'conv1/kernel:0':0.})
_, bits, _ = get_stats(masked_layers, 0., 'erdos_renyi_kernel', width=0.353)
print(sparse_bits/bits)
print_stats(masked_layers, 0., 'erdos_renyi_kernel', is_debug=False, width=0.353)
print_stats(masked_layers, 0.85, 'erdos_renyi_kernel', {'conv1/kernel:0':0.}, is_debug=False, width=1)
_, sparse_bits, _ = get_stats(masked_layers, 0.9, 'erdos_renyi_kernel', {'conv1/kernel:0':0.})
_, bits, _ = get_stats(masked_layers, 0., 'erdos_renyi_kernel', width=0.285)
print(sparse_bits/bits)
print_stats(masked_layers, 0., 'erdos_renyi_kernel', is_debug=False, width=0.285)
print_stats(masked_layers, 0.9, 'erdos_renyi_kernel', {'conv1/kernel:0':0.}, is_debug=False, width=1)
_, sparse_bits, _ = get_stats(masked_layers, 0.95, 'erdos_renyi_kernel', {'conv1/kernel:0':0.})
_, bits, _ = get_stats(masked_layers, 0., 'erdos_renyi_kernel', width=0.204)
print(sparse_bits/bits)
print_stats(masked_layers, 0., 'erdos_renyi_kernel', is_debug=False, width=0.204)
print_stats(masked_layers, 0.95, 'erdos_renyi_kernel', {'conv1/kernel:0':0.}, is_debug=False, width=1)<jupyter_output>0.9982463429660301
Method: erdos_renyi_kernel, Sparsity: 0.000000
Total Flops: 56.617 MFlops
Total Size: 1.396 Mbytes
Real Sparsity: 0.000
Method: erdos_renyi_kernel, Sparsity: 0.950000
Total Flops: 180.359 MFlops
Total Size: 1.393 Mbytes
Real Sparsity: 0.950<jupyter_text>Big-Sparse Networks<jupyter_code># BIGGER
_, sparse_bits, _ = get_stats(masked_layers, 0., 'erdos_renyi_kernel')
_, bits, _ = get_stats(masked_layers, 0.75, 'erdos_renyi_kernel', {'conv1/kernel:0':0.}, width=1.98)
print(sparse_bits/bits)
print_stats(masked_layers, 0.75, 'erdos_renyi_kernel', {'conv1/kernel:0':0.}, is_debug=False, width=1.98)
print_stats(masked_layers, 0.75, 'random', {'conv_preds/kernel:0':0.8, 'conv1/kernel:0':0.}, is_debug=False, width=1.98)
print_stats(masked_layers, 0., 'erdos_renyi_kernel', is_debug=False, width=1)
_, sparse_bits, _ = get_stats(masked_layers, 0., 'erdos_renyi_kernel')
_, bits, _ = get_stats(masked_layers, 0.85, 'erdos_renyi_kernel', {'conv1/kernel:0':0.}, width=2.52)
print(sparse_bits/bits)
print_stats(masked_layers, 0.85, 'erdos_renyi_kernel', {'conv1/kernel:0':0.}, is_debug=False, width=2.52)
print_stats(masked_layers, 0., 'erdos_renyi_kernel', is_debug=False, width=1)
_, sparse_bits, _ = get_stats(masked_layers, 0., 'erdos_renyi_kernel')
_, bits, _ = get_stats(masked_layers, 0.9, 'erdos_renyi_kernel', {'conv1/kernel:0':0.}, width=3.)
print(sparse_bits/bits)
print_stats(masked_layers, 0.9, 'erdos_renyi_kernel', {'conv1/kernel:0':0.}, is_debug=False, width=3.)
print_stats(masked_layers, 0.9, 'random', {'conv_preds/kernel:0':0.8, 'conv1/kernel:0':0.}, is_debug=False, width=3.)
print_stats(masked_layers, 0., 'erdos_renyi_kernel', is_debug=False, width=1)
_, sparse_bits, _ = get_stats(masked_layers, 0., 'erdos_renyi_kernel')
_, bits, _ = get_stats(masked_layers, 0.95, 'erdos_renyi_kernel', {'conv1/kernel:0':0.}, width=3.98)
print(sparse_bits/bits)
print_stats(masked_layers, 0.95, 'erdos_renyi_kernel', {'conv1/kernel:0':0.}, is_debug=False, width=3.98)
print_stats(masked_layers, 0., 'erdos_renyi_kernel', is_debug=False, width=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1465467208357516, 'fraction_numerical': 0.051944283226929774, 'mean_word_length': 7.74746192893401, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '409951', 'license': 'Apache-2.0', 'n_tokens_mistral': 2979, 'n_tokens_neox': 2732, 'n_words': 447, 'path': 'rigl/imagenet_resnet/colabs/MobileNet_Counting.ipynb', 'repo_name': 'xhchrn/rigl', 'size': '6892'}