# Path: rigl/experimental/jax/pruning/init.py
# def init(rng, shape, dtype=dtype):
#   if mask is None:
#     return base_init(rng, shape, dtype)
# 
#   # Find the ablated neurons in the mask, to determine correct fan_out.
#   neuron_weight_count = jnp.sum(
#       jnp.reshape(mask, (-1, mask.shape[-1])), axis=0)
#   non_zero_neurons = jnp.sum(neuron_weight_count != 0)
# 
#   # Special case of completely ablated weight matrix/layer.
#   if jnp.sum(non_zero_neurons) == 0:
#     print('Empty weight mask!')
#     return jnp.zeros(shape, dtype)
# 
#   # Neurons have different fan_in w/mask, build up initialization per-unit.
#   init_cols = []
#   rng, *split_rngs = jax.random.split(rng, mask.shape[-1] + 1)
#   for i in range(mask.shape[-1]):
#     # Special case of ablated neuron.
#     if neuron_weight_count[i] == 0:
#       init_cols.append(jnp.zeros(shape[:-1] + (1,), dtype))
#       continue
# 
#     # Fake shape of weight matrix with correct fan_in, and fan_out.
#     sparse_shape = (int(neuron_weight_count[i]), int(non_zero_neurons))
# 
#     # Use only the first column of init from initializer, since faked fan_out.
#     init = base_init(split_rngs[i], sparse_shape, dtype)[Ellipsis, 0]
# 
#     # Expand out to full sparse array.
#     expanded_init = jnp.zeros(
#         mask[Ellipsis, i].shape,
#         dtype).flatten().at[jnp.where(mask[Ellipsis, i].flatten() == 1)].set(init)
#     expanded_init = jnp.reshape(expanded_init, mask[Ellipsis, i].shape)
#     init_cols.append(expanded_init[Ellipsis, jnp.newaxis])
# 
#   return jnp.concatenate(init_cols, axis=-1)
# 
# Path: rigl/experimental/jax/pruning/masked.py
# def masked(module, mask):
#   """Convenience function for masking a FLAX module with MaskedModule."""
#   return MaskedModule.partial(wrapped_module=module, mask=mask)


from typing import Callable, Mapping, Optional, Sequence
from absl import logging
from rigl.experimental.jax.pruning import init
from rigl.experimental.jax.pruning import masked
import flax
import jax.numpy as jnp
      if f'MaskedModule_{i}' in masks:
        logging.info('Layer %d is masked in model', i)
        mask = masks[f'MaskedModule_{i}']
        inputs = masked.masked(flax.deprecated.nn.Conv, mask)(
            inputs,
            features=filter_num,
            kernel_size=filter_shape,
            kernel_init=init.sparse_init(
                init_fn(), mask['kernel'] if mask is not None else None))
      else:
        inputs = flax.deprecated.nn.Conv(
            inputs,
            features=filter_num,
            kernel_size=filter_shape,
            kernel_init=init_fn())
      inputs = batch_norm(inputs, name='bn_conv_{}'.format(i))
      inputs = activation_fn(inputs)

      if i < len(filters) - 1:
        inputs = flax.deprecated.nn.max_pool(
            inputs, window_shape=(2, 2), strides=(2, 2), padding='VALID')

    # Global average pool at end of convolutional layers.
    inputs = flax.deprecated.nn.avg_pool(
        inputs, window_shape=inputs.shape[1:-1], padding='VALID')

    # This is effectively a Dense layer, but we cast it as a convolution layer
    # to allow us to easily propagate masks, avoiding b/156135283.
    if f'MaskedModule_{depth - 2}' in masks:
      mask_dense_1 = masks[f'MaskedModule_{depth - 2}']

# Path: rigl/experimental/jax/pruning/init.py
# def init(rng, shape, dtype=dtype):
#   if mask is None:
#     return base_init(rng, shape, dtype)
# 
#   # Find the ablated neurons in the mask, to determine correct fan_out.
#   neuron_weight_count = jnp.sum(
#       jnp.reshape(mask, (-1, mask.shape[-1])), axis=0)
#   non_zero_neurons = jnp.sum(neuron_weight_count != 0)
# 
#   # Special case of completely ablated weight matrix/layer.
#   if jnp.sum(non_zero_neurons) == 0:
#     print('Empty weight mask!')
#     return jnp.zeros(shape, dtype)
# 
#   # Neurons have different fan_in w/mask, build up initialization per-unit.
#   init_cols = []
#   rng, *split_rngs = jax.random.split(rng, mask.shape[-1] + 1)
#   for i in range(mask.shape[-1]):
#     # Special case of ablated neuron.
#     if neuron_weight_count[i] == 0:
#       init_cols.append(jnp.zeros(shape[:-1] + (1,), dtype))
#       continue
# 
#     # Fake shape of weight matrix with correct fan_in, and fan_out.
#     sparse_shape = (int(neuron_weight_count[i]), int(non_zero_neurons))
# 
#     # Use only the first column of init from initializer, since faked fan_out.
#     init = base_init(split_rngs[i], sparse_shape, dtype)[Ellipsis, 0]
# 
#     # Expand out to full sparse array.
#     expanded_init = jnp.zeros(
#         mask[Ellipsis, i].shape,
#         dtype).flatten().at[jnp.where(mask[Ellipsis, i].flatten() == 1)].set(init)
#     expanded_init = jnp.reshape(expanded_init, mask[Ellipsis, i].shape)
#     init_cols.append(expanded_init[Ellipsis, jnp.newaxis])
# 
#   return jnp.concatenate(init_cols, axis=-1)
# 
# Path: rigl/experimental/jax/pruning/masked.py
# def masked(module, mask):
#   """Convenience function for masking a FLAX module with MaskedModule."""
#   return MaskedModule.partial(wrapped_module=module, mask=mask)


# Path: rigl/experimental/jax/models/mnist_cnn.py
from typing import Callable, Mapping, Optional, Sequence
from absl import logging
from rigl.experimental.jax.pruning import init
from rigl.experimental.jax.pruning import masked
import flax
import jax.numpy as jnp

      if f'MaskedModule_{i}' in masks:
        logging.info('Layer %d is masked in model', i)
        mask = masks[f'MaskedModule_{i}']
        inputs = masked.masked(flax.deprecated.nn.Conv, mask)(
            inputs,
            features=filter_num,
            kernel_size=filter_shape,
            kernel_init=init.sparse_init(
                init_fn(), mask['kernel'] if mask is not None else None))
      else:
        inputs = flax.deprecated.nn.Conv(
            inputs,
            features=filter_num,
            kernel_size=filter_shape,
            kernel_init=init_fn())
      inputs = batch_norm(inputs, name='bn_conv_{}'.format(i))
      inputs = activation_fn(inputs)

      if i < len(filters) - 1:
        inputs = flax.deprecated.nn.max_pool(
            inputs, window_shape=(2, 2), strides=(2, 2), padding='VALID')

    # Global average pool at end of convolutional layers.
    inputs = flax.deprecated.nn.avg_pool(
        inputs, window_shape=inputs.shape[1:-1], padding='VALID')

    # This is effectively a Dense layer, but we cast it as a convolution layer
    # to allow us to easily propagate masks, avoiding b/156135283.
    if f'MaskedModule_{depth - 2}' in masks:
      mask_dense_1 = masks[f'MaskedModule_{depth - 2}']

      inputs = masked.masked(flax.deprecated.nn.Conv, mask_dense_1)(