<jupyter_start><jupyter_text>Parameter Management<jupyter_code>from mxnet import init, nd
from mxnet.gluon import nn

net = nn.Sequential()
net.add(nn.Dense(256, activation='relu'))
net.add(nn.Dense(10))
net.initialize() 

x = nd.random.uniform(shape=(2, 20))
net(x)<jupyter_output><empty_output><jupyter_text>Parameter Access<jupyter_code>print(net[0].params)
print(net[1].params)<jupyter_output>dense0_ (
  Parameter dense0_weight (shape=(256, 20), dtype=float32)
  Parameter dense0_bias (shape=(256,), dtype=float32)
)
dense1_ (
  Parameter dense1_weight (shape=(10, 256), dtype=float32)
  Parameter dense1_bias (shape=(10,), dtype=float32)
)<jupyter_text>Access by Names<jupyter_code>print(net[0].params['dense0_weight'])
print(net[0].params['dense0_weight'].data())<jupyter_output>Parameter dense0_weight (shape=(256, 20), dtype=float32)

[[ 0.06700657 -0.00369488  0.0418822  ... -0.05517294 -0.01194733
  -0.00369594]
 [-0.03296221 -0.04391347  0.03839272 ...  0.05636378  0.02545484
  -0.007007  ]
 [-0.0196689   0.01582889 -0.00881553 ...  0.01509629 -0.01908049
  -0.02449339]
 ...
 [ 0.00010955  0.0439323  -0.04911506 ...  0.06975312  0.0449558
  -0.03283203]
 [ 0.04106557  0.05671307 -0.00066976 ...  0.06387014 -0.01292654
   0.00974177]
 [ 0.00297424 -0.0281784  -0.06881659 ... -0.04047417  0.00457048
   0.05696651]]
<NDArray 256x20 @cpu(0)><jupyter_text>Targeted Parameters<jupyter_code>print(net[1].bias)
print(net[1].bias.data())<jupyter_output>Parameter dense1_bias (shape=(10,), dtype=float32)

[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
<NDArray 10 @cpu(0)><jupyter_text>Access Gradient<jupyter_code>net[0].weight.grad()<jupyter_output><empty_output><jupyter_text>All Parameters at Once<jupyter_code>print(net.collect_params())<jupyter_output>sequential0_ (
  Parameter dense0_weight (shape=(256, 20), dtype=float32)
  Parameter dense0_bias (shape=(256,), dtype=float32)
  Parameter dense1_weight (shape=(10, 256), dtype=float32)
  Parameter dense1_bias (shape=(10,), dtype=float32)
)<jupyter_text>Access by Name Again<jupyter_code>net.collect_params()['dense1_bias'].data()<jupyter_output><empty_output><jupyter_text>Access by Name Patterns<jupyter_code>print(net.collect_params('.*weight'))
print(net.collect_params('dense0.*'))<jupyter_output>sequential0_ (
  Parameter dense0_weight (shape=(256, 20), dtype=float32)
  Parameter dense1_weight (shape=(10, 256), dtype=float32)
)
sequential0_ (
  Parameter dense0_weight (shape=(256, 20), dtype=float32)
  Parameter dense0_bias (shape=(256,), dtype=float32)
)<jupyter_text>Parameter Initialization Gaussian random Initialization<jupyter_code># force_reinit ensures that the variables are initialized again
net.initialize(init=init.Normal(sigma=0.01), force_reinit=True)
net[0].weight.data()[0]<jupyter_output><empty_output><jupyter_text>Constant Initialization<jupyter_code>net.initialize(init=init.Constant(1), force_reinit=True)
net[0].weight.data()[0]<jupyter_output><empty_output><jupyter_text>Multiple Initializations<jupyter_code>net[1].initialize(init=init.Constant(42), force_reinit=True)
net[0].weight.initialize(init=init.Xavier(), force_reinit=True)
print(net[1].weight.data()[0,0])
print(net[0].weight.data()[0])<jupyter_output>[42.]
<NDArray 1 @cpu(0)>

[ 0.00512482 -0.06579044 -0.10849719 -0.09586414  0.06394844  0.06029618
 -0.03065033 -0.01086642  0.01929168  0.1003869  -0.09339568 -0.08703034
 -0.10472868 -0.09879824 -0.00352201 -0.11063069 -0.04257748  0.06548801
  0.12987629 -0.13846186]
<NDArray 20 @cpu(0)><jupyter_text>Custom Initialization$$\begin{aligned}    w \sim \begin{cases}        U[5, 10] & \text{ with probability } \frac{1}{4} \\            0    & \text{ with probability } \frac{1}{2} \\        U[-10, -5] & \text{ with probability } \frac{1}{4}    \end{cases}\end{aligned}$$<jupyter_code>class MyInit(init.Initializer):
    def _init_weight(self, name, data):
        print('Init', name, data.shape)
        data[:] = nd.random.uniform(low=-10, high=10, shape=data.shape)
        data *= data.abs() >= 5
net.initialize(MyInit(), force_reinit=True)
net[0].weight.data()[0]<jupyter_output>Init dense0_weight (256, 20)
Init dense1_weight (10, 256)<jupyter_text>Modify Parameters Directly<jupyter_code>net[0].weight.data()[:] += 1
net[0].weight.data()[0,0] = 42
net[0].weight.data()[0]<jupyter_output><empty_output><jupyter_text>Tied Parameters<jupyter_code>net = nn.Sequential()
# we need to give the shared layer a name such that we can reference its parameters
shared = nn.Dense(8, activation='relu')
net.add(nn.Dense(8, activation='relu'),
        shared,
        nn.Dense(8, activation='relu', params=shared.params),
        nn.Dense(10))
net.initialize()

x = nd.random.uniform(shape=(2, 20))
net(x)

# Check whether the parameters are the same
print(net[1].weight.data()[0] == net[2].weight.data()[0])
net[1].weight.data()[0,0] = 100
# And make sure that they're actually the same object rather than just having the same value.
print(net[1].weight.data()[0] == net[2].weight.data()[0])<jupyter_output>[1. 1. 1. 1. 1. 1. 1. 1.]
<NDArray 8 @cpu(0)>

[1. 1. 1. 1. 1. 1. 1. 1.]
<NDArray 8 @cpu(0)>
{'fraction_non_alphanumeric': 0.16862745098039217, 'fraction_numerical': 0.14568627450980393, 'mean_word_length': 5.489821882951654, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '426432', 'license': 'Apache-2.0', 'n_tokens_mistral': 2408, 'n_tokens_neox': 2011, 'n_words': 343, 'path': 'slides/2_21/parameters.ipynb', 'repo_name': 'yang-chenyu104/berkeley-stat-157', 'size': '5100'}