<jupyter_start><jupyter_text>对比一下不同的初始化情况<jupyter_code>import numpy as np
import matplotlib.pyplot as plt

%matplotlib inline

D = np.random.randn(1000, 500)
hidden_layer_sizes = [500]*10
#***************active funtion "tanh"
#nonlinearities = ['tanh']*len(hidden_layer_sizes)
#***************active funtion "tanh"
#nonlinearities = ['relu']*len(hidden_layer_sizes)

act = {'relu':lambda x:np.maximum(0, x), 'tanh':lambda x:np.tanh(x)}
#***************active funtion "tanh"
nonlinearities = ['tanh']*len(hidden_layer_sizes)

Hs = {}
for i in xrange(len(hidden_layer_sizes)):
    X = D if i == 0 else Hs[i-1] #input layer
    fan_in = X.shape[1]
    fan_out = hidden_layer_sizes[i]

    #-------little random number
    W = np.random.randn(fan_in, fan_out) * 0.01
    #-------zero
    #W = 0
    #-------big number 
    #W = np.random.randn(fan_in, fan_out) * 1
    #-------xavier and tanh (xavier and relu)
    #W = np.random.randn(fan_in, fan_out) / np.sqrt(fan_in)
    #-------Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification by He et al., 2015 
    #W = np.random.randn(fan_in, fan_out) / np.sqrt(fan_in/2)

    H = np.dot(X, W)
    H = act[nonlinearities[i]](H)
    Hs[i] = H # cache result on this layer

print 'input layer had mean %f and std %f' % (np.mean(D), np.std(D))
layer_means = [np.mean(H) for i,H in Hs.iteritems()]
layer_stds = [np.std(H) for i,H in Hs.iteritems()]
for i,H in Hs.iteritems():
    print 'hidden layer %d had mean %f and std %f' % (i+1, layer_means[i], layer_stds[i])
    
plt.figure(figsize=(15,5))
plt.subplot(121)
plt.plot(Hs.keys(), layer_means, 'ob-')
plt.title('layer mean')
plt.subplot(122)
plt.plot(Hs.keys(), layer_stds, 'or-')
plt.title('layer std')

#plot the raw distributions
plt.figure(figsize=(15,5))
for i,H in Hs.iteritems():
    plt.subplot(1,len(Hs),i+1)
    plt.hist(H.ravel(), 30, range=(-1,1), histtype='bar')
#***************active funtion "tanh"
nonlinearities = ['tanh']*len(hidden_layer_sizes)

Hs = {}
for i in xrange(len(hidden_layer_sizes)):
    X = D if i == 0 else Hs[i-1] #input layer
    fan_in = X.shape[1]
    fan_out = hidden_layer_sizes[i]

    #-------zero
    W = 0
    #-------big number 
    #W = np.random.randn(fan_in, fan_out) * 1
    #-------xavier and tanh (xavier and relu)
    #W = np.random.randn(fan_in, fan_out) / np.sqrt(fan_in)
    #-------Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification by He et al., 2015 
    #W = np.random.randn(fan_in, fan_out) / np.sqrt(fan_in/2)

    H = np.dot(X, W)
    H = act[nonlinearities[i]](H)
    Hs[i] = H # cache result on this layer

print 'input layer had mean %f and std %f' % (np.mean(D), np.std(D))
layer_means = [np.mean(H) for i,H in Hs.iteritems()]
layer_stds = [np.std(H) for i,H in Hs.iteritems()]
for i,H in Hs.iteritems():
    print 'hidden layer %d had mean %f and std %f' % (i+1, layer_means[i], layer_stds[i])
    
plt.figure(figsize=(15,5))
plt.subplot(121)
plt.plot(Hs.keys(), layer_means, 'ob-')
plt.title('layer mean')
plt.subplot(122)
plt.plot(Hs.keys(), layer_stds, 'or-')
plt.title('layer std')

#plot the raw distributions
plt.figure(figsize=(15,5))
for i,H in Hs.iteritems():
    plt.subplot(1,len(Hs),i+1)
    plt.hist(H.ravel(), 30, range=(-1,1), histtype='bar')
    
#***************active funtion "tanh"
nonlinearities = ['tanh']*len(hidden_layer_sizes)

Hs = {}
for i in xrange(len(hidden_layer_sizes)):
    X = D if i == 0 else Hs[i-1] #input layer
    fan_in = X.shape[1]
    fan_out = hidden_layer_sizes[i]

    #-------big number 
    W = np.random.randn(fan_in, fan_out) * 1
    #-------xavier and tanh (xavier and relu)
    #W = np.random.randn(fan_in, fan_out) / np.sqrt(fan_in)
    #-------Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification by He et al., 2015 
    #W = np.random.randn(fan_in, fan_out) / np.sqrt(fan_in/2)

    H = np.dot(X, W)
    H = act[nonlinearities[i]](H)
    Hs[i] = H # cache result on this layer

print 'input layer had mean %f and std %f' % (np.mean(D), np.std(D))
layer_means = [np.mean(H) for i,H in Hs.iteritems()]
layer_stds = [np.std(H) for i,H in Hs.iteritems()]
for i,H in Hs.iteritems():
    print 'hidden layer %d had mean %f and std %f' % (i+1, layer_means[i], layer_stds[i])
    
plt.figure(figsize=(15,5))
plt.subplot(121)
plt.plot(Hs.keys(), layer_means, 'ob-')
plt.title('layer mean')
plt.subplot(122)
plt.plot(Hs.keys(), layer_stds, 'or-')
plt.title('layer std')

#plot the raw distributions
plt.figure(figsize=(15,5))
for i,H in Hs.iteritems():
    plt.subplot(1,len(Hs),i+1)
    plt.hist(H.ravel(), 30, range=(-1,1), histtype='bar')
#***************active funtion "tanh"
nonlinearities = ['tanh']*len(hidden_layer_sizes)

Hs = {}
for i in xrange(len(hidden_layer_sizes)):
    X = D if i == 0 else Hs[i-1] #input layer
    fan_in = X.shape[1]
    fan_out = hidden_layer_sizes[i]

    #-------xavier and tanh (xavier and relu)
    W = np.random.randn(fan_in, fan_out) / np.sqrt(fan_in)
    #-------Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification by He et al., 2015 
    #W = np.random.randn(fan_in, fan_out) / np.sqrt(fan_in/2)

    H = np.dot(X, W)
    H = act[nonlinearities[i]](H)
    Hs[i] = H # cache result on this layer

print 'input layer had mean %f and std %f' % (np.mean(D), np.std(D))
layer_means = [np.mean(H) for i,H in Hs.iteritems()]
layer_stds = [np.std(H) for i,H in Hs.iteritems()]
for i,H in Hs.iteritems():
    print 'hidden layer %d had mean %f and std %f' % (i+1, layer_means[i], layer_stds[i])
    
plt.figure(figsize=(15,5))
plt.subplot(121)
plt.plot(Hs.keys(), layer_means, 'ob-')
plt.title('layer mean')
plt.subplot(122)
plt.plot(Hs.keys(), layer_stds, 'or-')
plt.title('layer std')

#plot the raw distributions
plt.figure(figsize=(15,5))
for i,H in Hs.iteritems():
    plt.subplot(1,len(Hs),i+1)
    plt.hist(H.ravel(), 30, range=(-1,1), histtype='bar')
#***************active funtion "tanh"
nonlinearities = ['relu']*len(hidden_layer_sizes)

Hs = {}
for i in xrange(len(hidden_layer_sizes)):
    X = D if i == 0 else Hs[i-1] #input layer
    fan_in = X.shape[1]
    fan_out = hidden_layer_sizes[i]

    #-------xavier and tanh (xavier and relu)
    W = np.random.randn(fan_in, fan_out) / np.sqrt(fan_in)
    #-------Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification by He et al., 2015 
    #W = np.random.randn(fan_in, fan_out) / np.sqrt(fan_in/2)

    H = np.dot(X, W)
    H = act[nonlinearities[i]](H)
    Hs[i] = H # cache result on this layer

print 'input layer had mean %f and std %f' % (np.mean(D), np.std(D))
layer_means = [np.mean(H) for i,H in Hs.iteritems()]
layer_stds = [np.std(H) for i,H in Hs.iteritems()]
for i,H in Hs.iteritems():
    print 'hidden layer %d had mean %f and std %f' % (i+1, layer_means[i], layer_stds[i])
    
plt.figure(figsize=(15,5))
plt.subplot(121)
plt.plot(Hs.keys(), layer_means, 'ob-')
plt.title('layer mean')
plt.subplot(122)
plt.plot(Hs.keys(), layer_stds, 'or-')
plt.title('layer std')

#plot the raw distributions
plt.figure(figsize=(15,5))
for i,H in Hs.iteritems():
    plt.subplot(1,len(Hs),i+1)
    plt.hist(H.ravel(), 30, range=(-1,1), histtype='bar')
#***************active funtion "tanh"
nonlinearities = ['tanh']*len(hidden_layer_sizes)

Hs = {}
for i in xrange(len(hidden_layer_sizes)):
    X = D if i == 0 else Hs[i-1] #input layer
    fan_in = X.shape[1]
    fan_out = hidden_layer_sizes[i]

    #-------Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification by He et al., 2015 
    W = np.random.randn(fan_in, fan_out) / np.sqrt(fan_in/2)

    H = np.dot(X, W)
    H = act[nonlinearities[i]](H)
    Hs[i] = H # cache result on this layer

print 'input layer had mean %f and std %f' % (np.mean(D), np.std(D))
layer_means = [np.mean(H) for i,H in Hs.iteritems()]
layer_stds = [np.std(H) for i,H in Hs.iteritems()]
for i,H in Hs.iteritems():
    print 'hidden layer %d had mean %f and std %f' % (i+1, layer_means[i], layer_stds[i])
    
plt.figure(figsize=(15,5))
plt.subplot(121)
plt.plot(Hs.keys(), layer_means, 'ob-')
plt.title('layer mean')
plt.subplot(122)
plt.plot(Hs.keys(), layer_stds, 'or-')
plt.title('layer std')

#plot the raw distributions
plt.figure(figsize=(15,5))
for i,H in Hs.iteritems():
    plt.subplot(1,len(Hs),i+1)
    plt.hist(H.ravel(), 30, range=(-1,1), histtype='bar')
#***************active funtion "tanh"
nonlinearities = ['relu']*len(hidden_layer_sizes)

Hs = {}
for i in xrange(len(hidden_layer_sizes)):
    X = D if i == 0 else Hs[i-1] #input layer
    fan_in = X.shape[1]
    fan_out = hidden_layer_sizes[i]

    #-------Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification by He et al., 2015 
    W = np.random.randn(fan_in, fan_out) / np.sqrt(fan_in/2)

    H = np.dot(X, W)
    H = act[nonlinearities[i]](H)
    Hs[i] = H # cache result on this layer

print 'input layer had mean %f and std %f' % (np.mean(D), np.std(D))
layer_means = [np.mean(H) for i,H in Hs.iteritems()]
layer_stds = [np.std(H) for i,H in Hs.iteritems()]
for i,H in Hs.iteritems():
    print 'hidden layer %d had mean %f and std %f' % (i+1, layer_means[i], layer_stds[i])
    
plt.figure(figsize=(15,5))
plt.subplot(121)
plt.plot(Hs.keys(), layer_means, 'ob-')
plt.title('layer mean')
plt.subplot(122)
plt.plot(Hs.keys(), layer_stds, 'or-')
plt.title('layer std')

#plot the raw distributions
plt.figure(figsize=(15,5))
for i,H in Hs.iteritems():
    plt.subplot(1,len(Hs),i+1)
    plt.hist(H.ravel(), 30, range=(-1,1), histtype='bar')<jupyter_output>input layer had mean -0.000536 and std 1.000232
hidden layer 1 had mean 0.564158 and std 0.824424
hidden layer 2 had mean 0.553223 and std 0.811223
hidden layer 3 had mean 0.548282 and std 0.810047
hidden layer 4 had mean 0.536891 and std 0.801896
hidden layer 5 had mean 0.506204 and std 0.771387
hidden layer 6 had mean 0.512734 and std 0.744881
hidden layer 7 had mean 0.522724 and std 0.760240
hidden layer 8 had mean 0.536649 and std 0.795890
hidden layer 9 had mean 0.591624 and std 0.828165
hidden layer 10 had mean 0.535407 and std 0.823562
{'fraction_non_alphanumeric': 0.18414070944286134, 'fraction_numerical': 0.03684779404539648, 'mean_word_length': 3.8351543942992876, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '52152', 'license': 'Apache-2.0', 'n_tokens_mistral': 4425, 'n_tokens_neox': 4079, 'n_words': 1056, 'path': 'nn_init_weights/.ipynb_checkpoints/weight-checkpoint.ipynb', 'repo_name': 'RayleighChen/practices', 'size': '10177'}