<jupyter_start><jupyter_text>Preprocessing for deep learning<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
plt.style.use('ggplot')
plt.rcParams['axes.facecolor'] ='w'
plt.rcParams['axes.edgecolor'] = '#D6D6D6'
plt.rcParams['axes.linewidth'] = 2<jupyter_output><empty_output><jupyter_text>1. Background  A. Variance and covariance  Example 1.<jupyter_code>A = np.array([[1, 3, 5], [5, 4, 1], [3, 8, 6]])
print(A)
print(np.cov(A, rowvar=False, bias=True))<jupyter_output>[[ 2.66666667  0.66666667 -2.66666667]
 [ 0.66666667  4.66666667  2.33333333]
 [-2.66666667  2.33333333  4.66666667]]<jupyter_text>Finding the covariance matrix with the dot product<jupyter_code>def calculateCovariance(X):
    meanX = np.mean(X, axis = 0)
    lenX = X.shape[0]
    X = X - meanX
    covariance = X.T.dot(X)/lenX
    return covariance
print(calculateCovariance(A))<jupyter_output>[[ 2.66666667  0.66666667 -2.66666667]
 [ 0.66666667  4.66666667  2.33333333]
 [-2.66666667  2.33333333  4.66666667]]<jupyter_text>B. Visualize data and covariance matrices<jupyter_code>def plotDataAndCov(data):
    ACov = np.cov(data, rowvar=False, bias=True)
    print('Covariance matrix:\n', ACov)

    fig, ax = plt.subplots(nrows=1, ncols=2)
    fig.set_size_inches(10, 10)

    ax0 = plt.subplot(2, 2, 1)
    
    # Choosing the colors
    cmap = sns.color_palette("GnBu", 10)
    sns.heatmap(ACov, cmap=cmap, vmin=0)

    ax1 = plt.subplot(2, 2, 2)
    
    # data can include the colors
    if data.shape[1]==3:
        c=data[:,2]
    else:
        c="#0A98BE"
    ax1.scatter(data[:,0], data[:,1], c=c, s=40)
    
    # Remove the top and right axes from the data plot
    ax1.spines['right'].set_visible(False)
    ax1.spines['top'].set_visible(False)<jupyter_output><empty_output><jupyter_text>C. Simulating data  Uncorrelated data<jupyter_code>np.random.seed(1234)
a1 = np.random.normal(2, 1, 300)
a2 = np.random.normal(1, 1, 300)
A = np.array([a1, a2]).T
A.shape
print(A[:10,:])
sns.distplot(A[:,0], color="#53BB04")
sns.distplot(A[:,1], color="#0A98BE")
plt.show()
plt.close()
plotDataAndCov(A)
plt.show()
plt.close()<jupyter_output>Covariance matrix:
 [[ 0.95171641 -0.0447816 ]
 [-0.0447816   0.87959853]]<jupyter_text>Correlated data<jupyter_code>np.random.seed(1234)
b1 =  np.random.normal(3, 1, 300)
b2 = b1 + np.random.normal(7, 1, 300)/2.
B = np.array([b1, b2]).T
plotDataAndCov(B)
plt.show()
plt.close()<jupyter_output>Covariance matrix:
 [[0.95171641 0.92932561]
 [0.92932561 1.12683445]]<jupyter_text>2. Preprocessing  A. Mean normalization<jupyter_code>def center(X):
    newX = X - np.mean(X, axis = 0)
    return newX
BCentered = center(B)

print('Before:\n\n')

plotDataAndCov(B)
plt.show()
plt.close()

print('After:\n\n')

plotDataAndCov(BCentered)
plt.show()
plt.close()<jupyter_output>Before:


Covariance matrix:
 [[0.95171641 0.92932561]
 [0.92932561 1.12683445]]<jupyter_text>B. Standardization<jupyter_code>def standardize(X):
    newX = center(X)/np.std(X, axis = 0)
    return newX
np.random.seed(1234)
c1 =  np.random.normal(3, 1, 300)
c2 = c1 + np.random.normal(7, 5, 300)/2.
C = np.array([c1, c2]).T

plotDataAndCov(C)
plt.xlim(0, 15)
plt.ylim(0, 15)
plt.show()
plt.close()
CStandardized = standardize(C)

plotDataAndCov(CStandardized)
plt.show()
plt.close()<jupyter_output>Covariance matrix:
 [[1.         0.34500274]
 [0.34500274 1.        ]]<jupyter_text>C. Whitening  1. Zero-centering<jupyter_code>CCentered = center(C)

plotDataAndCov(CCentered)
plt.show()
plt.close()<jupyter_output>Covariance matrix:
 [[0.95171641 0.83976242]
 [0.83976242 6.22529922]]<jupyter_text>2. Decorrelate<jupyter_code>def decorrelate(X):
    cov = X.T.dot(X)/float(X.shape[0])
    # Calculate the eigenvalues and eigenvectors of the covariance matrix
    eigVals, eigVecs = np.linalg.eig(cov)
    # Apply the eigenvectors to X
    decorrelated = X.dot(eigVecs)
    return decorrelated
plotDataAndCov(C)
plt.show()
plt.close()

CDecorrelated = decorrelate(CCentered)
plotDataAndCov(CDecorrelated)
plt.xlim(-5,5)
plt.ylim(-5,5)
plt.show()
plt.close()<jupyter_output>Covariance matrix:
 [[0.95171641 0.83976242]
 [0.83976242 6.22529922]]<jupyter_text>3. Rescale the data<jupyter_code>def whiten(X):
    cov = X.T.dot(X)/float(X.shape[0])
    # Calculate the eigenvalues and eigenvectors of the covariance matrix
    eigVals, eigVecs = np.linalg.eig(cov)
    # Apply the eigenvectors to X
    decorrelated = X.dot(eigVecs)
    # Rescale the decorrelated data
    whitened = decorrelated / np.sqrt(eigVals + 1e-5)
    return whitened
CWhitened = whiten(CCentered)

plotDataAndCov(CWhitened)
plt.xlim(-5,5)
plt.ylim(-5,5)
plt.show()
plt.close()<jupyter_output>Covariance matrix:
 [[9.99987823e-01 2.51650552e-17]
 [2.51650552e-17 9.99998427e-01]]<jupyter_text>3. Image whitening<jupyter_code>from keras.datasets import cifar10

(X_train, y_train), (X_test, y_test) = cifar10.load_data()

X_train.shape
X = X_train[:1000]
print(X.shape)
X = X.reshape(X.shape[0], X.shape[1]*X.shape[2]*X.shape[3])
print(X.shape)
def plotImage(X):
    plt.rcParams["axes.grid"] = False
    plt.figure(figsize=(1.5, 1.5))
    plt.imshow(X.reshape(32,32,3))
    plt.show()
    plt.close()
plotImage(X[12, :])
X_norm = X / 255.
print('X.min()', X_norm.min())
print('X.max()', X_norm.max())
X_norm.mean(axis=0).shape
print(X_norm.mean(axis=0))
X_norm = X_norm - X_norm.mean(axis=0)
print(X_norm.mean(axis=0))
cov = np.cov(X_norm, rowvar=True)
cov.shape
U,S,V = np.linalg.svd(cov)
print(U.shape, S.shape)
print(np.diag(S))
print('\nshape:', np.diag(S).shape)
epsilon = 0.1
X_ZCA = U.dot(np.diag(1.0/np.sqrt(S + epsilon))).dot(U.T).dot(X_norm)
plotImage(X[12, :])
plotImage(X_ZCA[12, :])
X_ZCA_rescaled = (X_ZCA - X_ZCA.min()) / (X_ZCA.max() - X_ZCA.min())
print('min:', X_ZCA_rescaled.min())
print('max:', X_ZCA_rescaled.max())
plotImage(X[12, :])
plotImage(X_ZCA_rescaled[12, :])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1688377601082727, 'fraction_numerical': 0.09930637793943495, 'mean_word_length': 4.728682170542636, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 18, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '4050', 'license': 'MIT', 'n_tokens_mistral': 2980, 'n_tokens_neox': 2589, 'n_words': 433, 'path': 'Preprocessing Tutorial.ipynb', 'repo_name': 'GAnagno/Unsupervised', 'size': '5911'}