<jupyter_start><jupyter_text>Le-Net 1 based architecture We start with 41X41 (I) after first convolution (9x9)we have 33X33 (L1). The next pooling layer reduces dimension with 3 to an output image of 11X11 with 4x4 pooling kernels (L2). Then we apply different types of convolution 4x4 kernels on the L2 layer resulting in 8x8 (L3) . Then followed by pooling 2X2 resulting in 4x4 output map (L4). So we have 16 connection for each element in layer L4 (which depend on the amount of different Covolutions in L3) \begin{equation}f(x)=\frac{1}{1+e^{-x}} \\F_{k}= f( \sum_{i} \mathbf{W^{k}_{i} \cdot y_{i}}-b_{k})\end{equation}\begin{equation}E=\sum_{k} \frac{1}{2}|t_k-F_{k}|^{2} \\\Delta W_{ij}= - \eta \frac{dE}{d W_{ij}}\end{equation}\begin{equation}\Delta W_{ij}= \sum_{k} - \eta \frac{dE}{d F_{k}} \frac{dF_{k}}{dx_{k}} \frac{dx_{k}}{dW_{ij}}=\sum_{k} \eta (t_{k}-F_{k})\frac{e^{-x_{k}}}{(1+e^{-x_{k}})^{2}} \frac{dx_{k}}{dW_{ij}} \\= \eta (t_{k}-F_{k})\frac{e^{-x_{k}}}{(1+e^{-x_{k}})^{2}} y_{ij}\end{equation}\begin{equation}\Delta b_{k}= - \eta \frac{dE}{d F_{k}} \frac{dF_{k}}{dx_{k}} \frac{dx_{k}}{b_{k}}=\eta (t_{k}-F_{k})\frac{e^{-x_{k}}}{(1+e^{-x_{k}})^{2}} \cdot-1\end{equation}Since $\frac{e^{-x_{k}}}{(1+e^{-x_{k}})^{2}}$ is always positive we can neglect this term in our programme\begin{equation}x_{k}=\sum_{ij} W^{k}[i,j] \; y^{4rb}[i,j] - b_{k}\end{equation}\begin{equation}y^{4rb}[i,j]= \sum_{u,v} W^{3rb}[u,v] \; y^{3rb} [2i+u,2j+v]\end{equation}\begin{equation}y^{3rb} [2i+u,2j+v]= f\left (x^{3rb}[2i+u,2j+v] \right)\end{equation}\begin{equation}x^{3rb}[2i+u,2j+v]=\sum_{nm} W^{2rb}[n,m] \; y^{2rb}[n+(2i+u),m+(2j+v)] -b^{3rb}[2i+u,2j+v]\end{equation}\begin{equation}\begin{split}\Delta W^{2rb}[n,m] =\sum_{k} - \eta  \frac{dE}{dF_{k}} \frac{dF_{k}}{dx_{k}} \sum_{ij} \frac{dx_{k}}{dy^{4rb}[i,j]}  \sum_{uv}\frac{dy^{4rb}[i,j]}{d y^{3rb} [2i+u,2j+v]} \frac{d y^{3rb} [2i+u,2j+v]}{d x^{3rb}[2i+u,2j+v]}\sum_{nm}\frac{d x^{3rb}[2i+u,2j+v]}{d W^{2rb}[n,m]}\end{split}\end{equation}\begin{equation}\begin{split}\Delta b^{3rb}[2i+u,2j+v] =\sum_{k} - \eta  \frac{dE}{dF_{k}} \frac{dF_{k}}{dx_{k}} \sum_{ij} \frac{dx_{k}}{dy^{4rb}[i,j]}  \sum_{uv}\frac{dy^{4rb}[i,j]}{d y^{3rb} [2i+u,2j+v]} \frac{d y^{3rb} [2i+u,2j+v]}{d x^{3rb}[2i+u,2j+v]}\frac{d x^{3rb}[2i+u,2j+v]}{d b^{3rb}[2i+u,2j+v]}\end{split}\end{equation}\begin{equation}    \frac{dx_{k}}{dy^{4rb}[i,j]} = W^{4rbk}[i,j]\\\end{equation}\begin{equation}    \frac{dy^{4rb}[i,j]}{d y^{3rb} [2i+u,2j+v]} = W^{3rb}[u,v] \\ \end{equation} \begin{equation}\frac{d y^{3rb} [2i+u,2j+v]}{d x^{3rb}[2i+u,2j+v]}=\frac{e^{-x^{3rb}[2i+u,2j+v]}}{(1+e^{-x^{3rb}[2i+u,2j+v]})^2}\end{equation}This term is first not included since it is always positive. If the training will not converge it might be possible to include this term \begin{equation}    \frac{d y^{3rb} [2i+u,2j+v]}{d W^{2rb}[n,m]}= y^{2rb} [n+(2i+u),m+(2j+v)] \\\end{equation}\begin{equation}\frac{d x^{3rb}[2i+u,2j+v]}{d b^{3rb}[2i+u,2j+v]}=-1\end{equation}<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
from numpy import linalg as lin
import scipy.signal as sig
from PIL import Image
import glob
import matplotlib.cm as cm
import itertools
########### Load Input ############################################################################################################################
# In this script I used the brightness to determine structures, instead of one RGB color:
# this is determined by: 0.2126*R + 0.7152*G + 0.0722*B
# Source: https://en.wikipedia.org/wiki/Relative_luminance

patchSize=41 # patchsize this must be 48 since our network can only handle this value

# Open forest
Amount_data= len(glob.glob('Forest/F*'))
dataPatchedF=[]
for k in range (0, Amount_data):
    name="Forest/F%d.png" % (k+1)
    img = Image.open(name)
    data=img.convert('RGB')
    data= np.asarray( data, dtype="int32" )
    data=0.2126*data[:,:,0]+0.7152*data[:,:,1]+0.0722*data[:,:,2]
    Yamount=data.shape[0]/patchSize # Counts how many times the windowsize fits in the picture
    Xamount=data.shape[1]/patchSize # Counts how many times the windowsize fits in the picture
    dataPatchedF.append(np.array([[data[j*patchSize:(j+1)*patchSize,i*patchSize:(i+1)*patchSize] for i in range(0,Xamount)] for j in range(0,Yamount)]))

# Open city
Amount_data= len(glob.glob('City/C*'))
dataPatchedC=[]
for k in range (0, Amount_data):
    name="City/C%d.png" % (k+1)
    img = Image.open(name)
    data=img.convert('RGB')
    data = np.asarray( data, dtype="int32" )
    data=0.2126*data[:,:,0]+0.7152*data[:,:,1]+0.0722*data[:,:,2]
    Yamount=data.shape[0]/patchSize # Counts how many times the windowsize fits in the picture
    Xamount=data.shape[1]/patchSize # Counts how many times the windowsize fits in the picture    
    dataPatchedC.append(np.array([[data[j*patchSize:(j+1)*patchSize,i*patchSize:(i+1)*patchSize] for i in range(0,Xamount)] for j in range(0,Yamount)]))

# Open water
dataPatchedW=[]
Amount_data= len(glob.glob('Water/W*'))
for k in range (0, Amount_data): 
    name="Water/W%d.png" % (k+1)
    img = Image.open(name)
    data=img.convert('RGB')
    data = np.asarray( data, dtype="int32" )
    data=0.2126*data[:,:,0]+0.7152*data[:,:,1]+0.0722*data[:,:,2]
    Yamount=data.shape[0]/patchSize # Counts how many times the windowsize fits in the picture
    Xamount=data.shape[1]/patchSize # Counts how many times the windowsize fits in the picture
    dataPatchedW.append(np.array([[data[j*patchSize:(j+1)*patchSize,i*patchSize:(i+1)*patchSize] for i in range(0,Xamount)] for j in range(0,Yamount)])) 
    
########### Functions ############################################################################################################################

# Define Activitation functions, pooling and convolution functions (the rules)

def Sigmoid(x): 
    return (1/(1+np.exp(-x)))

def TanH(x):
    return (1-np.exp(-x))/(1+np.exp(-x))


def Pool(I,W):
    PoolImg=np.zeros((len(I)/len(W),len(I)/len(W))) # W must fit an integer times into I.
    for i in range(0,len(PoolImg)):
        for j in range(0,len(PoolImg)):
            SelAr=I[i*len(W):(i+1)*len(W),j*len(W):(j+1)*len(W)]
            PoolImg[i,j]=np.inner(SelAr.flatten(),W.flatten()) # Now this is just an inner product since we have vectors
    return PoolImg

# To automatically make Gaussian kernels
def makeGaussian(size, fwhm = 3, center=None):
    x = np.arange(0, size, 1, float)
    y = x[:,np.newaxis]

    if center is None:
        x0 = y0 = size // 2
    else:
        x0 = center[0]
        y0 = center[1]

    return np.exp(-4*np.log(2) * ((x-x0)**2 + (y-y0)**2) / fwhm**2)

# To automatically define pooling nodes
def Pool_node(N):
    s=(N,N)
    a=float(N)*float(N)
    return (1.0/a)*np.ones(s) 


#################### Define pooling layers ###########################################################################
P12=Pool_node(3)*(1.0/100.0) #factor 1000 added to lower values more
P34=Pool_node(2)

#################### Define Convolution layers #######################################################################

######### First C layer #########
C1=[]

## First Kernel

# Inspiration: http://en.wikipedia.org/wiki/Sobel_operator
# http://stackoverflow.com/questions/9567882/sobel-filter-kernel-of-large-size

Kernel=np.array([[4,3,2,1,0,-1,-2,-3,-4],
                 [5,4,3,2,0,-2,-3,-4,-5], 
                 [6,5,4,3,0,-3,-4,-5,-6],
                 [7,6,5,4,0,-4,-5,-6,-7], 
                 [8,7,6,5,0,-5,-6,-7,-8],
                 [7,6,5,4,0,-4,-5,-6,-7],
                 [6,5,4,3,0,-3,-4,-5,-6],
                 [5,4,3,2,0,-2,-3,-4,-5],
                 [4,3,2,1,0,-1,-2,-3,-4]])

C1.append(Kernel)

## Second Kernel
# Kernel=np.matrix.transpose(Kernel)
Kernel= [[np.sin(x) for x in np.linspace(0,15*np.pi,9)] for i in range(0,9)]
C1.append(Kernel)

## Third Kernel
Kernel=makeGaussian(9,5)
Kernel=(1/np.sum(Kernel))*Kernel
C1.append(Kernel)

## Fourth Kernel
Kernel=np.zeros((9,9))
Kernel[4:5,:]=1
Kernel[5:9,:]=2
Kernel=(1/np.sum(Kernel))*Kernel         
C1.append(Kernel)
        
## Fifth kernel
Kernel=np.zeros((9,9))
for i in range(0,9):
    for j in range(0,9):
        if(i==j):Kernel[i,j]=1
        if(i>j): Kernel[i,j]=2
Kernel=(1/np.sum(Kernel))*Kernel         
C1.append(Kernel)

#Sixth Kernel
Kernel=makeGaussian(9,2)
Kernel=(1/np.sum(Kernel))*Kernel
C1.append(Kernel)

# ## Seventh Kernel
# Kernel= np.transpose([[np.sin(x) for x in np.linspace(0,5*np.pi,9)] for i in range(0,9)])+[[np.sin(x) for x in np.linspace(0,5*np.pi,9)] for i in range(0,9)]
# Kernel=(1/np.sum(Kernel))*Kernel    
# C1.append(Kernel)


# ## Eigth Kernel
# Kernel= np.transpose([[np.sin(x) for x in np.linspace(0,5*np.pi,9)] for i in range(0,9)])
# Kernel=(1/np.sum(Kernel))*Kernel    
# C1.append(Kernel)

# ## Ninth Kernel
# Kernel= np.fliplr(C1[4])
# Kernel=(1/np.sum(Kernel))*Kernel    
# C1.append(Kernel)


######### Initialize output weights and biases #########

# Initialisation, since this layer should be trained!
C2=[] # second convolutio weigths
H3_bias=[]
W=[] # end-weight from output to classifier-neurons
# Define the number of branches in one row
N_branches=2
ClassAmount=3 # Forest, City, Water

for i in range (0, len(C1)):
    C2.append(np.random.rand(N_branches,4,4)*(1.0/100.0))
    H3_bias.append(np.random.rand(N_branches,8,8)*(1.0/10.0)) 
    W.append(np.random.rand(N_branches,ClassAmount,4,4)*(1.0/100.0)) # W[r][b,k,:,:]

bias=np.zeros((ClassAmount))# bias on the three classes
N_plts=len(C1)
for i in range(0,N_plts):
    plt.subplot(4,3,i+1)
    plt.imshow(C1[i])    
#################### Start training  #######################################################################

# Determines which patch is chosen when
N_F=len(dataPatchedF)
Patches_F=[]
j=0;
for i in range(0,N_F):
    N_ff=dataPatchedF[i].shape[0]*dataPatchedF[i].shape[1]
    Patches_F.append(np.reshape(dataPatchedF[i], (N_ff, patchSize, patchSize)))
    j=j+N_ff
N_F=j

N_C=len(dataPatchedC)
Patches_C=[]
j=0;
for i in range(0,N_C):
    N_cc=dataPatchedC[i].shape[0]*dataPatchedC[i].shape[1]
    j=j+N_cc
    Patches_C.append(np.reshape(dataPatchedC[i], (N_cc, patchSize, patchSize)))
N_C=j

N_W=len(dataPatchedW)
Patches_W=[]
j=0;
for i in range(0,N_W):
    N_ww=dataPatchedW[i].shape[0]*dataPatchedW[i].shape[1]
    Patches_W.append(np.reshape(dataPatchedW[i], (N_ww, patchSize, patchSize)))
    j=j+N_ww
N_W=j

N_total=N_F+N_C+N_W
Sequence = np.arange(N_total)
Sequence = np.random.permutation(Sequence)

# Make patches tractable
Patches_F = list(itertools.chain.from_iterable(Patches_F))
Patches_F=np.asarray(Patches_F)

Patches_C = list(itertools.chain.from_iterable(Patches_C))
Patches_C=np.asarray(Patches_C)

Patches_W = list(itertools.chain.from_iterable(Patches_W))
Patches_W=np.asarray(Patches_W)

len(Sequence)
from itertools import product
###### Chooses patch and defines label #####
for PP in range(0,len(Sequence)):
       # 1 has to be changed to the number of training data
    SS=Sequence[PP]
    if SS<N_F:
        Class_label=np.array([1,0,0])
        inputPatch=Patches_F[SS]
    elif(SS>=N_F) and (SS<(N_F+N_C)):
        Class_label=np.array([0,1,0])
        inputPatch=Patches_C[SS-N_F]
    else:
        Class_label=np.array([0,0,1])
        inputPatch=Patches_W[SS-N_F-N_C]

        
    ### Layer 1 ###
    H1=[]
    H2=[]
    H3=[[np.zeros((8,8)) for b in range(0,N_branches)] for r in range(0,len(C1))]
    H4=[[np.zeros((8,8)) for b in range(0,N_branches)] for r in range(0,len(C1))]
    x=np.zeros(ClassAmount)
    f=np.zeros(ClassAmount)
    #learning rates
    n_bias=1*10**-3
    n_W=2*10**-6
    n_C2=1*10**-7
    n_H3_bias=1*10**-3

    
 
    for r in range (0, len(C1)):
        H1.append(sig.convolve(inputPatch, C1[r], 'valid'))
        H2.append(Pool(H1[r], P12))
        for b in range(0,N_branches):
            H3[r][b]=Sigmoid(sig.convolve(H2[r], C2[r][b],'valid')-H3_bias[r][b])
            H4[r][b]=Pool(H3[r][b],P34)
            
    for k in range(0,ClassAmount):
        for r in range (0, len(C1)):
            for b in range(0,N_branches):
                x[k]=x[k]+np.inner(H4[r][b].flatten(),W[r][b,k].flatten())
        f[k]=Sigmoid(x[k]-bias[k])
    
    
    ###### Back-propagation #####
    for k in range(0, ClassAmount):
        bias[k]=bias[k]-n_bias*(Class_label[k]-f[k])
        for r in range (0, len(C1)):
            for b in range(0,N_branches):
                W[r][b,k]=W[r][b,k]+n_W*(Class_label[k]-f[k])*H4[r][b]
                for i, j, u, v, n, m in product(range(0,4), range(0,4), range(0,2), range(0,2), range(0,4), range(0,4)):
                    C2[r][b,n,m]=C2[r][b,n,m]+n_C2*(Class_label[k]-f[k])*W[r][b,k,i,j]*H2[r][n+(2*j+u),m+(2*j+v)]
                    H3_bias[r][b,2*i+u,2*j+v]=H3_bias[r][b,2*i+u,2*j+v]-n_H3_bias*(Class_label[k]-f[k])*W[r][b,k,i,j]
    
    print(Class_label,f)
    if (PP%100==0):print(PP)
    

sig.convolve(H2[2], C2[2][0],'valid')-H3_bias[2][0]
x
bias
W
plt.figure(figsize=(10,10))
for i in range(0,len(C1)):
    for j in range(0,N_branches):
        plt.subplot(len(C1),N_branches,(i*N_branches)+j)
        plt.imshow(H4[i][j])   
plt.figure(figsize=(10,10))
for i in range(0,len(C1)):
    for j in range(0,N_branches):
        plt.subplot(len(C1),N_branches,(i*N_branches)+j)
        plt.imshow(C2[i][j])<jupyter_output><empty_output><jupyter_text>save training parameters<jupyter_code>with open("C2.txt", 'w') as f:
    f.write(str(C2))
with open("W.txt", 'w') as f:
    f.write(str(W))
with open("H3_bias.txt", 'w') as f:
    f.write(str(H3_bias))
with open("Bias.txt", 'w') as f:
    f.write(str(bias))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.24812692816218598, 'fraction_numerical': 0.04517408550022036, 'mean_word_length': 4.443822471011595, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '23401', 'license': 'Unlicense', 'n_tokens_mistral': 5866, 'n_tokens_neox': 5689, 'n_words': 986, 'path': 'code/CNN/.ipynb_checkpoints/CNN3-BP-Michiel_2-checkpoint.ipynb', 'repo_name': 'ttphan/The-Cupcakes-3000', 'size': '13614'}