<jupyter_start><jupyter_text>Plotting Sigmoid Function<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors
import pandas as pd
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score,mean_squared_error
from tqdm import tqdm_notebook

def sigmoid(x,w,b):
  return 1/(1+np.exp(-(w*x+b)))
sigmoid(1,0.5,0)
w= -0.2     #@param {type: "slider", min: -2, max: 2,step: 0.1}
b = -1.5     #@param {type: "slider", min: -2, max: 2,step: 0.1} 
X = np.linspace(-10,10,100)
Y = sigmoid(X,w,b)
plt.plot(X,Y)
plt.show()
type(Y)
plt.plot(X,Y)
plt.show()
def sigmoid_2d(x1,x2,w1,w2,b):
  return 1/(1+np.exp(-(w1*x1+w2*x2+b)))
sigmoid_2d(1,0,0.5,0,0)
from mpl_toolkits import mplot3d
X1 = np.linspace(-10,10,100)
X2 = np.linspace(-10,10,100)                                                                                          

XX1,XX2 = np.meshgrid(X1, X2)

print(X1.shape,X2.shape,XX1.shape,XX2.shape)
type(XX1)
type(XX2)
w1 = 0.5
w2 = 0.5
b = 0

Y = sigmoid_2d(XX1,XX2,w1,w2,b)
#print(Y)
my_cmap = matplotlib.colors.LinearSegmentedColormap.from_list("",["red","yellow","green"])
#https://stackoverflow.com/questions/16834861/create-own-colormap-using-matplotlib-and-plot-color-scale
plt.contourf(XX1,XX2,Y,cmap=my_cmap,alpha=0.6 )  #contourf not contour
plt.show()
fig = plt.figure()
ax = plt.axes(projection='3d')
ax.plot_surface(XX1, XX2, Y, rstride=1, cstride=1, cmap='viridis')
ax.set_xlabel('x1')
ax.set_ylabel('x2')
ax.set_zlabel('y');

ax.view_init(30,180)<jupyter_output><empty_output><jupyter_text>Compute Loss for Given Dataset<jupyter_code>#given input and output, identify different params, then identify the params (optimal)
w_unknown = 0.5
b_unknown = 0.25

X = np.random.random(25)*20 -10  #random generates numbers between -1 and 1 
Y = sigmoid(X,w_unknown,b_unknown)          
plt.scatter(X,Y)
plt.show()
def calculate_loss(X,Y,w_est,b_est):
  loss = 0
  for x,y in zip(X,Y):
    loss +=  (y-sigmoid(x,w_est,b_est))**2
  return loss  
W = np.linspace(0,2,101)
B =  np.linspace(-1,1,101)

WW,BB  =  np.meshgrid(W,B)
Loss = np.zeros(WW.shape)
WW.shape
for i in range(WW.shape[0]):
  for j in range(WW.shape[1]):
    Loss[i,j] = calculate_loss(X,Y,WW[i,j],BB[i,j])

fig = plt.figure()
ax = plt.axes(projection='3d')
ax.plot_surface(WW, BB, Loss,cmap='viridis')
ax.set_xlabel('w')
ax.set_ylabel('b')
ax.set_zlabel('Loss');

ax.view_init(30,270)
ij = np.argmin(Loss)  #coordinate of number which is minimum in list
i = int(np.floor(ij/Loss.shape[1])) 
j = ij - i*Loss.shape[1]  
print(i,j)
print(WW[i,j],BB[i,j])<jupyter_output>0.5 0.26<jupyter_text>Class for Sigmoid neuron<jupyter_code>class SigmoidNeuron:
  def __init__(self):
    self.w =None
    self.b = None
  def perceptron(self,x):
    return np.dot(x,self.w.T)+self.b
  def sigmoid(self,x):
    return 1.0/(1.0+np.exp(-x))

  def grad_w(self,x,y):
    y_pred = self.sigmoid(self.perceptron(x))  
    return (y_pred-y)*y_pred*(1-y_pred)*x
  def grad_b(self,x,y):
    y_pred = self.sigmoid(self.perceptron(x))  
    return (y_pred-y)*y_pred*(1-y_pred)

  def fit(self, X, Y, epochs =1, learning_rate=1, initialise = True, display_loss = False):
    #initialize w,b
    if initialise:
      self.w = np.random.randn(1,X.shape[1])
      self.b=0
    if display_loss:
      loss = []

    for i in tqdm_notebook(range(epochs),total =epochs,unit = "epoch"):
      dw=0
      db=0
      for x,y in zip(X,Y):
        dw += self.grad_w(x,y)
        db += self.grad_b(x,y)

      self.w -= learning_rate *dw 
      self.b -= learning_rate *db    

      if display_loss:
        Y_pred = self.sigmoid(self.perceptron(X))
        loss.append(mean_squared_error(Y_pred,Y))
    
    #print("\nhahahahaha ", type(loss.values()))
    #loss plot 
    
    if display_loss:
      plt.plot(loss) # just loss.values() not working
      plt.xlabel('Epochs')
      plt.ylabel('Mean_Squared_Error')
      plt.show()    
  
  def predict(self,X):
    Y_pred=[]
    for x in X:
      y_pred = self.sigmoid(self.perceptron(x))
      Y_pred.append(y_pred)
    return np.array(Y_pred)<jupyter_output><empty_output><jupyter_text>Fit for ToyData<jupyter_code>X = np.asarray([[2.5,2.5],[4,-1],[1,-4],[-3,1.25],[-2,-4],[1,5]]) 
Y= [1,1,1,0,0,0]
print(X.shape[1])
sn = SigmoidNeuron()
sn.fit(X,Y,1,0.25,True)
print(sn.w,sn.b)
def plot_sn(X,Y,sn,ax): #sn is sn learnt so far
  X1 = np.linspace(-10,10,100)
  X2 = np.linspace(-10,10,100)
  XX1,XX2 = np.meshgrid(X1,X2)
  YY = np.zeros(XX1.shape)

  for i in range(X2.size):
    for j in range(X1.size):
      val = np.asarray([X1[j],X2[i]])
      YY[i,j]=sn.sigmoid(sn.perceptron(val))
  ax.contourf(XX1,XX2,YY,cmap = my_cmap,alpha=0.6)
  ax.scatter(X[:,0],X[:,1],c=Y,cmap=my_cmap)
  ax.plot()    

sn.fit(X,Y,1,0.05,True)
N=20
plt.figure(figsize=(10,N*5))
print(sn.w,sn.b)
for i in range(N):
  print(sn.w,sn.b)
  ax = plt.subplot(N,1,i+1)
  plot_sn(X,Y,sn,ax)  #we would like to separate plots into individual sub-plots. Can use sub-plots. Have to pass handler instead of plt
  sn.fit(X,Y,1,0.5,False)  #because learning rates are larger, changes are coming much faster

  #Exercise: Do an animation using the html...<jupyter_output>[[-0.25427194 -0.76706088]] [0.00288725]
[[-0.25427194 -0.76706088]] [0.00288725]
[[ 0.29501228 -0.78359266]] [0.03572646]
[[ 0.57740457 -0.51036311]] [0.07205715]
[[ 0.86901909 -0.13534951]] [0.04599024]
[[ 0.88398094 -0.43366137]] [-0.02424445]
[[ 1.05093936 -0.2317981 ]] [-0.07764557]
[[ 1.05844573 -0.4496559 ]] [-0.139483]
[[ 1.17594515 -0.31956871]] [-0.18504845]
[[ 1.19508124 -0.44399865]] [-0.23376458]
[[ 1.26850922 -0.39294821]] [-0.27162585]
[[ 1.30074664 -0.44339893]] [-0.30922795]
[[ 1.34774057 -0.43986071]] [-0.34165984]
[[ 1.38346718 -0.45847142]] [-0.37233552]
[[ 1.41952049 -0.46782073]] [-0.40039684]
[[ 1.4523728  -0.47894316]] [-0.426563]
[[ 1.48339619 -0.48881527]] [-0.45093771]
[[ 1.51256764 -0.49821176]] [-0.47374665]
[[ 1.54012156 -0.50707334]] [-0.49515231]
[[ 1.56621619 -0.51546736]] [-0.5152995]
[[ 1.59099255 -0.52343792]] [-0.53431271]<jupyter_text>Load Data<jupyter_code>!ls
data =pd.read_csv('mobile_cleaned.csv')
data.head()
data.shape
X =data.drop('Rating',axis=1)
Y = data['Rating'].values
#Y.head()
threshold = 4.2
data['Class'] = (data['Rating']>=threshold).astype(np.int)
data['Class']  #will contain 1s and 0s instead of true and False
data['Class'].value_counts(normalize =True) #skewed dataset, many more 1s than 0s
Y_binarised =data['Class'].values<jupyter_output><empty_output><jupyter_text>Standardization<jupyter_code>#Each columns has 0 mean and std 1
print(Y)
R = np.random.random([100,1])
#print(R)
plt.plot(R)
plt.show()
np.mean(R)
np.std(R)
#note: its not a direct appln, have to import AN INStance first
scaler =StandardScaler()
scaler.fit(R)
scaler.mean_
RT =scaler.transform(R)
np.mean(RT)
np.std(RT)
plt.plot(RT)
plt.show()
#should split first and then standardise
X_train,X_test,Y_train,Y_test = train_test_split(X,Y,random_state = 0,stratify =Y_binarised )
print(X_train.shape,X_test.shape)
#first, fit scaler on X_train and then use these mean and std values to transform X_test
scaler = StandardScaler()
X_scaled_train = scaler.fit_transform(X_train)
X_scaled_test = scaler.transform(X_test)
minmax_scaler =MinMaxScaler()
Y_scaled_train = minmax_scaler.fit_transform(Y_train.reshape(-1,1))
np.max(Y_scaled_train)
Y_scaled_test = minmax_scaler.transform(Y_test.reshape(-1,1))
np.min(Y_scaled_test)
scaled_threshold = list(minmax_scaler.transform(np.array([threshold]).reshape(-1,1)))[0][0]
scaled_threshold
Y_binarised_train = (Y_scaled_train>scaled_threshold).astype("int").ravel()
Y_binarised_train
Y_binarised_test = (Y_scaled_test>scaled_threshold).astype("int").ravel()<jupyter_output><empty_output><jupyter_text>Train on real data<jupyter_code>sn = SigmoidNeuron()
#Loss Plot

sn.fit(X_scaled_train,Y_scaled_train,epochs =2000 , learning_rate=0.015, display_loss=True)
Y_pred_train = sn.predict(X_scaled_train)
Y_pred_test = sn.predict(X_scaled_test)
Y_pred_binarised_train = (Y_pred_train>scaled_threshold).astype("int").ravel()
Y_pred_binarised_test = (Y_pred_test>scaled_threshold).astype("int").ravel()
accuracy_train = accuracy_score(Y_pred_binarised_train,Y_binarised_train)
accuracy_test = accuracy_score(Y_pred_binarised_test,Y_binarised_test)
print(accuracy_train, accuracy_test)
'''
1) Plot loss plot for different datasets under compute loss section
2) Use different toy POINTS, case when points are NOT linearly separable
3) Try different thresholds
4) Different Hyperparameters
5) Plot Accuracy vs Epochs 
6) Table:  Epochs, Learning Rate, MSE, Accuracy

* What happens when weights are updates after seeing A point instead of all the points(as in an epoch)
This means we will get 6 plots between the 2 consecutive contour plots corresponding to the two epochs
This will give a better intuition on how the decision boundary is learnt wrt each point, that is a red point already 
in the red zone doesnt affect much but the other case affects the decision boundary!  

*Do animation of these contour plots

*In the Loss Vs (W,b)meshgrid plot, indicate the loss value points for each (w,b) set learnt in an epoch.
And using an animation, see how the loss finally plunges into the valley!

My guess: This must be Stochastic gradient Descent 

'''<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15647921760391198, 'fraction_numerical': 0.08801955990220049, 'mean_word_length': 4.603335318642049, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 32, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '44120', 'license': 'MIT', 'n_tokens_mistral': 4396, 'n_tokens_neox': 3942, 'n_words': 774, 'path': 'SigmoidNeuron.ipynb', 'repo_name': 'pp30/PadhAI-Course-Deep-Learning-Fundamentals', 'size': '9407'}