<jupyter_start><jupyter_text>=========== Part 1: Loading and Visualizing Data =============<jupyter_code>dam_data = loadmat('ex5data1.mat')
print dam_data.keys()
features = dam_data['X'].ravel()
y = dam_data['y'].ravel()
m = features.shape[0]
print features.shape, y.shape
plt.scatter(features,y, c='r',marker='x', s=80, linewidth=3.)

plt.xlabel('Change in waterlevel (x)')
plt.ylabel('Water flowing out of the dam (x)')

fig = plt.gcf()
fig.set_size_inches(18.5, 10.5)
#fig.savefig('test2png.png', dpi=100)
plt.autoscale(tight=True)
plt.grid()
plt.show()<jupyter_output><empty_output><jupyter_text>=========== Part 2: Regularized Linear Regression Cost =============<jupyter_code>def linear_reg_cost_function(theta, X, y, _lambda):
    m = y.size
    if theta.ndim != 1:
        return
    theta_tmp = theta[:, np.newaxis]
    hX = X.dot(theta_tmp)
    
    regularization_term = (_lambda/(2*m)) * np.sum((theta_tmp*theta_tmp)[1:])
    return np.sum((hX-y)**2)/(2*m) + regularization_term

theta = np.array([1, 1])
J = linear_reg_cost_function(theta, np.c_[np.ones((m)), features], y[:, np.newaxis], 1)

print 'Cost at theta = [1 ; 1]: %f \n(this value should be about 303.993192)\n', J<jupyter_output>Cost at theta = [1 ; 1]: %f 
(this value should be about 303.993192)
303.99319222<jupyter_text>=========== Part 3: Regularized Linear Regression Gradient =============<jupyter_code>def linear_reg_gradient(theta, X, y, _lambda):
    m = y.size
    if theta.ndim != 1:
        return
    
    theta_tmp = theta[:, np.newaxis]
    hX = X.dot(theta_tmp)
#     print hX.shape, y.shape, theta_tmp.shape
    grad = X.T.dot(hX-y)/m +  (_lambda/m)*np.r_[[[0]],theta_tmp[1:]]
    return grad.ravel()
    
    
    
    
theta = np.array([1,1])
grad = linear_reg_gradient(theta, np.c_[np.ones((m)), features], y[:, np.newaxis], 1)

print 'Gradient at theta = [1 ; 1]:  [%f; %f] \n(this value should be about [-15.303016; 598.250744])\n', grad<jupyter_output>Gradient at theta = [1 ; 1]:  [%f; %f] 
(this value should be about [-15.303016; 598.250744])
[ -15.30301567  598.25074417]<jupyter_text>=========== Part 4: Train Linear Regression =============<jupyter_code>def trainLinearReg(X, y, _lambda):
    initial_theta = np.zeros((X.shape[1]))
    res = minimize(fun=linear_reg_cost_function, x0 =initial_theta, 
               args=(X, y, _lambda), method='CG', 
               jac=linear_reg_gradient, options={'maxiter':200})
    return res.x
features = dam_data['X'].ravel()
param = trainLinearReg(np.c_[np.ones((m)), features], y[:, np.newaxis], 0)
y = dam_data['y'].ravel()
m = features.shape[0]

#  Plot fit over the data
plt.scatter(features, y, c='r',marker='x',s=40, linewidths=1.5) #, ,  
plt.xlabel('Change in water level (x)');
plt.ylabel('Water flowing out of the dam (y)');

plt.plot(features, np.c_[np.ones((m)),features].dot(param[:, np.newaxis]), '--', 
         linewidth= 1, label='curve fitting')

fig = plt.gcf()
fig.set_size_inches(18.5, 10.5)
#fig.savefig('test2png.png', dpi=100)
plt.autoscale(tight=True)
plt.grid()

plt.show()<jupyter_output><empty_output><jupyter_text>=========== Part 5: Learning Curve for Linear Regression =============<jupyter_code>Xtrain = dam_data['X'].ravel()
ytrain = dam_data['y'].ravel()

Xval = dam_data['Xval'].ravel()
yval = dam_data['yval'].ravel()

J_train_values = list()
J_cv_values = list()
for i in np.arange(1, m+1):
    param = trainLinearReg(np.c_[np.ones((i)), features[:i]], y[:i][:, np.newaxis], 0)
    #print param.shape
    J_train = linear_reg_cost_function(param, np.c_[np.ones((m)), Xtrain][:i,:], ytrain[:, np.newaxis][:i,:], 0)
    J_cv = linear_reg_cost_function(param, np.c_[np.ones((yval.size)), Xval], yval[:, np.newaxis], 0)
    
    J_train_values.append(J_train)
    J_cv_values.append(J_cv)
    print param, 3*'-', J_train,3*'-', J_cv

    
plt.plot(np.arange(1, m+1), J_train_values, label='Training')
plt.plot(np.arange(1, m+1), J_cv_values, label='Cross-validation')
plt.legend()


fig = plt.gcf()
fig.set_size_inches(14.5, 10.5)
#fig.savefig('test2png.png', dpi=100)
#plt.autoscale(tight=True)

plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.grid()
plt.axis([0, 13, 0, 150])
plt.show()<jupyter_output>[ 0.00837049 -0.13339852] --- 2.52435489671e-29 --- 205.121095745
[ 3.29319566  0.07271775] --- 1.4559414082e-28 --- 110.300366108
[ 14.15498881   0.53912265] --- 3.2865950455 --- 45.0102318103
[ 14.62429829   0.55985727] --- 2.8426776894 --- 48.3689113698
[ 17.21375092   0.44974972] --- 13.1540488092 --- 35.8651411238
[ 15.40484629   0.45741708] --- 19.4439625125 --- 33.82995682
[ 14.26677698   0.4368173 ] --- 20.0985216551 --- 31.9709867845
[ 14.62343992   0.42312838] --- 18.1728586952 --- 30.8624463238
[ 13.29661011   0.41299409] --- 22.609405425 --- 31.135997955
[ 13.85366299   0.37780979] --- 23.2614615926 --- 28.9362074683
[ 12.93724155   0.36342988] --- 24.317249588 --- 29.551431622
[ 13.08790351   0.36777923] --- 22.3739064951 --- 29.4338181299<jupyter_text>=========== Part 6: Feature Mapping for Polynomial Regression =============<jupyter_code>def poly_features(features, dimension):
    new_features = np.zeros((features.size, dimension))
    for i in np.arange(dimension):
        new_features[:,i] = np.power(features,i+1)
    else:
        return new_features

def features_normalize(features):
    if features.ndim == 2:
        col_count = features.shape[1]
    
    mu = np.mean(features, axis=0)
    features_norm = features-mu
    sigma = np.std(features_norm, axis=0, ddof=1)
    
    return np.divide(features_norm, sigma), mu, sigma

# def feature_map(features, dimension, mu, sigma):
#     features_poly = poly_features(features, p)
#     normalized = np.divide(features_poly-mu, sigma)
#     return np.c_[np.ones((normalized.shape[0])), normalized]<jupyter_output><empty_output><jupyter_text>Feature Mapping (AKA Feature Normalization) via Scikit-learn<jupyter_code>from sklearn.preprocessing import MinMaxScaler, StandardScaler

fts = np.random.randint(1000, size=(4,2)) #  examples X features
print(fts)

mms = MinMaxScaler()
print(mms.fit_transform(fts))

ss = StandardScaler()      # as andrew's mean=0 and std = 1
print(ss.fit_transform(fts))
p = 8
features_poly = poly_features(features, p)
normalized_features_poly, features_mu, features_std = features_normalize(features_poly)
#print features_mu, features_std

X_poly = np.c_[np.ones((normalized_features_poly.shape[0])), normalized_features_poly]
X_poly.shape

# Map X_poly_test and normalize (using mu and sigma)
Xtest = dam_data['Xtest'].ravel()
X_poly_test = poly_features(Xtest, p)
#X_poly_test = bsxfun(@minus, X_poly_test, mu);
#X_poly_test = bsxfun(@rdivide, X_poly_test, sigma);
X_poly_test = np.divide(X_poly_test-features_mu, features_std)
X_poly_test = np.c_[np.ones((X_poly_test.shape[0])), X_poly_test]

# Map X_poly_val and normalize (using mu and sigma)
Xval = dam_data['Xval'].ravel()
X_poly_val = poly_features(Xval, p)
X_poly_val = np.divide(X_poly_val-features_mu, features_std)
X_poly_val = np.c_[np.ones((np.size(X_poly_val, 0))), X_poly_val]<jupyter_output><empty_output><jupyter_text>=========== Part 7: Learning Curve for Polynomial Regression =============<jupyter_code>_lambda = 0
#XX = np.c_[np.ones((m)), features]
param_poly = trainLinearReg(X_poly, y[:, np.newaxis], _lambda)
print param_poly
#print X_poly
param_octave = np.array([ [11.2176],[11.2813],[14.7554],[11.4739],[-12.4539],[-15.3507],
    [7.6451], [3.9019], [-4.9419] ])

d_features = X_poly[:,1].ravel()
y = dam_data['y'].ravel()
m = d_features.shape[0]

#  Plot fit over the data
#plt.scatter(d_features, y, c='r',marker='x',s=40, linewidths=1.5) #, ,  
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')

xp = np.linspace(features.min(), features.max(), 100)
xp_poly = poly_features(xp, p)
xp_poly_norm = np.divide(xp_poly-features_mu, features_std)
xp_poly2 = np.c_[np.ones((np.size(xp_poly, 0))), xp_poly]
print xp_poly2.shape, param_poly.shape
dfd = xp_poly2.dot(param_octave)
dfd2 = xp_poly2.dot(param_poly)
#print dfd
plt.plot(X_poly[:,1], y,'ko',xp, dfd,'-', xp, dfd2,'--')


fig = plt.gcf()
fig.set_size_inches(18.5, 10.5)
#fig.savefig('test2png.png', dpi=100)
#plt.autoscale(tight=True)
plt.grid()
#plt.axis([-60, 50, 0, 50])
plt.axis([-2, 2, 0, 40])
plt.show()
J_train_values = list()
J_cv_values = list()
for i in np.arange(1, m+1):
    param_poly = trainLinearReg(X_poly[:i,:], y[:i][:, np.newaxis], 0)
    # print param_poly.shape
    #param = trainLinearReg(np.c_[np.ones((i)), features[:i]], y[:i][:, np.newaxis], 0)
    
    J_train = linear_reg_cost_function(param_poly, X_poly[:i,:], ytrain[:, np.newaxis][:i,:], 0)
    J_cv = linear_reg_cost_function(param_poly, X_poly_val, yval[:, np.newaxis], 0)
    
    J_train_values.append(J_train)
    J_cv_values.append(J_cv)
    print  3*'-', J_train,3*'-', J_cv

    
plt.plot(np.arange(1, m+1), J_train_values, label='Training')
plt.plot(np.arange(1, m+1), J_cv_values, label='Cross-validation')
plt.legend()


fig = plt.gcf()
fig.set_size_inches(14.5, 10.5)
#fig.savefig('test2png.png', dpi=100)
#plt.autoscale(tight=True)

plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.grid()
plt.axis([0, 13, 0, 150])
plt.show()
x = dam_data['X'].ravel()
y = dam_data['y'].ravel()

#z = np.polyfit(x, y, 2)
#p = np.poly1d(z)
print param_octave
po = np.poly1d(param_octave.ravel()[::-1])
ps = np.poly1d(param_poly.ravel()[::-1])
p = np.poly1d(np.polyfit(X_poly[:,1], y, 8))
print p
print p.coeffs
p30 = np.poly1d(np.polyfit(X_poly[:,1], y, 30))

xp = np.linspace(X_poly[:,1].min(), X_poly[:,1].max(), 100)
_ = plt.plot(X_poly[:,1], y, '.',xp, p(xp), '-', xp, p30(xp), '--', xp, po(xp), xp,ps(xp) )
plt.ylim(y.min()-20, y.max()+20)

#plt.legend()
fig = plt.gcf()
fig.set_size_inches(18.5, 10.5)
plt.grid()
plt.show()

#param_poly.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19677452074246882, 'fraction_numerical': 0.0871285120194746, 'mean_word_length': 4.7459207459207455, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '301362', 'license': 'MIT', 'n_tokens_mistral': 4629, 'n_tokens_neox': 4027, 'n_words': 745, 'path': 'ml/ml_ex5.ipynb', 'repo_name': 'disooqi/jupyter-notebooks', 'size': '9859'}