<jupyter_start><jupyter_text>warmUpExercise-----------------<jupyter_code>import numpy as np
from numpy import ones, zeros, dot, newaxis, column_stack, linspace, mean, std, divide
from numpy.linalg import pinv
import matplotlib.pyplot as plt
from sklearn import datasets, linear_model
from sklearn.metrics import r2_score, mean_squared_error as mse

%matplotlib inline
np.identity(5)<jupyter_output><empty_output><jupyter_text>Part 2: Plotting-----------------<jupyter_code>file_path = '/home/disooqi/ml/machine-learning-ex1/ex1/ex1data1.txt'
# Load data from a text file.
data = np.loadtxt('ex1data1.txt', delimiter=',')

# Load data from a text file, with missing values handled as specified
data01 = np.genfromtxt('ex1data1.txt', delimiter=',')

# Construct an array from data in a text or binary file.
data02 = np.fromfile('ex1data1.txt')
# X = data(:, 1); y = data(:, 2);
X1 = data[:,0]
y = data[:, 1]
m = len(y)
plt.scatter(X1,y, color='r', marker='x')
plt.title("Scatter plot of training data")

plt.xlabel("Profit in $10,000s")
plt.ylabel("Population of City in 10,000s")
plt.autoscale(tight=True)
plt.grid()

# plt.xticks(())
# plt.yticks(())
plt.show()<jupyter_output><empty_output><jupyter_text>Part 3: Gradient descent---------------------------<jupyter_code>#% Add a column of ones to x to take into account the intercept term (θ0)
X = np.column_stack((np.ones((m,1)),data[:,0]))

#% initialize fitting parameters
theta = np.zeros((2, 1))
#% Some gradient descent settings
iterations = 1500
alpha = 0.01
# feature scaling and mean normalization
normalized_X = (X[:,1] - np.mean(X, axis=0)[1])/(np.max(X[:,1])-np.min(X[:,1]))
normalized_X = np.column_stack((np.ones((m,1)),normalized_X))
#% compute and display initial cost
def computeCost(X, y, theta):
    m = len(y)
    h = np.dot(X,theta)
    return np.sum((h-y[:,newaxis])**2)/(2*m)
def gradientDescent(X, y, theta, alpha, num_iters):
    '''GRADIENTDESCENT Performs gradient descent to learn theta
    theta = GRADIENTDESENT(X, y, theta, alpha, num_iters) updates theta by 
    taking num_iters gradient steps with learning rate alpha'''
    J_history = zeros((num_iters, 1))
    m = len(y)
    for i in range(num_iters):   # changed from xrange to range
        J_history[i] = computeCost(X, y, theta)
        h = dot(X, theta)
        diff = h-y[:,newaxis]
        # B = np.repeat(diff,2,1)
        # theta = theta - (alpha/m) * (((X * theta) - y)'  *  X)';
        theta = theta - (alpha/m)*dot(diff.T, X).T
           
    return theta, J_history
optimized_theta, J_history = gradientDescent(X, y, theta, alpha, iterations)
predict1 = np.dot([1, 3.5], optimized_theta);
predict2 = np.dot([1, 7], optimized_theta);

print ('For population = 35,000, we predict a profit of ', 10000*predict1[0])
print ('For population = 70,000, we predict a profit of ', 10000*predict2[0])
# plot the linear fit
plt.scatter(X1,y, color='black', marker='x')
# draw vertical line from (70,100) to (70, 250)
plt.plot(X1, dot(X,optimized_theta), 'r-', lw=0.1)
plt.title("Scatter plot of training data")

plt.xlabel("Profit in $10,000s")
plt.ylabel("Population of City in 10,000s")
plt.autoscale(tight=True)
plt.grid()

# plt.xticks(())
# plt.yticks(())
plt.show()
alphas = linspace(0.00001, 0.00009, 10) #
js = list()
for a in alphas.flat:
    optimized_theta, J_history = gradientDescent(X, y, theta, a, iterations)
    js.append(J_history)
print (len(js))
js[0].shape
# learning plot
#plt.scatter(X1,y, color='black', marker='x')
# draw vertical line from (70,100) to (70, 250)
for i in range(len(js)):
    plt.plot(np.arange(-1,1499), js[i], 'r-', lw=1)
plt.title("Scatter plot of training data")

plt.xlabel("No. of Iterations")
plt.ylabel("Value of the Cost function")
plt.autoscale(tight=True)
plt.grid()

# plt.xticks(())
# plt.yticks(())
plt.show()<jupyter_output><empty_output><jupyter_text>Examining the Effect of the Learning Rate using Keras<jupyter_code>import keras.backend as K
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.optimizers import SGD, Adam
alpha_cost_fns = list()
for learning_rate in alphas:
    K.clear_session()

    model = Sequential()
    model.add(Dense(1, input_shape=(1,), activation='linear'))

    model.compile(SGD(lr=learning_rate), loss='mean_squared_error')
                  #optimizer='sgd')

    history = model.fit(X1, y, epochs=100, verbose=0)
    plt.plot(np.arange(-1,99), history.history['loss'], 'r-', lw=1)

plt.title("Scatter plot of training data")

plt.xlabel("No. of epochs")
plt.ylabel("Value of the loss")
plt.autoscale(tight=True)
plt.grid()

# plt.xticks(())
# plt.yticks(())
plt.show()
len(history.history['loss'])
#http://scikit-learn.org/stable/auto_examples/linear_model/plot_ols.html
# Load the diabetes dataset
diabetes = datasets.load_diabetes()
# Use only one feature
diabetes_X = diabetes.data[:, np.newaxis, 2]
# Split the data into training/testing sets
diabetes_X_train = diabetes_X[:-20]
diabetes_X_test = diabetes_X[-20:]

# Split the targets into training/testing sets
diabetes_y_train = diabetes.target[:-20]
diabetes_y_test = diabetes.target[-20:]
diabetes_X_train.shape, diabetes_y_train.shape
# Create linear regression object
regr = linear_model.LinearRegression()

# Train the model using the training sets
regr.fit(X1[:,newaxis], y)

print ('Coefficients: ', regr.coef_)
print("Residual sum of squares: %.2f" % np.mean((regr.predict(X1[:,newaxis]) - y) ** 2))
# Explained variance score: 1 is perfect prediction
print('Variance score: %.2f' % regr.score(X1[:,newaxis], y))
print("The R2 score is {:0.3f}".format(r2_score(y, regr.predict(X1[:,newaxis]))))
print("The Mean squared Error score is {:0.3f}".format(mse(y, regr.predict(X1[:,newaxis]))))
# Plot outputs
plt.scatter(X1[:,newaxis], y,  color='black')
plt.plot(X1[:,newaxis], regr.predict(X1[:,newaxis]), color='blue',
         linewidth=3)

plt.xticks(())
plt.yticks(())

plt.show()
# plot the linear fit
plt.scatter(X1,y, color='black', marker='x')
# draw vertical line from (70,100) to (70, 250)
plt.plot(X1, dot(X,optimized_theta), 'r-', lw=0.1)
plt.plot(X1[:,newaxis], regr.predict(X1[:,newaxis]), color='blue',
         linewidth=1)
plt.title("Scatter plot of training data")

plt.xlabel("Profit in $10,000s")
plt.ylabel("Population of City in 10,000s")
plt.autoscale(tight=True)
plt.grid()

# plt.xticks(())
# plt.yticks(())
plt.show()<jupyter_output><empty_output><jupyter_text>------Optional Exercises=========Linear regression with multiple variables<jupyter_code># Load data from a text file.
data = np.loadtxt('ex1data2.txt', delimiter=',')

# Load data from a text file, with missing values handled as specified
data01 = np.genfromtxt('ex1data2.txt', delimiter=',')

# Construct an array from data in a text or binary file.
data02 = np.fromfile('ex1data2.txt')
X = data[:,:2]
y = data[:, 2]
m = len(y)
print 'First 10 examples from the dataset:'
print X[:10,:]
print y[:10][:,newaxis]<jupyter_output>First 10 examples from the dataset:
[[  2.10400000e+03   3.00000000e+00]
 [  1.60000000e+03   3.00000000e+00]
 [  2.40000000e+03   3.00000000e+00]
 [  1.41600000e+03   2.00000000e+00]
 [  3.00000000e+03   4.00000000e+00]
 [  1.98500000e+03   4.00000000e+00]
 [  1.53400000e+03   3.00000000e+00]
 [  1.42700000e+03   3.00000000e+00]
 [  1.38000000e+03   3.00000000e+00]
 [  1.49400000e+03   3.00000000e+00]]
[[ 399900.]
 [ 329900.]
 [ 369000.]
 [ 232000.]
 [ 539900.]
 [ 299900.]
 [ 314900.]
 [ 198999.]
 [ 212000.]
 [ 242500.]]<jupyter_text>Normalizing Features--------------------<jupyter_code>def featureNormalize(X):
    if X.ndim == 2:
        col_count = X.shape[1]
    mu = zeros((1, col_count))
    sigma = zeros((1, col_count))
    
    mu = mean(X, axis=0)
    sigma = std(X, axis=0)
    
    return divide(X-mu, sigma), mu, sigma
normalized_X, X_mean, X_sigma = featureNormalize(X)
X = column_stack((ones((m, 1)), normalized_X))
alpha = 0.01
num_iters = 400
theta = zeros((3,1))
optimized_theta, J_history = gradientDescent(X, y, theta, alpha, num_iters)
# learning plot
#plt.scatter(X1,y, color='black', marker='x')
# draw vertical line from (70,100) to (70, 250)
#for i in range(10):
plt.plot(np.arange(-1,num_iters-1), J_history, 'g-', lw=3)
plt.title("Scatter plot of training data")

plt.xlabel("No. of Iterations")
plt.ylabel("Value of the Cost function")
#plt.autoscale(tight=True)
plt.grid()

# plt.xticks(())
# plt.yticks(())
plt.show()
print 'Predicted price of a 1650 sq-ft, 3 br house (using gradient descent):'
x = np.array([1650, 3]).reshape((1,2))
norm_x = divide(x-X_mean, X_sigma)

print dot(column_stack(([1], norm_x)), optimized_theta)[0,0]<jupyter_output>Predicted price of a 1650 sq-ft, 3 br house (using gradient descent):
289221.547371<jupyter_text>Part 3: Normal Equations==<jupyter_code>def normalEqn(X, y):
    '''return: optimized theta'''
    return dot(dot(pinv(dot(X.T, X)), X.T), y)
otheta = normalEqn(X, y)[:,newaxis]
print 'Theta computed from the normal equations: ', otheta
print 'Predicted price of a 1650 sq-ft, 3 br house (using Normal Equations):', \
dot(column_stack(([1], norm_x)),otheta)[0,0]
# Create linear regression object
regr = linear_model.LinearRegression()

# Train the model using the training sets
regr.fit(X[:,1:],y)

regr.predict(norm_x)<jupyter_output><empty_output><jupyter_text>http://scikit-learn.org/stable/auto_examples/linear_model/plot_polynomial_interpolation.html  Linear Regression with Keras<jupyter_code>from keras.models import Sequential     # adding elements into this model in a sequence
from keras.layers import Dense
from keras.optimizers import Adam, SGD
model = Sequential()
model.add(Dense(1, input_shape=(1,)))
model.summary()
model.compile(Adam(lr=0.8), 'mean_squared_error')
model.fit(X1, y, epochs=40)

y_pred = model.predict(X1)

ax = plt.scatter(X1, y)
plt.plot(X1, y_pred, color='blue',
         linewidth=1)
plt.xlabel("Time (min)")
plt.ylabel("Buy")
plt.autoscale(tight=True)
plt.grid()

W, B = model.get_weights()
print 'W = ', W
print 'B = ', B<jupyter_output>W =  [[ 1.28640068]]
B =  [-3.35283661]
{'fraction_non_alphanumeric': 0.16948814977096197, 'fraction_numerical': 0.06094403505277833, 'mean_word_length': 4.7421383647798745, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 13, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '212124', 'license': 'MIT', 'n_tokens_mistral': 4217, 'n_tokens_neox': 3754, 'n_words': 911, 'path': 'ml/ml_ass01.ipynb', 'repo_name': 'disooqi/jupyter-notebooks', 'size': '10042'}