<jupyter_start><jupyter_text>2. Get the data<jupyter_code>import pandas as pd
import numpy as np
df = pd.read_csv(open("F:\Sem 3\EE-258\Homework\Advertising.csv"), header=0,index_col=0)
df.info()
print(df)<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 200 entries, 1 to 200
Data columns (total 4 columns):
TV           200 non-null float64
radio        200 non-null float64
newspaper    200 non-null float64
sales        200 non-null float64
dtypes: float64(4)
memory usage: 7.8 KB
        TV  radio  newspaper  sales
1    230.1   37.8       69.2   22.1
2     44.5   39.3       45.1   10.4
3     17.2   45.9       69.3    9.3
4    151.5   41.3       58.5   18.5
5    180.8   10.8       58.4   12.9
6      8.7   48.9       75.0    7.2
7     57.5   32.8       23.5   11.8
8    120.2   19.6       11.6   13.2
9      8.6    2.1        1.0    4.8
10   199.8    2.6       21.2   10.6
11    66.1    5.8       24.2    8.6
12   214.7   24.0        4.0   17.4
13    23.8   35.1       65.9    9.2
14    97.5    7.6        7.2    9.7
15   204.1   32.9       46.0   19.0
16   195.4   47.7       52.9   22.4
17    67.8   36.6      114.0   12.5
18   281.4   39.6       55.8   24.4
19    69.2   20.5       18.3   11.3
[...]<jupyter_text>3. Linear regression using Normal Equations for a single feature:       a. Find the linear regression coefficient using normal equations assuming TV as the feature (input) and sales as the output variable.<jupyter_code>X = df.drop(['newspaper','sales','radio'],axis=1)
y = df.drop(['newspaper','TV','radio'],axis=1)

from sklearn.cross_validation import train_test_split
X_train, X_test, Y_train, Y_test = train_test_split(X, y, test_size=0.2,random_state = 10)
X_train.shape

y1 = Y_train
x1 = X_train
from sklearn.linear_model import LinearRegression

linereg = LinearRegression()

linereg.fit(X_train,Y_train)
print (linereg.intercept_)
print (linereg.coef_)
y_pred = linereg.predict(X_test)
print(y_pred)
import numpy.linalg
X_b = np.c_[np.ones((160, 1)), X_train] # add x0 = 1 to each instance
theta_best = np.linalg.inv(X_b.T.dot(X_b)).dot(X_b.T).dot(Y_train)
theta_best
X_new_b = np.c_[np.ones((40, 1)), X_test] # add x0 = 1 to each instance
y_predict = X_new_b.dot(theta_best)
print(y_predict)
print(Y_test.values)<jupyter_output>[[ 16.84129141]
 [  7.56876758]
 [ 17.194749  ]
 [ 20.18766263]
 [ 17.10294183]
 [ 13.93100422]
 [ 15.8176415 ]
 [ 16.29962912]
 [ 17.12130326]
 [  7.95894804]
 [ 20.46767449]
 [ 10.20363326]
 [  7.94517696]
 [ 17.29114652]
 [ 17.70427877]
 [ 11.88370441]
 [ 18.89318158]
 [  7.41728575]
 [ 18.26430249]
 [ 16.57964098]
 [ 15.79009935]
 [  7.20153891]
 [ 19.16401272]
 [ 12.25093308]
 [ 20.03159044]
 [  8.03239377]
 [  8.34453814]
 [  9.21211586]
 [ 18.18167604]
 [  8.06911664]
 [ 13.21490833]
 [ 17.6354234 ]
 [  8.43175495]
 [  8.48224889]
 [ 20.51357807]
 [ 19.8709279 ]
 [ 10.85087379]
 [ 20.21979514]
 [ 11.28695783]
 [ 17.37836333]]
[[ 18.4]
 [  7.2]
 [ 18. ]
 [ 25.5]
 [ 22.6]
 [ 14.6]
 [ 14.9]
 [ 23.7]
 [ 22.3]
 [  9.3]
 [ 25.4]
 [  8.6]
 [  8.7]
 [ 20.1]
 [ 19.7]
 [ 14. ]
 [ 19.8]
 [  5.3]
 [ 21.8]
 [ 22.6]
 [ 15.6]
 [  1.6]
 [ 24.2]
 [ 16. ]
 [ 14.8]
 [  7. ]
 [  9.5]
 [ 10.4]
 [ 23.2]
 [  7.6]
 [ 12.9]
 [ 21.5]
 [  6.9]
 [  7.3]
 [ 12.8]
 [ 11.8]
 [  8.8]
 [ 15. ]
 [ 10.6]
 [ 11.5][...]<jupyter_text>Have compared the normal equations and using inbuilt library function        b. Display data points (sales vs TV)       c. On the same figure, display the regression line.<jupyter_code>import matplotlib.pyplot as plt
%matplotlib inline
plt.xlabel('TV')
plt.ylabel('sales')
plt.plot(X_test, y_predict, "k-")
plt.plot(X_test, Y_test, "r.")
plt.plot(X_train, Y_train, "b.")
plt.show()
print("red for test data and blue for training data")<jupyter_output><empty_output><jupyter_text>d. Repeat parts a-b for replacing TV with radio and newspaper.            Radio<jupyter_code>radio = df.drop(['TV','sales','radio'],axis=1)
radio_sales = df.drop(['newspaper','TV','radio'],axis=1)

from sklearn.cross_validation import train_test_split
radio1_train, radio1_test, radio2_train, radio2_test = train_test_split(radio,radio_sales,test_size=0.2,random_state = 10)

radio1_train = radio1_train.values
radio2_train = radio2_train.values

radio_b = np.c_[np.ones((160, 1)), radio1_train] # add x0 = 1 to each instance
theta_radio = np.linalg.inv(radio_b.T.dot(radio_b)).dot(radio_b.T).dot(radio2_train)
print(theta_radio)
radio_new_b = np.c_[np.ones((40, 1)), radio2_test] # add x0 = 1 to each instance
radio_predict = radio_new_b.dot(theta_radio)
print(radio_predict)
from sklearn.linear_model import LinearRegression

radio2 = LinearRegression()

radiomodel = radio2.fit(radio1_train,radio2_train)
radio_predict = radio2.predict(radio2_test)
radio_predict[:]
radiomodel.intercept_
import matplotlib.pyplot as plt
%matplotlib inline
plt.xlabel('radio')
plt.ylabel('sales')
plt.plot(radio1_test, radio2_test, "k.")
plt.plot(radio1_test, radio_predict, "b-",linewidth=4)
plt.show()<jupyter_output><empty_output><jupyter_text>Newspaper<jupyter_code>news = df.drop(['TV','sales','newspaper'],axis=1)
news_sales = df.drop(['newspaper','TV','radio'],axis=1)

from sklearn.cross_validation import train_test_split
news1_train, news1_test, news2_train, news2_test = train_test_split(news,news_sales,test_size=0.1,random_state = 5)

news1_train = news1_train.values
news2_train = news2_train.values

news_b = np.c_[np.ones((180, 1)), news1_train] # add x0 = 1 to each instance
theta_news = np.linalg.inv(news_b.T.dot(news_b)).dot(news_b.T).dot(news2_train)
print(theta_news)
news_new_b = np.c_[np.ones((20, 1)), news2_test] # add x0 = 1 to each instance
news_predict = news_new_b.dot(theta_radio)
print(news_predict)
from sklearn.linear_model import LinearRegression

news2 = LinearRegression()

newsmodel = news2.fit(news1_train,news2_train)
news_predict = news2.predict(news2_test)
print(newsmodel.intercept_)
news_predict[:]
import matplotlib.pyplot as plt
%matplotlib inline
plt.xlabel('radio')
plt.ylabel('sales')
plt.plot(news1_test, news2_test, "k.")
plt.plot(news1_test, news_predict, "b-",linewidth=2)
plt.show()<jupyter_output><empty_output><jupyter_text>With radio and newspaper both<jupyter_code>X2 = df.drop(['TV','sales'],axis=1)
y2 = df.drop(['newspaper','TV','radio'],axis=1)

from sklearn.cross_validation import train_test_split
X2_train, X2_test, Y2_train, Y2_test = train_test_split(X2,y2,test_size=0.1,random_state = 10)
X2_train.shape
x2 = X2_train.values
y2 = Y2_train.values
X2_b = np.c_[np.ones((180, 1)),x2] # add x0 = 1 to each instance
theta_best = np.linalg.inv(X2_b.T.dot(X2_b)).dot(X2_b.T).dot(y2)
print(theta_best,theta_best.shape)
X2_new_b = np.c_[np.ones((20, 1)), X2_test] # add x0 = 1 to each instance
y2_predict = X2_new_b.dot(theta_best)
print(y2_predict)
print(Y2_test.values)
from sklearn.linear_model import LinearRegression

linereg2 = LinearRegression()

model = linereg2.fit(X2_train,Y2_train)
predictions = linereg2.predict(X2_test)
predictions[:]
print (linereg2.intercept_)
print (linereg2.coef_)
print(linereg2.score(X2_train,Y2_train))
import matplotlib.pyplot as plt
%matplotlib inline
plt.xlabel('newspaper and radio')
plt.ylabel('sales')
plt.plot(X2_test, y2_predict, "k.")
plt.plot(X2_test, Y2_test, "r.")
plt.show()<jupyter_output><empty_output><jupyter_text>e. Discuss the complexity of the Normal Equations computation. Ans. In normal equations we computed inverse of X transpose dot X where X is the features. Now if the number of features increase then the matrix size will increase, increasing the complexty of implementation. Generally computational complexity of inverting a matrix is typically about O(n2.4) to O(n3). So normal equations get very slow when there are many features. But normal equations are linear so if they fit in the memory then large training sets can be computed.       4. Linear regression using Gradient Descent:       a. Find the linear regression coefficient using gradient descent assuming TV as the feature (input) and sales as the output variable.<jupyter_code>eta = 0.000001 # 1st learning rate
X4_b = np.c_[np.ones((160, 1)), x1] # add x0 = 1 to each instance
m = 160
theta = np.random.randn(2,1) # random initialization
print(y1.shape,X4_b.T.shape,X4_b.shape,theta.shape)
print(theta)
n_iterations = 10
for iteration in range(n_iterations):
    gradients = 2/m * X4_b.T.dot(X4_b.dot(theta) - y1)
    theta = theta - eta * gradients
    
print(theta)

X_new_b = np.c_[np.ones((40, 1)), X_test]
y_predict3 = X_new_b.dot(theta)
y_predict3
from matplotlib import pyplot as plt
%matplotlib inline

plt.xlabel('TV')
plt.ylabel('sales')
plt.plot(X_test,Y_test,'b.')
plt.plot(X_test,y_predict3,'r-')
plt.show()
eta2 = 0.001 # 2nd learning rate
X4_b = np.c_[np.ones((160, 1)), x1] # add x0 = 1 to each instance
m = 160
theta2 = np.random.randn(2,1) # random initialization
print(y1.shape,X4_b.T.shape,X4_b.shape,theta.shape)
n_iterations = 10
for iteration in range(n_iterations):
    gradients2 = 2/m * X4_b.T.dot(X4_b.dot(theta2) - y1)
    theta2 = theta2 - eta2 * gradients2
    
print(theta2)
X_new_b = np.c_[np.ones((40, 1)), X_test]
y_predict4 = X_new_b.dot(theta)
y_predict4
from matplotlib import pyplot as plt
%matplotlib inline

plt.xlabel('TV')
plt.ylabel('sales')
plt.plot(X_test,Y_test,'b.')
plt.plot(X_test,y_predict4,'r-')
plt.show()
eta3 = 0.1 # 3rd learning rate
X6_b = np.c_[np.ones((160, 1)), x1] # add x0 = 1 to each instance
m = 160
theta3 = np.random.randn(2,1) # random initialization
print(y1.shape,X6_b.T.shape,X6_b.shape,theta.shape)
n_iterations = 10
for iteration in range(n_iterations):
    gradients3 = 2/m * X6_b.T.dot(X6_b.dot(theta3) - y1)
    theta3 = theta2 - eta3 * gradients3
    
print(theta3)
X_new_b = np.c_[np.ones((40, 1)), X_test]
y_predict6 = X_new_b.dot(theta)
y_predict6
from matplotlib import pyplot as plt
%matplotlib inline

plt.xlabel('TV')
plt.ylabel('sales')
plt.plot(X_test,Y_test,'b.')
plt.plot(X_test,y_predict4,'r-')
plt.show()<jupyter_output><empty_output><jupyter_text>5. Multiple linear regression:         a. Use all three features (TV, radio, newspaper) and find a linear relation between the sales and all three features.<jupyter_code>X5 = df.drop(['sales'],axis=1)
y5 = df.drop(['newspaper','TV','radio'],axis=1)

from sklearn.cross_validation import train_test_split
X5_train, X5_test, Y5_train, Y5_test = train_test_split(X5, y5, test_size=0.1,random_state = 10)
X5_train.shape
x5 = X5_train.values
y5 = Y5_train.values
X5_b = np.c_[np.ones((180, 1)), x5] # add x0 = 1 to each instance
theta_best = np.linalg.inv(X5_b.T.dot(X5_b)).dot(X5_b.T).dot(y5)
theta_best
X_new_b = np.c_[np.ones((20,1)),X5_test]
y_predict5 = X_new_b.dot(theta_best)
y_predict5
from sklearn import linear_model
clf = linear_model.SGDRegressor(eta0=0.000001,n_iter=10, penalty='none')# normal learning 
clf.fit(X5_train, Y5_train)

clf.predict(X5_test)
from matplotlib import pyplot as plt
%matplotlib inline

plt.xlabel('All 3 features')
plt.ylabel('sales')
plt.plot(X5_test,Y5_test,'b.')
plt.plot(X5_test,y_predict5,'r.')
plt.show()<jupyter_output><empty_output><jupyter_text>b. Compare your results (cost and coefficients) with the results in Problem 3     Ans. The coefficients in problem 3 are very different than in this problem.        c. Interpret your results using the correlation coefficient matrix<jupyter_code>corr_matrix = df.corr()
print (corr_matrix)<jupyter_output>TV     radio  newspaper     sales
TV         1.000000  0.054809   0.056648  0.782224
radio      0.054809  1.000000   0.354104  0.576223
newspaper  0.056648  0.354104   1.000000  0.228299
sales      0.782224  0.576223   0.228299  1.000000<jupyter_text>7. Linear Regression using tensor-flow:        a. Repeat Problem 5 using tensorflow<jupyter_code>import tensorflow as tf

X = tf.constant(X5_b, dtype=tf.float32, name="X")
y = tf.constant(Y5_train.values.reshape(-1, 1), dtype=tf.float32, name="y")
XT = tf.transpose(X)
theta = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(XT, X)), XT), y)

with tf.Session() as sess:
    theta_value = theta.eval()
    
print(theta_value)<jupyter_output>[[ 3.12943363]
 [ 0.04341276]
 [ 0.18859901]
 [ 0.0039553 ]]<jupyter_text>b. Compare the results with Problem 5 Ans. The results are the same as problem 5 i.e. theta values are the same. But in tensorflow we got values much faster.<jupyter_code>from sklearn.preprocessing import StandardScaler
m=200
n=3
Scaler1 = StandardScaler()
scalerdata = Scaler1.fit_transform(X5)
scalerdata1=np.c_[np.ones((m,1)),scalerdata]
y7 = df.sales
n_epochs= 1000
tf.reset_default_graph()

val_1 = tf.constant(scalerdata1, dtype=tf.float32, name="val_1")
val_2 = tf.constant(y7.values.reshape(-1, 1), dtype=tf.float32, name="val_2")
theta = tf.Variable(tf.random_uniform([n+1, 1], -1.0, 1.0),dtype=tf.float32, name="theta")

y_pred = tf.matmul(val_1, theta, name="predictions")
error = y_pred - val_2
mse = tf.reduce_mean(tf.square(error), name="mse")
gradients = 2/m * tf.matmul(tf.transpose(val_1), error)
training_op = tf.assign(theta, theta - learning_rate * gradients)


init = tf.global_variables_initializer()
with tf.Session() as sess:
    sess.run(init)
    
    for epoch in range(n_epochs):
        if epoch % 100 == 0:
            print("Epoch", epoch, "MSE =", mse.eval())
            sess.run(training_op)
    best_theta = theta.eval()<jupyter_output><empty_output><jupyter_text>c. Visualize training stats using Tensor board<jupyter_code>now = datetime.utcnow().strftime('%Y%m%d%H%M%S')
root_logdir = 'tf_hw3'
logdir = '{}/run-{}'.format(root_logdir, now)
mse_summary =tf.summary.scalar('MSE', mse)
file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())
m=180
with tf.name_scope('input'):
    x = tf.placeholder(tf.float32, shape=[m,n], name="x-input") 

with tf.name_scope('input'):
    y = tf.placeholder(tf.float32, shape=[m,1], name="y-input")
    
with tf.name_scope("weights"):
    W = tf.Variable(tf.ones([n,1]))
    b = tf.Variable(tf.zeros([1]))

with tf.name_scope("Wx_b") as scope:
    y_ = tf.matmul(x,W)+b

with tf.name_scope("cost") as scope:
    loss = tf.reduce_mean(tf.square(y - y_))

with tf.name_scope('train') as scope:
    optimizer = tf.train.GradientDescentOptimizer(0.00001)
    train = optimizer.minimize(loss)

we = tf.summary.histogram("weights",W)
bi = tf.summary.histogram("biases",b)
lo = tf.summary.histogram("cost",loss)

merged = tf.summary.merge_all()
writer = tf.summary.FileWriter(logdir, sess.graph)

with tf.Session() as sess:
    init = tf.global_variables_initializer()
    sess.run(init)
    vals = []
    
    for i in range(m):
        if i%10 == 0:
            result = we.eval(feed_dict={x:X5_train, y:Y5_train})
            writer.add_summary(result, i)
            result1 = bi.eval(feed_dict={x:X5_train, y:Y5_train})
            writer.add_summary(result1, i)
            result2 = bi.eval(feed_dict={x:X5_train, y:Y5_train})
            writer.add_summary(result2, i)
        else:
            val = sess.run([train,loss], feed_dict={x:X5_train, y:Y5_train})
            vals.append(val)
            W1 = sess.run(W)
            B1 = sess.run(b)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14022775423728814, 'fraction_numerical': 0.0892478813559322, 'mean_word_length': 4.046775810223855, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '68209', 'license': 'MIT', 'n_tokens_mistral': 6781, 'n_tokens_neox': 6036, 'n_words': 1289, 'path': 'Homeworks/EE-258 HW 3.ipynb', 'repo_name': 'Dharmikas/Machine-learning', 'size': '15104'}