<jupyter_start><jupyter_text><NAME><jupyter_code>%run helper.py

# General functions for this problem
def get_decision_boundary(weights, x):
    w0 = weights[0]
    w1 = weights[1]
    w2 = weights[2]

    y = -(w0 + w1 * x) / w2
    
    return y
    
def sigmoid(z):
    return 1 / (1 + np.exp(-z))

def predict(weight, data):
    predicted_values = []

    for i in range(0, len(data)):
        total = weight[0] + data['x1'][i] * weight[1] +  data['x2'][i] * weight[2]
        predicted_value = 0 if sigmoid(total) <= 0.5 else 1
        predicted_values.append(predicted_value)
    
    return predicted_values<jupyter_output><empty_output><jupyter_text>Problem 7 Part AWrite down a code in Python whose input is a trainingdataset ${(x_1, y_1), . . . ,(x_N , y_N )}$ and itsoutput is the weight vectorw in the logistic regression model $y = \sigma(w \lt x)$.<jupyter_code>"""
    Input: pandas data frame of [x, y]
    Output: weight vector
    Assumption:
        1. Not for multiclass
"""

def logistic_regression(X, y, steps, learning_rate):
    intercept = np.ones((X.shape[0], 1))
    X = np.hstack((intercept, X))

    weights = np.zeros(X.shape[1])

    for step in range(0, steps):
        z = np.dot(X, weights)
        
        predictions = sigmoid(z)

        # Update weights with log likelihood gradient
        output_error_signal = y - predictions
        
        gradient = np.dot(X.T, output_error_signal)
        weights += learning_rate * gradient

    return weights<jupyter_output><empty_output><jupyter_text>Part BDownload thedataset on the course webpage. Use ‘dataset1’. Run thecode on the trainingdataset to compute $w$ and evaluate on the test dataset.Report $w$,classification error on the trainingset and classification error onthe testset. Plot the data (use different colors for data in differentclasses)and plotthe decision boundary found by the logistic regressions.<jupyter_code># Plotting the training dataset
# Load dataset 1
ds1 = scipy.io.loadmat('data1.mat')
ds1_train_X = ds1['X_trn']
ds1_test_X = ds1['X_tst']
ds1_train_Y = ds1['Y_trn']
ds1_test_Y = ds1['Y_tst']

# Combine them all into one data frame
ds1_train = to_data_frame(ds1_train_X, ds1_train_Y)

# Plot the graph
plt.scatter(ds1_train['x1'], ds1_train['x2'], c = ds1_train['y'])
plt.show()

# Ok, seems like linearly separable
# There are only two classes
alpha = 5 * (10**-5)
weights = logistic_regression(ds1_train[ds1_train.columns[0:2]], ds1_train['y'],
                     steps = 50000, learning_rate = alpha)
print(weights)
# Comparing the results to make sure the correctness
model = linear_model.LogisticRegression(fit_intercept=True, C = 1e15)
model.fit(ds1_train[ds1_train.columns[0:2]], ds1_train['y'])
# Print values using sklearn (Train dataset)
print("Prediction using sklearn (Train):")
model_predicted_values_train = model.predict(ds1_train[ds1_train.columns[0:2]])
print(model.predict(ds1_train[ds1_train.columns[0:2]]))

# Print values using our custom model (Train dataset)
predicted_values_train = predict(weights, ds1_train[ds1_train.columns[0:2]])
print("Prediction using our model (Train):")
print(predicted_values_train)

# Calculate the error:
print("The error for the train data using sklearn logistic regression is (%):")
print(count_error(model_predicted_values_train, ds1_train['y']) * 100)

# Calculate the error:
print("The error for the train data using our logistic regression is (%):")
print(count_error(predicted_values_train, ds1_train['y']) * 100)
# Combine them all into one data frame
ds1_test = to_data_frame(ds1_test_X, ds1_test_Y)

# Print values using sklearn (Test dataset)
print("Prediction using sklearn (Test):")
print(model.predict(ds1_test[ds1_test.columns[0:2]]))

# Print values using our custom model (Test dataset)
predicted_values_test = predict(weights, ds1_test[ds1_test.columns[0:2]])
print("Prediction using our model (Test):")
print(predicted_values_test)

# Calculate the error:
print("The error for the test data using our logistic regression is (%):")
print(count_error(predicted_values_test, ds1_test['y']) * 100)
# For our train model:
y = get_decision_boundary(weights, ds1_train['x1'])
plt.scatter(ds1_train['x1'], ds1_train['x2'], c = ds1_train['y'])
plt.plot(ds1_train['x1'], y)
plt.show()
# For our test model:
y = get_decision_boundary(weights, ds1_test['x1'])
plt.scatter(ds1_test['x1'], ds1_test['x2'], c = ds1_test['y'])
plt.plot(ds1_test['x1'], y)
plt.show()<jupyter_output><empty_output><jupyter_text>Part CRepeat part B using ‘dataset2’. Explain the differences in resultsbetween part A and B and justify your observations/results.<jupyter_code># Load dataset 2
ds2 = scipy.io.loadmat('data2.mat')
ds2_train_X = ds2['X_trn']
ds2_test_X = ds2['X_tst']
ds2_train_Y = ds2['Y_trn']
ds2_test_Y = ds2['Y_tst']

# Combine them all into one data frame
ds2_train = to_data_frame(ds2_train_X, ds2_train_Y)

# Plot the graph
plt.scatter(ds2_train['x1'], ds2_train['x2'], c = ds2_train['y'])
plt.show()

# Ok, this one is not linearly separable
# There are only two classes
alpha = 5 * (10**-5)
weights = logistic_regression(ds2_train[ds2_train.columns[0:2]], ds2_train['y'],
                     steps = 50000, learning_rate = alpha)
print(weights)
# Print values using our custom model (Train dataset)
predicted_values_train = predict(weights, ds1_train[ds1_train.columns[0:2]])
print("Prediction using our model (Train):")
print(predicted_values_train)

# Calculate the error:
print("The error for the train data using our logistic regression is (%):")
print(count_error(predicted_values_train, ds1_train['y']) * 100)
# Combine them all into one data frame
ds2_test = to_data_frame(ds2_test_X, ds2_test_Y)

# Print values using our custom model (Test dataset)
predicted_values_test = predict(weights, ds2_test[ds2_test.columns[0:2]])
print("Prediction using our model (Test):")
print(predicted_values_test)

# Calculate the error:
print("The error for the test data using our logistic regression is (%):")
print(count_error(predicted_values_test, ds2_test['y']) * 100)
# For our train model:
y = get_decision_boundary(weights, ds2_train['x1'])
plt.scatter(ds2_train['x1'], ds2_train['x2'], c = ds2_train['y'])
plt.plot(ds2_train['x1'], y)
plt.show()
# For our test model:
y = get_decision_boundary(weights, ds2_test['x1'])
plt.scatter(ds2_test['x1'], ds2_test['x2'], c = ds2_test['y'])
plt.plot(ds2_test['x1'], y)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12832477834811012, 'fraction_numerical': 0.03033131124591694, 'mean_word_length': 5.303921568627451, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '381305', 'license': 'MIT', 'n_tokens_mistral': 2340, 'n_tokens_neox': 2236, 'n_words': 586, 'path': '.ipynb_checkpoints/LogisticRegression-checkpoint.ipynb', 'repo_name': 'prdx/MLFS', 'size': '6457'}