<jupyter_start><jupyter_text><jupyter_code>%time
gpu_info = !nvidia-smi
gpu_info = '\n'.join(gpu_info)
if gpu_info.find('failed') >= 0:
  print('Select the Runtime > "Change runtime type" menu to enable a GPU accelerator, ')
  print('and then re-execute this cell.')
else:
  print(gpu_info)
from psutil import virtual_memory
ram_gb = virtual_memory().total / 1e9
print('Your runtime has {:.1f} gigabytes of available RAM\n'.format(ram_gb))

if ram_gb < 20:
  print('To enable a high-RAM runtime, select the Runtime > "Change runtime type"')
  print('menu, and then select High-RAM in the Runtime shape dropdown. Then, ')
  print('re-execute this cell.')
else:
  print('You are using a high-RAM runtime!')
import pandas as pd
import numpy as np
from sklearn.metrics import mean_squared_error
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.metrics import*
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import plot_confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn import model_selection
import statsmodels.api as sms;

# Starter Code --DO NOT ADJUST--
col_names = ["age", "workclass", "fnlwgt", "education", "education_num", "marital_status", "ooccupation", "relationship", "race", "sex", "capital_gain", "capital_loss", "hours_per_week", "native_country", "target"]
training = pd.read_csv("https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data", names=col_names).drop("race", axis=1)
test = pd.read_csv("https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.test", names=col_names, skiprows=1).drop("race", axis=1)


training["target"] = training["target"].apply(lambda x: x.strip())
test["target"] = test["target"].apply(lambda x: x.strip()[:-1])


# Create Train
X_train = training.drop("target", axis=1)
y_train = training["target"]

# Create Test
X_test = test.drop("target", axis=1)
y_test = test["target"]<jupyter_output><empty_output><jupyter_text>1. EDA
   - **What is this data?**
- It is about  Predict-customer-retention-ML.
-Predict whether income exceeds $50K/yr based on census data.<jupyter_code>training.sample(5)
training.describe()<jupyter_output><empty_output><jupyter_text>Visualization the Data: Visualize the `target` column based on Occupation<jupyter_code># select color
sns.set_palette('pastel')
# deterimne figure size
fig = plt.figure(figsize = (18,8))
ax=sns.countplot(training['ooccupation'], hue=training['target'])
# title
ax.set_title('Occupation vs Income', size=20)
plt.xlabel("Occupation",fontsize = 10)
plt.xticks(rotation=90)
plt.ylabel('Count');<jupyter_output>/usr/local/lib/python3.6/dist-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  FutureWarning<jupyter_text>Visualize `age` column as a groups<jupyter_code># select color
sns.set_palette('pastel')
# split age column into groups
age1 = training.loc[training.age <= 18].age.count()
age2 = training.loc[((training.age > 19)&(training.age <= 36))].age.count()
age3 = training.loc[((training.age > 37)&(training.age <= 54))].age.count()
age4 = training.loc[((training.age > 55)&(training.age <= 72))].age.count()
Age_list = [age1 , age2  , age3 , age4]
# deterimne figure size
plt.figure(figsize=[7,7])
plt.pie(Age_list, explode=[0.3,0,0,0], labels=['0-18 Years', '19-36 Years', '37-54 Years', '55 & Above' ] ,
        autopct='%1.1f%%',
              wedgeprops = {"edgecolor" : "black",
                      'linewidth': 1, 
                      'antialiased': True})
# title
plt.title("The Percentage of Age Groups");<jupyter_output><empty_output><jupyter_text>Visualize number of stages of education by gender<jupyter_code># select color
sns.set_palette('pastel')
# deterimne figure size
plt.figure(figsize=(17, 9))
sns.countplot(y='education', hue='sex', data=training )
plt.title("Stages of Education by Gender", size=20);<jupyter_output><empty_output><jupyter_text>Visualize the relationship between `workclass` and `hours_per_week`<jupyter_code>sns.set_palette('Set3')
g =sns.catplot(x = "workclass",y = "hours_per_week",data=training)
# deterimne figureheight and figurewidth size
g.fig.set_figheight(8)
g.fig.set_figwidth(10)
# title
plt.title( "The relationship between Workclass and Hours per Week") 
plt.xticks(
    rotation=45, 
    horizontalalignment='right',
    fontweight='light',
    fontsize='x-large');<jupyter_output><empty_output><jupyter_text>Visulaize the unique values in `relationship` column<jupyter_code># select color
sns.set_palette('pastel')
# deterimne figure size
plt.figure(figsize=(7,7));
labels = ['Not-in-family', 'Husband', 'Wife', 'Own-child',
          'Unmarried', 'Other-relative']
plt.pie(training.relationship.value_counts(), autopct='%1.1f%%' ,
        wedgeprops = {"edgecolor" : "white", 
                      'linewidth': 2, 
                      'antialiased': True})
# title
plt.title("The Percentage of Relationship")
plt.legend(labels, loc ='lower left');<jupyter_output><empty_output><jupyter_text>Visualize the relationship between `education_num` and `target`<jupyter_code># deterimne figure size
plt.figure(figsize=(7,5))
sns.barplot(y='education_num', x='target', data=training)
# title
plt.title('Education vs Income');<jupyter_output><empty_output><jupyter_text>2. Data prep & `train_test_split()`
     Check if there is any missing values<jupyter_code>training.isnull().sum()<jupyter_output><empty_output><jupyter_text>Remove Abnprmal values from `native_country` column<jupyter_code>training.drop(training.loc[training['native_country']==' ?'].index, inplace=True)<jupyter_output><empty_output><jupyter_text>Convert `target` column from categorical to int<jupyter_code>training['target']= training['target'].map({'<=50K':1, '>50K':0}).astype(int)
training.target<jupyter_output><empty_output><jupyter_text>Display categorical columns<jupyter_code>cat_col = [col for col in training.columns if training[col].dtype=='O']
cat_col<jupyter_output><empty_output><jupyter_text>Convert `sex` column from categorical to int<jupyter_code>training['sex'] = training['sex'].map({' Male':0, ' Female':1}).astype(int)<jupyter_output><empty_output><jupyter_text>Save object columns in variable<jupyter_code>categorical = training[cat_col]<jupyter_output><empty_output><jupyter_text>Convert categorical columns into dummy/ indictor variables<jupyter_code>categorical_dum = pd.get_dummies(categorical)<jupyter_output><empty_output><jupyter_text>Concat new datafame with training dataset<jupyter_code>df_new = pd.concat(
    [training,
     categorical_dum],
      axis=1)
df_new<jupyter_output><empty_output><jupyter_text>Drop categorical columns from dataframe<jupyter_code>df_new.drop(['workclass',
 'education',
 'marital_status',
 'ooccupation',
 'relationship',
 'native_country'], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>3. Create a model
We used Classification model:
- KNeighborsClassifier
- LogisticRegression
- SGDClassifier


 4. Optimization methods: 
**GridSearchCV** is a function that comes in Scikit-learn’s(or SK-learn) model_selection package. This function helps to loop through predefined hyperparameters and fit your estimator (model) on your training set. So, in the end, we can select the best parameters from the listed hyperparameters.

**RandomizedSearchCV** Random search is a technique where random combinations of the hyperparameters are used to find the best solution for the built model. It is similar to grid search, and yet it has proven to yield better results comparatively.<jupyter_code>%time
# import train_test_split to split dataset
from sklearn.model_selection import train_test_split

X = df_new.drop(['target'], axis=1)
#X = df_new[['relationship_ Husband', 'marital_status_ Married-civ-spouse', 'marital_status_ Never-married','education_num', 'age', 'hours_per_week']]
y = df_new.target

X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=.7)
# import standard scaler
from sklearn.preprocessing import StandardScaler
# import standard scaler
from sklearn.model_selection import RandomizedSearchCV

#Standardize features by removing the mean and scaling to unit variance
sc = StandardScaler()<jupyter_output><empty_output><jupyter_text>Model 1
 **KNeighborsClassifier:**<jupyter_code># import KNN model
from sklearn.neighbors import KNeighborsClassifier
#Compute the mean and std to be used for later scaling
X_train = sc.fit_transform(X_train)
#Fit to data, then transform it.
X_test = sc.transform(X_test)
knn = KNeighborsClassifier(leaf_size=2, n_neighbors=9, metric='minkowski', p=1, n_jobs=-1)

knn.fit(X_train, y_train)
# prediction for X_test
%%time
pred_knn = knn.predict(X_test)
# deterimne figure size
plt.figure(figsize=(16,10))
plt.scatter(x=df_new['hours_per_week'], y=df_new['fnlwgt'],
            c=df_new['target'], cmap=('Set3'),  s = 45)
# title
plt.title('Scatter K-Neighbors Classifier', fontsize=16);<jupyter_output><empty_output><jupyter_text>Visualization Confusion Matrix<jupyter_code>%%time 
# deterimne figure size
fig, ax = plt.subplots(figsize=(13, 6))
# title
plt.title('Confusion Matrix for KNN', fontsize=16)
# make confusion matrix
plot_confusion_matrix(knn, X=X_test, y_true=y_test, cmap='PuBuGn', ax=ax, display_labels=['<=50K', '>50K'], normalize='true', values_format='.0%');
#print accuracy for KNN
print("KNN Accuracy is {:.2f}%".format(accuracy_score(y_test, pred_knn)*100))<jupyter_output>KNN Accuracy is 82.92%<jupyter_text>LogisticRegression Optimization with RandomizedSearchCV:<jupyter_code>#RandomizedSearchCV parameters
param_grid = {'n_neighbors':[9,10,11,12],'weights': ['uniform', 'distance']}
#applying RandomizedSearchCV in KNN model
random_reg = RandomizedSearchCV(knn,param_grid,n_iter=15,n_jobs=-1,cv=5)
#fit traoning dato into RandomizedSearchCV
 %%time
 random_reg.fit(X_train,y_train)


#outcomes of RandomizedSearchCV in KNN to find best parameters and score
%%time
RandomizedSearchCV(cv=5, error_score='raise-deprecating',
          estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',
                                        n_neighbors=9, p=1,
                                        weights='uniform'),iid='warn', n_iter=15, n_jobs=-1,
                   param_distributions={'n_neighbors': [9, 10, 11, 12], 'weights': ['uniform', 'distance']})
#RandomizedSearchCV in KNN to find best parameters
random_reg.best_estimator_

#RandomizedSearchCV in KNN to find best score
random_reg.best_score_<jupyter_output><empty_output><jupyter_text>**KNN Report:**<jupyter_code># calculate the performance metrics for knn
classification_report(y_test, pred_knn)<jupyter_output><empty_output><jupyter_text>Model 2:
 **LogisticRegression :**<jupyter_code># import Logistic Regression model
from sklearn.linear_model import LogisticRegression
# logistic regression is a classifier, for our case, True (1) or False (0)
lr = LogisticRegression()
#fit training set into LogisticRegression model 
lr.fit(X_train, y_train)
# predict the y values from X test data
pred_lr = lr.predict(X_test)
#cross val score
from sklearn.model_selection import cross_val_score
cross_val_score(lr, X_train, y_train, cv=10)
# deterimne figure size
fig, ax = plt.subplots(figsize=(13, 6))
# title
plt.title('Confusion Matrix for LR', fontsize=16)
# make confusion matrix
plot_confusion_matrix(lr, X=X_test, y_true=y_test, cmap='PuBuGn', ax=ax, display_labels=['<=50K', '>50K'], normalize='true', values_format='.0%');
# print accuracy for SGDC
print("Logistics Regression Accuracy is {:.2f}%".format(accuracy_score(y_test, lr.predict(X_test))*100))<jupyter_output>Logistics Regression Accuracy is 85.03%<jupyter_text>LogisticRegression Optimization with RandomizedSearchCV:<jupyter_code>#RandomizedSearchCV parameters  for logisticregresssion
log_params = {'penalty': ['l1','l2'],
              'C': [0.001,0.01,0.1,0.5,1.0],
              'class_weight': ['balanced',None],
              'multi_class': ['auto', 'ovr'],
              'solver': ['newton-cg', 'lbfgs']
             }
#RandomizedSearchCV for logisticregresssion with previous parameters 
random_reg = RandomizedSearchCV(lr,log_params,n_jobs=-1,cv=10)
#fit training set into RandomizedSearchCV
random_reg.fit(X_train,y_train)

#RandomizedSearchCV's best parameters for LogisticRegression
random_reg.best_estimator_

#RandomizedSearchCV's best score for LogisticRegression
random_reg.best_score_<jupyter_output><empty_output><jupyter_text>Check out the accuracy of LogisticRegression model after using RandomizedSearchCV optimization:<jupyter_code>#hyperparameters of LogisticRegression
lr = LogisticRegression(C=0.1, class_weight=None, dual=False, fit_intercept=True,
                   intercept_scaling=1, l1_ratio=None, max_iter=100,
                   multi_class='auto', n_jobs=None, penalty='l2',
                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,
                   warm_start=False)
#fit rainig set into LogisticRegression after RandomizedSearchCV
lr.fit(X_train, y_train)
#prediction of X_test 
pred_lr = lr.predict(X_test)

#Accuracy of Logistics Regression
print("Logistics Regression Accuracy is {:.2f}%".format(accuracy_score(y_test, lr.predict(X_test))*100))<jupyter_output>Logistics Regression Accuracy is 85.04%<jupyter_text>LogisticRegression Optimization with GridSearchCV:<jupyter_code>param_grid ={'solver': ['liblinear', 'newton-cg'],
             'fit_intercept':[True, False]}
#Make GridSearchCV
grid_search =GridSearchCV(lr, param_grid, cv=10)
#fit X_train and y_train into grid_search
grid_search= grid_search.fit(X_train, y_train)
grid_search
#best parameters of LogisticRegression after using GridSearchCV
grid_search.best_params_
#best score of LogisticRegression after using GridSearchCV
grid_search.best_score_<jupyter_output><empty_output><jupyter_text>Check out the accuracy of LogisticRegression model after using GridSearchCV's optimization:<jupyter_code>#LogisticRegression with best parameters
lr=LogisticRegression(fit_intercept= True, solver= 'liblinear')
#fit training set into LogisticRegression
lr.fit(X_train, y_train)
#prediction of x_test
pred_lr = lr.predict(X_test)
print("LogisticRegression Accuracy after using GridSearchCV is {:.2f}%".format(accuracy_score(y_test, lr.predict(X_test))*100))<jupyter_output>LogisticRegression Accuracy after using GridSearchCV is 85.03%<jupyter_text>LogisticRegression Report:<jupyter_code># calculate the performance metrics for logistics regression
classification_report(y_test, pred_lr)<jupyter_output><empty_output><jupyter_text>Model 3:
  **Stochastic Gradient Descent:**<jupyter_code># import SGD Classifier model
from sklearn.linear_model import SGDClassifier
sgdc = SGDClassifier(max_iter=1000)
#fit training set into SGD model
sgdc.fit(X_train, y_train)
#prediction of X_test
pred_sgdc = sgdc.predict(X_test)
# print accuracy for SGDC
print("Stochastic Gradient Descent Accuracy is {:.2f}%".format(accuracy_score(y_test, pred_sgdc)*100))<jupyter_output>Stochastic Gradient Descent Accuracy is 84.07%<jupyter_text>Stochastic Gradient Descent Report:<jupyter_code># calculate the performance metrics for sgdc
classification_report(y_test, pred_sgdc)
# deterimne figure size
fig, ax = plt.subplots(figsize=(13, 6))
# title
plt.title('Confusion Matrix for SGDC', fontsize=16)
# make confusion matrix
plot_confusion_matrix(sgdc, X=X_test, y_true=y_test, cmap='PuBuGn', ax=ax, display_labels=['<=50K', '>50K'], normalize='true', values_format='.0%');<jupyter_output><empty_output><jupyter_text>Stochastic Gradient Descent Optimization with RandomizedSearchCV:<jupyter_code>#param_grid = {'n_neighbors':[9,10,11,12],'weights': ['uniform', 'distance']}

sgdc_params = {'loss': ['log', 'huber', 'epsilon_insensitive', 'squared_epsilon_insensitive'],
              'max_iter': [10, 100, 250, 1000, 10000],
               'penalty':['l2', 'l1']}
#optimize SGDClassifier by RandomizedSearchCV
random_reg = RandomizedSearchCV(sgdc,sgdc_params,n_jobs=-1,cv=10)

%%time
#fit training set into RandomizedSearchCV
random_reg.fit(X_train,y_train)

# best patamers of SGDClassifier after using RandomizedSearchCV
random_reg.best_estimator_
# best score of SGDClassifier after using RandomizedSearchCV
random_reg.best_score_<jupyter_output><empty_output><jupyter_text>Check out the accuracy of SGDClassifier model after using RandomizedSearchCV optimization:<jupyter_code>sgdc = SGDClassifier(alpha=0.0001, average=False, class_weight=None,
              early_stopping=False, epsilon=0.1, eta0=0.0, fit_intercept=True,
              l1_ratio=0.15, learning_rate='optimal', loss='log', max_iter=100,
              n_iter_no_change=5, n_jobs=None, penalty='l1', power_t=0.5,
              random_state=None, shuffle=True, tol=0.001,
              validation_fraction=0.1, verbose=0, warm_start=False)
#fit trainig set into SGDClassifier after using RandomizedSearchCV
sgdc.fit(X_train, y_train)

#prediction of x_test 
pred_sgdc = sgdc.predict(X_test)
#accuracy of Stochastic Gradient Descent
print("Stochastic Gradient Descent Accuracy after using RandomizedSearchCV optimization is {:.2f}%".format(accuracy_score(y_test, pred_sgdc)*100))<jupyter_output>Stochastic Gradient Descent Accuracy after using RandomizedSearchCV optimization is 84.98%<jupyter_text>Stochastic Gradient Descent Optimization with GridSearchCV:<jupyter_code>from sklearn.model_selection import GridSearchCV
#GridSearchCV optimization for Stochastic Gradient Descent model 
param_dict = {'loss': ['log', 'huber', 'epsilon_insensitive', 'squared_epsilon_insensitive'],
              'max_iter': [1000, 10000]}

grid_sgdc = GridSearchCV(sgdc, param_dict)
grid_sgdc
#fit X_train and y_train into grid_search
%%time
grid_sgdc.fit(X_train, y_train)
#sgdc best parameters after the GridSearchCV optimization
grid_sgdc.best_params_
#sgdc best score after the GridSearchCV optimization
grid_sgdc.best_score_<jupyter_output><empty_output><jupyter_text>Check out the accuracy of SGDClassifier model after using GridSearchCV optimization:<jupyter_code>#SGDClassifier with best parameters after using GridSearchCV
sgdc=SGDClassifier(loss= 'log', max_iter= 10000)
#fit training set into sgdc
sgdc.fit(X_train, y_train)
#prediction of X_test
pred_sgdc = sgdc.predict(X_test)
print("Stochastic Gradient Descent Accuracy after using GridSearchCV optimization is {:.2f}%".format(accuracy_score(y_test, pred_sgdc)*100))<jupyter_output>Stochastic Gradient Descent Accuracy after using GridSearchCV optimization is 84.52%<jupyter_text>5. Baseline model:

Baseline is a model that is both simple to set up and has a reasonable chance of providing decent results. Experimenting with them is usually quick and low cost, since implementations are widely available in popular packages.<jupyter_code># function baseline
def baseline_model(n_predictions, value_to_predict):
    """
    just predict a single value (e.g. mean) for everything
    """
    # create array to save predictions
    baseline_preds = []
    # loop to add baseline predictions in array baseline_preds
    for i in range(n_predictions):
        baseline_preds.append(value_to_predict)
    # return predictions
    return pd.Series(baseline_preds)
n_predictions = len(y_test)  # how many predictions to make?
baseline_value = y_train.value_counts().index[0]  # what value to predict? 
# make predictions baseline
baseline_preds = baseline_model(n_predictions, baseline_value)
baseline_preds
# baseline accuracy
baseline_acc = accuracy_score(y_test, baseline_preds)
print("Baseline Accuracy is {:.2f}%".format(baseline_acc*100))
# report for y_test and preds
print(classification_report(y_test, pred_knn))<jupyter_output><empty_output><jupyter_text>5. Optimize your model
    - [GridSearchCV](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html)
        - Bonus: [RandomizedSearchCV](https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.RandomizedSearchCV.html)
    - Compare your results to the baseline
        - How did you do this? Why?
    - If you don't beat the baseline, try improving your model or using a different model
        - Were you able to beat the baseline? Why?  Comparison between classification models accuracy:<jupyter_code># create empty array
models = []
# add models in array
models.append(('LogisticRegression', LogisticRegression()))
models.append(('SGDClassifier', SGDClassifier()))
models.append(('KNeighborsClassifier', KNeighborsClassifier()))
# create 2 empty array to append results and names for models
results = []
names = []
# determine type scoring
scoring = 'accuracy'
# loop to 
for name, model in models:
  kfold = model_selection.KFold(n_splits=10, random_state=None)
  cv_results = model_selection.cross_val_score(model, X_train, y_train, cv=kfold, scoring=scoring)
  results.append(cv_results)
  names.append(name)
  msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
  print(msg)
# determine figure size
fig = plt.figure(figsize=(12,9))
# title
fig.suptitle('Comparison between classification algorithms')
ax = fig.add_subplot(111)
# create boxplot
plt.boxplot(results)
ax.set_xticklabels(names)
# show plot
plt.show()<jupyter_output><empty_output><jupyter_text>ROC and AUC for SGDClassifier, KNeighborsRegressor, and LogisticRegression ( SGDC, KNN, LR) :<jupyter_code># import libraries roc_curve and area under curve
from sklearn.metrics import roc_curve, auc

# calculate roc curve
false_positive_rate_knn, true_positive_rate_knn, thresholds_knn = roc_curve(y_test, pred_knn)
false_positive_rate_lr, true_positive_rate_lr, thresholds_lr = roc_curve(y_test, pred_lr)
false_positive_rate_sgdc, true_positive_rate_sgdc, thresholds_sgdc = roc_curve(y_test, pred_sgdc)


# calculate area under curve 
roc_auc_knn = auc(false_positive_rate_knn, true_positive_rate_knn)
roc_auc_lr = auc(false_positive_rate_lr, true_positive_rate_lr)
roc_auc_sgdc = auc(false_positive_rate_sgdc, true_positive_rate_sgdc)

# figure size
plt.figure(figsize=(10,6))
# plot roc curve
plt.plot(false_positive_rate_knn, true_positive_rate_knn, color='green', alpha=0.5, label='ROC curve (KNN AUC= %0.2f)' % roc_auc_knn)
plt.plot(false_positive_rate_lr, true_positive_rate_lr, color='blue', alpha=0.5, label='ROC curve (LR AUC = %0.2f)' % roc_auc_lr)
plt.plot(false_positive_rate_sgdc, true_positive_rate_sgdc, color='red', alpha=0.5, label='ROC curve (SGDC AUC = %0.2f)' % roc_auc_sgdc)


# add labels
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')

# add title and specific location of legend
plt.title('ROC Curve', fontsize=16)
# show the plot
plt.legend(loc="lower right")
plt.show()<jupyter_output><empty_output><jupyter_text>Conclusion:

To conclude, we obtain `KNeighborsClassifier`, `LogisticRegression`, `SGDClassifier` recorded accuracy with `83.0%`, `85.03%`, and `84.52%`respectively after using `GridSearchCV` and `RandomizedSearchCV`. As we have seen above with no significant difference among them.

To compare these models with the `Baseline`, we illustrate that. Baseline showed the lowest accuracy with `75.97%`.

From the previous chart, the Area Under Curve (AUC) for `LogisticRegression` model LR is showing the best choice rather than `K-Nearest Neighbors` algorithm KNN.  Resources:
* [K-Nearest Neighbors](https://www.analyticsvidhya.com/blog/2015/08/learning-concept-knn-algorithms-programming/)
* [Logistic Regression](https://searchbusinessanalytics.techtarget.com/definition/logistic-regression)
* [SGD Classifier](https://www.tutorialspoint.com/scikit_learn/scikit_learn_stochastic_gradient_descent.htm)
* [Baseline](https://blog.insightdatascience.com/always-start-with-a-stupid-model-no-exceptions-3a22314b9aaa)
* [GridSearchCV](https://www.mygreatlearning.com/blog/gridsearchcv/)
* [RandomizedSearchCV](https://scikit-learn.org/0.16/modules/generated/sklearn.grid_search.RandomizedSearchCV.html)
* [Difference between GridSearchCV and RandomizedSearchCV](https://www.linkedin.com/pulse/what-gridsearchcv-randomizedsearchcv-differences-between-cheruku)
* [Comparing Machine Learning Methods](https://pythondata.com/comparing-machine-learning-methods/) **We tried some plot of KNN**<jupyter_code>#%%time
# # creating list of K for KNN
# k_list = list(range(1,50,2))
# # creating list of cv scores
# cv_scores = []

# # perform 10-fold cross validation
 #for k in k_list:
   #  knn = KNeighborsClassifier(n_neighbors=k)
  #   scores = cross_val_score(knn, X_train, y_train, cv=10, scoring='accuracy')
    # cv_scores.append(scores.mean())

# # changing to misclassification error
# MSE = [1 - x for x in cv_scores]

 #plt.figure()
 #plt.figure(figsize=(15,10))
 #plt.title('The optimal number of neighbors', fontsize=20, fontweight='bold')
 #plt.xlabel('Number of Neighbors K', fontsize=15)
 #plt.ylabel('Misclassification Error', fontsize=15)
 #sns.set_style("whitegrid")
 #plt.plot(k_list, MSE)

 #plt.show();
# error_rate = []
# # Might take some time
# for i in range(1,40):
    
#     knn = KNeighborsClassifier(n_neighbors=i)
#     knn.fit(X_train,y_train)
#     pred_i = knn.predict(X_test)
#     error_rate.append(np.mean(pred_i != y_test))

# plt.figure(figsize=(10,6))
# plt.plot(range(1,40),error_rate,color='blue', linestyle='dashed', marker='o',
#          markerfacecolor='red', markersize=10)
# plt.title('Error Rate vs. K Value')
# plt.xlabel('K')
# plt.ylabel('Error Rate')<jupyter_output><empty_output><jupyter_text>**In this part, we tried to use GridSerach for finding best parameters, and it consumed a long time to run:**<jupyter_code># #List Hyperparameters that we want to tune.
# leaf_size = list(range(1,30))
# n_neighbors = list(range(1,10))
# p=[1,2]
# #Convert to dictionary
# hyperparameters = dict(leaf_size=leaf_size, n_neighbors=n_neighbors, p=p)
# #Use GridSearch
# clf = GridSearchCV(knn, hyperparameters, cv=10)
# #Fit the model
# best_model = clf.fit(X,y)
# #Print The value of best Hyperparameters
# print('Best leaf_size:', best_model.best_estimator_.get_params()['leaf_size'])
# print('Best p:', best_model.best_estimator_.get_params()['p'])
# print('Best n_neighbors:', best_model.best_estimator_.get_params()['n_neighbors'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11710162267254917, 'fraction_numerical': 0.01701188962835564, 'mean_word_length': 5.802390640895219, 'pattern_counts': {'":': 0, '<': 158, '<?xml version=': 0, '>': 159, 'https://': 12, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '39', 'dir': 'jupyter-structured-clean-dedup', 'id': '588663', 'license': 'Apache-2.0', 'n_tokens_mistral': 9539, 'n_tokens_neox': 9027, 'n_words': 2033, 'path': 'W2_D5_income_bracket_prediction.ipynb', 'repo_name': 'ShaimaM/Intro_to_ML', 'size': '26746'}