<jupyter_start><jupyter_text>[Tutorial 06: Classification](https://franciszheng.com/dsper2020/tutorials/tutorial06/)  [[1] Getting Started](https://franciszheng.com/dsper2020/tutorials/tutorial06/getting-started)  [[1a] Importing Libraries](https://franciszheng.com/dsper2020/tutorials/tutorial06/importing-libraries)<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import statsmodels.api as sm
import statsmodels.formula.api as smf
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn import neighbors
from sklearn import preprocessing
from IPython.display import Markdown, display<jupyter_output><empty_output><jupyter_text>[[1b] Plot and Output Settings](https://franciszheng.com/dsper2020/tutorials/tutorial06/plot-and-output-settings)<jupyter_code># Reset all styles to the default:
plt.rcParams.update(plt.rcParamsDefault)
# Then make graphs inline:
%matplotlib inline

# Useful function for Jupyter to display text in bold:
def displaybd(text):
    display(Markdown("**" + text + "**"))
    
# Set custom style settings:
# NB: We need to separate "matplotlib inline" call and these settings into different
# cells, otherwise the parameters are not set. This is a bug somewhere in Jupyter
plt.rcParams['figure.figsize'] = (7, 6)
plt.rcParams['font.size'] = 24
plt.rcParams['legend.fontsize'] = 'large'
plt.rcParams['figure.titlesize'] = 'large'
plt.rcParams['lines.markersize'] = 10<jupyter_output><empty_output><jupyter_text>[[1c] Our Dataset](https://franciszheng.com/dsper2020/tutorials/tutorial06/our-dataset)<jupyter_code>smarket = pd.read_csv('Smarket.csv', parse_dates=False)

# Create direction codes as Up=1 and Down=0 to be sure about the interpretation in regressions:
smarket["DirectionCode"] = np.where(smarket["Direction"].str.contains("Up"), 1, 0)
display(smarket[1:10])

display(smarket.describe())

displaybd("Correlations matrix:")
display(smarket.corr())

smarket["Volume"].plot()
plt.xlabel("Day");
plt.ylabel("Volume");<jupyter_output><empty_output><jupyter_text>[[2] Logit](https://franciszheng.com/dsper2020/tutorials/tutorial06/logit)  [[2a] Running Logit via GLM](https://franciszheng.com/dsper2020/tutorials/tutorial06/running-logit-via-glm)<jupyter_code>model = smf.glm("DirectionCode~Lag1+Lag2+Lag3+Lag4+Lag5+Volume", data=smarket,
                family=sm.families.Binomial())
res = model.fit()
display(res.summary())<jupyter_output><empty_output><jupyter_text>[[2b] Predicted Probabilities and Confusion Matrix](https://franciszheng.com/dsper2020/tutorials/tutorial06/predicted-probabilities-and-confusion-matrix)<jupyter_code>displaybd("Predicted probabilities for the first observations:")
DirectionProbs = res.predict()
print(DirectionProbs[0:10])

DirectionHat = np.where(DirectionProbs > 0.5, "Up", "Down")
confusionDF = pd.crosstab(DirectionHat, smarket["Direction"],
                          rownames=['Predicted'], colnames=['Actual'],
                          margins=True)
display(Markdown("***"))
displaybd("Confusion matrix:")
display(confusionDF)

displaybd("Share of correctly predicted market movements:")
print(np.mean(smarket['Direction'] == DirectionHat))<jupyter_output><empty_output><jupyter_text>[[2c] Estimation of Test Error](https://franciszheng.com/dsper2020/tutorials/tutorial06/estimation-of-test-error)<jupyter_code>train = (smarket['Year'] < 2005)
smarket2005 = smarket[~train]
displaybd("Dimensions of the validation set:")
print(smarket2005.shape)

model = smf.glm("DirectionCode~Lag1+Lag2+Lag3+Lag4+Lag5+Volume", data=smarket,
                family=sm.families.Binomial(), subset=train)
res = model.fit()

DirectionProbsTets = res.predict(smarket2005)
DirectionTestHat = np.where(DirectionProbsTets > 0.5, "Up", "Down")
displaybd("Share of correctly predicted market movements in 2005:")
print(np.mean(smarket2005['Direction'] == DirectionTestHat))<jupyter_output><empty_output><jupyter_text>[[3] Linear Discriminant Analysis](https://franciszheng.com/dsper2020/tutorials/tutorial06/linear-discriminant-analysis)  [[3a] Custom Output Functions](https://franciszheng.com/dsper2020/tutorials/tutorial06/custom-output-functions)<jupyter_code>def printPriorProbabilities(ldaClasses, ldaPriors):
    priorsDF = pd.DataFrame()
    for cIdx, cName in enumerate(ldaClasses):
        priorsDF[cName] = [ldaPriors[cIdx]];
    displaybd('Prior probablities of groups:')
    display(Markdown(priorsDF.to_html(index=False)))
    
def printGroupMeans(ldaClasses, featuresNames, ldaGroupMeans):
    displaybd("Group means:")
    groupMeansDF = pd.DataFrame(index=ldaClasses)
    for fIdx, fName in enumerate(featuresNames):
         groupMeansDF[fName] = ldaGroupMeans[:, fIdx]
    display(groupMeansDF)

def printLDACoeffs(featuresNames, ldaCoeffs):
    coeffDF = pd.DataFrame(index=featuresNames)
    for cIdx in range(ldaCoeffs.shape[0]):
        colName = "LDA" + str(cIdx + 1)
        coeffDF[colName] = ldaCoeffs[cIdx]
    displaybd("Coefficients of linear discriminants:")
    display(coeffDF)<jupyter_output><empty_output><jupyter_text>[[3b] Fitting an LDA Model](https://franciszheng.com/dsper2020/tutorials/tutorial06/fitting-an-lda-model)<jupyter_code>outcomeName = 'Direction'
featuresNames = ['Lag1', 'Lag2'];

X_train = smarket.loc[train, featuresNames]
y_train = smarket.loc[train, outcomeName]

lda = LinearDiscriminantAnalysis()
ldaFit = lda.fit(X_train, y_train);

printPriorProbabilities(ldaFit.classes_, ldaFit.priors_)
printGroupMeans(ldaFit.classes_, featuresNames, ldaFit.means_)
printLDACoeffs(featuresNames, ldaFit.coef_)
# Coefficients calcualted by Python's LDA are different from R's lda.
# But they are proportional:
printLDACoeffs(featuresNames, 11.580267503964166 * ldaFit.coef_)
# See this: https://stats.stackexchange.com/questions/87479/what-are-coefficients-of-linear-discriminants-in-lda<jupyter_output><empty_output><jupyter_text>[[3c] LDA Predictions](https://franciszheng.com/dsper2020/tutorials/tutorial06/lda-predictions)<jupyter_code>X_test = smarket2005.loc[~train, featuresNames]
y_test = smarket.loc[~train, outcomeName]
y_hat = ldaFit.predict(X_test)

confusionDF = pd.crosstab(y_hat, y_test,
                          rownames=['Predicted'], colnames=['Actual'],
                          margins=True)
displaybd("Confusion matrix:")
display(confusionDF)

displaybd("Share of correctly predicted market movements:")
print(np.mean(y_test == y_hat))<jupyter_output><empty_output><jupyter_text>[[3d] Posterior Probabilities](https://franciszheng.com/dsper2020/tutorials/tutorial06/posterior-probabilities)<jupyter_code>pred_p = lda.predict_proba(X_test)
# pred_p is an array of shape (number of observations) x (number of classes)

upNmb = np.sum(pred_p[:, 1] > 0.5)
displaybd("Number of upward movements with threshold 0.5: " + str(upNmb))

upNmb = np.sum(pred_p[:, 1] > 0.9)
displaybd("Number of upward movements with threshold 0.9: " + str(upNmb))<jupyter_output><empty_output><jupyter_text>[[4] Quadratic Discriminant Analysis](https://franciszheng.com/dsper2020/tutorials/tutorial06/quadratic-discriminant-analysis)  [[4a] Fitting a QDA Model](https://franciszheng.com/dsper2020/tutorials/tutorial06/fitting-a-qda-model)<jupyter_code>qda = QuadraticDiscriminantAnalysis()
qdaFit = qda.fit(X_train, y_train);
printPriorProbabilities(qdaFit.classes_, qdaFit.priors_)
printGroupMeans(qdaFit.classes_, featuresNames, qdaFit.means_)<jupyter_output><empty_output><jupyter_text>[[4b] QDA Predictions](https://franciszheng.com/dsper2020/tutorials/tutorial06/qda-predictions)<jupyter_code>y_hat = qdaFit.predict(X_test)
confusionDF = pd.crosstab(y_hat, y_test,
                          rownames=['Predicted'], colnames=['Actual'],
                          margins=True)
displaybd("Confusion matrix:")
display(confusionDF)
displaybd("Share of correctly predicted market movements:")
print(np.mean(y_test == y_hat))<jupyter_output><empty_output><jupyter_text>[[5] k-Nearest Neighbors](https://franciszheng.com/dsper2020/tutorials/tutorial06/k-nearest-neighbors)  [[5a] One Neighbor](https://franciszheng.com/dsper2020/tutorials/tutorial06/one-neighbor)<jupyter_code>knn = neighbors.KNeighborsClassifier(n_neighbors=1)
y_hat = knn.fit(X_train, y_train).predict(X_test)
confusionDF = pd.crosstab(y_hat, y_test,
                          rownames=['Predicted'], colnames=['Actual'],
                          margins=True)
displaybd("Confusion matrix:")
display(confusionDF)
displaybd("Share of correctly predicted market movements:")
print(np.mean(y_test == y_hat))<jupyter_output><empty_output><jupyter_text>[[5b] Three Neighbors](https://franciszheng.com/dsper2020/tutorials/tutorial06/three-neighbors)<jupyter_code>knn = neighbors.KNeighborsClassifier(n_neighbors=3)
y_hat = knn.fit(X_train, y_train).predict(X_test)
confusionDF = pd.crosstab(y_hat, y_test,
                          rownames=['Predicted'], colnames=['Actual'],
                          margins=True)
displaybd("Confusion matrix:")
display(confusionDF)
displaybd("Share of correctly predicted market movements:")
print(np.mean(y_test == y_hat))<jupyter_output><empty_output><jupyter_text>[[6] An Application to Caravan Insurance Data](https://franciszheng.com/dsper2020/tutorials/tutorial06/an-application-to-caravan-insurance-data)  [[6a] A New Dataset](https://franciszheng.com/dsper2020/tutorials/tutorial06/a-new-dataset)  [[6aa] Loading Our Dataset](https://franciszheng.com/dsper2020/tutorials/tutorial06/loading-our-dataset)<jupyter_code>caravan = pd.read_csv('Caravan.csv', index_col=0)

display(caravan.describe())
display(caravan.describe(include=[np.object]))<jupyter_output><empty_output><jupyter_text>[[6ab] Standardizing Our Data](https://franciszheng.com/dsper2020/tutorials/tutorial06/standardizing-our-data)<jupyter_code>y = caravan.Purchase
X = caravan.drop('Purchase', axis=1).astype('float64')
X_scaled = preprocessing.scale(X)<jupyter_output><empty_output><jupyter_text>[[6ac] Splitting Data into Train and Test Data](https://franciszheng.com/dsper2020/tutorials/tutorial06/splitting-data-into-train-and-test-data)<jupyter_code>X_train = X_scaled[1000:,:]
y_train = y[1000:]
X_test = X_scaled[:1000,:]
y_test = y[:1000]<jupyter_output><empty_output><jupyter_text>[[6b] Using KNN for Prediction](https://franciszheng.com/dsper2020/tutorials/tutorial06/using-knn-for-prediction)<jupyter_code>knn = neighbors.KNeighborsClassifier(n_neighbors=1)
y_hat = knn.fit(X_train, y_train).predict(X_test)
confusionDF = pd.crosstab(y_hat, y_test,
                          rownames=['Predicted'], colnames=['Actual'],
                          margins=True)
displaybd("Confusion matrix:")
display(confusionDF)
displaybd("Share of correctly predicted purchases:")
print(np.mean(y_test == y_hat))<jupyter_output><empty_output><jupyter_text>[[6c] Logit](https://franciszheng.com/dsper2020/tutorials/tutorial06/logit-1)<jupyter_code>X_train_w_constant = sm.add_constant(X_train)
X_test_w_constant = sm.add_constant(X_test, has_constant='add')

y_train_code = np.where(y_train == "No", 0, 1)

res = sm.GLM(y_train_code, X_train_w_constant, family=sm.families.Binomial()).fit()
y_hat_code = res.predict(X_test_w_constant)
PurchaseHat = np.where(y_hat_code > 0.25, "Yes", "No")

confusionDF = pd.crosstab(PurchaseHat, y_test,
                          rownames=['Predicted'], colnames=['Actual'],
                          margins=True)
displaybd("Confusion matrix:")
display(confusionDF)<jupyter_output><empty_output><jupyter_text>[[7] More Iris Classification](https://franciszheng.com/dsper2020/tutorials/tutorial06/more-iris-classification)  [[7a] Our Dataset](https://franciszheng.com/dsper2020/tutorials/tutorial06/our-dataset-1)  [[7aa] Importing Our Dataset](https://franciszheng.com/dsper2020/tutorials/tutorial06/importing-our-dataset)<jupyter_code>url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data"
names = ['sepal-length', 'sepal-width', 'petal-length', 'petal-width', 'type']
iris_df = pd.read_csv(url, names=names)<jupyter_output><empty_output><jupyter_text>[[7ab] Splitting Data into Train and Test Data](https://franciszheng.com/dsper2020/tutorials/tutorial06/splitting-data-into-train-and-test-data-1)<jupyter_code>X = iris_df.iloc[:, :-1] #attributes, iloc[:, :-1] means until the last column
y = iris_df['type'] #labels
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.80)<jupyter_output><empty_output><jupyter_text>[[7ac] Feature Scaling](https://franciszheng.com/dsper2020/tutorials/tutorial06/feature-scaling)<jupyter_code>from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaler.fit(X_train)

X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)<jupyter_output><empty_output><jupyter_text>[[7b] Logit](https://franciszheng.com/dsper2020/tutorials/tutorial06/logit-2)  [[7ba]  Fitting Our Model](https://franciszheng.com/dsper2020/tutorials/tutorial06/fitting-our-model)<jupyter_code>from sklearn.linear_model import LogisticRegression
logit_model = LogisticRegression()
logit_model.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>[[7bb] Making Predictions](https://franciszheng.com/dsper2020/tutorials/tutorial06/making-predictions)<jupyter_code>y_pred = logit_model.predict(X_test)<jupyter_output><empty_output><jupyter_text>[[7bc] Evaluating Our Predictions](https://franciszheng.com/dsper2020/tutorials/tutorial06/evaluating-our-predictions)<jupyter_code>from sklearn.metrics import classification_report
print(classification_report(y_test, y_pred))
from sklearn.metrics import confusion_matrix

cm = confusion_matrix(y_test, y_pred)
cm_df = pd.DataFrame(cm,
                     index = ['setosa','versicolor','virginica'], 
                     columns = ['setosa','versicolor','virginica'])

sns.heatmap(cm_df, annot=True)
plt.ylabel('Actual')
plt.xlabel('Predicted')
plt.show()<jupyter_output><empty_output><jupyter_text>[[7c] Linear Discriminant Analysis](https://franciszheng.com/dsper2020/tutorials/tutorial06/linear-discriminant-analysis-1)  [[7ca] Fitting Our Model](https://franciszheng.com/dsper2020/tutorials/tutorial06/fitting-our-model-1)<jupyter_code>lda_model = LinearDiscriminantAnalysis()
lda_model.fit(X_train, y_train);<jupyter_output><empty_output><jupyter_text>[[7cb] Making Predictions](https://franciszheng.com/dsper2020/tutorials/tutorial06/making-predictions-1)<jupyter_code>y_pred = lda_model.predict(X_test)<jupyter_output><empty_output><jupyter_text>[[7cd] Evaluating Our Predictions](https://franciszheng.com/dsper2020/tutorials/tutorial06/evaluating-our-predictions-1)<jupyter_code>print(classification_report(y_test, y_pred))
cm = confusion_matrix(y_test, y_pred)
cm_df = pd.DataFrame(cm,
                     index = ['setosa','versicolor','virginica'], 
                     columns = ['setosa','versicolor','virginica'])

sns.heatmap(cm_df, annot=True)
plt.ylabel('Actual')
plt.xlabel('Predicted')
plt.show()<jupyter_output><empty_output><jupyter_text>[[7d] Quadratic Discriminant Analysis](https://franciszheng.com/dsper2020/tutorials/tutorial06/quadratic-discriminant-analysis-1)  [[7da] Fitting Our Model](https://franciszheng.com/dsper2020/tutorials/tutorial06/fitting-our-model-2)<jupyter_code>qda_model = QuadraticDiscriminantAnalysis()
qda_model.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>[[7db] Making Predictions](https://franciszheng.com/dsper2020/tutorials/tutorial06/making-predictions-2)<jupyter_code>y_pred = qda_model.predict(X_test)<jupyter_output><empty_output><jupyter_text>[[7dc] Evaluating Our Predictions](https://franciszheng.com/dsper2020/tutorials/tutorial06/evaluating-our-predictions-2)<jupyter_code>print(classification_report(y_test, y_pred))
cm = confusion_matrix(y_test, y_pred)
cm_df = pd.DataFrame(cm,
                     index = ['setosa','versicolor','virginica'], 
                     columns = ['setosa','versicolor','virginica'])

sns.heatmap(cm_df, annot=True)
plt.ylabel('Actual')
plt.xlabel('Predicted')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13807401906176506, 'fraction_numerical': 0.027726203738086395, 'mean_word_length': 6.550934579439252, 'pattern_counts': {'":': 0, '<': 126, '<?xml version=': 0, '>': 130, 'https://': 46, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '31', 'dir': 'jupyter-structured-clean-dedup', 'id': '388181', 'license': 'MIT', 'n_tokens_mistral': 6295, 'n_tokens_neox': 5775, 'n_words': 761, 'path': 'tutorials/tutorial06/tutorial06.ipynb', 'repo_name': 'francisdzheng/dsper2020', 'size': '16158'}