<jupyter_start><jupyter_text>Imports<jupyter_code>from __future__ import print_function
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import pickle
from itertools import cycle
from IPython.display import Image
from sklearn.cross_validation import train_test_split
from sklearn import svm
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
%matplotlib inline
from __future__ import division
from patsy import dmatrices
from sklearn import linear_model as lm
from sklearn.linear_model import LogisticRegression
from sklearn import preprocessing
from sklearn import cross_validation
from sklearn import metrics
from sklearn.metrics import confusion_matrix, accuracy_score
from sklearn.metrics import roc_curve, auc
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import LinearSVC
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
import matplotlib.cm as cm
from sklearn import naive_bayes
from sklearn.metrics import accuracy_score, classification_report
# from catboost import CatBoostRegressor, CatBoostClassifier, Pool<jupyter_output>/Users/cyrusrustomji/anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)<jupyter_text>Modeling  Open and sort dataframe<jupyter_code>with open('/Users/cyrusrustomji/Documents/metis/metisgh/metis/spotify_3/Data/pipeline.pkl', 'rb') as handle:
    data = pickle.load(handle)
data.head()
df = data.drop(columns=['popularity_log','followers_log'])
df = df.sort_values(by=['followers'],ascending=False)
df.head()
df.head()<jupyter_output><empty_output><jupyter_text>x = df['popularity']min_max_scaler = preprocessing.MinMaxScaler()x_scaled = min_max_scaler.fit_transform(x.values)rating_df = pd.DataFrame(x_scaled)rating_df<jupyter_code>scaled_features = df.copy()
col_names = ['popularity', 'followers']
features = scaled_features[col_names]
scaler = StandardScaler().fit(features.values)
features = scaler.transform(features.values)
scaled_features[col_names] = features
scaled_features.head()<jupyter_output>/Users/cyrusrustomji/anaconda3/lib/python3.6/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.
  warnings.warn(msg, DataConversionWarning)<jupyter_text>Use the train/test info below for all models<jupyter_code>y,X=dmatrices('platinum ~ followers + popularity',data=df,return_type='dataframe')
xtrain, xtest, ytrain, ytest = cross_validation.train_test_split(X, y, test_size=0.2, random_state=1234)<jupyter_output><empty_output><jupyter_text>Logistic Regression  Question for JB/CS if my test size is large, the accuracy will increase, right?  Should I randomly shuffle the train sets below?<jupyter_code># 1. Fix the below to make sure it does not include repeat variable names
# 2. Leave changes to variables seperate in each model
def plot_confusion_matrix(cm,title='Confusion matrix', cmap=plt.cm.Reds):
    plt.imshow(cm, interpolation='nearest',cmap=cmap)
    plt.title(title)
    plt.colorbar()
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

#Could be a typical function for classifying:

def train_score(classifier,x,y):
    xtrain, xtest, ytrain, ytest2 = cross_validation.train_test_split(x, y, test_size=0.2, random_state=1234)
    ytrain2=np.ravel(ytrain)
    clf = classifier.fit(xtrain, ytrain2)
    # accuracy for test & train:
    train_acc=clf.score(xtrain, ytrain2)
    test_acc=clf.score(xtest,ytest2)
    print("Training Data Accuracy: %0.2f" %(train_acc))
    print("Test Data Accuracy:     %0.2f" %(test_acc))
    
    y_true = ytest
    y_pred = clf.predict(xtest)


    conf = confusion_matrix(y_true, y_pred)
    print(conf)

    print ('\n')
    print ("Precision:              %0.2f" %(conf[0, 0] / (conf[0, 0] + conf[1, 0])))
    print ("Recall:                 %0.2f"% (conf[0, 0] / (conf[0, 0] + conf[0, 1])))
    
    cm=confusion_matrix(y_true, y_pred, labels=None)
    
    plt.figure()
    plot_confusion_matrix(cm)
log_clf=LogisticRegression()
train_score(log_clf,X,y)<jupyter_output>Training Data Accuracy: 0.36
Test Data Accuracy:     0.45
[[0 6]
 [0 5]]


Precision:              nan
Recall:                 0.00<jupyter_text>ROC Curve<jupyter_code>log = LogisticRegression()
log.fit(xtrain,np.ravel(ytrain))
y_score=log.predict_proba(xtest)[:,1]

fpr, tpr,thres = roc_curve(ytest, y_score)
roc_auc = auc(fpr, tpr)

plt.figure()
# Plotting our Baseline..
plt.plot([0,1],[0,1])
plt.plot(fpr,tpr)
plt.xlabel('FPR')
plt.ylabel('TPR')
tpr
thres
1-thres
df.head()<jupyter_output><empty_output><jupyter_text>Gradient Descent  Does GD implement theta, l1,l2, and combo?  SVM  Question for JB/CS: run for loop to find most efficient gamma?  SVC with linear kernel<jupyter_code># fit linear model
model_svm = svm.SVC(kernel='linear')
ytrain3 = np.ravel(ytrain)
model_svm.fit(xtrain, ytrain3)
# predict out of sample
y_pred = model_svm.predict(xtest)
# check accuracy
accuracy_score(ytest,y_pred)
model_svm.coef_
from sklearn.metrics import average_precision_score
average_precision = average_precision_score(ytest, y_pred)

print('Average precision-recall score:', round(average_precision,2))
from sklearn.metrics import precision_recall_curve

precision, recall, _ = precision_recall_curve(ytest, y_pred)

plt.step(recall, precision, color='b', alpha=0.2,
         where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2,
                 color='b')

plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(
          average_precision))<jupyter_output><empty_output><jupyter_text>SVC with RBF kernel<jupyter_code># fit rbf model
model_svm2 = svm.SVC(kernel='rbf')
model_svm2.fit(xtrain, ytrain3)
y_pred2 = model_svm2.predict(xtest)
y_pred2
accuracy_score(ytest,y_pred2)
confusion_matrix(ytest,y_pred2)
from sklearn.metrics import precision_recall_curve

precision, recall, _ = precision_recall_curve(ytest, y_pred2)

plt.step(recall, precision, color='b', alpha=0.2,
         where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2,
                 color='b')

plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(
          average_precision))<jupyter_output><empty_output><jupyter_text>SVC with poly kernel<jupyter_code># fit poly model
model_svm3 = svm.SVC(kernel='poly')
model_svm3.fit(xtrain, ytrain3)
y_pred3 = model_svm3.predict(xtest)
y_pred3
accuracy_score(ytest,y_pred3)
confusion_matrix(ytest,y_pred3)<jupyter_output><empty_output><jupyter_text>SVM<jupyter_code>y,X=dmatrices('platinum ~ followers + popularity',data=df,return_type='dataframe')
def quick_test(model, X, y):
    xtrain, xtest, ytrain, ytest = train_test_split(X, y, test_size=0.2)
    model.fit(xtrain, ytrain)
    return model.score(xtest, ytest)

def quick_test_afew_times(model, X, y, n=10):
    return np.mean([quick_test(model, X, y) for j in range(n)])
linearsvc = LinearSVC()
quick_test_afew_times(linearsvc, X, y)<jupyter_output>/Users/cyrusrustomji/anaconda3/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)<jupyter_text>Naive Bayes<jupyter_code>y,X=dmatrices('platinum ~ followers + popularity',data=df,return_type='dataframe')
xtrain, xtest, ytrain, ytest = cross_validation.train_test_split(X, y, test_size=0.2, random_state=1234)<jupyter_output><empty_output><jupyter_text>Binomial NB<jupyter_code>model = naive_bayes.BernoulliNB()
ytrain = np.ravel(ytrain)
model.fit(xtrain, ytrain)

print("Accuracy: %.3f"% accuracy_score(ytest, model.predict(xtest)))
print(classification_report(ytest, model.predict(xtest)))<jupyter_output>Accuracy: 0.545
             precision    recall  f1-score   support

        0.0       0.55      1.00      0.71         6
        1.0       0.00      0.00      0.00         5

avg / total       0.30      0.55      0.39        11<jupyter_text>Multinomial NB<jupyter_code># this would work the best because the outcome of one album going platinum, will not affect
# the outcome of another album going platinum

model = naive_bayes.MultinomialNB()
model.fit(xtrain, ytrain)

print("Accuracy: %.3f"% accuracy_score(ytest, model.predict(xtest)))
print(classification_report(ytest, model.predict(xtest)))
def plot_confusion_matrix(cm,title='Confusion matrix', cmap=plt.cm.Reds):
    plt.imshow(cm, interpolation='nearest',cmap=cmap)
    plt.title(title)
    plt.colorbar()
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

#Could be a typical function for classifying:

def train_score(classifier,x,y):
    xtrain, xtest, ytrain, ytest2 = cross_validation.train_test_split(x, y, test_size=0.2, random_state=1234)
    ytrain2=np.ravel(ytrain)
    clf = classifier.fit(xtrain, ytrain2)
    # accuracy for test & train:
    train_acc=clf.score(xtrain, ytrain2)
    test_acc=clf.score(xtest,ytest2)
    print("Training Data Accuracy: %0.2f" %(train_acc))
    print("Test Data Accuracy:     %0.2f" %(test_acc))
    
    y_true = ytest
    y_pred = clf.predict(xtest)


    conf = confusion_matrix(y_true, y_pred)
    print(conf)

    print ('\n')
    print ("Precision:              %0.2f" %(conf[0, 0] / (conf[0, 0] + conf[1, 0])))
    print ("Recall:                 %0.2f"% (conf[0, 0] / (conf[0, 0] + conf[0, 1])))
    
    cm=confusion_matrix(y_true, y_pred, labels=None)
    
    plt.figure()
    plot_confusion_matrix(cm)
log_clf=naive_bayes.MultinomialNB()
train_score(log_clf,X,y)
model.fit(xtrain, ytrain)
print(classification_report(ytest, model.predict(xtest)))<jupyter_output>precision    recall  f1-score   support

        0.0       0.75      1.00      0.86         6
        1.0       1.00      0.60      0.75         5

avg / total       0.86      0.82      0.81        11<jupyter_text>SVCs Part 2  Linear SVC<jupyter_code>from sklearn.svm import LinearSVC, SVC
from sklearn.preprocessing import scale

xtrain = scale(xtrain)
xtest = scale(xtest)

model = LinearSVC()
model.fit(xtrain, ytrain)

print("Accuracy: %.3f"% accuracy_score(ytest, model.predict(xtest)))
print(classification_report(ytest, model.predict(xtest)))<jupyter_output>Accuracy: 0.727
             precision    recall  f1-score   support

        0.0       0.71      0.83      0.77         6
        1.0       0.75      0.60      0.67         5

avg / total       0.73      0.73      0.72        11<jupyter_text>SVC<jupyter_code>model = SVC()
model.fit(xtrain, ytrain)

print("Accuracy: %.3f"% accuracy_score(ytest, model.predict(xtest)))
print(classification_report(ytest, model.predict(xtest)))<jupyter_output>Accuracy: 0.818
             precision    recall  f1-score   support

        0.0       0.75      1.00      0.86         6
        1.0       1.00      0.60      0.75         5

avg / total       0.86      0.82      0.81        11<jupyter_text>Decision Trees<jupyter_code>y,X=dmatrices('platinum ~ followers + popularity',data=df,return_type='dataframe')
xtrain, xtest, ytrain, ytest = cross_validation.train_test_split(X, y, test_size=0.2, random_state=1234)<jupyter_output><empty_output><jupyter_text>How to pick the right number of trees<jupyter_code>for i in range(1,20,1):
    decisiontree = DecisionTreeClassifier(max_depth=i)
    print(i,quick_test_afew_times(decisiontree, X, y))<jupyter_output>1 0.845454545455
2 0.809090909091
3 0.836363636364
4 0.827272727273
5 0.854545454545
6 0.790909090909
7 0.818181818182
8 0.827272727273
9 0.818181818182
10 0.790909090909
11 0.790909090909
12 0.790909090909
13 0.781818181818
14 0.763636363636
15 0.8
16 0.845454545455
17 0.809090909091
18 0.845454545455
19 0.9<jupyter_text>This is random every time, right?<jupyter_code>for i in range(1,10,1):
    randomforest = RandomForestClassifier(n_estimators=i)
    yrf = np.ravel(y)
    print(i, quick_test_afew_times(randomforest, X, yrf))
decisiontree = DecisionTreeClassifier()
quick_test_afew_times(decisiontree, X, y)
linearsvc = LinearSVC(loss='hinge')
quick_test_afew_times(linearsvc, X, yrf)
svc = SVC()
quick_test_afew_times(svc, X, yrf)
s2 = SVC()
X2 = (0.5-X) * 2
quick_test_afew_times(s2, X2, yrf)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11413466023946509, 'fraction_numerical': 0.04851500544238843, 'mean_word_length': 4.61703056768559, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '73762', 'license': 'Apache-2.0', 'n_tokens_mistral': 5101, 'n_tokens_neox': 4708, 'n_words': 989, 'path': 'Album_Sales_Prediction/Code/.ipynb_checkpoints/Modeling-checkpoint.ipynb', 'repo_name': 'cyrus-rustomji/metis', 'size': '12862'}