<jupyter_start><jupyter_text>Supervised Learning Setting imports<jupyter_code>%matplotlib inline
import os

os.environ['KMP_DUPLICATE_LIB_OK']='True'

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from scipy import stats
import seaborn as sns
import statsmodels.api as sm
from sklearn import svm
from sklearn import linear_model
from statsmodels.formula.api import ols
from sklearn import svm # SVM classifier
from sklearn.neural_network import MLPClassifier # MLP classifier
from sklearn import cross_validation # used to test classifier
from sklearn.cross_validation import KFold, cross_val_score, train_test_split
from sklearn import metrics
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import KNeighborsClassifier 
from sklearn.tree import DecisionTreeClassifier 
from sklearn.ensemble import RandomForestClassifier 
from sklearn.naive_bayes import MultinomialNB, GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_curve # ROC Curves
from sklearn.metrics import auc # Calculating AUC for ROC's!

from sklearn.model_selection import GridSearchCV, KFold, train_test_split
from sklearn.metrics import make_scorer, accuracy_score
from sklearn.tree import DecisionTreeClassifier

import warnings
warnings.filterwarnings('ignore')<jupyter_output>/anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)<jupyter_text>Read data from csv file.<jupyter_code>original = pd.read_csv('clean_data_new.csv')
original.head()<jupyter_output><empty_output><jupyter_text>Drop coloum that would not be used.<jupyter_code>data_valid = original.drop(['id','runtime','vote_count'],axis=1)
data_valid.head()<jupyter_output><empty_output><jupyter_text>Supervised Learning  Data scaling function to make data to [0,1]<jupyter_code>def scaled_df(df):
    scaled = pd.DataFrame()
    for item in df:
        if item in df.select_dtypes(include=[np.float]):
            scaled[item] = ((df[item] - df[item].min()) / 
            (df[item].max() - df[item].min()))
        else: 
            scaled[item] = df[item]
    return scaled

data_valid_scaled = scaled_df(data_valid)<jupyter_output><empty_output><jupyter_text>Getting box plot<jupyter_code>f, ax = plt.subplots(figsize=(11, 15))

ax.set_facecolor('#FFFFFF')
plt.title("Box Plot Data Scaled")
ax.set(xlim=(-.05, 1.05))
ax = sns.boxplot(data = data_valid_scaled[0:40], 
  orient = 'h', 
  palette = 'Set3')<jupyter_output><empty_output><jupyter_text>Decision Tree  rank_predictor function to predict the relationship of each factor<jupyter_code>data_names=['budget','popularity','revenue']

def rank_predictors(dat,l,f='if_popular'):
    rank={}
    max_vals=dat.max()
    median_vals=dat.groupby(f).median()  # We are using the median as the mean is sensitive to outliers
    for p in l:
        score=np.abs((median_vals[p][0]-median_vals[p][1])/max_vals[p])
        rank[p]=score
    return rank

cat_rank = rank_predictors(data_valid,data_names)
cat_rank
cat_rank=sorted(cat_rank.items(), key=lambda x: x[1])
cat_rank<jupyter_output><empty_output><jupyter_text>Take the top predictors based on median difference<jupyter_code># Take the top predictors based on median difference
ranked_predictors=[]
for f in cat_rank[0:]:
    ranked_predictors.append(f[0])
ranked_predictors<jupyter_output><empty_output><jupyter_text>Setting X, y values<jupyter_code>X = data_valid_scaled[data_names]
#setting target
y = data_valid_scaled['if_popular']<jupyter_output><empty_output><jupyter_text>Import library that we use.<jupyter_code>import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import SGD<jupyter_output>Using TensorFlow backend.<jupyter_text>Do first time of decision tree.<jupyter_code>#dividing data to have a training and a testing set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size= 0.2, random_state=0)
# Decision Tree classifier

DTm = DecisionTreeClassifier()
                                
# Decision Tree cross validation

print("KfoldCrossVal mean score using Decision Tree is %s" %cross_val_score(DTm,X,y,cv=10).mean())

# Decision Tree metrics
sm = DTm.fit(X_train, y_train)

y_pred = sm.predict(X_test)
print("Accuracy score using Decision Tree is %s" %metrics.accuracy_score(y_test, y_pred))<jupyter_output>KfoldCrossVal mean score using Decision Tree is 0.6839457121124657
Accuracy score using Decision Tree is 0.7127991675338189<jupyter_text>Optimize the model, changing parameters.<jupyter_code>regressor = DecisionTreeClassifier(random_state=0)
parameters = {'max_depth': range(1, 6),
             'random_state':range(0,5,1),
             'min_samples_split':range(2,5,1),
             'min_samples_leaf':range(1,5,1)}
scoring_fnc = make_scorer(accuracy_score)
kfold = KFold(n_splits=10)
grid = GridSearchCV(regressor, parameters, scoring_fnc, cv=kfold)
grid = grid.fit(X_train, y_train)
reg = grid.best_estimator_
print('best score: %f'%grid.best_score_)
print('best parameters:')
for key in parameters.keys():
    print('%s: %d'%(key, reg.get_params()[key]))
print('test score: %f'%reg.score(X_test, y_test))
import pandas as pd
pd.DataFrame(grid.cv_results_).T<jupyter_output>best score: 0.805830
best parameters:
max_depth: 5
random_state: 0
min_samples_split: 2
min_samples_leaf: 2
test score: 0.812695<jupyter_text>Put best parameters into model.<jupyter_code># Decision Tree classifier

DTm = DecisionTreeClassifier(max_depth=5,random_state=0,min_samples_split=2,min_samples_leaf=2)
                                
# Decision Tree cross validation

print("KfoldCrossVal mean score using Decision Tree is %s" %cross_val_score(DTm,X,y,cv=10).mean())

# Decision Tree metrics
sm = DTm.fit(X_train, y_train)

y_pred = sm.predict(X_test)
print("Accuracy score using Decision Tree is %s" %metrics.accuracy_score(y_test, y_pred))<jupyter_output>KfoldCrossVal mean score using Decision Tree is 0.7888778542875621
Accuracy score using Decision Tree is 0.8126951092611863<jupyter_text>Random Forest  Setting parameters of model and get result.<jupyter_code># Random Forest classifier

RFm = RandomForestClassifier( random_state = 42,
                                criterion='gini',
                                n_estimators = 50)
                                
# Random Forest cross validation

print("KfoldCrossVal mean score using Random Forest is %s" %cross_val_score(RFm,X,y,cv=10).mean())

# Random Forest metrics
sm = RFm.fit(X_train, y_train)

y_pred = sm.predict(X_test)
print("Accuracy score using Random Forest is %s" %metrics.accuracy_score(y_test, y_pred))<jupyter_output>KfoldCrossVal mean score using Random Forest is 0.7447197814081947
Accuracy score using Random Forest is 0.8033298647242456<jupyter_text>Optimize the model, changing parameters.<jupyter_code>regressor = RandomForestClassifier()
parameters = {'n_estimators': range(10,50,10),
             'random_state':range(20,50,10),
             'min_samples_split':range(2,5,1),
             'min_samples_leaf':range(1,5,1)}
scoring_fnc = make_scorer(accuracy_score)
kfold = KFold(n_splits=10)
grid = GridSearchCV(regressor, parameters, scoring_fnc, cv=kfold)
grid = grid.fit(X_train, y_train)
reg = grid.best_estimator_
print('best score: %f'%grid.best_score_)
print('best parameters:')
for key in parameters.keys():
    print('%s: %d'%(key, reg.get_params()[key]))
print('test score: %f'%reg.score(X_test, y_test))
import pandas as pd
pd.DataFrame(grid.cv_results_).T<jupyter_output>best score: 0.812337
best parameters:
n_estimators: 40
random_state: 20
min_samples_split: 2
min_samples_leaf: 3
test score: 0.823101<jupyter_text>Put best parameters into model.<jupyter_code># Random Forest classifier

RFm = RandomForestClassifier( random_state = 20,
                                criterion='gini',
                                n_estimators = 40,
                                min_samples_split = 2,
                                min_samples_leaf = 3)
                                
# Random Forest cross validation

print("KfoldCrossVal mean score using Random Forest is %s" %cross_val_score(RFm,X,y,cv=10).mean())

# Random Forest metrics
sm = RFm.fit(X_train, y_train)

y_pred = sm.predict(X_test)
print("Accuracy score using Random Forest is %s" %metrics.accuracy_score(y_test, y_pred))<jupyter_output>KfoldCrossVal mean score using Random Forest is 0.7736751389256609
Accuracy score using Random Forest is 0.8231009365244537
{'fraction_non_alphanumeric': 0.09644957033016735, 'fraction_numerical': 0.030303030303030304, 'mean_word_length': 5.033424283765348, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '402102', 'license': 'MIT', 'n_tokens_mistral': 3150, 'n_tokens_neox': 2915, 'n_words': 720, 'path': 'Supervised Learning/Supervised Learning.ipynb', 'repo_name': 'BoyangWei/INFO_7390', 'size': '8844'}