<jupyter_start><jupyter_text>Scaling<jupyter_code># import packages
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
# read dataframe in
df = pd.read_csv('data/kickstarter_preprocess.csv')
df.columns<jupyter_output><empty_output><jupyter_text>features to keep: preparation, duration_days, goal, pledged_per_backer, parent_name, blurb_len_w, slug_len_w, 'launched_month'<jupyter_code># drop unimportant features
df.drop(['backers_count', 'country', 'usd_pledged', 'blurb_len_c', 'slug_len_c', 'cat_in_slug', 
         'category_parent_id', 'category_id', 'category_name', 'created_year', 'created_month', 'deadline_year', 
         'deadline_month', 'launched_year', 'rel_pledged_goal', 'filled_parent', 'staff_pick'], 
        axis=1, inplace=True)
df.columns
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 177593 entries, 0 to 177592
Data columns (total 9 columns):
 #   Column              Non-Null Count   Dtype  
---  ------              --------------   -----  
 0   goal                177593 non-null  float64
 1   state               177593 non-null  object 
 2   blurb_len_w         177593 non-null  int64  
 3   slug_len_w          177593 non-null  int64  
 4   launched_month      177593 non-null  int64  
 5   duration_days       177593 non-null  int64  
 6   preparation         177593 non-null  int64  
 7   pledged_per_backer  177593 non-null  int64  
 8   parent_name         177593 non-null  object 
dtypes: float64(1), int64(6), object(2)
memory usage: 12.2+ MB<jupyter_text>drop rows with state == canceled, rows with wrong categories<jupyter_code>df = df.drop(df[df['state'] == "canceled" ].index)
df.shape
categories = ["Games", "Art", "Photography", "Film & Video", "Design", "Technology"]
df = df[df.parent_name.isin(categories)]
df.shape<jupyter_output><empty_output><jupyter_text>make dummies (state, category_name)<jupyter_code>#df.staff_pick = df.staff_pick.astype('int')
df['state'] = np.where(df['state'] == 'successful', 1, 0)
df.groupby('state').state.count()
# convert the categorical variable parent_name into dummy/indicator variables
df_dum2 = pd.get_dummies(df.parent_name, prefix='parent_name')
df = df.drop(['parent_name'], axis=1)
df = pd.concat([df, df_dum2], axis=1)
# making a categorical variable for launched_month q1, q2, q3, q4 
df.loc[df['launched_month'] <  4, 'time_yr'] = 'q1'
df.loc[(df['launched_month'] >=  4) & (df['launched_month'] <  7), 'time_yr'] = 'q2'
df.loc[(df['launched_month'] >=  7) & (df['launched_month'] <  10), 'time_yr'] = 'q3'
df.loc[df['launched_month'] >  9, 'time_yr'] = 'q4'
df_dum3 = pd.get_dummies(df.time_yr, prefix='time_yr')
df = df.drop(['time_yr'], axis=1)
df = df.drop(['launched_month'], axis=1)
df = pd.concat([df, df_dum3], axis=1)
df.columns
df.info()
df.head()<jupyter_output><empty_output><jupyter_text>Train-Test-Split<jupyter_code>from sklearn.model_selection import train_test_split, cross_val_score
y = df.state
X = df.drop('state', axis=1)
# Train-test-split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)<jupyter_output><empty_output><jupyter_text>Scaling<jupyter_code>from sklearn.preprocessing import StandardScaler
# we have to define which columns we want to scale.
col_scale = ['goal', 'blurb_len_w', 'slug_len_w', 'duration_days', 'preparation', 'pledged_per_backer']<jupyter_output><empty_output><jupyter_text>Data standardization<jupyter_code># Scaling with standard scaler
scaler = StandardScaler()
X_train_scaled_st = scaler.fit_transform(X_train[col_scale])
X_test_scaled_st = scaler.transform(X_test[col_scale])
# Concatenating scaled and dummy columns 
X_train_preprocessed_st = np.concatenate([X_train_scaled_st, X_train.drop(col_scale, axis=1)], axis=1)
X_test_preprocessed_st = np.concatenate([X_test_scaled_st, X_test.drop(col_scale, axis=1)], axis=1)<jupyter_output><empty_output><jupyter_text>Data normalization  Scaling with MinMaxScaler Try to scale you data with the MinMaxScaler() from sklearn.  It follows the same syntax as the StandardScaler. Don't forget: you have to import the scaler at the top of your notebook. from sklearn.preprocessing import MinMaxScalerscaler = MinMaxScaler()X_train_scaled_nor = scaler.fit_transform(X_train[col_scale])X_test_scaled_nor = scaler.transform(X_test[col_scale])  Concatenating scaled and dummy columns X_train_preprocessed_nor = np.concatenate([X_train_scaled_nor, X_train.drop(col_scale, axis=1)], axis=1)X_test_preprocessed_nor = np.concatenate([X_test_scaled_nor, X_test.drop(col_scale, axis=1)], axis=1)<jupyter_code>df.groupby('state').state.count()<jupyter_output><empty_output><jupyter_text>Model Classification and Gridsearch (tuning hyperparameters) Logistic Regression<jupyter_code>from sklearn.datasets import make_classification
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.model_selection import GridSearchCV
# fit model
lr = LogisticRegression()
lr.fit(X_train_preprocessed_st, y_train)
y_pred = lr.predict(X_test_preprocessed_st)
confusion_matrix(y_test, y_pred)
# normalization
#print (classification_report(y_test, y_pred))
# standardization
print classification_report(y_test, y_pred))
# Gridsearch https://www.kaggle.com/enespolat/grid-search-with-logistic-regression
grid = {"C":np.logspace(-3,3,7), "penalty":["l1","l2"]} # l1 lasso l2 ridge
logreg = LogisticRegression()
logreg_cv = GridSearchCV(logreg,grid,cv=10)
logreg_cv.fit(X_train_preprocessed_st,y_train)

print("tuned hpyerparameters :(best parameters) ",logreg_cv.best_params_)
print("accuracy :",logreg_cv.best_score_)
# fit model
lr2 = LogisticRegression(C=1000.0,penalty="l2")
lr2.fit(X_train_preprocessed_st, y_train)
y_pred = lr2.predict(X_test_preprocessed_st)
confusion_matrix(y_test, y_pred)
print(classification_report(y_test, y_pred))<jupyter_output>precision    recall  f1-score   support

           0       0.75      0.69      0.72      7732
           1       0.74      0.80      0.77      8676

    accuracy                           0.74     16408
   macro avg       0.75      0.74      0.74     16408
weighted avg       0.74      0.74      0.74     16408<jupyter_text>Kernel SVM<jupyter_code>import pylab as pl
import scipy.optimize as opt
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=4)
print ('Train set:', X_train_preprocessed_st.shape,  y_train.shape)
print ('Test set:', X_test_preprocessed_st.shape,  y_test.shape)
from sklearn import svm

clf = svm.SVC(kernel='rbf')
clf.fit(X_train_preprocessed_st, y_train) 
from sklearn.metrics import classification_report, confusion_matrix
import itertools
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
# Compute confusion matrix
cnf_matrix = confusion_matrix(y_test, y_pred, labels=[0,1])
np.set_printoptions(precision=2)

print (classification_report(y_test, y_pred))

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=['failed','successful'],normalize= False,  title='Confusion matrix')
param_grid = [{'kernel': ['rbf'], 
               'gamma': [0.0001, 0.001, 0.01, 0.1, 1],
               'C': [1, 10, 100, 1000]},
              {'kernel': ['linear'], 
               'C': [1, 10, 100, 1000]}]

grid = GridSearchCV(clf, param_grid, verbose=True, n_jobs=-1)

result = grid.fit(X_train_preprocessed_st, y_train)
# Print best parameters 
print('Best Parameters:', result.best_params_)

# Print best score
print('Best Score:', result.best_score_)
clf2 = svm.SVC(kernel='rbf')
clf2.fit(X_train_preprocessed_st, y_train)
# Compute confusion matrix
cnf_matrix = confusion_matrix(y_test, y_pred, labels=[0,1])
np.set_printoptions(precision=2)

print (classification_report(y_test, y_pred))

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=['failed','successful'],normalize= False,  title='Confusion matrix')<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RandomizedSearchCV
# Create the model with 100 trees
model = RandomForestClassifier(n_estimators=100, 
                               random_state=42, 
                               max_features = 'sqrt',
                               n_jobs=-1, verbose = 1)
# Fit on training data
model.fit(X_train_preprocessed_st, y_train)
y_pred = model.predict(X_test_preprocessed_st)
# Training predictions (to demonstrate overfitting)
train_rf_predictions = model.predict(X_train_preprocessed_st)
train_rf_probs = model.predict_proba(X_train_preprocessed_st)[:, 1]

# Testing predictions (to determine performance)
rf_predictions = model.predict(X_test_preprocessed_st)
rf_probs = model.predict_proba(X_test_preprocessed_st)[:, 1]
# Compute confusion matrix
cnf_matrix = confusion_matrix(y_test, y_pred, labels=[0,1])
np.set_printoptions(precision=2)

print (classification_report(y_test, y_pred))

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=['failed','successful'],normalize= False,  title='Confusion matrix')
print (classification_report(y_test, y_pred))<jupyter_output>precision    recall  f1-score   support

           0       0.86      0.73      0.79      7732
           1       0.79      0.89      0.84      8676

    accuracy                           0.81     16408
   macro avg       0.82      0.81      0.81     16408
weighted avg       0.82      0.81      0.81     16408<jupyter_text>Random Forest: Optimization through Random Search<jupyter_code># Hyperparameter grid
param_grid = {
    'n_estimators': np.linspace(10, 200).astype(int),
    'max_depth': [None] + list(np.linspace(3, 20).astype(int)),
    'max_features': ['auto', 'sqrt', None] + list(np.arange(0.5, 1, 0.1)),
    'max_leaf_nodes': [None] + list(np.linspace(10, 50, 500).astype(int)),
    'min_samples_split': [2, 5, 10],
    'bootstrap': [True, False]
}

# Estimator for use in random search
estimator = RandomForestClassifier(random_state = 42)

# Create the random search model
rs = RandomizedSearchCV(estimator, param_grid, n_jobs = -1, 
                        scoring = 'roc_auc', cv = 3, 
                        n_iter = 10, verbose = 5, random_state=42)

# Fit 
rs.fit(X_train_preprocessed_st, y_train)
rs.best_params_
# Create the model with 100 trees
model = RandomForestClassifier(n_estimators=196, 
                               random_state=42, 
                               min_samples_split=10,
                               max_leaf_nodes=49,
                               max_features=0.7,
                               max_depth=17,
                               bootstrap=True,
                               n_jobs=-1, verbose = 1)
# Fit on training data
model.fit(X_train_preprocessed_st, y_train)
y_pred = model.predict(X_test_preprocessed_st)
# Training predictions (to demonstrate overfitting)
train_rf_predictions = model.predict(X_train_preprocessed_st)
train_rf_probs = model.predict_proba(X_train_preprocessed_st)[:, 1]

# Testing predictions (to determine performance)
rf_predictions = model.predict(X_test_preprocessed_st)
rf_probs = model.predict_proba(X_test_preprocessed_st)[:, 1]
# Compute confusion matrix
cnf_matrix = confusion_matrix(y_test, y_pred, labels=[0,1])
np.set_printoptions(precision=2)

print (classification_report(y_test, y_pred))

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=['failed','successful'],normalize= False,  title='Confusion matrix')<jupyter_output>precision    recall  f1-score   support

           0       0.88      0.69      0.77      7732
           1       0.77      0.92      0.84      8676

    accuracy                           0.81     16408
   macro avg       0.83      0.80      0.81     16408
weighted avg       0.82      0.81      0.81     16408<jupyter_text>Use best model<jupyter_code>best_model = rs.best_estimator_
train_rf_predictions = best_model.predict(X_train_preprocessed_st)
train_rf_probs = best_model.predict_proba(X_train_preprocessed_st)[:, 1]

rf_predictions = best_model.predict(X_test_preprocessed_st)
rf_probs = best_model.predict_proba(X_test_preprocessed_st)[:, 1]
n_nodes = []
max_depths = []

for ind_tree in best_model.estimators_:
    n_nodes.append(ind_tree.tree_.node_count)
    max_depths.append(ind_tree.tree_.max_depth)
    
print(f'Average number of nodes {int(np.mean(n_nodes))}')
print(f'Average maximum depth {int(np.mean(max_depths))}')
def evaluate_model(predictions, probs, train_predictions, train_probs):
    """Compare machine learning model to baseline performance.
    Computes statistics and shows ROC curve."""
    
    baseline = {}
    
    baseline['recall'] = recall_score(y_test, [1 for _ in range(len(y_test))])
    baseline['precision'] = precision_score(y_test, [1 for _ in range(len(y_test))])
    baseline['roc'] = 0.5
    
    results = {}
    
    results['recall'] = recall_score(y_test, predictions)
    results['precision'] = precision_score(y_test, predictions)
    results['roc'] = roc_auc_score(y_test, probs)
    
    train_results = {}
    train_results['recall'] = recall_score(y_train, train_predictions)
    train_results['precision'] = precision_score(y_train, train_predictions)
    train_results['roc'] = roc_auc_score(y_train, train_probs)
    
    for metric in ['recall', 'precision', 'roc']:
        print(f'{metric.capitalize()} Baseline: {round(baseline[metric], 2)} Test: {round(results[metric], 2)} Train: {round(train_results[metric], 2)}')
    
    # Calculate false positive rates and true positive rates
    base_fpr, base_tpr, _ = roc_curve(y_test, [1 for _ in range(len(y_test))])
    model_fpr, model_tpr, _ = roc_curve(y_test, probs)

    plt.figure(figsize = (8, 6))
    plt.rcParams['font.size'] = 16
    
    # Plot both curves
    plt.plot(base_fpr, base_tpr, 'b', label = 'baseline')
    plt.plot(model_fpr, model_tpr, 'r', label = 'model')
    plt.legend();
    plt.xlabel('False Positive Rate'); plt.ylabel('True Positive Rate'); plt.title('ROC Curves');


evaluate_model(rf_predictions, rf_probs, train_rf_predictions, train_rf_probs)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11702402479979333, 'fraction_numerical': 0.031839318005683286, 'mean_word_length': 4.055501142670584, 'pattern_counts': {'":': 2, '<': 53, '<?xml version=': 0, '>': 54, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '50036', 'license': 'MIT', 'n_tokens_mistral': 5663, 'n_tokens_neox': 5299, 'n_words': 1085, 'path': 'Class_scaled_Gridsearch.ipynb', 'repo_name': 'olehansen538/MPO-Kickstarter', 'size': '15484'}