<jupyter_start><jupyter_text>Import libraries & dataset<jupyter_code>import pandas as pd
import numpy as np
import datetime
import matplotlib.pyplot as plt
import time
import statistics as stats
import requests
import pickle
import json

from sklearn.svm import LinearSVC
import xgboost as xgb
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
import tensorflow as tf

from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.compose import ColumnTransformer
from sklearn.feature_selection import SelectFromModel
from sklearn.metrics import classification_report
from sklearn.base import clone
from sklearn.externals.joblib import dump, load

from mlxtend.feature_selection import ExhaustiveFeatureSelector as EFS
from imblearn.over_sampling import SMOTE, SMOTENC, ADASYN, BorderlineSMOTE
from imblearn.under_sampling import NearMiss, RandomUnderSampler

df = pd.read_csv('http://puma.swstats.info/files/kickstarter_with_trends.csv', index_col="ID")
df.columns<jupyter_output>/usr/local/lib/python3.6/dist-packages/sklearn/externals/joblib/__init__.py:15: FutureWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.
  warnings.warn(msg, category=FutureWarning)
/usr/local/lib/python3.6/dist-packages/sklearn/externals/six.py:31: FutureWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).
  "(https://pypi.org/project/six/).", FutureWarning)
/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.neighbors.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functio[...]<jupyter_text>Prepare for data cleaning<jupyter_code>link = 'https://3l7z4wecia.execute-api.us-east-1.amazonaws.com/default/api-dynamodb/'

get_categories = {
  "operation": "list",
  "table": "categories",
}

categories = requests.post(link, json=get_categories)

categories = categories.json()['Items']
categories_proper = dict()
for item in categories:
  categories_proper[item['name']] = item['id'] # map NAME to ID

get_main_categories = {
  "operation": "list",
  "table": "maincategories",
}

main_categories = requests.post(link, json=get_main_categories)

main_categories = main_categories.json()['Items']
main_categories_proper = dict()
for item in main_categories:
  main_categories_proper[item['name']] = item['id'] # map NAME to ID

get_countries = {
  "operation": "list",
  "table": "countries",
}

countries = requests.post(link, json=get_countries)

countries = countries.json()['Items']
countries_proper = dict()
for item in countries:
  countries_proper[item['name']] = item['id'] # map NAME to ID<jupyter_output><empty_output><jupyter_text>Clean & prepare data* Calculate campaign length * Delete all incomplete data (like country == N,0")* Delete all kickstarter projects with different state than 'failed' and 'successful' * Cast to numerical types all non-numerical features and drop all empty data* Use Label Encoding or One-Hot Encoding<jupyter_code>df_clean = df.copy()

indexes = df_clean[df_clean['country'] == 'N,0"'].index
df_clean.drop(indexes, inplace=True)
# drop live & undefined states
indexes = df_clean[(df_clean['state'] == 'live') | (df_clean['state'] == 'undefined')].index
df_clean.drop(indexes, inplace=True)

df_clean['campaign_length'] = pd.to_timedelta((pd.to_datetime(df_clean['deadline']) - pd.to_datetime(df_clean['launched'])), unit='days').dt.days
# df_clean = df_clean[(df_clean['usd_goal_real'] >= 10) & (df_clean['campaign_length'] >= 7)] # drop all with lower goal than 10$ and shorter than week

##########################################################
# """ Label Encoding - if you want to run this, just comment lines with quotation marks
jsons = dict()
map_dict = {
    'category': categories_proper,
    'main_category': main_categories_proper,
    'country': countries_proper,
}
for key, val in map_dict.items():
  df_clean[key] = df_clean[key].map(val)

json.dump(jsons, open('categories.json', 'w'))

df_clean.drop(['tokenized_name', 'currency', 'name'], inplace=True, axis=1)
df_clean.dropna(inplace=True)
# """
##########################################################

###########################################################
""" One-Hot Encoding - if you want to run this, just comment lines with quotation marks
column_transformer = ColumnTransformer([('encoder', OneHotEncoder(), ['category', 'main_category', 'currency', 'country'])], sparse_threshold=0, n_jobs=-1)
onehot = pd.DataFrame(column_transformer.fit_transform(df_clean)).set_index(df_clean.index)
new_cols_encoding = [col.replace('encoder__x0_', '').replace('encoder__x1_', '').replace('encoder__x2_', '').replace('encoder__x3_', '') for col in column_transformer.get_feature_names()]
onehot.columns = new_cols_encoding
df_clean = pd.concat([df_clean, onehot], axis=1)
df_clean.drop(['category', 'main_category', 'currency', 'country', 'tokenized_name'], inplace=True, axis=1)
df_clean = df_clean.loc[:,~df_clean.columns.duplicated()]
"""
##########################################################

df_xd = df_clean[~df_clean['state'].str.contains('successful')].index
df_clean.loc[df_clean['state'].str.contains('successful'), 'state'] = 1
df_clean.loc[df_xd, 'state'] = 0
df_clean['state'] = df_clean['state'].astype(int)


df_clean<jupyter_output><empty_output><jupyter_text>Check features correlationWe say features are dependant, if abs(correlation) > .5<jupyter_code>corr = df_clean.corr()

plt.matshow(corr)
plt.show()

corr[(corr > .5) | (corr < -.5)]<jupyter_output><empty_output><jupyter_text>Delete unnecessary featuresWe delete dupe features (like converted goal value) and the ones that user won't be able to provide, like backers.<jupyter_code>df_shortened = df_clean.copy()
df_shortened.drop(['pledged', 'backers', 'usd pledged', 'deadline', 'launched', 'usd_pledged_real', 'goal'], axis=1, inplace=True)
df_shortened<jupyter_output><empty_output><jupyter_text>Split dataSplit data for training & test set, with 10% being in test set. 30k is enough for testing.<jupyter_code>X = df_shortened.drop('state', axis=1)
y = df_shortened['state']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.1, random_state=2137) # 90%:10%
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=.1, random_state=2137) # 81%:9% -> 90%
X_train<jupyter_output><empty_output><jupyter_text>Data Over/Undersampling<jupyter_code>print(pd.Series(train_data['y']).value_counts())
def sample_data(sampler, X_train, y_train, cols):
    start = time.time()
    X_train_new, y_train_new = sampler.fit_sample(X_train, y_train)
    X_train_new = pd.DataFrame(X_train_new)
    X_train_new.columns = cols
    print(f"SMOTENC done in {round(time.time() - start, 2)} seconds")

    return {
      'x': X_train_new,
      'y': y_train_new,
    }


train_data = sample_data(SMOTENC([0, 1, 2], n_jobs=-1), X_train, y_train, X_train.columns)
test_data = { 'x': X_test, 'y': y_test }
val_data = { 'x': X_val, 'y': y_val }

print(pd.Series(train_data['y']).value_counts())
print(pd.Series(test_data['y']).value_counts())
print(pd.Series(val_data['y']).value_counts())<jupyter_output>/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.
  warnings.warn(msg, category=FutureWarning)
/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function safe_indexing is deprecated; safe_indexing is deprecated in version 0.22 and will be removed in version 0.24.
  warnings.warn(msg, category=FutureWarning)<jupyter_text>(Optional) Delete all irrelevant featuresDelete all irrelevant features, but keep AT MAX 5<jupyter_code>""" If you want to use this cell, just comment lines with quotation marks at the beginning
logistic = LogisticRegression(C=1, penalty="l2", max_iter=1000).fit(X_train, y_train)
model = SelectFromModel(logistic, prefit=True, max_features=5)

X_new = model.transform(X_train)

selected_features = pd.DataFrame(model.inverse_transform(X_new), index=X_train.index, columns=X_train.columns)
selected_columns = selected_features.columns[selected_features.var() != 0]

X_train = X_train[selected_columns]
X_test = X_test[selected_columns]

selected_features
"""<jupyter_output><empty_output><jupyter_text>Standarization & min-max scalingStandarization -> mean-stdMin-Max scaling -> min-max<jupyter_code>def standarize(X_train, X_test, X_val):
    cols = X_train.columns
    indexes_x_train = X_train.index
    indexes_x_test = X_test.index
    indexes_x_val = X_val.index

    X_train_categorical = X_train[['category', 'main_category', 'country']]
    X_test_categorical = X_test[['category', 'main_category', 'country']]
    X_val_categorical = X_val[['category', 'main_category', 'country']]

    scaler = StandardScaler()
    scaler.fit(X_train.drop(['category', 'main_category', 'country'], axis=1))
    
    X_train = pd.concat([X_train_categorical, pd.DataFrame(scaler.transform(X_train.drop(['category', 'main_category', 'country'], axis=1))).set_index(indexes_x_train)], axis=1)    
    X_test = pd.concat([X_test_categorical, pd.DataFrame(scaler.transform(X_test.drop(['category', 'main_category', 'country'], axis=1))).set_index(indexes_x_test)], axis=1)
    X_val = pd.concat([X_val_categorical, pd.DataFrame(scaler.transform(X_val.drop(['category', 'main_category', 'country'], axis=1))).set_index(indexes_x_val)], axis=1)
    
    X_train.columns = cols
    X_test.columns = cols
    X_val.columns = cols

    return X_train, X_test, X_val, scaler

train_data['x'], test_data['x'], val_data['x'], standarizer = standarize(train_data['x'], test_data['x'], val_data['x'])
test_data['x']<jupyter_output><empty_output><jupyter_text>Load Standarizer (Scaler) from Web Server Deep Learning<jupyter_code>! pip install -q tensorflow-model-optimization
from tensorflow_model_optimization.sparsity import keras as sparsity
l = tf.keras.layers
batch_size = 1024
epochs = 500
end_step = np.ceil(1.0 * train_data['x'].shape[0] / batch_size).astype(np.int32) * epochs
pruning_params = {
      'pruning_schedule': sparsity.PolynomialDecay(initial_sparsity=0.01,
                                                   final_sparsity=0.2,
                                                   begin_step=round(end_step/epochs/2),
                                                   end_step=end_step,
                                                   frequency=end_step/epochs)
}
tf.random.set_seed(2137)
pruned_model = tf.keras.Sequential([
    sparsity.prune_low_magnitude(
        tf.keras.layers.Dense(12, input_dim=train_data['x'].shape[1], activation='selu'),
        **pruning_params),
    l.BatchNormalization(),
    sparsity.prune_low_magnitude(
        tf.keras.layers.Dense(12, activation='relu'),**pruning_params),
    l.Flatten(),
    sparsity.prune_low_magnitude(
        tf.keras.layers.Dense(12*train_data['x'].shape[1], activation='selu'),**pruning_params),
    l.Dropout(0.001),
    sparsity.prune_low_magnitude(tf.keras.layers.Dense(1, activation='sigmoid'),
                                 **pruning_params)
])

pruned_model.summary()

pruned_model.compile(
    loss=tf.keras.losses.binary_crossentropy,
    optimizer='Adam',
    metrics=['accuracy'])

# Add a pruning step callback to peg the pruning step to the optimizer's
# step. Also add a callback to add pruning summaries to tensorboard
callbacks = [
    sparsity.UpdatePruningStep(),
    sparsity.PruningSummaries(log_dir=logdir, profile_batch=0)
]

pruned_model.fit(train_data['x'],train_data['y'],
          batch_size=batch_size,
          epochs=epochs,
          verbose=1,
          callbacks=callbacks,
          validation_data=(val_data['x'],val_data['y']))

score = pruned_model.evaluate(test_data['x'],test_data['y'], verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
# Zapisanie modelu sieci
pruned_model.save('Model_Sieci_Glebokiego_Uczenia')
# Wczytanie modelu sieci
siec = tf.keras.models.load_model('Model_Sieci_Glebokiego_Uczenia')

import seaborn as sns
y_pred=siec.predict_classes(train_data['x'])
con_mat = tf.math.confusion_matrix(labels=train_data['y'], predictions=y_pred).numpy()

con_mat_norm = np.around(con_mat.astype('float') / con_mat.sum(axis=1)[:, np.newaxis], decimals=2)
 
con_mat_df = pd.DataFrame(con_mat_norm)


figure = plt.figure(figsize=(8, 8))
sns.heatmap(con_mat_df, annot=True,cmap=plt.cm.Blues)
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1474109662167315, 'fraction_numerical': 0.013040494166094716, 'mean_word_length': 5.6602336211274755, 'pattern_counts': {'":': 6, '<': 40, '<?xml version=': 0, '>': 46, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '594095', 'license': 'MIT', 'n_tokens_mistral': 4544, 'n_tokens_neox': 4370, 'n_words': 895, 'path': 'Deep_Learning_Classifier_model.ipynb', 'repo_name': 'QuatZo/kickstarter-predictor', 'size': '13113'}