<jupyter_start><jupyter_text>ModelingML Tasks<jupyter_code>import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Input<jupyter_code>file_path = "../data/df_model01.pkl"
df = pd.read_pickle(file_path)

print(df.shape)
df.head()<jupyter_output>(75, 36)<jupyter_text>__Binning__<jupyter_code>bins = [10, 40, 120, 180] # SR, LNZ, SZG, VIE, >VIE

df["binned_distance"] = np.digitize(df.distance.values, bins=bins)<jupyter_output><empty_output><jupyter_text>__Conversion for Scikit-learn__Feature selection based on expert knowledge. Model-based hardly interpretable, at least confirmed "binned_distance" as relevant feature.<jupyter_code>feature_names = ["buzzwordy_title", "main_topic_Daten", "binned_distance"]
X = df[feature_names].values
y = df.rating.map(lambda x: 1 if x>5 else 0).values # binary target: >5 (better as all the same) was worth attending

print("X:", X.shape, "y:", y.shape)<jupyter_output>X: (75, 3) y: (75,)<jupyter_text>Train-Test Split<jupyter_code>from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=23, 
                                                    test_size=0.5) # 50% split, small dataset size<jupyter_output><empty_output><jupyter_text>Modeling Model<jupyter_code>from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeClassifier

linreg = LinearRegression() # Benchmark model
dec_tree = DecisionTreeClassifier() # Actual model<jupyter_output><empty_output><jupyter_text>Benchmark_Linear Regression_<jupyter_code># Benchmark
linreg.fit(X_train, y_train)
print("Score (r^2): {:.3f}".format(linreg.score(X_test, y_test)))
print("Coef: {}".format(linreg.coef_))<jupyter_output>Score (r^2): -0.220
Coef: [ 0.02485585  0.04313925  0.08000478]<jupyter_text>=> Really bad perfomance  Model_Decision Tree CV_<= Ability to visualize, hasn't performed much worse than other applied models __Parameter Tuning__<jupyter_code>from sklearn.model_selection import GridSearchCV

parameter_grid = {"criterion": ["gini", "entropy"], 
                  "max_depth": [None, 1, 2, 3, 4, 5, 6], 
                  "min_samples_leaf": list(range(1, 14)), 
                  "max_leaf_nodes": list(range(3, 25))}

grid_search = GridSearchCV(DecisionTreeClassifier(presort=True), parameter_grid, cv=5) # 5 fold cross-val
grid_search.fit(X_train, y_train)

print("Score (Accuracy): {:.3f}".format(grid_search.score(X_test, y_test)))
print("Best Estimator: {}".format(grid_search.best_estimator_))
print("Best Parameters: {}".format(grid_search.best_params_))<jupyter_output>Score (Accuracy): 0.632
Best Estimator: DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,
            max_features=None, max_leaf_nodes=5, min_impurity_split=1e-07,
            min_samples_leaf=2, min_samples_split=2,
            min_weight_fraction_leaf=0.0, presort=True, random_state=None,
            splitter='best')
Best Parameters: {'criterion': 'gini', 'max_depth': None, 'max_leaf_nodes': 5, 'min_samples_leaf': 2}<jupyter_text>=> Not that good accuracy, but at least better than random draw __Build Model__<jupyter_code>model = DecisionTreeClassifier(presort=True, criterion="gini", max_depth=None, 
                               min_samples_leaf=2, max_leaf_nodes=5)
model.fit(X_train, y_train)

print("Score (Accuracy): {:.3f}".format(model.score(X_test, y_test)))<jupyter_output>Score (Accuracy): 0.632<jupyter_text>__Print Decision Tree__<jupyter_code>from sklearn.tree import export_graphviz

export_graphviz(model, class_names=True, feature_names=feature_names,
                rounded=True, filled=True, label="root", impurity=False, proportion=True,
                out_file="plots/dectree_Model_best.dot")<jupyter_output><empty_output><jupyter_text>Model Evaluation __Evaluation Scores__<jupyter_code>from sklearn.metrics import classification_report

y_pred = model.predict(X_test)
print(classification_report(y_test, y_pred))<jupyter_output>precision    recall  f1-score   support

          0       0.71      0.77      0.74        26
          1       0.40      0.33      0.36        12

avg / total       0.62      0.63      0.62        38<jupyter_text>=> Not so good in predicting class 1 ("worth attending), better in predicting class 0. Weighted average of precision and recall 0.62 (F1 Score) __ROC Curve__<jupyter_code>from sklearn.metrics import roc_curve, roc_auc_score

fpr, tpr, thresholds = roc_curve(y_test, y_pred)

plt.plot(fpr, tpr, label="ROC")
plt.plot([0, 1], c="r", label="Chance level")
plt.title("ROC Curve")
plt.xlabel("False Positive Rate")
plt.ylabel("Recall")
plt.legend(loc=4)

plt.savefig("plots/ROC_Curve_Model.png", dpi=180)
plt.show()

print("AUC: {:.3f}".format(roc_auc_score(y_test, y_pred)))<jupyter_output><empty_output><jupyter_text>=> Indeed only slightly better than random guess, ~ 0.05 percentage points (AUC)<jupyter_code>from sklearn.model_selection import cross_val_score

scores_benchmark = cross_val_score(model, X, y, cv=5) # 5 folds

print("Cross-Val Scores (Accuracy): {}".format(scores_benchmark))
print("Cross-Val Mean (Accuracy): {}".format(scores_benchmark.mean()))<jupyter_output>Cross-Val Scores (Accuracy): [ 0.5625      0.625       0.46666667  0.71428571  0.64285714]
Cross-Val Mean (Accuracy): 0.6022619047619048<jupyter_text>=> Generalization acceptable, but not good (as expected from a Decision Tree)  Persist Model<jupyter_code>from sklearn.externals import joblib

file_path = "../data/model_trained.pkl"

joblib.dump(model, file_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1226950354609929, 'fraction_numerical': 0.03687943262411347, 'mean_word_length': 5.316909294512878, 'pattern_counts': {'":': 4, '<': 51, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '406082', 'license': 'Apache-2.0', 'n_tokens_mistral': 2169, 'n_tokens_neox': 1972, 'n_words': 400, 'path': 'EventDec/event_dec/notebook/2_Modeling.ipynb', 'repo_name': 'datadonK23/pyData-Projects', 'size': '5640'}