<jupyter_start><jupyter_text>Splitting the Dataset and Feature Scaling<jupyter_code>X_frame  = gen_model_for(post_dataset)
X = X_frame.iloc[:, 2:-1].values
y = X_frame.iloc[:, -1].values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
print(f'Training dataset is: {len(X_train)} rows')
train_X, test_X = X_train, X_test<jupyter_output><empty_output><jupyter_text>Dimensionality Reduction<jupyter_code>def do_lda(train_X, train_Y, test_X):
    lda = LinearDiscriminantAnalysis(n_components=2)
    train_X = lda.fit_transform(train_X, train_Y)
    test_X = lda.transform(test_X)
    return train_X, test_X
pca = PCA(n_components=None)  
train_X = pca.fit_transform(X_train)
test_X = pca.transform(X_test)
# cumsum = np.cumsum(pca.explained_variance_ratio_)
# trace = go.Scatter(x=[i for i in range(len(cumsum))], y=cumsum,
#                      marker=dict(color='rgb(150, 25, 120)'))
# fig = go.Figure(data=[trace])
# plot(fig, filename='cdf-dataset')
kpca = KernelPCA(kernel='rbf', n_components=3)  
train_X = kpca.fit_transform(X_train)
test_X = kpca.transform(X_test)<jupyter_output><empty_output><jupyter_text>Visualizing the Principal Components<jupyter_code>def visualize_principal_axis(X, Y, title):
    trace0_x, trace0_y, trace1_x, trace1_y = [], [], [], []
    for x_val, y_val in zip(X, Y):
        if y_val == 0:
            trace0_x.append(x_val[0])
            trace0_y.append(x_val[1])
        else:
            trace1_x.append(x_val[0])
            trace1_y.append(x_val[1])
    trace0 = go.Scatter(
        x = trace0_x,
        y = trace0_y,
        mode = 'markers',
        name="Not Credible"
    )
    trace1 = go.Scatter(
        x = trace1_x,
        y = trace1_y,
        mode = 'markers',
        name="Credible"
    )
    layout = dict(
        title=title,
    )
    return dict(data=[trace0, trace1], layout=layout)

fig = visualize_principal_axis(X_test, y_train, "PCA")
plot(fig, filename='pca')<jupyter_output><empty_output><jupyter_text>Performance Metrics<jupyter_code>def run_metrics(estimator, trainX, testX, trainY, testY):
    # Performance Evaluation - Confusion Matrix Test Evaluation
    pred_test_y = estimator.predict(testX)
    cm = confusion_matrix(testY, pred_test_y)
    print(f'Test Data Confusion Matrix: {cm}')
    # Performance Evaluation - Confusion Matrix Train Evaluation
    pred_train_y = estimator.predict(trainX)
    cm = confusion_matrix(trainY, pred_train_y)
    print(f'Train Data Confusion Matrix: {cm}')
    # Performance Evaluation - K Means
    accuracies = cross_val_score(estimator = estimator, X=trainX, y=trainY, cv=10)
    print(f'K-Means Accuracy Average: {accuracies.mean()}')
    print(f'K-Means Variance: {accuracies.std()}')<jupyter_output><empty_output><jupyter_text>SVM Testbench<jupyter_code>classifier = SVC(kernel = 'rbf', C=2.1, gamma= 0.04, probability=True)
classifier.fit(X_train, y_train)
run_metrics(classifier, X_train, X_test, y_train, y_test)<jupyter_output><empty_output><jupyter_text>Hyper parameter Tuning Grid Search: SVM<jupyter_code># C_vals = [0.1 * i+1 for i in range(20)]
def num_generator(start_num, intervals, interval_size):
    nums = [start_num]
    while intervals > 0:
        nums.append(nums[-1] + interval_size)
        intervals-=1
    return nums
C_vals = num_generator(0.1, 20, 0.1)
gamma = num_generator(0, 10, 0.01)
parameters = [
#     {
#         'C': C_vals,
#         'kernel': ['linear']
#     },
    {
        'C': C_vals,
        'kernel': [
            'rbf',
            'poly',
            'sigmoid',
        ],
        'gamma': gamma
    }
]
grid_search = GridSearchCV(estimator=classifier, param_grid=parameters, scoring='accuracy', cv=10, n_jobs=-1, verbose=10)
grid_search = grid_search.fit(X_train, y_train)
print(grid_search.best_score_, grid_search.best_params_)<jupyter_output><empty_output><jupyter_text>Random Forest  Hyper Parameter Tuning RandomForest<jupyter_code>parameters = [
    {
        'n_estimators': [i for i in range(1, 100)],
        'criterion': ['entropy']
    }
]
classifier = RandomForestClassifier()
grid_search = GridSearchCV(estimator=classifier, param_grid=parameters, scoring='accuracy', cv=10, n_jobs=-1, verbose=10)
grid_search = grid_search.fit(X_train, y_train)
best_accuracy = grid_search.best_score_
print(grid_search.best_score_, grid_search.best_params_)
classifier = RandomForestClassifier(n_estimators = 47, criterion = 'entropy', random_state = 0)
classifier.fit(X_train, y_train)
run_metrics(classifier, X_train, X_test, y_train, y_test)<jupyter_output><empty_output><jupyter_text>XGBoost<jupyter_code>classifier = XGBClassifier(learning_rate=0.21, max_depth=4, n_estimators=138)
classifier.fit(X_train, y_train)
run_metrics(classifier, X_train, X_test, y_train, y_test)
classifier = XGBClassifier()
classifier.fit(X_train, y_train)
run_metrics(classifier, X_train, X_test, y_train, y_test)
importances = Df({'importance': classifier.feature_importances_, 'col_names': list(X_frame.columns[2:-1])})
importances.sort_values(by=['importance'], ascending=False)<jupyter_output><empty_output><jupyter_text>Visualize Boosted Tree<jupyter_code>plot_tree(classifier)
fig = plt.gcf()
fig.set_size_inches(40, 40)
fig.savefig('tree.png')<jupyter_output><empty_output><jupyter_text>Feature Importances in Boosted Tree<jupyter_code>feature_importances = Df({'columns': X_frame.columns[2:-1], 'importances': classifier.feature_importances_})
feature_importances.sort_values(by=['importances'], ascending=False)<jupyter_output><empty_output><jupyter_text>Hyper Parameter Tuning XGBoost<jupyter_code>learning_rates = num_generator(0.1, 20, 0.01)
num_estimators = num_generator(90, 20, 3)
learning_rates, num_estimators
parameters = [
    {
        'max_depth': [i for i in range(5)],
        'learning_rate': learning_rates,
        'n_estimators': num_estimators,        
    }
]
classifier = XGBClassifier()
grid_search = GridSearchCV(estimator=classifier, param_grid=parameters, scoring='accuracy', cv=10, n_jobs=-1, verbose=10)
grid_search = grid_search.fit(X_train, y_train)
best_accuracy = grid_search.best_score_
print(grid_search.best_score_, grid_search.best_params_)<jupyter_output><empty_output><jupyter_text>CAP Curve<jupyter_code>best_xgb = XGBClassifier(learning_rate=0.21, max_depth=4, n_estimators=138)
best_svm = SVC(kernel = 'rbf', C=2.1, gamma= 0.04, probability=True)
best_random_forest = RandomForestClassifier(n_estimators = 47, criterion = 'entropy', random_state = 0)
def get_model_for_classifier(classifier, name):
    classifier.fit(X_train, y_train)
    test_data = Df(X_test)
    total = len(test_data)
    test_data['actual'] = y_test
    test_data['predicted'] = classifier.predict_proba(X_test)[:, 1]
    prob_sorted = test_data.sort_values(by=['predicted'], ascending=False)
    model_y = list(prob_sorted.actual) 
    return go.Scatter(
        x=np.arange(0, total + 1),
        y=np.append([0], np.cumsum(model_y)),
        name=name,
        mode='lines'
    )


def get_model_random():
    unique, counts = np.unique(y_test, return_counts=True)
    class_1_count = dict(zip(unique, counts))[1]    
    return {
        "type": "line",
        "x0": 0,
        "x1": len(y_test),
        "y0": 0,
        "y1": class_1_count,
        "line": {
            "color": 'rgb(256, 0, 0)',
            "width": 4,
            "dash": "dot"
        }
    }


def get_model_perfect():
    unique, counts = np.unique(y_test, return_counts=True)
    class_1_count = dict(zip(unique, counts))[1]    
    return go.Scatter(
        x=[0, class_1_count, len(y_test)],
        y= [0, class_1_count, class_1_count],
        mode='lines',
        name='Perfect Model'
    )

def do_plot():
    plt.figure(figsize=(20, 12))


    fig = {
        "data": [
            get_model_for_classifier(best_xgb, 'Tuned XGBoost'),
            get_model_for_classifier(best_svm, 'Tuned SVM'),
            get_model_for_classifier(best_random_forest, 'Tuned Random Forest'),
            get_model_perfect()
        ],
        "layout": go.Layout(
            title=go.layout.Title(text="CAP Curves"),
            xaxis=go.layout.XAxis(title=go.layout.xaxis.Title(text='Observations')),
            yaxis=go.layout.YAxis(title=go.layout.yaxis.Title(text='Misinformative Observations')),
            shapes=[get_model_random()]
        )
    }
    plot(fig, filename='CAP Curves of Various Models')
#     offline.plot(fig, image='png', image_filename='CAP Curves', output_type='file')
do_plot()
def plot_error_rates():
    misinfo_missed = [32.8, 27.14, 14.1, 7.8, 12, 9.2]
    false_positives = [6.1, 4.83, 0.7, 0.2, 0.5, 0.5]
    labels = ['Kernel SVM', 'Tuned SVM', 'XGB', 'Tuned XGB', 'Random Forest', 'Tuned Random Forest']
    trace0 = go.Bar(
        x=labels,
        y=false_positives,
        text=false_positives,
        textposition = 'auto',
        marker=dict(
        color='rgb(158,202,225)',
        line=dict(
            color='rgb(8,48,107)',
            width=1.5),
        ),
        name='False Positives / Type 1 Error'
    )
    trace1 = go.Bar(
        x=labels,
        y=misinfo_missed,
        text=misinfo_missed,
        textposition = 'auto',
        name='False Negatives / Type 2 Error',
        marker=dict(
        color='rgb(58,200,225)',
        line=dict(
            color='rgb(8,48,107)',
            width=1.5),
        ),
    )
    layout = go.Layout(
        title=go.layout.Title(text="Error Types and Counts across Models"),
        xaxis=go.layout.XAxis(title=go.layout.xaxis.Title(text='Model Type')),
        yaxis=go.layout.YAxis(title=go.layout.yaxis.Title(text='Error %')),
    )
    fig = {'data': [trace0, trace1], 'layout': layout}
    plot(fig, filename='Error Rate Bar Chart')
#     offline.plot(fig, image='png', image_filename='error_rate_bar', output_type='file')
plot_error_rates()
plot_tree(best_xgb)
fig = plt.gcf()
fig.set_size_inches(40, 40)
fig.savefig('tree.png')
feature_importances = Df({'columns': X_frame.columns[2:-1], 'importances': best_xgb.feature_importances_})
feature_importances.sort_values(by=['importances'], ascending=False).head<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13725681884837226, 'fraction_numerical': 0.02277837520774269, 'mean_word_length': 4.017165277096616, 'pattern_counts': {'":': 11, '<': 49, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '207754', 'license': 'MIT', 'n_tokens_mistral': 3914, 'n_tokens_neox': 3719, 'n_words': 625, 'path': 'Model.ipynb', 'repo_name': 'andersonpaac/Credd', 'size': '10229'}