<jupyter_start><jupyter_text>Task 2 Prediction ModelIn this notebook I'll implement the (i) K-Means clustering (ii) Prediction Model<jupyter_code>import numpy as np 
import pandas as pd 
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
df_train = pd.read_pickle('../data/processed/multiclass/train_all_first.pickle')
df_train.head()
df_test = pd.read_pickle('../data/processed/multiclass/test_all_first.pickle')
df_test.head()
# Concat all df_train and df_test
# We use the whole DataFrame because we create own train and test sets
df = pd.concat([df_train, df_test])
assert len(df) == len(df_train) + len(df_test), 'Length is not the same!'
df.head()
!mkdir ../data/interim/task2/<jupyter_output><empty_output><jupyter_text>K-Means cluster on pid table<jupyter_code>df_pid = pd.read_csv('../data/raw/profiles.csv')
df_pid.head()
y_hat = KMeans(n_clusters=4).fit_predict(df_pid.iloc[:,1:])
df_pid_labels = df_pid.assign(label = y_hat)
plt.bar(df_pid_labels.label.unique(), df_pid_labels.groupby('label').count()['pid']);
plt.xlabel('Unique label');
plt.ylabel('Count of entries');
plt.xticks(range(0,4));
plt.title("Nr. of entries in each label");
label_matrix = df_pid_labels[['pid', 'label']]
label_matrix.columns = ['pid', 'kmeans_4']
label_matrix.to_pickle('../data/interim/label_matrix.pickle')<jupyter_output><empty_output><jupyter_text>Prepare Data- Delete all without PID- Random test, train set<jupyter_code>count_na = df[df.pid == -1].shape[0]
y = [(len(df) - count_na), count_na]

y_percentage = [(len(df) - count_na) / (len(df)), count_na / len(df)]

fig, ax = plt.subplots();
ax.bar(['PID available', 'PID not available'], y);

rects = ax.patches



for rect, label1, label2 in zip(rects, y, y_percentage):
    height = rect.get_height()
    ax.text(rect.get_x() + rect.get_width() / 2, height + 5, "{:,} | {:.2f}%".format(label1, (label2*100)),
            ha='center', va='bottom')
# Delete all without PID
df_processed = df[df.pid != -1]
df_processed.shape
df_processed.to_pickle('../data/interim/task2/df_pp.pickle')
df_pid_labels.head()
# Join df_processed and df_pid_labels to get the labels
df_processed = pd.merge(df_processed, df_pid_labels[['pid', 'label']], on='pid')
df_processed.to_pickle('../data/processed/multiclass/task2_df_example.pickle')<jupyter_output><empty_output><jupyter_text>Read in feature file<jupyter_code>features = pd.read_pickle('../data/processed/features/task_2.pickle')
df_processed = df_processed[features]
df_processed.head()<jupyter_output><empty_output><jupyter_text>Test, train set<jupyter_code>from sklearn.model_selection import train_test_split
X = df_processed.iloc[:,:-1].values
y = df_processed.iloc[:, -1].values
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.25, random_state=42, shuffle=True)<jupyter_output><empty_output><jupyter_text>Random Forest Model<jupyter_code>from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(n_estimators=100)
rf.fit(X_train, y_train)
y_preds = rf.predict(X_test)
from sklearn.metrics import f1_score
f1_score(y_test, y_preds, average='weighted')<jupyter_output><empty_output><jupyter_text>K-Fold<jupyter_code>from sklearn.model_selection import KFold

kf = KFold(n_splits=5, random_state=42, shuffle=True)
kf.get_n_splits(X, y)

print(kf)  

count_iter = 0

for train_index, test_index in kf.split(X, y):
    count_iter += 1
    print('Split Nr. {}'.format(count_iter))
    print("Train size: {}, Test size: {}".format(len(train_index), len(test_index)))
    X_train, X_test = X[train_index], X[test_index]
    y_train, y_test = y[train_index], y[test_index]
    rf = RandomForestClassifier(n_estimators=10)
    rf.fit(X_train, y_train)
    y_preds = rf.predict(X_test)
    print("F1 Score at split {} is {:.4f} %".format(count_iter, f1_score(y_test, y_preds, average='weighted')))
    print("====================================================================")<jupyter_output>Split Nr. 1
Train size: 268816, Test size: 67205
F1 Score at split 1 is 0.4497 %
====================================================================
Split Nr. 2
Train size: 268817, Test size: 67204
F1 Score at split 2 is 0.4442 %
====================================================================
Split Nr. 3
Train size: 268817, Test size: 67204
F1 Score at split 3 is 0.4425 %
====================================================================
Split Nr. 4
Train size: 268817, Test size: 67204
F1 Score at split 4 is 0.4473 %
====================================================================
Split Nr. 5
Train size: 268817, Test size: 67204
F1 Score at split 5 is 0.4471 %
====================================================================
{'fraction_non_alphanumeric': 0.217678381256656, 'fraction_numerical': 0.029818956336528223, 'mean_word_length': 6.406940063091483, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '246425', 'license': 'MIT', 'n_tokens_mistral': 1798, 'n_tokens_neox': 1660, 'n_words': 354, 'path': 'notebooks/130_task2_prediction_model.ipynb', 'repo_name': 'AlessandroVol23/kdd-cup-2019', 'size': '4695'}