<jupyter_start><jupyter_text>Model using data aggregation<jupyter_code>df_aggregate_data = df_positions_2019.loc[
    (df_positions_2019['activity_label'].isin(['Standby', 'DP', 'Port', 'TransitCombine'])), 
    ['id', 'vessel_id', 'speed','new_position_received_time',
     'prev_receive_time', 'prev_coord', 'curr_coord', 'mile_since','hour_since', 
     'speed_nm', 'fuel_consumption_average', 'activity_label', 'activity_label2',
     'time_period', 'ref_date', 'transit_type', 'daily_vessel_id']].sort_values(by=['vessel_id', 
                                                                                    'new_position_received_time'])
df_aggregate_data['ref_date'] = pd.to_datetime(df_aggregate_data.ref_date, errors='coerce').dt.date
df_aggregate_data = df_aggregate_data.groupby(by=['daily_vessel_id']).agg({'mile_since': 'sum',
                                                                           'hour_since': 'sum',
                                                                           'new_position_received_time': 'min',
                                                                           'speed': ['max', 'mean'], 
                                                                           'speed_nm': ['max', 'median', 'mean'], 
                                                                           'fuel_consumption_average': ['max', 'mean'],
                                                                           'curr_coord': ['first', 'last'],
                                                                           'activity_label': 'first',
                                                                           'activity_label2': 'first',
                                                                           'transit_type': 'first'
                                                                 }).reset_index(col_level=0)

df_aggregate_data.columns = ['_'.join(col).strip() for col in df_aggregate_data.columns.values]
df_aggregate_data['period_total_distance'] = df_aggregate_data.apply(lambda x: gd.distance(x['curr_coord_first'],
                                                                                          x['curr_coord_last']
                                                                                          ).nm, axis=1) 
df_aggregate_data.loc[df_aggregate_data['period_total_distance'] == 0, 'period_total_distance'] = 0.00001
df_aggregate_data['distance_ratio'] = df_aggregate_data.mile_since_sum/df_aggregate_data.period_total_distance

df_aggregate_data.to_pickle("data/[PSV]df_aggregate_data_1.pkl")
df_aggregate_data = pd.read_pickle("./data/[PSV]df_aggregate_data_1.pkl")
df_vessel_daily_data = pd.read_pickle("./data/df_vessel_daily_2019_complete.pkl")
df_aggregate_data.columns
df_aggregate_data = df_aggregate_data.loc[(df_aggregate_data['activity_label_first'].isin(['Standby', 'DP', 'Port', 'TransitCombine']))
                                          &
                                          (df_aggregate_data['new_position_received_time_min']<'2019-12-22') 
                                          , ['activity_label_first', 'transit_type_first',
                                                                                                             'hour_since_sum',
                                                                                                             'mile_since_sum',
                                                                                                             'period_total_distance',
                                                                                                             'distance_ratio',
                                                                                                             'fuel_consumption_average_max', 
                                                                                                             'fuel_consumption_average_mean',
                                                                                                             'speed_nm_max', 'speed_nm_median', 
                                                                                                             'speed_nm_mean',
                                                                                                             'daily_vessel_id_',
                                                                                                             'new_position_received_time_min'
                                                                                                    ]]

df_aggregate_data.loc[df_aggregate_data['transit_type_first'].isin(['Transit Max']), 'activity_label_first'] = df_aggregate_data['transit_type_first']
# df_aggregate_data.activity_label_first.value_counts()
df_aggregate_data.count()
Q1_DP = df_aggregate_data.loc[(df_aggregate_data['activity_label_first'] == 'DP'), ['fuel_consumption_average_mean','distance_ratio']].quantile(0.25)
Q3_DP = df_aggregate_data.loc[(df_aggregate_data['activity_label_first'] == 'DP'), ['fuel_consumption_average_mean','distance_ratio']].quantile(0.75)
IQR_DP = Q3_DP - Q1_DP

Q1_Port = df_aggregate_data.loc[(df_aggregate_data['activity_label_first'] == 'Port'), ['fuel_consumption_average_mean', 'period_total_distance']].quantile(0.25)
Q3_Port = df_aggregate_data.loc[(df_aggregate_data['activity_label_first'] == 'Port'), ['fuel_consumption_average_mean', 'period_total_distance']].quantile(0.75)
IQR_Port = Q3_Port - Q1_Port

Q1_Standby = df_aggregate_data.loc[(df_aggregate_data['activity_label_first'] == 'Standby'), ['fuel_consumption_average_mean','distance_ratio']].quantile(0.25)
Q3_Standby = df_aggregate_data.loc[(df_aggregate_data['activity_label_first'] == 'Standby'), ['fuel_consumption_average_mean','distance_ratio']].quantile(0.75)
IQR_Standby = Q3_Standby - Q1_Standby

# Q1_TransitEco = df_aggregate_data.loc[(df_aggregate_data['activity_label_first'] == 'Transit Eco'), ['fuel_consumption_average_mean', 'period_total_distance', 'speed_nm_mean', 'distance_ratio']].quantile(0.25)
# Q3_TransitEco = df_aggregate_data.loc[(df_aggregate_data['activity_label_first'] == 'Transit Eco'), ['fuel_consumption_average_mean', 'period_total_distance', 'speed_nm_mean', 'distance_ratio']].quantile(0.75)
# IQR_TransitEco = Q3_TransitEco - Q1_TransitEco

Q1_Transit = df_aggregate_data.loc[(df_aggregate_data['activity_label_first'] == 'TransitCombine'), ['fuel_consumption_average_mean','distance_ratio']].quantile(0.25)
Q3_Transit = df_aggregate_data.loc[(df_aggregate_data['activity_label_first'] == 'TransitCombine'), ['fuel_consumption_average_mean','distance_ratio']].quantile(0.75)
IQR_Transit = Q3_Transit - Q1_Transit

Q1_TransitMax = df_aggregate_data.loc[(df_aggregate_data['activity_label_first'] == 'Transit Max'), ['hour_since_sum', 'mile_since_sum','period_total_distance', 'distance_ratio','fuel_consumption_average_max', 'fuel_consumption_average_mean','speed_nm_max', 'speed_nm_median', 'speed_nm_mean']].quantile(0.25)
Q3_TransitMax = df_aggregate_data.loc[(df_aggregate_data['activity_label_first'] == 'Transit Max'), ['hour_since_sum', 'mile_since_sum','period_total_distance', 'distance_ratio','fuel_consumption_average_max', 'fuel_consumption_average_mean','speed_nm_max', 'speed_nm_median', 'speed_nm_mean']].quantile(0.75)
IQR_TransitMax = Q3_TransitMax - Q1_TransitMax
df_aggregate_data = df_aggregate_data[
   ((df_aggregate_data['activity_label_first'] == 'DP') & (~((df_aggregate_data[['fuel_consumption_average_mean','distance_ratio']] < (Q1_DP - 1.5 * IQR_DP)) | (df_aggregate_data[['fuel_consumption_average_mean','distance_ratio']] > (Q3_DP + 1.5 * IQR_DP))).any(axis=1)))
   |
   ((df_aggregate_data['activity_label_first'] == 'Port') & (~((df_aggregate_data[['fuel_consumption_average_mean', 'period_total_distance']] < (Q1_Port - 1.5 * IQR_Port)) | (df_aggregate_data[['fuel_consumption_average_mean', 'period_total_distance']] > (Q3_Port + 1.5 * IQR_Port))).any(axis=1)))
   |
   ((df_aggregate_data['activity_label_first'] == 'Standby') & (~((df_aggregate_data[['fuel_consumption_average_mean','distance_ratio']] < (Q1_Standby - 1.5 * IQR_Standby)) | (df_aggregate_data[['fuel_consumption_average_mean','distance_ratio']] > (Q3_Standby + 1.5 * IQR_Standby))).any(axis=1)))
#     |
#     ((df_aggregate_data['activity_label_first'] == 'Transit Eco') & (~((df_aggregate_data < (Q1_TransitEco - 1.5 * IQR_TransitEco)) | (df_aggregate_data > (Q3_TransitEco + 1.5 * IQR_TransitEco))).any(axis=1)))
    |
    ((df_aggregate_data['activity_label_first'] == 'TransitCombine') & (~((df_aggregate_data[['fuel_consumption_average_mean','distance_ratio']] < (Q1_Transit - 1.5 * IQR_Transit)) | (df_aggregate_data[['fuel_consumption_average_mean','distance_ratio']] > (Q3_Transit + 1.5 * IQR_Transit))).any(axis=1)))
    |
    ((df_aggregate_data['activity_label_first'] == 'Transit Max') & (~((df_aggregate_data[['hour_since_sum', 'mile_since_sum','period_total_distance', 'distance_ratio','fuel_consumption_average_max', 'fuel_consumption_average_mean','speed_nm_max', 'speed_nm_median', 'speed_nm_mean']] < (Q1_TransitMax - 1.5 * IQR_TransitMax)) | (df_aggregate_data[['hour_since_sum', 'mile_since_sum','period_total_distance', 'distance_ratio','fuel_consumption_average_max', 'fuel_consumption_average_mean','speed_nm_max', 'speed_nm_median', 'speed_nm_mean']] > (Q3_TransitMax + 1.5 * IQR_TransitMax))).any(axis=1)))
    
  ]
df_aggregate_data['activity_label_first'].value_counts()
df_aggregate_data.activity_label_first.factorize()
df_aggregate_data['activity_label_code'] = df_aggregate_data.activity_label_first.factorize()[0]

#split train and test dataset
# train_dataset2 = df_aggregate_data
# test_dataset2 = df_aggregate_data.loc[(df_aggregate_data['new_position_received_time_min']>'2019-11-1')]

# test_dataset2 = df_vessel_plot_AH_agg.loc[(df_vessel_plot_AH_agg['new_position_received_time_']>='2019-10-1')
#                                   &
#                                    (df_vessel_plot_AH_agg['new_position_received_time_']<'2019-12-1')]
train_dataset_x = df_aggregate_data.loc[(df_aggregate_data['speed_nm_max'].isna()==False)
                                        &
                                        (df_aggregate_data['new_position_received_time_min']<'2019-12-22') 
                                        , 
                                     ['hour_since_sum', 'mile_since_sum',
       'period_total_distance', 'distance_ratio',
       'fuel_consumption_average_max', 'fuel_consumption_average_mean',
       'speed_nm_max', 'speed_nm_median', 'speed_nm_mean'
                                     ]]

train_dataset_y = df_aggregate_data.loc[(df_aggregate_data['speed_nm_max'].isna()==False)
                                        &
                                        (df_aggregate_data['new_position_received_time_min']<'2019-12-22')
                                        , 'activity_label_code']
validate_dataset_x = df_aggregate_data.loc[(df_aggregate_data['speed_nm_max'].isna()==False)
                                           &
                                           (df_aggregate_data['new_position_received_time_min']>='2019-12-22') , 
                                     ['hour_since_sum', 'mile_since_sum',
       'period_total_distance', 'distance_ratio',
       'fuel_consumption_average_max', 'fuel_consumption_average_mean',
       'speed_nm_max', 'speed_nm_median', 'speed_nm_mean'
                                     ]]



validate_dataset_y = df_aggregate_data.loc[(df_aggregate_data['speed_nm_max'].isna()==False)
                                           &
                                           (df_aggregate_data['new_position_received_time_min']>='2019-12-22') 
                                           , 'activity_label_code']
from sklearn.ensemble import RandomForestClassifier 
# from sklearn.metrics import accuracy_score
# from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import train_test_split
# from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import RobustScaler
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
# from sklearn.gaussian_process import GaussianProcessClassifier
# from sklearn.gaussian_process.kernels import RBF


# split the training data to train and validate
train_x, validate_x, train_y, validate_y = train_test_split(train_dataset_x, train_dataset_y, test_size = 0.25, random_state = 42, stratify=train_dataset_y)

# # split the SCALED training data to train and validate 
# # train_x_scaled, validate_x_scaled, train_y_scaled, validate_y_scaled = train_test_split(X_scaled, train_dataset_y, test_size = 0.25, random_state = 42)

# # Variable X then fitted to the scaler
scaler = RobustScaler().fit(train_x)
# # Then Transform the X into array X_scaled, this variable later will be the parameter for model
train_X_scaled = scaler.transform(train_x)
val_X_scaled = scaler.transform(validate_x)
randomforrest_clf = RandomForestClassifier(max_depth=50, max_features=6, min_samples_leaf=2,
                       min_samples_split=4, n_estimators=500)

randomforrest_clf.fit(train_dataset_x, train_dataset_y)
# randomforrest_clf.fit(train_X_scaled, train_y)

print(randomforrest_clf.score(train_dataset_x, train_dataset_y))
rf_predictions = randomforrest_clf.predict(train_dataset_x)
# serialized the model
file_name = '[PSV]random_forest_model.pkl'
pickle.dump(randomforrest_clf, open('model/'+file_name, 'wb'))
knn_clf = KNeighborsClassifier(n_neighbors=30, algorithm='brute', weights='distance', p=1)
knn_clf.fit(train_dataset_x, train_dataset_y)
knn_training_score = knn_clf.score(train_dataset_x, train_dataset_y)

print(knn_training_score)
svm_clf = SVC(gamma=2, C=1)
svm_clf.fit(train_dataset_x, train_dataset_y)
svm_training_score = svm_clf.score(train_dataset_x, train_dataset_y)

print(svm_training_score)
import xgboost as xgb

class_weights = list([ 1.00020111, 0.68789765, 0.94257557, 0.72526431, 9.36629002])

w_array = np.ones(train_dataset_y.shape[0])
for i, val in enumerate(train_dataset_y):
    w_array[i] = class_weights[val-1]

xgb_clf = xgb.XGBClassifier()

xgb_clf.fit(train_dataset_x, train_dataset_y)

# make predictions for test data
xgb_predictions = xgb_clf.predict(train_dataset_x)
# xgb_predictions = xgb_clf.predict(val_X_scaled)

# evaluate predictions
xgb_training_score = xgb_clf.score(train_dataset_x, train_dataset_y)
# xgb_training_score = xgb_clf.score(val_X_scaled, validate_y)
print(xgb_training_score)
# serialized the model
file_name2 = '[PSV]xg_boost_model.pkl'
pickle.dump(xgb_clf, open('model/'+file_name2, 'wb'))
from sklearn.ensemble import VotingClassifier

# List of models to vote
four_models_voting_clf = VotingClassifier(estimators=[('xgboost', xgb_clf), 
                                                      ('randomforest', randomforrest_clf), 
                                                      ('knearest', knn_clf), 
                                                      ('svm', svm_clf)],
                                          voting='hard', weights=[1.5,1.3,1,1])

four_models_voting_clf.fit(train_dataset_x, train_dataset_y) 
print(four_models_voting_clf.score(train_dataset_x, train_dataset_y))
# serialized the ensemble model
file_name_ense = '[PSV]ensemble_model3.pkl'
pickle.dump(four_models_voting_clf, open('model/'+file_name_ense, 'wb'))
# from sklearn.metrics import confusion_matrix, plot_confusion_matrix

# confusion_matrix(validate_y, predictions)
cm_1 = plot_confusion_matrix(randomforrest_clf, val_X_scaled, validate_y, 
                      display_labels=['Standby', 'DP', 'Port', 'TransitCombine', 'Towing', 'AH',
        'Transit Max'], 
                      cmap=plt.cm.Blues, xticks_rotation='vertical')

cm_1.ax_.set_title('Confusion Matrix')

print(cm_1.confusion_matrix)
plt.show()

from sklearn.metrics import precision_score, recall_score, f1_score

precision_macro = precision_score(train_dataset_y, rf_predictions, average='macro')
precision_weighted = precision_score(train_dataset_y, rf_predictions, average='weighted')

recall_macro = recall_score(train_dataset_y, rf_predictions, average='macro')
recall_weighted = recall_score(train_dataset_y, rf_predictions, average='weighted')

f1_score = f1_score(train_dataset_y, rf_predictions, average='macro')

print('Precision Score:' + str(precision_macro))
print('Recall Score:' + str(recall_macro))
print('F1 Score:' + str(f1_score))<jupyter_output>Precision Score:0.9870661385089555
Recall Score:0.9479174148854426
F1 Score:0.965056906600313
{'fraction_non_alphanumeric': 0.11202695385355875, 'fraction_numerical': 0.01985440105890139, 'mean_word_length': 2.721898790864308, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 128, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '546943', 'license': 'MIT', 'n_tokens_mistral': 5410, 'n_tokens_neox': 4975, 'n_words': 614, 'path': 'model_training.ipynb', 'repo_name': 'esilaban/vessel_activity_classification', 'size': '16621'}