<jupyter_start><jupyter_text>Vessel Classification on January DataSince we do not have the capability to compare the classification here to a single row classification directly we will only do the voyage view for the entire monthlong dataset.<jupyter_code>import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from sklearn.tree import DecisionTreeClassifier, plot_tree, export_graphviz
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
dir_path = './data/voyage_agg/'
all_files = os.listdir(dir_path)
all_files = [dir_path + f for f in all_files]
other_dir = './data/voyage_agg_sub/'
first = True
for f in all_files:
    temp = pd.read_csv(f)
    temp = temp[temp.VesselType.isin([37,51,35,30])]
    if first:
        temp.to_csv(other_dir + 'binary_set.csv',index=False)
        first = False
    else:
        temp.to_csv(other_dir + 'binary_set.csv',index=False,header=False,mode='a')
del temp
ais = pd.read_csv(other_dir + 'binary_set.csv')
ais = ais.dropna()
ais['VesselType'] = ais['VesselType'].astype(str)
ais.head()
ais.shape
ais.VesselName.nunique()<jupyter_output><empty_output><jupyter_text>Decision Tree<jupyter_code>features = ['LAT','LON','SOG','Heading', 'acceleration', 'bearing_rate']
df = ais.copy()
for feat in features:
    df = df[np.isfinite(df[feat])]
X_train = df[features]
Y_train = df['VesselType'].astype(str).tolist()
X_train, X_test, Y_train, Y_test = train_test_split(X_train, Y_train, test_size=0.3, random_state=123)
dt_model = DecisionTreeClassifier(max_depth=3,random_state=123)
dt_model.fit(X_train, Y_train)
plot_tree(dt_model)
print('Accuracy of decision tree classifier on train set: {:.2f}'.format(dt_model.score(X_train, Y_train)))
print('Accuracy of decision tree classifier on test set: {:.2f}'.format(dt_model.score(X_test, Y_test)))
print('F1 score of decision tree classifier on test set: {:.2f}'.format(f1_score(Y_test, dt_model.predict(X_test), average='weighted')))
from sklearn.metrics import confusion_matrix
Y_pred = dt_model.predict(X_test)
confusion_matrix = confusion_matrix(Y_test, Y_pred)
print(confusion_matrix)<jupyter_output>[[14208     0  7388     0]
 [   93     0   209     0]
 [ 1572     0 39889     0]
 [   77     0   513     0]]<jupyter_text>Logistic Regression<jupyter_code>from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
coefficients = pd.concat([pd.DataFrame(X_train.columns),pd.DataFrame(np.transpose(logreg.coef_))], axis = 1)
coefficients
print('Accuracy of logistic regression classifier on train set: {:.2f}'.format(logreg.score(X_train, Y_train)))
print('Accuracy of logistic regression classifier on test set: {:.2f}'.format(logreg.score(X_test, Y_test)))
print('F1 score of logistic regresssion classifier on test set: {:.2f}'.format(f1_score(Y_test, logreg.predict(X_test), average='weighted')))
from sklearn.metrics import confusion_matrix
Y_pred = logreg.predict(X_test)
confusion_matrix = confusion_matrix(Y_test, Y_pred)
print(confusion_matrix)<jupyter_output>[[ 8703     0 12891     2]
 [   67     0   232     3]
 [ 4392     0 37028    41]
 [  298     0   287     5]]<jupyter_text>Boosted<jupyter_code>from sklearn.ensemble import AdaBoostClassifier
ada = AdaBoostClassifier()
ada.fit(X_train, Y_train)
print('Accuracy of AdaBoost classifier on train set: {:.2f}'.format(ada.score(X_train, Y_train)))
print('Accuracy of AdaBoost classifier on test set: {:.2f}'.format(ada.score(X_test, Y_test)))
print('F1 score of AdaBoost classifier on test set: {:.2f}'.format(f1_score(Y_test, ada.predict(X_test), average='weighted')))
from sklearn.metrics import confusion_matrix
Y_pred = ada.predict(X_test)
confusion_matrix = confusion_matrix(Y_test, Y_pred)
print(confusion_matrix)<jupyter_output>[[15119     0  6473     4]
 [   34     0   266     2]
 [ 1358     0 40018    85]
 [   66     1   494    29]]
{'fraction_non_alphanumeric': 0.11921207469941161, 'fraction_numerical': 0.03734970580711179, 'mean_word_length': 5.368078175895765, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '542184', 'license': 'MIT', 'n_tokens_mistral': 1508, 'n_tokens_neox': 1383, 'n_words': 323, 'path': 'MonthClassification.ipynb', 'repo_name': 'bevvvvv/vessel-classification', 'size': '3909'}