<jupyter_start><jupyter_text>Naive Identifier: Final modelThis notebook features the most interesting model using NaiveIdentifier.<jupyter_code>%run notebook_utils.py

from sklearn.feature_extraction.text import CountVectorizer

%cd ..
from langid import NaiveIdentifier, np_sanitize
%cd - 
import matplotlib
# set big font in plots
SMALL_SIZE = 20
matplotlib.rc('font', size=SMALL_SIZE)
matplotlib.rc('axes', titlesize=SMALL_SIZE)
# load data
X_train, X_test, y_train, y_test = load_split_data()
sms_any_X, sms_any_y = load_sms_any()
X_valid, y_valid = load_validation_data()<jupyter_output><empty_output><jupyter_text>Model training<jupyter_code># train the model
clf = NaiveIdentifier(
    klass=CountVectorizer, 
    analyzer='word', 
    ngram_range=(1,3), 
    max_features=3000)
clf.fit(np_sanitize(X_train), y_train)<jupyter_output><empty_output><jupyter_text>Model evaluation<jupyter_code>print("TEST SET")
print("========")
y_pred = clf.predict(np_sanitize(X_test))
print_results(y_test, y_pred)
plot_confusion_matrix(y_test, y_pred, normalised=True, 
    title="Test, accuracy = %.2f%%" % (metrics.accuracy_score(y_test, y_pred) * 100))
print("SMS RECALL")
print("==========")

test_recall_with_sms(clf)

print("\n\nSMS ANY")
print("==========")
y_sms_any_pred = clf.predict(np_sanitize(sms_any_X))
print_results(sms_any_y, y_sms_any_pred)
plot_confusion_matrix(sms_any_y, y_sms_any_pred, normalised=True,
    title="SMS, accuracy = %.2f%%" % (metrics.accuracy_score(sms_any_y, y_sms_any_pred) * 100))

print("\n\nVALIDATION SET")
print("===============")
y_valid_pred = clf.predict(np_sanitize(X_valid))
print_results(y_valid, y_valid_pred)
plot_confusion_matrix(y_valid, y_valid_pred, normalised=True,
    title="Validation, accuracy = %.2f%%" % (metrics.accuracy_score(y_valid, y_valid_pred) * 100))<jupyter_output>TEST SET
========
accuracy: 0.9816

             precision    recall  f1-score   support

         de     0.9293    0.9979    0.9624      1462
         fr     0.9946    1.0000    0.9973      1468
         en     0.9942    0.9967    0.9955      1538
         it     0.9980    0.9980    0.9980      1482
         sg     0.9962    0.9131    0.9528      1438

avg / total     0.9826    0.9816    0.9815      7388<jupyter_text>Model export<jupyter_code># model export
import pickle

filename = "Sanitize-CountVec_feat3000_1-3wordgrams-NaiveIdentifier.pickle"
with open(filename, 'wb') as f: pickle.dump(clf, f)<jupyter_output><empty_output><jupyter_text>A peek at the features<jupyter_code>clf.vectorizers[0].get_feature_names()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13013168086754454, 'fraction_numerical': 0.05693261037955073, 'mean_word_length': 4.857142857142857, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '151820', 'license': 'Apache-2.0', 'n_tokens_mistral': 1070, 'n_tokens_neox': 993, 'n_words': 170, 'path': 'language-detection/notebooks/09-FinalModel-NaiveIdentifier.ipynb', 'repo_name': 'derlin/SwigSpot_Schwyzertuutsch-Spotting', 'size': '2582'}