<jupyter_start><jupyter_text>Load and preprocess data<jupyter_code># # Load data
# df = (
#     pd.read_csv(
#         "../../data/02_intermediate/intermediate_tweets.csv",
#     )
#     .replace(abbr_dict, regex=True)
#     .replace(slang_dict, regex=True)
#     .replace("'", "")
#     .replace("\s+", " ", regex=True)
#     .dropna()
# )

# # Reduce memory consumption
# df = reduce_memory_usage(df, convert_to_category=False, verbose=False)

# # Train test split
# X_train, X_test, y_train, y_test = train_test_split(
#     df["text"], df["target"], test_size=0.33, random_state=42
# )

# # Instantiate tfidf vectorizer
# tfidf_vectorizer = TfidfVectorizer(
#     analyzer="word", strip_accents="ascii", stop_words="english"
# )

# # Vectorize data
# X_train_tfidf = tfidf_vectorizer.fit_transform(X_train)
# X_test_tfidf = tfidf_vectorizer.transform(X_test)

# # Write data to disk
# y_train.to_csv("../../data/05_model_input/y_train.csv")
# y_test.to_csv("../../data/05_model_input/y_test.csv")
# scipy.sparse.save_npz('../../data/05_model_input/X_train_tfidf.npz', X_train_tfidf)
# scipy.sparse.save_npz('../../data/05_model_input/X_test_tfidf.npz', X_test_tfidf)
X_train = scipy.sparse.load_npz("../../data/05_model_input/X_train_tfidf.npz")
y_train = pd.read_csv("../../data/05_model_input/y_train.csv")
X_test = scipy.sparse.load_npz("../../data/05_model_input/X_test_tfidf.npz")
y_test = pd.read_csv("../../data/05_model_input/y_test.csv")<jupyter_output><empty_output><jupyter_text>Train Model<jupyter_code># Train model
clf = MultinomialNB().fit(X_train_tfidf, y_train)

# Vectorize test data
X_test_tfidf = tfidf_vectorizer.transform(X_test)


print(classification_report(y_test, clf.predict(X_test_tfidf)))
svm = SGDClassifier(
    loss="hinge", penalty="l2", alpha=1e-3, random_state=42, max_iter=5, tol=None
).fit(X_train_tfidf, y_train)


print(classification_report(y_test, svm.predict(X_test_tfidf)))
random_forest  = RandomForestClassifier(max_depth=2, random_state=0).fit(X_train_tfidf, y_train)


print(classification_report(y_test, random_forest.predict(X_test_tfidf)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1629769194536034, 'fraction_numerical': 0.01460197833254828, 'mean_word_length': 5.475609756097561, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '83485', 'license': 'MIT', 'n_tokens_mistral': 895, 'n_tokens_neox': 870, 'n_words': 99, 'path': 'notebooks/Modelling/01-felix-modelling.ipynb', 'repo_name': 'FelixBehne/sentiment_analysis_twitter', 'size': '2123'}