<jupyter_start><jupyter_text>Import dataset from Google Drive to Colab<jupyter_code>!rm cleaned_train.csv
!rm cleaned_test.csv

!cp drive/MyDrive/CI_FinalProject/cleaned_train.csv ./
!cp drive/MyDrive/CI_FinalProject/cleaned_test.csv ./<jupyter_output>rm: cannot remove 'cleaned_train.csv': No such file or directory
rm: cannot remove 'cleaned_test.csv': No such file or directory<jupyter_text>Import all important things<jupyter_code>import numpy as np
import pandas as pd

from sklearn.feature_extraction.text import TfidfVectorizer

import torch
import torch.nn as nn
import torch.nn.functional as F

from tqdm.notebook import tqdm<jupyter_output><empty_output><jupyter_text>Preprocessing  Load cleaned train data<jupyter_code>train_data = pd.read_csv('cleaned_train.csv', usecols=['clean_text', 'Category'])
# train_data.drop(columns=['Unnamed: 0', 'Unnamed: 0.1'], inplace=True)
train_data['category_id'] = train_data['Category'].factorize()[0]
train_data.head()
test_data = pd.read_csv('cleaned_test.csv', usecols=['Id', 'clean_text'])
# test_data.drop(columns=['Unnamed: 0'], inplace=True)
test_data.head()
label2id = {label: i for i, label in enumerate(train_data['Category'].unique())}
id2label = {v: k for k, v in label2id.items()}

print(f'label2id: {label2id}')
print(f'id2label: {id2label}')
vectorizer = TfidfVectorizer(sublinear_tf=True, min_df=30, norm='l2', ngram_range=(1, 2))
# vectorizer = HashingVectorizer(norm='l2', ngram_range=(1, 2))
features = vectorizer.fit_transform(train_data['clean_text'])
labels = train_data['category_id']
import pickle

pickle.dump(features, open("drive/MyDrive/CI_FinalProject/TfidfVectorized_features.p", "wb"))
from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier(n_estimators=200, max_depth=100, random_state=0, verbose=1)
model.fit(features, labels)
from joblib import dump, load

dump(model, 'drive/MyDrive/CI_FinalProject/RandomForestModel.joblib')
test_features = vectorizer.transform(test_data['clean_text'])
# y_pred_proba = model.predict_proba(test_features)
y_pred = model.predict(test_features)
Y_pred_name =[]
for cat_id in y_pred:
  Y_pred_name.append(id2label[cat_id])
submission = pd.DataFrame({
        "Id": test_data["Id"],
        "Category": Y_pred_name
    })
submission.head()
submission.to_csv('submission.csv', index=False)
from sklearn.linear_model import LogisticRegression
LRModel = LogisticRegression(random_state=30)
LRModel.fit(features, labels)
from joblib import dump, load
dump(LRModel, 'drive/MyDrive/CI_FinalProject/LogisticRegressionModel.joblib')
y_pred = LRModel.predict(test_features)
Y_pred_name =[]
for cat_id in y_pred:
  Y_pred_name.append(id2label[cat_id])
submission = pd.DataFrame({
        "Id": test_data["Id"],
        "Category": Y_pred_name
    })
submission.head()
submission.to_csv('submission.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12624016421484777, 'fraction_numerical': 0.01094765651727677, 'mean_word_length': 6.654450261780105, 'pattern_counts': {'":': 4, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '464869', 'license': 'MIT', 'n_tokens_mistral': 1075, 'n_tokens_neox': 1038, 'n_words': 195, 'path': 'FinalProject/CI_FinalProject_tfidf.ipynb', 'repo_name': 'ParsaHejabi/ComputationalIntelligence-ComputerAssignments', 'size': '2923'}