<jupyter_start><jupyter_text>Loading Cleaned dataset, splitting it into train test<jupyter_code>df_cl = pd.read_csv("../input/cleaned-df/cleaned_df.csv", engine="python")
df_cl["label"] = df_cl["dialect"]

df_cl.drop(df_cl[df_cl.text == '[]'].index, inplace=True, axis=0)
df_cl.dropna(inplace=True)

df_cl.label = LabelEncoder().fit_transform(df_cl.label)

df_train, df_test = train_test_split(df_cl, test_size=0.3, random_state=911, shuffle=True)
df_train[["label", "text"]].to_csv("df_train.csv", encoding="utf-8-sig", index=False)<jupyter_output><empty_output><jupyter_text>Loading the training dataset using hugging-face dataset<jupyter_code>df = load_dataset('csv', script_version="master", data_files=["./df_train.csv"], delimiter=",", split="train")
df = df.train_test_split()
df<jupyter_output><empty_output><jupyter_text>Using MARBERT Tokenizer<jupyter_code>tokenizer = AutoTokenizer.from_pretrained("UBC-NLP/MARBERT")

def preprocess_function(examples):
    return tokenizer(examples["text"], truncation=True)
tokenized_df = df.map(preprocess_function, batched=True)
tokenized_df = tokenized_df.remove_columns(["text"]) 
data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors="tf")
tf_train_dataset = tokenized_df["train"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "label"],
    shuffle=True,
    batch_size=16,
    collate_fn=data_collator,
)

tf_validation_dataset = tokenized_df["test"].to_tf_dataset(
    columns=["attention_mask", "input_ids", "label"],
    shuffle=False,
    batch_size=16,
    collate_fn=data_collator,
)
batch_size = 16
num_epochs = 5
batches_per_epoch = len(tokenized_df["train"]) // batch_size
total_train_steps = int(batches_per_epoch * num_epochs)
optimizer, schedule = create_optimizer(init_lr=2e-5, num_warmup_steps=0, num_train_steps=total_train_steps)
model = TFAutoModelForSequenceClassification.from_pretrained("UBC-NLP/MARBERT", num_labels=18)

model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
    filepath='./',
    save_weights_only=True,
    monitor='val_loss',
    mode='min',
    save_best_only=True)

callbacks = [model_checkpoint_callback]<jupyter_output><empty_output><jupyter_text>Model Training<jupyter_code>model.compile(optimizer=optimizer)

model.fit(
    tf_train_dataset,
    validation_data=tf_validation_dataset,
    epochs=3
)
# model.save_pretrained("./")
model.load_weights("./tf_model.h5")
# predict_input = tokenizer.encode(df_cl["text"][0],
#                                  truncation=True,
#                                  padding=True,
#                                  return_tensors="tf")<jupyter_output><empty_output><jupyter_text>Encoding the test data text, and doing the evaluation<jupyter_code>input_seq_test = [tokenizer.encode(lst, truncation=True, padding=True, return_tensors="tf") 
                      for lst in df_test["text"]]
len(input_seq_test)
input_seq_test[0]
tf_output = [np.argmax(model.predict(lst)[0], axis=1) for lst in input_seq_test]
prediction = [lst[0] for lst in tf_output]
dialects_dict = {
    3: "EG",
    11: "PL",
    6: "KW", 
    8: "LY",
    12: "QA",
    5: "JO",
    7: "LB",
    13: "SA",
    0: "AE",
    1: "BH",
    10: "OM",
    15: "SY",
    2: "DZ",
    4: "IQ",
    9: "MA",
    17: "YE",
    16: "TN",
    14: "SD"
}
from imblearn.metrics import classification_report_imbalanced

print(classification_report_imbalanced(df_test["label"], prediction))<jupyter_output>pre       rec       spe        f1       geo       iba       sup

          0       0.47      0.48      0.97      0.48      0.68      0.44      7970
          1       0.44      0.36      0.97      0.40      0.60      0.33      8002
          2       0.61      0.60      0.99      0.60      0.77      0.57      4855
          3       0.77      0.87      0.96      0.82      0.92      0.83     17287
          4       0.63      0.63      0.99      0.63      0.79      0.60      4654
          5       0.45      0.39      0.97      0.42      0.62      0.36      8390
          6       0.57      0.58      0.96      0.58      0.75      0.54     12620
          7       0.69      0.66      0.98      0.67      0.80      0.63      8247
          8       0.73      0.74      0.98      0.74      0.85      0.71     10903
          9       0.84      0.61      1.00      0.71      0.78      0.59      3452
         10       0.48      0.44      0.98      0.46      0.66      0.41      5738
   [...]
{'fraction_non_alphanumeric': 0.12660798916723087, 'fraction_numerical': 0.0699616339426766, 'mean_word_length': 2.5091053048297702, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '63478', 'license': 'Unlicense', 'n_tokens_mistral': 1737, 'n_tokens_neox': 1606, 'n_words': 282, 'path': 'Notebooks/MARBERT-FineTuning.ipynb', 'repo_name': 'Jabalov/Arabic-Dialects-Identification', 'size': '4431'}