<jupyter_start><jupyter_text>Arabic_Dialect_Identification_NLP_AIM_Task**`<NAME>`**  Installing requirements<jupyter_code>%%capture
!pip install transformers
!pip install pyarabic
%cd /content/drive/MyDrive/Aim_NLP_Task/Arabic_Dialect_Classification_Task
import torch
# If there's a GPU available...
if torch.cuda.is_available():    

    # Tell PyTorch to use the GPU.    
    device = torch.device("cuda")

    print('There are %d GPU(s) available.' % torch.cuda.device_count())

    print('We will use the GPU:', torch.cuda.get_device_name(0))

# If not...
else:
    print('No GPU available, using the CPU instead.')
    device = torch.device("cpu")
import gc 
import re
import time
import logging
import aranorm
import numpy as np
import pandas as pd
import dialectUtils
import preprocess_arabert
from transformers import AutoTokenizer, BertForSequenceClassification,BertTokenizer
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning) 
warnings.filterwarnings("ignore")
dialectUtils.set_global_logging_level(logging.ERROR, ["transformers", "nlp", "torch", "tensorflow", "tensorboard", "wandb"])
logging.getLogger("transformers.tokenization_utils_base").setLevel(logging.ERROR)
train_tweets,train_labels = dialectUtils.read_csv("../Data/preProcessedTweets_trainF.csv",
                                                  "text_preprocessed","label")
valid_tweets,valid_labels = dialectUtils.read_csv("../Data/preProcessedTweets_validF.csv",
                                                  "text_preprocessed","label")
test_tweets,test_labels   = dialectUtils.read_csv("../Data/preProcessedTweets_testF.csv",
                                                  "text_preprocessed","label")<jupyter_output><empty_output><jupyter_text>Bert `Base` AraBert  Train<jupyter_code>preTrainedBertModel_base="aubmindlab/bert-base-arabertv02-twitter"

tokenizer_base= AutoTokenizer.from_pretrained(preTrainedBertModel_base)
tokenizer_obj_base = dialectUtils.Tokenizer(tokenizer_base)


train_data_base = tokenizer_obj_base.bert_tokenize_data(train_tweets,train_labels)
valid_data_base = tokenizer_obj_base.bert_tokenize_data(valid_tweets ,valid_labels)
test_data_base  = tokenizer_obj_base.bert_tokenize_data(test_tweets,test_labels)

train_loader_base,valid_loader_base=dialectUtils.create_bert_dataloader(train_data_base,
                                                                        valid_data_base,
                                                                        batch_size=32,
                                                                        split_train= False)

test_loader_base=dialectUtils.create_bert_dataloader(test_data_base,valid=None,batch_size=32,
                                                split_train=False,test=True)
BERTmodel_base=dialectUtils.get_model(preTrainedBertModel_base)
dialectUtils.printModel_Parameters(BERTmodel_base)
print("\n")
torch.cuda.empty_cache()
print(torch.cuda.memory_summary(device=None, abbreviated=False))
dialectUtils.train(train_loader_base,valid_loader_base,model=BERTmodel_base,epochs=5,learning_rate=2e-5,eps=1e-8,device ="cuda")<jupyter_output>/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning
  FutureWarning,<jupyter_text>Predict on Test<jupyter_code>pred_base,actual = dialectUtils.Batchpredict(BERTmodel_base,test_loader_base)
dialectUtils.get_report(pred_base,actual)<jupyter_output>precision    recall  f1-score   support

          EG       0.78      0.88      0.83      5756
          PL       0.55      0.57      0.56      4368
          KW       0.57      0.60      0.58      4200
          LY       0.76      0.74      0.75      3647
          QA       0.53      0.54      0.54      3105
          JO       0.46      0.42      0.44      2784
          LB       0.72      0.68      0.70      2757
          SA       0.48      0.56      0.51      2678
          AE       0.48      0.50      0.49      2620
          BH       0.42      0.41      0.41      2612
          OM       0.49      0.49      0.49      1904
          SY       0.49      0.43      0.46      1621
          DZ       0.64      0.62      0.63      1614
          IQ       0.67      0.61      0.64      1548
          SD       0.75      0.66      0.70      1438
          MA       0.79      0.64      0.71      1153
          YE       0.40      0.31      0.35       988
          TN       0.71    [...]<jupyter_text>Save Base Model<jupyter_code>import os

# Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()

output_dir = '/content/drive/MyDrive/Aim_NLP_Task/BERT_Fine_Tuning_dialect_base_0.60'

# Create output directory if needed
if not os.path.exists(output_dir):
    os.makedirs(output_dir)

print("Saving model to %s" % output_dir)

# Save a trained model, configuration and tokenizer using `save_pretrained()`.
# They can then be reloaded using `from_pretrained()`
model_to_save = BERTmodel_base.module if hasattr(BERTmodel_base, 'module') else BERTmodel_base  # Take care of distributed/parallel training
model_to_save.save_pretrained(output_dir)
tokenizer_obj_base.save_pretrained(output_dir)

# Good practice: save your training arguments together with the trained model
# torch.save(args, os.path.join(output_dir, 'training_args.bin'))

path = "/content/drive/MyDrive/Aim_NLP_Task/Models_Weight/model_base_0.60.pth"
torch.save(BERTmodel_base.cpu().state_dict(), path) # saving model
BERTmodel_base.cuda()<jupyter_output><empty_output><jupyter_text>Load Pretrained Model<jupyter_code>output_dir_base="/content/drive/MyDrive/Aim_NLP_Task/BERT_Fine_Tuning_dialect_base_0.60"
test_tweets,test_labels = dialectUtils.read_csv("../Data/preProcessedTweets_testF.csv",
                                                  "text_preprocessed","label")
tokenizer_class_base = BertTokenizer.from_pretrained(output_dir_base)
load_BERTmodel_base  = BertForSequenceClassification.from_pretrained(output_dir_base)
load_BERTmodel_base.to(device)
tokenizer_obj_base = dialectUtils.Tokenizer(tokenizer_class_base)
test_data_base  = tokenizer_obj_base.bert_tokenize_data(test_tweets,test_labels)
test_loader_base=dialectUtils.create_bert_dataloader(test_data_base,valid=None,batch_size=32,
                                                split_train=False,test=True)
pred_base,actual = dialectUtils.Batchpredict(load_BERTmodel_base,test_loader_base)
dialectUtils.get_report(pred_base,actual)<jupyter_output>precision    recall  f1-score   support

          EG       0.78      0.88      0.83      5756
          PL       0.55      0.57      0.56      4368
          KW       0.57      0.60      0.58      4200
          LY       0.76      0.74      0.75      3647
          QA       0.53      0.54      0.54      3105
          JO       0.46      0.42      0.44      2784
          LB       0.72      0.68      0.70      2757
          SA       0.48      0.56      0.51      2678
          AE       0.48      0.50      0.49      2620
          BH       0.42      0.41      0.41      2612
          OM       0.49      0.49      0.49      1904
          SY       0.49      0.43      0.46      1621
          DZ       0.64      0.62      0.63      1614
          IQ       0.67      0.61      0.64      1548
          SD       0.75      0.66      0.70      1438
          MA       0.79      0.64      0.71      1153
          YE       0.40      0.31      0.35       988
          TN       0.71    [...]<jupyter_text>________________________________________________________________________  Bert `Large` AraBert  Train<jupyter_code>preTrainedBertModel_large="aubmindlab/bert-large-arabertv02-twitter"

tokenizer_large= AutoTokenizer.from_pretrained(preTrainedBertModel_large)
tokenizer_obj_large = dialectUtils.Tokenizer(tokenizer_large)


train_data_large = tokenizer_obj_large.bert_tokenize_data(train_tweets,train_labels)
valid_data_large = tokenizer_obj_large.bert_tokenize_data(valid_tweets ,valid_labels)
test_data_large  = tokenizer_obj_large.bert_tokenize_data(test_tweets,test_labels)

train_loader_large,valid_loader_large=dialectUtils.create_bert_dataloader(train_data_large,
                                                                        valid_data_large,
                                                                        batch_size=64,
                                                                        split_train= False)

test_loader_large=dialectUtils.create_bert_dataloader(test_data_large,valid=None,batch_size=64,
                                                split_train=False,test=True)

BERTmodel_large=dialectUtils.get_model(preTrainedBertModel_large)
dialectUtils.printModel_Parameters(BERTmodel_large)
print("\n")
torch.cuda.empty_cache()
print(torch.cuda.memory_summary(device=None, abbreviated=False))
dialectUtils.train(train_loader_large,valid_loader_large,model=BERTmodel_large,epochs=5,learning_rate=2e-5,eps=1e-8,device ="cuda")<jupyter_output>/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning
  FutureWarning,<jupyter_text>Predict on Test<jupyter_code>pred_large,actual = dialectUtils.Batchpredict(BERTmodel_large,test_loader_large)
dialectUtils.get_report(pred_large,actual)<jupyter_output>precision    recall  f1-score   support

          EG       0.79      0.88      0.83      5764
          PL       0.58      0.59      0.59      4374
          KW       0.56      0.62      0.59      4211
          LY       0.75      0.73      0.74      3650
          QA       0.53      0.57      0.55      3107
          JO       0.48      0.40      0.44      2792
          LB       0.72      0.70      0.71      2762
          SA       0.48      0.55      0.51      2683
          AE       0.47      0.50      0.48      2630
          BH       0.44      0.40      0.42      2629
          OM       0.48      0.50      0.49      1911
          SY       0.54      0.43      0.48      1624
          DZ       0.64      0.60      0.62      1619
          IQ       0.68      0.64      0.66      1550
          SD       0.74      0.68      0.71      1443
          MA       0.78      0.66      0.71      1154
          YE       0.43      0.32      0.37       993
          TN       0.71    [...]<jupyter_text>Save Large Model<jupyter_code>import os

# Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()

output_dir = '/content/drive/MyDrive/Aim_NLP_Task/BERT_Fine_Tuning_dialect_large_061'

# Create output directory if needed
if not os.path.exists(output_dir):
    os.makedirs(output_dir)

print("Saving model to %s" % output_dir)

# Save a trained model, configuration and tokenizer using `save_pretrained()`.
# They can then be reloaded using `from_pretrained()`
model_to_save = BERTmodel_large.module if hasattr(BERTmodel_large, 'module') else BERTmodel_large  # Take care of distributed/parallel training
model_to_save.save_pretrained(output_dir)
tokenizer_obj_large.save_pretrained(output_dir)

# Good practice: save your training arguments together with the trained model
# torch.save(args, os.path.join(output_dir, 'training_args.bin'))
path = "/content/drive/MyDrive/Aim_NLP_Task/Models_Weight/model_large_0.61.pth"
torch.save(BERTmodel_large.cpu().state_dict(), path) # saving model
BERTmodel_large.cuda()<jupyter_output><empty_output><jupyter_text>Load Pretrained Model<jupyter_code>output_dir_large="/content/drive/MyDrive/Aim_NLP_Task/BERT_Fine_Tuning_dialect_large_0.61"
test_tweets,test_labels = dialectUtils.read_csv("../Data/preProcessedTweets_testF.csv",
                                                  "text_preprocessed","label")
tokenizer_class_large = BertTokenizer.from_pretrained(output_dir_large)
load_BERTmodel_large = BertForSequenceClassification.from_pretrained(output_dir_large)
load_BERTmodel_large.to(device)
tokenizer_obj_large = dialectUtils.Tokenizer(tokenizer_class_large)
test_data_large  = tokenizer_obj_large.bert_tokenize_data(test_tweets,test_labels)
test_loader_large=dialectUtils.create_bert_dataloader(test_data_large,valid=None,batch_size=64,
                                                split_train=False,test=True)
pred_large,actual = dialectUtils.Batchpredict(BERTmodel_large,test_loader_large)
dialectUtils.get_report(pred_large,actual)<jupyter_output>precision    recall  f1-score   support

          EG       0.79      0.88      0.83      5764
          PL       0.58      0.59      0.59      4374
          KW       0.56      0.62      0.59      4211
          LY       0.75      0.73      0.74      3650
          QA       0.53      0.57      0.55      3107
          JO       0.48      0.40      0.44      2792
          LB       0.72      0.70      0.71      2762
          SA       0.48      0.55      0.51      2683
          AE       0.47      0.50      0.48      2630
          BH       0.44      0.40      0.42      2629
          OM       0.48      0.50      0.49      1911
          SY       0.54      0.43      0.48      1624
          DZ       0.64      0.60      0.62      1619
          IQ       0.68      0.64      0.66      1550
          SD       0.74      0.68      0.71      1443
          MA       0.78      0.66      0.71      1154
          YE       0.43      0.32      0.37       993
          TN       0.71    [...]<jupyter_text>___________________________________________________________________  Load 2 Models to Test on Single Sentence<jupyter_code>path_large="../Models_Weight/model_large_0.61.pth"
path_base="../Models_Weight/model_base_0.60.pth"
tweet="من الاخر اللعيبة. اللي النادي الاهلي بيفكر فيهم ولهم علاقة بنادي الزمالك هم. مصطفي محمد مصطفي فتحي. بن شرقي فقط لاغير"
dialectUtils.predictDialect(tweet,path_large,"large")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.07769600905745826, 'fraction_numerical': 0.06778941409566941, 'mean_word_length': 2.1233149171270718, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 104, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '181474', 'license': 'MIT', 'n_tokens_mistral': 5017, 'n_tokens_neox': 4521, 'n_words': 831, 'path': 'NoteBooks/4_Deep_Learning_Model-Training-AraBERT.ipynb', 'repo_name': 'MohamedSebaie/Arabic_Dialect_Identification_NLP-AIM-Task', 'size': '14155'}