<jupyter_start><jupyter_text>Libraries Required<jupyter_code>import re
import time
import aranorm
import random
import numpy as np
import pandas as pd
import seaborn as sns
import preprocess_arabert
from sklearn.model_selection import train_test_split
seed=42
random.seed(seed)
df= pd.read_csv("fetchingTweets_dialect_dataset.csv")
df.head()
print("DataFrame Shape:",df.shape,"\n")
print("  DataFrame Info","\n","-"*20)
df.info()
print("")
df[df.isnull().any(axis=1)]
tweets = list(df["tweets"])
s=time.time()
text_preprocessed = [preprocess_arabert.preprocess(txt, do_farasa_tokenization=True) for txt in tweets] 
text_preprocessed=[aranorm.normalize_arabic_text(txt) for txt in text_preprocessed]
e=time.time()
print((e-s)/60,"  Minutes")
tweets[:5]
text_preprocessed[:5]
df["text_preprocessed"]=text_preprocessed
df.sample(5)
print("DataFrame Shape:",df.shape,"\n")
print("  DataFrame Info","\n","-"*20)
df.info()
print("")
df[df.isnull().any(axis=1)]
df.dialect.value_counts()
df[df.duplicated(['text_preprocessed'])]
duplicates=df[df.duplicated(['text_preprocessed'])]
duplicates.dialect.value_counts()
df_clean_prep= df.drop_duplicates('text_preprocessed').reset_index(drop=True)
df_clean_prep[df_clean_prep.duplicated(['text_preprocessed'])]
df_clean_prep[df_clean_prep.isnull().any(axis=1)]
df_clean_prep.sample(5)
print("DataFrame Shape:",df_clean_prep.shape,"\n")
print("  DataFrame Info","\n","-"*20)
df_clean_prep.info()
print("")
keys_dictionary   = {0:"EG",1:"PL",2:"KW",3:"LY",4:"QA",5:"JO",6:"LB",7:"SA",8:"AE",9:"BH",
                     10:"OM",11:"SY",12:"DZ",13:"IQ",14:"SD",15:"MA",16:"YE",17:"TN"}

labels_dictionary = {"EG":0,"PL":1,"KW":2,"LY":3,"QA":4,"JO":5,"LB":6,"SA":7,"AE":8,"BH":9,
                     "OM":10,"SY":11,"DZ":12,"IQ":13,"SD":14,"MA":15,"YE":16,"TN":17}
df_clean_prep['label'] = df_clean_prep['dialect'].apply(lambda x : labels_dictionary.get(x))
df_clean_prep.head(5)
df_ = df_clean_prep.sample(frac=1).reset_index(drop=True)
df_.head(5)
df_.to_csv('Cleaned_PreProcessed_Tweets_dialect_dataset.csv', encoding='utf-8',index = None, header=True)
dfTotal=df_[["text_preprocessed","label"]]
dfTotal.head()
dfTotal.label.value_counts()
def randomSampling_df(df,percent):
    text=[]
    label=[]
    for i in range(18):
        df2=df[df["label"]==i].sample(frac=percent)
        text.extend(list(df2["text_preprocessed"]))
        label.extend(list(df2["label"]))
    return text, label
        
text,label=randomSampling_df(dfTotal,0.1)
df_10_percent=pd.DataFrame([text,label], index=['text_preprocessed', 'label']).T
df_10_percent.label.value_counts()
df_10_percent= df_10_percent.sample(frac=1).reset_index(drop=True)
df_10_percent.head()
def random_split(data, features, output, fraction, seed=42):
    X_train, X_test, y_train, y_test = train_test_split(data[features],
                                                        data[output],
                                                        stratify = data[output],
                                                        random_state=seed,
                                                        train_size=fraction
                                                       )
    train_data = pd.DataFrame(data=X_train, columns=features)
    train_data[output] = y_train
    test_data = pd.DataFrame(data=X_test, columns=features)
    test_data[output] = y_test
    
    return train_data, test_data
train_fraction = 0.80 
val_fraction   = 0.50   

output = 'label' 

features = dfTotal.columns.tolist()

features.remove(output)
print('output:', output)
print('features:', features)
dfTrain, tmp    = random_split(dfTotal, features, output, train_fraction, seed)
dfValid, dfTest = random_split(tmp, features, output, val_fraction, seed)

print(len(dfTrain))
print(len(dfValid))
print(len(dfTest))
print(len(dfTrain)+len(dfValid)+len(dfTest))
print(len(dfTotal))
dfTrain_10_percent, tmp_10_percent    = random_split(df_10_percent, features, output, train_fraction, seed)
dfValid_10_percent, dfTest_10_percent = random_split(tmp_10_percent, features, output, val_fraction, seed)

print(len(dfTrain_10_percent))
print(len(dfValid_10_percent))
print(len(dfTest_10_percent))
print(len(dfTrain_10_percent)+len(dfValid_10_percent)+len(dfTest_10_percent))
print(len(df_10_percent))
dfTrain.reset_index(drop=True,inplace=True)
dfValid.reset_index(drop=True,inplace=True)
dfTest.reset_index(drop=True,inplace=True)

dfTrain_10_percent.reset_index(drop=True,inplace=True)
dfValid_10_percent.reset_index(drop=True,inplace=True)
dfTest_10_percent.reset_index(drop=True,inplace=True)
dfTrain.head()
print("trainData")
pd.DataFrame(data=np.column_stack((dfTrain.label.value_counts(), (dfTrain.label.value_counts()/len(dfTrain)*100))),columns=['Values','Percentage'])
dfTrain.label.value_counts().plot(kind='pie');
sns.countplot(x="label",data=dfTrain);
dfValid.head()
print("validData")
pd.DataFrame(data=np.column_stack((dfValid.label.value_counts(), (dfValid.label.value_counts()/len(dfValid)*100))),columns=['Values','Percentage'])
dfValid.label.value_counts().plot(kind='pie');
sns.countplot(x="label",data=dfValid);
dfTest.head()
print("testData")
pd.DataFrame(data=np.column_stack((dfTest.label.value_counts(), (dfTest.label.value_counts()/len(dfTest)*100))),columns=['Values','Percentage'])
dfTest.label.value_counts().plot(kind='pie');
sns.countplot(x="label",data=dfTest);
dfTrain.to_csv('preProcessedTweets_trainF.csv', encoding='utf-8',index = None, header=True)
dfValid.to_csv('preProcessedTweets_validF.csv', encoding='utf-8',index = None, header=True)
dfTest.to_csv('preProcessedTweets_testF.csv'  , encoding='utf-8',index = None, header=True)
dfTrain_10_percent.to_csv('preProcessedTweets_trainF_10_percent.csv', encoding='utf-8',index = None, header=True)
dfValid_10_percent.to_csv('preProcessedTweets_validF_10_percent.csv', encoding='utf-8',index = None, header=True)
dfTest_10_percent.to_csv('preProcessedTweets_testF_10_percent.csv'  , encoding='utf-8',index = None, header=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16724854530340816, 'fraction_numerical': 0.026267664172901082, 'mean_word_length': 6.085983510011778, 'pattern_counts': {'":': 18, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '536135', 'license': 'MIT', 'n_tokens_mistral': 2295, 'n_tokens_neox': 2152, 'n_words': 257, 'path': 'NoteBooks/2_Data-Preprocessing.ipynb', 'repo_name': 'MohamedSebaie/Arabic_Dialect_Identification_NLP-AIM-Task', 'size': '6015'}