<jupyter_start><jupyter_text><jupyter_code>from urllib.request import Request, urlopen
from bs4 import BeautifulSoup
from google.colab import drive
import csv
import re
import nltk
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from nltk.stem.isri import ISRIStemmer
from nltk.corpus import stopwords
from nltk.tokenize import wordpunct_tokenize<jupyter_output><empty_output><jupyter_text>Mounting Google Drive:<jupyter_code>drive.mount('/content/drive')
project_path = "/content/drive/My Drive/nlp-project/"
data_path = "/content/drive/My Drive/nlp-project/data/"<jupyter_output><empty_output><jupyter_text>Copy files from google drive to github repo:  Since Colab backend is not persistent, we need a permanent storage solution. In addition to that, we also need a version control system so we can keep track of changes.To do so, we will choose Google Drive for storing our data and models. At the same time, Github is better suited for code, thus we will use it for notebook and libraries.<jupyter_code>%cd /content/drive/'My Drive'/nlp-project
%rm -rf .git
!git init
!git config --global user.email "<EMAIL>"
!git config --global user.name "<NAME>"
!git add .
!git commit -m "First commit"
!git remote add origin https://h3t1:psswrd@github.com/h3t1/nlp-project.git

!git push -f origin master
%cd /content/<jupyter_output><empty_output><jupyter_text>Parsing and extracting all the links related to covid-19:<jupyter_code>def extract_links(i):
  req = Request('https://www.hespress.com/index.html?search_options=YTo2OntzOjEyOiJzZWFyY2hfbGltaXQiO2k6MTA7czoxNDoic2VhcmNoX29wdGlvbnMiO2E6Nzp7czoyMToic2VhcmNoX2NyZWF0ZWRfZmlsdGVyIjtpOjE7czoxOToic2VhcmNoX3N0YXR1c19saW1pdCI7aToxO3M6MTA6InNlYXJjaF9jaWQiO2E6MTp7aTowO3M6MToiMCI7fXM6MjQ6InNlYXJjaF9hdXRob3JfZXhhY3RfbmFtZSI7czoxOiIxIjtzOjEyOiJzZWFyY2hfcXVlcnkiO3M6MTI6ItmD2YjYsdmI2YbYpyI7czoxOToic2VhcmNoX2JlZm9yZV9hZnRlciI7czoxOiIxIjtzOjE0OiJnZW5lcmljX3NlYXJjaCI7YTowOnt9fXM6MTQ6InNlYXJjaF9zb3J0X2J5IjtzOjk6Im9yZGVyX251bSI7czoxMjoic2VhcmNoX29yZGVyIjtzOjEwOiJkZXNjZW5kaW5nIjtzOjY6Im9mZnNldCI7aTowO3M6MTg6InNlYXJjaF9kb19hZHZhbmNlZCI7YjoxO30&pg={}'.format(i), headers={'User-Agent': 'Mozilla/5.0'})
  webpage = urlopen(req).read().decode('utf-8')
  soup = BeautifulSoup(webpage,"lxml")
  return [a.get("href") for a in soup.select(".section_title a")]

def save_links_to_file():
  i=1
  while len(extract_links(i))>0:
    tmp_list = extract_links(i)
    i+=1
    with open('hespress-links.txt', 'a') as f:
      for link in tmp_list:
        print(link, file=f, sep='\n')
      
#save_links_to_file()<jupyter_output><empty_output><jupyter_text>Extracting and filtering comments  from all collected links:*   Group comments (with at least one word related to covid-19) by month and write them into CSV files with their corresponding vote.*   Each CSV file will represent visitors' comments within a month.<jupyter_code>%cd /content/
def get_comments():
  with open(data_path+"hespress-links.txt") as source, open('comments-january.csv', 'a', newline='') as january, open('comments-february.csv', 'a', newline='') as february, open('comments-march.csv', 'a', newline='') as march, open('comments-april.csv', 'a', newline='') as april, open('comments-may.csv', 'a', newline='') as may, open('comments-june.csv', 'a', newline='') as june, open('comments-july.csv', 'a', newline='') as july :
    
    january_writer = csv.DictWriter(january, fieldnames=['vote', 'comment'])
    february_writer = csv.DictWriter(february, fieldnames=['vote', 'comment'])
    march_writer = csv.DictWriter(march, fieldnames=['vote', 'comment'])
    april_writer = csv.DictWriter(april, fieldnames=['vote', 'comment'])
    may_writer = csv.DictWriter(may, fieldnames=['vote', 'comment'])
    june_writer = csv.DictWriter(june, fieldnames=['vote', 'comment'])
    july_writer = csv.DictWriter(july, fieldnames=['vote', 'comment'])
    
    january_writer.writeheader()
    february_writer.writeheader()
    march_writer.writeheader()
    april_writer.writeheader()
    may_writer.writeheader()
    june_writer.writeheader()
    july_writer.writeheader()
 
    words = ("كورون","مرض","حجر","عزل","جائحة","داء","وباء","وبئة","علم","سارس","مستجد","سعال","لتهاب","خفاش","كوفي","عدوى","كمام","صحي",
"مصح","تنفس","عراض","تحاليل","تحليل","كشف","سلامة","لتزام","حالة","مناع","حالات","تعقيم","علاج","شفا","تشاف","موت","وفاة","وفايات","نهيار",
"سعار","لقاح","دواء","صاب","صين","كرونا","كورن","حمى", "تدابير","وقا","مستشف","مساف","رشاد","رصد","تباعد","جتماع","تهوية","مخالف","بؤر","مؤامر","مؤمر",
"طب","نظافة","غلاق","غلق","زمة","صندوق","رميد","مهيكل","تعليمات","مضاد","هيدر","كسي","كلور","كوين","رتفاع","مساعد","حدود","شكاي","تضامن","حاد")


    for link in source:
      req = Request('https://www.hespress.com{}'.format(link), headers={'User-Agent': 'Mozilla/5.0 (THE OUTSIDERS; Android 503; 404) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.136 Mobile Safari/537.36'})
      webpage = urlopen(req).read().decode('utf-8')
      soup = BeautifulSoup(webpage,"lxml")

      votes = [v.text.strip("\n") for v in soup.select('.result')]
      comments = [c.text.strip("\n") for c in soup.select('.comment_text')]
      dates = [d.text.strip("\n") for d in soup.select('.comment_date')]
      
      r = range(len(votes))
      for i in r:
          month = dates[i].split(" ")[2]
          for w in words:
              if w in comments[i]:
                  comments[i] = comments[i].replace("\n","")
                  if month=="يناير":
                    january_writer.writerow({'vote': votes[i], 'comment': comments[i]})
                    break
                  elif month=="فبراير":
                    february_writer.writerow({'vote': votes[i], 'comment': comments[i]})
                    break
                  elif month=="مارس":
                    march_writer.writerow({'vote': votes[i], 'comment': comments[i]})
                    break
                  elif month=="أبريل":
                    april_writer.writerow({'vote': votes[i], 'comment': comments[i]})
                    break
                  elif month=="ماي":
                    may_writer.writerow({'vote': votes[i], 'comment': comments[i]})
                    break
                  elif month=="يونيو":
                    june_writer.writerow({'vote': votes[i], 'comment': comments[i]})
                    break
                  elif month=="يوليوز":
                    july_writer.writerow({'vote': votes[i], 'comment': comments[i]})
                    break

#get_comments()<jupyter_output><empty_output><jupyter_text>Reading comments from CSV files<jupyter_code>df1 = pd.read_csv('https://github.com/h3t1/nlp-project/raw/master/data/comments-january.csv', sep=',',encoding='utf-8')
df2 = pd.read_csv('https://github.com/h3t1/nlp-project/raw/master/data/comments-february.csv', sep=',', encoding='utf-8')
df3 = pd.read_csv('https://github.com/h3t1/nlp-project/raw/master/data/comments-march.csv', sep=',', encoding='utf-8')
df4 = pd.read_csv('https://github.com/h3t1/nlp-project/raw/master/data/comments-april.csv', sep=',', encoding='utf-8')
df5 = pd.read_csv('https://github.com/h3t1/nlp-project/raw/master/data/comments-may.csv', sep=',', encoding='utf-8')
df6 = pd.read_csv('https://github.com/h3t1/nlp-project/raw/master/data/comments-june.csv', sep=',', encoding='utf-8')
df7 = pd.read_csv('https://github.com/h3t1/nlp-project/raw/master/data/comments-july.csv', sep=',', encoding='utf-8')

df5.head(10)<jupyter_output><empty_output><jupyter_text>Plotting the evolution of the number of comments related to covid-19 per month:<jupyter_code>labels = ("January","February","March","April","May","June","July[01-18]")
size = (len(df1.index), len(df2.index), len(df3.index), len(df4.index), len(df5.index), len(df6.index), len(df7.index))
ypos = np.arange(len(labels))
plt.bar(ypos,size)
plt.xticks(ypos,labels)
plt.ylabel("number of comments")
plt.title("Evolution of number of comments related to covid-19 per month")
plt.show()<jupyter_output><empty_output><jupyter_text>Distribution of votes on comments:In this snippet, we will try to answer the question of the usefulness of comments based on their corresponding votes.<jupyter_code>def classify(x):
      if x > 10:
        return "Excellent"
      elif x > 0:
        return "Good"
      elif x < -10:
        return "Terrible"
      elif x < 0:
        return "Disappointing"
      else:
        return "Neutral"
        
order=["Terrible","Disappointing","Neutral","Good","Excellent"]
palette=["#d24d57","#f1828d","#efedf5","#65c6bb","#89c4f4"]
fig, axes = plt.subplots(nrows=3,ncols=3,figsize=(24,20))

sns.countplot(df1["vote"].apply(classify), label="count", ax = axes[0,0], order=order, palette=palette)
axes[0,0].title.set_text('Distribution of votes in January')
sns.countplot(df2["vote"].apply(classify), label="count", ax = axes[0,1], order=order, palette=palette)
axes[0,1].title.set_text('Distribution of votes in Febrary')
sns.countplot(df3["vote"].apply(classify), label="count", ax = axes[0,2], order=order, palette=palette)
axes[0,2].title.set_text('Distribution of votes in March')
sns.countplot(df4["vote"].apply(classify), label="count", ax = axes[1,0], order=order, palette=palette)
axes[1,0].title.set_text('Distribution of votes in April')
sns.countplot(df5["vote"].apply(classify), label="count", ax = axes[1,1], order=order, palette=palette)
axes[1,1].title.set_text('Distribution of votes in May')
sns.countplot(df6["vote"].apply(classify), label="count", ax = axes[1,2], order=order, palette=palette)
axes[1,2].title.set_text('Distribution of votes in June')
sns.countplot(df7["vote"].apply(classify), label="count", ax = axes[2,0], order=order, palette=palette)
axes[2,0].title.set_text('Distribution of votes between the period of July 1 - July 18')
axes[2, 1].axis('off')
axes[2, 2].axis('off')

plt.show()<jupyter_output><empty_output><jupyter_text>Average length of comments per month:<jupyter_code>df1["length"] = df1["comment"].apply(len)
df2["length"] = df2["comment"].apply(len)
df3["length"] = df3["comment"].apply(len)
df4["length"] = df4["comment"].apply(len)
df5["length"] = df5["comment"].apply(len)
df6["length"] = df6["comment"].apply(len)
df7["length"] = df7["comment"].apply(len)

fig, axes = plt.subplots(nrows=3,ncols=3,figsize=(24,20))

axes[0,0].title.set_text("Average length of comments of January")
axes[0,1].title.set_text("Average length of comments of February")
axes[0,2].title.set_text("Average length of comments of March")
axes[1,0].title.set_text("Average length of comments of April")
axes[1,1].title.set_text("Average length of comments of May")
axes[1,2].title.set_text("Average length of comments of June")
axes[2,0].title.set_text("Average length of comments between the period of July 1 - July 18")

df1.length.plot(bins=100, kind="hist",range=(0,1000), ax = axes[0,0])
df2.length.plot(bins=100, kind="hist",range=(0,1000), ax = axes[0,1])
df3.length.plot(bins=100, kind="hist",range=(0,1000), ax = axes[0,2])
df4.length.plot(bins=100, kind="hist",range=(0,1000), ax = axes[1,0])
df5.length.plot(bins=100, kind="hist",range=(0,1000), ax = axes[1,1])
df6.length.plot(bins=100, kind="hist",range=(0,1000), ax = axes[1,2])
df7.length.plot(bins=100, kind="hist",range=(0,1000), ax = axes[2,0])
axes[2, 1].axis('off')
axes[2, 2].axis('off')
print("January summary:")
df1.describe()<jupyter_output>January summary:<jupyter_text>Preprocessing Steps:1.   Normalize hashtags (Example: لا لتمديد الحجر الصحي→ لا_لتمديد_الحجر_الصحي)2.   Remove mentions (Example: @user123)3.   Allow only arabic characters4.   Remove punctuations5.   Remove diacritics6.   Remove consecutive characters from a string7.   Remove words length less than 2 characters8.   Remove stopwords9.   Stem words<jupyter_code>try:
  stopwords = nltk.corpus.stopwords.words('arabic')
except:
  nltk.download('stopwords')
  stopwords = nltk.corpus.stopwords.words('arabic')

sw_file = urlopen("https://raw.githubusercontent.com/mohataher/arabic-stop-words/master/list.txt").read().decode('utf-8')
sw = [x for x in sw_file.split("\n")]
stopwords = set(stopwords + sw)

def remove_repeating_char(text):
    return re.sub(r'(.)\1+', r'\1\1', text)# keep only 1 repeat

arabic_diacritics = re.compile(""" ّ    | # Tashdid
                             َ    | # Fatha
                             ً    | # Tanwin Fath
                             ُ    | # Damma
                             ٌ    | # Tanwin Damm
                             ِ    | # Kasra
                             ٍ    | # Tanwin Kasr
                             ْ    | # Sukun
                             ـ     # Tatwil/Kashida
                         """, re.VERBOSE)

def remove_lt2(text):
  return re.sub(r'\b\w{,2}\b', '',text)

def remove_diacritics(text):
    return re.sub(arabic_diacritics, '', text)
    
def remove_punctuations(text):
    arabic_punctuations = '''`÷×؛<>_()*&^%][ـ،/:"؟.,'{}~¦+|!”…“–ـ'''
    #english_punctuations = string.punctuation
    #all_punctuations = set(arabic_punctuations + english_punctuations)
    for p in arabic_punctuations:
        if p in text:
            text = text.replace(p, '')
    return text

def remove_mention(text):
    return re.sub(r'@\S+', '', text)

def allow_only_ar(text):
    return re.sub(r'[^\u0600-\u06ff\u0750-\u077f\ufb50-\ufbc1\ufbd3-\ufd3f\ufd50-\ufd8f\ufd50-\ufd8f\ufe70-\ufefc\uFDF0-\uFDFD]+', ' ', text)

def hashtag_match(match_object):
        return match_object.group(1).replace('_', ' ')

def normalize_hashtag(text):
    return re.sub(r'#(\S+)', hashtag_match, text)

def remove_stopwords(text):
  word_tokens = wordpunct_tokenize(text)
  filtered_sentence = [w for w in word_tokens if not w in stopwords]
  return " ".join(filtered_sentence)

def process_text(text):
    
    clean_text = normalize_hashtag(text)
    clean_text = remove_mention(clean_text)
    clean_text = allow_only_ar(clean_text)
    clean_text = remove_punctuations(clean_text)
    clean_text = remove_diacritics(clean_text)
    clean_text = remove_repeating_char(clean_text)
    clean_text = remove_lt2(clean_text)
    clean_text = remove_stopwords(clean_text)
    
    tokens = list()
    st = ISRIStemmer()
    for w in wordpunct_tokenize(clean_text):
      tokens.append(st.stem(w))
    return tokens<jupyter_output><empty_output><jupyter_text>Word cloud:<jupyter_code>try:
  from ar_wordcloud import ArabicWordCloud
  from ar_wordcloud.utils import read_mask_image
except:
  !pip install ar_wordcloud
  from ar_wordcloud import ArabicWordCloud
  from ar_wordcloud.utils import read_mask_image

from imageio import imread

image = imread('https://i.imgur.com/cOegrUM.png')

mask_img = np.array(image) 

sentences  = df5["comment"].tolist()
sentences_as_one_string = " ".join(sentences)

sentences_as_one_string = allow_only_ar(sentences_as_one_string)
sentences_as_one_string = remove_punctuations(sentences_as_one_string)
sentences_as_one_string = remove_stopwords(sentences_as_one_string)

awc = ArabicWordCloud(background_color="white",mask=mask_img)
plt.figure(figsize=(17,17))
wc = awc.from_text(sentences_as_one_string)
wc.to_file('may_wc.png')
plt.imshow(wc)
plt.title("May's word cloud")<jupyter_output><empty_output><jupyter_text>Multinomial Naive Bayes classifier<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
frames = [df1, df2, df6, df7]
df = pd.concat(frames)
df = df[(df.vote > 2) | (df.vote < -1)]

X = CountVectorizer(analyzer = process_text, dtype="uint8").fit_transform(df["comment"]).toarray()
y = df["vote"].apply(lambda x: 'positive' if x >= 0 else 'negative')


X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.023)

NB_classifier = MultinomialNB()
NB_classifier.fit(X_train,y_train)
from sklearn.metrics import classification_report, confusion_matrix
y_predict_test = NB_classifier.predict(X_test)
cm = confusion_matrix(y_test,y_predict_test)
sns.heatmap(cm, annot=True, xticklabels = ['Negative', 'Positive'],yticklabels=['Negative','Positive'])
print(classification_report(y_test,y_predict_test))<jupyter_output>precision    recall  f1-score   support

    negative       0.61      0.42      0.50       256
    positive       0.87      0.93      0.90      1032

    accuracy                           0.83      1288
   macro avg       0.74      0.68      0.70      1288
weighted avg       0.82      0.83      0.82      1288
{'fraction_non_alphanumeric': 0.15590254571966705, 'fraction_numerical': 0.029831703019624522, 'mean_word_length': 4.400262467191601, 'pattern_counts': {'":': 7, '<': 53, '<?xml version=': 0, '>': 55, 'https://': 12, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '266088', 'license': 'MIT', 'n_tokens_mistral': 6498, 'n_tokens_neox': 6146, 'n_words': 1079, 'path': 'covid_19_feedbacks.ipynb', 'repo_name': 'h3t1/nlp-project', 'size': '16484'}