<jupyter_start><jupyter_text>Analyse<jupyter_code>df.head()
df.tail()
print("Formde du dataframe : ", df.shape)
df.isna().sum()
print("Pourcentage des valeurs nulles :")
print("Train set : ")
print((df.isna().sum() / df.shape[0]) * 100)
df.dtypes
view_size = 30
view_df = df.iloc[:view_size,:].copy()

for _, row in view_df.iterrows():
  print(row["review_body"])
# plt.figure(figsize=(15,10))
# df["review_date"].size().sort_values(ascending=False).plot.bar()
# plt.xticks(rotation=50)
# plt.xlabel("Date du commentaire")
# plt.ylabel("Nombre de commentaires")
# plt.show();
from PIL import Image
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
wordcloud = WordCloud().generate(' '.join(df['review_body']))
plt.figure(figsize=(10,8))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()<jupyter_output><empty_output><jupyter_text>Nettoyage 1. Emojis<jupyter_code>def remove_emoji(text):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    return emoji_pattern.sub(r'', text)

remove_emoji("Omg another Earthquake 😔😔")
# Suppression des Emojis
df["review_body"] = df["review_body"].apply(remove_emoji)<jupyter_output><empty_output><jupyter_text>2. Suppression des hashtags<jupyter_code># Suppression des Hashtags
df["has_hashtag"] = df["review_body"].str.contains("#")
df["review_body"] = df['review_body'].str.replace('#', '')
# Encodage de la colonne hashtag
hasghtag_lbl = LabelEncoder()
if "has_hashtag" in df.columns:
    df["has_hashtag"] = hasghtag_lbl.fit_transform(df["has_hashtag"])
df.head()
df.tail()
print("Nombre de commentaires qui ont possédé un hashtag avant le nettoyage : ", df["has_hashtag"].sum())<jupyter_output>Nombre de commentaires qui ont possédé un hashtag avant le nettoyage :  2<jupyter_text>3. Liens<jupyter_code>def remove_urls(text):
  #return re.sub(r'http\S+', '', text)
   return re.compile(r'https?://\S+|www\.\S+').sub(r'',text)
# Suppression des liens
df["review_body"] = df["review_body"].apply(remove_urls)
df.head()<jupyter_output><empty_output><jupyter_text>4. Ponctutation<jupyter_code>df["review_body"] = df["review_body"].str.replace('[{}]'.format(string.punctuation), '')
for _, row in df.iloc[:20,:].iterrows():
  print(row["review_body"])<jupyter_output>Have to admit I am not a great fan of McDonalds but it is a great place for a quickish snack I doubt it can really be classed as a Fast Food establishment anymoreThe one on the Champs Elysees occupies an ideal position from the Companys point of view just a 100 mtrs from the Arc de Triomphe it is extremely busy but a large restaurant with an upstairs seating area which means I was coping even at lunchtimeThe food was no different to any other McDonalds I have ever visited and well priced compared with any place to eat near a Paris landmarktourist attraction we saw coffee on sale near the Eifel Tower at 16 euros per cup were charge 22 euros for a cup of tea small beer and small bottle of water more than half a mile from the tower so that in itself is a major pointOrdering is almost totally be cardautomatic ordering machine which I personally find a little difficult to understand but would be childs play to most regular uses and there does seem to be ample machines within the takeawaycaf[...]<jupyter_text>5. Tags HTML<jupyter_code>def remove_html(text):
    html=re.compile(r'<.*?>')
    return html.sub(r'',text)

df["review_body"] = df["review_body"].astype(str)
df['review_body'] = df['review_body'].apply(lambda x : remove_html(x))<jupyter_output><empty_output><jupyter_text>6. Correction de l'ortographe<jupyter_code>spell = SpellChecker()
def correct_spellings(text):
    corrected_text = []
    misspelled_words = spell.unknown(text.split())
    for word in text.split():
        if word in misspelled_words:
            corrected_text.append(spell.correction(word))
        else:
            corrected_text.append(word)
    return " ".join(corrected_text)

# df['review_body'] = df['review_body'].apply(lambda x : correct_spellings(x))<jupyter_output><empty_output><jupyter_text>Visualisation des mots les plus courants :<jupyter_code>data = df["review_body"].copy()
data.head()
import matplotlib
top_N = 10

txt = data.str.lower().str.replace(r'\|', ' ').str.cat(sep=' ')
words_ = nltk.tokenize.word_tokenize(txt)
word_dist = nltk.FreqDist(words_)

stopwords_ = nltk.corpus.stopwords.words('english')
words_except_stop_dist = nltk.FreqDist(w for w in words_ if w not in stopwords_) 

print('All frequencies, including STOPWORDS:')
print('=' * 60)
rslt = pd.DataFrame(word_dist.most_common(top_N),
                    columns=['Word', 'Frequency'])
print(rslt)
print('=' * 60)

rslt = pd.DataFrame(words_except_stop_dist.most_common(top_N),
                    columns=['Word', 'Frequency']).set_index('Word')

matplotlib.style.use('ggplot')

rslt.plot.bar(rot=0);<jupyter_output>All frequencies, including STOPWORDS:
============================================================
  Word  Frequency
0  the       1850
1  and       1120
2    a        980
3   to        960
4  was        692
5   in        592
6    i        578
7   of        512
8   it        458
9   is        416
============================================================<jupyter_text>7. Tokenization<jupyter_code>def tokenize(text):
  return nltk.word_tokenize(text)

df['review_body'] = df['review_body'].apply(tokenize)
df.head()<jupyter_output><empty_output><jupyter_text>8. Chiffres<jupyter_code>do_they = list()
for _, row in df.iterrows():
  result = False
  for s in row["review_body"]:
    if s.isdigit():
        result = True
  do_they.append(result)
df["had_number"] = do_they
df.head()<jupyter_output><empty_output><jupyter_text>9. Stopwords<jupyter_code>sp = spacy.load('en_core_web_sm')

all_stopwords = sp.Defaults.stop_words

def remove_stopwords(text):
  return [word for word in text if not word in all_stopwords]
df['review_body'] = df['review_body'].apply(remove_stopwords)<jupyter_output><empty_output><jupyter_text>10. Chiffres<jupyter_code>for _, row in df.iterrows():
  for index, word in enumerate(row["review_body"]):
    if word.isdigit():
      row["review_body"].pop(index)
df.head()
df.isna().sum()<jupyter_output><empty_output><jupyter_text>Count Vectorizer (BOW)<jupyter_code>def text_process(mess):
    # Check characters to see if they are in punctuation
    nopunc = [char for char in mess if char not in string.punctuation]

    # Join the characters again to form the string.
    nopunc = ''.join(nopunc)
    
    # Now just remove any stopwords
    return [word for word in nopunc.split() if word.lower() not in stopwords.words('english')]
df.head()
bow_transformer = CountVectorizer(analyzer=text_process).fit(df['review_body'])
bow_transformer.vocabulary_
print("Nombre de mots dans le vocabulaire : ", len(bow_transformer.vocabulary_))
commentaire = df['review_body'][3]
print(commentaire)
print(bow_transformer.transform([commentaire]))
commentaires_bow = bow_transformer.transform(df['review_body'])
print('Taille de la matrice ', commentaires_bow.shape)
print(commentaires_bow.nnz)<jupyter_output>540<jupyter_text>TF-IDF<jupyter_code>tfidf_transformer = TfidfTransformer().fit(commentaires_bow)
tfidf = tfidf_transformer.transform(commentaires_bow)
print(tfidf)
tfidf_transformer = TfidfTransformer().fit(commentaires_bow)
commentaires_tfidf = tfidf_transformer.transform(commentaires_bow)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13998977243671695, 'fraction_numerical': 0.021222193812324215, 'mean_word_length': 4.803412462908012, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '657538', 'license': 'MIT', 'n_tokens_mistral': 2795, 'n_tokens_neox': 2659, 'n_words': 605, 'path': 'CM.ipynb', 'repo_name': 'MoezAbid/Community_Management_McDonald-s', 'size': '7822'}