<jupyter_start><jupyter_text>1. Text Cleaning<jupyter_code>#########
# Import
#########

%%capture
import os
import sys
import pandas as pd 
import numpy as np
########################
# Set directories paths
########################

YOUR_PATH = "..."
proj_dir = os.path.join(YOUR_PATH, "net2020-main")
base_dir = os.path.join(YOUR_PATH, "net2020-main/Semantic_Group/text_preprocessing")
####################################
# Import text-cleaning utils folder
####################################

sys.path.append(os.path.join(base_dir, "utils"))
##############
# Import data
##############

os.chdir(proj_dir)

# import the csv file with all the comments and post togheter
comDB = pd.read_csv(r"database/com_liwc.csv", sep='\t', engine='python', encoding='utf-8')

# import the csv file with JUST the politicians post
postDB = pd.read_csv(r"database/postDB.csv", engine='python')
# create the Data Frame
df = pd.DataFrame(data=comDB)
df_post = pd.DataFrame(data=postDB)

df

# add a new colum with sequence numbers
df['Count']=1
df_post['Count']=1

# print all the DF
pd.set_option('display.max_columns', None)
pd.set_option('display.max_row', 5)

#df.head()
########################
# Set working directory
########################

os.chdir(base_dir)<jupyter_output><empty_output><jupyter_text>Data Analysis  Dimensions check Shape of original datasets<jupyter_code>print(f'Comments DF shape: \t{comDB.shape}')
print(f'Post DF shape: \t\t{postDB.shape}')<jupyter_output>Comments DF shape: 	(78175, 244)
Post DF shape: 		(10103, 246)<jupyter_text>Shape of datasets after appending column 'Count'<jupyter_code>print(f'Comments DF shape: \t{df.shape}')
print(f'Post DF shape: \t\t{df_post.shape}')<jupyter_output>Comments DF shape: 	(78175, 244)
Post DF shape: 		(10103, 246)<jupyter_text>Why adding column 'Count' we still have 246 columns in df_post dataframe?<jupyter_code>postDB.columns
postDB[postDB['Count']!=1].shape<jupyter_output><empty_output><jupyter_text>postDB dataframe already has a column 'Count' whose entry are all 1.  Columns comparison<jupyter_code>col_df = set([col for col in df.columns])
col_df_post = set([col for col in df_post.columns])

print(f'N. columns in df:\t{len(col_df)}')
print(f'N. columns in df_post:\t{len(col_df_post)}')

print(f'\nColumns in df not in df_post:\t{col_df - col_df_post}')
print(f'Columns in df_post not in df:\t{col_df_post - col_df}')<jupyter_output>N. columns in df:	244
N. columns in df_post:	246

Columns in df not in df_post:	set()
Columns in df_post not in df:	{'target1_2', 'Unnamed: 0'}<jupyter_text>What do these columns refer to? *   'Unnamed: 0': *   'target1_2' :   NaN values<jupyter_code>print('Columns with Nan in df:\n', [(col, df[col].isna().sum()) for col in df.columns if df[col].isna().sum()>0], '\n')
print('Columns with Nan in df_post:\n', [(col, df_post[col].isna().sum()) for col in df_post.columns if df_post[col].isna().sum()>0])<jupyter_output>Columns with Nan in df:
 [('p_GRUPPO_PE', 4215), ('p_PARTITO', 1692), ('p_text', 2), ('Target1', 40762), ('Target2', 71745), ('target1_s-p', 40868), ('target1_pol', 40868), ('c_text', 1), ('c_replyToUser', 314), ('c_replyToText', 733), ('Unnamed: 38', 78175)] 

Columns with Nan in df_post:
 [('p_GRUPPO_PE', 420), ('p_PARTITO', 286), ('p_text', 1), ('Target1', 6507), ('Target2', 9538), ('target1_s-p', 6513), ('target1_pol', 6513), ('c_replyToUser', 24), ('c_replyToText', 64), ('Unnamed: 38', 10103), ('target1_2', 9538)]<jupyter_text>For the moment we are concerned about the NaN in the columns related to posts and comments text.  NaN in comments dataframe<jupyter_code># Identify rows with NaN in post text in df (comments dataframe)
df[df['p_text'].isna()][['Origin_file_order']]
# Identify rows with NaN in comment text in df (comments dataframe)
df[df['c_text'].isna()][['Origin_file_order']]<jupyter_output><empty_output><jupyter_text>Row 45804 in comments dataframe can be removed since we have neither the text of the post nor the text of the comment associated with it.<jupyter_code>print('df shape before dropping row: \t', df.shape)
df = df[df['c_text'].notna()]
print('df shape after dropping row: \t', df.shape)
print('Number of Nan in comments text: ', df['c_text'].isna().sum())
df.shape<jupyter_output><empty_output><jupyter_text>NaN in posts dataframe<jupyter_code># Identify rows with NaN in post text in df_post (posts dataframe)
df_post[df_post['p_text'].isna()][['Origin_file_order']]<jupyter_output><empty_output><jupyter_text>Comments Text Preprocessing Let us create a dataframe containing only the comments' text<jupyter_code># comments = df[['c_text']].sample(n=1000, random_state=1).copy() # work with a sample
comments = df[['c_text']].copy()
comments.rename(columns={'c_text':'text'}, inplace=True)
###############################
# Set seed for reproducibility
###############################

import random
random.seed(0)
np.random.seed(0)
for i in list(np.random.choice(list(comments.index), 5)):
    print(f'Comment {i}')
    print(comments.loc[i]['text'], '\n')<jupyter_output>Comment 68269
@matteosalvinimi Grande capitano Avanti! !!!dai più valore alle forze dell ordine! !!!!LA DIVISA E LA LEGGE VA RISPETTATA DA TUTTI NESSUNO ESCLUSO! !!!!LA VOLETE CAPIREEEEE! !!!!! 

Comment 43567
@Ale_Mussolini_ Ma che dici! Forse tu fai,giustamente,propaganda a FI che non appoggia certo il governo! 

Comment 42613
Siamo è saremo sempre con lei comandante <NAME> <NAME> vero comandante a destra 🇮🇹 

Comment 45892
Si coprono l'uno con l'altro 

Comment 21243
Ha torto perché tenta una difesa peraltro senza argomentare,di un fatto che andrebbe condannato e basta e la sua patetica insistenza priva di argomenti sta diventando pure ridicola.Punto<jupyter_text>Word cloud with raw data What if we generate a word cloud with no-preprocessed text?<jupyter_code>from PIL import Image
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator

import matplotlib.pyplot as plt
% matplotlib inline
full_text = " ".join(comm for comm in comments['text'])

wordcloud = WordCloud(max_font_size=50, max_words=100, background_color="white").generate(full_text)
plt.figure()
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()<jupyter_output><empty_output><jupyter_text>The word cloud we get is full of so-called stop words, the only significant words we can recognize are names of parties or politicians. A bit of text pre-processing is therefore mandatory.  Text pre-processing There are differnt types of text preprocessing steps which can be applied and the choice of these steps depends on the tasks to be performed.For this initial step, our goal is to identify the most used words in the comments and the main topics of discussion.<jupyter_code>comments["text_clean"] = comments["text"].copy()<jupyter_output><empty_output><jupyter_text>Lower casing<jupyter_code>def lower_casing(text):
    return(text.lower())
comments["text_clean"] = comments["text_clean"].apply(lambda text: lower_casing(text))<jupyter_output><empty_output><jupyter_text>Removal of redundant spaces<jupyter_code>def remove_spaces(text):
    return ' '.join([token for token in text.split()])
comments["text_clean"] = comments["text_clean"].apply(lambda text: remove_spaces(text))<jupyter_output><empty_output><jupyter_text>Removal of patterns<jupyter_code>import re
import regex
from collections import Counter

def remove_patterns(text, patterns):
    for pattern in patterns:
        try: r = re.findall(pattern, text)
        except: r = regex.findall(pattern, text)
        for i in r:
            try: text = re.sub(re.escape(i), ' ', text)
            except: text = regex.sub(regex.escape(i), ' ', text)
    return text

def pattern_freq(docs, pattern):
    p_freq = Counter()
    for text in docs:
        p_found= re.findall(pattern, text)
        for p in p_found:
            p_freq[p] += 1
    return p_freq
%%capture
!pip install emoji
import emoji
PATTERNS = {'urls': re.compile(r'https?://\S+|www\.\S+'),
            'users': re.compile(r'@[\w]*'),
            #'hashtags': re.compile(r'#[\w]*'),
            'emojis': emoji.get_emoji_regexp(),
            'laughs': re.compile(r'\b(?:(?:hah+|ah+|hah+a|ah+a|lo+l) ?)+\b'),
            'blabla': re.compile(r'\b(?:(?:bla+) ?)+\b'),
            'dates': re.compile(r'\b(?:(?:[0-9]+[:\/,\.])+[0-9]+)+\b'),
            'digits': r'#\S+(*SKIP)(*FAIL)|\b\d+\s|\s\d+\s|\s\d+|\b\d+\W|\W\d+\W|\W\d+$' 
            }<jupyter_output><empty_output><jupyter_text>Before removing patterns we can answer following questions:* Which are the most used hashtags?* Which are most tagged users?* Are there frequent URLs?* Which are most frequent emojis/emoticons?<jupyter_code>hashtags_patt = re.compile(r'#[\w]*')

hashtags_freq = pattern_freq(comments['text'].values, hashtags_patt)       
hashtags_freq.most_common(10)
wordcloud = WordCloud(max_font_size=50, max_words=100, background_color="white").generate_from_frequencies(hashtags_freq)
plt.figure()
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()
users_freq = pattern_freq(comments['text'].values, PATTERNS['users'])        
users_freq.most_common(10)
wordcloud = WordCloud(max_font_size=50, max_words=100, background_color="white").generate_from_frequencies(users_freq)
plt.figure()
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()
urls_freq = pattern_freq(comments['text'].values, PATTERNS['urls'])        
urls_freq.most_common(10)
emojis_freq = pattern_freq(comments['text'].values, PATTERNS['emojis']) 
emojis_freq.most_common(10)
comments["text_clean"] = comments["text_clean"].apply(lambda text: remove_patterns(text, PATTERNS.values()))<jupyter_output><empty_output><jupyter_text>Unicode replacer<jupyter_code>from unicode_replacer import *
comments['text_clean'] = comments['text_clean'].apply(lambda text: UnicodeReplacer().replace(text))<jupyter_output><empty_output><jupyter_text>Removal of punctuation<jupyter_code>import string
punct = string.punctuation
print(f"Punctuation symbols: {punct}")
def remove_punctuation(text, keep=[]):
    punctuation = string.punctuation
    for p in keep:
        punctuation = punctuation.replace(p, "")
    return(''.join([char if not char in punctuation else ' ' for char in text]))
comments['text_clean'] = comments['text_clean'].apply(lambda text: remove_punctuation(text, keep="#',.")).apply(lambda text: remove_spaces(text))
for i in list(np.random.choice(list(comments.index), 5)):
    print(f'Comment {i}')
    print(comments.loc[i]['text'])
    print(comments.loc[i]['text_clean'], '\n')
    print()<jupyter_output>Comment 41993
Buona Pasqua!!!😃
buona pasqua 


Comment 55027
L'immagine è bellissima. Spero che quei giovani abbiano buone gambe.
l'immagine è bellissima. spero che quei giovani abbiano buone gambe. 


Comment 48601
E' proprio vero come dice Checco a destra c' e' la gnocca!
e' proprio vero come dice checco a destra c' e' la gnocca 


Comment 39512
Mo  lo buttano giu' tutto ..e faranno una moschea
mo lo buttano giu' tutto ..e faranno una moschea 


Comment 52621
Grande ,<NAME>
grande ,<NAME><jupyter_text>NLP with Spacy<jupyter_code>##########
# Install
##########

%%capture
!pip install --upgrade spacy
!python -m spacy download it_core_news_lg
#!python -m spacy download it_core_news_sm
import it_core_news_lg
nlp = it_core_news_lg.load()<jupyter_output><empty_output><jupyter_text>**Stop words collection**<jupyter_code>#################################################
# Import list of stopwords from it_stop_words.py
#################################################

from it_stop_words import get_italian_stop_words
my_it_stop_words = get_italian_stop_words()
#######################################
# Import stopwords from spacy and nltk
#######################################

from spacy.lang.it.stop_words import STOP_WORDS as it_spacy_stopwords

import nltk
nltk.download('stopwords')
it_nltk_stopwords = nltk.corpus.stopwords.words('italian')

# Gather stopwords
it_stopwords = set(it_spacy_stopwords) | set(it_nltk_stopwords) | my_it_stop_words<jupyter_output><empty_output><jupyter_text>**Stop words to keep**<jupyter_code>keep_words = set(['casa', 'citta', 'città', 'governo', 'lontano', 'milione',
                  'persona', 'piedi', 'torino', 'vicino'])

it_stopwords = it_stopwords - keep_words
for stopword in keep_words:
    nlp_vocab = nlp.vocab[stopword]
    nlp_vocab.is_stop = False

for stopword in it_stopwords:
    nlp_vocab = nlp.vocab[stopword]
    nlp_vocab.is_stop = True<jupyter_output><empty_output><jupyter_text>**Lemmatization errors**<jupyter_code>###########################################################
# Import lemmatization corrections from it_lemma_errors.py
###########################################################

from it_lemma_errors import get_it_lemma_errors
lemma_errors = get_it_lemma_errors()
def lemma(token):
    if token.text in lemma_errors:
        if token.pos_ in lemma_errors[token.text]:
            return lemma_errors[token.text][token.pos_]
    return token.lemma_<jupyter_output><empty_output><jupyter_text>**Hashtags as tokens**<jupyter_code>from spacy.tokenizer import _get_regex_pattern

# get default pattern for tokens that don't get split
re_token_match = _get_regex_pattern(nlp.Defaults.token_match)
# add your patterns (here: hashtags and in-word hyphens)
re_token_match = f"({re_token_match}|#\w+|\w+-\w+)"
# overwrite token_match function of the tokenizer
nlp.tokenizer.token_match = re.compile(re_token_match).match<jupyter_output><empty_output><jupyter_text>**Text processing**<jupyter_code>text_nlp = comments["text_clean"].apply(lambda text: nlp(text))
comments['text_nlp'] = text_nlp
# Run this cell to save the nlp-comments

# comments['text_nlp'].to_pickle(os.path.join(base_dir, 'comments_nlp'))
# Run this cell to load the nlp-comments

# comments['text_nlp'] = pd.read_pickle(os.path.join(base_dir, 'comments_nlp'))
print(f"{'Token':<20}\t{'Lemma':<20}\t{'POS':<20}\t{'is-stop':<8}\t{'is-punct':<8}")
for token in comments['text_nlp'].loc[0]:
    print(f"{token.text:<20}\t{lemma(token):<20}\t{token.pos_:<20}\t{token.is_stop or lemma(token) in it_stopwords:^8}\t{token.is_punct:^8}")<jupyter_output>Token               	Lemma               	POS                 	is-stop 	is-punct
c'                  	c'                  	PRON                	   1    	   0    
è                   	essere              	VERB                	   1    	   0    
poco                	poco                	ADV                 	   1    	   0    
da                  	da                  	ADP                 	   1    	   0    
dire                	dire                	AUX                 	   1    	   0    
questa              	questo              	PRON                	   1    	   0    
è                   	essere              	VERB                	   1    	   0    
gente               	gente               	NOUN                	   0    	   0    
che                 	che                 	SCONJ               	   1    	   0    
la                  	la                  	DET                 	   1    	   0    
guerra              	guerra              	NOUN                	   0    	   0    
c'                  	c'     [...]<jupyter_text>Removal of Stop-Words<jupyter_code>def remove_stop(tokens):
    return(' '.join([lemma(token) for token in tokens if not (token.is_stop or lemma(token) in it_stopwords)])) 
comments['text_clean'] = comments["text_nlp"].apply(lambda tokens: remove_stop(tokens))<jupyter_output><empty_output><jupyter_text>Further removal of punctuation<jupyter_code>comments['text_clean'] = comments['text_clean'].apply(lambda text: remove_punctuation(text, keep=['#']))
comments["text_clean"] = comments['text_clean'].apply(lambda text: remove_spaces(text))
import random
for i in list(np.random.choice(list(comments.index), 20)):
    print(f'Comment {i}')
    print(comments.loc[i]['text'])
    print(comments.loc[i]['text_clean'])
    print()<jupyter_output>Comment 684
Iniziative per europee? E cioè? Ciao
iniziativa europeo cioè ciao

Comment 55761
Forza presidente Zingaretti, noi siamo con lei
presidente zingaretti

Comment 60407
Buon lavoro Mara 😘💚💚💚
buon mara

Comment 61007
sono perfatamento dacordo con voi forza sempre aventi
perfatamento dacordo

Comment 64483
@DSantanche https://t.co/LlkwGWtmeM


Comment 26247
@cozzolino62 @pdnetwork Visto che si stanno dimostrando meglio dei vostri, pensi a come siete ridotti voi sinistri.
dimostrare ridurre sinistro

Comment 64222
@Ale_Mussolini_ @Utente_Generico @Utente_Generico @Utente_Generico Forza Alessandra! :-)
alessandra

Comment 35951
@mrctrdsh A che ti aspetti dal "Fatto"? Bordin un gigante vicino a quei scribacchini.
aspetto bordare gigante vicino scribacchini

Comment 55583
Molte mamme hanno e crescono i figli da sole....ci sono anche mamme che non sono capaci di farlo come ci sono papà che è meglio perderli che averli, il discorso è ampio.comunque ognuno  può pensarla come crede .
mam[...]<jupyter_text>Resulting word-cloud<jupyter_code>full_cleaned_text = ' '.join([doc for doc in comments['text_clean']])

wordcloud = WordCloud(max_font_size=50, max_words=100, background_color="white").generate(full_cleaned_text)
plt.figure()
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()<jupyter_output><empty_output><jupyter_text>Save cleaned text<jupyter_code># comments['text_clean'].to_pickle(os.path.join(base_dir, 'comments_cleaned'))
# comments['text_clean'] = pd.read_pickle(os.path.join(base_dir, 'comments_cleaned'))<jupyter_output><empty_output><jupyter_text>Text Cleaning from function See text_preprocessing folder<jupyter_code># Note: text_preprocessing folder has to be updasted. We'll do it after fixing text-preprocessing problems
'''
# Import list of stopwords from it_stop_words.py
import sys
sys.path.append(os.path.join(base_dir, "Semantic_Group/text_preprocessing"))
from text_cleaning import *

cleaned_text = clean_content(comments['text'])
'''
'''
full_cleaned_text = ' '.join(cleaned_text)

wordcloud = WordCloud(max_font_size=50, max_words=100, background_color="white").generate(full_cleaned_text)
plt.figure()
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()
'''<jupyter_output><empty_output><jupyter_text>Cleaned Data Analysis  Most popular words<jupyter_code>def words_freq(docs):
    words_count = Counter()
    for text in docs:
        for word in text.split():
            words_count[word] += 1
    return words_count
words_count = words_freq(comments['text_clean'])
print('Total words\n')
len(words_count.most_common())
print('Most common words\n')
words_count.most_common(50)
less_common = [(w, wc) for (w, wc) in words_count.most_common()[:-51:-1]]
for (w, wc) in less_common: print((w, wc), end='\n')
import seaborn as sns
counts = [wc for (w, wc) in words_count.most_common()]
sns.histplot(counts, bins=[1,2,3,4,5,6,7,8,9,10], discrete=True, binrange=[1,10])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15428330877709956, 'fraction_numerical': 0.019101241843822354, 'mean_word_length': 4.671441360787824, 'pattern_counts': {'":': 0, '<': 125, '<?xml version=': 0, '>': 119, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '30', 'dir': 'jupyter-structured-clean-dedup', 'id': '272503', 'license': 'CC0-1.0', 'n_tokens_mistral': 6941, 'n_tokens_neox': 6492, 'n_words': 1546, 'path': 'Semantic_Group/text_preprocessing/text_cleaning.ipynb', 'repo_name': 'Nat-aho/net2020', 'size': '19041'}