<jupyter_start><jupyter_text>Preprocessing<jupyter_code>import pandas as pd
import csv
from bs4 import BeautifulSoup
import re

import sys
import os

#src
#root_project = "/content/ReSt/"
root_project = "/Users/Alessandro/Dev/repos/ReSt/"
#root_project = "/home/jupyter/ReSt/"
sys.path.append(root_project)

from src.data.preprocessing.preprocessor import PreProcessor

%load_ext autoreload
%autoreload 2<jupyter_output>/Users/Alessandro/anaconda3/lib/python3.7/site-packages/ekphrasis/classes/exmanager.py:14: FutureWarning: Possible nested set at position 42
  regexes = {k.lower(): re.compile(self.expressions[k]) for k, v in<jupyter_text>load dataset<jupyter_code># names of files to read from
train_val_AB_TSV        = root_project + '/dataset/haspeede2/raw/haspeede2_dev/haspeede2_dev_taskAB.tsv'

test_tweets_AB_TSV      = root_project + '/dataset/haspeede2/raw/haspeede2_test/haspeede2_test_taskAB-tweets.tsv'
test_news_AB_TSV        = root_project + '/dataset/haspeede2/raw/haspeede2_test/haspeede2-test_taskAB-news.tsv'

reference_tweets_AB_TSV = root_project + '/dataset/haspeede2/raw/haspeede2_reference/haspeede2_reference_taskAB-tweets.tsv'
reference_news_AB_TSV   = root_project + '/dataset/haspeede2/raw/haspeede2_reference/haspeede2_reference_taskAB-news.tsv'<jupyter_output><empty_output><jupyter_text>dev set<jupyter_code>#df = pd.read_csv(train_val_AB_TSV, sep='\t')
#df.rename(columns={"text ": "text"}, inplace=True)
df = pd.read_csv(train_val_AB_TSV, delimiter=r'\t', header=None, engine='python')
df.columns =['id', 'text', 'hs', 'stereotype'] 
df = df.drop([0])
print('Number of dev sentences: {:,}\n'.format(df.shape[0]))<jupyter_output>Number of dev sentences: 6,839<jupyter_text>test set<jupyter_code>#Test Tweets
df_tweets_test = pd.read_csv(test_tweets_AB_TSV, delimiter=r'\t', header=None, engine='python')

#Test News
df_news_test = pd.read_csv(test_news_AB_TSV, delimiter=r'\t', header=None, engine='python')<jupyter_output><empty_output><jupyter_text>reference set<jupyter_code>#Reference Tweets
df_tweets_reference = pd.read_csv(reference_tweets_AB_TSV, delimiter=r'\t', header=None, engine='python')
df_tweets_reference.columns =['id', 'text', 'hs', 'stereotype'] 

#Reference News
df_news_reference = pd.read_csv(reference_news_AB_TSV, delimiter=r'\t', header=None, engine='python')
df_news_reference.columns =['id', 'text', 'hs', 'stereotype'] 
# for windows
#Reference Tweets
#df_tweets_reference = pd.read_csv(reference_tweets_AB_TSV, delimiter=r'\t', header=None, engine='python')
#df_app1 = pd.DataFrame([[11834, "@user A me pare una scelta politica suicida puntare tutto su una battaglia sicuramente perdente in favore dell'immigrazione incontrollata...Meglio così, spariranno più velocemente!", 1, 0]], columns=['id', 'text', 'hs', 'stereotype'])
#df_tweets_reference.rename(columns={'11834': 'id', "@user A me pare una scelta politica suicida puntare tutto su una battaglia sicuramente perdente in favore dell'immigrazione incontrollata...Meglio così, spariranno più velocemente!": 'text', '1':'hs', '0':'stereotype'}, inplace=True)
#df_tweets_reference = df_tweets_reference.append(df_app1, ignore_index=True)

#Reference News
#df_news_reference = pd.read_csv(reference_news_AB_TSV, delimiter=r'\t', header=None, engine='python')
#df_app2 =pd.DataFrame([[11976, "Andate pure là, tanto quei fessi degli italiani.... Capito perché ci invadono? Il clandestino confessa", 1, 0]], columns=['id', 'text', 'hs', 'stereotype'])
#df_news_reference.rename(columns={'11976': 'id', "Andate pure là, tanto quei fessi degli italiani.... Capito perché ci invadono? Il clandestino confessa": 'text', '1':'hs', '0':'stereotype'}, inplace=True)
#df_news_reference = df_news_reference.append(df_app2, ignore_index=True)
prp = PreProcessor()
df_test = df.loc[4833:4834].copy()
df_test<jupyter_output><empty_output><jupyter_text>Dev<jupyter_code>save_path =  "/Users/Alessandro/Dev/repos/ReSt/dataset/haspeede2/preprocessed/dev/dev.csv"
prp.preprocess_df(df=df, column_name='text', path=save_path, verbose=True)<jupyter_output>0%|          | 0/6839 [00:00<?, ?it/s]<jupyter_text>Reference Twitter<jupyter_code>save_path =  "/Users/Alessandro/Dev/repos/ReSt/dataset/haspeede2/preprocessed/reference/reference_tweets.csv"
prp.preprocess_df(df=df_tweets_reference, column_name='text', path=save_path, verbose=True)<jupyter_output>0%|          | 0/1263 [00:00<?, ?it/s]<jupyter_text>Reference News<jupyter_code>save_path =  "/Users/Alessandro/Dev/repos/ReSt/dataset/haspeede2/preprocessed/reference/reference_news.csv"
prp.preprocess_df(df=df_news_reference, column_name='text', path=save_path, verbose=True)<jupyter_output>11%|█         | 55/500 [00:00<00:00, 546.43it/s]<jupyter_text>-----------------------------------------------------------------------  TEST<jupyter_code>from ekphrasis.classes.preprocessor import TextPreProcessor
from ekphrasis.classes.tokenizer import SocialTokenizer
from ekphrasis.dicts.emoticons import emoticons
from nltk.tokenize import TweetTokenizer
social_token = SocialTokenizer(lowercase=False).tokenize

text_processor = TextPreProcessor(
    normalize=['url', 'email', 'user', 'percent', 'money', 'phone', 'time', 'date'],
    #annotate={, "elongated", 'emphasis', 'censored'},
    fix_html=True,  # fix HTML tokens
    #unpack_hashtags=True, 
    # select a tokenizer. You can use SocialTokenizer, or pass your own
    # the tokenizer, should take as input a string and return a list of tokens
    tokenizer=SocialTokenizer(lowercase=False).tokenize, 
    #tokenizer=TweetTokenizer().tokenize,
    
    # list of dictionaries, for replacing tokens extracted from the text,
    # with other expressions. You can pass more than one dictionaries.
    dicts=[emoticons]
)


text = "@user Questo per dimostrare agli idioti buonisti di sinistra filoimmigrazionisti che nessuno vuole "
text_processor.pre_process_doc(text)
social_token(":):)")

import wordninja
from src.data.preprocessing.dicts.hashtags import know_hashtag_words, fixed_hashtags
from src.data.preprocessing.string_manipulation import add_space_before_hashtag, add_space_before_capital_words
segmenter = wordninja.LanguageModel(italian_gzip)

def split_hashtag(hashtag, segmenter, delimiter = ("<",">")):
    """
    Splits the hashtags in the text into words and encloses them between < and >.

    Example: #iostoconsalvini -> < io sto con salvini >

    Parameters
    ----------
    text : str

    Returns
    -------
    str
    """
    
    #remove hashtag symb
    hashtag = re.sub("#","",hashtag)
    if hashtag.lower() not in know_hashtag_words:
        splitted_hashtag = ' '.join(segmenter.split(add_space_before_capital_words(hashtag)))
        if splitted_hashtag in fixed_hashtags:
            splitted_hashtag = fixed_hashtags[splitted_hashtag]
        return delimiter[0]+" "+splitted_hashtag+" "+delimiter[1] 
    else:
        return delimiter[0]+" "+hashtag+" "+delimiter[1] 

def get_splitted_hashtags(text, segmenter):
    hashtags = re.findall(r'#\S+', add_space_before_hashtag(text))
    return {hashtag: split_hashtag(hashtag, segmenter) for hashtag in hashtags}

def split_hashtags(text, segmenter):
    hashtags = get_splitted_hashtags(text, segmenter)
    return ' '.join([hashtags[word] if word in hashtags else word for word in add_space_before_hashtag(text).split()])


text = "Studentessa cinese morta fermato un nomade di 20anni e URL denunciato nomade sedicenne tutti del vicino campo nomade SkyTG24" 
text = text_processor.pre_process_doc(text)
text = re.sub(r"(?i)(?<=\d)(?=[a-z])|(?<=[a-z])(?=\d)"," ", ' '.join(text))
split_hashtags(text, segmenter)
text = "!!!! ADDIO:) c*zzo l'ciao come stai:) 22/22 20€ €19 Far West a Roma': rom 100euro 25enne senza patente semina il panico, inseguimento, incidenti e feriti – webitalia360... URL "
#text = "#MovimentOnesti #cIaOGiubileo#GiubileodellaMisericordia #Roma "
text_processor.pre_process_doc(text)
text = "#MovimentOnesti #ciaoCiaoCome#ristoranteitaliano #Giubileo#GiubileodellaMisericordia #Roma "
split_hashtags(add_space_before_hashtag(text))
from ekphrasis.classes.exmanager import ExManager
import re

regexes = ExManager().get_compiled()
text = "prova :) (: :):)"
for item in ["ltr_face", "rtl_face"]:
        text = re.sub(regexes[item], r' \g<0> ', text)
text

regexes
def handle_elongated_match(m):
        text = m
        # normalize to at most 2 repeating chars
        text = regexes["normalize_elong"].sub(r'\1\1', text)
        text = re.compile(r'([!?.,;]){2,}', re.UNICODE).sub(r' \1\1 ', text)
        text = re.compile(r'([∞¶°ç◊\(\)€$/:?.,;{~!"^`\[\]])', re.UNICODE).sub(r' \1 ', text)
        return text
text = "!!!!!!!!- {[◊@#¶Ç∞◊ciao]}()_ci-ao_ 20$€$ #ciao-ciao??? ,,,,,ciiiiiaooooooooo....."
' '.join(handle_elongated_match(text).split())
import emoji

EMOJI_PATTERN = re.compile(
    "(["
    "\U0001F1E0-\U0001F1FF"  # flags (iOS)
    "\U0001F300-\U0001F5FF"  # symbols & pictographs
    "\U0001F600-\U0001F64F"  # emoticons
    "\U0001F680-\U0001F6FF"  # transport & map symbols
    "\U0001F700-\U0001F77F"  # alchemical symbols
    "\U0001F780-\U0001F7FF"  # Geometric Shapes Extended
    "\U0001F800-\U0001F8FF"  # Supplemental Arrows-C
    "\U0001F900-\U0001F9FF"  # Supplemental Symbols and Pictographs
    "\U0001FA00-\U0001FA6F"  # Chess Symbols
    "\U0001FA70-\U0001FAFF"  # Symbols and Pictographs Extended-A
    "\U00002702-\U000027B0"  # Dingbats
    "\u2600-\u26FF"
    "])"
    )

def add_space_between_emojies(text):
    text = re.sub(EMOJI_PATTERN, r' \1 ', text)
    return text

text = "video▶️ URL"
add_space_between_emojies(text)
regexes["emoji"].sub(r' \g<0> ', "video▶️ URL")
from ekphrasis.classes.segmenter import Segmenter

text = "l’immigrazione’"
text.replace("’", "'")
import wordninja

wn = wordninja.LanguageModel(italian_gzip)
wn.split("Cos'è")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1475193167954453, 'fraction_numerical': 0.029280195201301342, 'mean_word_length': 5.27359693877551, 'pattern_counts': {'":': 4, '<': 42, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '390465', 'license': 'MIT', 'n_tokens_mistral': 3892, 'n_tokens_neox': 3662, 'n_words': 683, 'path': 'notebooks/preprocessing/generate_preprocessed_data.ipynb', 'repo_name': 'alessandrocuda/ReSt', 'size': '9836'}