<jupyter_start><jupyter_text>Translate es --> ar --> es<jupyter_code>import pandas as pd
from tqdm.auto import tqdm
from googletrans import Translator
translator = Translator()
df = pd.read_csv("data/train_with_translations_clean.csv")
df_test = pd.read_csv("data/test_with_translations_clean.csv")
sentences_es = list(df.Pregunta.values)
print(f'Amount of sentences {len(sentences_es)}')
sentences_es[:2]
sentences_es_test = list(df_test.Pregunta.values)
print(f'Amount of sentences {len(sentences_es_test)}')
sentences_es_test[:2]
translations_ar = []
for sent in tqdm(sentences_es):
    translation = translator.translate(sent, src="es", dest="ar").text
    translations_ar.append(translation)
print(f'Amount sentences en: {len(translations_ar)}')

translations_es_back = []
for sent in tqdm(translations_ar):
    translation = translator.translate(sent, src="ar", dest="es").text
    translations_es_back.append(translation)
print(f'Amount sentences en: {len(translations_es_back)}')

df["Pregunta_T4_ar"] = translations_es_back


translations_ar = []
for sent in tqdm(sentences_es_test):
    translation = translator.translate(sent, src="es", dest="ar").text
    translations_ar.append(translation)
print(f'Amount sentences en: {len(translations_ar)}')

translations_es_back = []
for sent in tqdm(translations_ar):
    translation = translator.translate(sent, src="ar", dest="es").text
    translations_es_back.append(translation)
print(f'Amount sentences en: {len(translations_es_back)}')

df_test["Pregunta_T4_ar"] = translations_es_back
!pip install joblib
from textacy.preprocess import preprocess_text as textacy_preprocess
import re
EMOJI_PATTERN = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           "]+", flags=re.UNICODE)


def remove_stopwords(text):
    """Remove stop words from list of tokenized words."""
    new_words = [word for word in text.split() if word not in STOPWORDS_SET]
    return ' '.join(new_words)


def remove_emoticons(text):
    """Remove emoticos."""
    return EMOJI_PATTERN.sub('', text)


def to_lowercase(text):
    """To lowercase."""
    return text.lower()


def preprocess_text(text, fix_unicode=True, lowercase=True,
                    no_urls=True, no_emails=True,
                    no_phone_numbers=True,
                    no_numbers=True, no_currency_symbols=True,
                    no_punct=True, no_accents=True):
    """Preprocess text."""
    clean_text = ""
    if text:
        clean_text = remove_emoticons(text)
        if lowercase:
            clean_text = to_lowercase(clean_text)
#        clean_text = remove_stopwords(clean_text)
        clean_text = textacy_preprocess(clean_text, fix_unicode=fix_unicode,
                                        lowercase=lowercase,
                                        no_urls=no_urls, no_emails=no_emails,
                                        no_phone_numbers=no_phone_numbers,
                                        no_numbers=no_numbers,
                                        no_currency_symbols=no_currency_symbols,
                                        no_punct=no_punct,
                                        no_accents=no_accents)
    return clean_text
df["clean_txt_T4_ar"] = df["Pregunta_T4_ar"].apply(lambda x: preprocess_text(x))
df_test["clean_txt_T4_ar"] = df_test["Pregunta_T4_ar"].apply(lambda x: preprocess_text(x))
df.head()
df_ = pd.read_csv("data/train_with_translations_clean.csv")
df_test_ = pd.read_csv("data/test_with_translations_clean.csv")
df_["clean_txt_T4_ar"] = df["clean_txt_T4_ar"]
df_test_["clean_txt_T4_ar"] = df_test["clean_txt_T4_ar"]
df_["Pregunta_T4_ar"] = df["Pregunta_T4_ar"]
df_test_["Pregunta_T4_ar"] = df_test["Pregunta_T4_ar"]
df_.to_csv("data/train_with_translations_clean.csv", index=False)
df_test_.to_csv("data/test_with_translations_clean.csv", index=False)
import seaborn as sns
columns = ["clean_txt_T1", "clean_txt_T2_fr", "clean_txt_T3_pt", "clean_txt_T4_ar"]
df = df_[columns + ["clean_txt", "Intencion"]]
df0 = df[["clean_txt", "Intencion"]].copy()
for col in columns:
    thr = df0["Intencion"].value_counts().values[0] // 2
    print(thr)
    cats_to_popu = [ k for k, v in dict(df0["Intencion"].value_counts()).items() if v < thr]
    print(f'Amount categories to populate : {len(cats_to_popu)}')
    df1 = df[df["Intencion"].isin(cats_to_popu)][[col, "Intencion"]].copy()
    df1.columns = ["clean_txt", "Intencion"]
    df0 = pd.concat([df0, df1])
sns.lineplot(data=pd.DataFrame(list(df0.Intencion.value_counts().values)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1233361064891847, 'fraction_numerical': 0.016430948419301164, 'mean_word_length': 3.502808988764045, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '489871', 'license': 'MIT', 'n_tokens_mistral': 1704, 'n_tokens_neox': 1640, 'n_words': 268, 'path': 'notebooks/EDA.ipynb', 'repo_name': 'nicoperetti/metadata-sadosky-santander', 'size': '4808'}