<jupyter_start><jupyter_text>Instalando pacotes Transformers<jupyter_code>! pip install transformers
! pip install sentencepiece<jupyter_output>Collecting sentencepiece
  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)
[K     |████████████████████████████████| 1.2 MB 5.0 MB/s 
[?25hInstalling collected packages: sentencepiece
Successfully installed sentencepiece-0.1.96<jupyter_text>Dataset<jupyter_code>! pip install datasets
! pip install simpletransformers<jupyter_output>Collecting simpletransformers
  Downloading simpletransformers-0.62.2-py3-none-any.whl (231 kB)
[?25l
[K     |█▍                              | 10 kB 28.1 MB/s eta 0:00:01
[K     |██▉                             | 20 kB 18.2 MB/s eta 0:00:01
[K     |████▎                           | 30 kB 10.5 MB/s eta 0:00:01
[K     |█████▊                          | 40 kB 8.9 MB/s eta 0:00:01
[K     |███████                         | 51 kB 5.1 MB/s eta 0:00:01
[K     |████████▌                       | 61 kB 5.6 MB/s eta 0:00:01
[K     |██████████                      | 71 kB 5.4 MB/s eta 0:00:01
[K     |███████████▍                    | 81 kB 6.1 MB/s eta 0:00:01
[K     |████████████▊                   | 92 kB 4.6 MB/s eta 0:00:01
[K     |██████████████▏                 | 102 kB 5.0 MB/s eta 0:00:01
[K     |███████████████▋                | 112 kB 5.0 MB/s eta 0:00:01
[K     |█████████████████               | 122 kB 5.0 MB/s eta 0:00:01
[K     |██████████████████▍             | 133 kB 5[...]<jupyter_text>Googletranslation<jupyter_code>!pip install googletrans==3.1.0a0
!pip install -U sentence-transformers
!pip install datasets
!pip install tensorboardX<jupyter_output>Collecting tensorboardX
  Downloading tensorboardX-2.4-py2.py3-none-any.whl (124 kB)
[?25l
[K     |██▋                             | 10 kB 31.0 MB/s eta 0:00:01
[K     |█████▎                          | 20 kB 16.7 MB/s eta 0:00:01
[K     |████████                        | 30 kB 10.3 MB/s eta 0:00:01
[K     |██████████▌                     | 40 kB 8.5 MB/s eta 0:00:01
[K     |█████████████▏                  | 51 kB 5.0 MB/s eta 0:00:01
[K     |███████████████▉                | 61 kB 5.5 MB/s eta 0:00:01
[K     |██████████████████▍             | 71 kB 5.3 MB/s eta 0:00:01
[K     |█████████████████████           | 81 kB 6.0 MB/s eta 0:00:01
[K     |███████████████████████▊        | 92 kB 4.6 MB/s eta 0:00:01
[K     |██████████████████████████▎     | 102 kB 4.9 MB/s eta 0:00:01
[K     |█████████████████████████████   | 112 kB 4.9 MB/s eta 0:00:01
[K     |███████████████████████████████▋| 122 kB 4.9 MB/s eta 0:00:01
[K     |████████████████████████████████| 124 kB 4.9 MB/s 
[[...]<jupyter_text>Baixando o Dataset<jupyter_code>from google.colab import drive
drive.mount('drive')
import pandas as pd
from datasets import load_dataset
from tqdm import tqdm
dataset = load_dataset('tapaco', 'pt')
tapaco = []
# The dataset has only train split.
for data in tqdm(dataset["train"]):
    keys = data.keys()
    tapaco.append([data[key] for key in keys])

tapaco_df = pd.DataFrame(
    data=tapaco,
    columns=[
       "paraphrase_set_id",
       "sentence_id",
        "paraphrase",
        "lists",
        "tags",
        "language",
    ],
)
tapaco_df.head()
def generate_tapaco_paraphrase_dataset(dataset):
    dataset_df = dataset[["paraphrase", "paraphrase_set_id"]]
    non_single_labels = (
        dataset_df["paraphrase_set_id"]
        .value_counts()[dataset_df["paraphrase_set_id"].value_counts() > 1]
        .index.tolist()
    )
    tapaco_df_sorted = dataset_df.loc[
        dataset_df["paraphrase_set_id"].isin(non_single_labels)
    ]
    tapaco_paraphrases_dataset = []

    for paraphrase_set_id in tqdm(tapaco_df_sorted["paraphrase_set_id"].unique()):
        id_wise_paraphrases = tapaco_df_sorted[
            tapaco_df_sorted["paraphrase_set_id"] == paraphrase_set_id
        ]
        len_id_wise_paraphrases = (
            id_wise_paraphrases.shape[0]
            if id_wise_paraphrases.shape[0] % 2 == 0
            else id_wise_paraphrases.shape[0] - 1
        )
        for ix in range(0, len_id_wise_paraphrases, 2):
            current_phrase = id_wise_paraphrases.iloc[ix][0]
            for count_ix in range(ix + 1, ix + 2):
                next_phrase = id_wise_paraphrases.iloc[ix + 1][0]
                tapaco_paraphrases_dataset.append([current_phrase, next_phrase])
    tapaco_paraphrases_dataset_df = pd.DataFrame(
        tapaco_paraphrases_dataset, columns=["Text", "Paraphrase"]
    )
    
    return tapaco_paraphrases_dataset_df
dataset_df = generate_tapaco_paraphrase_dataset(tapaco_df)
dataset_df.head()
from simpletransformers.t5 import T5Model
from sklearn.model_selection import train_test_split
import sklearn
dataset_df.columns = ["input_text","target_text"]
dataset_df["prefix"] = "parafrase"
dataset_df.shape
dataset_df.head()
from sentence_transformers import SentenceTransformer, util

model = SentenceTransformer('sentence-transformers/paraphrase-multilingual-mpnet-base-v2')
from tqdm import tqdm
tqdm.pandas()
original_embedding = model.encode(dataset_df['input_text'])
dataset_df['original_embedding'] = [original_embedding[i] for i in range(original_embedding.shape[0])]


parafrase_embedding = model.encode(dataset_df['target_text'])
dataset_df['parafrase_embedding'] = [parafrase_embedding[i] for i in range(parafrase_embedding.shape[0])]

dataset_df['similiaridade_parafrase'] = dataset_df.progress_apply(lambda x:util.cos_sim(x['original_embedding'], x['parafrase_embedding']).tolist()[0][0], axis=1)
dataset_df.head()
dataset_df['similiaridade_parafrase'].describe()
fim = dataset_df[dataset_df['similiaridade_parafrase'] >= 0.8].copy()
fim = fim [["input_text","target_text", "prefix"]].copy()
train_data,test_data = train_test_split(dataset_df,test_size=0.1)
train_data.shape
args = {
    "reprocess_input_data": True,
    "overwrite_output_dir": True,
    "max_seq_length": 256,
    "num_train_epochs": 4,
    "num_beams": None,
    "do_sample": True,
    "top_k": 50,
    "top_p": 0.85,
    "use_multiprocessing": False,
    "save_steps": -1,
    "save_eval_checkpoints": False,
    "evaluate_during_training": False,
    'adam_epsilon': 1e-08,
    'eval_batch_size': 6,
    'fp_16': True,
    'gradient_accumulation_steps': 16,
    'learning_rate': 0.0003,
    'max_grad_norm': 1.0,
    'n_gpu': 1,
    'seed': 42,
    'train_batch_size': 6,
    'warmup_steps': 0,
    'weight_decay': 0.0
}
model = T5Model("t5",'unicamp-dl/ptt5-base-portuguese-vocab', args=args)
model.train_model(train_data, eval_data=test_data, use_cuda=True,acc=sklearn.metrics.accuracy_score)
prefix = "parafrase"
pred = model.predict([f"{prefix}: Adoro treinar modelos no python."])
print(pred)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1788128122245078, 'fraction_numerical': 0.05230678812812224, 'mean_word_length': 3.078490113840623, 'pattern_counts': {'":': 12, '<': 14, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '381087', 'license': 'MIT', 'n_tokens_mistral': 3139, 'n_tokens_neox': 2620, 'n_words': 469, 'path': 'treinando_ptt5_parafraseador_vf.ipynb', 'repo_name': 'lucasfaop/ptt5_paraphraser', 'size': '6806'}