{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/conda/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } ], "source": [ "from transformers import AutoTokenizer\n", "from transformers.models.bart.modeling_bart import BartForConditionalGeneration" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "tokenizer = AutoTokenizer.from_pretrained(\"facebook/bart-large\")" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "('bart/tokenizer/tokenizer_config.json',\n", " 'bart/tokenizer/special_tokens_map.json',\n", " 'bart/tokenizer/vocab.json',\n", " 'bart/tokenizer/merges.txt',\n", " 'bart/tokenizer/added_tokens.json',\n", " 'bart/tokenizer/tokenizer.json')" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenizer.save_pretrained(\"bart/tokenizer\")" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "model = BartForConditionalGeneration.from_pretrained(\"facebook/bart-large\", forced_bos_token_id=0)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Some weights of AudioBartForConditionalGeneration were not initialized from the model checkpoint at bart/model/ and are newly initialized: ['model.encodec_embeddings.3.weight', 'model.encodec_embeddings.4.weight', 'model.encodec_embeddings.1.weight', 'model.encodec_embeddings.0.weight', 'model.encoder.encodec_embeddings.7.weight', 'model.encodec_embeddings.2.weight', 'model.encodec_embeddings.6.weight', 'model.encoder.encodec_embeddings.0.weight', 'model.encodec_embeddings.7.weight', 'model.encoder.encodec_embeddings.4.weight', 'model.encoder.encodec_embeddings.2.weight', 'model.encoder.encodec_embeddings.3.weight', 'model.encodec_embeddings.5.weight', 'model.encoder.encodec_embeddings.5.weight', 'model.encoder.encodec_embeddings.1.weight', 'model.encoder.encodec_embeddings.6.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" ] } ], "source": [ "from modeling.audiobart import AudioBartForConditionalGeneration\n", "model = AudioBartForConditionalGeneration.from_pretrained(\"bart/model/\")" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'input_ids': tensor([[ 0, 31414, 127, 50264, 32440, 3807, 118, 32440, 3807, 118,\n", " 25610, 2]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n" ] } ], "source": [ "text = \"Hello my yeppi yeppi yo\"\n", "input = tokenizer(text, return_tensors='pt')\n", "print(input)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "generated_ids = model.generate(input[\"input_ids\"])" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [], "source": [ "ids = output.logits.detach().numpy().argmax(-1)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['Hello my friends, yeppi yeppiiyeppiyeppii ye']" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenizer.batch_decode(generated_ids, skip_special_tokens=True)" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [], "source": [ "model.save_pretrained(\"bart/model\")" ] }, { "cell_type": "code", "execution_count": 51, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'input_ids': tensor([[ 0, 7842, 330, 506, 1536, 267, 131, 6634, 36807, 571,\n", " 20920, 127, 766, 16, 32440, 3807, 118, 32440, 3807, 118,\n", " 25610, 2]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n" ] } ], "source": [ "text = \"adskfalsj;lsdfg Hello my name is yeppi yeppi yo\"\n", "input = tokenizer(text, return_tensors='pt')\n", "print(input)" ] }, { "cell_type": "code", "execution_count": 52, "metadata": {}, "outputs": [], "source": [ "output = model.forward(**input)" ] }, { "cell_type": "code", "execution_count": 54, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/conda/lib/python3.9/site-packages/transformers/generation/utils.py:1353: UserWarning: Using `max_length`'s default (20) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n", " warnings.warn(\n" ] }, { "data": { "text/plain": [ "['adskfalsj;lsdfg Hello my name is yeppi ye']" ] }, "execution_count": 54, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenizer.batch_decode(model.generate(input['input_ids']))" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['Hello my name is yeppi yeppi yo']" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenizer.batch_decode(input['input_ids'])" ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [], "source": [ "from transformers.models.bart.modeling_bart import shift_tokens_right" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "tensor([[ 0, 0, 31414, 127, 766, 16, 32440, 3807, 118, 32440,\n", " 3807, 118, 25610]])\n" ] } ], "source": [ "print(shift_tokens_right(input['input_ids'], pad_token_id=1, decoder_start_token_id=0))" ] }, { "cell_type": "code", "execution_count": 55, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Downloading (…)lve/main/config.json: 100%|██████████| 1.58k/1.58k [00:00<00:00, 589kB/s]\n", "Downloading (…)olve/main/vocab.json: 100%|██████████| 899k/899k [00:00<00:00, 1.29MB/s]\n", "Downloading (…)olve/main/merges.txt: 100%|██████████| 456k/456k [00:00<00:00, 884kB/s]\n", "Downloading (…)/main/tokenizer.json: 100%|██████████| 1.36M/1.36M [00:00<00:00, 7.43MB/s]\n" ] } ], "source": [ "cnn_tokenizer = AutoTokenizer.from_pretrained(\"facebook/bart-large-cnn\")" ] }, { "cell_type": "code", "execution_count": 63, "metadata": {}, "outputs": [], "source": [ "original_text = \"ArithmeticErrorThe tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct.\"" ] }, { "cell_type": "code", "execution_count": 68, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['ArithmeticErrorThe tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct.']\n" ] } ], "source": [ "input = cnn_tokenizer(text=original_text, return_tensors='pt')\n", "print(cnn_tokenizer.batch_decode(input['input_ids']))" ] }, { "cell_type": "code", "execution_count": 65, "metadata": {}, "outputs": [], "source": [ "cnn_model = BartForConditionalGeneration.from_pretrained(\"facebook/bart-large-cnn\")" ] }, { "cell_type": "code", "execution_count": 66, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/conda/lib/python3.9/site-packages/transformers/generation/utils.py:1353: UserWarning: Using `max_length`'s default (142) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n", " warnings.warn(\n" ] }, { "data": { "text/plain": [ "['The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world. It was the first structure to reach a height of 300 metres.']" ] }, "execution_count": 66, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cnn_tokenizer.batch_decode(cnn_model.generate(input['input_ids']))" ] } ], "metadata": { "kernelspec": { "display_name": "base", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.12" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }