diff --git "a/all-data-200k (1).ipynb" "b/all-data-200k (1).ipynb" new file mode 100644--- /dev/null +++ "b/all-data-200k (1).ipynb" @@ -0,0 +1 @@ +{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# installing tensorflow_text \n!pip install tensorflow_text ","metadata":{"id":"DGFTkuRvzWqc","outputId":"5cd555ad-f28d-40fc-8931-effd87043ffd","execution":{"iopub.status.busy":"2023-03-16T11:42:55.387809Z","iopub.execute_input":"2023-03-16T11:42:55.388211Z","iopub.status.idle":"2023-03-16T11:43:17.345494Z","shell.execute_reply.started":"2023-03-16T11:42:55.388174Z","shell.execute_reply":"2023-03-16T11:43:17.344235Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Requirement already satisfied: tensorflow_text in /opt/conda/lib/python3.7/site-packages (2.11.0)\nRequirement already satisfied: tensorflow-hub>=0.8.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow_text) (0.12.0)\nRequirement already satisfied: tensorflow<2.12,>=2.11.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow_text) (2.11.0)\nRequirement already satisfied: grpcio<2.0,>=1.24.3 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (1.51.1)\nRequirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (1.16.0)\nRequirement already satisfied: libclang>=13.0.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (15.0.6.1)\nRequirement already satisfied: flatbuffers>=2.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (23.1.21)\nRequirement already satisfied: setuptools in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (59.8.0)\nRequirement already satisfied: keras<2.12,>=2.11.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (2.11.0)\nRequirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (0.29.0)\nRequirement already satisfied: opt-einsum>=2.3.2 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (3.3.0)\nRequirement already satisfied: absl-py>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (1.4.0)\nRequirement already satisfied: gast<=0.4.0,>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (0.4.0)\nRequirement already satisfied: tensorflow-estimator<2.12,>=2.11.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (2.11.0)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (23.0)\nRequirement already satisfied: wrapt>=1.11.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (1.14.1)\nRequirement already satisfied: astunparse>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (1.6.3)\nRequirement already satisfied: h5py>=2.9.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (3.8.0)\nRequirement already satisfied: tensorboard<2.12,>=2.11 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (2.11.2)\nCollecting protobuf<3.20,>=3.9.2\n Downloading protobuf-3.19.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m28.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n\u001b[?25hRequirement already satisfied: typing-extensions>=3.6.6 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (4.4.0)\nRequirement already satisfied: termcolor>=1.1.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (2.2.0)\nRequirement already satisfied: numpy>=1.20 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (1.21.6)\nRequirement already satisfied: google-pasta>=0.1.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow<2.12,>=2.11.0->tensorflow_text) (0.2.0)\nRequirement already satisfied: wheel<1.0,>=0.23.0 in /opt/conda/lib/python3.7/site-packages (from astunparse>=1.6.0->tensorflow<2.12,>=2.11.0->tensorflow_text) (0.38.4)\nRequirement already satisfied: google-auth<3,>=1.6.3 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (1.35.0)\nRequirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (0.6.1)\nRequirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (1.8.1)\nRequirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (3.4.1)\nRequirement already satisfied: werkzeug>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (2.2.3)\nRequirement already satisfied: requests<3,>=2.21.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (2.28.2)\nRequirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (0.4.6)\nRequirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (0.2.8)\nRequirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (4.9)\nRequirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (4.2.4)\nRequirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (1.3.1)\nRequirement already satisfied: importlib-metadata>=4.4 in /opt/conda/lib/python3.7/site-packages (from markdown>=2.6.8->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (4.11.4)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (1.26.14)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (2022.12.7)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (3.4)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (2.1.1)\nRequirement already satisfied: MarkupSafe>=2.1.1 in /opt/conda/lib/python3.7/site-packages (from werkzeug>=1.0.1->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (2.1.1)\nRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (3.11.0)\nRequirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (0.4.8)\nRequirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow_text) (3.2.2)\nInstalling collected packages: protobuf\n Attempting uninstall: protobuf\n Found existing installation: protobuf 3.20.3\n Uninstalling protobuf-3.20.3:\n Successfully uninstalled protobuf-3.20.3\n\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\ncudf 21.12.2 requires cupy-cuda115, which is not installed.\ntfx-bsl 1.12.0 requires google-api-python-client<2,>=1.7.11, but you have google-api-python-client 2.79.0 which is incompatible.\ntfx-bsl 1.12.0 requires pyarrow<7,>=6, but you have pyarrow 5.0.0 which is incompatible.\ntensorflow-transform 1.12.0 requires pyarrow<7,>=6, but you have pyarrow 5.0.0 which is incompatible.\nonnx 1.13.1 requires protobuf<4,>=3.20.2, but you have protobuf 3.19.6 which is incompatible.\napache-beam 2.44.0 requires dill<0.3.2,>=0.3.1.1, but you have dill 0.3.6 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0mSuccessfully installed protobuf-3.19.6\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"# importing required libraries\nimport numpy as np\n\nimport typing\nfrom typing import Any, Tuple\n\nimport tensorflow as tf\n\nimport tensorflow_text as tf_text\n\nimport matplotlib.pyplot as plt\nimport matplotlib.ticker as ticker","metadata":{"id":"tnxXKDjq3jEL","execution":{"iopub.status.busy":"2023-03-16T11:43:17.349024Z","iopub.execute_input":"2023-03-16T11:43:17.349386Z","iopub.status.idle":"2023-03-16T11:43:24.129905Z","shell.execute_reply.started":"2023-03-16T11:43:17.349353Z","shell.execute_reply":"2023-03-16T11:43:24.128829Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"# using builtin implementations\nuse_builtins = True","metadata":{"id":"KPJ9J7iPUchc","execution":{"iopub.status.busy":"2023-03-16T11:43:24.131282Z","iopub.execute_input":"2023-03-16T11:43:24.132076Z","iopub.status.idle":"2023-03-16T11:43:24.137016Z","shell.execute_reply.started":"2023-03-16T11:43:24.132030Z","shell.execute_reply":"2023-03-16T11:43:24.135628Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"#@title Shape checker\nclass ShapeChecker():\n def __init__(self):\n # Keep a cache of every axis-name seen\n self.shapes = {}\n\n def __call__(self, tensor, names, broadcast=False):\n if not tf.executing_eagerly():\n return\n\n if isinstance(names, str):\n names = (names,)\n\n shape = tf.shape(tensor)\n rank = tf.rank(tensor)\n\n if rank != len(names):\n raise ValueError(f'Rank mismatch:\\n'\n f' found {rank}: {shape.numpy()}\\n'\n f' expected {len(names)}: {names}\\n')\n\n for i, name in enumerate(names):\n if isinstance(name, int):\n old_dim = name\n else:\n old_dim = self.shapes.get(name, None)\n new_dim = shape[i]\n\n if (broadcast and new_dim == 1):\n continue\n\n if old_dim is None:\n # If the axis name is new, add its length to the cache.\n self.shapes[name] = new_dim\n continue\n\n if new_dim != old_dim:\n raise ValueError(f\"Shape mismatch for dimension: '{name}'\\n\"\n f\" found: {new_dim}\\n\"\n f\" expected: {old_dim}\\n\")","metadata":{"id":"KqFqKi4fqN9X","execution":{"iopub.status.busy":"2023-03-16T11:43:24.141505Z","iopub.execute_input":"2023-03-16T11:43:24.142344Z","iopub.status.idle":"2023-03-16T11:43:24.155793Z","shell.execute_reply.started":"2023-03-16T11:43:24.142304Z","shell.execute_reply":"2023-03-16T11:43:24.154779Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"!pip install Metaphone\nfrom metaphone import doublemetaphone\nimport pandas as pd\nimport re","metadata":{"id":"rOduTbljY-4e","outputId":"b95ed0e1-fb62-464b-f0a4-738b6ffc11a2","execution":{"iopub.status.busy":"2023-03-16T11:43:24.157133Z","iopub.execute_input":"2023-03-16T11:43:24.158036Z","iopub.status.idle":"2023-03-16T11:43:35.424695Z","shell.execute_reply.started":"2023-03-16T11:43:24.157970Z","shell.execute_reply":"2023-03-16T11:43:35.423443Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"Collecting Metaphone\n Downloading Metaphone-0.6.tar.gz (14 kB)\n Preparing metadata (setup.py) ... \u001b[?25ldone\n\u001b[?25hBuilding wheels for collected packages: Metaphone\n Building wheel for Metaphone (setup.py) ... \u001b[?25ldone\n\u001b[?25h Created wheel for Metaphone: filename=Metaphone-0.6-py3-none-any.whl size=13919 sha256=0ea3a5d91487d74185eb933acaaff4b0ad7165cf87bdbc87c56a244527ff6eb8\n Stored in directory: /root/.cache/pip/wheels/3a/2c/57/cb7d38aa6c80a870a0341ccf4265c2921fb55b1f7f4b76b46b\nSuccessfully built Metaphone\nInstalling collected packages: Metaphone\nSuccessfully installed Metaphone-0.6\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"data1=pd.read_csv(\"/kaggle/input/google-reasearch/train.tsv\",sep='\\t')\ndata1","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:35.428421Z","iopub.execute_input":"2023-03-16T11:43:35.428743Z","iopub.status.idle":"2023-03-16T11:43:36.846977Z","shell.execute_reply.started":"2023-03-16T11:43:35.428710Z","shell.execute_reply":"2023-03-16T11:43:36.845910Z"},"trusted":true},"execution_count":6,"outputs":[{"execution_count":6,"output_type":"execute_result","data":{"text/plain":" en_query \\\n0 Who is the featured performer at The Grand Ole... \n1 Remind me of my meeting at 12 am \n2 set an alarm for 5 am \n3 What ’ s will weather be like at Sydney at 2 pm \n4 What is the weather in Canada ? \n... ... \n170078 How many miles from Humble to Brenham \n170079 What is traffic so bad on Ronald Reagan \n170080 Concerts by The Strokes for this weekend \n170081 How many mm of rain do we expect for tomorrow ? \n170082 cancel all alarms \n\n cs_query \\\n0 Is Saturday night The Grand Ole Opry me featur... \n1 mujhe 12 am ko meri meeting ke baare me yaad d... \n2 subha 5 baje ke liye ek alarm set karen \n3 dopaher 2 baje Sydney me mausam kaisa hoga \n4 Canada me mausam kaisa hai ? \n... ... \n170078 Humble se Brenham tak kitne miles hai \n170079 Ronald Reagan par traffic kyu kharaab hai \n170080 Is weekend The Strokes ke hone wale Concerts \n170081 kal kitni mm baarish ki ummeed hai ? \n170082 sarey alarms cance kare \n\n en_parses \\\n0 [IN:GET_EVENT Who is [SL:CATEGORY_EVENT the fe... \n1 [IN:CREATE_REMINDER Remind [SL:PERSON_REMINDED... \n2 [IN:CREATE_ALARM set an alarm [SL:DATE_TIME fo... \n3 [IN:GET_WEATHER What ’s will weather be like a... \n4 [IN:GET_WEATHER What is the weather in [SL:LOC... \n... ... \n170078 [IN:GET_DISTANCE How many [SL:UNIT_DISTANCE mi... \n170079 [IN:GET_INFO_TRAFFIC What is traffic so bad on... \n170080 [IN:GET_EVENT [SL:CATEGORY_EVENT Concerts ] by... \n170081 [IN:UNSUPPORTED_WEATHER How many mm of [SL:WEA... \n170082 [IN:DELETE_ALARM cancel [SL:AMOUNT all ] alarms ] \n\n cs_parses \n0 [IN:GET_EVENT [SL:DATE_TIME Is Saturday night ... \n1 [IN:CREATE_REMINDER [SL:PERSON_REMINDED mujhe ... \n2 [IN:CREATE_ALARM [SL:DATE_TIME subha 5 baje ke... \n3 [IN:GET_WEATHER [SL:DATE_TIME dopaher 2 baje ]... \n4 [IN:GET_WEATHER [SL:LOCATION Canada ] me mausa... \n... ... \n170078 [IN:GET_DISTANCE [SL:SOURCE Humble ] se [SL:DE... \n170079 [IN:GET_INFO_TRAFFIC [SL:LOCATION Ronald Reaga... \n170080 [IN:GET_EVENT [SL:DATE_TIME Is weekend ] [SL:N... \n170081 [IN:UNSUPPORTED_WEATHER kal kitni mm [SL:WEATH... \n170082 [IN:DELETE_ALARM [SL:AMOUNT sarey ] alarms can... \n\n[170083 rows x 4 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
en_querycs_queryen_parsescs_parses
0Who is the featured performer at The Grand Ole...Is Saturday night The Grand Ole Opry me featur...[IN:GET_EVENT Who is [SL:CATEGORY_EVENT the fe...[IN:GET_EVENT [SL:DATE_TIME Is Saturday night ...
1Remind me of my meeting at 12 ammujhe 12 am ko meri meeting ke baare me yaad d...[IN:CREATE_REMINDER Remind [SL:PERSON_REMINDED...[IN:CREATE_REMINDER [SL:PERSON_REMINDED mujhe ...
2set an alarm for 5 amsubha 5 baje ke liye ek alarm set karen[IN:CREATE_ALARM set an alarm [SL:DATE_TIME fo...[IN:CREATE_ALARM [SL:DATE_TIME subha 5 baje ke...
3What ’ s will weather be like at Sydney at 2 pmdopaher 2 baje Sydney me mausam kaisa hoga[IN:GET_WEATHER What ’s will weather be like a...[IN:GET_WEATHER [SL:DATE_TIME dopaher 2 baje ]...
4What is the weather in Canada ?Canada me mausam kaisa hai ?[IN:GET_WEATHER What is the weather in [SL:LOC...[IN:GET_WEATHER [SL:LOCATION Canada ] me mausa...
...............
170078How many miles from Humble to BrenhamHumble se Brenham tak kitne miles hai[IN:GET_DISTANCE How many [SL:UNIT_DISTANCE mi...[IN:GET_DISTANCE [SL:SOURCE Humble ] se [SL:DE...
170079What is traffic so bad on Ronald ReaganRonald Reagan par traffic kyu kharaab hai[IN:GET_INFO_TRAFFIC What is traffic so bad on...[IN:GET_INFO_TRAFFIC [SL:LOCATION Ronald Reaga...
170080Concerts by The Strokes for this weekendIs weekend The Strokes ke hone wale Concerts[IN:GET_EVENT [SL:CATEGORY_EVENT Concerts ] by...[IN:GET_EVENT [SL:DATE_TIME Is weekend ] [SL:N...
170081How many mm of rain do we expect for tomorrow ?kal kitni mm baarish ki ummeed hai ?[IN:UNSUPPORTED_WEATHER How many mm of [SL:WEA...[IN:UNSUPPORTED_WEATHER kal kitni mm [SL:WEATH...
170082cancel all alarmssarey alarms cance kare[IN:DELETE_ALARM cancel [SL:AMOUNT all ] alarms ][IN:DELETE_ALARM [SL:AMOUNT sarey ] alarms can...
\n

170083 rows × 4 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"data1['en_query']=data1['en_query'].apply(lambda x: str(x.strip()).lower())\ndata1['cs_query']=data1['cs_query'].apply(lambda x: str(x.strip()).lower())","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:36.848736Z","iopub.execute_input":"2023-03-16T11:43:36.849470Z","iopub.status.idle":"2023-03-16T11:43:37.029500Z","shell.execute_reply.started":"2023-03-16T11:43:36.849430Z","shell.execute_reply":"2023-03-16T11:43:37.028424Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"data1=data1[['cs_query','en_query']]\ndata1.rename(columns = {'cs_query':'Sentence','en_query':'trr'},inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.031152Z","iopub.execute_input":"2023-03-16T11:43:37.031552Z","iopub.status.idle":"2023-03-16T11:43:37.051279Z","shell.execute_reply.started":"2023-03-16T11:43:37.031512Z","shell.execute_reply":"2023-03-16T11:43:37.049916Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/pandas/core/frame.py:5047: SettingWithCopyWarning: \nA value is trying to be set on a copy of a slice from a DataFrame\n\nSee the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n errors=errors,\n","output_type":"stream"}]},{"cell_type":"code","source":"data1","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.052792Z","iopub.execute_input":"2023-03-16T11:43:37.053596Z","iopub.status.idle":"2023-03-16T11:43:37.066177Z","shell.execute_reply.started":"2023-03-16T11:43:37.053557Z","shell.execute_reply":"2023-03-16T11:43:37.064925Z"},"trusted":true},"execution_count":9,"outputs":[{"execution_count":9,"output_type":"execute_result","data":{"text/plain":" Sentence \\\n0 is saturday night the grand ole opry me featur... \n1 mujhe 12 am ko meri meeting ke baare me yaad d... \n2 subha 5 baje ke liye ek alarm set karen \n3 dopaher 2 baje sydney me mausam kaisa hoga \n4 canada me mausam kaisa hai ? \n... ... \n170078 humble se brenham tak kitne miles hai \n170079 ronald reagan par traffic kyu kharaab hai \n170080 is weekend the strokes ke hone wale concerts \n170081 kal kitni mm baarish ki ummeed hai ? \n170082 sarey alarms cance kare \n\n trr \n0 who is the featured performer at the grand ole... \n1 remind me of my meeting at 12 am \n2 set an alarm for 5 am \n3 what ’ s will weather be like at sydney at 2 pm \n4 what is the weather in canada ? \n... ... \n170078 how many miles from humble to brenham \n170079 what is traffic so bad on ronald reagan \n170080 concerts by the strokes for this weekend \n170081 how many mm of rain do we expect for tomorrow ? \n170082 cancel all alarms \n\n[170083 rows x 2 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
Sentencetrr
0is saturday night the grand ole opry me featur...who is the featured performer at the grand ole...
1mujhe 12 am ko meri meeting ke baare me yaad d...remind me of my meeting at 12 am
2subha 5 baje ke liye ek alarm set karenset an alarm for 5 am
3dopaher 2 baje sydney me mausam kaisa hogawhat ’ s will weather be like at sydney at 2 pm
4canada me mausam kaisa hai ?what is the weather in canada ?
.........
170078humble se brenham tak kitne miles haihow many miles from humble to brenham
170079ronald reagan par traffic kyu kharaab haiwhat is traffic so bad on ronald reagan
170080is weekend the strokes ke hone wale concertsconcerts by the strokes for this weekend
170081kal kitni mm baarish ki ummeed hai ?how many mm of rain do we expect for tomorrow ?
170082sarey alarms cance karecancel all alarms
\n

170083 rows × 2 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"data=pd.read_csv(\"/kaggle/input/file-2/file2.csv\")\ndata","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.071478Z","iopub.execute_input":"2023-03-16T11:43:37.071828Z","iopub.status.idle":"2023-03-16T11:43:37.179204Z","shell.execute_reply.started":"2023-03-16T11:43:37.071794Z","shell.execute_reply":"2023-03-16T11:43:37.178042Z"},"trusted":true},"execution_count":10,"outputs":[{"execution_count":10,"output_type":"execute_result","data":{"text/plain":" Sentence \\\n0 congratulations on you celebrating british kid... \n1 uske liye toh bahot kuch karna padega ye pappi... \n2 yehi to hum semjhane ki koshish kar rahe hain.... \n3 cake kaha hai ?? \n4 jeet ka jashn aur shubah ki shuruat eating bre... \n... ... \n12143 dr kumar vishwas: \"koi deewana kehta hai.. koi... \n12144 me: aaj kuch toofani karte hai.mom: pani ki bo... \n12145 pyar mangi to jaan dengi,milk mango to kher de... \n12146 kaale kaale baal gaal gore gore \n12147 ye sab aunty'on ke saath? \n\n English_Translation \\\n0 users congratulate you for celebrating British... \n1 rat we should a lot more for that, by this evi... \n2 ehi, this is what i'm expecting you to underst... \n3 is Dramebajakudi where is the cake? \n4 the celebration of a victory and the start of ... \n... ... \n12143 Dr Kumar Vishwas: \"Some used to say lover.. So... \n12144 Me: Let's do some stormy today.Mom: Fill the w... \n12145 Ask for love we'll give life, ask for milk we'... \n12146 black hair cheeks fair fair \n12147 All this with aunties? \n\n trr \\\n0 congratulations on you celebrating british kid... \n1 you will have to do a lot for that, it will no... \n2 this is what we are trying to understand. peop... \n3 where is the cake?? \n4 victory celebration and early morning eating b... \n... ... \n12143 dr. kumar vishwas: \"some say crazy..some think... \n12144 me: let's do something stormy today. mother: f... \n12145 if you ask for love, you will give your life, ... \n12146 kale kale bal gal gore gore \n12147 all this with the aunties? \n\n tr \n0 congratulations on you celebrating british kid... \n1 you will have to do a lot for that, it will no... \n2 this is what we are trying to understand. peop... \n3 where is the cake?? \n4 victory celebration and early morning eating b... \n... ... \n12143 dr. kumar vishwas: \"some say crazy..some think... \n12144 me: let's do something stormy today. mother: f... \n12145 if you ask for love, you will give your life, ... \n12146 kale kale bal gal gore gore \n12147 all this with the aunties? \n\n[12148 rows x 4 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
SentenceEnglish_Translationtrrtr
0congratulations on you celebrating british kid...users congratulate you for celebrating British...congratulations on you celebrating british kid...congratulations on you celebrating british kid...
1uske liye toh bahot kuch karna padega ye pappi...rat we should a lot more for that, by this evi...you will have to do a lot for that, it will no...you will have to do a lot for that, it will no...
2yehi to hum semjhane ki koshish kar rahe hain....ehi, this is what i'm expecting you to underst...this is what we are trying to understand. peop...this is what we are trying to understand. peop...
3cake kaha hai ??is Dramebajakudi where is the cake?where is the cake??where is the cake??
4jeet ka jashn aur shubah ki shuruat eating bre...the celebration of a victory and the start of ...victory celebration and early morning eating b...victory celebration and early morning eating b...
...............
12143dr kumar vishwas: \"koi deewana kehta hai.. koi...Dr Kumar Vishwas: \"Some used to say lover.. So...dr. kumar vishwas: \"some say crazy..some think...dr. kumar vishwas: \"some say crazy..some think...
12144me: aaj kuch toofani karte hai.mom: pani ki bo...Me: Let's do some stormy today.Mom: Fill the w...me: let's do something stormy today. mother: f...me: let's do something stormy today. mother: f...
12145pyar mangi to jaan dengi,milk mango to kher de...Ask for love we'll give life, ask for milk we'...if you ask for love, you will give your life, ...if you ask for love, you will give your life, ...
12146kaale kaale baal gaal gore goreblack hair cheeks fair fairkale kale bal gal gore gorekale kale bal gal gore gore
12147ye sab aunty'on ke saath?All this with aunties?all this with the aunties?all this with the aunties?
\n

12148 rows × 4 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"data=data[['Sentence','trr']]\ndata","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.180595Z","iopub.execute_input":"2023-03-16T11:43:37.181056Z","iopub.status.idle":"2023-03-16T11:43:37.196234Z","shell.execute_reply.started":"2023-03-16T11:43:37.181017Z","shell.execute_reply":"2023-03-16T11:43:37.194971Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":" Sentence \\\n0 congratulations on you celebrating british kid... \n1 uske liye toh bahot kuch karna padega ye pappi... \n2 yehi to hum semjhane ki koshish kar rahe hain.... \n3 cake kaha hai ?? \n4 jeet ka jashn aur shubah ki shuruat eating bre... \n... ... \n12143 dr kumar vishwas: \"koi deewana kehta hai.. koi... \n12144 me: aaj kuch toofani karte hai.mom: pani ki bo... \n12145 pyar mangi to jaan dengi,milk mango to kher de... \n12146 kaale kaale baal gaal gore gore \n12147 ye sab aunty'on ke saath? \n\n trr \n0 congratulations on you celebrating british kid... \n1 you will have to do a lot for that, it will no... \n2 this is what we are trying to understand. peop... \n3 where is the cake?? \n4 victory celebration and early morning eating b... \n... ... \n12143 dr. kumar vishwas: \"some say crazy..some think... \n12144 me: let's do something stormy today. mother: f... \n12145 if you ask for love, you will give your life, ... \n12146 kale kale bal gal gore gore \n12147 all this with the aunties? \n\n[12148 rows x 2 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
Sentencetrr
0congratulations on you celebrating british kid...congratulations on you celebrating british kid...
1uske liye toh bahot kuch karna padega ye pappi...you will have to do a lot for that, it will no...
2yehi to hum semjhane ki koshish kar rahe hain....this is what we are trying to understand. peop...
3cake kaha hai ??where is the cake??
4jeet ka jashn aur shubah ki shuruat eating bre...victory celebration and early morning eating b...
.........
12143dr kumar vishwas: \"koi deewana kehta hai.. koi...dr. kumar vishwas: \"some say crazy..some think...
12144me: aaj kuch toofani karte hai.mom: pani ki bo...me: let's do something stormy today. mother: f...
12145pyar mangi to jaan dengi,milk mango to kher de...if you ask for love, you will give your life, ...
12146kaale kaale baal gaal gore gorekale kale bal gal gore gore
12147ye sab aunty'on ke saath?all this with the aunties?
\n

12148 rows × 2 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"file = open('/kaggle/input/anuvaad/anuvaad.txt','r')\n# print(file.read())\n# lst=targ, lst2=inp\nfileStrings = file.readlines()\ntarg = []\ninp = []\nfor line in fileStrings:\n # print(line)\n line = line.replace('\\n','')\n lstO = line.split('\\t')\n targ.append(lstO[0])\n inp.append(lstO[1])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.197815Z","iopub.execute_input":"2023-03-16T11:43:37.198915Z","iopub.status.idle":"2023-03-16T11:43:37.212932Z","shell.execute_reply.started":"2023-03-16T11:43:37.198867Z","shell.execute_reply":"2023-03-16T11:43:37.211924Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"data_anu=pd.DataFrame(data={'Sentence':inp,'trr':targ})\ndata_anu","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.216354Z","iopub.execute_input":"2023-03-16T11:43:37.216636Z","iopub.status.idle":"2023-03-16T11:43:37.233188Z","shell.execute_reply.started":"2023-03-16T11:43:37.216609Z","shell.execute_reply":"2023-03-16T11:43:37.232043Z"},"trusted":true},"execution_count":13,"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":" Sentence \\\n0 Come on boys, waqt hai shine karne ka \n1 Yeh heart Maange More! \n2 She was bhunno-ing the masala-s jub phone ki g... \n3 You know me na... \n4 Woh to mai sambhal lunga \n... ... \n1299 muze vo bohot pasand hai \n1300 mujhe psnd hai \n1301 mujhe market se vegetables lekar aane hain \n1302 sabse best friend \n1303 best friend sabse \n\n trr \n0 Come on boys, it is time to shine. \n1 This heart asks for more! \n2 She was roasting the spices when the phone rang \n3 You know me right \n4 I will handle that \n... ... \n1299 I like it a lot \n1300 I like it a lot \n1301 I have to bring vegetables from the market \n1302 Most best friend \n1303 most best friend \n\n[1304 rows x 2 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
Sentencetrr
0Come on boys, waqt hai shine karne kaCome on boys, it is time to shine.
1Yeh heart Maange More!This heart asks for more!
2She was bhunno-ing the masala-s jub phone ki g...She was roasting the spices when the phone rang
3You know me na...You know me right
4Woh to mai sambhal lungaI will handle that
.........
1299muze vo bohot pasand haiI like it a lot
1300mujhe psnd haiI like it a lot
1301mujhe market se vegetables lekar aane hainI have to bring vegetables from the market
1302sabse best friendMost best friend
1303best friend sabsemost best friend
\n

1304 rows × 2 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"data_anu['Sentence']=data_anu['Sentence'].apply(lambda x: str(x.strip()).lower())\ndata_anu['trr']=data_anu['trr'].apply(lambda x: str(x.strip()).lower())","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.236242Z","iopub.execute_input":"2023-03-16T11:43:37.236568Z","iopub.status.idle":"2023-03-16T11:43:37.245140Z","shell.execute_reply.started":"2023-03-16T11:43:37.236540Z","shell.execute_reply":"2023-03-16T11:43:37.244018Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"df=pd.concat([data,data1,data_anu],axis=0)\ndf","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.246257Z","iopub.execute_input":"2023-03-16T11:43:37.246745Z","iopub.status.idle":"2023-03-16T11:43:37.272633Z","shell.execute_reply.started":"2023-03-16T11:43:37.246706Z","shell.execute_reply":"2023-03-16T11:43:37.271664Z"},"trusted":true},"execution_count":15,"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":" Sentence \\\n0 congratulations on you celebrating british kid... \n1 uske liye toh bahot kuch karna padega ye pappi... \n2 yehi to hum semjhane ki koshish kar rahe hain.... \n3 cake kaha hai ?? \n4 jeet ka jashn aur shubah ki shuruat eating bre... \n... ... \n1299 muze vo bohot pasand hai \n1300 mujhe psnd hai \n1301 mujhe market se vegetables lekar aane hain \n1302 sabse best friend \n1303 best friend sabse \n\n trr \n0 congratulations on you celebrating british kid... \n1 you will have to do a lot for that, it will no... \n2 this is what we are trying to understand. peop... \n3 where is the cake?? \n4 victory celebration and early morning eating b... \n... ... \n1299 i like it a lot \n1300 i like it a lot \n1301 i have to bring vegetables from the market \n1302 most best friend \n1303 most best friend \n\n[183535 rows x 2 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
Sentencetrr
0congratulations on you celebrating british kid...congratulations on you celebrating british kid...
1uske liye toh bahot kuch karna padega ye pappi...you will have to do a lot for that, it will no...
2yehi to hum semjhane ki koshish kar rahe hain....this is what we are trying to understand. peop...
3cake kaha hai ??where is the cake??
4jeet ka jashn aur shubah ki shuruat eating bre...victory celebration and early morning eating b...
.........
1299muze vo bohot pasand haii like it a lot
1300mujhe psnd haii like it a lot
1301mujhe market se vegetables lekar aane haini have to bring vegetables from the market
1302sabse best friendmost best friend
1303best friend sabsemost best friend
\n

183535 rows × 2 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"hin_vocab=dict()\nfor i in df['Sentence'].apply(lambda x: re.sub(\"[.!?\\\\-\\'\\\"]\", \"\", x)):\n for j in i.split(' '):\n hin_vocab[doublemetaphone(j)[0]+'*'+doublemetaphone(j[::-1])[0]+'*'+j[:2]+'*'+j[len(j)-1:]]=j\nfor i in hin_vocab:\n a=set()\n a.add(hin_vocab[i])\n hin_vocab[i]=a\nfor i in df['Sentence'].apply(lambda x: re.sub(\"[.!?\\\\-\\'\\\"]\", \"\", x)):\n for j in i.split(' '):\n hin_vocab[doublemetaphone(j)[0]+'*'+doublemetaphone(j[::-1])[0]+'*'+j[:2]+'*'+j[len(j)-1:]].add(j)\nprint(len(hin_vocab))\nhin_vocab","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:43:37.274284Z","iopub.execute_input":"2023-03-16T11:43:37.274892Z","iopub.status.idle":"2023-03-16T11:45:07.573197Z","shell.execute_reply.started":"2023-03-16T11:43:37.274851Z","shell.execute_reply":"2023-03-16T11:45:07.572045Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stdout","text":"26683\n","output_type":"stream"},{"execution_count":16,"output_type":"execute_result","data":{"text/plain":"{'KNKRTLXNS*SNTLTRNK*co*s': {'congratulations'},\n 'AN*N*on*n': {'on'},\n 'A*A*yo*u': {'you', 'youuuuu', 'youuuuuu'},\n 'SLPRTNK*NTRPLK*ce*g': {'celebrating'},\n 'PRTX*STRP*br*h': {'british'},\n 'KT*TK*ki*d': {'kid', 'kidd'},\n 'SNKRS*SRNS*si*s': {'singers'},\n 'SF*APS*so*a': {'sophia'},\n 'KRSS*SKRK*gr*s': {'graces', 'grocess'},\n 'ANT*TN*an*d': {'and'},\n 'RSS*SSR*ro*s': {'roses', 'rosies'},\n 'ST*TS*st*t': {'st'},\n 'ANFRSR*ARSRFN*an*y': {'anniversary'},\n 'AF*F*of*f': {'of', 'off'},\n 'A*A*a*a': {'a'},\n 'FST*TSF*vi*t': {'visit', 'vist'},\n 'AR*R*yo*r': {'yor', 'your'},\n 'X*AS*sh*w': {'shaw', 'show'},\n '***': {''},\n 'H*A*ho*w': {'how'},\n 'ASK*AKS*us*e': {'uske'},\n 'L*AL*li*e': {'lie', 'liye', 'liyee', 'liyeee', 'liyye'},\n 'T*HT*to*h': {'toh'},\n 'PHT*THP*ba*t': {'bahat', 'bahaut', 'bahot', 'bahuat', 'bahut'},\n 'KX*KK*ku*h': {'kucch', 'kucchh', 'kuch'},\n 'KRN*ANRK*ka*a': {'karana', 'kareena', 'karna', 'karwana'},\n 'PTK*AJTP*pa*a': {'padega', 'padhega', 'padhiyega'},\n 'A*A*ye*e': {'ye', 'yee', 'yeeeeee'},\n 'PPN*NPP*pa*n': {'paapon', 'pappiyan', 'pappiyon'},\n 'S*AS*se*e': {'se', 'see', 'sewee'},\n 'KM*MK*ka*m': {'kaam', 'kaayam', 'kam', 'kamm', 'kaum', 'kayam'},\n 'NH*AHN*na*i': {'naahi', 'nahi'},\n 'XLK*AJLK*ch*a': {'chalayega', 'chalega', 'chilayega'},\n 'AH*AH*ye*i': {'yehi'},\n 'T*AT*to*o': {'to', 'too', 'tooo'},\n 'HM*M*hu*m': {'hum'},\n 'SMJN*ANMS*se*e': {'semjhane'},\n 'K*AK*ki*i': {'ki', 'kii', 'kiwi'},\n 'KXX*SSK*ko*h': {'koshish', 'koshishh'},\n 'KR*RK*ka*r': {'kaar', 'kar', 'karr', 'kaur'},\n 'RH*AHR*ra*e': {'rahe', 'rahee', 'rahiye'},\n 'HN*N*ha*n': {'haan',\n 'haann',\n 'hain',\n 'han',\n 'hawaiian',\n 'hawaiyan',\n 'hawaon',\n 'hawayein',\n 'hawon'},\n 'LK*KL*lo*g': {'log', 'loog'},\n 'SP*PS*sa*b': {'saab', 'sab'},\n 'K*AK*ko*o': {'ko', 'koo'},\n 'AS*AS*is*i': {'isaai', 'isi', 'issi'},\n 'MN*NM*me*n': {'mean', 'mein', 'men', 'meun'},\n 'TL*LT*to*l': {'tol', 'toll', 'tool', 'towel'},\n 'TT*ATT*de*e': {'deate', 'dede', 'dete'},\n 'KNTKSTT*TKSTNK*co*:': {'context:'},\n 'KK*AKK*ca*e': {'cake'},\n 'KH*AHK*ka*a': {'kaha', 'kahaa'},\n 'H*A*ha*i': {'hai',\n 'haii',\n 'haiii',\n 'haiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii',\n 'hawai',\n 'hawaii'},\n 'JT*TJ*je*t': {'jeet', 'jet', 'jett'},\n 'K*AK*ka*a': {'ka', 'kaa', 'kauwa', 'kauwaa', 'kaya'},\n 'JXN*NSJ*ja*n': {'jashn'},\n 'AR*R*au*r': {'aur'},\n 'XP*HPS*sh*h': {'shubah'},\n 'XRT*TRS*sh*t': {'shart',\n 'shirt',\n 'shohrat',\n 'short',\n 'shroowat',\n 'shuruaat',\n 'shuruat'},\n 'ATNK*NT*ea*g': {'eating'},\n 'PRT*TRP*br*d': {'brad', 'bread', 'breed', 'brid', 'broad'},\n 'PKT*ATKP*pa*a': {'pakadta', 'pakata', 'pakda', 'pakoda'},\n 'AT*T*at*t': {'at', 'att'},\n 'TNTNS*SNTNT*ta*s': {'tandons'},\n 'KTJ*AKTK*co*e': {'cottage'},\n 'FXL*ALSF*va*i': {'vaishali'},\n 'TM*HMT*da*h': {'damoh'},\n 'SK*AKS*so*a': {'soca', 'sookha'},\n 'PR*ARP*pu*i': {'puri', 'puuri'},\n 'TN*ANT*de*a': {'deana', 'deanna', 'deewana', 'deewaniya', 'dena', 'denia'},\n 'MTLP*PLTM*ma*b': {'matlab', 'matlb'},\n 'TM*MT*tu*m': {'tum'},\n 'KRXM*AMSRK*ka*a': {'karishma'},\n 'NHN*NHN*na*n': {'nahin'},\n 'RFN*ANFR*ra*a': {'ravana', 'raveena', 'ravenna', 'ravinia'},\n 'H*A*ho*o': {'ho', 'hoo', 'hooo', 'hooooo'},\n 'A*H*wo*h': {'woh'},\n 'J*AJ*jo*o': {'jo', 'joao', 'joo'},\n '0*T*th*,': {'tha,', 'thaa,', 'the,', 'thi,'},\n 'AP*P*ab*b': {'ab', 'abb'},\n 'KHN*NHK*ka*n': {'kahaaaan',\n 'kahaan',\n 'kahaen',\n 'kahan',\n 'kaheen',\n 'kahein',\n 'kahen',\n 'kahin',\n 'kahun'},\n 'SK*AKS*sa*i': {'saakhi', 'sachai', 'sachhai', 'sachhi', 'sakai', 'saki'},\n 'HKNK*AKNK*ho*o': {'hogayi*aankho'},\n 'ANS*SN*aa**': {'aansoo*', 'aansu*'},\n 'SH*AHS*sa*i': {'sahi'},\n 'M*AM*me*e': {'me', 'mee'},\n 'AR*R*ya*r': {'yaaar', 'yaar', 'yar', 'yarr'},\n 'KN*NK*ka*n': {'kaan', 'kahn', 'kan', 'kaun', 'kawn', 'kayon'},\n 'SLL*LLS*za*l': {'zaleel'},\n 'ANSN*NSN*in*n': {'insaan', 'insan'},\n 'HNN*N*ha*,': {'haan,', 'hain,'},\n 'KTK*AKTK*kh*i': {'khidki'},\n 'KL*ALK*kh*i': {'khaali', 'khali', 'kheli', 'kholi', 'khuli'},\n '0*AT*th*i': {'thai', 'the/hai', 'thei', 'thi', 'thii'},\n 'JSMN*NMSJ*ji*n': {'jismein'},\n 'PPR*ARPP*ba*o': {'baburao'},\n 'JNK*NJ*jh*k': {'jhaank'},\n 'RH*AHR*ra*a': {'raha', 'rahaa'},\n '0*AT*th*a': {'tha', 'thaa', 'thha'},\n 'KNFS*SFNK*co*s': {'confess'},\n 'KRT*ATRK*ka*e': {'karate',\n 'karatee',\n 'karde',\n 'karode',\n 'karte',\n 'karwaate',\n 'karwate'},\n 'A*A*ya*a': {'ya', 'yaa'},\n 'TRTR*ARTRT*to*e': {'torture'},\n 'ATN*ANT*it*a': {'itana', 'itna', 'itnaa'},\n 'PT*ATP*ba*a': {'baadhaa',\n 'bada',\n 'badaaa',\n 'badaya',\n 'baddua',\n 'badha',\n 'badhaiya',\n 'badhaya',\n 'badhiya',\n 'badhya',\n 'badiya',\n 'badta',\n 'bata',\n 'bataa',\n 'bataya'},\n 'KNFSN*NSFNK*co*n': {'confession', 'conffesn', 'confusion'},\n 'TJ*AJT*tu*e': {'tuje', 'tujhe'},\n 'SN*NJS*si*n': {'sign'},\n 'AP*P*up*p': {'up'},\n 'K*AK*ke*e': {'ke', 'kee'},\n 'RS*SR*rs*s': {'rs', 'rss'},\n 'MR*ARM*me*e': {'mere', 'meree'},\n 'AKNT*TNK*ac*t': {'account'},\n 'TLN*ANLT*da*e': {'daalne', 'dalane', 'dalne', 'dalwane'},\n 'PTNJJ*NTP*pa*,': {'padenge,'},\n 'TP*PT*ta*b': {'tab'},\n 'H*A*hi*i': {'hi', 'hii', 'hiii'},\n 'RXRJ*AKRSR*re*e': {'recharge'},\n 'PK*AJP*pa*a': {'paaega', 'paayega', 'paega', 'payega'},\n 'MS*SM*mi*s': {'mis', 'miss'},\n 'A*A*u*u': {'u'},\n 'MST*TSM*ms*d': {'msd'},\n 'ATT*TT*aa*t': {'aadat'},\n 'K*AK*ga*i': {'gai', 'gaii', 'gayi', 'gayii'},\n 'APK*AKP*aa*o': {'aaapko', 'aapako', 'aapko'},\n 'KPTN*NTPK*ca*n': {'capetown', 'captain'},\n 'TKT*A0KT*de*e': {'dekhte'},\n 'H*A*hu*e': {'hue', 'huee', 'huwe', 'huye'},\n 'NT*ATN*no*e': {'note'},\n 'APLKXN*NTSLP*ap*n': {'application'},\n 'KKL*ALKK*go*e': {'google'},\n 'PLSTR*ARTSLP*pl*e': {'playstore'},\n 'TNLT*TLNT*do*d': {'donald', 'download'},\n 'HJ*AK*ho*i': {'hogi', 'hogyi'},\n '**,*,': {','},\n 'FK*AKF*fa*e': {'fake'},\n 'SKT*ATKS*sa*i': {'sakti'},\n 'PS*SP*ba*s': {'baas', 'bas', 'bass', 'baus'},\n 'KP*APK*ka*i': {'kabhi', 'kabi'},\n 'KMNT*TNMK*gh*d': {'ghamand'},\n 'K*AK*ki*a': {'kia', 'kiya', 'kiyaa', 'kiyaaa'},\n 'PX*ASP*bh*a': {'bhaasha', 'bhasha'},\n 'SK*AKS*si*a': {'sikha', 'sikhaya', 'sikka'},\n 'XNL*LNK*ch*l': {'chainal', 'chanel', 'channel'},\n 'PR*RP*pa*r': {'paar', 'pair', 'par', 'parr', 'pawar', 'payar'},\n 'KP*PK*ka*b': {'kab', 'kabb'},\n 'AT*AT*aa*a': {'aadha', 'aata', 'aatta'},\n 'PT*TP*ba*,': {'badho,', 'bata,', 'batao,'},\n 'SR*ARS*za*a': {'zaara', 'zara', 'zaraa', 'zariya'},\n 'P*AP*bh*i': {'bhaaaaiiiiii',\n 'bhaaaiiiii',\n 'bhaai',\n 'bhai',\n 'bhaii',\n 'bhaiiii',\n 'bhaui',\n 'bhi',\n 'bhii',\n 'bhoi'},\n 'TK*AKT*de*e': {'deke', 'dekhe', 'dekhiye', 'dekhye'},\n 'AKR*RK*ag*r': {'agar', 'ager', 'agr'},\n 'AS*AS*ay*i': {'ayesi'},\n 'H*A*he*e': {'he', 'hee', 'heeee'},\n 'PT*TP*ba*t': {'baaaat', 'baaat', 'baat', 'bat', 'baut'},\n 'PKL*ALSP*pe*y': {'peechley'},\n 'SL*ALS*sa*o': {'saalo', 'salo'},\n 'PJP*PP*bj*p': {'bjp'},\n 'AL*AL*wa*y': {'waaley', 'walay', 'waley', 'waly'},\n 'KNKRS*SRNK*co*s': {'congres', 'congress'},\n 'KLF*FLK*kh*f': {'khelaaf', 'khilaaf', 'khilaf'},\n 'APN*ANP*ap*a': {'apana', 'apanaa', 'apna', 'apnaya'},\n 'PRTX*ASTRP*pr*i': {'prattyashi'},\n 'K*AK*ky*u': {'kyu'},\n 'KR*ARK*kh*a': {'khara', 'kheera'},\n 'KRT*ATRK*ka*y': {'karrtey', 'kartay', 'kartey'},\n '0N*NT*th*n': {'than', 'thein', 'then', 'thin', 'thon'},\n 'TL*ALT*de*i': {'dehli', 'delhi', 'deli'},\n 'FTNSP*APSNTF*vi*a': {'vidhansabha'},\n 'ALKXN*NTSL*el*n': {'election'},\n 'K*AK*ky*a': {'kya', 'kyaa', 'kyaaa'},\n 'HK*AK*ho*a': {'hoga', 'hogaya'},\n 'AS*AS*ai*a': {'aisa'},\n 'ANN*ANN*un*e': {'unhone'},\n 'PL*ALP*pe*e': {'peele', 'pehle', 'pelee'},\n 'PK*SP*pi*c': {'pic'},\n 'N*AN*ni*i': {'ni', 'nii'},\n 'TK*AKT*dk*i': {'dkhi'},\n 'T*AT*ti*i': {'ti'},\n 'TMR*ARMT*tm*i': {'tmhari'},\n 'MLN*ANLM*mi*e': {'milane', 'milne'},\n 'K*K*k*k': {'k'},\n 'PT*TP*ba*d': {'baaad', 'baad', 'bad'},\n 'AKNR*ARNJ*ig*e': {'ignore'},\n 'TM*AMT*tm*e': {'tmhe'},\n 'M*AM*ma*i': {'mai', 'maui'},\n 'ATN*ANT*it*e': {'itane', 'itanee', 'itne', 'itnee'},\n 'KMS*SMK*ga*s': {'games'},\n 'P*P*b*b': {'b'},\n 'NNJ*ANN*ni*a': {'ninja'},\n 'TRTLS*SLTRT*tu*s': {'turtles'},\n 'STRT*TRTS*st*t': {'start', 'stewart', 'street', 'stuart'},\n 'FTR*R0JF*fi*r': {'fighter'},\n 'KR*RK*ca*r': {'caar', 'car', 'carr'},\n 'RS*AKR*ra*e': {'race'},\n 'MR*ARM*ma*o': {'maaro', 'mario', 'maro'},\n 'KL*LK*kh*l': {'khaal',\n 'khail',\n 'khal',\n 'khayaal',\n 'khayal',\n 'khel',\n 'khol',\n 'khool',\n 'khul',\n 'khyaal'},\n 'KX*SK*kh*h': {'khich', 'khush'},\n 'PK*AKP*ba*i': {'baaaki',\n 'baachhi',\n 'baakhi',\n 'baaki',\n 'bachai',\n 'bachhi',\n 'bakhi',\n 'baki',\n 'bakiii',\n 'baqi'},\n 'TN*NT*dh*n': {'dhan',\n 'dhawan',\n 'dhayan',\n 'dhayn',\n 'dhiyaan',\n 'dhuaan',\n 'dhuaon',\n 'dhyaan',\n 'dhyan'},\n 'TN*ANT*de*e': {'deewane', 'dene', 'dennie'},\n 'TM*MT*ty*m': {'tym'},\n 'KSK*AKSK*ki*e': {'kisake', 'kisike', 'kiske'},\n 'PS*SP*pa*s': {'paas', 'pas', 'pass', 'paws'},\n 'AS*S*is*s': {'is', 'iss'},\n 'RXTT*T0SR*ri*d': {'rishtewaad'},\n 'AST*ATS*az*i': {'azaadi', 'azadi'},\n 'APS*ASP*aa*e': {'aapse'},\n 'XT*A0K*ch*e': {'chahte'},\n 'HPLSSS*SSSLP*ha*z': {'haiplzzzzzz'},\n 'TNT*TNT*dn*t': {'dnt'},\n 'PRK*KRP*br*k': {'break', 'brk', 'brook'},\n '0R*RT*th*r': {'their', 'thor', 'thr', 'thur'},\n 'HRTS*STR*he*s': {'hearts'},\n 'SR*RS*si*r': {'sir'},\n 'PLSS*SSLP*pl*z': {'plzzz', 'plzzzz'},\n 'F*HF*vo*h': {'voh'},\n 'PL*ALP*bo*a': {'bola', 'boleya'},\n 'JSN*ANSJ*ji*e': {'jisne'},\n 'KS*ASK*ki*i': {'kisi', 'kisii', 'kissi'},\n 'PRT*ATRP*pa*y': {'parody', 'party'},\n 'SPRT*TRPS*su*t': {'suparhit', 'superhit', 'support'},\n 'KRN*ANRK*ka*e': {'karaane',\n 'karane',\n 'karene',\n 'karine',\n 'karne',\n 'karonye',\n 'karwaane',\n 'karwane'},\n 'MN*ANM*ma*a': {'maana', 'mana', 'mania', 'manna', 'manwa'},\n 'KRT*ATRK*ka*a': {'karadiya', 'karata', 'kardiya', 'karta', 'karwata'},\n 'FLP*PLF*fl*p': {'flap', 'flip', 'flop'},\n 'ANJ*AJN*an*i': {'annaji'},\n 'KN*ANK*ko*e': {'kone'},\n 'AL*AL*wa*e': {'waale', 'wale', 'walee', 'walle'},\n 'PLT*TLP*pl*t': {'plot'},\n 'JK*HKJ*ja*h': {'jagah'},\n 'ST*ATS*zy*a': {'zyaada', 'zyaadaa', 'zyada'},\n 'HT*AT*ho*i': {'hoti'},\n 'SL*ALS*sa*a': {'saaala', 'saala', 'sala', 'salaaa', 'salla'},\n 'KM*AMK*ga*e': {'game'},\n 'PR*ARP*bh*e': {'bhare'},\n 'RT*A0R*re*e': {'rehte'},\n '0*AT*th*e': {'the', 'thee', 'thhe'},\n 'ASM*AMS*is*e': {'isme'},\n 'TKPS*SPKT*dh*z': {'dhokebaaz'},\n '**>_*<': {'>_<'},\n 'X*AK*ch*a': {'cha', 'chaa', 'chaiyya', 'chiya', 'chya'},\n 'ST*ATS*se*a': {'seedha', 'seeta'},\n 'NSR*RSN*na*r': {'nasr', 'nazar'},\n 'AN*AN*aa*e': {'aaine', 'aane'},\n 'LK*AKL*la*a': {'laga', 'lagaa', 'lagaya', 'lagwa', 'laoga'},\n 'KSP*PSK*ka*b': {'kasab'},\n 'TRMP*PMRT*tr*p': {'trump'},\n 'TK*KT*ta*k': {'taak', 'tak'},\n 'TM*AMT*ti*e': {'time'},\n 'SN*ANS*su*a': {'suna', 'sunha', 'sunna'},\n 'AK*AK*ac*a': {'achha'},\n 'SMJ*AMS*sa*a': {'samja', 'samjha', 'samjhaa', 'samjhaya'},\n 'KP*PK*ka*,': {'kabhi,'},\n 'SR*ARS*sa*e': {'saaare', 'saare', 'sare', 'saree', 'sayre'},\n 'XSN*NSK*ch*n': {'cheezein',\n 'cheezen',\n 'cheezon',\n 'chizein',\n 'chizen',\n 'chosen'},\n 'TL*LT*de*l': {'deal', 'del', 'dell', 'deol'},\n 'XT*TS*sh*d': {'shayad',\n 'shayd',\n 'shed',\n 'shoud',\n 'showed',\n 'shud',\n 'shudd',\n 'shyad'},\n 'LF*AFL*li*e': {'life', 'live'},\n 'SKSSFL*LFSKS*su*l': {'succesful', 'succesfull', 'successfull'},\n 'HT*AT*ho*a': {'hota'},\n '0RK*KRT*th*k': {'tharak'},\n 'SL*ALS*sa*e': {'saale', 'sale'},\n 'HR*R*ha*r': {'haar', 'haaur', 'haiaur', 'hair', 'har', 'harr'},\n 'LTK*AKTL*la*i': {'ladki', 'latki'},\n 'MNXN*NTNM*me*n': {'mention'},\n 'AJ*J*aa*j': {'aaaj', 'aaj', 'aawaj'},\n 'MX*KTM*ma*h': {'match'},\n 'N*AN*na*a': {'na', 'naa', 'naaa', 'naiya', 'naiyya', 'naya', 'nayyiaa'},\n 'KL*LK*ka*l': {'kaal', 'kael', 'kal'},\n 'LTK*AKTL*la*e': {'ladakee', 'ladke', 'latke'},\n 'AN*AN*in*e': {'inhe', 'inne'},\n 'SNT*ATNS*se*i': {'sendai', 'senti'},\n 'KRN*ANRK*kr*e': {'krane', 'krenhe', 'krne'},\n 'KRT*ATRK*kr*e': {'krde', 'krtae', 'krte', 'krwate'},\n '**:(*(': {':('},\n 'TKST*TKST*te*t': {'text'},\n 'KR*ARK*ka*o': {'kaaro',\n 'karao',\n 'kariyo',\n 'karo',\n 'karoo',\n 'karooooo',\n 'karwao'},\n 'FR*RP*ph*r': {'pher', 'phir', 'phor', 'phr'},\n 'TKT*A0KT*de*y': {'dekhtay'},\n 'TT*ATT*di*i': {'didi'},\n 'AP*AP*ab*i': {'abhi', 'abhiii', 'abi'},\n 'APN*ANP*ap*i': {'apani', 'apni'},\n 'AP*AP*ab*e': {'abbe', 'abbie', 'abe'},\n 'KNKRSNN*NSRNK*co*,': {'congressiyon,'},\n 'NJRNS*SNRJN*ni*s': {'nigerians'},\n 'HT*AT*ho*e': {'hoodie', 'hootie', 'hote'},\n 'PNTR*RTNP*ba*r': {'bandar'},\n 'KN*NK*ky*n': {'kyon', 'kyoon', 'kyoun', 'kyun'},\n 'PR*ARP*bu*a': {'bura', 'buraiya'},\n 'MN*NM*ma*n': {'maan', 'maiin', 'main', 'man', 'mann', 'maun', 'mayan'},\n 'K*AK*ga*e': {'gaaye', 'gae', 'gaye', 'gayee'},\n 'KTN*ANTK*ki*e': {'kitane', 'kitne'},\n 'TK*KT*du*h': {'dukh'},\n 'TFRS*AKRFT*di*e': {'divorce'},\n 'ASM*AMS*us*e': {'usme', 'usshme'},\n 'K*AK*ko*i': {'koi', 'koii', 'koyi'},\n 'MTFT*ATFTM*ma*e': {'matvate'},\n 'RNTK*STNJRR*/r*c': {'/rajnetic'},\n 'HN*AN*ho*a': {'hona', 'honna'},\n 'XH*AHK*ch*e': {'chaahiye', 'chahe', 'chahie', 'chahiye', 'chahye', 'chuhe'},\n 'MHL*ALHM*ma*o': {'mahilao', 'mahilayo'},\n 'XRF*PRS*sh*h': {'sharaph'},\n 'SMN*NMS*sa*n': {'saamaan', 'samaan', 'saman', 'samman'},\n 'TRPL*ALPRT*tr*e': {'triple', 'trouble'},\n 'TLK*KLT*ta*q': {'talaq', 'taluq'},\n 'RK*AKR*ro*e': {'roke'},\n 'KRJ*AKRK*ka*e': {'karaoge', 'karge', 'karoge', 'karwaoge'},\n 'S0*TS*sa*h': {'saath', 'sath'},\n 'MN*ANM*ma*e': {'maane',\n 'maine',\n 'mainne',\n 'manaye',\n 'mane',\n 'manne',\n 'mayne'},\n 'TM*AMT*tu*e': {'tumhe'},\n 'ALT*ATL*ul*a': {'ulta'},\n 'HT*T*ha*h': {'hadh'},\n 'AS*AS*wi*e': {'wise'},\n 'T*AT*tu*u': {'tu'},\n 'PL*ALP*bo*i': {'boli'},\n 'N*AN*na*i': {'naayi', 'nai', 'nayi'},\n 'LNXR*RKNL*lu*r': {'lunchar', 'luncher'},\n 'PJ*AJP*pa*i': {'paaji', 'paegi', 'paji', 'payegi'},\n 'X*AK*ch*i': {'chaai', 'chaayi', 'chai', 'chaiwai', 'chi', 'chiii'},\n 'PSKT*TKSP*bi*t': {'biscuit', 'biskoot', 'bizkit'},\n 'K*AK*kh*a': {'kha', 'khaa', 'khaya', 'khoya'},\n '0K*KT*th*k': {'thaak', 'thak', 'thakk', 'theek', 'thik', 'thk', 'thok'},\n 'H**ha*,': {'haaye,', 'hai,', 'hai,,'},\n 'AS*AS*ai*e': {'aise'},\n 'TXRT*TRST*ts*t': {'tshirt'},\n 'KS*ASK*ka*e': {'kaise', 'kaisee', 'kassie'},\n 'TL*LT*da*l': {'daal', 'dal', 'dayal'},\n 'SKT*ATKS*sa*a': {'sakta'},\n 'TTR*RTT*tw*r': {'twiter', 'twitter', 'twtr'},\n 'ALN*NL*wa*n': {'waalon', 'walloon', 'walon', 'waylon'},\n 'KXNSP*APSNSK*kh*i': {'khushnaseebi'},\n 'HP*AP*ha*y': {'happy', 'happyyy'},\n 'PR0T*ATTRP*bi*y': {'birthday', 'birthdayyyy'},\n 'TN*ANT*dh*i': {'dhoni', 'dhunayi'},\n '**&*&': {'&'},\n 'HL*AL*he*o': {'hello', 'helo'},\n 'SKX*ASKS*sa*i': {'saakshi', 'sakshi'},\n 'PP*APP*bh*i': {'bhabhi', 'bhabi', 'bhaibhai'},\n 'KRT*TRK*kh*t': {'khairaat', 'khairiyat'},\n 'LLL*LL*lo*,': {'lol,'},\n 'FNK*NP*ph*k': {'phenk', 'phoonk'},\n 'TT*ATT*de*a': {'deta'},\n 'PXR*ARSP*be*e': {'bechare'},\n 'KT*TK*ka*t': {'kaat', 'kat', 'katt'},\n 'JT*ATJ*ja*a': {'jaada',\n 'jaaeta',\n 'jaata',\n 'jaataa',\n 'jada',\n 'jaida',\n 'jata',\n 'jataa',\n 'jataya',\n 'jayada',\n 'jayda'},\n 'PLL*LP*bi*,': {'bill,'},\n 'ASL*ALS*is*e': {'iseeliye', 'isiliye'},\n 'LT*ATL*la*a': {'ladaa', 'ladta', 'lata', 'lataa', 'lauta'},\n 'T*AT*di*a': {'dia', 'diya', 'diyaa'},\n 'XP*PK*ch*p': {'chaap', 'chap', 'cheap', 'chep', 'chip', 'chup', 'chupp'},\n 'TT*TT*di*d': {'did', 'died'},\n 'ARN*NR*ar*n': {'aran', 'aren', 'arun'},\n 'LL*LL*la*l': {'laal', 'lal'},\n 'S*S*sa*,': {'sa,', 'say,'},\n 'PNKLTX*ASTLNP*ba*i': {'bangladeshi'},\n 'PLR*RLP*pl*r': {'player'},\n 'KX*ASK*kh*i': {'khushi'},\n 'APN*ANP*ap*e': {'apne'},\n 'ANTKX*SKTN*an*h': {'andkosh'},\n 'TKR*ARKT*ta*a': {'takra'},\n 'XL*LK*ch*l': {'chaal', 'chal', 'chaul', 'chawal', 'chill'},\n 'AXNK*KNK*ac*k': {'achaanak', 'achanak'},\n 'SN*ANS*so*y': {'sonny', 'sony'},\n 'TF*FT*tv*v': {'tv'},\n 'TLL*ALLT*di*e': {'dilwale'},\n 'MF*AFM*mo*e': {'moive', 'moovee', 'move', 'movie', 'moviee'},\n 'TLKST*TSSLT*te*t': {'telecast'},\n 'FL*ALF*va*i': {'vaalii', 'vali'},\n 'RT*TR*ra*t': {'raat', 'raitt', 'rat', 'rawat'},\n 'A*A*wh*y': {'why'},\n 'HMX*ASM*ha*a': {'hamesha', 'hameshaa'},\n 'SN*ANS*su*y': {'sunny'},\n 'LN*ANL*le*e': {'leanne', 'lene', 'leone'},\n 'TK*AKT*di*u': {'dikhau'},\n 'PT*ATP*pa*a': {'pada',\n 'padha',\n 'padhaa',\n 'padhaya',\n 'padta',\n 'padwa',\n 'paida',\n 'pata',\n 'pataa',\n 'patta'},\n 'STTNTS*STNTTS*st*s': {'students'},\n 'MHNJ*AKNHM*me*e': {'mehenge'},\n 'TN*NT*di*n': {'dian', 'din', 'dion'},\n 'AK*AK*ag*e': {'agaye', 'agye'},\n 'SX*AKS*so*a': {'socha'},\n 'ANK*KN*aa*h': {'aankh'},\n 'PNT*TNP*ba*d': {'band', 'banned'},\n 'XRT*ATRS*sh*i': {'shruti'},\n 'HSN*NS*ha*n': {'hasan', 'haseen', 'hassan'},\n 'AT*AT*aa*i': {'aadhi', 'aati'},\n 'KT*TK*gu*d': {'gud'},\n 'MRNK*NRM*mr*g': {'mrng'},\n 'MHXFRTR*ARTRFSHM*ma*i': {'mahashivaratri', 'mahashivratri'},\n 'HRTK*KTR*ha*k': {'hardik'},\n 'MNKLKMN*ANMKLKNM*ma*e': {'mangalkamnaye'},\n 'SR*RS*so*,': {'sorry,'},\n 'MJ*AJM*mu*e': {'muje', 'mujhe'},\n 'PJ*AJP*ba*e': {'bajaaye', 'bajawe', 'bajaye', 'baje', 'bajee', 'bajhe'},\n 'FN*ANP*ph*e': {'phone'},\n 'ALPPR*RPPL*wa*r': {'wallpaper'},\n 'SNTJ*AKTNS*zi*i': {'zindagi'},\n 'RH*AHR*ra*i': {'raahi', 'rahi', 'rahii'},\n 'PTL*ALTP*ba*i': {'baatli', 'badhali'},\n 'TMPRR*ARRPMT*te*y': {'temporary'},\n 'PN*NP*ba*n': {'baan', 'baayen', 'ban', 'bann', 'bayaan', 'bayan'},\n 'LK*KL*la*g': {'laag', 'lag', 'lagg'},\n 'XK*AKK*ch*a': {'chacha', 'chakka', 'chauka', 'chokha', 'chuka', 'chukaya'},\n 'AK*AK*ac*i': {'achhai', 'achhhi', 'achhi'},\n 'NHH*HN*na*,': {'nahi,'},\n 'M0N*NTM*mi*n': {'mithun', 'mitthaiyan'},\n 'KL*ALK*kh*o': {'khayaalo', 'khelo', 'khilao', 'kholo'},\n 'AMK*KM*om*g': {'omg'},\n 'N*AN*ne*e': {'ne', 'nee'},\n 'RT*TR*rt*t': {'rt'},\n 'SPR*ARPS*sp*e': {'spare', 'spree'},\n '0NKS*SKNT*th*s': {'thanks', 'thinks', 'thnks'},\n 'KX*ASKK**k*i': {'*khushi'},\n 'AMT*TM*am*t': {'amit'},\n 'MHMT*TMHM*mu*d': {'muhammad'},\n 'AL*AL*al*i': {'ali'},\n 'PKL*ALKP*pa*i': {'pagli'},\n 'PLS*ASLP*pl*e': {'please'},\n 'FL*ALF*fo*w': {'follow', 'folow'},\n 'FR*ARF*fr*e': {'free'},\n 'TR*ART*da*u': {'daaru', 'daru', 'daruu'},\n 'PLT*ATLP*pi*e': {'pilaate'},\n 'HS*AS*hi*a': {'hisa', 'hissa'},\n 'AL*AL*wa*a': {'waala', 'wala', 'walaaaaa', 'walla'},\n 'TKLF*FLKT*ta*f': {'takleef', 'taklif', 'taqleef'},\n 'TK*AJT*de*a': {'dega', 'degaaa'},\n 'A*A*wo*o': {'wo'},\n 'KS*ASK*ki*a': {'kisa', 'kissa'},\n 'HMR*ARM*ha*i': {'hamaari', 'hamari'},\n 'KXT*A0SK*ka*i': {'kashti'},\n 'AHN*NH*wa*n': {'wahaan', 'wahan', 'wahin'},\n 'TP*APT*do*i': {'doobi'},\n 'JHN*NHJ*ja*n': {'jahaan', 'jahan'},\n 'PN*ANP*pa*i': {'paaani', 'paani', 'pani'},\n 'H*A*hu*u': {'hu', 'huuuu'},\n 'MR*ARM*me*a': {'mehra', 'meira', 'mera'},\n 'MTF*AFTM*mo*e': {'motive'},\n 'AR*R*er*r': {'er', 'err'},\n 'L*AL*le*e': {'le', 'lee', 'leye'},\n 'AK*K*ek*k': {'ek'},\n 'KR*ARK*gu*u': {'guru'},\n 'J*AJ*ji*i': {'ji', 'jii'},\n 'PXN*ANXP*pe*e': {'pehchaane', 'pehchane', 'pehchanne'},\n 'ANKR*RKN*in*r': {'inkaar', 'inkar'},\n 'LKN*NKL*lo*n': {'logan', 'logon', 'loogoun'},\n 'PKSTN*ANTSKP*pa*i': {'pakistani'},\n 'FS*ASF*vi*a': {'visa'},\n 'PSPRT*TRPSP*pa*t': {'passport'},\n 'MFT*TFM*mu*t': {'muft'},\n 'J*AJ*ja*e': {'jaae',\n 'jaaiye',\n 'jaaye',\n 'jaayiye',\n 'jae',\n 'jaie',\n 'jaiye',\n 'jaye'},\n 'PRNM*MNRP*pr*m': {'pranaam', 'pranam'},\n 'ARR*R*ya*,': {'yaar,'},\n 'T*AT*do*o': {'do', 'doo'},\n 'A*H*ye*h': {'yeah', 'yeh'},\n 'J*AJ*ja*a': {'ja', 'jaa', 'jaya'},\n '**_/*_': {'_/\\\\_'},\n 'RLJ*AKLR*ru*e': {'rulaoge'},\n 'ANT*ATN*in*a': {'india'},\n 'SR*ARS*sr*i': {'sri'},\n 'LNK*AKNL*la*a': {'laaunga', 'lanka', 'launga'},\n 'PL*ALP*bu*o': {'bulao'},\n 'FT*ATF*vo*e': {'vote'},\n 'RFS*SFR*re*s': {'reviews'},\n 'SRRT*TRRS*za*t': {'zaroorat', 'zarorat', 'zarurat'},\n 'KF*AFK*ka*i': {'kaafi', 'kafi', 'kaufii', 'kavi'},\n 'PRTSHN*NHSTRP*pr*n': {'protsahan'},\n 'MLT*ATLM*mi*a': {'milata', 'milta'},\n 'AN*N*in*n': {'in', 'inn'},\n 'PTN*NTP*ba*n': {'baatein',\n 'baaten',\n 'baaton',\n 'badhaiyaan',\n 'badhaiyan',\n 'badhen',\n 'badhun',\n 'badon',\n 'bataayein',\n 'bataen',\n 'batan',\n 'batayen',\n 'batein',\n 'baten',\n 'baton',\n 'batten',\n 'battiyan'},\n 'ATM*AMT*aa*i': {'aadami', 'aadamii', 'aadmi'},\n 'MLN*ANLM*mi*a': {'milania', 'milena', 'milna'},\n 'PHT*THP*bo*t': {'bohat', 'bohoot', 'bohot', 'bohut'},\n 'SLMNNN*NNNMLS*sa*n': {'salmannnnn', 'salmannnnnn'},\n 'A*A*aa*a': {'aa', 'aaa', 'aaaa', 'aaaaa', 'aaya', 'aayaa', 'aaye,aa'},\n 'ANTSR*RSTN*in*r': {'intazaar',\n 'intazar',\n 'intezaar',\n 'intezar',\n 'intzaar',\n 'intzar'},\n 'ATR*RT*id*r': {'idhar'},\n 'TR*ART*te*e': {'tere', 'terre'},\n 'AR*AR*ar*y': {'aray', 'arey', 'array', 'arrey', 'arry', 'ary'},\n 'AP*P*aa*p': {'aap'},\n 'TNK*NT*dh*g': {'dhang'},\n 'HHH*AHH*ha*a': {'hahaha',\n 'hahahaa',\n 'hahahaaa',\n 'hahahaaaa',\n 'hahahahha',\n 'hai_hahaha'},\n 'XF*AFS*sh*e': {'shave', 'shivaye'},\n 'LN*ANL*le*a': {'leeanna', 'leena', 'lena'},\n 'TR*ART*te*a': {'tera', 'teraa', 'terra'},\n 'SHR*ARHS*sa*a': {'sahaara', 'sahara'},\n 'HH*AH*ha*a': {'haaha',\n 'haahaa',\n 'haha',\n 'hahaa',\n 'hahahha',\n 'hahahhaa',\n 'hahahhahha'},\n 'TT*ATT*da*e': {'daate', 'date', 'daudte'},\n 'AT*T*ya*d': {'yaad', 'yad'},\n 'TLJ*AKLT*di*e': {'dialouge', 'dilaoge'},\n 'KS*ASK*kh*i': {'khaasi', 'khusi'},\n 'H*A*hu*i': {'huawei', 'hui', 'huiii', 'huwi', 'huyi'},\n 'TMR*ARMT*tu*a': {'tumara', 'tumhaaara', 'tumhaara', 'tumhara', 'tumhra'},\n 'JNMTN*NTMNJ*ja*n': {'janamdin', 'janmdin'},\n 'AS*AS*us*e': {'use', 'usse', 'ussee'},\n 'KT*ATK*ga*e': {'gaddhe', 'gadhe', 'gate'},\n 'PHR*RHP*ba*r': {'baahar', 'bahaar', 'bahar', 'baher'},\n 'ATRN*ANRT*ut*a': {'utarna'},\n 'KMPLSR*ARSLPMK*co*y': {'compulsory'},\n 'KR0*ATRK*ke*i': {'keerthi'},\n 'LTS*STL*le*s': {'leads', 'leeds', 'lets'},\n 'APN*NP*op*n': {'opan', 'open', 'oppn'},\n 'NK*AKN*ng*o': {'ngo'},\n 'TK0R*RTKT*to*r': {'together'},\n 'TT*ATT*to*e': {'toadie', 'tode', 'todte', 'toote'},\n 'TL*LT*di*l': {'dial', 'dil', 'dill'},\n 'MRSN*NSRM*ma*n': {'marezon'},\n 'L*AL*li*a': {'lia', 'liya', 'liyaa', 'liyea'},\n 'KP*APK*kb*i': {'kbhi', 'kbi'},\n 'XN*ANK*ch*a': {'chana', 'chanaa', 'china', 'chuna'},\n 'PX*ASP*pe*e': {'peche', 'peeche', 'peeeche', 'peshwe'},\n 'N*AN*nh*i': {'nhai', 'nhi', 'nhii'},\n 'KR*RK*kr*r': {'kr', 'krr'},\n '0T*TT*th*t': {'thaat', 'that'},\n 'SPN*NPS*sp*n': {'spain', 'spann', 'spin', 'spoon'},\n 'RF*AFR*ra*i': {'raffi', 'rafi', 'ravi'},\n 'AXN*NS*as*n': {'ashwin'},\n 'AFRK*ASRF*af*a': {'africa'},\n 'ARLT*TLR*wo*d': {'world'},\n 'KPS*SPK*cu*s': {'cubs', 'cups'},\n 'M*AM*me*i': {'mei'},\n 'T*T*t*t': {'t'},\n 'F*F*v*v': {'v'},\n 'KT*TK*gu*:': {'guddu:'},\n 'MM*AMM*mu*y': {'mummy'},\n 'KN*ANK*kh*e': {'khaane', 'khane', 'khanye'},\n 'XLK*KLK*ch*k': {'chalaak', 'chalk'},\n 'XRN*ANRK*ch*e': {'churane'},\n 'MS*ASM*ma*a': {'maaza', 'mausa', 'maza', 'mazaa', 'mazya', 'mazza'},\n 'PLKLL*LKLP*bi*,': {'bilkul,'},\n 'SKT*ATKS*sa*e': {'sakate', 'sakte'},\n 'SPK*AKPS*sa*o': {'sabhiko', 'sabko'},\n 'PK*AKP*ba*a': {'bacha', 'bachaa', 'bachha', 'baga', 'bakaya'},\n 'KHHRF*FRSSHK*ka*f': {'kaho,sirf'},\n 'AL*AL*il*a': {'ilawa', 'ilha'},\n 'AK*K*ok*k': {'ok', 'okk'},\n 'ML*ALM*mi*i': {'mili'},\n '**\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
Sentencetrr
0congratulations on you celebrating british kid...congratulations on you celebrating british kid...
1uske liye toh bahat kuch karwana padega yee pa...you will have to do a lot for that, it will no...
2yehi tooo hum semjhane ki koshish karr rahee h...this is what we are trying to understand. peop...
3cake kaha haiwhere is the cake??
4jett kauwa jashn aur shubah ki shroowat eating...victory celebration and early morning eating b...
.........
1299muse vo bohut pasand haii like it a lot
1300muje psnd haii like it a lot
1301muje maarkit see vegtables lekar aaine hawoni have to bring vegetables from the market
1302sabse best frndmost best friend
1303best frnd sabsemost best friend
\n

183535 rows × 2 columns

\n"},"metadata":{}}]},{"cell_type":"code","source":"df = df.sample(frac = 1)","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:54.284580Z","iopub.execute_input":"2023-03-16T11:45:54.285329Z","iopub.status.idle":"2023-03-16T11:45:54.321449Z","shell.execute_reply.started":"2023-03-16T11:45:54.285289Z","shell.execute_reply":"2023-03-16T11:45:54.320417Z"},"trusted":true},"execution_count":23,"outputs":[]},{"cell_type":"code","source":"df['len_sen']=df['Sentence'].apply(lambda x: len(x.split(' ')))\ndf=df[df['len_sen']<=20]","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:54.323403Z","iopub.execute_input":"2023-03-16T11:45:54.323788Z","iopub.status.idle":"2023-03-16T11:45:54.763578Z","shell.execute_reply.started":"2023-03-16T11:45:54.323750Z","shell.execute_reply":"2023-03-16T11:45:54.762523Z"},"trusted":true},"execution_count":24,"outputs":[]},{"cell_type":"code","source":"inp=list(df['Sentence'])\ntarg=list(df['trr'])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:54.765394Z","iopub.execute_input":"2023-03-16T11:45:54.765774Z","iopub.status.idle":"2023-03-16T11:45:54.833711Z","shell.execute_reply.started":"2023-03-16T11:45:54.765737Z","shell.execute_reply":"2023-03-16T11:45:54.832417Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"code","source":"# printing last english sentence\nprint(targ[-1])","metadata":{"id":"lH_dPY8TRp3c","outputId":"a4486ac6-bd0f-4b6f-c685-5bdf1983e769","execution":{"iopub.status.busy":"2023-03-16T11:45:54.835218Z","iopub.execute_input":"2023-03-16T11:45:54.835825Z","iopub.status.idle":"2023-03-16T11:45:54.845721Z","shell.execute_reply.started":"2023-03-16T11:45:54.835785Z","shell.execute_reply":"2023-03-16T11:45:54.844518Z"},"trusted":true},"execution_count":26,"outputs":[{"name":"stdout","text":"remove all alarms and reminders\n","output_type":"stream"}]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:54.847682Z","iopub.execute_input":"2023-03-16T11:45:54.848100Z","iopub.status.idle":"2023-03-16T11:45:55.842937Z","shell.execute_reply.started":"2023-03-16T11:45:54.848038Z","shell.execute_reply":"2023-03-16T11:45:55.841702Z"},"trusted":true},"execution_count":27,"outputs":[{"name":"stdout","text":"Thu Mar 16 11:45:55 2023 \n+-----------------------------------------------------------------------------+\n| NVIDIA-SMI 470.82.01 Driver Version: 470.82.01 CUDA Version: 11.4 |\n|-------------------------------+----------------------+----------------------+\n| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n| | | MIG M. |\n|===============================+======================+======================|\n| 0 Tesla P100-PCIE... Off | 00000000:00:04.0 Off | 0 |\n| N/A 35C P0 27W / 250W | 0MiB / 16280MiB | 0% Default |\n| | | N/A |\n+-------------------------------+----------------------+----------------------+\n \n+-----------------------------------------------------------------------------+\n| Processes: |\n| GPU GI CI PID Type Process name GPU Memory |\n| ID ID Usage |\n|=============================================================================|\n| No running processes found |\n+-----------------------------------------------------------------------------+\n","output_type":"stream"}]},{"cell_type":"code","source":"gpus = tf.config.experimental.list_physical_devices('GPU')\nfor gpu in gpus:\n print(\"Name:\", gpu.name, \" Type:\", gpu.device_type)","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:55.866108Z","iopub.execute_input":"2023-03-16T11:45:55.866509Z","iopub.status.idle":"2023-03-16T11:45:56.134202Z","shell.execute_reply.started":"2023-03-16T11:45:55.866449Z","shell.execute_reply":"2023-03-16T11:45:56.133058Z"},"trusted":true},"execution_count":28,"outputs":[{"name":"stdout","text":"Name: /physical_device:GPU:0 Type: GPU\n","output_type":"stream"}]},{"cell_type":"code","source":"from tensorflow.python.client import device_lib\n\ndevice_lib.list_local_devices()","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:56.135612Z","iopub.execute_input":"2023-03-16T11:45:56.136601Z","iopub.status.idle":"2023-03-16T11:45:58.548906Z","shell.execute_reply.started":"2023-03-16T11:45:56.136559Z","shell.execute_reply":"2023-03-16T11:45:58.547810Z"},"trusted":true},"execution_count":29,"outputs":[{"execution_count":29,"output_type":"execute_result","data":{"text/plain":"[name: \"/device:CPU:0\"\n device_type: \"CPU\"\n memory_limit: 268435456\n locality {\n }\n incarnation: 7469856977397221377\n xla_global_id: -1,\n name: \"/device:GPU:0\"\n device_type: \"GPU\"\n memory_limit: 16105865216\n locality {\n bus_id: 1\n links {\n }\n }\n incarnation: 16859620279429889827\n physical_device_desc: \"device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0\"\n xla_global_id: 416903419]"},"metadata":{}}]},{"cell_type":"code","source":"tf.test.is_gpu_available()","metadata":{"execution":{"iopub.status.busy":"2023-03-16T11:45:58.552088Z","iopub.execute_input":"2023-03-16T11:45:58.552542Z","iopub.status.idle":"2023-03-16T11:45:58.567023Z","shell.execute_reply.started":"2023-03-16T11:45:58.552496Z","shell.execute_reply":"2023-03-16T11:45:58.566034Z"},"trusted":true},"execution_count":30,"outputs":[{"execution_count":30,"output_type":"execute_result","data":{"text/plain":"True"},"metadata":{}}]},{"cell_type":"code","source":"# printing first 100 english sentence\nprint(targ[:100])","metadata":{"id":"WtYLR0ZWXDld","outputId":"afcfed6b-ee0f-4007-9b3d-9cafc3eac537","execution":{"iopub.status.busy":"2023-03-16T11:45:58.570720Z","iopub.execute_input":"2023-03-16T11:45:58.571137Z","iopub.status.idle":"2023-03-16T11:45:58.579466Z","shell.execute_reply.started":"2023-03-16T11:45:58.571108Z","shell.execute_reply":"2023-03-16T11:45:58.578494Z"},"trusted":true},"execution_count":31,"outputs":[{"name":"stdout","text":"['set a new alarm', 'set an alarm for 9 am', 'resume my timer', 'set a recurring reminder every morning at 9 : 00 to take my vitamins', 'any parties in philly tonight', 'when is my alarm set for on monday ?', 'set up an alarm to go to sleep tonight .', 'how long will it take me to get to syracuse downtown if i leave here in 10 minutes ?', 'modiji has hand in this.', 'should i buy emergency supplies at the store tonight', 'how long is my drive home', 'are there any accidents on the 33', 'set up a 30 minute meditation timer', 'why is the turnpike backed up', 'vancouver night market next weekend', 'is it going to be hot today', 'set up a nightly sleep reminder for 10 pm', 'events in new york', \"remind me that i didn ' t finish my tax return .\", 'open a message to peter asking him to call me after the meeting .', 'set an alert for me to go and pick up the catering order at 11 : 45 pm', 'can you please remind me to pay the utility bill on friday at 2 pm ?', 'didi.. agreed.. both of them were seen as nirma super.', 'cancel reminder 4 pm tomorrow .', 'set a timer to go off every hour', 'i want to listen to classical music for 1 hour', 'set up a reminder to get my passport application started by the end of the week', 'play me some jazz', 'scrapbooking parties in atlanta next weekend', 'can you set a reminder for my eye appointment on august 15th at 2 pm ?', 'concerts by new edition', 'please set the alarm at 6 am for tomorrow .', 'play some rock music from the 60s .', 'run this timer twice', 'how far from cocoa beach to port canaveral', 'set an egg timer for 5 minutes ?', \"what ' s the average temp in virginia this time of year\", \"set up a recurring reminder about little matt ' s football practice monday thru friday at 5 : 30 pm\", 'should i bring a jacket with me ?', 'can i tour the princeton art museum today and park in front of the university ?', 'delete my daily 3 pm and 4 pm alarms', 'cancel tomorrows alarm .', 'how much of a delay for my trip home today', 'skip the next track', 'set timer for 20 minutes from now', 'how is the weather on saturday', 'is 75 faster than 45 right now', 'can i get from here to dover by 9 pm tonight ?', 'message tim asking if he has found my wallet at his house', 'set timer for the laundry', 'please create a weekly alarm for thursday at 5 pm', 'what temp will is be in washington dc this week', 'remind me to be at the meeting at 6 pm today', 'prateek johri dialogue is not in your control. ,', 'suspend my alarm for the weekend', 'restart all timers', 'text mom and tell her ill be at the show next thursday', 'any free xmas events i can go to this weekend', 'set the snooze setting on the alarm clock to 9 minutes .', 'set an alarm daily at 6 am for the following 2 weeks', 'is there a scenic backroad drive from jamesville to highlands , nc ?', 'turn off my saturday alarms .', 'start timer for 5 hours', 'replay this track', 'delete all reminders about the wedding .', 'i wanna text robert', 'how long will it take me to get to utsa by bicycle from the rim ?', 'cancel my reminder about the anniversary gift', 'what is the seven day forecast ?', 'when will the sun rise today ?', 'do i need a jacket today ?', 'change order of songs', 'remind me to make a salad dish for my social dinner tomrorow at 12 pm .', 'text steve , how are you doing ?', 'could you wake us up', 'set alarm for every hour', 'how cold is it for paris this weekend in degree c', 'what is the temperature today ? i am going to walk down to the casino .', 'set a reminder for a party tonight', 'remind me i have a dentist appointment on thursday', 'how is traffic from irvine to tustin', 'sound an alarm once timer is complete', 'restart timer', 'add another ten minutes to the timer .', 'cancel my alarm ( s ) for saturday', 'how hot will it be tomorrow afternoon ?', \"how long will it take me to get to john ' s house if i leave in 15 minutes ?\", 'please tell me what hours traffic is the heaviest in kansas city', 'set up an wake up alarm at midnight today .', 'looks like someone has made him an uncle in shares', 'start a new alarm for very tuesday at noon .', 'thumbs down the current song playing on pandora', 'what time is my doctor appointment reminder ?', 'look at my messages on whatsapp .', \"when will i arrive at bob ' s dinner party if i leave right now using the highway ?\", 'expressway is costlier or cheaper', 'set an alarm to wake up at 4 am .', 'how many miles are we from salt lake city', 'let me see meetup reminders for the history group .', 'what is the temperature today']\n","output_type":"stream"}]},{"cell_type":"code","source":"# printing first 100 hinglish sentences\nprint(inp[-1:])","metadata":{"id":"WdC25G6AZXr9","outputId":"26bc2c3e-c413-4715-f08a-65b5ad1ba001","execution":{"iopub.status.busy":"2023-03-16T11:45:58.580765Z","iopub.execute_input":"2023-03-16T11:45:58.582330Z","iopub.status.idle":"2023-03-16T11:45:58.591359Z","shell.execute_reply.started":"2023-03-16T11:45:58.582288Z","shell.execute_reply":"2023-03-16T11:45:58.589997Z"},"trusted":true},"execution_count":32,"outputs":[{"name":"stdout","text":"['sabhi alarms aur remnders koo hatya deon']\n","output_type":"stream"}]},{"cell_type":"code","source":"# arrays of strings are shuffled and batches are made\nBUFFER_SIZE = len(inp)\nBATCH_SIZE = 64\n\ndataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\ndataset = dataset.batch(BATCH_SIZE)","metadata":{"id":"TqHsArVZ3jFS","execution":{"iopub.status.busy":"2023-03-16T11:45:58.593109Z","iopub.execute_input":"2023-03-16T11:45:58.593511Z","iopub.status.idle":"2023-03-16T11:46:00.512669Z","shell.execute_reply.started":"2023-03-16T11:45:58.593475Z","shell.execute_reply":"2023-03-16T11:46:00.511527Z"},"trusted":true},"execution_count":33,"outputs":[]},{"cell_type":"code","source":"for example_input_batch, example_target_batch in dataset.take(1):\n print(example_input_batch[:5])\n print()\n print(example_target_batch[:5])\n break","metadata":{"id":"qc6-NK1GtWQt","outputId":"4059f9ad-4d6e-4204-d40a-840261b82f90","execution":{"iopub.status.busy":"2023-03-16T11:46:00.515424Z","iopub.execute_input":"2023-03-16T11:46:00.516310Z","iopub.status.idle":"2023-03-16T11:46:00.828098Z","shell.execute_reply.started":"2023-03-16T11:46:00.516262Z","shell.execute_reply":"2023-03-16T11:46:00.826860Z"},"trusted":true},"execution_count":34,"outputs":[{"name":"stdout","text":"tf.Tensor(\n[b'taylor swift koo play karwayee' b'1 hour kee liye timer seat karein'\n b'kyaa aj koii live music hai'\n b'kyaa aap muje meree location kee liye traffic updates dee sakate heeee'\n b'9 a m kee liye alaram koo cancel karein'], shape=(5,), dtype=string)\n\ntf.Tensor(\n[b'play taylor swift' b'set the timer for 1 hour'\n b'any live music tonight'\n b'can you give me traffic updates for my location'\n b'cancel alarm for 9 a . m .'], shape=(5,), dtype=string)\n","output_type":"stream"}]},{"cell_type":"code","source":"# unicode normalization operation - first step to standardize data\nexample_text = tf.constant('Yahi haal hai')\n\nprint(example_text.numpy())\nprint(tf_text.normalize_utf8(example_text, 'NFKD').numpy())","metadata":{"id":"mD0e-DWGQ2Vo","outputId":"52e95336-e962-4e89-da30-956c44458493","execution":{"iopub.status.busy":"2023-03-16T11:46:00.829577Z","iopub.execute_input":"2023-03-16T11:46:00.859628Z","iopub.status.idle":"2023-03-16T11:46:00.870845Z","shell.execute_reply.started":"2023-03-16T11:46:00.859578Z","shell.execute_reply":"2023-03-16T11:46:00.869271Z"},"trusted":true},"execution_count":35,"outputs":[{"name":"stdout","text":"b'Yahi haal hai'\nb'Yahi haal hai'\n","output_type":"stream"}]},{"cell_type":"code","source":"def tf_lower_and_split_punct(text):\n # Split accecented characters.\n text = tf_text.normalize_utf8(text, 'NFKD')\n text = tf.strings.lower(text)\n # Keep space, a to z, and select punctuation.\n text = tf.strings.regex_replace(text, '[^ a-z.?!,¿]', '')\n # Add spaces around punctuation.\n text = tf.strings.regex_replace(text, '[.?!,¿]', r' \\0 ')\n # Strip whitespace.\n text = tf.strings.strip(text)\n\n text = tf.strings.join(['[START]', text, '[END]'], separator=' ')\n return text","metadata":{"id":"chTF5N885F0P","execution":{"iopub.status.busy":"2023-03-16T11:46:00.872665Z","iopub.execute_input":"2023-03-16T11:46:00.874045Z","iopub.status.idle":"2023-03-16T11:46:00.881931Z","shell.execute_reply.started":"2023-03-16T11:46:00.873999Z","shell.execute_reply":"2023-03-16T11:46:00.880784Z"},"trusted":true},"execution_count":36,"outputs":[]},{"cell_type":"code","source":"print(example_text.numpy().decode())\nprint(tf_lower_and_split_punct(example_text).numpy().decode())","metadata":{"id":"UREvDg3sEKYa","outputId":"c0abfd72-ff76-4661-809c-d1b40f6debb8","execution":{"iopub.status.busy":"2023-03-16T11:46:00.883659Z","iopub.execute_input":"2023-03-16T11:46:00.884080Z","iopub.status.idle":"2023-03-16T11:46:00.907442Z","shell.execute_reply.started":"2023-03-16T11:46:00.884041Z","shell.execute_reply":"2023-03-16T11:46:00.906256Z"},"trusted":true},"execution_count":37,"outputs":[{"name":"stdout","text":"Yahi haal hai\n[START] yahi haal hai [END]\n","output_type":"stream"}]},{"cell_type":"code","source":"max_vocab_size = 53000\n\ninput_text_processor = tf.keras.layers.TextVectorization( # handles vocabulary extraction & conversion of i/p text to sequences of tokens\n standardize=tf_lower_and_split_punct,\n max_tokens=max_vocab_size)","metadata":{"id":"eAY9k49G3jE_","execution":{"iopub.status.busy":"2023-03-16T11:46:00.909139Z","iopub.execute_input":"2023-03-16T11:46:00.909823Z","iopub.status.idle":"2023-03-16T11:46:00.948831Z","shell.execute_reply.started":"2023-03-16T11:46:00.909781Z","shell.execute_reply":"2023-03-16T11:46:00.947635Z"},"trusted":true},"execution_count":38,"outputs":[]},{"cell_type":"code","source":"input_text_processor.adapt(inp) # initializes the layer based on the data\n\n# Here are the first 10 words from the vocabulary:\ninput_text_processor.get_vocabulary()[:10]\n# Hinglish TextVectorization layer","metadata":{"id":"bmsI1Yql8FYe","outputId":"89009883-0834-449f-d408-0d947cd033b2","execution":{"iopub.status.busy":"2023-03-16T11:46:00.951091Z","iopub.execute_input":"2023-03-16T11:46:00.951727Z","iopub.status.idle":"2023-03-16T11:46:13.843270Z","shell.execute_reply.started":"2023-03-16T11:46:00.951684Z","shell.execute_reply":"2023-03-16T11:46:13.842283Z"},"trusted":true},"execution_count":39,"outputs":[{"execution_count":39,"output_type":"execute_result","data":{"text/plain":"['', '[UNK]', '[START]', '[END]', 'kee', 'liye', 'hai', 'koo', 'kyaa', 'mee']"},"metadata":{}}]},{"cell_type":"code","source":"output_text_processor = tf.keras.layers.TextVectorization(\n standardize=tf_lower_and_split_punct,\n max_tokens=max_vocab_size)\n\noutput_text_processor.adapt(targ)\noutput_text_processor.get_vocabulary()[:10]\n# English TextVectorization layer","metadata":{"id":"jlC4xuZnKLBS","outputId":"9b307cdd-52db-48bf-f9bc-2510dd84bd19","execution":{"iopub.status.busy":"2023-03-16T11:46:13.844725Z","iopub.execute_input":"2023-03-16T11:46:13.845119Z","iopub.status.idle":"2023-03-16T11:46:35.612054Z","shell.execute_reply.started":"2023-03-16T11:46:13.845081Z","shell.execute_reply":"2023-03-16T11:46:35.610936Z"},"trusted":true},"execution_count":40,"outputs":[{"execution_count":40,"output_type":"execute_result","data":{"text/plain":"['', '[UNK]', '[START]', '[END]', 'the', 'to', '.', 'for', 'is', '?']"},"metadata":{}}]},{"cell_type":"code","source":"# the above layers convert a batch of strings into a batch of token IDs\nexample_tokens = input_text_processor(example_input_batch)\nexample_tokens[:3, :10]","metadata":{"id":"9KZxj8IrNZ9S","outputId":"ed7ca51e-cd13-4c35-dd4e-fd706b1fb34a","execution":{"iopub.status.busy":"2023-03-16T11:46:35.613717Z","iopub.execute_input":"2023-03-16T11:46:35.614109Z","iopub.status.idle":"2023-03-16T11:46:35.701463Z","shell.execute_reply.started":"2023-03-16T11:46:35.614070Z","shell.execute_reply":"2023-03-16T11:46:35.700450Z"},"trusted":true},"execution_count":41,"outputs":[{"execution_count":41,"output_type":"execute_result","data":{"text/plain":""},"metadata":{}}]},{"cell_type":"code","source":"# get_vocabulary - converts token IDs back to text\ninput_vocab = np.array(input_text_processor.get_vocabulary())\ntokens = input_vocab[example_tokens[0].numpy()]\n' '.join(tokens)","metadata":{"id":"98g9rcxGQY0I","outputId":"43515fd5-6811-41ec-d9e9-8e1c4d920d81","execution":{"iopub.status.busy":"2023-03-16T11:46:35.703028Z","iopub.execute_input":"2023-03-16T11:46:35.703389Z","iopub.status.idle":"2023-03-16T11:46:35.776070Z","shell.execute_reply.started":"2023-03-16T11:46:35.703352Z","shell.execute_reply":"2023-03-16T11:46:35.775051Z"},"trusted":true},"execution_count":42,"outputs":[{"execution_count":42,"output_type":"execute_result","data":{"text/plain":"'[START] taylor swift koo play karwayee [END] '"},"metadata":{}}]},{"cell_type":"code","source":"# token IDs converted into a mask\nplt.subplot(1, 2, 1)\nplt.pcolormesh(example_tokens)\nplt.title('Token IDs')\n\nplt.subplot(1, 2, 2)\nplt.pcolormesh(example_tokens != 0)\nplt.title('Mask')","metadata":{"id":"_jx4Or_eFRSz","outputId":"fd68e833-9404-407b-a54b-333bb35ffd2b","execution":{"iopub.status.busy":"2023-03-16T11:46:35.778833Z","iopub.execute_input":"2023-03-16T11:46:35.779871Z","iopub.status.idle":"2023-03-16T11:46:36.103825Z","shell.execute_reply.started":"2023-03-16T11:46:35.779830Z","shell.execute_reply":"2023-03-16T11:46:36.101972Z"},"trusted":true},"execution_count":43,"outputs":[{"execution_count":43,"output_type":"execute_result","data":{"text/plain":"Text(0.5, 1.0, 'Mask')"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"
","image/png":"iVBORw0KGgoAAAANSUhEUgAAAh8AAAGxCAYAAADCo9TSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAvyklEQVR4nO3de3hU9Z3H8c/kwpA7csskCBhtvACKVTCC0qQqsWhZW+iKYhXUKhZ0m7JKC/iUYN0gWFncjcXV9YKtij7demltleyi0cpa4wVLsRWVcGkxRNOQBAgJSX77h81sxoDmB3N+c3u/nmeex5z5zpnvYfDLd34533N8xhgjAAAAR5IinQAAAEgsNB8AAMApmg8AAOAUzQcAAHCK5gMAADhF8wEAAJyi+QAAAE7RfAAAAKdoPgAAgFM0HzHM5/P16fHSSy/1aV833nij90n3QUlJicaMGROy7bjjjgseT1JSknJycnTKKafoqquu0rp16yKUKYCHH374c2uNMUZf+tKX5PP5VFJSEvb337Ztm3w+n37yk5+Efd/wTkqkE8CR+9///d+Qn3/84x/rxRdf1Pr160O2jxo1ymVanjnnnHOCBWbv3r167733tHbtWl144YWaPn26Hn/8caWmpkY4SyAxZWVl6YEHHujVYFRXV+vDDz9UVlZWZBJDVKL5iGFnn312yM9DhgxRUlJSr+3xYsCAASHHdsEFF2jevHkqLy/X0qVLdeutt2r58uURzBBIXDNmzNCjjz6qe+65R9nZ2cHtDzzwgCZMmKDm5uYIZodow69d4tzf/vY3zZ07V8OGDVO/fv10/PHHa/HixWpra/vc1xljtGjRIqWmpur+++8Pbn/iiSc0YcIEZWRkKDMzUxdeeKHefvvtkNfOnj1bmZmZ+uCDD3TRRRcpMzNTw4cP1z//8z9/4fseifLyco0ePVqVlZU6cOBAcPvq1as1duxYZWZmKisrSyeffLIWLVoU9vcHIF1++eWSpMcffzy4rampSf/1X/+la665plf80qVLVVRUpIEDByo7O1tnnHGGHnjgAX32Xqfr169XSUmJBg0apLS0NI0YMULTp0/X/v37D5vLwYMHNWvWLGVmZurXv/51mI4Q4UTzEccOHDigr371q3rkkUc0f/58Pffcc/r2t7+tFStWaNq0aYd9XVtbm2bOnKnKykr96le/0nXXXSdJqqio0OWXX65Ro0bpySef1M9+9jO1tLRo0qRJevfdd0P2cfDgQf3DP/yDzj//fD3zzDO65ppr9K//+q+erUxMnTpV+/fv1xtvvCFJWrt2rebOnavi4mI99dRTevrpp/X9739f+/bt8+T9gUSXnZ2tb33rW3rwwQeD2x5//HElJSVpxowZveK3bdumOXPm6Mknn9Qvf/lLTZs2TTfddJN+/OMfh8RcfPHF6tevnx588EE9//zzuuOOO5SRkaH29vZD5rFnzx5deOGFWrdunaqrq/X1r389/AeLo2cQN2bNmmUyMjKCP997771GknnyySdD4pYvX24kmXXr1gW3STLz5s0zDQ0N5txzzzXDhg0zGzduDD6/Y8cOk5KSYm666aaQfbW0tJhAIGAuvfTSkDwO9b4XXXSROemkk77wOIqLi83o0aNDto0cOdJcfPHFh33N6tWrjSTzxBNPGGOMufHGG82AAQO+8L0AHJ2HHnrISDI1NTXmxRdfNJLMH//4R2OMMePHjzezZ882xhgzevRoU1xcfMh9dHZ2moMHD5rbbrvNDBo0yHR1dRljjPnFL35hJIXUos+qra01ksydd95pamtrzahRo8yoUaPMtm3bwnugCCtWPuLY+vXrlZGRoW9961sh22fPni1J+p//+Z+Q7bW1tcHfzb722msaO3Zs8LkXXnhBHR0duuqqq9TR0RF89O/fX8XFxb3Ocvf5fJo6dWrIttNOO03bt28P3wH2YD6zVHvWWWdpz549uvzyy/XMM8/ok08+8eR9Afy/4uJinXDCCXrwwQe1adMm1dTUHPJXLtKn9emCCy5QTk6OkpOTlZqaqh/96EdqaGhQfX29JOn0009Xv379dP3112vNmjXaunXrYd/7rbfe0tlnn63c3Fy9+uqrGjlypCfHiPCg+YhjDQ0NCgQC8vl8IduHDh2qlJQUNTQ0hGx//fXXtWXLFs2YMUPHHntsyHO7d++WJI0fP16pqakhjyeeeKLXP+7p6enq379/yDa/3x9yTkY4dTc1+fn5kqQrr7xSDz74oLZv367p06dr6NChKioqUlVVlSfvD+DTLx1XX321fv7zn+vee+/ViSeeqEmTJvWKe/3111VaWipJuv/++/Xqq6+qpqZGixcvliS1trZKkk444QT993//t4YOHap58+bphBNO0AknnKC777671z6rqqq0e/dufec739GAAQO8O0iEBdMucWzQoEH6/e9/L2NMSANSX1+vjo4ODR48OCR+xowZCgQCWrx4sbq6unTrrbcGn+uO/cUvfhF13yiMMfrVr36ljIwMjRs3Lrj96quv1tVXX619+/bp5Zdf1pIlS/T1r39dW7ZsibpjAOLF7Nmz9aMf/Uj33nuv/uVf/uWQMWvXrlVqaqp+/etfh3xJefrpp3vFTpo0SZMmTVJnZ6feeOMN/fu//7vKysqUm5uryy67LBh3yy236MMPPwyuzl511VVhPzaED81HHDv//PP15JNP6umnn9Y3v/nN4PZHHnkk+Pxn3XrrrcrKygqenLls2TJJ0oUXXqiUlBR9+OGHmj59upsD6KOlS5fq3Xff1aJFi3qttkhSRkaGpkyZovb2dn3jG9/Q5s2baT4AjwwbNky33HKL/vznP2vWrFmHjPH5fEpJSVFycnJwW2trq372s58ddr/JyckqKirSySefrEcffVRvvfVWSPORlJSk//iP/1BmZqZmz56tffv26bvf/W74DgxhRfMRx6666irdc889mjVrlrZt26ZTTz1Vv/vd71RRUaGLLrpIF1xwwSFf973vfU+ZmZm6/vrrtXfvXv3bv/2bjjvuON12221avHixtm7dqq997Ws65phjtHv3br3++uvKyMjQ0qVLPT2ePXv26LXXXpMk7du3L3iRsVdeeUWXXnppyPtfd911SktL0znnnKO8vDzV1dVp2bJlysnJ0fjx4z3NE0h0d9xxx+c+f/HFF2vlypWaOXOmrr/+ejU0NOgnP/mJ/H5/SNy9996r9evX6+KLL9aIESN04MCB4DTN4erXXXfdpaysLM2dO1d79+7VLbfcEp6DQljRfMSx/v3768UXX9TixYt155136uOPP9awYcN08803a8mSJZ/72muvvVYZGRm68sortW/fPv3nf/6nFi5cqFGjRunuu+/W448/rra2NgUCAY0fP1433HCD58fz6quvasKECfL5fMrIyNCwYcN01lln6dZbbw3+/rjbpEmT9PDDD+vJJ59UY2OjBg8erHPPPVePPPKIhgwZ4nmuAA7vvPPO04MPPqjly5dr6tSpGjZsmK677joNHTpU1157bTDu9NNP17p167RkyRLV1dUpMzNTY8aM0bPPPtvr//meysvLlZmZqVtuuUV79+71/IsR7PnMZ8cEAAAAPMS0CwAAcIrmAwAAOEXzAQAAnKL5AAAATtF8AAAAp2g+AACAU1F3nY+uri7t2rVLWVlZve5JAsANY4xaWlqUn5+vpKTY+I5C7QAiy6ZuRF3zsWvXLg0fPjzSaQCQtHPnzl43GYxW1A4gOvSlbkRd85GVlSVJOlcXKUWpfXpNcmaG1Xt07t1nFe9L6Vse3Uxnp1W8TJddvC+Kvona5o6Y0KGD+p1+E/z/MRZ057r9reOUnRlF/48g6n3zxFMjnUJcsKkbUdd8dC+XpihVKb4+Nh++fpbv0W4Zb9l8WDcHMdx82OaO2PD36x7H0q8vunPNzkxSdlbyF0QD/6+v/9bgC1jUjWj6VwwAACQAmg8AAOBU1P3a5Uh07t3r6f5Nx0FP92/N8jyLJH//Psd2tR2wzQYAos6F+WMjnQI+BysfAADAKZoPAADgFM0HAABwiuYDAAA4RfMBAACciotpF1vJmZlW8V5P09iyveIqEywAog3TKImNlQ8AAOAUzQcAAHCK5gMAADhF8wEAAJyi+QAAAE7FxbSL7fRHtE2v2Iq6e80AiDtMo8BLrHwAAACnrJuPv/71r/r2t7+tQYMGKT09XaeffrrefPPN4PPGGJWXlys/P19paWkqKSnR5s2bw5o0gNhD7QDQzar5aGxs1DnnnKPU1FT99re/1bvvvqu77rpLAwYMCMasWLFCK1euVGVlpWpqahQIBDR58mS1tLSEO3cAMYLaAaAnq3M+li9fruHDh+uhhx4KbjvuuOOC/22M0apVq7R48WJNmzZNkrRmzRrl5ubqscce05w5c8KTNYCYQu0A0JPVysezzz6rcePG6R//8R81dOhQffnLX9b9998ffL62tlZ1dXUqLS0NbvP7/SouLtaGDRsOuc+2tjY1NzeHPADEF2oHgJ6sVj62bt2q1atXa/78+Vq0aJFef/11/dM//ZP8fr+uuuoq1dXVSZJyc3NDXpebm6vt27cfcp/Lli3T0qVLjzD9T9lOf/iSk+3239lpt3/L6Rvb/ct0WYWn5A7tc2zH7nq7fZ9caBXf8ef3reKT0tOt4rv277eKTxk0yCq+o6HBKh6fitbagcN7Ydc7kU4BMaa5pVPHnNi3WKuVj66uLp1xxhmqqKjQl7/8Zc2ZM0fXXXedVq9eHRLn8/lCfjbG9NrWbeHChWpqago+du7caZMSgBhA7QDQk1XzkZeXp1GjRoVsO+WUU7Rjxw5JUiAQkKTgt5hu9fX1vb7RdPP7/crOzg55AIgv1A4APVk1H+ecc47ee++9kG1btmzRyJEjJUkFBQUKBAKqqqoKPt/e3q7q6mpNnDgxDOkCiEXUDgA9WZ3z8f3vf18TJ05URUWFLr30Ur3++uu67777dN9990n6dMm0rKxMFRUVKiwsVGFhoSoqKpSenq6ZM2d6cgAAoh+1A0BPVs3H+PHj9dRTT2nhwoW67bbbVFBQoFWrVumKK64IxixYsECtra2aO3euGhsbVVRUpHXr1ikrKyvsyQOIDdQOAD35jDEm0kn01NzcrJycHJXoEqX47KZG+iplxLFW8V0f2004dLW2WsXHMq8nhxAZHeagXtIzampqiplzKbprR+OW45WdZff3EomN+9iEh03d4N4uAADAKZoPAADgFM0HAABwiuYDAAA4ZTXtEq1sL8HdseMvHmXiiM+yZ7S4HHtSWprVrm1PrrX9rEy73aXzk4dYXi79o7ovDgIQ17y+lDwntPbGygcAAHCK5gMAADhF8wEAAJyi+QAAAE7RfAAAAKfiYtqla/9+q/iUk75kFX9gxAC7/Ve9YRXvNZtLoJv2dg8zsf+sbDG9AsAW0yjusfIBAACcovkAAABO0XwAAACnaD4AAIBTNB8AAMCpuJh2sdXx3gdW8SlboqtHs5lekSTT0ff7oyRnZtqmY6Vz715P9w8Atry+t4utRJi+ia5/VQEAQNyj+QAAAE7RfAAAAKdoPgAAgFM0HwAAwKmEnHaRz7LnMl3e5NHNMp+kjHSr+M6mpr7HMo0CIAolwgRIImHlAwAAOEXzAQAAnKL5AAAATtF8AAAApxLzhFOP2V7+PHnoELs3SLX82CxOOI150XYyMYCwiLZLoEeTWDwZl5UPAADgFM0HAABwiuYDAAA4RfMBAACcovkAAABOJea0i+WEQ1JamlV814E2q/iOj+qs4q0nOhIJ0ytARMTixAUih3/FAACAUzQfAADAKZoPAADgFM0HAABwiuYDAAA4FRfTLil5Aat42+mSAf+dbhX/t3NareJt2d47xnRYTIBYTtL4knx2uXQZq3hrTLsAEcG9Vw6PSaDeWPkAAABO0XwAAACnaD4AAIBTNB8AAMApmg8AAOBUXEy7WN8bxdLfzmnwdP/JA4+xiu9qavYoE1lPi5hOj/Loxn1sAEQZpleOHpUdAAA4ZdV8lJeXy+fzhTwCgf+/xoYxRuXl5crPz1daWppKSkq0efPmsCcNILZQOwD0ZL3yMXr0aH300UfBx6ZNm4LPrVixQitXrlRlZaVqamoUCAQ0efJktbS0hDVpALGH2gGgm3XzkZKSokAgEHwMGTJE0qffXFatWqXFixdr2rRpGjNmjNasWaP9+/frscceC3viAGILtQNAN+vm4/3331d+fr4KCgp02WWXaevWrZKk2tpa1dXVqbS0NBjr9/tVXFysDRs2HHZ/bW1tam5uDnkAiD/UDgDdrKZdioqK9Mgjj+jEE0/U7t27dfvtt2vixInavHmz6uo+nTjJzc0NeU1ubq62b99+2H0uW7ZMS5cuPYLU40fn3xojnUKQLyXV7gW291IZN9pu96//0W7/ttMx3AvGCWoH4onX97FJhGkaq0o9ZcoUTZ8+XaeeeqouuOACPffcc5KkNWvWBGN8vtAbjRljem3raeHChWpqago+du7caZMSgBhA7QDQ01GN2mZkZOjUU0/V+++/HzxzvftbTLf6+vpe32h68vv9ys7ODnkAiG/UDiCxHVXz0dbWpj/96U/Ky8tTQUGBAoGAqqqqgs+3t7erurpaEydOPOpEAcQPageQ2KzO+bj55ps1depUjRgxQvX19br99tvV3NysWbNmyefzqaysTBUVFSosLFRhYaEqKiqUnp6umTNnepU/gBhA7QDQk1Xz8Ze//EWXX365PvnkEw0ZMkRnn322XnvtNY0cOVKStGDBArW2tmru3LlqbGxUUVGR1q1bp6ysLE+SP1Ipw/Kt4rs+/sQuvr3dKj6amI6D3r7B7//g7f4RleKldiSSRDjpEZHjM8aYSCfRU3Nzs3JyclSiS5Tis5y86COaD+DzdZiDeknPqKmpKWbOpeiuHY1bjld2VnKk04l5NB+wZVM3uLcLAABwiuYDAAA4RfMBAACcovkAAABOWU27RKvkMSdZxXdsft8q3pd0+KssJjpfst2Jfaaz06NMAIST15cQ9xonzEY3Vj4AAIBTNB8AAMApmg8AAOAUzQcAAHCK5gMAADgVF9MunX98z9P9GwY0DovpFQAuML0SX1j5AAAATtF8AAAAp2g+AACAUzQfAADAKZoPAADgVFxMu9hKzsqyiu9sabF7A59lT2e67OIt+VJS+xxrOg56mAkAAKx8AAAAx2g+AACAUzQfAADAKZoPAADgVEKecGp9Aqkt2xNIPT5BlZNIAQDRhJUPAADgFM0HAABwiuYDAAA4RfMBAACcovkAAABOxcW0S8rQIVbxZsgxVvGdm7dYxfuSk63iTWenVTwA2Lowf2ykUwCCWPkAAABO0XwAAACnaD4AAIBTNB8AAMApmg8AAOBUXEy7dNR/bPeCjxu8SeTvmF4BEG1e2PVOpFOIWkwCucfKBwAAcIrmAwAAOEXzAQAAnKL5AAAATtF8AAAAp+Ji2sWa6fJ0957f28Vn2TN6fLwAwMQIbLDyAQAAnKL5AAAATtF8AAAAp2g+AACAUzQfAADAqcScdvGY5/d2YXoFABDDjmrlY9myZfL5fCorKwtuM8aovLxc+fn5SktLU0lJiTZv3ny0eQKIE9QNAEfcfNTU1Oi+++7TaaedFrJ9xYoVWrlypSorK1VTU6NAIKDJkyerpaXlqJMFENuoGwCkI2w+9u7dqyuuuEL333+/jjnmmOB2Y4xWrVqlxYsXa9q0aRozZozWrFmj/fv367HHHgtb0gBiD3UDQLcjaj7mzZuniy++WBdccEHI9traWtXV1am0tDS4ze/3q7i4WBs2bDjkvtra2tTc3BzyABB/wlk3JGoHEMusTzhdu3at3nrrLdXU1PR6rq6uTpKUm5sbsj03N1fbt28/5P6WLVumpUuX2qaR0Gwv354cyP3ioL/r2FVnlwwnv6IPwl03JGpHtHlh1zuRTiEEl3uPblYrHzt37tT3vvc9/fznP1f//v0PG+fz+UJ+Nsb02tZt4cKFampqCj527txpkxKAKOdF3ZCoHUAss1r5ePPNN1VfX68zzzwzuK2zs1Mvv/yyKisr9d5770n69JtMXl5eMKa+vr7Xt5pufr9ffr//SHIHEAO8qBsStQOIZVYrH+eff742bdqkjRs3Bh/jxo3TFVdcoY0bN+r4449XIBBQVVVV8DXt7e2qrq7WxIkTw548gOhH3QDwWVYrH1lZWRozZkzItoyMDA0aNCi4vaysTBUVFSosLFRhYaEqKiqUnp6umTNnhi9rADGDugHgs8J+hdMFCxaotbVVc+fOVWNjo4qKirRu3TplZWWF+60AxAnqBpBYfMYYE+kkempublZOTo5KdIlSfKmevEeS//AnvR1KV9sBT/IAolWHOaiX9IyampqUnZ0d6XT6pLt2NG45XtlZdhNhcI9plPhjUze4sRwAAHCK5gMAADhF8wEAAJyi+QAAAE7RfAAAAKfCPmobCSlDBlvFd3z8iUeZuOFLsZsCSh44oM+xHfUf2+VieZ8ZW8m5Q63iO3fXW8Wbzk6rePn63q8npdr975UUsDvWjh1/sYoHokm03QsGh+fFZBIrHwAAwCmaDwAA4BTNBwAAcIrmAwAAOEXzAQAAnIqLaZdEm15Jys60iu9s+JtVvA3raRFLHbs+8nT/1kxXn0O72tutdt3F9AoQl7iPTW+sfAAAAKdoPgAAgFM0HwAAwCmaDwAA4FRcnHDqNdtLiNuehGk6DlrFdzY2WcUDQLThJMzExsoHAABwiuYDAAA4RfMBAACcovkAAABO0XwAAACnmHbpA9vLn9tOu6QMGmQV39HQYLf/Y4f1fd9/+avVvuWz7F8tLk8OIH69sOudSKcQxOSNe6x8AAAAp2g+AACAUzQfAADAKZoPAADgFM0HAABwKi6mXby+90pX2wGreFu20yu20zc2Eyxe/1kCgAtMsEQ3Vj4AAIBTNB8AAMApmg8AAOAUzQcAAHCK5gMAADgVF9MuthMXyZmZVvFdra1W8V5PgJiOg97tm+kVIC4x/YFowsoHAABwiuYDAAA4RfMBAACcovkAAABOxcUJp7ZsT6q0jU/q188qvqu93SoeAGy9sOsdq3hOUIWXWPkAAABO0XwAAACnaD4AAIBTNB8AAMApmg8AAOBUQk672F4uPcnf327/bQes4uWz7AFNl93uk5P7vmsurw5A9tMxXmP6Jr6w8gEAAJyyaj5Wr16t0047TdnZ2crOztaECRP029/+Nvi8MUbl5eXKz89XWlqaSkpKtHnz5rAnDSC2UDsA9GTVfBx77LG644479MYbb+iNN97Qeeedp0suuSRYJFasWKGVK1eqsrJSNTU1CgQCmjx5slpaWjxJHkBsoHYA6MlnjDFHs4OBAwfqzjvv1DXXXKP8/HyVlZXpBz/4gSSpra1Nubm5Wr58uebMmdOn/TU3NysnJ0clukQpvtSjSS1sOOcDiabDHNRLekZNTU3Kzs725D28qh2NW45Xdlbf/84jNnDOR/SzqRtHfM5HZ2en1q5dq3379mnChAmqra1VXV2dSktLgzF+v1/FxcXasGHDYffT1tam5ubmkAeA+EXtAGA97bJp0yZNmDBBBw4cUGZmpp566imNGjUqWCRyc3ND4nNzc7V9+/bD7m/ZsmVaunSpbRohUk76klV8x3sfWMXb3nsl2u7tYrWa4fEqDBJXNNYOAJFhvfJx0kknaePGjXrttdf03e9+V7NmzdK7774bfN7n84XEG2N6betp4cKFampqCj527txpmxKAGEDtANDNeuWjX79++tKXPl1pGDdunGpqanT33XcHf1dbV1envLy8YHx9fX2vbzQ9+f1++f1+2zQAxBhqB4BuR32dD2OM2traVFBQoEAgoKqqquBz7e3tqq6u1sSJE4/2bQDEGWoHkLisVj4WLVqkKVOmaPjw4WppadHatWv10ksv6fnnn5fP51NZWZkqKipUWFiowsJCVVRUKD09XTNnzvQqfwAxgNoBoCer5mP37t268sor9dFHHyknJ0ennXaann/+eU2ePFmStGDBArW2tmru3LlqbGxUUVGR1q1bp6ysLE+SBxAbqB0Aejrq63yEWzRe5yNlWL5VfMdfd3mUyZGxus5Hl+VfB6Zd4pKL63yEG9f5QE9cF8Q9J9f5AAAAOBI0HwAAwCmaDwAA4BTNBwAAcIrmAwAAOGV9hdNoZDPNIdnfubWzbrdVfMqgQVbx+9bajRP6lwywik9qbu1zbOcf37Pbt+0dfz2+j03KcLvJpK6PG+ziW/v+Z5lsOSbqGzjAKl7tB63COz6qs9s/EMNe2PVOpFNIOM0tnTrmxL7FsvIBAACcovkAAABO0XwAAACnaD4AAIBTcXHCqe0JpPLZ9VzWJ6ju2WMV759sd9KjLZ/F5eFtT5LsbGmxTcdTHTv+EukUgqz/bKLszxJAZMTqpeE7zEFJW/sUy8oHAABwiuYDAAA4RfMBAACcovkAAABO0XwAAACn4mLaxZrp8nb3ltMxSWlpVvE2l/iWpI6/7rKKBwBbsTqhgchg5QMAADhF8wEAAJyi+QAAAE7RfAAAAKdoPgAAgFMJOe2S5O9vFd/VdsAqPnn0iVbx5v1tVvG+lFS7/XcctIoHAFsv7Hon0ik4w2TP0WPlAwAAOEXzAQAAnKL5AAAATtF8AAAAp2g+AACAUwk57WI7vZLUr59VfOfmLVbxvuRku3jLfJh2AeA1JkBgg5UPAADgFM0HAABwiuYDAAA4RfMBAACcSsgTTm11tbd7un/T2WkX39rqUSYAYgUneCKWsfIBAACcovkAAABO0XwAAACnaD4AAIBTNB8AAMCpxJx28Vn2XKbLmzz+zpeSahVvfbl0m+P1+Fij7c8eiFUv7Hon0inEDSaH3GPlAwAAOEXzAQAAnKL5AAAATtF8AAAAp2g+AACAU4k57RJlExTW0yvWbxBFxxtNuQA4LCZA4CVWPgAAgFNWzceyZcs0fvx4ZWVlaejQofrGN76h9957LyTGGKPy8nLl5+crLS1NJSUl2rx5c1iTBhBbqB0AerJqPqqrqzVv3jy99tprqqqqUkdHh0pLS7Vv375gzIoVK7Ry5UpVVlaqpqZGgUBAkydPVktLS9iTBxAbqB0AevIZY8yRvvjjjz/W0KFDVV1dra985Ssyxig/P19lZWX6wQ9+IElqa2tTbm6uli9frjlz5nzhPpubm5WTk6MSXaIUn92VPwGER4c5qJf0jJqampSdnR32/XtZOxq3HK/srOSw55xoOOcDtmzqxlGd89HU1CRJGjhwoCSptrZWdXV1Ki0tDcb4/X4VFxdrw4YNh9xHW1ubmpubQx4A4hu1A0hsR9x8GGM0f/58nXvuuRozZowkqa6uTpKUm5sbEpubmxt87rOWLVumnJyc4GP48OHWuST162f18Fq05QNEk2iqHQAi44ibjxtvvFF/+MMf9Pjjj/d6zufzhfxsjOm1rdvChQvV1NQUfOzcufNIUwIQA6gdAI7oOh833XSTnn32Wb388ss69thjg9sDgYCkT7/F5OXlBbfX19f3+kbTze/3y+/3H0kaAGIMtQOAZLnyYYzRjTfeqF/+8pdav369CgoKQp4vKChQIBBQVVVVcFt7e7uqq6s1ceLE8GQMIOZQOwD0ZLXyMW/ePD322GN65plnlJWVFfxdbE5OjtLS0uTz+VRWVqaKigoVFhaqsLBQFRUVSk9P18yZMz05AADRj9oBoCer5mP16tWSpJKSkpDtDz30kGbPni1JWrBggVpbWzV37lw1NjaqqKhI69atU1ZWVlgSBhB7qB0Aejqq63x4get8AJHn9XU+vMB1PiKL64LA2XU+AAAAbNF8AAAAp2g+AACAUzQfAADAqSO6yFi0Sc7MtIrv3LvXKt72Euhd7e1W8fJZ9oCmy273yX0/+c50dtrlAgCSXtj1jqf754TW+MLKBwAAcIrmAwAAOEXzAQAAnKL5AAAATtF8AAAAp+Ji2sV2esWW7fSKzXSJJJkub69wzwQLAFtMl8BLrHwAAACnaD4AAIBTNB8AAMApmg8AAOAUzQcAAHAqLqZdvL43ii3r6RLb/G3Z7N/yzyb5tFOs4jv/8CereFvWk0bRNAkUZX+Pkdi8vlcLYocXk0+sfAAAAKdoPgAAgFM0HwAAwCmaDwAA4BTNBwAAcCo+pl2ijC8l1SredBz0KJNPJaX2/WO2nf7wenrFVlRNr9hiegU4LO41E19Y+QAAAE7RfAAAAKdoPgAAgFM0HwAAwKn4OOHU9kQ9jy9j7fUJpLa62tsjnQIAHBUvL/fOyazusfIBAACcovkAAABO0XwAAACnaD4AAIBTNB8AAMCp+Jh2scVlrAEkGCY6EE1Y+QAAAE7RfAAAAKdoPgAAgFM0HwAAwCmaDwAA4FRCTruYSV+2ik/Z+IFVvC8n2yre7N9vFd/5t0ar+A/+7ew+x478dYfVvtc//IBVPGfcA5Hh5b1RohG1Jrqx8gEAAJyi+QAAAE7RfAAAAKdoPgAAgFM0HwAAwKmEnHbx/c7urO9O23vBtLTYxdvy2fWMX/qn1zxKhDPKARwZakdiY+UDAAA4Zd18vPzyy5o6dary8/Pl8/n09NNPhzxvjFF5ebny8/OVlpamkpISbd68OVz5AohB1A0APVk3H/v27dPYsWNVWVl5yOdXrFihlStXqrKyUjU1NQoEApo8ebJavP5VBICoRd0A0JP1OR9TpkzRlClTDvmcMUarVq3S4sWLNW3aNEnSmjVrlJubq8cee0xz5sw5umwBxCTqBoCewnrOR21trerq6lRaWhrc5vf7VVxcrA0bNhzyNW1tbWpubg55AEgcR1I3JGoHEMvCOu1SV1cnScrNzQ3Znpubq+3btx/yNcuWLdPSpUvDmcYXs5xeSfL3t4rvajsQVfu3kTxggFV85549nuSBxHEkdUOKUO1A2Njea4bpmPjiybSLz+cL+dkY02tbt4ULF6qpqSn42LlzpxcpAYhyNnVDonYAsSysKx+BQEDSp99k8vLygtvr6+t7favp5vf75ff7w5kGgBhyJHVDonYAsSysKx8FBQUKBAKqqqoKbmtvb1d1dbUmTpwYzrcCECeoG0DisV752Lt3rz744IPgz7W1tdq4caMGDhyoESNGqKysTBUVFSosLFRhYaEqKiqUnp6umTNnhjVxALGDugGgJ+vm44033tBXv/rV4M/z58+XJM2aNUsPP/ywFixYoNbWVs2dO1eNjY0qKirSunXrlJWVFb6sP8vycuO2TMdBT/fv5QmktjiBFF6IyrqBiOIE0sTmM8aYSCfRU3Nzs3JyclSiS5TiS+3bizxuPnxJhz/p7VBMZ6dHmQBudJiDeknPqKmpSdnZ2ZFOp0+6a0fjluOVnZUc6XTwBWg+4o9N3eDeLgAAwCmaDwAA4BTNBwAAcIrmAwAAOBXWi4zFDMvLqxvb80dtT4C1zMfz/duIsmNNGTrYKr5jd71VvA1fSh9PmP476xOVvfxcAY/ZXl4d0a+5pVPHnNi3WFY+AACAUzQfAADAKZoPAADgFM0HAABwiuYDAAA4FR/TLl5PUAweaBXf8fEnVvFJ6el28WlpVvEdDQ1W8Va8nriw3L+X0yu2vL4nEBBNuFw6OsxBSVv7FMvKBwAAcIrmAwAAOEXzAQAAnKL5AAAATtF8AAAAp+Jj2sXj+4vYTq/Y6tq/39N4APBatN2rhemb6MbKBwAAcIrmAwAAOEXzAQAAnKL5AAAATsXFCae+JJ/lK5Ktok1np+X+AQDhxAmk8YWVDwAA4BTNBwAAcIrmAwAAOEXzAQAAnKL5AAAATsXFtEusT6Mk9etnFd/V3u5RJvaSBwywiu9qabGKN13GKt720vkAYkO0Xb7dBpM6vbHyAQAAnKL5AAAATtF8AAAAp2g+AACAUzQfAADAqbiYdok2Sf7+VvFdbQc8ysR7nXv2RDoFAFGAiQ7YYOUDAAA4RfMBAACcovkAAABO0XwAAACnaD4AAIBTcTHt4ktJtYo3HQc9yuRTsTy9AgCA11j5AAAATtF8AAAAp2g+AACAUzQfAADAKZoPAADgVHxMu6TaHYZtfFdrq1U8AMQ67tUCL3m28vHTn/5UBQUF6t+/v84880y98sorXr0VgDhB3QASgyfNxxNPPKGysjItXrxYb7/9tiZNmqQpU6Zox44dXrwdgDhA3QAShyfNx8qVK3XttdfqO9/5jk455RStWrVKw4cP1+rVq714OwBxgLoBJI6wn/PR3t6uN998Uz/84Q9DtpeWlmrDhg294tva2tTW1hb8uampSZLUoYOS6dt7JpnkI0+4D7qMt1dEBaJNhz79O29MH/8nPEq2dUM6fO1o3tvlXaIJpIO6B0s2dSPszccnn3yizs5O5ebmhmzPzc1VXV1dr/hly5Zp6dKlvbb/Tr/p+5tyPijgiZaWFuXk5Hj+PrZ1Qzp87Rh5xjYvUkxAWyOdAGJUX+qGZ9MuPp8v5GdjTK9tkrRw4ULNnz8/+POePXs0cuRI7dixw0nRi7Tm5mYNHz5cO3fuVHZ2dqTT8RTHGjuMMWppaVF+fr7T9+1r3ZASu3bE+t8vW4l0vLF8rDZ1I+zNx+DBg5WcnNzr20p9fX2vbzWS5Pf75ff7e23PycmJuT/4o5GdnZ0wx8uxxgaX/4Db1g2J2iHF9t+vI5FIxxurx9rXuhH2E0779eunM888U1VVVSHbq6qqNHHixHC/HYA4QN0AEosnv3aZP3++rrzySo0bN04TJkzQfffdpx07duiGG27w4u0AxAHqBpA4PGk+ZsyYoYaGBt1222366KOPNGbMGP3mN7/RyJEjv/C1fr9fS5YsOeRyajxKpOPlWPF5jqZuSIn1Z55Ixyol1vEmyrH6jKtZOgAAAHFjOQAA4BjNBwAAcIrmAwAAOEXzAQAAnKL5AAAATkVd8/HTn/5UBQUF6t+/v84880y98sorkU4p7MrLy+Xz+UIegUAg0mmFzcsvv6ypU6cqPz9fPp9PTz/9dMjzxhiVl5crPz9faWlpKikp0ebNmyOT7FH6omOdPXt2r8/67LPPjkyycSwR6oYU37UjkeqGRO2IqubjiSeeUFlZmRYvXqy3335bkyZN0pQpU7Rjx45IpxZ2o0eP1kcffRR8bNq0KdIphc2+ffs0duxYVVZWHvL5FStWaOXKlaqsrFRNTY0CgYAmT56slpYWx5kevS86Vkn62te+FvJZ/+Y3FjdNxBdKpLohxW/tSKS6IVE7ZKLIWWedZW644YaQbSeffLL54Q9/GKGMvLFkyRIzduzYSKfhhCTz1FNPBX/u6uoygUDA3HHHHcFtBw4cMDk5Oebee++NQIbh89ljNcaYWbNmmUsuuSQi+SSKRKkbxiRO7UikumFMYtaOqFn5aG9v15tvvqnS0tKQ7aWlpdqwYUOEsvLO+++/r/z8fBUUFOiyyy7T1q2Jcfvq2tpa1dXVhXzOfr9fxcXFcfk5S9JLL72koUOH6sQTT9R1112n+vr6SKcUNxKtbkiJWTsSsW5I8V07oqb5+OSTT9TZ2dnrDpa5ubm97nQZ64qKivTII4/ohRde0P3336+6ujpNnDhRDQ0NkU7Nc92fZSJ8zpI0ZcoUPfroo1q/fr3uuusu1dTU6LzzzlNbW1ukU4sLiVQ3pMStHYlWN6T4rx2e3NvlaPh8vpCfjTG9tsW6KVOmBP/71FNP1YQJE3TCCSdozZo1mj9/fgQzcycRPmfp0/uVdBszZozGjRunkSNH6rnnntO0adMimFl8SZS/T4leOxLlc5biv3ZEzcrH4MGDlZyc3KuLra+v79XtxpuMjAydeuqpev/99yOdiue6z8xPxM9ZkvLy8jRy5MiE+KxdSOS6ISVO7Uj0uiHFX+2ImuajX79+OvPMM1VVVRWyvaqqShMnToxQVm60tbXpT3/6k/Ly8iKdiucKCgoUCARCPuf29nZVV1fH/ecsSQ0NDdq5c2dCfNYuJHLdkBKndiR63ZDir3ZE1a9d5s+fryuvvFLjxo3ThAkTdN9992nHjh264YYbIp1aWN18882aOnWqRowYofr6et1+++1qbm7WrFmzIp1aWOzdu1cffPBB8Ofa2lpt3LhRAwcO1IgRI1RWVqaKigoVFhaqsLBQFRUVSk9P18yZMyOY9ZH5vGMdOHCgysvLNX36dOXl5Wnbtm1atGiRBg8erG9+85sRzDq+JErdkOK7diRS3ZCoHVE1amuMMffcc48ZOXKk6devnznjjDNMdXV1pFMKuxkzZpi8vDyTmppq8vPzzbRp08zmzZsjnVbYvPjii0ZSr8esWbOMMZ+OzS1ZssQEAgHj9/vNV77yFbNp06bIJn2EPu9Y9+/fb0pLS82QIUNMamqqGTFihJk1a5bZsWNHpNOOO4lQN4yJ79qRSHXDGGqHzxhj3LY7AAAgkUXNOR8AACAx0HwAAACnaD4AAIBTNB8AAMApmg8AAOAUzQcAAHCK5gMAADhF8wEAAJyi+QAAAE7RfAAAAKdoPgAAgFP/B2IplhRAsSS+AAAAAElFTkSuQmCC\n"},"metadata":{}}]},{"cell_type":"code","source":"# defining constants for the model\nembedding_dim = 256\nunits = 1024","metadata":{"id":"_a9uNz3-IrF-","execution":{"iopub.status.busy":"2023-03-16T11:46:36.105561Z","iopub.execute_input":"2023-03-16T11:46:36.105951Z","iopub.status.idle":"2023-03-16T11:46:36.111169Z","shell.execute_reply.started":"2023-03-16T11:46:36.105911Z","shell.execute_reply":"2023-03-16T11:46:36.109957Z"},"trusted":true},"execution_count":44,"outputs":[]},{"cell_type":"markdown","source":"Encoding","metadata":{"id":"6tQ8xnOH7mFc"}},{"cell_type":"code","source":"class Encoder(tf.keras.layers.Layer):\n def __init__(self, input_vocab_size, embedding_dim, enc_units):\n super(Encoder, self).__init__()\n self.enc_units = enc_units\n self.input_vocab_size = input_vocab_size\n\n # The embedding layer converts tokens to vectors\n self.embedding = tf.keras.layers.Embedding(self.input_vocab_size,\n embedding_dim)\n\n # The GRU RNN layer processes those vectors sequentially.\n self.gru = tf.keras.layers.GRU(self.enc_units,\n # Return the sequence and state\n return_sequences=True,\n return_state=True,\n recurrent_initializer='glorot_uniform',\n recurrent_dropout=0.1)\n\n def call(self, tokens, state=None):\n shape_checker = ShapeChecker()\n shape_checker(tokens, ('batch', 's'))\n\n # 2. The embedding layer looks up the embedding for each token.\n vectors = self.embedding(tokens)\n shape_checker(vectors, ('batch', 's', 'embed_dim'))\n\n # 3. The GRU processes the embedding sequence.\n # output shape: (batch, s, enc_units)\n # state shape: (batch, enc_units)\n output, state = self.gru(vectors, initial_state=state)\n shape_checker(output, ('batch', 's', 'enc_units'))\n shape_checker(state, ('batch', 'enc_units'))\n\n # 4. Returns the new sequence and its state.\n return output, state","metadata":{"id":"nZ2rI24i3jFg","execution":{"iopub.status.busy":"2023-03-16T11:46:36.113169Z","iopub.execute_input":"2023-03-16T11:46:36.113539Z","iopub.status.idle":"2023-03-16T11:46:36.124161Z","shell.execute_reply.started":"2023-03-16T11:46:36.113502Z","shell.execute_reply":"2023-03-16T11:46:36.122978Z"},"trusted":true},"execution_count":45,"outputs":[]},{"cell_type":"code","source":"# Convert the input text to tokens.\nexample_tokens = input_text_processor(example_input_batch)\n\n# Encode the input sequence.\nencoder = Encoder(input_text_processor.vocabulary_size(),\n embedding_dim, units)\nexample_enc_output, example_enc_state = encoder(example_tokens)\n\nprint(f'Input batch, shape (batch): {example_input_batch.shape}')\nprint(f'Input batch tokens, shape (batch, s): {example_tokens.shape}')\nprint(f'Encoder output, shape (batch, s, units): {example_enc_output.shape}')\nprint(f'Encoder state, shape (batch, units): {example_enc_state.shape}')","metadata":{"id":"60gSVh05Jl6l","outputId":"99c5ac95-5e34-4591-e00a-16b2d068052d","execution":{"iopub.status.busy":"2023-03-16T11:46:36.125926Z","iopub.execute_input":"2023-03-16T11:46:36.126373Z","iopub.status.idle":"2023-03-16T11:46:38.342644Z","shell.execute_reply.started":"2023-03-16T11:46:36.126335Z","shell.execute_reply":"2023-03-16T11:46:38.341549Z"},"trusted":true},"execution_count":46,"outputs":[{"name":"stdout","text":"Input batch, shape (batch): (64,)\nInput batch tokens, shape (batch, s): (64, 19)\nEncoder output, shape (batch, s, units): (64, 19, 1024)\nEncoder state, shape (batch, units): (64, 1024)\n","output_type":"stream"}]},{"cell_type":"markdown","source":"Attention layer","metadata":{"id":"mI7Zgyrp89GK"}},{"cell_type":"code","source":"class BahdanauAttention(tf.keras.layers.Layer):\n def __init__(self, units):\n super().__init__()\n # For Eqn. (4), the Bahdanau attention\n self.W1 = tf.keras.layers.Dense(units, use_bias=False)\n self.W2 = tf.keras.layers.Dense(units, use_bias=False)\n\n self.attention = tf.keras.layers.AdditiveAttention()\n\n def call(self, query, value, mask):\n shape_checker = ShapeChecker()\n shape_checker(query, ('batch', 't', 'query_units'))\n shape_checker(value, ('batch', 's', 'value_units'))\n shape_checker(mask, ('batch', 's'))\n\n # From Eqn. (4), `W1@ht`.\n w1_query = self.W1(query)\n shape_checker(w1_query, ('batch', 't', 'attn_units'))\n\n # From Eqn. (4), `W2@hs`.\n w2_key = self.W2(value)\n shape_checker(w2_key, ('batch', 's', 'attn_units'))\n\n query_mask = tf.ones(tf.shape(query)[:-1], dtype=bool)\n value_mask = mask\n\n context_vector, attention_weights = self.attention(\n inputs = [w1_query, value, w2_key],\n mask=[query_mask, value_mask],\n return_attention_scores = True,\n )\n shape_checker(context_vector, ('batch', 't', 'value_units'))\n shape_checker(attention_weights, ('batch', 't', 's'))\n\n return context_vector, attention_weights","metadata":{"id":"momiE59lXo6U","execution":{"iopub.status.busy":"2023-03-16T11:46:38.344306Z","iopub.execute_input":"2023-03-16T11:46:38.344676Z","iopub.status.idle":"2023-03-16T11:46:38.354659Z","shell.execute_reply.started":"2023-03-16T11:46:38.344638Z","shell.execute_reply":"2023-03-16T11:46:38.353267Z"},"trusted":true},"execution_count":47,"outputs":[]},{"cell_type":"code","source":"# creating BahdanauAttention layer\nattention_layer = BahdanauAttention(units)","metadata":{"id":"t4QMlOp8Gidh","execution":{"iopub.status.busy":"2023-03-16T11:46:38.356518Z","iopub.execute_input":"2023-03-16T11:46:38.357309Z","iopub.status.idle":"2023-03-16T11:46:38.369756Z","shell.execute_reply.started":"2023-03-16T11:46:38.357220Z","shell.execute_reply":"2023-03-16T11:46:38.368696Z"},"trusted":true},"execution_count":48,"outputs":[]},{"cell_type":"code","source":"(example_tokens != 0).shape","metadata":{"id":"DYSHqmORgVFo","outputId":"8feaa6f2-f454-4fa6-bf69-29762aacebee","execution":{"iopub.status.busy":"2023-03-16T11:46:38.371579Z","iopub.execute_input":"2023-03-16T11:46:38.372037Z","iopub.status.idle":"2023-03-16T11:46:38.381523Z","shell.execute_reply.started":"2023-03-16T11:46:38.371917Z","shell.execute_reply":"2023-03-16T11:46:38.380268Z"},"trusted":true},"execution_count":49,"outputs":[{"execution_count":49,"output_type":"execute_result","data":{"text/plain":"TensorShape([64, 19])"},"metadata":{}}]},{"cell_type":"code","source":"# Later, the decoder will generate this attention query\nexample_attention_query = tf.random.normal(shape=[len(example_tokens), 2, 10])\n\n# Attend to the encoded tokens\n\ncontext_vector, attention_weights = attention_layer(\n query=example_attention_query,\n value=example_enc_output,\n mask=(example_tokens != 0))\n\nprint(f'Attention result shape: (batch_size, query_seq_length, units): {context_vector.shape}')\nprint(f'Attention weights shape: (batch_size, query_seq_length, value_seq_length): {attention_weights.shape}')","metadata":{"id":"7y7hjPkNMmHh","outputId":"600899f2-e9d6-467a-ba4a-04ab287d4f08","execution":{"iopub.status.busy":"2023-03-16T11:46:38.383542Z","iopub.execute_input":"2023-03-16T11:46:38.383972Z","iopub.status.idle":"2023-03-16T11:46:38.461258Z","shell.execute_reply.started":"2023-03-16T11:46:38.383936Z","shell.execute_reply":"2023-03-16T11:46:38.460196Z"},"trusted":true},"execution_count":50,"outputs":[{"name":"stdout","text":"Attention result shape: (batch_size, query_seq_length, units): (64, 2, 1024)\nAttention weights shape: (batch_size, query_seq_length, value_seq_length): (64, 2, 19)\n","output_type":"stream"}]},{"cell_type":"code","source":"plt.subplot(1, 2, 1)\nplt.pcolormesh(attention_weights[:, 0, :])\nplt.title('Attention weights')\n\nplt.subplot(1, 2, 2)\nplt.pcolormesh(example_tokens != 0)\nplt.title('Mask')\n","metadata":{"id":"Rqr8XGsAJlf6","outputId":"a8f59ec0-d2a6-49a2-de95-aa80d7f7a7b7","execution":{"iopub.status.busy":"2023-03-16T11:46:38.462954Z","iopub.execute_input":"2023-03-16T11:46:38.463333Z","iopub.status.idle":"2023-03-16T11:46:38.765069Z","shell.execute_reply.started":"2023-03-16T11:46:38.463296Z","shell.execute_reply":"2023-03-16T11:46:38.763953Z"},"trusted":true},"execution_count":51,"outputs":[{"execution_count":51,"output_type":"execute_result","data":{"text/plain":"Text(0.5, 1.0, 'Mask')"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"
","image/png":"iVBORw0KGgoAAAANSUhEUgAAAh8AAAGxCAYAAADCo9TSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA1a0lEQVR4nO3de3hU1b3/8c8kJJOEXASUTCIQIg0qBW+ASChNqCQWKb9W1NbSQ0OvKtia0laLtBosBovKD4+0WCwqVlFP6/VYL8QL0V8DNWi1NPaxqBFiIUZpTMItIWH9/uBkTmKCZsHsNTOZ9+t55nnInpW9v3uGrHxnZX3X8hljjAAAAByJC3cAAAAgtpB8AAAAp0g+AACAUyQfAADAKZIPAADgFMkHAABwiuQDAAA4RfIBAACcIvkAAABOkXyEwH/+53/K5/Np7NixvT7/xhtvqKysTO+++26P59avX6+VK1d6G2Af4pg3b55GjhzpJA6X3n33Xfl8Pt19991H9f0+n09XXHHFp7arqqpSWVmZPvroo6O6DhCt7r77bvl8Pvl8Pm3cuLHH88YYfeYzn5HP51NhYWHIr9/5M37zzTeH/NzwDslHCNx5552SpJqaGv3lL3/p8fwbb7yhJUuWRETycaQ4fvGLX+iRRx5xEodLWVlZ2rRpk2bOnOnpdaqqqrRkyRKSD8SstLQ0rV27tsfxyspKvf3220pLSwtDVIhUJB/HaMuWLXr99deDv9x6++GLBqNGjdKZZ54Z7jBCzu/365xzztEJJ5wQ7lCAfu1rX/uaHnroITU3N3c7vnbtWk2ePFkjRowIU2SIRCQfx6gz2bjxxhuVn5+vBx54QPv27Qs+f/fdd+viiy+WJE2bNi04PHn33XersLBQf/rTn7R9+/bgcZ/PF/zetrY2LV26VKeccor8fr9OOOEEfetb39IHH3zQLYaRI0fqS1/6kp5++mmdddZZSk5O1imnnBIckfm0OKTe/+xy4MABLVq0SLm5uUpMTNSJJ56oBQsW9Ph035frH8nEiRN7jEqMGzdOPp9P1dXVwWMPP/ywfD6ftm7dGjy2bds2zZkzR0OHDpXf79epp56qX//6193OdaQ/uzz22GM67bTT5Pf7ddJJJ+nWW29VWVlZt9e/q9///vc69dRTlZKSotNPP11PPPFE8LmysjL99Kc/lSTl5ub2GIJ+/vnnVVhYqCFDhig5OVkjRozQhRde2O3/CRDtvv71r0uS7r///uCxpqYmPfTQQ/r2t7/do/2SJUs0adIkDR48WOnp6TrrrLO0du1afXyv06P5+Tl48KBKSkqUmpra7WcVEcTgqO3bt89kZGSYiRMnGmOM+d3vfmckmbvvvjvYpqGhwZSXlxtJ5te//rXZtGmT2bRpk2loaDA1NTVmypQpJhAIBI9v2rTJGGNMR0eH+eIXv2gGDhxolixZYioqKszvfvc7c+KJJ5oxY8aYffv2Ba+Rk5Njhg0bZsaMGWPuuece88wzz5iLL77YSDKVlZWfGocxxpSUlJicnJzgOQ8dOmTOO+88M2DAAPOLX/zCbNiwwdx8881m4MCB5swzzzQHDhywuv6R/OxnPzOpqammra3NGGNMfX29kWSSk5PNDTfcEGx3+eWXm8zMzODXNTU1JiMjw4wbN87cc889ZsOGDebHP/6xiYuLM2VlZcF2tbW1RpK56667gseeeuopExcXZwoLC80jjzxi/vCHP5hJkyaZkSNHmo//SEgyI0eONGeffbb5r//6L/Pkk0+awsJCM2DAAPP2228bY4ypq6szP/jBD4wk8/DDDwdf26amJlNbW2uSkpJMUVGRefTRR83GjRvNfffdZ+bOnWsaGxs/8bUBosFdd91lJJnq6mozd+5cc/bZZwefW716tRk4cKBpbm42n/3sZ01BQUHwuXnz5pm1a9eaiooKU1FRYX75y1+a5ORks2TJkmCbvvz8dP6M33TTTcYYYxobG820adNMIBAwW7ZscfIawB7JxzG45557jCRz++23G2OMaWlpMampqWbq1Knd2v3hD38wkswLL7zQ4xwzZ87s9ku/0/33328kmYceeqjb8erqaiPJ/OY3vwkey8nJMUlJSWb79u3BY/v37zeDBw82l156aZ/i+Hjy8fTTTxtJZvny5d3aPfjgg0aSWbNmjfX1e/Pss88aSebFF180xhhz7733mrS0NDN//nwzbdq0YLu8vDwzZ86c4NfnnXeeGTZsmGlqaup2viuuuMIkJSWZf//738aY3pOPiRMnmuHDh5vW1tbgsZaWFjNkyJBek4/MzEzT3NwcPFZfX2/i4uLMsmXLgsduuukmI8nU1tZ2+/4//vGPRpJ57bXXPvF1AKJV1+TjhRdeMJLM3//+d2PM4Z+1efPmGWNMj+Sjq46ODnPw4EFz/fXXmyFDhphDhw4ZY/r289M1+aitrTVjxowxY8aMMe+++25obxQhxZ9djsHatWuVnJysSy65RJKUmpqqiy++WC+99JK2bdt2TOd+4okndNxxx2nWrFlqb28PPs444wwFAoEes8rPOOOMbn9TTUpK0ujRo7V9+/ajuv7zzz8v6fCfY7q6+OKLNXDgQD333HMhuf6UKVOUlJSkZ599VpJUUVGhwsJCffGLX1RVVZX27dunuro6bdu2TdOnT5d0+M9Bzz33nC644AKlpKR0e33OP/98HThwQJs3b+71env37tWWLVv0la98RYmJicHjqampmjVrVq/fM23atG6T5TIzMzV06NA+vbZnnHGGEhMT9f3vf1/r1q3TO++886nfA0SrgoICjRo1Snfeeae2bt2q6urqXv/kIh3uY6ZPn66MjAzFx8crISFB1157rXbv3q2GhgZJdj8/r776qs455xxlZmbqz3/+s3Jycjy5R4QGycdReuutt/Tiiy9q5syZMsboo48+0kcffaSLLrpIkvo03+GTvP/++/roo4+UmJiohISEbo/6+np9+OGH3doPGTKkxzn8fr/2799/VNffvXu3BgwY0GOips/nUyAQ0O7du0Ny/aSkJE2ZMiWYfDz33HMqKipSYWGhOjo69NJLL6miokKSgsnH7t271d7erttuu63Ha3P++edLUo/Xp1NjY6OMMcrMzOzxXG/HjuXepMMTeZ999lkNHTpUCxYs0KhRozRq1Cjdeuutn/q9QLTx+Xz61re+pXvvvVe33367Ro8eralTp/Zo9/LLL6u4uFiSdMcdd+jPf/6zqqurtXjxYkkK/mzZ/PxUVFTo/fff13e/+10dd9xx3t0kQmJAuAOIVnfeeaeMMfrjH/+oP/7xjz2eX7dunZYuXar4+PijOv/xxx+vIUOG6Omnn+71ea/L1oYMGaL29nZ98MEH3RIQY4zq6+s1ceLEkF3r3HPP1bXXXquXX35Z7733noqKipSWlqaJEyeqoqJCO3fu1OjRozV8+HBJ0qBBgxQfH6+5c+dqwYIFvZ4zNze31+ODBg2Sz+fT+++/3+O5+vr6kN1TV1OnTtXUqVPV0dGhLVu26LbbblNpaakyMzODo2ZAfzFv3jxde+21uv3223XDDTf02uaBBx5QQkKCnnjiCSUlJQWPP/rooz3a9vXn56c//anefvttffOb31R7e7u++c1vhvzeEDqMfByFjo4OrVu3TqNGjdILL7zQ4/HjH/9Yu3bt0lNPPSXp8KdkSb1+Uj7SJ+gvfelL2r17tzo6OjRhwoQej5NPPtk67k+K4+POPfdcSdK9997b7fhDDz2kvXv3Bp8PhenTp6u9vV2/+MUvNGzYMJ1yyinB488++2xweLZTSkqKpk2bpr/+9a867bTTen19ehutkKSBAwdqwoQJevTRR9XW1hY8vmfPnmOaFd+X1zY+Pl6TJk0KVuS8+uqrR309IFKdeOKJ+ulPf6pZs2appKSk1zY+n08DBgzo9uFs//79+v3vf3/E837az09cXJx++9vf6sorr9S8efO0evXqENwNvMLIx1F46qmntHPnTv3qV7/qdcW+sWPHatWqVVq7dq2+9KUvBVc+XbNmjdLS0pSUlKTc3FwNGTJE48aN08MPP6zVq1dr/PjxiouL04QJE3TJJZfovvvu0/nnn68rr7xSZ599thISEvTee+/phRde0Je//GVdcMEFVnF/UhwfV1RUpPPOO09XX321mpubNWXKFP3tb3/TddddpzPPPFNz5861f+GOYPz48Ro0aJA2bNigb33rW8Hj06dP1y9/+cvgv7u69dZb9bnPfU5Tp07V5ZdfrpEjR6qlpUVvvfWW/vu//zs4Z6U3119/vWbOnKnzzjtPV155pTo6OnTTTTcpNTVV//73v4/qHsaNGxeMq6SkRAkJCTr55JN133336fnnn9fMmTM1YsQIHThwIPgnuY/fE9Bf3HjjjZ/4/MyZM7VixQrNmTNH3//+97V7927dfPPNwSS+0+23327983PLLbcoLS1N8+fP1549e4Jl8Igw4Z3vGp2+8pWvmMTExGCZam8uueQSM2DAAFNfX2+MMWblypUmNzfXxMfHd6u++Pe//20uuugic9xxxxmfz9et2uLgwYPm5ptvNqeffrpJSkoyqamp5pRTTjGXXnqp2bZtW7BdTk6OmTlzZo8YCgoKeswuP1IcH692MeZwxcrVV19tcnJyTEJCgsnKyjKXX355jxJRm+sfyQUXXGAkmfvuuy94rK2tzQwcONDExcX1WpZaW1trvv3tb5sTTzzRJCQkmBNOOMHk5+ebpUuXdmujj1W7GGPMI488YsaNG2cSExPNiBEjzI033mh++MMfmkGDBnVrJ8ksWLCgx7VzcnJMSUlJt2OLFi0y2dnZJi4uLlhVtGnTJnPBBReYnJwc4/f7zZAhQ0xBQYF5/PHH+/S6AJGua7XLJ/l4tcudd95pTj75ZOP3+81JJ51kli1bZtauXdutaqwvPz8fL7Xt1FmBdu2114bsXhE6PmM+tqILEIMOHjyoM844QyeeeKI2bNgQ7nAAoF/jzy6ISd/5zndUVFSkrKws1dfX6/bbb9c//vEPqlAAwAGSD8SklpYW/eQnP9EHH3yghIQEnXXWWXryySeZhwEADvBnFwAA4BSltgAAwCmSDwAA4BTJBwAAcCriJpweOnRIO3fuVFpamnw+X7jDAWKSMUYtLS3Kzs5WXFx0fEah7wDCy6bfiLjkY+fOncE9PACEV11dnYYNGxbuMPqEvgOIDH3pNyIu+ejcMG3UFdcq3p/0Ka3/h8cfcozl+X2HLC9geX7beLzk87hWyvZeT7zpL94EEmPadVD/T096voFhKHXGuv3VkUpPjY7RGkSGC0aPC3cI/YJNvxFxyUfncGm8P4nk4whIPo5sgC/Bm0Bizf+8r9H054vOWNNT45SednS7SSM20W+EiEW/wccDAADgFMkHAABwKuL+7NKpPUkyffyri7VIG0m2jcf2Tx027b1+bSzPn/PzKm/iANCvnZd9erhDwCdg5AMAADhF8gEAAJwi+QAAAE6RfAAAAKdIPgAAgFMRW+3SFjiouOTIWCjIZ5mimQ7bVck8XqnLgu295n2n2ptAAPRrVKPENkY+AACAUyQfAADAKZIPAADgFMkHAABwiuQDAAA4FbHVLqNP2qUBA/19anvIct/1AXF2e95bn99neX7LDU9sz3+goN6qPQBQjQIvMfIBAACcsk4+/vWvf+k//uM/NGTIEKWkpOiMM87QK6+8EnzeGKOysjJlZ2crOTlZhYWFqqmpCWnQAKIPfQeATlbJR2Njo6ZMmaKEhAQ99dRTeuONN3TLLbfouOOOC7ZZvny5VqxYoVWrVqm6ulqBQEBFRUVqaWkJdewAogR9B4CurOZ8/OpXv9Lw4cN11113BY+NHDky+G9jjFauXKnFixdr9uzZkqR169YpMzNT69ev16WXXhqaqAFEFfoOAF1ZjXw8/vjjmjBhgi6++GINHTpUZ555pu64447g87W1taqvr1dxcXHwmN/vV0FBgaqqqno9Z2trq5qbm7s9APQv9B0AurIa+XjnnXe0evVqLVy4UNdcc41efvll/fCHP5Tf79c3v/lN1dcfrqrIzMzs9n2ZmZnavn17r+dctmyZlixZ0uP4njXDNCAhySY8HMmFIz07tWUhUMSJoG11rF9L29htzt9+8ID06GN2F/gELvsOhMYzO18PdwiIMs0tHRo0um9trUY+Dh06pLPOOkvl5eU688wzdemll+p73/ueVq9e3a2dz9e9lzPG9DjWadGiRWpqago+6urqbEICEAXoOwB0ZZV8ZGVlacyYMd2OnXrqqdqxY4ckKRAISFLwU0ynhoaGHp9oOvn9fqWnp3d7AOhf6DsAdGWVfEyZMkVvvvlmt2P//Oc/lZOTI0nKzc1VIBBQRUVF8Pm2tjZVVlYqPz8/BOECiEb0HQC6sprz8aMf/Uj5+fkqLy/XV7/6Vb388stas2aN1qxZI+nwkGlpaanKy8uVl5envLw8lZeXKyUlRXPmzPHkBgBEPvoOAF1ZJR8TJ07UI488okWLFun6669Xbm6uVq5cqW984xvBNldddZX279+v+fPnq7GxUZMmTdKGDRuUlpYW8uABRAf6DgBd+YwxETTfX2publZGRoZG/7hc8f4+VrtEecWF17ysSPG6WsQ29mE39F6WCTvt5qA26jE1NTVFzVyKzr6j8Z8nKT0tPtzhIIqwj01o2PQb7O0CAACcIvkAAABOkXwAAACnSD4AAIBTVtUuLiW2SPGtfWsb7Ut8e87i9fEd8i4MSTK26a7lhNaGKyzXhLD8vzP0Nia0Av2N10vJM6G1J0Y+AACAUyQfAADAKZIPAADgFMkHAABwiuQDAAA4FbHVLm3pUrzfo5N7vaC8ZQWFbbWO7ZLmNue3rnbxuNKI5dUBeI1qFPcY+QAAAE6RfAAAAKdIPgAAgFMkHwAAwCmSDwAA4FTEVrsk7ZbiE/vY2OvqFVuxtNeM1/dq+d7u/r7l3i62LO53yG+pvAGigdd7u9iKheobRj4AAIBTJB8AAMApkg8AAOAUyQcAAHCK5AMAADgVsdUuTXlGcUl9LHXwOoWKsL1grNnEb3uvlq/9qB9tsrwAAMRGBUgsYeQDAAA4RfIBAACcIvkAAABOkXwAAACnInbC6eAai+XVLZlYWv7clu1rYzlB9aN5k+1ObxmPzzKeSPq/MOguJuMCRxJpS6BHkmicjMvIBwAAcIrkAwAAOEXyAQAAnCL5AAAATpF8AAAApyK22qWpeK/iUzo8Obcvztv10j0uGPF0NfbhF2318OwA+qtorLhA+DDyAQAAnCL5AAAATpF8AAAAp0g+AACAUyQfAADAqYitdkn5c6riE5P61Nbr/Tmieb8QSVblMQ1X5HsXh+RtqY4DQ2+rCncIQERi75UjoxKoJ0Y+AACAUyQfAADAKZIPAADgFMkHAABwiuQDAAA4FbHVLh2Jkvx9bBxJm6McjQiK37ZSx7YSyGu28WfdQvUKADtUrxw7Rj4AAIBTVslHWVmZfD5ft0cgEAg+b4xRWVmZsrOzlZycrMLCQtXU1IQ8aADRhb4DQFfWIx+f/exntWvXruBj69b/3YJ9+fLlWrFihVatWqXq6moFAgEVFRWppaUlpEEDiD70HQA6WScfAwYMUCAQCD5OOOEESYc/uaxcuVKLFy/W7NmzNXbsWK1bt0779u3T+vXrQx44gOhC3wGgk3XysW3bNmVnZys3N1eXXHKJ3nnnHUlSbW2t6uvrVVxcHGzr9/tVUFCgqqojT+prbW1Vc3NztweA/oe+A0Anq2qXSZMm6Z577tHo0aP1/vvva+nSpcrPz1dNTY3q6+slSZmZmd2+JzMzU9u3bz/iOZctW6YlS5b0OJ6wV4o/2MfAvK4WsW3vdTxeVph4Xb0SYZVGu79vuZeNRfxDfkslTSeXfQfgNa/3sYmFahqrkY8ZM2bowgsv1Lhx4zR9+nT96U9/kiStW7cu2Mbn6947G2N6HOtq0aJFampqCj7q6upsQgIQBeg7AHR1TKW2AwcO1Lhx47Rt27bgzPXOTzGdGhoaenyi6crv9ys9Pb3bA0D/Rt8BxLZjSj5aW1v1j3/8Q1lZWcrNzVUgEFBFRUXw+ba2NlVWVio/3+Nt2gFEFfoOILZZzfn4yU9+olmzZmnEiBFqaGjQ0qVL1dzcrJKSEvl8PpWWlqq8vFx5eXnKy8tTeXm5UlJSNGfOHK/iBxAF6DsAdGWVfLz33nv6+te/rg8//FAnnHCCzjnnHG3evFk5OTmSpKuuukr79+/X/Pnz1djYqEmTJmnDhg1KS0uzDmzwhXUaMLBv66sPiDtkff5IcshyTfABPrv7bSvcZdUeCDWXfQdCIxYmPSJ8fMaYiNqdo7m5WRkZGSp84nKSjyMg+YDX2s1BbdRjampqipq5FJ19R+M/T1J6Wny4w4l6JB+wZdNvsLcLAABwiuQDAAA4RfIBAACcIvkAAABOWVW7uLRj03DFJyX1rbFtCuX1/FSvlxC3nSL8y1xPwjgqlq9Nzs9ZohwIB6+XEPcaE2YjGyMfAADAKZIPAADgFMkHAABwiuQDAAA4RfIBAACcithql46BRibJo5Xfva6OsT2/1wvce1l947MLflTpZo8CAdCfUb3SvzDyAQAAnCL5AAAATpF8AAAAp0g+AACAUyQfAADAqYitdvGUbXVJpFWv2PI0Hq83sgEA9DeMfAAAAKdIPgAAgFMkHwAAwCmSDwAA4BQTTvsi0uZURlI8lsurAwDAyAcAAHCK5AMAADhF8gEAAJwi+QAAAE6RfAAAAKcittrlu9OfU1Jq38I7ZOxyqDjfIav2B028VfsEX4en7Z8YM8iqPQCcl316uEMAghj5AAAATpF8AAAAp0g+AACAUyQfAADAKZIPAADgVMRWu/z+wemK9yf1rbHlXifGsr1lcYz38Vxj195qL5tI2jdG9q/NsBuqvAkEiHLP7Hw93CFELCqB3GPkAwAAOEXyAQAAnCL5AAAATpF8AAAAp0g+AACAUxFb7dJ++h6ZlHZvTm5T/SFZV4DE+WwvYMnD8w+78O+enRtA/0XFCGww8gEAAJwi+QAAAE6RfAAAAKdIPgAAgFMkHwAAwKmIrXaxYln84Yuz+wZju8GIJevaFct4fF5X3wAAYOGYRj6WLVsmn8+n0tLS4DFjjMrKypSdna3k5GQVFhaqpqbmWOME0E/QbwA46uSjurpaa9as0Wmnndbt+PLly7VixQqtWrVK1dXVCgQCKioqUktLyzEHCyC60W8AkI4y+dizZ4++8Y1v6I477tCgQYOCx40xWrlypRYvXqzZs2dr7NixWrdunfbt26f169eHLGgA0Yd+A0Cno0o+FixYoJkzZ2r69OndjtfW1qq+vl7FxcXBY36/XwUFBaqqqur1XK2trWpubu72AND/hLLfkOg7gGhmPeH0gQce0Kuvvqrq6uoez9XX10uSMjMzux3PzMzU9u3bez3fsmXLtGTJkp5PvJkqJSXZhtc3tvMvLVO0Q5ant43Hev6rxfm3L823PLkd29hHLj7yLx9Ej1D3G9In9B0Ii2d2vh7uELphuffIZvVrta6uTldeeaXuvfdeJX1CYuDzdf8NY4zpcazTokWL1NTUFHzU1dXZhAQgwnnRb0j0HUA0sxr5eOWVV9TQ0KDx48cHj3V0dOjFF1/UqlWr9Oabb0o6/EkmKysr2KahoaHHp5pOfr9ffr//aGIHEAW86Dck+g4gmlmNfJx77rnaunWrXnvtteBjwoQJ+sY3vqHXXntNJ510kgKBgCoqKoLf09bWpsrKSuXnezucDyAy0W8A+DirkY+0tDSNHTu227GBAwdqyJAhweOlpaUqLy9XXl6e8vLyVF5erpSUFM2ZMyd0UQOIGvQbAD4u5CucXnXVVdq/f7/mz5+vxsZGTZo0SRs2bFBaWlqoLwWgn6DfAGKLzxgTUWtvNzc3KyMjQ8NuW6K4ZK+qXSxLLiyXY7dfL92Oz7L6xuYd/oT5fb3K+07P6gVEv3ZzUBv1mJqampSenh7ucPqks+9o/OdJSk+LD3c4+BRUo/Q/Nv0GG8sBAACnSD4AAIBTJB8AAMApkg8AAOAUyQcAAHAq5KW2oRLYGK8BCX2csW6710mksa2Osbxfm+Ien2Usey45x+4bcETWe/ZYSrt/s7cXACxE2l4wODIvKpMY+QAAAE6RfAAAAKdIPgAAgFMkHwAAwCmSDwAA4FTEVrscGBKn+ERvciPrqgKvq2kiaXcd23u1jN22msaW1+/t0NuqLC8AINaxj01PjHwAAACnSD4AAIBTJB8AAMApkg8AAOBUxE44/WjcQcUl93F5dVu2sxIjbZZknF08PpvTW95r3re3WLUHAIlJmLGOkQ8AAOAUyQcAAHCK5AMAADhF8gEAAJwi+QAAAE5FbLVLyvYExfsT+tQ20opXbFkvCe4h29fmX4vyvQnkfxjL9HjYDSx/DkSDZ3a+Hu4Qgqi8cY+RDwAA4BTJBwAAcIrkAwAAOEXyAQAAnCL5AAAATkVstUu7XzJJfWzsdbWI7fltq2m8rr6x2tvF8ty2sVueP+fnVK8AsEcFS2Rj5AMAADhF8gEAAJwi+QAAAE6RfAAAAKdIPgAAgFMRW+1S8n+eV1KqN+HF+Q5ZtT9ku8GIJX/cQav2T4wZ5FEkAPorqj8QSRj5AAAATpF8AAAAp0g+AACAUyQfAADAqYidcPrS7lEacMDfp7aHjLfrq8f57NYQ9zqepMoOq/YHCuo9igRAtHhm5+tW7ZmgCi8x8gEAAJwi+QAAAE6RfAAAAKdIPgAAgFMkHwAAwKmIrXbZ9voIxSUleXNyu+IVybZ4Jc7yAoe8rY7Rity+t7UNxbISaFTpZssLAAgH2+oYr1F9078w8gEAAJyySj5Wr16t0047Tenp6UpPT9fkyZP11FNPBZ83xqisrEzZ2dlKTk5WYWGhampqQh40gOhC3wGgK6vkY9iwYbrxxhu1ZcsWbdmyRV/4whf05S9/OdhJLF++XCtWrNCqVatUXV2tQCCgoqIitbS0eBI8gOhA3wGgK6vkY9asWTr//PM1evRojR49WjfccINSU1O1efNmGWO0cuVKLV68WLNnz9bYsWO1bt067du3T+vXr/cqfgBRgL4DQFdHPeejo6NDDzzwgPbu3avJkyertrZW9fX1Ki4uDrbx+/0qKChQVVXVEc/T2tqq5ubmbg8A/Rd9BwDr5GPr1q1KTU2V3+/XZZddpkceeURjxoxRff3h/UMyMzO7tc/MzAw+15tly5YpIyMj+Bg+fLhtSIcrNGwecZYPWx0+u4fXbF4bY/vw2T0QsyKy7wAQFta/Wk8++WS99tpr2rx5sy6//HKVlJTojTfeCD7v83X/BWOM6XGsq0WLFqmpqSn4qKursw0JQBSg7wDQyXqdj8TERH3mM5+RJE2YMEHV1dW69dZbdfXVV0uS6uvrlZWVFWzf0NDQ4xNNV36/X35/33avBRC96DsAdDrmdT6MMWptbVVubq4CgYAqKiqCz7W1tamyslL5+fnHehkA/Qx9BxC7rEY+rrnmGs2YMUPDhw9XS0uLHnjgAW3cuFFPP/20fD6fSktLVV5erry8POXl5am8vFwpKSmaM2eOV/EDiAL0HQC6sko+3n//fc2dO1e7du1SRkaGTjvtND399NMqKiqSJF111VXav3+/5s+fr8bGRk2aNEkbNmxQWlqaJ8EDiA70HQC68hljbHc68VRzc7MyMjJ00uJyxXu1t4vXRRe25/d6r5lIeoctY8/5+ZFLLeGddnNQG/WYmpqalJ6eHu5w+qSz72j850lKT4sPdzgIM/aCcc+m32BvFwAA4BTJBwAAcIrkAwAAOEXyAQAAnCL5AAAATlmvcOrK8X8/pAEJh8IdhqTI25LEF0nVKx7be9E5Vu1j6bWxZfP/uP3gAenRx7wLBvDYMztfD3cIMae5pUODRvetLSMfAADAKZIPAADgFMkHAABwiuQDAAA4FbETTg8MjlN8Yh9zI6+XJ/ea18ul27SP9qXnvWYR/5DfRu/S8O3mYLhDAGJWtC4Nf7jfeKdPbRn5AAAATpF8AAAAp0g+AACAUyQfAADAKZIPAADgVMRWu+yZtkfxKe2enNvn8RrcxnI99vg4u2Xkbc9vc7/DLvy71bkBQIreCg2EByMfAADAKZIPAADgFMkHAABwiuQDAAA4RfIBAACcithqF/9fUhXvT/Lm5JG2t4sly2IX2RT37PpxvmfndsH2tcm6JXr3XwEiyTM7Xw93CM5Q2XPsGPkAAABOkXwAAACnSD4AAIBTJB8AAMApkg8AAOBUxFa7tJ3ToviUg31qa7vXiS3bvWBso/G6YMQmnuEXbfUsDgD9FxUgsMHIBwAAcIrkAwAAOEXyAQAAnCL5AAAATkXshNONE+9RelrfcqN4n90UT78vwap9nMc5GhO1ANii30A0Y+QDAAA4RfIBAACcIvkAAABOkXwAAACnSD4AAIBTEVvtcu7vFijen+TNyW3XM/d29XbpGrvmtqvJ26wO7+W5j+b8tobdUOXtBYAI8czO18MdQr9B5ZB7jHwAAACnSD4AAIBTJB8AAMApkg8AAOAUyQcAAHAqYqtdAgXvacBAf5/aJsZ3WJ07zrrcxU67scvp4ixLRmzjbyvcZdUeAKgAgZcY+QAAAE5ZJR/Lli3TxIkTlZaWpqFDh+orX/mK3nzzzW5tjDEqKytTdna2kpOTVVhYqJqampAGDSC60HcA6Moq+aisrNSCBQu0efNmVVRUqL29XcXFxdq7d2+wzfLly7VixQqtWrVK1dXVCgQCKioqUktLS8iDBxAd6DsAdOUzxhz1BIgPPvhAQ4cOVWVlpT7/+c/LGKPs7GyVlpbq6quvliS1trYqMzNTv/rVr3TppZd+6jmbm5uVkZGhz//3fOZ8HKk9cz7gsXZzUBv1mJqampSenh7y83vZdzT+8ySlp8WHPOZYw5wP2LLpN45pzkdTU5MkafDgwZKk2tpa1dfXq7i4ONjG7/eroKBAVVW9L3vd2tqq5ubmbg8A/Rt9BxDbjjr5MMZo4cKF+tznPqexY8dKkurr6yVJmZmZ3dpmZmYGn/u4ZcuWKSMjI/gYPnz44cB8ps8PW4fk8/Zh7B5ALPG67wAQ+Y46+bjiiiv0t7/9Tffff3+P53y+7r9QjTE9jnVatGiRmpqago+6urqjDQlAFKDvAHBU63z84Ac/0OOPP64XX3xRw4YNCx4PBAKSDn+KycrKCh5vaGjo8Ymmk9/vl9/ft7kdAKIbfQcAyXLkwxijK664Qg8//LCef/555ebmdns+NzdXgUBAFRUVwWNtbW2qrKxUfn5+aCIGEHXoOwB0ZTXysWDBAq1fv16PPfaY0tLSgn+LzcjIUHJysnw+n0pLS1VeXq68vDzl5eWpvLxcKSkpmjNnjic3ACDy0XcA6Moq+Vi9erUkqbCwsNvxu+66S/PmzZMkXXXVVdq/f7/mz5+vxsZGTZo0SRs2bFBaWlpIAgYQfeg7AHR1TOt8eKGzVv+ke65RfEqSJ9ewrZCxrUjxHUUFjtX5bdtbxDPswr9bnh39kdfrfHiBdT7Ci3VB4GydDwAAAFskHwAAwCmSDwAA4BTJBwAAcOqoFhlzoeNfA2WSvJlw6jnbGaFeT/m1iOftFZPtzh1nF/yo0s125wcQFZ7Z+bqn52dCa//CyAcAAHCK5AMAADhF8gEAAJwi+QAAAE6RfAAAAKcittpl1Gl1GjDQm+2yvV5e3fb8tuIsy2PaCnd5FAmA/orqEniJkQ8AAOAUyQcAAHCK5AMAADhF8gEAAJwi+QAAAE5FbLVL853DNCChj3u72O6lEmEsi2lkXUxzSY7lN3jH+l4PeRPH/17A4/N7yPa1TLuffXXQd17v1YLo4UXlEyMfAADAKZIPAADgFMkHAABwiuQDAAA4RfIBAACcithqlw8m+BSX1Mfp/LYVC7YVFLbnj/N2bxfrMgcvK0Ys09dRP9rkTRwA+jX2mulfGPkAAABOkXwAAACnSD4AAIBTJB8AAMCpiJ1w6v/Qp3i/RxNOPWZ8dgHZLpfu+XLsHnpvcb6n5x92Q5Wn5wcQHl4u985kVvcY+QAAAE6RfAAAAKdIPgAAgFMkHwAAwCmSDwAA4FTEVrtc8tXnlZSa0Ke28bIr54jzebneuPfxPDFmkFV7AKCiA5GEkQ8AAOAUyQcAAHCK5AMAADhF8gEAAJwi+QAAAE5FbLXLvY9/QfFJSX1rbLu3SwTtdSLJOn7fErv2NnvBmHjLWDx+LXN+zl4tQCh4uTdKJKK6J7Ix8gEAAJwi+QAAAE6RfAAAAKdIPgAAgFMkHwAAwKmIrXZpyzqouOQ+ll5EefWKNQ/v12eZjuZ9p9qbQAD0a1SjxDZGPgAAgFPWyceLL76oWbNmKTs7Wz6fT48++mi3540xKisrU3Z2tpKTk1VYWKiamppQxQsgCtFvAOjKOvnYu3evTj/9dK1atarX55cvX64VK1Zo1apVqq6uViAQUFFRkVpaWo45WADRiX4DQFfWcz5mzJihGTNm9PqcMUYrV67U4sWLNXv2bEnSunXrlJmZqfXr1+vSSy89tmgBRCX6DQBdhXTOR21trerr61VcXBw85vf7VVBQoKqq3pfJbm1tVXNzc7cHgNhxNP2GRN8BRLOQVrvU19dLkjIzM7sdz8zM1Pbt23v9nmXLlmnJkp6blcS3DFDcwT6G5/XeLpFWveLl/VrG8vb/nWzVftSPNtldAP3e0fQb0pH7DkQH271mqI7pXzypdvH5uv92NMb0ONZp0aJFampqCj7q6uq8CAlAhLPpNyT6DiCahXTkIxAISDr8SSYrKyt4vKGhocenmk5+v19+vz+UYQCIIkfTb0j0HUA0C+nIR25urgKBgCoqKoLH2traVFlZqfz8/FBeCkA/Qb8BxB7rkY89e/borbfeCn5dW1ur1157TYMHD9aIESNUWlqq8vJy5eXlKS8vT+Xl5UpJSdGcOXNCGjiA6EG/AaAr6+Rjy5YtmjZtWvDrhQsXSpJKSkp0991366qrrtL+/fs1f/58NTY2atKkSdqwYYPS0tKsrtMxsEMmuaNvjX3erq9uu+S4sQ0nkiacWp579HdZXh2fzlW/gejBBNLY5jPG+lelp5qbm5WRkaFh/3m94pKT+vZNJB+hOz/JByS1m4PaqMfU1NSk9PT0cIfTJ519R+M/T1J6Wh/3hULYkHz0Pzb9Bnu7AAAAp0g+AACAUyQfAADAKZIPAADgVEgXGQul7OfiNCAhOnMjYzlp0+P5sp7ae9E54Q6hG9vX0vq9OmRxgU9YnTMcbO61/eAB6dHHvAsGMc92eXVEvuaWDg0a3be20fnbHQAARC2SDwAA4BTJBwAAcIrkAwAAOEXyAQAAnIrYapcPL9yn+JRDnpzb53F5SZzl+Q/ZllxYsoln2IV/9zASRIt2czDcISDKsFw6Dvcb7/SpLSMfAADAKZIPAADgFMkHAABwiuQDAAA4RfIBAACcithqF72ZKiUl9a2tZbGIdXGJZXFMh20xjdfxWJx/+y/z7U5uy/Jec35e5U0cAEIq0vZqofomsjHyAQAAnCL5AAAATpF8AAAAp0g+AACAUxE74bRjoJFJ6uPMSm9XJ/d8QmhEsb1Xy6XkR5VutrwAADCBtL9h5AMAADhF8gEAAJwi+QAAAE6RfAAAAKdIPgAAgFMRW+2Sut2n+MS+lV5YFlzYL69uyfN4PKy+sY49zi6Yhis8Xr7dluVrOfQ2lnsHwiHSlm+3QaVOT4x8AAAAp0g+AACAUyQfAADAKZIPAADgFMkHAABwKmKrXfZP2aP4lHZPzh1nW9JhyeutXYxleYzN/Q6/aKttOABARQesMPIBAACcIvkAAABOkXwAAACnSD4AAIBTJB8AAMCpiK12sWJbXmK5n4fX1SvWW7tYVuvYtgcAwEuMfAAAAKdIPgAAgFMkHwAAwCmSDwAA4BTJBwAAcCpiq11GDG7UgIH+PrUdEHfI6tyHLPdGsWW7d0ycZT2N7fkPFNRbtQcA9mqBlzwb+fjNb36j3NxcJSUlafz48XrppZe8uhSAfoJ+A4gNniQfDz74oEpLS7V48WL99a9/1dSpUzVjxgzt2LHDi8sB6AfoN4DY4UnysWLFCn3nO9/Rd7/7XZ166qlauXKlhg8frtWrV3txOQD9AP0GEDtCPuejra1Nr7zyin72s591O15cXKyqqqoe7VtbW9Xa2hr8uqmpSZLUvq+t7xeN8jkfttHYnr/dHLS8AmJduw7/nzHGzeq4tv2GdOS+o3mPXX+A3tFvwJZNvxHy5OPDDz9UR0eHMjMzux3PzMxUfX3PiY/Lli3TkiVLehz/f19dG+rQAFhqaWlRRkaG59ex7TekI/cdOWe960WIMeidcAeAKNWXfsOzahefr/vneWNMj2OStGjRIi1cuDD49UcffaScnBzt2LHDSacXbs3NzRo+fLjq6uqUnp4e7nA8xb1GD2OMWlpalJ2d7fS6fe03pNjuO6L9/5etWLrfaL5Xm34j5MnH8ccfr/j4+B6fVhoaGnp8qpEkv98vv79nSW1GRkbUvfDHIj09PWbul3uNDi5/gdv2GxJ9hxTd/7+ORizdb7Tea1/7jZBPOE1MTNT48eNVUVHR7XhFRYXy8/NDfTkA/QD9BhBbPPmzy8KFCzV37lxNmDBBkydP1po1a7Rjxw5ddtllXlwOQD9AvwHEDk+Sj6997WvavXu3rr/+eu3atUtjx47Vk08+qZycnE/9Xr/fr+uuu67X4dT+KJbul3vFJzmWfkOKrdc8lu5Viq37jZV79RlXtXQAAABiYzkAAOAYyQcAAHCK5AMAADhF8gEAAJwi+QAAAE5FXPLxm9/8Rrm5uUpKStL48eP10ksvhTukkCsrK5PP5+v2CAQC4Q4rZF588UXNmjVL2dnZ8vl8evTRR7s9b4xRWVmZsrOzlZycrMLCQtXU1IQn2GP0afc6b968Hu/1OeecE55g+7FY6Dek/t13xFK/IdF3RFTy8eCDD6q0tFSLFy/WX//6V02dOlUzZszQjh07wh1ayH32s5/Vrl27go+tW7eGO6SQ2bt3r04//XStWrWq1+eXL1+uFStWaNWqVaqurlYgEFBRUZFaWlocR3rsPu1eJemLX/xit/f6ySefdBhh/xdL/YbUf/uOWOo3JPoOmQhy9tlnm8suu6zbsVNOOcX87Gc/C1NE3rjuuuvM6aefHu4wnJBkHnnkkeDXhw4dMoFAwNx4443BYwcOHDAZGRnm9ttvD0OEofPxezXGmJKSEvPlL385LPHEiljpN4yJnb4jlvoNY2Kz74iYkY+2tja98sorKi4u7na8uLhYVVVVYYrKO9u2bVN2drZyc3N1ySWX6J13YmP76traWtXX13d7n/1+vwoKCvrl+yxJGzdu1NChQzV69Gh973vfU0NDQ7hD6jdird+QYrPviMV+Q+rffUfEJB8ffvihOjo6euxgmZmZ2WOny2g3adIk3XPPPXrmmWd0xx13qL6+Xvn5+dq9e3e4Q/Nc53sZC++zJM2YMUP33Xefnn/+ed1yyy2qrq7WF77wBbW2toY7tH4hlvoNKXb7jljrN6T+33d4srfLsfD5fN2+Nsb0OBbtZsyYEfz3uHHjNHnyZI0aNUrr1q3TwoULwxiZO7HwPkuH9yvpNHbsWE2YMEE5OTn605/+pNmzZ4cxsv4lVv4/xXrfESvvs9T/+46IGfk4/vjjFR8f3yOLbWho6JHt9jcDBw7UuHHjtG3btnCH4rnOmfmx+D5LUlZWlnJycmLivXYhlvsNKXb6jljvN6T+13dETPKRmJio8ePHq6KiotvxiooK5efnhykqN1pbW/WPf/xDWVlZ4Q7Fc7m5uQoEAt3e57a2NlVWVvb791mSdu/erbq6uph4r12I5X5Dip2+I9b7Dan/9R0R9WeXhQsXau7cuZowYYImT56sNWvWaMeOHbrsssvCHVpI/eQnP9GsWbM0YsQINTQ0aOnSpWpublZJSUm4QwuJPXv26K233gp+XVtbq9dee02DBw/WiBEjVFpaqvLycuXl5SkvL0/l5eVKSUnRnDlzwhj10fmkex08eLDKysp04YUXKisrS++++66uueYaHX/88brgggvCGHX/Eiv9htS/+45Y6jck+o6IKrU1xphf//rXJicnxyQmJpqzzjrLVFZWhjukkPva175msrKyTEJCgsnOzjazZ882NTU14Q4rZF544QUjqcejpKTEGHO4bO66664zgUDA+P1+8/nPf95s3bo1vEEfpU+613379pni4mJzwgknmISEBDNixAhTUlJiduzYEe6w+51Y6DeM6d99Ryz1G8bQd/iMMcZtugMAAGJZxMz5AAAAsYHkAwAAOEXyAQAAnCL5AAAATpF8AAAAp0g+AACAUyQfAADAKZIPAADgFMkHAABwiuQDAAA4RfIBAACc+v/aa9Q0LfNgawAAAABJRU5ErkJggg==\n"},"metadata":{}}]},{"cell_type":"code","source":"attention_weights.shape","metadata":{"id":"ZuzrCdmYlTcJ","outputId":"8dfa3d00-cd0c-48e8-8718-af89ed82d106","execution":{"iopub.status.busy":"2023-03-16T11:46:38.766676Z","iopub.execute_input":"2023-03-16T11:46:38.767075Z","iopub.status.idle":"2023-03-16T11:46:38.773967Z","shell.execute_reply.started":"2023-03-16T11:46:38.767036Z","shell.execute_reply":"2023-03-16T11:46:38.772787Z"},"trusted":true},"execution_count":52,"outputs":[{"execution_count":52,"output_type":"execute_result","data":{"text/plain":"TensorShape([64, 2, 19])"},"metadata":{}}]},{"cell_type":"code","source":"attention_slice = attention_weights[0, 0].numpy()\nattention_slice = attention_slice[attention_slice != 0]","metadata":{"id":"qIMwC-f-ZC8N","execution":{"iopub.status.busy":"2023-03-16T11:46:38.775961Z","iopub.execute_input":"2023-03-16T11:46:38.776857Z","iopub.status.idle":"2023-03-16T11:46:38.786165Z","shell.execute_reply.started":"2023-03-16T11:46:38.776814Z","shell.execute_reply":"2023-03-16T11:46:38.785038Z"},"trusted":true},"execution_count":53,"outputs":[]},{"cell_type":"code","source":"#@title\nplt.suptitle('Attention weights for one sequence')\n\nplt.figure(figsize=(12, 6))\na1 = plt.subplot(1, 2, 1)\nplt.bar(range(len(attention_slice)), attention_slice)\n# freeze the xlim\nplt.xlim(plt.xlim())\nplt.xlabel('Attention weights')\n\na2 = plt.subplot(1, 2, 2)\nplt.bar(range(len(attention_slice)), attention_slice)\nplt.xlabel('Attention weights, zoomed')\n\n# zoom in\ntop = max(a1.get_ylim())\nzoom = 0.85*top\na2.set_ylim([0.90*top, top])\na1.plot(a1.get_xlim(), [zoom, zoom], color='k')","metadata":{"id":"ysWDPO6hOS8X","outputId":"9f941203-0962-4219-98cb-efdfe46a5faa","execution":{"iopub.status.busy":"2023-03-16T11:46:38.788275Z","iopub.execute_input":"2023-03-16T11:46:38.788715Z","iopub.status.idle":"2023-03-16T11:46:39.182116Z","shell.execute_reply.started":"2023-03-16T11:46:38.788670Z","shell.execute_reply":"2023-03-16T11:46:39.181038Z"},"trusted":true},"execution_count":54,"outputs":[{"execution_count":54,"output_type":"execute_result","data":{"text/plain":"[]"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"
","image/png":"iVBORw0KGgoAAAANSUhEUgAAA94AAAINCAYAAADIsKceAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABSMElEQVR4nO3df1zUZb7//+cIwpgp/oBQCwGtVYxSd2gLjH5shcfKtK1dSsV2g1oWN0WOe5KwNekHW5mL/QA/kMa6pbK3Y51qY0vqpGm4WyJTntav9kPE4w43xC1R24WE6/uHN+c0O2AM8nZgeNxvt/ftMNf7el/v10WevXx6zbzHZowxAgAAAAAAlujn7wIAAAAAAAhkBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACwU7O8CuktbW5v+9re/adCgQbLZbP4uBwDQxxljdPToUY0aNUr9+vHv3N2BtR4A0NN0dr0PmOD9t7/9TVFRUf4uAwAADwcOHNAFF1zg7zICAms9AKCn+q71PmCC96BBgySdnPDgwYP9XA0AoK9rampSVFSUe33CmWOtBwD0NJ1d7wMmeJ96y9ngwYNZjAEAPQZvie4+rPUAgJ7qu9Z7PnQGAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFgv1dAAAgcMUsfsPfJfis9jc3+bsEAAAQYNjxBgAAAADAQux4A4AfsSMMAAAQ+NjxBgAAAADAQgRvAAAAAAAsxFvN+6je+PZWibe49jW98c8pf0YBAADwrwjeQC/VG0OpRDAFAABA30PwRsDqjcGUUAoAAAAEHoJ3BwhtAAAAAIDu0KWHqxUVFSk2NlZ2u10Oh0Nbt27tsK/L5dKsWbM0btw49evXT9nZ2acde8OGDbLZbJo5c2ZXSgMAAAAAoEfxOXiXl5crOztbeXl5qqmpUXJysqZNm6a6urp2+zc3NysiIkJ5eXmaOHHiacfev3+/Fi1apOTkZF/LAgAAAACgR/I5eK9YsULp6enKyMhQXFycCgsLFRUVpeLi4nb7x8TEaOXKlZo7d67CwsI6HLe1tVWzZ8/WsmXLNGbMGF/LAgAAAACgR/IpeLe0tKi6ulopKSke7SkpKaqqqjqjQvLz8xUREaH09PRO9W9ublZTU5PHAQAAAABAT+NT8G5sbFRra6siIyM92iMjI1VfX9/lIt5//32tXr1apaWlnb6moKBAYWFh7iMqKqrL9wcAAAAAwCpderiazWbzeG2M8WrrrKNHj2rOnDkqLS1VeHh4p6/Lzc3VkSNH3MeBAwe6dH8AAAAAAKzk09eJhYeHKygoyGt3u6GhwWsXvLM+//xz1dbWavr06e62tra2k8UFB2vPnj0aO3as13WhoaEKDQ3t0j0BAAAAADhbfNrxDgkJkcPhUGVlpUd7ZWWlkpKSulTA+PHjtWvXLjmdTvdxyy236Nprr5XT6eQt5AAAAACAXs2nHW9JysnJUVpamhISEpSYmKiSkhLV1dUpMzNT0sm3gB88eFBr1651X+N0OiVJx44d06FDh+R0OhUSEqIJEybIbrcrPj7e4x5DhgyRJK92AAAAAAB6G5+Dd2pqqg4fPqz8/Hy5XC7Fx8eroqJC0dHRkiSXy+X1nd6TJ092/1xdXa1169YpOjpatbW1Z1Y9AAAAAAA9nM/BW5KysrKUlZXV7rmysjKvNmOMT+O3NwYAAAAAAL1Rl55qDgAAAAAAOofgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AQB9TVFSk2NhY2e12ORwObd26tcO+LpdLs2bN0rhx49SvXz9lZ2d79SkrK5PNZvM6/vnPf3b5vgAABBKCNwAAfUh5ebmys7OVl5enmpoaJScna9q0aaqrq2u3f3NzsyIiIpSXl6eJEyd2OO7gwYPlcrk8Drvd3uX7AgAQSAjeAAD0IStWrFB6eroyMjIUFxenwsJCRUVFqbi4uN3+MTExWrlypebOnauwsLAOx7XZbBoxYoTHcSb3BQAgkBC8AQDoI1paWlRdXa2UlBSP9pSUFFVVVZ3R2MeOHVN0dLQuuOAC3XzzzaqpqTnj+zY3N6upqcnjAACgNyJ4AwDQRzQ2Nqq1tVWRkZEe7ZGRkaqvr+/yuOPHj1dZWZlee+01rV+/Xna7XVOmTNGnn356RvctKChQWFiY+4iKiupyjQAA+BPBGwCAPsZms3m8NsZ4tfniiiuu0Jw5czRx4kQlJyfrD3/4g773ve/pmWeeOaP75ubm6siRI+7jwIEDXa4RAAB/CvZ3AQAA4OwIDw9XUFCQ1y5zQ0OD1270mejXr58uu+wy9453V+8bGhqq0NDQbqsLAAB/YccbAIA+IiQkRA6HQ5WVlR7tlZWVSkpK6rb7GGPkdDo1cuTIs3pfAAB6Kna8AQDoQ3JycpSWlqaEhAQlJiaqpKREdXV1yszMlHTy7d0HDx7U2rVr3dc4nU5JJx+gdujQITmdToWEhGjChAmSpGXLlumKK67QRRddpKamJj399NNyOp167rnnOn1fAAACGcEbAIA+JDU1VYcPH1Z+fr5cLpfi4+NVUVGh6OhoSZLL5fL6bu3Jkye7f66urta6desUHR2t2tpaSdJXX32le++9V/X19QoLC9PkyZP13nvv6Qc/+EGn7wsAQCAjeAMA0MdkZWUpKyur3XNlZWVebcaY047329/+Vr/97W/P6L4AAAQyPuMNAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIWC/V1Adzt+/LiCgoLOeJy2ln92QzVn1/HjxzvdtzfOTwr8OQb6/KTAn6Mv85MCf46BPr+zMQ4AAOj9bMYY4+8iukNTU5PCwsL8XQYAAB6OHDmiwYMH+7uMgHBqred3CgDoKTq7NvFWcwAAAAAALBRwbzX/29/+1i3/Ch734JvdUM3Ztfvhf+t03944Pynw5xjo85MCf46+zE8K/DkG+vxOp6mpSaNGjeqWsQAAQO8WcMF74MCBGjhw4BmP0y/E3g3VnF2+zLs3zk8K/DkG+vykwJ+jr//7E+hzDPT5nU5ra2u3jAMAAHo/3moOAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFioS8G7qKhIsbGxstvtcjgc2rp1a4d9XS6XZs2apXHjxqlfv37Kzs726lNaWqrk5GQNHTpUQ4cO1fXXX68PPvigK6UBAAAAANCj+By8y8vLlZ2drby8PNXU1Cg5OVnTpk1TXV1du/2bm5sVERGhvLw8TZw4sd0+mzdv1p133ql3331X27dv1+jRo5WSkqKDBw/6Wh4AAAAAAD2Kz8F7xYoVSk9PV0ZGhuLi4lRYWKioqCgVFxe32z8mJkYrV67U3LlzFRYW1m6fl156SVlZWZo0aZLGjx+v0tJStbW16Z133vG1PAAAAAAAehSfgndLS4uqq6uVkpLi0Z6SkqKqqqpuK+rrr7/WN998o2HDhnXYp7m5WU1NTR4HAAAAAAA9jU/Bu7GxUa2trYqMjPRoj4yMVH19fbcVtXjxYp1//vm6/vrrO+xTUFCgsLAw9xEVFdVt9wcAAAAAoLt06eFqNpvN47Uxxqutq5544gmtX79eL7/8sux2e4f9cnNzdeTIEfdx4MCBbrk/AAAAAADdKdiXzuHh4QoKCvLa3W5oaPDaBe+K5cuX67HHHtPbb7+tSy+99LR9Q0NDFRoaesb3BAAAAADASj7teIeEhMjhcKiystKjvbKyUklJSWdUyJNPPqmHH35Yb775phISEs5oLAAAAAAAegqfdrwlKScnR2lpaUpISFBiYqJKSkpUV1enzMxMSSffAn7w4EGtXbvWfY3T6ZQkHTt2TIcOHZLT6VRISIgmTJgg6eTbyx988EGtW7dOMTEx7h31c889V+eee+6ZzhEAAAAAAL/xOXinpqbq8OHDys/Pl8vlUnx8vCoqKhQdHS1JcrlcXt/pPXnyZPfP1dXVWrdunaKjo1VbWytJKioqUktLi26//XaP65YuXaqHHnrI1xIBAAAAAOgxfA7ekpSVlaWsrKx2z5WVlXm1GWNOO96pAA4AAAAAQKDp0lPNAQAAAABA5xC8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEA6GOKiooUGxsru90uh8OhrVu3dtjX5XJp1qxZGjdunPr166fs7OzTjr1hwwbZbDbNnDnTo/3EiRNasmSJYmNjNWDAAI0ZM0b5+flqa2vrhhkBANCzEbwBAOhDysvLlZ2drby8PNXU1Cg5OVnTpk1TXV1du/2bm5sVERGhvLw8TZw48bRj79+/X4sWLVJycrLXuccff1yrVq3Ss88+q927d+uJJ57Qk08+qWeeeaZb5gUAQE9G8AYAoA9ZsWKF0tPTlZGRobi4OBUWFioqKkrFxcXt9o+JidHKlSs1d+5chYWFdThua2urZs+erWXLlmnMmDFe57dv364ZM2bopptuUkxMjG6//XalpKRox44d3TY3AAB6KoI3AAB9REtLi6qrq5WSkuLRnpKSoqqqqjMaOz8/XxEREUpPT2/3/JVXXql33nlHe/fulSR99NFH2rZtm2688cYOx2xublZTU5PHAQBAbxTs7wIAAMDZ0djYqNbWVkVGRnq0R0ZGqr6+vsvjvv/++1q9erWcTmeHfe6//34dOXJE48ePV1BQkFpbW/Xoo4/qzjvv7PCagoICLVu2rMt1AQDQU7DjDQBAH2Oz2TxeG2O82jrr6NGjmjNnjkpLSxUeHt5hv/Lycr344otat26ddu7cqd/97ndavny5fve733V4TW5uro4cOeI+Dhw40KUaAQDwN3a8AQDoI8LDwxUUFOS1u93Q0OC1C95Zn3/+uWprazV9+nR326knlQcHB2vPnj0aO3asfvWrX2nx4sW64447JEmXXHKJ9u/fr4KCAt11113tjh0aGqrQ0NAu1QUAQE/CjjcAAH1ESEiIHA6HKisrPdorKyuVlJTUpTHHjx+vXbt2yel0uo9bbrlF1157rZxOp6KioiRJX3/9tfr18/xrR1BQEF8nBgDoE9jxBgCgD8nJyVFaWpoSEhKUmJiokpIS1dXVKTMzU9LJt3cfPHhQa9eudV9z6rPbx44d06FDh+R0OhUSEqIJEybIbrcrPj7e4x5DhgyRJI/26dOn69FHH9Xo0aN18cUXq6amRitWrNDdd99t7YQBAOgBCN4AAPQhqampOnz4sPLz8+VyuRQfH6+KigpFR0dLklwul9d3ek+ePNn9c3V1tdatW6fo6GjV1tZ2+r7PPPOMHnzwQWVlZamhoUGjRo3Sz3/+c/3617/ulnkBANCTEbwBAOhjsrKylJWV1e65srIyrzZjjE/jtzfGoEGDVFhYqMLCQp/GAgAgEPAZbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCXQreRUVFio2Nld1ul8Ph0NatWzvs63K5NGvWLI0bN079+vVTdnZ2u/02btyoCRMmKDQ0VBMmTNArr7zSldIAAAAAAOhRfA7e5eXlys7OVl5enmpqapScnKxp06aprq6u3f7Nzc2KiIhQXl6eJk6c2G6f7du3KzU1VWlpafroo4+Ulpamn/zkJ/rLX/7ia3kAAAAAAPQoPgfvFStWKD09XRkZGYqLi1NhYaGioqJUXFzcbv+YmBitXLlSc+fOVVhYWLt9CgsLdcMNNyg3N1fjx49Xbm6urrvuOhUWFvpaHgAAAAAAPYpPwbulpUXV1dVKSUnxaE9JSVFVVVWXi9i+fbvXmFOnTj3tmM3NzWpqavI4AAAAAADoaXwK3o2NjWptbVVkZKRHe2RkpOrr67tcRH19vc9jFhQUKCwszH1ERUV1+f4AAAAAAFilSw9Xs9lsHq+NMV5tVo+Zm5urI0eOuI8DBw6c0f0BAAAAALBCsC+dw8PDFRQU5LUT3dDQ4LVj7YsRI0b4PGZoaKhCQ0O7fE8AAAAAAM4Gn3a8Q0JC5HA4VFlZ6dFeWVmppKSkLheRmJjoNeamTZvOaEwAAAAAAHoCn3a8JSknJ0dpaWlKSEhQYmKiSkpKVFdXp8zMTEkn3wJ+8OBBrV271n2N0+mUJB07dkyHDh2S0+lUSEiIJkyYIElasGCBrrrqKj3++OOaMWOGXn31Vb399tvatm1bN0wRAAAAAAD/8Tl4p6am6vDhw8rPz5fL5VJ8fLwqKioUHR0tSXK5XF7f6T158mT3z9XV1Vq3bp2io6NVW1srSUpKStKGDRu0ZMkSPfjggxo7dqzKy8t1+eWXn8HUAAAAAADwP5+DtyRlZWUpKyur3XNlZWVebcaY7xzz9ttv1+23396VcgAAAAAA6LG69FRzAAAAAADQOQRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAA+piioiLFxsbKbrfL4XBo69atHfZ1uVyaNWuWxo0bp379+ik7O/u0Y2/YsEE2m00zZ870Onfw4EHNmTNHw4cP1znnnKNJkyapurr6DGcDAEDPR/AGAKAPKS8vV3Z2tvLy8lRTU6Pk5GRNmzZNdXV17fZvbm5WRESE8vLyNHHixNOOvX//fi1atEjJycle57788ktNmTJF/fv315/+9Cf99a9/1VNPPaUhQ4Z0x7QAAOjRuvQ93gAAoHdasWKF0tPTlZGRIUkqLCzUW2+9peLiYhUUFHj1j4mJ0cqVKyVJa9as6XDc1tZWzZ49W8uWLdPWrVv11VdfeZx//PHHFRUVpRdeeMFjbAAA+gJ2vAEA6CNaWlpUXV2tlJQUj/aUlBRVVVWd0dj5+fmKiIhQenp6u+dfe+01JSQk6Mc//rHOO+88TZ48WaWlpacds7m5WU1NTR4HAAC9EcEbAIA+orGxUa2trYqMjPRoj4yMVH19fZfHff/997V69erTBukvvvhCxcXFuuiii/TWW28pMzNT8+fP19q1azu8pqCgQGFhYe4jKiqqyzUCAOBPBG8AAPoYm83m8doY49XWWUePHtWcOXNUWlqq8PDwDvu1tbXp+9//vh577DFNnjxZP//5z3XPPfeouLi4w2tyc3N15MgR93HgwIEu1QgAgL/xGW8AAPqI8PBwBQUFee1uNzQ0eO2Cd9bnn3+u2tpaTZ8+3d3W1tYmSQoODtaePXs0duxYjRw5UhMmTPC4Ni4uThs3buxw7NDQUIWGhnapLgAAehJ2vAEA6CNCQkLkcDhUWVnp0V5ZWamkpKQujTl+/Hjt2rVLTqfTfdxyyy269tpr5XQ63W8PnzJlivbs2eNx7d69exUdHd21yQAA0Iuw4w0AQB+Sk5OjtLQ0JSQkKDExUSUlJaqrq1NmZqakk2/vPnjwoMdnr51OpyTp2LFjOnTokJxOp0JCQjRhwgTZ7XbFx8d73OPUV4R9u33hwoVKSkrSY489pp/85Cf64IMPVFJSopKSEmsnDABAD0DwBgCgD0lNTdXhw4eVn58vl8ul+Ph4VVRUuHeeXS6X13d6T5482f1zdXW11q1bp+joaNXW1nb6vpdddpleeeUV5ebmKj8/X7GxsSosLNTs2bO7ZV4AAPRkBG8AAPqYrKwsZWVltXuurKzMq80Y49P47Y0hSTfffLNuvvlmn8YCACAQ8BlvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEJdCt5FRUWKjY2V3W6Xw+HQ1q1bT9t/y5YtcjgcstvtGjNmjFatWuXVp7CwUOPGjdOAAQMUFRWlhQsX6p///GdXygMAAAAAoMfwOXiXl5crOztbeXl5qqmpUXJysqZNm6a6urp2++/bt0833nijkpOTVVNTowceeEDz58/Xxo0b3X1eeuklLV68WEuXLtXu3bu1evVqlZeXKzc3t+szAwAAAACgBwj29YIVK1YoPT1dGRkZkk7uVL/11lsqLi5WQUGBV/9Vq1Zp9OjRKiwslCTFxcVpx44dWr58uW677TZJ0vbt2zVlyhTNmjVLkhQTE6M777xTH3zwQVfnBQAAAABAj+DTjndLS4uqq6uVkpLi0Z6SkqKqqqp2r9m+fbtX/6lTp2rHjh365ptvJElXXnmlqqur3UH7iy++UEVFhW666aYOa2lublZTU5PHAQAAAABAT+PTjndjY6NaW1sVGRnp0R4ZGan6+vp2r6mvr2+3/4kTJ9TY2KiRI0fqjjvu0KFDh3TllVfKGKMTJ07oF7/4hRYvXtxhLQUFBVq2bJkv5QMAAAAAcNZ16eFqNpvN47Uxxqvtu/p/u33z5s169NFHVVRUpJ07d+rll1/WH//4Rz388MMdjpmbm6sjR464jwMHDnRlKgAAAAAAWMqnHe/w8HAFBQV57W43NDR47WqfMmLEiHb7BwcHa/jw4ZKkBx98UGlpae7PjV9yySU6fvy47r33XuXl5alfP+9/HwgNDVVoaKgv5QMAAAAAulHM4jf8XUKX1P6m4481W8GnHe+QkBA5HA5VVlZ6tFdWViopKandaxITE736b9q0SQkJCerfv78k6euvv/YK10FBQTLGuHfHAQAAAADojXx+q3lOTo6ef/55rVmzRrt379bChQtVV1enzMxMSSffAj537lx3/8zMTO3fv185OTnavXu31qxZo9WrV2vRokXuPtOnT1dxcbE2bNigffv2qbKyUg8++KBuueUWBQUFdcM0AQAAAADwD5+/Tiw1NVWHDx9Wfn6+XC6X4uPjVVFRoejoaEmSy+Xy+E7v2NhYVVRUaOHChXruuec0atQoPf300+6vEpOkJUuWyGazacmSJTp48KAiIiI0ffp0Pfroo90wRQAAAAAA/Mfn4C1JWVlZysrKavdcWVmZV9vVV1+tnTt3dlxEcLCWLl2qpUuXdqUcAAAAAAB6rC491RwAAAAAAHQOwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQDoY4qKihQbGyu73S6Hw6GtW7d22NflcmnWrFkaN26c+vXrp+zs7NOOvWHDBtlsNs2cObPDPgUFBbLZbN85FgAAgYLgDQBAH1JeXq7s7Gzl5eWppqZGycnJmjZtmsdXgX5bc3OzIiIilJeXp4kTJ5527P3792vRokVKTk7usM+HH36okpISXXrppWc0DwAAehOCNwAAfciKFSuUnp6ujIwMxcXFqbCwUFFRUSouLm63f0xMjFauXKm5c+cqLCysw3FbW1s1e/ZsLVu2TGPGjGm3z7FjxzR79myVlpZq6NCh3TIfAAB6A4I3AAB9REtLi6qrq5WSkuLRnpKSoqqqqjMaOz8/XxEREUpPT++wz7x583TTTTfp+uuvP6N7AQDQ2wT7uwAAAHB2NDY2qrW1VZGRkR7tkZGRqq+v7/K477//vlavXi2n09lhnw0bNmjnzp368MMPOz1uc3Ozmpub3a+bmpq6XCMAAP7EjjcAAH2MzWbzeG2M8WrrrKNHj2rOnDkqLS1VeHh4u30OHDigBQsW6MUXX5Tdbu/02AUFBQoLC3MfUVFRXaoRAAB/Y8cbAIA+Ijw8XEFBQV672w0NDV674J31+eefq7a2VtOnT3e3tbW1SZKCg4O1Z88e7dq1Sw0NDXI4HO4+ra2teu+99/Tss8+qublZQUFBXmPn5uYqJyfH/bqpqYnwDQDolQjeAAD0ESEhIXI4HKqsrNStt97qbq+srNSMGTO6NOb48eO1a9cuj7YlS5bo6NGjWrlypaKionTeeed59fnZz36m8ePH6/777283dEtSaGioQkNDu1QXAAA9CcEbAIA+JCcnR2lpaUpISFBiYqJKSkpUV1enzMxMSSd3mQ8ePKi1a9e6rzn12e1jx47p0KFDcjqdCgkJ0YQJE2S32xUfH+9xjyFDhkiSuz0kJMSrz8CBAzV8+HCvdgAAAhHBGwCAPiQ1NVWHDx9Wfn6+XC6X4uPjVVFRoejoaEmSy+Xy+k7vyZMnu3+urq7WunXrFB0drdra2rNZOgAAvRbBGwCAPiYrK0tZWVntnisrK/NqM8b4NH57Y/yrzZs3+zQmAAC9GcEbAAAAACwSs/gNf5fgs9rf3OTvEgIOXycGAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhYL9XQAAAACsEbP4DX+X4LPa39zk7xJwFvFnFH0FO94AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhboUvIuKihQbGyu73S6Hw6GtW7eetv+WLVvkcDhkt9s1ZswYrVq1yqvPV199pXnz5mnkyJGy2+2Ki4tTRUVFV8oDAAAAAKDH8Dl4l5eXKzs7W3l5eaqpqVFycrKmTZumurq6dvvv27dPN954o5KTk1VTU6MHHnhA8+fP18aNG919WlpadMMNN6i2tlb/+Z//qT179qi0tFTnn39+12cGAAAAAEAP4PPXia1YsULp6enKyMiQJBUWFuqtt95ScXGxCgoKvPqvWrVKo0ePVmFhoSQpLi5OO3bs0PLly3XbbbdJktasWaO///3vqqqqUv/+/SVJ0dHRXZ0TAAAAAAA9hk873i0tLaqurlZKSopHe0pKiqqqqtq9Zvv27V79p06dqh07duibb76RJL322mtKTEzUvHnzFBkZqfj4eD322GNqbW3tsJbm5mY1NTV5HAAAAAAA9DQ+Be/Gxka1trYqMjLSoz0yMlL19fXtXlNfX99u/xMnTqixsVGS9MUXX+g///M/1draqoqKCi1ZskRPPfWUHn300Q5rKSgoUFhYmPuIioryZSoAAAAAAJwVXXq4ms1m83htjPFq+67+325va2vTeeedp5KSEjkcDt1xxx3Ky8tTcXFxh2Pm5ubqyJEj7uPAgQNdmQoAAAAAAJby6TPe4eHhCgoK8trdbmho8NrVPmXEiBHt9g8ODtbw4cMlSSNHjlT//v0VFBTk7hMXF6f6+nq1tLQoJCTEa9zQ0FCFhob6Uj4AAADQq8QsfsPfJfis9jc3+bsEoMfxacc7JCREDodDlZWVHu2VlZVKSkpq95rExESv/ps2bVJCQoL7QWpTpkzRZ599pra2NnefvXv3auTIke2GbgAAAAAAeguf32qek5Oj559/XmvWrNHu3bu1cOFC1dXVKTMzU9LJt4DPnTvX3T8zM1P79+9XTk6Odu/erTVr1mj16tVatGiRu88vfvELHT58WAsWLNDevXv1xhtv6LHHHtO8efO6YYoAAAAAAPiPz18nlpqaqsOHDys/P18ul0vx8fGqqKhwf/2Xy+Xy+E7v2NhYVVRUaOHChXruuec0atQoPf300+6vEpOkqKgobdq0SQsXLtSll16q888/XwsWLND999/fDVMEAAAAAMB/fA7ekpSVlaWsrKx2z5WVlXm1XX311dq5c+dpx0xMTNSf//znrpQDAAAAAECP1aWnmgMAAAAAgM4heAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAB9TFFRkWJjY2W32+VwOLR169YO+7pcLs2aNUvjxo1Tv379lJ2dfdqxN2zYIJvNppkzZ3q0FxQU6LLLLtOgQYN03nnnaebMmdqzZ083zAYAgJ6P4A0AQB9SXl6u7Oxs5eXlqaamRsnJyZo2bZrHV4F+W3NzsyIiIpSXl6eJEyeeduz9+/dr0aJFSk5O9jq3ZcsWzZs3T3/+859VWVmpEydOKCUlRcePH++WeQEA0JMRvAEA6ENWrFih9PR0ZWRkKC4uToWFhYqKilJxcXG7/WNiYrRy5UrNnTtXYWFhHY7b2tqq2bNna9myZRozZozX+TfffFM//elPdfHFF2vixIl64YUXVFdXp+rq6m6bGwAAPRXBGwCAPqKlpUXV1dVKSUnxaE9JSVFVVdUZjZ2fn6+IiAilp6d3qv+RI0ckScOGDTuj+wIA0BsE+7sAAABwdjQ2Nqq1tVWRkZEe7ZGRkaqvr+/yuO+//75Wr14tp9PZqf7GGOXk5OjKK69UfHx8h/2am5vV3Nzsft3U1NTlGgEA8Cd2vAEA6GNsNpvHa2OMV1tnHT16VHPmzFFpaanCw8M7dc0vf/lLffzxx1q/fv1p+xUUFCgsLMx9REVFdalGAAD8jR1vAAD6iPDwcAUFBXntbjc0NHjtgnfW559/rtraWk2fPt3d1tbWJkkKDg7Wnj17NHbsWPe5++67T6+99pree+89XXDBBacdOzc3Vzk5Oe7XTU1NhG94iFn8hr9L8Fntb27ydwkA/IDgDQBAHxESEiKHw6HKykrdeuut7vbKykrNmDGjS2OOHz9eu3bt8mhbsmSJjh49qpUrV7qDsjFG9913n1555RVt3rxZsbGx3zl2aGioQkNDu1QXAAA9CcEbAIA+JCcnR2lpaUpISFBiYqJKSkpUV1enzMxMSSd3mQ8ePKi1a9e6rzn12e1jx47p0KFDcjqdCgkJ0YQJE2S3270+pz1kyBBJ8mifN2+e1q1bp1dffVWDBg1y77qHhYVpwIABFs4YAAD/I3gDANCHpKam6vDhw8rPz5fL5VJ8fLwqKioUHR0tSXK5XF7f6T158mT3z9XV1Vq3bp2io6NVW1vb6fue+rqya665xqP9hRde0E9/+tMuzQUAgN6C4A0AQB+TlZWlrKysds+VlZV5tRljfBq/O8YAACCQ8FRzAAAAAAAsRPAGAAAAAMBCvNUcAAD0SXwVFQDgbGHHGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwUJeCd1FRkWJjY2W32+VwOLR169bT9t+yZYscDofsdrvGjBmjVatWddh3w4YNstlsmjlzZldKAwAAAACgR/E5eJeXlys7O1t5eXmqqalRcnKypk2bprq6unb779u3TzfeeKOSk5NVU1OjBx54QPPnz9fGjRu9+u7fv1+LFi1ScnKy7zMBAAAAAKAH8jl4r1ixQunp6crIyFBcXJwKCwsVFRWl4uLidvuvWrVKo0ePVmFhoeLi4pSRkaG7775by5cv9+jX2tqq2bNna9myZRozZkzXZgMAAAAAQA/jU/BuaWlRdXW1UlJSPNpTUlJUVVXV7jXbt2/36j916lTt2LFD33zzjbstPz9fERERSk9P71Qtzc3Nampq8jgAAAAAAOhpfArejY2Nam1tVWRkpEd7ZGSk6uvr272mvr6+3f4nTpxQY2OjJOn999/X6tWrVVpa2ulaCgoKFBYW5j6ioqJ8mQoAAAAAAGdFlx6uZrPZPF4bY7zavqv/qfajR49qzpw5Ki0tVXh4eKdryM3N1ZEjR9zHgQMHfJgBAAAAAABnR7AvncPDwxUUFOS1u93Q0OC1q33KiBEj2u0fHBys4cOH65NPPlFtba2mT5/uPt/W1nayuOBg7dmzR2PHjvUaNzQ0VKGhob6UDwAAAADAWefTjndISIgcDocqKys92isrK5WUlNTuNYmJiV79N23apISEBPXv31/jx4/Xrl275HQ63cctt9yia6+9Vk6nk7eQAwAAAAB6NZ92vCUpJydHaWlpSkhIUGJiokpKSlRXV6fMzExJJ98CfvDgQa1du1aSlJmZqWeffVY5OTm65557tH37dq1evVrr16+XJNntdsXHx3vcY8iQIZLk1Q4AAAAAQG/jc/BOTU3V4cOHlZ+fL5fLpfj4eFVUVCg6OlqS5HK5PL7TOzY2VhUVFVq4cKGee+45jRo1Sk8//bRuu+227psFAAAAAAA9lM/BW5KysrKUlZXV7rmysjKvtquvvlo7d+7s9PjtjQEAAAAAQG/UpaeaAwAAAACAziF4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAH1MUVGRYmNjZbfb5XA4tHXr1g77ulwuzZo1S+PGjVO/fv2UnZ192rE3bNggm82mmTNnntF9AQAIJARvAAD6kPLycmVnZysvL081NTVKTk7WtGnTPL4K9Nuam5sVERGhvLw8TZw48bRj79+/X4sWLVJycvIZ3xcAgEBC8AYAoA9ZsWKF0tPTlZGRobi4OBUWFioqKkrFxcXt9o+JidHKlSs1d+5chYWFdThua2urZs+erWXLlmnMmDFnfF8AAAIJwRsAgD6ipaVF1dXVSklJ8WhPSUlRVVXVGY2dn5+viIgIpaend9t9m5ub1dTU5HEAANAbEbwBAOgjGhsb1draqsjISI/2yMhI1dfXd3nc999/X6tXr1ZpaWm33regoEBhYWHuIyoqqss1AgDgTwRvAAD6GJvN5vHaGOPV1llHjx7VnDlzVFpaqvDw8G69b25uro4cOeI+Dhw40KUaAQDwt2B/FwAAAM6O8PBwBQUFee0yNzQ0eO1Gd9bnn3+u2tpaTZ8+3d3W1tYmSQoODtaePXsUFRXVpfuGhoYqNDS0S3UBANCTsOMNAEAfERISIofDocrKSo/2yspKJSUldWnM8ePHa9euXXI6ne7jlltu0bXXXiun06moqChL7gsAQG/CjjcAAH1ITk6O0tLSlJCQoMTERJWUlKiurk6ZmZmSTr69++DBg1q7dq37GqfTKUk6duyYDh06JKfTqZCQEE2YMEF2u13x8fEe9xgyZIgkebR/130BAAhkBG8AAPqQ1NRUHT58WPn5+XK5XIqPj1dFRYWio6MlSS6Xy+u7tSdPnuz+ubq6WuvWrVN0dLRqa2u77b4AAAQygjcAAH1MVlaWsrKy2j1XVlbm1WaM8Wn89sb4rvsCABDI+Iw3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFupS8C4qKlJsbKzsdrscDoe2bt162v5btmyRw+GQ3W7XmDFjtGrVKo/zpaWlSk5O1tChQzV06FBdf/31+uCDD7pSGgAAAAAAPYrPwbu8vFzZ2dnKy8tTTU2NkpOTNW3aNNXV1bXbf9++fbrxxhuVnJysmpoaPfDAA5o/f742btzo7rN582bdeeedevfdd7V9+3aNHj1aKSkpOnjwYNdnBgAAAABAD+Bz8F6xYoXS09OVkZGhuLg4FRYWKioqSsXFxe32X7VqlUaPHq3CwkLFxcUpIyNDd999t5YvX+7u89JLLykrK0uTJk3S+PHjVVpaqra2Nr3zzjtdnxkAAAAAAD2AT8G7paVF1dXVSklJ8WhPSUlRVVVVu9ds377dq//UqVO1Y8cOffPNN+1e8/XXX+ubb77RsGHDOqylublZTU1NHgcAAAAAAD2NT8G7sbFRra2tioyM9GiPjIxUfX19u9fU19e32//EiRNqbGxs95rFixfr/PPP1/XXX99hLQUFBQoLC3MfUVFRvkwFAAAAAICzoksPV7PZbB6vjTFebd/Vv712SXriiSe0fv16vfzyy7Lb7R2OmZubqyNHjriPAwcO+DIFAAAAAADOimBfOoeHhysoKMhrd7uhocFrV/uUESNGtNs/ODhYw4cP92hfvny5HnvsMb399tu69NJLT1tLaGioQkNDfSkfAAAAAICzzqcd75CQEDkcDlVWVnq0V1ZWKikpqd1rEhMTvfpv2rRJCQkJ6t+/v7vtySef1MMPP6w333xTCQkJvpQFAAAAAECP5fNbzXNycvT8889rzZo12r17txYuXKi6ujplZmZKOvkW8Llz57r7Z2Zmav/+/crJydHu3bu1Zs0arV69WosWLXL3eeKJJ7RkyRKtWbNGMTExqq+vV319vY4dO9YNUwQAAAAAwH98equ5JKWmpurw4cPKz8+Xy+VSfHy8KioqFB0dLUlyuVwe3+kdGxuriooKLVy4UM8995xGjRqlp59+Wrfddpu7T1FRkVpaWnT77bd73Gvp0qV66KGHujg1AAAAAAD8z+fgLUlZWVnKyspq91xZWZlX29VXX62dO3d2OF5tbW1XygAAAAAAoMfr0lPNAQAAAABA5xC8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAgD6mqKhIsbGxstvtcjgc2rp1a4d9XS6XZs2apXHjxqlfv37Kzs726vPyyy8rISFBQ4YM0cCBAzVp0iT9/ve/9+hz4sQJLVmyRLGxsRowYIDGjBmj/Px8tbW1dff0AADocQjeAAD0IeXl5crOzlZeXp5qamqUnJysadOmeXwV6Lc1NzcrIiJCeXl5mjhxYrt9hg0bpry8PG3fvl0ff/yxfvazn+lnP/uZ3nrrLXefxx9/XKtWrdKzzz6r3bt364knntCTTz6pZ555xpJ5AgDQkxC8AQDoQ1asWKH09HRlZGQoLi5OhYWFioqKUnFxcbv9Y2JitHLlSs2dO1dhYWHt9rnmmmt06623Ki4uTmPHjtWCBQt06aWXatu2be4+27dv14wZM3TTTTcpJiZGt99+u1JSUrRjxw5L5gkAQE9C8AYAoI9oaWlRdXW1UlJSPNpTUlJUVVXVLfcwxuidd97Rnj17dNVVV7nbr7zySr3zzjvau3evJOmjjz7Stm3bdOONN3Y4VnNzs5qamjwOAAB6o2B/FwAAAM6OxsZGtba2KjIy0qM9MjJS9fX1ZzT2kSNHdP7556u5uVlBQUEqKirSDTfc4D5///3368iRIxo/fryCgoLU2tqqRx99VHfeeWeHYxYUFGjZsmVnVBcAAD0BwRsAgD7GZrN5vDbGeLX5atCgQXI6nTp27Jjeeecd5eTkaMyYMbrmmmsknfxs+Ysvvqh169bp4osvltPpVHZ2tkaNGqW77rqr3TFzc3OVk5Pjft3U1KSoqKgzqhMAAH8geAMA0EeEh4crKCjIa3e7oaHBaxfcV/369dOFF14oSZo0aZJ2796tgoICd/D+1a9+pcWLF+uOO+6QJF1yySXav3+/CgoKOgzeoaGhCg0NPaO6AADoCfiMNwAAfURISIgcDocqKys92isrK5WUlNSt9zLGqLm52f3666+/Vr9+nn/tCAoK4uvEAAB9AjveAAD0ITk5OUpLS1NCQoISExNVUlKiuro6ZWZmSjr59u6DBw9q7dq17mucTqck6dixYzp06JCcTqdCQkI0YcIESSc/i52QkKCxY8eqpaVFFRUVWrt2rceT0qdPn65HH31Uo0eP1sUXX6yamhqtWLFCd99999mbPAAAfkLwBgCgD0lNTdXhw4eVn58vl8ul+Ph4VVRUKDo6WpLkcrm8vtN78uTJ7p+rq6u1bt06RUdHq7a2VpJ0/PhxZWVl6X//9381YMAAjR8/Xi+++KJSU1Pd1z3zzDN68MEHlZWVpYaGBo0aNUo///nP9etf/9r6SQMA4GcEbwAA+pisrCxlZWW1e66srMyrzRhz2vEeeeQRPfLII6ftM2jQIBUWFqqwsLCzZQIAEDD4jDcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAW6lLwLioqUmxsrOx2uxwOh7Zu3Xra/lu2bJHD4ZDdbteYMWO0atUqrz4bN27UhAkTFBoaqgkTJuiVV17pSmkAAAAAAPQoPgfv8vJyZWdnKy8vTzU1NUpOTta0adNUV1fXbv99+/bpxhtvVHJysmpqavTAAw9o/vz52rhxo7vP9u3blZqaqrS0NH300UdKS0vTT37yE/3lL3/p+swAAAAAAOgBfA7eK1asUHp6ujIyMhQXF6fCwkJFRUWpuLi43f6rVq3S6NGjVVhYqLi4OGVkZOjuu+/W8uXL3X0KCwt1ww03KDc3V+PHj1dubq6uu+46FRYWdnliAAAAAAD0BMG+dG5paVF1dbUWL17s0Z6SkqKqqqp2r9m+fbtSUlI82qZOnarVq1frm2++Uf/+/bV9+3YtXLjQq8/pgndzc7Oam5vdr48cOSJJampq8mVKHWpr/rpbxjmbfJl7b5yfFPhzDPT5SYE/R1//NyjQ5xjo8+vMOMaYbhkP//e7ZK3vvECfY6DPTwr8OQb6/KTAn2NvnJ909td7n4J3Y2OjWltbFRkZ6dEeGRmp+vr6dq+pr69vt/+JEyfU2NiokSNHdtinozElqaCgQMuWLfNqj4qK6ux0Ak5Yob8rsF6gzzHQ5ycF/hwDfX5S4M+xu+d39OhRhYWFde+gfdTRo0clsdYHukCfY6DPTwr8OQb6/CTm2BXftd77FLxPsdlsHq+NMV5t39X/X9t9HTM3N1c5OTnu121tbfr73/+u4cOHn/Y6f2tqalJUVJQOHDigwYMH+7ucbhfo85MCf46BPj8p8OcY6POTesccjTE6evSoRo0a5e9SAsaoUaN04MABDRo0iLXejwJ9flLgzzHQ5ycF/hwDfX5S75ljZ9d7n4J3eHi4goKCvHaiGxoavHasTxkxYkS7/YODgzV8+PDT9uloTEkKDQ1VaGioR9uQIUM6OxW/Gzx4cI/+A3SmAn1+UuDPMdDnJwX+HAN9flLPnyM73d2rX79+uuCCC/xdRqf19D+fZyrQ5ycF/hwDfX5S4M8x0Ocn9Y45dma99+nhaiEhIXI4HKqsrPRor6ysVFJSUrvXJCYmevXftGmTEhIS1L9//9P26WhMAAAAAAB6C5/fap6Tk6O0tDQlJCQoMTFRJSUlqqurU2ZmpqSTbwE/ePCg1q5dK0nKzMzUs88+q5ycHN1zzz3avn27Vq9erfXr17vHXLBgga666io9/vjjmjFjhl599VW9/fbb2rZtWzdNEwAAAAAA//A5eKempurw4cPKz8+Xy+VSfHy8KioqFB0dLUlyuVwe3+kdGxuriooKLVy4UM8995xGjRqlp59+Wrfddpu7T1JSkjZs2KAlS5bowQcf1NixY1VeXq7LL7+8G6bYs4SGhmrp0qVeb5MPFIE+Pynw5xjo85MCf46BPj+pb8wRvVeg//kM9PlJgT/HQJ+fFPhzDPT5SYE3R5vhe04AAAAAALCMT5/xBgAAAAAAviF4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgvdZVFRUpNjYWNntdjkcDm3dutXfJXWb9957T9OnT9eoUaNks9n0X//1X/4uqVsVFBTosssu06BBg3Teeedp5syZ2rNnj7/L6lbFxcW69NJLNXjwYA0ePFiJiYn605/+5O+yLFNQUCCbzabs7Gx/l9JtHnroIdlsNo9jxIgR/i6rWx08eFBz5szR8OHDdc4552jSpEmqrq72d1mAG2t978VaH3hY63unQF3rCd5nSXl5ubKzs5WXl6eamholJydr2rRpHl+91psdP35cEydO1LPPPuvvUiyxZcsWzZs3T3/+859VWVmpEydOKCUlRcePH/d3ad3mggsu0G9+8xvt2LFDO3bs0A9/+EPNmDFDn3zyib9L63YffvihSkpKdOmll/q7lG538cUXy+VyuY9du3b5u6Ru8+WXX2rKlCnq37+//vSnP+mvf/2rnnrqKQ0ZMsTfpQGSWOt7O9b6wMJa3zsF9FpvcFb84Ac/MJmZmR5t48ePN4sXL/ZTRdaRZF555RV/l2GphoYGI8ls2bLF36VYaujQoeb555/3dxnd6ujRo+aiiy4ylZWV5uqrrzYLFizwd0ndZunSpWbixIn+LsMy999/v7nyyiv9XQbQIdb6wMJa33ux1vdegbzWs+N9FrS0tKi6ulopKSke7SkpKaqqqvJTVTgTR44ckSQNGzbMz5VYo7W1VRs2bNDx48eVmJjo73K61bx583TTTTfp+uuv93cplvj00081atQoxcbG6o477tAXX3zh75K6zWuvvaaEhAT9+Mc/1nnnnafJkyertLTU32UBkljrAxFrfe/FWt97BfJaT/A+CxobG9Xa2qrIyEiP9sjISNXX1/upKnSVMUY5OTm68sorFR8f7+9yutWuXbt07rnnKjQ0VJmZmXrllVc0YcIEf5fVbTZs2KCdO3eqoKDA36VY4vLLL9fatWv11ltvqbS0VPX19UpKStLhw4f9XVq3+OKLL1RcXKyLLrpIb731ljIzMzV//nytXbvW36UBrPUBhrW+92Kt790Cea0P9ncBfYnNZvN4bYzxakPP98tf/lIff/yxtm3b5u9Sut24cePkdDr11VdfaePGjbrrrru0ZcuWgFiQDxw4oAULFmjTpk2y2+3+LscS06ZNc/98ySWXKDExUWPHjtXvfvc75eTk+LGy7tHW1qaEhAQ99thjkqTJkyfrk08+UXFxsebOnevn6oCTWOsDA2t978Raz1rfk7HjfRaEh4crKCjI61+8GxoavP5lHD3bfffdp9dee03vvvuuLrjgAn+X0+1CQkJ04YUXKiEhQQUFBZo4caJWrlzp77K6RXV1tRoaGuRwOBQcHKzg4GBt2bJFTz/9tIKDg9Xa2urvErvdwIEDdckll+jTTz/1dyndYuTIkV5/MYyLiwuYB1ehd2OtDxys9b0Xa33vF8hrPcH7LAgJCZHD4VBlZaVHe2VlpZKSkvxUFXxhjNEvf/lLvfzyy/rv//5vxcbG+ruks8IYo+bmZn+X0S2uu+467dq1S06n030kJCRo9uzZcjqdCgoK8neJ3a65uVm7d+/WyJEj/V1Kt5gyZYrXV/vs3btX0dHRfqoI+D+s9b0fa33vx1rf+wXyWs9bzc+SnJwcpaWlKSEhQYmJiSopKVFdXZ0yMzP9XVq3OHbsmD777DP363379snpdGrYsGEaPXq0HyvrHvPmzdO6dev06quvatCgQe4djbCwMA0YMMDP1XWPBx54QNOmTVNUVJSOHj2qDRs2aPPmzXrzzTf9XVq3GDRokNfn9AYOHKjhw4cHzOf3Fi1apOnTp2v06NFqaGjQI488oqamJt11113+Lq1bLFy4UElJSXrsscf0k5/8RB988IFKSkpUUlLi79IASaz1vR1rfe/HWt/7BfRa778Hqvc9zz33nImOjjYhISHm+9//fkB9PcW7775rJHkdd911l79L6xbtzU2SeeGFF/xdWre5++673X8+IyIizHXXXWc2bdrk77IsFWhfMZKammpGjhxp+vfvb0aNGmV+9KMfmU8++cTfZXWr119/3cTHx5vQ0FAzfvx4U1JS4u+SAA+s9b0Xa31gYq3vfQJ1rbcZY8zZDPoAAAAAAPQlfMYbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbyBAHPNNdcoOzvb32X4pKysTEOGDPHpmp/+9KeaOXOmJfUAAHA6rLXorNraWtlsNjmdTn+XAj8jeAM+qKqqUlBQkP7t3/7N69xDDz2kSZMmebXbbDb913/9V7fXsnnzZtlsNn311Vce7S+//LIefvjhbr+flVJTU7V3795uHzcmJkaFhYXdPi4AwDqstdZgrQX8i+AN+GDNmjW67777tG3bNtXV1fm7nHYNGzZMgwYN8ncZPhkwYIDOO+88f5cBAOgBWGutwVoL+BfBG+ik48eP6w9/+IN+8Ytf6Oabb1ZZWZn7XFlZmZYtW6aPPvpINptNNptNZWVliomJkSTdeuutstls7teS9Prrr8vhcMhut2vMmDFatmyZTpw44T5vs9n0/PPP69Zbb9U555yjiy66SK+99pqkk29buvbaayVJQ4cOlc1m009/+lNJ3m9/+/LLLzV37lwNHTpU55xzjqZNm6ZPP/3Uo/YhQ4borbfeUlxcnM4991z927/9m1wuV4e/C4fDoaeeesr9eubMmQoODlZTU5Mkqb6+XjabTXv27JEktbS06D/+4z90/vnna+DAgbr88su1efNmrxq+7ZFHHtF5552nQYMGKSMjQ4sXL253l2P58uUaOXKkhg8frnnz5umbb75x/x7279+vhQsXuv+bSNL+/fs1ffp0DR06VAMHDtTFF1+sioqKDucKADh7WGv/T19ea2NiYtzjffs4ZdeuXfrhD3+oAQMGaPjw4br33nt17Ngx9/m2tjbl5+frggsuUGhoqCZNmqQ333zTff7U27//8Ic/KDk5WQMGDNBll12mvXv36sMPP1RCQoL7v9GhQ4c8anvhhRcUFxcnu92u8ePHq6ioyOP8Bx98oMmTJ8tutyshIUE1NTWdnjcCnAHQKatXrzYJCQnGGGNef/11ExMTY9ra2owxxnz99dfm3//9383FF19sXC6Xcblc5uuvvzYNDQ1GknnhhReMy+UyDQ0Nxhhj3nzzTTN48GBTVlZmPv/8c7Np0yYTExNjHnroIff9JJkLLrjArFu3znz66adm/vz55txzzzWHDx82J06cMBs3bjSSzJ49e4zL5TJfffWVMcaYq6++2ixYsMA9zi233GLi4uLMe++9Z5xOp5k6daq58MILTUtLizHGmBdeeMH079/fXH/99ebDDz801dXVJi4uzsyaNavD30VOTo65+eabjTHGtLW1mWHDhpnw8HDzxhtvGGOMWbdunRkxYoS7/6xZs0xSUpJ57733zGeffWaefPJJExoaavbu3euuISwszN3/xRdfNHa73axZs8bs2bPHLFu2zAwePNhMnDjR3eeuu+4ygwcPNpmZmWb37t3m9ddfN+ecc44pKSkxxhhz+PBhc8EFF5j8/Hz3fxNjjLnpppvMDTfcYD7++GPz+eefm9dff91s2bKlk38KAABWYq39P315rW1oaHCP97//+7/miiuuMMnJycYYY44fP25GjRplfvSjH5ldu3aZd955x8TGxpq77rrLff2KFSvM4MGDzfr1683/9//9f+Y//uM/TP/+/d2/i3379hlJZvz48ebNN980f/3rX80VV1xhvv/975trrrnGbNu2zezcudNceOGFJjMz0z1uSUmJGTlypNm4caP54osvzMaNG82wYcNMWVmZMcaYY8eOmYiICJOammr+53/+x7z++utmzJgxRpKpqanp9PwRmAjeQCclJSWZwsJCY4wx33zzjQkPDzeVlZXu80uXLvVYrE6RZF555RWPtuTkZPPYY495tP3+9783I0eO9LhuyZIl7tfHjh0zNpvN/OlPfzLGGPPuu+8aSebLL7/0GOfbfxnYu3evkWTef/999/nGxkYzYMAA84c//MEYc3IhlmQ+++wzd5/nnnvOREZGdvi7eO2110xYWJhpbW01TqfTREREmIULF5pf/epXxhhj7r33XpOammqMMeazzz4zNpvNHDx40GOM6667zuTm5rpr+PZfBi6//HIzb948j/5Tpkzx+stAdHS0OXHihLvtxz/+sfu+xhgTHR1tfvvb33qMc8kll3j8pQsA0HOw1v4f1tqT5s+fb6Kjo93/oFJSUmKGDh1qjh075u7zxhtvmH79+pn6+npjjDGjRo0yjz76qMc4l112mcnKyjLG/F/wfv75593n169fbySZd955x91WUFBgxo0b534dFRVl1q1b5zHuww8/bBITE40xxvy///f/zLBhw8zx48fd54uLiwneMMYYw1vNgU7Ys2ePPvjgA91xxx2SpODgYKWmpmrNmjVdGq+6ulr5+fk699xz3cc999wjl8ulr7/+2t3v0ksvdf88cOBADRo0SA0NDZ2+z+7duxUcHKzLL7/c3TZ8+HCNGzdOu3fvdredc845Gjt2rPv1yJEjT3ufq666SkePHlVNTY22bNmiq6++Wtdee622bNki6eTDaK6++mpJ0s6dO2WM0fe+9z2P+W7ZskWff/55u+Pv2bNHP/jBDzza/vW1JF188cUKCgrqdN2SNH/+fD3yyCOaMmWKli5dqo8//vi0/QEAZwdrrSfWWqmkpESrV6/Wq6++qoiICEknf98TJ07UwIED3f2mTJmitrY27dmzR01NTfrb3/6mKVOmeIw1ZcoUj/8ekud/+8jISEnSJZdc4tF2aq6HDh3SgQMHlJ6e7vE7fuSRR9y/41O1nXPOOe4xEhMTuzR3BJ5gfxcA9AarV6/WiRMndP7557vbjDHq37+/vvzySw0dOtSn8dra2rRs2TL96Ec/8jpnt9vdP/fv39/jnM1mU1tbW6fvY4zpsP3bn5Vq7z4dXStJYWFhmjRpkjZv3qyqqir98Ic/VHJyspxOpz799FPt3btX11xzjaSTcw0KClJ1dbXHwi1J5557bof3+HZ9Hc2lK7+fjIwMTZ06VW+88YY2bdqkgoICPfXUU7rvvvtOex0AwFqstZ76+lq7efNm3XfffVq/fr0mTpzoUeO/1v3t2tr7uaPrvj23U+f+te3UXE/939LSUo9/ZJHk/p2f7r8nwI438B1OnDihtWvX6qmnnpLT6XQfH330kaKjo/XSSy9JkkJCQtTa2up1ff/+/b3av//972vPnj268MILvY5+/Tr3/5YhISGS1O49T5kwYYJOnDihv/zlL+62w4cPa+/evYqLi+vUfTpyzTXX6N1339V7772na665RkOGDNGECRPcD2o5Nf7kyZPV2tqqhoYGr7mOGDGi3bHHjRunDz74wKNtx44dPtfY0X+TqKgoZWZm6uWXX9a///u/q7S01OexAQDdh7W2fX11rf3ss89022236YEHHvD6h5MJEybI6XTq+PHj7rb3339f/fr10/e+9z0NHjxYo0aN0rZt2zyuq6qqOqP/HpGRkTr//PP1xRdfeP2OY2Nj3bV99NFH+sc//uG+7s9//nOX74nAQvAGvsMf//hHffnll0pPT1d8fLzHcfvtt2v16tWSTj6Bc9++fXI6nWpsbFRzc7O7/Z133lF9fb2+/PJLSdKvf/1rrV27Vg899JA++eQT7d69W+Xl5VqyZEmn64qOjpbNZtMf//hHHTp0yONpnqdcdNFFmjFjhu655x5t27ZNH330kebMmaPzzz9fM2bMOKPfyzXXXKM333xTNptNEyZMcLe99NJL7re+SdL3vvc9zZ49W3PnztXLL7+sffv26cMPP9Tjjz/e4RNO77vvPq1evVq/+93v9Omnn+qRRx7Rxx9/3OG/cHckJiZG7733ng4ePKjGxkZJUnZ2tt566y3t27dPO3fu1H//93+f8V+MAABnhrW2fX1xrf3HP/6h6dOna9KkSbr33ntVX1/vPiRp9uzZstvtuuuuu/Q///M/evfdd3XfffcpLS3N/XbxX/3qV3r88cdVXl6uPXv2aPHixXI6nVqwYIFPc/tXDz30kAoKCrRy5Urt3btXu3bt0gsvvKAVK1ZIkmbNmqV+/fopPT1df/3rX1VRUaHly5ef0T0RQPzwuXKgV7n55pvNjTfe2O656upqI8lUV1ebf/7zn+a2224zQ4YMcT9d1ZiTD0e58MILTXBwsImOjnZf++abb5qkpCQzYMAAM3jwYPODH/zA/ZRQY9p/UExYWJh7XGOMyc/PNyNGjDA2m839NM9/fdLq3//+d5OWlmbCwsLMgAEDzNSpU91P9TTG+2ErxhjzyiuvmO/6n4evvvrKBAUFmdtvv93rumeffdajb0tLi/n1r39tYmJiTP/+/c2IESPMrbfeaj7++OMOa8jPzzfh4eHm3HPPNXfffbeZP3++ueKKK9zn77rrLjNjxgyPaxYsWGCuvvpq9+vt27ebSy+91ISGhrrn88tf/tKMHTvWhIaGmoiICJOWlmYaGxtPO1cAgLVYa9sXqGttdHS0Wbp0abtzPvXgs/aOUz7++GNz7bXXGrvdboYNG2buuecec/ToUff51tZWs2zZMnP++eeb/v37m4kTJ7ofmPfte3z7gWftPUivvd/ZSy+9ZCZNmmRCQkLM0KFDzVVXXWVefvllj9/HxIkTTUhIiJk0aZL7yfg8XA02Y/gwAoCe74YbbtCIESP0+9//3t+lAAAQkM7GWvuPf/xDw4YNU0VFhft70oG+gIerAehxvv76a61atUpTp05VUFCQ1q9fr7fffluVlZX+Lg0AgIDgr7V2y5Yt+uEPf0joRp/DjjeAHufU57t27typ5uZmjRs3TkuWLGn3ybQAAMB3rLXA2UXwBgAAAADAQjzVHAAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAAL/f+lwFwXgyVx1gAAAABJRU5ErkJggg==\n"},"metadata":{}}]},{"cell_type":"markdown","source":"Decoder","metadata":{"id":"g68xf3c_-gBS"}},{"cell_type":"code","source":"class Decoder(tf.keras.layers.Layer):\n def __init__(self, output_vocab_size, embedding_dim, dec_units):\n super(Decoder, self).__init__()\n self.dec_units = dec_units\n self.output_vocab_size = output_vocab_size\n self.embedding_dim = embedding_dim\n\n # For Step 1. The embedding layer converts token IDs to vectors\n self.embedding = tf.keras.layers.Embedding(self.output_vocab_size,\n embedding_dim)\n\n # For Step 2. The RNN keeps track of what's been generated so far.\n self.gru = tf.keras.layers.GRU(self.dec_units,\n return_sequences=True,\n return_state=True,\n recurrent_initializer='glorot_uniform',\n recurrent_dropout=0.1)\n\n # For step 3. The RNN output will be the query for the attention layer.\n self.attention = BahdanauAttention(self.dec_units)\n\n # For step 4. Eqn. (3): converting `ct` to `at`\n self.Wc = tf.keras.layers.Dense(dec_units, activation=tf.math.tanh,\n use_bias=False)\n\n # For step 5. This fully connected layer produces the logits for each output token.\n self.fc = tf.keras.layers.Dense(self.output_vocab_size)","metadata":{"id":"erYvHIgAl8kh","execution":{"iopub.status.busy":"2023-03-16T11:46:39.183861Z","iopub.execute_input":"2023-03-16T11:46:39.184272Z","iopub.status.idle":"2023-03-16T11:46:39.192243Z","shell.execute_reply.started":"2023-03-16T11:46:39.184233Z","shell.execute_reply":"2023-03-16T11:46:39.191028Z"},"trusted":true},"execution_count":55,"outputs":[]},{"cell_type":"code","source":"class DecoderInput(typing.NamedTuple):\n new_tokens: Any\n enc_output: Any\n mask: Any\n\nclass DecoderOutput(typing.NamedTuple):\n logits: Any\n attention_weights: Any","metadata":{"id":"7WfSIb2sArRT","execution":{"iopub.status.busy":"2023-03-16T11:46:39.193966Z","iopub.execute_input":"2023-03-16T11:46:39.194651Z","iopub.status.idle":"2023-03-16T11:46:39.206347Z","shell.execute_reply.started":"2023-03-16T11:46:39.194612Z","shell.execute_reply":"2023-03-16T11:46:39.205356Z"},"trusted":true},"execution_count":56,"outputs":[]},{"cell_type":"code","source":"def call(self,\n inputs: DecoderInput,\n state=None) -> Tuple[DecoderOutput, tf.Tensor]:\n shape_checker = ShapeChecker()\n shape_checker(inputs.new_tokens, ('batch', 't'))\n shape_checker(inputs.enc_output, ('batch', 's', 'enc_units'))\n shape_checker(inputs.mask, ('batch', 's'))\n\n if state is not None:\n shape_checker(state, ('batch', 'dec_units'))\n\n # Step 1. Lookup the embeddings\n vectors = self.embedding(inputs.new_tokens)\n shape_checker(vectors, ('batch', 't', 'embedding_dim'))\n\n # Step 2. Process one step with the RNN\n rnn_output, state = self.gru(vectors, initial_state=state)\n\n shape_checker(rnn_output, ('batch', 't', 'dec_units'))\n shape_checker(state, ('batch', 'dec_units'))\n\n # Step 3. Use the RNN output as the query for the attention over the\n # encoder output.\n context_vector, attention_weights = self.attention(\n query=rnn_output, value=inputs.enc_output, mask=inputs.mask)\n shape_checker(context_vector, ('batch', 't', 'dec_units'))\n shape_checker(attention_weights, ('batch', 't', 's'))\n\n # Step 4. Eqn. (3): Join the context_vector and rnn_output\n # [ct; ht] shape: (batch t, value_units + query_units)\n context_and_rnn_output = tf.concat([context_vector, rnn_output], axis=-1)\n\n # Step 4. Eqn. (3): `at = tanh(Wc@[ct; ht])`\n attention_vector = self.Wc(context_and_rnn_output)\n shape_checker(attention_vector, ('batch', 't', 'dec_units'))\n\n # Step 5. Generate logit predictions:\n logits = self.fc(attention_vector)\n shape_checker(logits, ('batch', 't', 'output_vocab_size'))\n\n return DecoderOutput(logits, attention_weights), state","metadata":{"id":"PJOi5btHAPNK","execution":{"iopub.status.busy":"2023-03-16T11:46:39.208199Z","iopub.execute_input":"2023-03-16T11:46:39.208844Z","iopub.status.idle":"2023-03-16T11:46:39.220786Z","shell.execute_reply.started":"2023-03-16T11:46:39.208582Z","shell.execute_reply":"2023-03-16T11:46:39.219581Z"},"trusted":true},"execution_count":57,"outputs":[]},{"cell_type":"code","source":"Decoder.call = call","metadata":{"id":"Ay_mTMPfnb2a","execution":{"iopub.status.busy":"2023-03-16T11:46:39.222435Z","iopub.execute_input":"2023-03-16T11:46:39.223014Z","iopub.status.idle":"2023-03-16T11:46:39.232325Z","shell.execute_reply.started":"2023-03-16T11:46:39.222949Z","shell.execute_reply":"2023-03-16T11:46:39.231281Z"},"trusted":true},"execution_count":58,"outputs":[]},{"cell_type":"code","source":"decoder = Decoder(output_text_processor.vocabulary_size(),\n embedding_dim, units)","metadata":{"id":"4ZUMbYXIEVeA","execution":{"iopub.status.busy":"2023-03-16T11:46:39.233853Z","iopub.execute_input":"2023-03-16T11:46:39.234238Z","iopub.status.idle":"2023-03-16T11:46:39.251412Z","shell.execute_reply.started":"2023-03-16T11:46:39.234201Z","shell.execute_reply":"2023-03-16T11:46:39.250373Z"},"trusted":true},"execution_count":59,"outputs":[]},{"cell_type":"code","source":"# Convert the target sequence, and collect the \"[START]\" tokens\nexample_output_tokens = output_text_processor(example_target_batch)\n\nstart_index = output_text_processor.get_vocabulary().index('[START]')\nfirst_token = tf.constant([[start_index]] * example_output_tokens.shape[0])","metadata":{"id":"4u6eJBU4GL40","execution":{"iopub.status.busy":"2023-03-16T11:46:39.252616Z","iopub.execute_input":"2023-03-16T11:46:39.253187Z","iopub.status.idle":"2023-03-16T11:46:39.336408Z","shell.execute_reply.started":"2023-03-16T11:46:39.253147Z","shell.execute_reply":"2023-03-16T11:46:39.335259Z"},"trusted":true},"execution_count":60,"outputs":[]},{"cell_type":"code","source":"# Run the decoder\ndec_result, dec_state = decoder(\n inputs = DecoderInput(new_tokens=first_token,\n enc_output=example_enc_output,\n mask=(example_tokens != 0)),\n state = example_enc_state\n)\n\nprint(f'logits shape: (batch_size, t, output_vocab_size) {dec_result.logits.shape}')\nprint(f'state shape: (batch_size, dec_units) {dec_state.shape}')","metadata":{"id":"E5hqvbR5FUCD","outputId":"ec292583-153d-4d60-d79e-082f2ed7a208","execution":{"iopub.status.busy":"2023-03-16T11:46:39.337807Z","iopub.execute_input":"2023-03-16T11:46:39.338547Z","iopub.status.idle":"2023-03-16T11:46:39.459382Z","shell.execute_reply.started":"2023-03-16T11:46:39.338507Z","shell.execute_reply":"2023-03-16T11:46:39.458221Z"},"trusted":true},"execution_count":61,"outputs":[{"name":"stdout","text":"logits shape: (batch_size, t, output_vocab_size) (64, 1, 23282)\nstate shape: (batch_size, dec_units) (64, 1024)\n","output_type":"stream"}]},{"cell_type":"code","source":"# sample token according to logits\nsampled_token = tf.random.categorical(dec_result.logits[:, 0, :], num_samples=1)","metadata":{"id":"P5UY8wko3jFp","execution":{"iopub.status.busy":"2023-03-16T11:46:39.460768Z","iopub.execute_input":"2023-03-16T11:46:39.461652Z","iopub.status.idle":"2023-03-16T11:46:39.470290Z","shell.execute_reply.started":"2023-03-16T11:46:39.461610Z","shell.execute_reply":"2023-03-16T11:46:39.469239Z"},"trusted":true},"execution_count":62,"outputs":[]},{"cell_type":"code","source":"# decode the token as first word of o/p\nvocab = np.array(output_text_processor.get_vocabulary())\nfirst_word = vocab[sampled_token.numpy()]\nfirst_word[:5]","metadata":{"id":"lKXTLYu4IV7I","outputId":"4c243ce7-23fc-4781-98c6-9d64e07e13d0","execution":{"iopub.status.busy":"2023-03-16T11:46:39.471965Z","iopub.execute_input":"2023-03-16T11:46:39.472379Z","iopub.status.idle":"2023-03-16T11:46:39.554000Z","shell.execute_reply.started":"2023-03-16T11:46:39.472340Z","shell.execute_reply":"2023-03-16T11:46:39.553028Z"},"trusted":true},"execution_count":63,"outputs":[{"execution_count":63,"output_type":"execute_result","data":{"text/plain":"array([['slogan'],\n ['foreigner'],\n ['disney'],\n ['ptag'],\n ['annoying']], dtype='}\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n\nCPU times: user 28.7 s, sys: 1.24 s, total: 30 s\nWall time: 29.1 s\n","output_type":"stream"}]},{"cell_type":"code","source":"# maximizing performance while training\n@tf.function(input_signature=[[tf.TensorSpec(dtype=tf.string, shape=[None]),\n tf.TensorSpec(dtype=tf.string, shape=[None])]])\ndef _tf_train_step(self, inputs):\n return self._train_step(inputs)","metadata":{"id":"UFUsTKQx0jaH","execution":{"iopub.status.busy":"2023-03-16T11:47:08.881371Z","iopub.execute_input":"2023-03-16T11:47:08.881738Z","iopub.status.idle":"2023-03-16T11:47:08.889074Z","shell.execute_reply.started":"2023-03-16T11:47:08.881701Z","shell.execute_reply":"2023-03-16T11:47:08.886914Z"},"trusted":true},"execution_count":77,"outputs":[]},{"cell_type":"code","source":"TrainTranslator._tf_train_step = _tf_train_step","metadata":{"id":"2-bgU59jrztQ","execution":{"iopub.status.busy":"2023-03-16T11:47:08.891048Z","iopub.execute_input":"2023-03-16T11:47:08.891867Z","iopub.status.idle":"2023-03-16T11:47:08.901275Z","shell.execute_reply.started":"2023-03-16T11:47:08.891829Z","shell.execute_reply":"2023-03-16T11:47:08.900192Z"},"trusted":true},"execution_count":78,"outputs":[]},{"cell_type":"code","source":"translator.use_tf_function = True","metadata":{"id":"KC8bRv_Gr3H9","execution":{"iopub.status.busy":"2023-03-16T11:47:08.902845Z","iopub.execute_input":"2023-03-16T11:47:08.904047Z","iopub.status.idle":"2023-03-16T11:47:08.910837Z","shell.execute_reply.started":"2023-03-16T11:47:08.904005Z","shell.execute_reply":"2023-03-16T11:47:08.909469Z"},"trusted":true},"execution_count":79,"outputs":[]},{"cell_type":"code","source":"translator.train_step([example_input_batch, example_target_batch])","metadata":{"id":"pLQZsX2dp1QK","outputId":"db5ec937-a3e4-40b2-ec54-9dea9c4f3519","execution":{"iopub.status.busy":"2023-03-16T11:47:08.912266Z","iopub.execute_input":"2023-03-16T11:47:08.912796Z","iopub.status.idle":"2023-03-16T11:47:15.710605Z","shell.execute_reply.started":"2023-03-16T11:47:08.912760Z","shell.execute_reply":"2023-03-16T11:47:15.709612Z"},"trusted":true},"execution_count":80,"outputs":[{"execution_count":80,"output_type":"execute_result","data":{"text/plain":"{'batch_loss': }"},"metadata":{}}]},{"cell_type":"code","source":"%%time\nfor n in range(10):\n print(translator.train_step([example_input_batch, example_target_batch]))\nprint()","metadata":{"id":"UzXXMwjXCqqh","outputId":"c536483a-a0b5-4c1b-b99b-3cc35a2caf48","execution":{"iopub.status.busy":"2023-03-16T11:47:15.712754Z","iopub.execute_input":"2023-03-16T11:47:15.713156Z","iopub.status.idle":"2023-03-16T11:47:19.076763Z","shell.execute_reply.started":"2023-03-16T11:47:15.713109Z","shell.execute_reply":"2023-03-16T11:47:19.075669Z"},"trusted":true},"execution_count":81,"outputs":[{"name":"stdout","text":"{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n{'batch_loss': }\n\nCPU times: user 4.32 s, sys: 389 ms, total: 4.71 s\nWall time: 3.36 s\n","output_type":"stream"}]},{"cell_type":"code","source":"losses = []\nfor n in range(100):\n print('.', end='')\n logs = translator.train_step([example_input_batch, example_target_batch])\n losses.append(logs['batch_loss'].numpy())\n\nprint()\nplt.plot(losses)","metadata":{"id":"U-dIWMIBqK7b","outputId":"6df08356-994b-48fa-b64c-ebcdae2c6291","execution":{"iopub.status.busy":"2023-03-16T11:47:19.087055Z","iopub.execute_input":"2023-03-16T11:47:19.087394Z","iopub.status.idle":"2023-03-16T11:47:51.493218Z","shell.execute_reply.started":"2023-03-16T11:47:19.087365Z","shell.execute_reply":"2023-03-16T11:47:51.491996Z"},"trusted":true},"execution_count":82,"outputs":[{"name":"stdout","text":"....................................................................................................\n","output_type":"stream"},{"execution_count":82,"output_type":"execute_result","data":{"text/plain":"[]"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"
","image/png":"iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABF3UlEQVR4nO3deVhU9f4H8PeZhRm2GVlEQBYFxQ1wARdcc+2mmV7LzL3FjLK09Vba79ZdTMvqWplmlmWblqaV5Z77ggqiIqiooOwgIgzrwMyc3x8ISaKyDJyZ4f16nnme6znfmfnM9/F23p7zXQRRFEUQERERSUQmdQFERETUsjGMEBERkaQYRoiIiEhSDCNEREQkKYYRIiIikhTDCBEREUmKYYSIiIgkxTBCREREklJIXUBdmEwmZGRkwNnZGYIgSF0OERER1YEoiigsLIS3tzdkstvf/7CKMJKRkQFfX1+pyyAiIqIGSE1NhY+Pz23PW0UYcXZ2BlD5YzQajcTVEBERUV3odDr4+vpWX8dvxyrCSNWjGY1GwzBCRERkZe42xIIDWImIiEhSDCNEREQkqUaFkUWLFkEQBDz//PN3bLdv3z6EhYVBrVYjICAAn376aWO+loiIiGxIg8PI8ePH8dlnnyE0NPSO7ZKTkzF69GgMGjQIsbGxmD9/PubOnYuffvqpoV9NRERENqRBYaSoqAhTp07FqlWr4OLicse2n376Kfz8/LB06VJ06dIFs2bNwuOPP4733nuvQQUTERGRbWlQGJkzZw7GjBmDESNG3LXtkSNHMGrUqBrH7r33XkRHR6OioqLW9+j1euh0uhovIiIisk31DiPr1q3DiRMnsGjRojq1z8rKQps2bWoca9OmDQwGA3Jzc2t9z6JFi6DVaqtfXPCMiIjIdtUrjKSmpmLevHn49ttvoVar6/y+v84vFkWx1uNVXn/9dRQUFFS/UlNT61MmERERWZF6LXoWExODnJwchIWFVR8zGo3Yv38/li1bBr1eD7lcXuM9np6eyMrKqnEsJycHCoUCbm5utX6PSqWCSqWqT2lERERkpeoVRoYPH464uLgaxx577DF07twZr7766i1BBAAiIiKwefPmGsd27NiB8PBwKJXKBpRMREREtqReYcTZ2RnBwcE1jjk6OsLNza36+Ouvv4709HR8/fXXAIDIyEgsW7YML774Ip588kkcOXIEX3zxBdauXWumn0BERETWzOwrsGZmZiIlJaX6z+3bt8eWLVuwd+9e9OjRA//5z3/w0Ucf4cEHHzT3VxMREZEVEsSq0aQWTKfTQavVoqCgwKwb5R1LzsPHuy/g/Ynd4aGp+4BcIiIiuru6Xr9b7N40oijinW3ncOBCLj7Zc1HqcoiIiFqsFhtGBEHAS6OCAADfH0tB2vUSiSsiIiJqmVpsGAGA/oHu6B/ohgqjiI//4N0RIiIiKbToMAIAL43qBADYcCINybnFEldDRETU8rT4MBLm74JhnT1gNIn4cFei1OUQERG1OC0+jADAiyMrx478cioDidmFEldDRETUsjCMAAhuq8V9wZ4QReCDHbw7QkRE1JwYRm54cWQQBAHYFp+FuLQCqcshIiJqMRhGbujYxhnje7QFACzZcR5WsBYcERGRTWAYucm84R2hkAnYn3gVXx66LHU5RERELQLDyE3auTtiwZguAIC3t5zF0aRrEldERERk+xhG/uLR/u0wroc3DCYRc76PRbauTOqSiIiIbBrDyF8IgoBFE0LQ2dMZuUV6PP1tDMoNJqnLIiIislkMI7VwsFNg5fQwaNQKnEjJx39+S5C6JCIiIpvFMHIb/m6OWPpIDwDAN1FX8NKPp3A+iwuiERERmZsgWsEcVp1OB61Wi4KCAmg0mmb97mW7L+C9mxZCGxzUGk8Oao8Bge7IKylHZn4ZMgtKUVBagRFd2sDF0a5Z6yMiIrJUdb1+M4zUQcyV6/j8QBK2x2fBdKO35DIBRlPNrhsT6oVPpvRq9vqIiIgsUV2v34pmrMlqhfm7IMw/DCnXSrD6UDJ+jE5FSbkRggC0dlKhtbMK8Rk67IzPRkFpBbT2SqlLJiIishq8M9IAxXoD8ksr4OGsglIugyiKuHfpfiRmF+GdB0Mwqbef1CUSERFJrq7Xbw5gbQBHlQJtW9lDKa/sPkEQMO7GUvI/x2ZIWRoREZHVYRgxk3E9vAEAUcnXkFlQKnE1RERE1oNhxEx8XBzQu50LRBH49STvjhAREdUVw4gZje9541ENwwgREVGdMYyY0ZgQLyjlAs5m6pCYzQXSiIiI6oJhxIxaOdhhSJAHAODn2HSJqyEiIrIODCNmNr5n5UDWX05mwGSy+FnTREREkmMYMbMRXdrASaVAen4poq9cl7ocIiIii8cwYmZqpRx/C/YEAPx8ko9qiIiI7oZhpAmMv7EA2u+nM7H7XDbKKowSV0RERGS5uDdNE4gIdIOnRo0sXRke/yoaaqUM/QPdMbSzBx7s1RYOdux2IiKiKrwz0gTkMgGrH+2Naf384K1Vo6zChN3ncvB/P5/BU9/ESF0eERGRReFGeU1MFEWczy7EH2dz8OGuCyg3mrA+MgK927lKXRoREVGT4kZ5FkIQBHT21GDO0A54MMwHAPDJnosSV0VERGQ5GEaaUeSQAMgEYO/5qziTXiB1OURERBaBYaQZ+bs5Ymz3ykXRVuy9JHE1REREloFhpJk9fU8gAGDLmUxculokcTVERETSYxhpZp09NRjRpQ1EEfiUd0eIiIgYRqTwzNDKuyObYtORnl8qcTVERETSYhiRQC8/F/QPdIPBJGLV/iSpyyEiIpIUw4hE5gztAABYeywFaddLJK6GiIhIOgwjEukf6Iaefq2gN5gwaWUULucWS10SERGRJBhGJCIIApZN6YV2bg5Izy/FxJVHcD6rUOqyiIiImh3DiITatrLHj5ER6OzpjKuFekz67AhOpeZLXRYREVGzYhiRmIezGutm90N331bIL6nA1M+P4mjSNanLIiIiajYMIxaglYMdvpvVFxEBbijSGzBrTTQXRCMiohaDYcRCOKkU+PKx3ujTzhWFegOeXBONgtIKqcsiIiJqcgwjFkStlGP5tF7w1qqRlFuMeetiYTSJUpdFRETUpBhGLIy7kwqfzQiHWinD3vNX8e72c1KXRERE1KQYRixQcFstljzUHQCwcl8SfjmZLnFFRERETYdhxEKN7e6NZ27s8PuPDadx+GKuxBURERE1DYYRC/bSqE4Y3tkDeoMJM1Yfw/dHU6QuiYiIyOwYRiyYXCbgk6m98EB3bxhMIuZvisO/NydwUCsREdkUhhELp1bK8eEjPfDSyCAAwOpDyXhizXEUlnHaLxER2QaGESsgCAKeG94Ry6f2qp5lM/HTI8gvKZe6NCIiokZjGLEio0O8sP6p/mjtrMK5rEI8+uVxFOsNUpdFRETUKAwjVibER4vvZvVFKwclTqbmY/Y30dAbjFKXRURE1GAMI1YoqI0zvnqsDxzt5Dh08Rrmro2FwWiSuiwiIqIGYRixUj18W2HVjHDYKWTYHp+N1zbGwcRZNkREZIUYRqxY/w7uWDa5J+QyARti0vDaxtO8Q0JERFaHYcTKjermiSUPhUImAD9Gp+Gpb2JQWs4xJEREZD0YRmzAhF4++HRaGFQKGf44l4Mpn0chr5jTfomIyDowjNiIUd088d2svtDaKxGbko+HPj2M1LwSqcsiIiK6K4YRGxLezhUbIiPgrVUj6WoxJn56hHdIiIjI4tUrjKxYsQKhoaHQaDTQaDSIiIjA1q1b7/ie7777Dt27d4eDgwO8vLzw2GOP4dq1a40qmm6vYxtn/PRMfwS4OyJLV4Z//nJG6pKIiIjuqF5hxMfHB4sXL0Z0dDSio6MxbNgwjBs3DvHx8bW2P3jwIGbMmIEnnngC8fHxWL9+PY4fP45Zs2aZpXiqnZfWHksf6QG5TMBvpzPx++lMqUsiIiK6rXqFkbFjx2L06NEICgpCUFAQFi5cCCcnJ0RFRdXaPioqCu3atcPcuXPRvn17DBw4EE899RSio6PNUjzdXqhPKzxzTyAA4P9+OYPcIr3EFREREdWuwWNGjEYj1q1bh+LiYkRERNTapn///khLS8OWLVsgiiKys7OxYcMGjBkz5o6frdfrodPparyo/p4b1hGdPZ2RV1yO//v5DESRi6IREZHlqXcYiYuLg5OTE1QqFSIjI7Fp0yZ07dq11rb9+/fHd999h0mTJsHOzg6enp5o1aoVPv744zt+x6JFi6DVaqtfvr6+9S2TANgpZHhvYncoZAK2nsnCb3xcQ0REFqjeYaRTp044efIkoqKi8PTTT2PmzJlISEiotW1CQgLmzp2Lf/7zn4iJicG2bduQnJyMyMjIO37H66+/joKCgupXampqfcukG4LbavHssA4AKh/X5BSWSVwRERFRTYLYyHv3I0aMQGBgIFauXHnLuenTp6OsrAzr16+vPnbw4EEMGjQIGRkZ8PLyqtN36HQ6aLVaFBQUQKPRNKbcFqnCaMK4ZYeQkKlDn3au+PqJPlAr5VKXRURENq6u1+9GrzMiiiL0+toHR5aUlEAmq/kVcrm8+n3UPJRyGZY+0gPOKgWOXc7DnO9OoIJ72BARkYWoVxiZP38+Dhw4gMuXLyMuLg4LFizA3r17MXXqVACVj1dmzJhR3X7s2LHYuHEjVqxYgaSkJBw6dAhz585Fnz594O3tbd5fQncU1MYZXzzau3rJ+H9sOM1dfomIyCIo6tM4Ozsb06dPR2ZmJrRaLUJDQ7Ft2zaMHDkSAJCZmYmUlJTq9o8++igKCwuxbNkyvPTSS2jVqhWGDRuGd955x7y/guqkT3tXLJ/aC7O/icGm2HRo7ZV4c2xXCIIgdWlERNSCNXrMSHPgmBHz+jk2Hc//cBIAMHtwACb0aot2bo4cR0JERGZV1+s3w0gLtebwZbz5658r5woC4OvigA4eTnh2WAf08nORsDoiIrIFdb1+1+sxDdmOmf3bwU4hw4/RqbiYU4TCMgNS8kqQkleCU6n52PPKPdColVKXSURELQDDSAs2uY8fJvfxgyiKyC0qx6WrRZi/KQ5JV4uxfM8lvHZfZ6lLJCKiFqDRU3vJ+gmCgNbOKvQLcMP8+7oAAFYfSkZqXonElRERUUvAMEI1DO/igYgAN5QbTFiy/bzU5RARUQvAMEI1CIKABWO6QBCAX09l4GRqvtQlERGRjWMYoVsEt9XiwV4+AID//pbA1XKJiKhJMYxQrV4e1Qn2Sjmir1zHtjNZUpdDREQ2jGGEauWpVePJwQEAgEVbz6GgtELiioiIyFYxjNBtPTU4AK2dVUjJK8E9S/ZgzeHL3GCPiIjMjmGEbstRpcDK6WHo4OGE6yUVePPXeNy7dD92JmRzHAkREZkNl4OnuzIYTVh7PBVLdybiWnE5AGBMiBc+fKQHFHLmWSIiql1dr9+8ktBdKeQyTO/njz2v3IOn7wmEnVyG3+My8dbmeN4hISKiRmMYoTrTqJV49W+d8eEjPSAIwLdRKVix75LUZRERkZVjGKF6uy/EC/83pisA4N1t5/HLyXSJKyIiImvGMEIN8vjA9nhiYHsAwMvrT+HwpVyJKyIiImvFMEINtmB0F4wO8USFUcRT38TgYk6R1CUREZEVYhihBpPJBHzwcA+E+7ugsMyAuWtjoTcYpS6LiIisDMMINYpaKcfyqb3g4qBEQqYO7+9IlLokIiKyMgwj1GgeGjXeeTAUAPDZ/iQcvMDxI0REVHcMI2QWo7p5YkpfPwDAS+tP4vqNxdGIiIjuhmGEzOaNMV0Q0NoR2To9Xtt4mguiERFRnTCMkNk42Cnw0SM9oZQL2B6fje+OpkhdEhERWQGGETKr4LZavDSqEwDgjZ/PYOrnUTiadE3iqoiIyJIppC6AbM/sQQHIKijDt1FXcOjiNRy6eA1927ti3vCOiAh0gyAIUpdIREQWhLv2UpNJu16CFXsv4cfoVFQYK/+aDe/sgbce6AZfVweJqyMioqZW1+s3wwg1uYz8Uny67xLWHktBhVGEWinD3OEdMWtgAOwUfFJIRGSrGEbI4lzMKcQbP59BVFIeAKCjhxMWTQhBeDtXiSsjIqKmUNfrN/9ZSs2mg4cz1j7ZDx883B1ujna4kFOERz7jAFciopaOYYSalSAImNDLB7tfugcju7aBwSTi6e9OIDWvROrSiIhIIgwjJAmtgxIfPdITIW21yCsux5NfR6NYb5C6LCIikgDDCEnG3k6Oz2aEobWzCueyCvHCDydhMln8ECYiIjIzhhGSlJfWHiunh8FOLsOOhGz8bxd3/SUiamkYRkhyvfxcsGhCCADg490XsSk2TeKKiIioOTGMkEV4MMwHswcHAABe+vEUAwkRUQvCMEIW47W/dcbkPr4wicCLP57Cj9GpUpdERETNgGGELIZMJmDh+BBM6+cHUQT+seE0vufOv0RENo8b5ZFFkckE/GdcMBQyGb46fBnzN8Uht0iPHr6t4KiSw1GlgNZeCS+tvdSlEhGRmTCMkMURBAFvju0KpVzAqgPJ+GDnrTNsHunti0UTQrgDMBGRDWAYIYskCALmj+4CT609tp3JRJHeiGK9AcV6A/JKyrHueCr6d3DHA929pS6ViIgaiRvlkdVZuisRS3ddgNZeiZ0vDIaHRi11SUREVAtulEc2a87QDghuq0FBaQVe3xgHK8jTRER0BwwjZHWUchnen9gDdnIZ/jiXgw0xXJOEiMiaMYyQVerk6YznR3YEAPx7cwIy8kslroiIiBqKYYSs1uxBAejh2wqFegNe/ek0H9cQEVkphhGyWgq5DO8/3B0qhQwHLuTin7/Ec9dfIiIrxDBCVi2wtRMW/j0EggB8E3UFr208DSMDCRGRVWEYIav3UJgP3p/YHTIB+DE6DS/+eBIGo0nqsoiIqI4YRsgmTOjlg48m94RCJuCXkxl4bm0syg0MJERE1oBhhGzG/aHeWD61F+zkMmw9k4VZX0ejoLRC6rKIiOguGEbIpozq5olVM8OhVsqwP/Eqxn9yCBdzCqUui4iI7oBhhGzOkKDW2BDZH95aNZJzizH+k8PYlZAtdVlERHQbDCNkk4LbavHrcwPRp50rivQGPPlNND7+4wKn/hIRWSCGEbJZ7k4qfDurL6b384coAu/vTMSYjw9iX+JVLpBGRGRBGEbIptkpZPjP+GC882AInNUKnM3UYebqY5j2xVHEpRVIXR4REQEQRCv4J2JdtyAmupPrxeX4ZM9FfH3kCspvrEMyOKg1BnZwQ0SAO7p6ayCXCRJXSURkO+p6/WYYoRYnNa8EH+xMxM8n03Hz336NWoF+AW6YN6IjunlrpSuQiMhGMIwQ3cWlq0XYcy4HRy5dw7HkPBTqDQAqH+28ObYrpvTxgyDwTgkRUUMxjBDVg8FoQnyGDh/9cQF/nMsBAIzt7o1FE0LgpFJIXB0RkXWq6/WbA1iJULkDcHffVlg1IxzzR3eGXCZg86kMjP34IBIydFKXR0Rk0xhGiG4ikwmYPTgQPz7V789F05YfwuqDyVyjhIioiTCMENUizN8Vv88dhOGdPVBuMOHfvyVg5pfHkKMrk7o0IiKbwzBCdBsujnb4fGY4/jM+GCqFDAcu5OLepfuxPT5L6tKIiGwKwwjRHQiCgOn9/PH73IHo5q3B9ZIKPPVNDD4/kCR1aURENqNeYWTFihUIDQ2FRqOBRqNBREQEtm7desf36PV6LFiwAP7+/lCpVAgMDMTq1asbVTRRc+vg4YxNzwzArIHtAQBLtp9HyrUSiasiIrIN9Zqz6OPjg8WLF6NDhw4AgDVr1mDcuHGIjY1Ft27dan3Pww8/jOzsbHzxxRfo0KEDcnJyYDAYGl85UTOzU8iwYEwXnM3S4dDFa3jz1zNY/WhvrkVCRNRIjV5nxNXVFUuWLMETTzxxy7lt27bhkUceQVJSElxdXRv8HVxnhCzJpatF+NvS/agwivh0Whj+FuwpdUlERBapydcZMRqNWLduHYqLixEREVFrm19//RXh4eF499130bZtWwQFBeHll19GaWnpHT9br9dDp9PVeBFZisDWTnhqcCAA4N+b41Gs550+IqLGqHcYiYuLg5OTE1QqFSIjI7Fp0yZ07dq11rZJSUk4ePAgzpw5g02bNmHp0qXYsGED5syZc8fvWLRoEbRabfXL19e3vmUSNalnh3WAr6s9MgrK8NEfF2qcyy3S48fjqThw4SoqbmzIR0REt1fvxzTl5eVISUlBfn4+fvrpJ3z++efYt29frYFk1KhROHDgALKysqDVVm48tnHjRjz00EMoLi6Gvb19rd+h1+uh1+ur/6zT6eDr68vHNGRRdp/LxuNfRUMhE/D73EEo0hvwzZHL2BKXVb0rsEatwPAubXBvtzYYHNQaDnZcWp6IWo5m25tmxIgRCAwMxMqVK285N3PmTBw6dAgXL16sPnb27Fl07doViYmJ6NixY52+g2NGyFI99U00tsdnw14pR2mFsfp4Vy8NcgrLkFtUXn3M3ckO6yP7o727oxSlEhE1u2bbm0YUxRp3MW42YMAAZGRkoKioqPpYYmIiZDIZfHx8GvvVRJL759hu1UHETiHDQ2E++PXZAdgybxCOzh+B9ZERmDWwPTw1auQWlWPZ7ot3/1AiohamXndG5s+fj/vuuw++vr4oLCzEunXrsHjxYmzbtg0jR47E66+/jvT0dHz99dcAgKKiInTp0gX9+vXDv/71L+Tm5mLWrFkYMmQIVq1aVecieWeELFnMlTyczSzEmBAvuDja1drmZGo+xn9yCAqZgH3/GIq2rWp/RElEZEua5M5IdnY2pk+fjk6dOmH48OE4evRodRABgMzMTKSkpFS3d3Jyws6dO5Gfn4/w8HBMnToVY8eOxUcffdTAn0VkecL8XTGtn/9tgwgA9PBthYgANxhMIr44kNyM1RERWb5GjxlpDrwzQrZgX+JVzFx9DPZKOQ6/NuyO4YWIyBY025gRIqqbwR3d0dVLg9IKI74+ckXqcoiILAbDCFEzEQQBkfdULpb21eFklJRzsTQiIoBhhKhZjQ72hK+rPa6XVODH46lSl0NEZBEYRoiakUIuw+wbS8mvOpBcvUJrWYURBy/kYn10KvKKy+/0EURENofLQRI1s4lhPvhwVyLS80vxyvpTyNKV4cSV/OpVW+2Vckzq7YsnBwdwCjARtQi8M0LUzNRKOR7t3w4A8PPJDEQl5aHcaIKnRo2gNk4orTDiq8OXMeTdPXjxh5NIulp05w8kIrJyvDNCJIGZ/dshPqNyN+r+HdwxINCtepn4gxdzsWLvJRy+dA0bY9OxIyEb62b3Q3BbrZQlExE1Ga4zQmShTqXm463N8YhNyYe7kx02RPZHO+5rQ0RWhOuMEFm57r6tsObxPujipUFuUTlmrD6GHF2Z1GUREZkdwwiRBdOolVjzeG/4uTogJa8EM788Dl1ZhdRlERGZFcMIkYXzcFbjmyf6wN1JhbOZOsxaE42yCqPUZRERmQ3DCJEV8HdzxJrHe8NZpcCx5Dz8a3O81CUREZkNwwiRlejmrcWKaWEQBGDtsVTsiM+SuiQiIrNgGCGyIgM7umP2oAAAwGsb45BTyAGtRGT9GEaIrMyLo4LQ1UuDvOJyvLL+NKxgdj4R0R0xjBBZGZVCjg8f6QGVQoZ9iVfx9ZEr1efyS8rx6b5LmLTyCA5cuCphlU2jpNyAzIJSqcsgIjPjCqxEVqhjG2e8fl9nvLU5AW9vOYs2GjX2JeZgU2w6yioq97i59MMp7Hl5CJzVSomrNZ+X15/CzoRsbJk7CB3bOEtdDhGZCe+MEFmpmf3bYUhQa+gNJkR+G4O1x1JRVmFCVy8NfFzskVukx9JdF6Qu02xMJhF7z19FhVHE/gu5UpdDRGbEMEJkpQRBwJKHQuHhrIJcJmBMiBfWR0bg97kDsfDvIQCArw5fRmJ2ocSVmseVvBKUlFeur3ImvUDiaojInPiYhsiKeWjU2PniEBhNIlwd7aqPDwlqjXu7tcH2+Gz885czWPtkPwiCIGGljXc2U1f9v+MYRohsCu+MEFk5rb2yRhCp8saYrlApZIhKysPm05kSVGZeCRl/hpFLV4tQpDdIWA0RmRPDCJGN8nV1wJyhHQAAC39PQLGVX7xvvjMiijXDCRFZN4YRIhs2e3AA/FwdkK3T46Pd1j2YNeFGGHF3UgHgoxoiW8IwQmTD1Eo53hzbFQDw2f4kzFpzHPsTr8Jksq6F0vJLypFZULna7IRebQEAcWn5ElZERObEMEJk44Z3aYPp/fwhisCuszmYsfoYRvxvH746lGw14y6q7or4uTogIsANAO+MENkShhGiFuA/44Ox+6UheLR/OzipFEi6Woy3Nidg0Du7sWLvJYsfT1I1PqSLlzOC22oBAEm5xVYTpojozhhGiFqIgNZOeOuBboiaPxz/GR+MAHdHXC+pwDvbzmHQu3vw2f5LKL2xjoelOZtZuVZKVy8tWjur4KVVQxSBeN4dIbIJDCNELYyTSoHp/fyx44XB+ODh7vB3c0BecTne3nIOg5fsweGLlre6adVjmi5elUvAV90d4aMaItvAMELUQinkMkzo5YM/XhyCJQ+FwtfVHlcL9Zj2xVEs33vRYga5lhtMuJhz486ItwYAEMowQmRTGEaIWjiFXIaJ4b7Y+cIQPBzuA5MIvLvtPGZ/E4OC0gqpy8PFnCJUGEVo1Aq0bWUPAAj2YRghsiUMI0QEoHIa8LsPdcfiCSGwU8iw62w2Hlh2EHFp0l7wqxY76+ylqV7SPuTGnZHk3GIUlt09MOkNRhgt5E4PEd2KYYSIanikjx9+iuyPtq3sceVaCcZ9chCvb4xDbpFeknqqxot09dJUH3N3UsG7ahDrXVZizS8px+B392Dq51FNWicRNRzDCBHdIsRHi9+eG4j7Q71gEoG1x1IwdMlerNx3CXpD8864OVtLGAH+HMR6tx189yVeRbZOj6ikPIt47EREt2IYIaJauTjaYdmUXlgfGYGQtloU6g1YtPUc7v3ffpzLap59YURR/PPOiHfNMBJSx0GsBy/8OTuoaiAsEVkWhhEiuqPe7Vzxy5wBeG9id3g4q3D5WgmmrDraLIEkS1eG/JIKyGUCOng41TgXUjWI9Q5jWkRRxMGbpiqfzypqmkKJqFEYRojormQyAQ+F+WDnC0MQ6qNFXnE5pqw6WmMn3aZQtfJqh9ZOUCvlNc6F3LQS6+0GsV66Wly9pw0AJGbzzgiRJWIYIaI60zoo8c0TfasDydTPmzaQnP3LYmc3c7sxiBW4/SDWgxeuAgBklZNwGEaILBTDCBHVi9a+ZiCZsiqqyab/Vi0D3+Uvg1er3O1RTdUjmlFdPQEAidl8TENkiRhGiKjebg4k10sqMHbZQYz56ACW7b5g1kGitxu8WuVOg1grjCZEJeUBAB4d0A4AkFukR15xudnqIyLzYBghogapCiQjunhAJlQ+KnlvRyJGfLAfIz7Yh6NJ1xr1+cV6Ay5fKwZw+zsjoT6tAACHLubessnfqdR8FOkNcHFQok87V/i4VK7eykc1RJaHYYSIGkxrr8TnM3vj+IIReOfBEAzt1Bp2chku5hRhxupj2H0uu8GffS6rEKIIeDir4O6kqrVNRKAb2rayx7Xicqw9llLj3IEbU3r7d3CHTCagU5vKcScXbhNGCssqoKvDaq5EZH4MI0TUaG5OKkzq7YcvH+uD6P8bgRFd2kBvMGH21zHYfCqj3p8niiI+3XcJABB6Y1xIbZRyGZ4ZGggAWLn/Esoq/rw7UjVeZFAHdwBAxxth5HwtYURvMOK+Dw/gvqUHbrnDQkRNj2GEiMxKo1ZixbReGNfDGwaTiLnrYrHuL3ct7uaH46nYmZANpVzACyOD7tj2oTAfeGnVyNbpsT4mDQCgK6vAydR8AMDAjpVhpJNn5ToltQ1ijbl8HWnXS5GeX4r9N2bgEFHzYRghIrNTymX44OEemNLXD6IIvLYxDkt3JdZpU7vk3GL8a3MCAODlUZ3Qzfv2d0YAQKWQI3JI5d2RFXsuotxgQtSlazCaRLR3d4SPiwMAoKNH5Z2RxOxCiGLNTfMO3LQw2vb4rLr/UCIyC4YRImoScpmAheOD8dSQAADA0l0X0GfhH3hl/SnEXMm7JRAAlTNgnv/hJEorjOgX4IpZgwLq9F2TevuitbMKGQVl2HgirfoRzcAbj2gAoIOHE2QCkF9Sgat/2fTv5iXj/zibgwqjqd6/l4gajmGEiJqMIAh47W+d8c6DIejg4YTSCiPWx6ThwRVHMPJ/+/HJnou4nFtc3X7Z7os4lZoPZ7UCHzzcA/Kq1cruQq2U46nBlcHlk70XsT+x8lFL1SOaqjb+bo4AgMSbloW/XlyOMxmVU4OdVQoUlFbgWHJe4344EdULwwgRNSlBEDCptx92vjAYPz0dgYlhPrBXynExpwhLtp/HPe/txf0fH8CirWexbM9FAMDCv4fAu5V9vb5nSl8/uDnaITWvFJevlUAuExAR6FajTVCbqnEjfw5iPXQpF6IIdGrjjNEhXgD4qIaouTGMEFGzEAQBYf6uWDKxO44tGI53HwzFoI7ukMsEnEnXYeW+JBhNIsb38MYD3b3r/fkOdooaj3W6+2ihUStrtAlq8+e4kSpVj2gGdnTHvcFtAFSGEZPp1sdIRNQ0FFIXQEQtj7NaiYd7++Lh3r64VqTH9vhsbInLhCAA/xoX3ODPnR7hj5X7LyG/pAIDO7a+5fxfw4goitXrkQzs6I6IADc42smRrdPjVFo+evq5NLgWIqo7hhEikpSbkwpT+vphSl+/Rn+Wk0qBheND8PWRy5jcx/eW80HVC58VQRRFJOcWIz2/FHZyGfq2d4VaKcfQzh747XQmtsdnM4wQNRM+piEimzIm1As/PBUBL+2tY07auztCIRNQqDcgs6CsetZNmL8LHOwq/212b7fKTfW2x2fVOuOHiMyPYYSIWgw7hQzt3Stn1JzPLqzxiKbKPTeWtE/OLcaFnLvv8nsqNb9O66cQ0e0xjBBRixLkWfmo5mymDlGXKjfzG3RTGHFWKzGgQ+UsnO1n7jyrZuOJNIz75BDe/CW+iaolahkYRoioRQm6sRLrhpg0FOoNaOWgvGWV178F33hUk3D7MGIwmvDRHxcAAHvO5/CRDlEjMIwQUYtStUdN0tXKxdYGBLrfsrjaiC5tIBOAM+k6pF0vqfVzfo/LxOVrleeul1Tg0tXiWtsR0d0xjBBRi1K1e2+Vm8eLVHFzUiG8nSsA4PfTmbecN5lEfHJjgTbhRo6JvsxVW4kaimGEiFoUf1cH2Cn+/E/fzfvX3Kxq4bUP/7iAizmFNc7tPJuNxOwiOKsUmN7PHwBw/PL1JqqYyPYxjBBRi6KQyxDYuvJRTXt3R/i6OtTabnIfP0QEuKGk3Iinvz2BknIDgMqF0pbtrrwrMqO/P4Z19gAARF/hnRGihmIYIaIWp/ONGTW3uysCVO46/OHkHvBwVuFCThHe2HQGoihi/4VcxKUXwF4px+MD2qOXvwsEAbhyrQQ5hWXN9ROIbArDCBG1OE/fE4gJvdrimaGBd2zn4azGx5N7Qi4TsDE2HeuOp2LZ7soZNFP6+sHNSQWNWonOnhoAQAwf1RA1CMMIEbU4QW2c8cHDPWpdpfWv+ga44eVRnQAA//fzGRy/fB12chlmD/5zU75w/8pl4zluhKhhGEaIiO7iqcEBGNHFA4YbO/lODPdBG426+nx4u8owwnEjRA3DMEJEdBcymYD3J/ZAOzcHOKkUiBxS8/FO7xvTgOMzdCjWG6Qokciq1SuMrFixAqGhodBoNNBoNIiIiMDWrVvr9N5Dhw5BoVCgR48eDamTiEhSWgclfp87CPv/MfSWGTjerezRtpU9jCYRJ1PzpSmQyIrVK4z4+Phg8eLFiI6ORnR0NIYNG4Zx48YhPv7O+zIUFBRgxowZGD58eKOKJSKSkqNKAVdHu1rPVT2qOc7Fz4jqrV5hZOzYsRg9ejSCgoIQFBSEhQsXwsnJCVFRUXd831NPPYUpU6YgIiKiUcUSEVmqqhVbY67cOojVaOK+NUR30uAxI0ajEevWrUNxcfEdQ8aXX36JS5cu4c0336zzZ+v1euh0uhovIiJLVjWj5sSV6zAYTdXHY65cR9+3d2HWmmiUVRilKo/IotU7jMTFxcHJyQkqlQqRkZHYtGkTunbtWmvbCxcu4LXXXsN3330HhUJR5+9YtGgRtFpt9cvX17e+ZRIRNaugNs5wVitQXG7EuazK5eOTc4sxa81x5BaVY9fZbMz57gQqbgoqRFSp3mGkU6dOOHnyJKKiovD0009j5syZSEhIuKWd0WjElClT8K9//QtBQUH1+o7XX38dBQUF1a/U1NT6lklE1KzkMgFh/n+OG8krLsdjXx7D9ZIKdPRwgkohwx/ncvD8Dyf52IboLwRRFBv1/4oRI0YgMDAQK1eurHE8Pz8fLi4ukMvl1cdMJhNEUYRcLseOHTswbNiwOn2HTqeDVqtFQUEBNBpNY8olImoyn+y5iCXbz2N4Zw9cLynHiZR8+LjYY9MzA3AmowCzv45GhVHExDAfvPNgKGQyQeqSiZpUXa/fdX92chuiKEKv199yXKPRIC4ursax5cuXY/fu3diwYQPat2/f2K8mIrIoVeNG/jiXAwDQqBX46rHeaO2swtBOHvh4ck/M+T4W62PS4KhS4M2xXSEIDCRE9Qoj8+fPx3333QdfX18UFhZi3bp12Lt3L7Zt2wag8vFKeno6vv76a8hkMgQHB9d4v4eHB9Rq9S3HiYhsQXffVlDKBVQYRdjJZfhsRjg6eDhXn/9bsBfem2jEiz+ewleHLyOwtSOmR7STrmAiC1GvMSPZ2dmYPn06OnXqhOHDh+Po0aPYtm0bRo4cCQDIzMxESkpKkxRKRGTp1Eo5BnVsDZkALJkYin4Bbre0+XtPH8y/rwsA4O0t53DlWnFzl0lkcRo9ZqQ5cMwIEVmL0nIj8krK0bbV7TfhM5lETP38KI4kXUPvdi5YNzsCco4fIRtU1+s396YhIjIjezv5HYMIULnXzbsPhcLRTo7jl6/jy0PJt7RJulqErw4lc68bahEYRoiIJODr6oD/u79yjaZ3t5/HxZzKtUkMRhOW772Iv314AG9tTsA/f7nzdhtEtoBhhIhIIpN6++KeTq1RbjDhpR9PIS6tAOOXH8K7286j3FC5ONrG2DQkZHAVarJtDCNERBIRBAGLJ4RCo1bgVFoBxi47iDPpOmjtlXh/YnfcH+oFUQQWbT0rdalETYphhIhIQp5aNf41rlv1n+8L9sTOFwfjwTAf/OPezlDKBRy4kIv9iVclrJKoaTV60TMiImqc8T3aQiGTwVmtwD2dPKqP+7k5YEZEO3xxMBlvbzmLAR3cOeuGbBLvjBARSUwQBIzt7l0jiFR5dmgHOKsVOJdViI0n0iSojqjpMYwQEVkwF0c7PDu0AwDg/R2JKKswSlwRkfkxjBARWbiZ/duhbSt7ZOnK8MXBW9ckIbJ2DCNERBZOrZTj5XuDAABLtp/HpJVHsD46lQuikc3gcvBERFbAZBLxyobT2Bibhqr/ajvYyfG3YE/08G0FX1cH+Lk6oG0re6iVcmmLJbqhrtdvhhEiIiuSkV+KTbHp2BCThuTc2jfZ6x/ohpXTw+CsVjZzdUQ1MYwQEdkwURRxIuU6tsdn43JuMVLySpCaV4Li8soBrrMHB2D+6C4SV0ktXV2v31xnhIjICgmCgDB/V4T5u1YfE0URu87m4Mmvo7H6YDIeDvdFBw8nCaskqhsOYCUishGCIGBk1zYY0cUDBpOIf22OhxXc/CZiGCEisjVvjOkKO7kMBy7kYkdCttTlEN0VwwgRkY1p5+6IJwe3BwD857eEGgulVRhNWHcsBR/sOM8F1MhicMwIEZENmjO0AzaeSEfa9VKs3JeEucM7YEtcFt7bcb56Fk6h3oA3x3a7yycRNT3OpiEislGbT2XgubWxUClk6OTpjNNpBQCAVg5K5JdUAAC+frwPBge1lrJMsmF1vX7zMQ0RkY26P9QL/QJcoTeYcDqtAI52cjw/oiMOvjoM0/v5AwBeXn8K14vLJa6UWjqGESIiGyUIAt7+ewjC/V3waP922PePoXh+RBCcVArMH90FAa0dkVOox/xNcZx1Q5LiYxoiohYqLq0Af19+CAaTiPcmdsdDYT5Sl0Q2ho9piIjojkJ8tHhhZOUGfG/+cgYp10okrohaKoYRIqIWLHJIIML9XVBcbsTkVVHYGpfJRzbU7BhGiIhaMLlMwP8m9UDbVvZIzy/F09+dwORVUTibqZO6NGpBOGaEiIhQUm7Ap/uSsHLfJegNJsgEYFJvXzzYywc9fFtBIee/Xan+uGsvERHVW9r1Eizaeg6/n86sPqa1V2JQR3cM7eSBUd3awFmtlLBCsiYMI0RE1GBHk67h26Mp2J94FQWlFdXH/d0c8MucAWjlYCdhdWQt6nr95nLwRER0i74Bbugb4AaD0YRTafnYc+4q1sek4sq1Erz04ymsmhEOmUyQukyyEXwISEREt6WQyxDm74qX7+2E1Y/2hp1Chj/O5eDT/ZekLo1sCMMIERHVSTdvLf79QOXGeu9tP48jl65JXBHZCoYRIiKqs6oZNiYReG5tLHJ0ZVKXRDaAY0aIiKjOBEHAf8cHIz6jAOeyCvHs97GYHuGP/JJyXC+pQH5JBbp4OWNiuK/UpZIVYRghIqJ6sbeTY/nUXnhg2SEcu5yHY5fzbmljEkVM6u0nQXVkjRhGiIio3gJaO+HjyT2xdFci1Eo5XBzs0MpBiYLSCmw9k4U3fj6Ddm6O6BvgJnWpZAW4zggREZmNySTiubWx+D0uE66OdvhlzgD4ujpIXRZJhLv2EhFRs5PJBLw3sTtC2mqRV1yOJ9YcR2FZxd3fSC0awwgREZmVvZ0cq2aEw8NZhcTsIsxbdxJGk8XfhCcJ8TENERE1iVOp+Xh45RHoDSa4O9mhb3s39A1wRb8AN3T0cIIgcAVXW8e9aYiISHJb4zLx8vpTKC431jge4O6IVTPDEdjaSaLKqDkwjBARkUXQG4w4lVqAqKRrOJp8DTFXrqOswgQ3RzusebwPgttqpS6RmgjDCBERWaRrRXrMWH0M8Rk6OKsU+OLR3ujT3lXqsqgJcDYNERFZJDcnFdbO7oc+7VxRqDdgxuqj2HM+R+qySEIMI0RE1Ow0aiXWPN4HQzu1RlmFCU+uicYXB5NRbjBJXRpJgGGEiIgkYW8nx2czwvFAd28YTCL+81sCRv5vHzafyoCJU4FbFIYRIiKSjFIuw9JJPfDf8cFwd1LhyrUSPLc2FuM+OYTDF3OlLo+aCQewEhGRRSjWG/DFwWSs3Hepeirwk4Pa49W/dYZCzn87WyMOYCUiIqviqFJg7vCO2PePoZjWr3LH31UHkjH186PIKSyTuDpqSgwjRERkUdydVPjv+BCsmNoLTioFjibn4f6PDiL6cp7UpVETYRghIiKLdF+IF355dgA6ejghp1CPRz6LwrLdF1BWYbz7m8mqMIwQEZHFCmzthJ/nDMD9oV4wmES8tyMRw9+vnHFjBUMeqY4YRoiIyKI5qhT4eHJPfPBwd3hq1EjPL8Vza2MxYcVhxFy5LnV5ZAacTUNERFajtNyIVQeS8Om+Syi5MePmqcEBePneTlByxo3F4WwaIiKyOfZ2cswd3hF7X74HE8N8AAAr9ydhyqooZBVwxo21YhghIiKr46FRY8nE7lh+Y8bN8cvXMfqjAzhw4arUpVEDMIwQEZHVGh3ihd+eG4iuXhrkFZdjxupjWLT1LIr0BqlLo3pgGCEiIqvWzt0RG5/pj8l9/CCKwMp9SbhnyR58d/QKDEZuvGcNOICViIhsxo74LLy95SwuXysBAHT0cML80V0wtLOHxJW1THW9fjOMEBGRTSk3mPD90Sv48I8LuF5SAQCYFO6Lf4/vBpVCLnF1LQtn0xARUYtkp5Dh0QHtsfeVoZg9OAAyAfghOhWTP4tCjo4zbiwRwwgREdkkrb0S80d3wZeP9YFGrcCJlHyMXXYQsSlcKM3SMIwQEZFNGxLUGr8+OxAdPZyQrdNj0soorI9OlbosugnDCBER2bx27o7YNGcARnZtg3KjCa9sOI2FvyfAaLL4YZMtQr3CyIoVKxAaGgqNRgONRoOIiAhs3br1tu03btyIkSNHonXr1tXtt2/f3uiiiYiI6stJpcDKaWGYO6wDAGDVgWTMWnMchWUVEldG9QojPj4+WLx4MaKjoxEdHY1hw4Zh3LhxiI+Pr7X9/v37MXLkSGzZsgUxMTEYOnQoxo4di9jYWLMUT0REVB8ymYAXR3XCx5N7QqWQYc/5q5iw/DBSbkwFJmk0emqvq6srlixZgieeeKJO7bt164ZJkybhn//8Z52/g1N7iYjI3E6n5ePJr6ORrdPDxUGJT6eFoW+Am9Rl2ZQmn9prNBqxbt06FBcXIyIiok7vMZlMKCwshKura0O/loiIyCxCfVrhlzkDEeqjxfWSCkz74ig2xaZJXVaLVO8wEhcXBycnJ6hUKkRGRmLTpk3o2rVrnd77/vvvo7i4GA8//PAd2+n1euh0uhovIiIic/PUqvHjUxEYHeKJCqOIF344haW7EmEF64HalHqHkU6dOuHkyZOIiorC008/jZkzZyIhIeGu71u7di3eeust/PDDD/DwuPOyvIsWLYJWq61++fr61rdMIiKiOlEr5Vg2uRcihwQCAJbuuoCXfjwFvcEocWUtR6PHjIwYMQKBgYFYuXLlbdv88MMPeOyxx7B+/XqMGTPmrp+p1+uh1+ur/6zT6eDr68sxI0RE1KTWHkvBGz+fgdEkItzfBQv/HoJOns5Sl2W1mm05eFEUawSHv1q7di0effRRfP/993UKIgCgUqmqpw9XvYiIiJra5D5++Oqx3nBWKRB95Tru+3A/FmyKQ27R7a9z1Hj1CiPz58/HgQMHcPnyZcTFxWHBggXYu3cvpk6dCgB4/fXXMWPGjOr2a9euxYwZM/D++++jX79+yMrKQlZWFgoKCsz7K4iIiMxkUMfW+H3uINwX7AmTCHx3NAX3LNmLFXsvoayCj26aQr3CSHZ2NqZPn45OnTph+PDhOHr0KLZt24aRI0cCADIzM5GSklLdfuXKlTAYDJgzZw68vLyqX/PmzTPvryAiIjIjPzcHrJgWhh9m90NIWy2K9Aa8s+0cxn58EGfS+Q9qc2v0mJHmwHVGiIhIKiaTiJ9PpuPtLeeQW6SHQibg+REdETkkEAo5d1W5k2YbM0JERGTLZDIBE3r5YMcLg3FfsCcMJhHv7UjEwyuP4HJusdTl2QSGESIiojpwdbTD8qm98MHD3eGsUuBESj5G/m8fZq2Jxq+nMlBSbmjQ567Yewmhb23H4Yu5Zq7YevAxDRERUT2l55fi1Q2ncfCmAGGvlGNE1zYYE+KFezq1hlopv+vnbIhJw8vrTwEAwv1dsOHp/k1WsxTqev1mGCEiImqg81mF2HwqA7+eykBK3p+b7TnYyTG0swdGB1cGE0eV4pb3HrqYi5mrj8Fg+vMyvOmZ/ujp59IstTcHhhEiIqJmIooiTqcVYPOpDGw9k4X0/NLqc/ZKOab188PswYFo7awCACRmF+LBFYdRWGbA2O7eUMoEbIxNx/2hXlg2pZdUP8PsGEaIiIgkUBVMtp7JwtYzmbhyrfKOiVopw9S+/pjQqy1mfx2D9PxS9G7ngm+e6Iukq8UY/dEByGUC9r1yD3xcHCT+FebBMEJERCQxURSxN/EqPtx1ASdT82uca+/uiI1P94eLox0AYOrnUTh08RqeHNQeC8bUbQNaS8epvURERBITBAFDO3lg0zP9sebxPujl1woA4OKgxJeP9q4OIgAwa2AAAGDdsVQUllVIUa5kbh1RQ0RERGYlCAKGBLXG4I7uOJ1WgDYaNTy16hpthgS1RmBrR1y6Wowfo9PwxMD2ElXb/HhnhIiIqJkIgoDuvq1uCSJA5eJqT9y4O/LloWQYjKbmLk8yDCNEREQWYkKvtnBxUCLteil2JGRLXU6zYRghIiKyEGqlHNP7+QMAVh1Ikria5sMwQkREZEGmRfjDTi5DbEo+TqRcl7qcZsEwQkREZEE8nNUY290bALD6YLLE1TQPhhEiIiIL8/jAdgCArWeykHHTaq62imGEiIjIwnTz1qJfgCuMJhFrjlyWupwmxzBCRERkgaqm+a49moKScoPE1TQthhEiIiILNLyzB/zdHKArM+CnmDSpy2lSDCNEREQWSCYT8Fj/dgCA1Ycuw2Sy+K3kGoxhhIiIyEJNDPeFs1qB5Nxi7E3MkbqcJsMwQkREZKEcVQo80tsXAPCFDU/zZRghIiKyYDP7t4NMAA5dvIbTaflSl9MkGEaIiIgsmI+LA+4L9gIATFh+GHPXxuJUar60RZmZQuoCiIiI6M7euL8LrhXrEZWUh19PZeDXUxkI93fBuB7ecFYroVbKYW8nh4OdHO5OKnhq1LC3k0tddp0Joiha/PBcnU4HrVaLgoICaDQaqcshIiKSxJn0Aqw+mIzNpzNQYbzz5Vtrr4SXVo2u3hq8/fcQqJXNH07qev1mGCEiIrIyOboyfHs0BQkZBSirMKG0wojSciNKyg3IKdSjpNxYo/3Cvwdjal//Zq+zrtdvPqYhIiKyMh4aNV4cGVTrOVEUUag3IKugDD/FpGHl/iR8c+QKpvTxgyAIzVxp3XAAKxERkQ0RBAEatRJBbZzxzD0doFbKcC6rENFXrktd2m0xjBAREdkorYMS47q3BQB8c+SKxNXcHsMIERGRDZseUTlWZOuZTFwt1EtcTe0YRoiIiGxYcFstevq1QoVRxA/HU6Qup1YMI0RERDZuer/KuyPfH02BwWiSuJpbMYwQERHZuNEhXnB1tENGQRn+OGd5G+4xjBAREdk4tVKOh8MrN9z7NsryBrIyjBAREbUAU/v6QRCAAxdykXS1SOpyamAYISIiagF8XR0wrJMHAODj3RctauwIwwgREVEL8diA9gCATbHpeOSzKKTmlUhcUSWGESIiohZiYEd3/G9SdzipFIi+ch33fXgAP8WkQept6hhGiIiIWpC/9/TB1nmDEO7vgiK9AS+tP4Vn18aioKRCspoYRoiIiFoYX1cHrJvdDy+PCoJCJuD305n44mCSZPVw114iIqIWSCGX4dlhHTGoY2ss33sRzwztIF0tkn0zERERSa67byusnB4uaQ18TENERESSYhghIiIiSTGMEBERkaQYRoiIiEhSDCNEREQkKYYRIiIikhTDCBEREUmKYYSIiIgkxTBCREREkmIYISIiIkkxjBAREZGkGEaIiIhIUgwjREREJCmr2LVXFEUAgE6nk7gSIiIiqquq63bVdfx2rCKMFBYWAgB8fX0lroSIiIjqq7CwEFqt9rbnBfFuccUCmEwmZGRkwNnZGYIgmO1zdTodfH19kZqaCo1GY7bPpVuxr5sX+7v5sK+bD/u6+Zirr0VRRGFhIby9vSGT3X5kiFXcGZHJZPDx8Wmyz9doNPyL3UzY182L/d182NfNh33dfMzR13e6I1KFA1iJiIhIUgwjREREJKkWHUZUKhXefPNNqFQqqUuxeezr5sX+bj7s6+bDvm4+zd3XVjGAlYiIiGxXi74zQkRERNJjGCEiIiJJMYwQERGRpBhGiIiISFItOowsX74c7du3h1qtRlhYGA4cOCB1SVZv0aJF6N27N5ydneHh4YHx48fj/PnzNdqIooi33noL3t7esLe3xz333IP4+HiJKrYNixYtgiAIeP7556uPsZ/NKz09HdOmTYObmxscHBzQo0cPxMTEVJ9nf5uHwWDAG2+8gfbt28Pe3h4BAQH497//DZPJVN2Gfd0w+/fvx9ixY+Ht7Q1BEPDzzz/XOF+XftXr9Xjuuefg7u4OR0dHPPDAA0hLS2t8cWILtW7dOlGpVIqrVq0SExISxHnz5omOjo7ilStXpC7Nqt17773il19+KZ45c0Y8efKkOGbMGNHPz08sKiqqbrN48WLR2dlZ/Omnn8S4uDhx0qRJopeXl6jT6SSs3HodO3ZMbNeunRgaGirOmzev+jj72Xzy8vJEf39/8dFHHxWPHj0qJicni7t27RIvXrxY3Yb9bR7//e9/RTc3N/G3334Tk5OTxfXr14tOTk7i0qVLq9uwrxtmy5Yt4oIFC8SffvpJBCBu2rSpxvm69GtkZKTYtm1bcefOneKJEyfEoUOHit27dxcNBkOjamuxYaRPnz5iZGRkjWOdO3cWX3vtNYkqsk05OTkiAHHfvn2iKIqiyWQSPT09xcWLF1e3KSsrE7Varfjpp59KVabVKiwsFDt27Cju3LlTHDJkSHUYYT+b16uvvioOHDjwtufZ3+YzZswY8fHHH69xbMKECeK0adNEUWRfm8tfw0hd+jU/P19UKpXiunXrqtukp6eLMplM3LZtW6PqaZGPacrLyxETE4NRo0bVOD5q1CgcPnxYoqpsU0FBAQDA1dUVAJCcnIysrKwafa9SqTBkyBD2fQPMmTMHY8aMwYgRI2ocZz+b16+//orw8HBMnDgRHh4e6NmzJ1atWlV9nv1tPgMHDsQff/yBxMREAMCpU6dw8OBBjB49GgD7uqnUpV9jYmJQUVFRo423tzeCg4Mb3fdWsVGeueXm5sJoNKJNmzY1jrdp0wZZWVkSVWV7RFHEiy++iIEDByI4OBgAqvu3tr6/cuVKs9dozdatW4cTJ07g+PHjt5xjP5tXUlISVqxYgRdffBHz58/HsWPHMHfuXKhUKsyYMYP9bUavvvoqCgoK0LlzZ8jlchiNRixcuBCTJ08GwL/bTaUu/ZqVlQU7Ozu4uLjc0qax184WGUaqCIJQ48+iKN5yjBru2WefxenTp3Hw4MFbzrHvGyc1NRXz5s3Djh07oFarb9uO/WweJpMJ4eHhePvttwEAPXv2RHx8PFasWIEZM2ZUt2N/N94PP/yAb7/9Ft9//z26deuGkydP4vnnn4e3tzdmzpxZ3Y593TQa0q/m6PsW+ZjG3d0dcrn8liSXk5NzSyqkhnnuuefw66+/Ys+ePfDx8ak+7unpCQDs+0aKiYlBTk4OwsLCoFAooFAosG/fPnz00UdQKBTVfcl+Ng8vLy907dq1xrEuXbogJSUFAP9em9Mrr7yC1157DY888ghCQkIwffp0vPDCC1i0aBEA9nVTqUu/enp6ory8HNevX79tm4ZqkWHEzs4OYWFh2LlzZ43jO3fuRP/+/SWqyjaIoohnn30WGzduxO7du9G+ffsa59u3bw9PT88afV9eXo59+/ax7+th+PDhiIuLw8mTJ6tf4eHhmDp1Kk6ePImAgAD2sxkNGDDglinqiYmJ8Pf3B8C/1+ZUUlICmazmpUkul1dP7WVfN4269GtYWBiUSmWNNpmZmThz5kzj+75Rw1+tWNXU3i+++EJMSEgQn3/+edHR0VG8fPmy1KVZtaefflrUarXi3r17xczMzOpXSUlJdZvFixeLWq1W3LhxoxgXFydOnjyZ0/LM4ObZNKLIfjanY8eOiQqFQly4cKF44cIF8bvvvhMdHBzEb7/9troN+9s8Zs6cKbZt27Z6au/GjRtFd3d38R//+Ed1G/Z1wxQWFoqxsbFibGysCED84IMPxNjY2OolLerSr5GRkaKPj4+4a9cu8cSJE+KwYcM4tbexPvnkE9Hf31+0s7MTe/XqVT39lBoOQK2vL7/8srqNyWQS33zzTdHT01NUqVTi4MGDxbi4OOmKthF/DSPsZ/PavHmzGBwcLKpUKrFz587iZ599VuM8+9s8dDqdOG/ePNHPz09Uq9ViQECAuGDBAlGv11e3YV83zJ49e2r97/PMmTNFUaxbv5aWlorPPvus6OrqKtrb24v333+/mJKS0ujaBFEUxcbdWyEiIiJquBY5ZoSIiIgsB8MIERERSYphhIiIiCTFMEJERESSYhghIiIiSTGMEBERkaQYRoiIiEhSDCNEREQkKYYRIiIikhTDCBEREUmKYYSIiIgkxTBCREREkvp/xdr/sdiC+N8AAAAASUVORK5CYII=\n"},"metadata":{}}]},{"cell_type":"code","source":"# building a fresh copy of the model to train from scratch since the training step is working\ntrain_translator = TrainTranslator(\n embedding_dim, units,\n input_text_processor=input_text_processor,\n output_text_processor=output_text_processor)\n\n# Configure the loss and optimizer\ntrain_translator.compile(\n optimizer=tf.optimizers.Adam(),\n loss=MaskedLoss(),\n)","metadata":{"id":"Emgfgh4tAmJt","execution":{"iopub.status.busy":"2023-03-16T11:47:51.494792Z","iopub.execute_input":"2023-03-16T11:47:51.495166Z","iopub.status.idle":"2023-03-16T11:47:51.524712Z","shell.execute_reply.started":"2023-03-16T11:47:51.495127Z","shell.execute_reply":"2023-03-16T11:47:51.523701Z"},"trusted":true},"execution_count":83,"outputs":[]},{"cell_type":"markdown","source":"Training the model","metadata":{"id":"1h7Dmiy1De8S"}},{"cell_type":"code","source":"class BatchLogs(tf.keras.callbacks.Callback):\n def __init__(self, key):\n self.key = key\n self.logs = []\n\n def on_train_batch_end(self, n, logs):\n self.logs.append(logs[self.key])\n\nbatch_loss = BatchLogs('batch_loss')","metadata":{"id":"J7m4mtnj80sq","execution":{"iopub.status.busy":"2023-03-16T11:47:51.526321Z","iopub.execute_input":"2023-03-16T11:47:51.526708Z","iopub.status.idle":"2023-03-16T11:47:51.532735Z","shell.execute_reply.started":"2023-03-16T11:47:51.526670Z","shell.execute_reply":"2023-03-16T11:47:51.531558Z"},"trusted":true},"execution_count":84,"outputs":[]},{"cell_type":"code","source":"train_translator.fit(dataset, epochs=14,\n callbacks=[batch_loss])","metadata":{"id":"BQd_esVVoSf3","outputId":"06c29d9e-195b-46a6-a401-92eb33ccdc94","execution":{"iopub.status.busy":"2023-03-16T11:47:51.534342Z","iopub.execute_input":"2023-03-16T11:47:51.535032Z","iopub.status.idle":"2023-03-16T15:34:16.598152Z","shell.execute_reply.started":"2023-03-16T11:47:51.534976Z","shell.execute_reply":"2023-03-16T15:34:16.597041Z"},"trusted":true},"execution_count":85,"outputs":[{"name":"stdout","text":"Epoch 1/14\n2820/2820 [==============================] - 989s 348ms/step - batch_loss: 3.3863\nEpoch 2/14\n2820/2820 [==============================] - 961s 341ms/step - batch_loss: 1.3731\nEpoch 3/14\n2820/2820 [==============================] - 958s 340ms/step - batch_loss: 0.9362\nEpoch 4/14\n2820/2820 [==============================] - 969s 343ms/step - batch_loss: 0.7388\nEpoch 5/14\n2820/2820 [==============================] - 959s 340ms/step - batch_loss: 0.6408\nEpoch 6/14\n2820/2820 [==============================] - 957s 339ms/step - batch_loss: 0.5801\nEpoch 7/14\n2820/2820 [==============================] - 956s 339ms/step - batch_loss: 0.5346\nEpoch 8/14\n2820/2820 [==============================] - 961s 341ms/step - batch_loss: 0.5017\nEpoch 9/14\n2820/2820 [==============================] - 956s 339ms/step - batch_loss: 0.4785\nEpoch 10/14\n2820/2820 [==============================] - 957s 339ms/step - batch_loss: 0.4618\nEpoch 11/14\n2820/2820 [==============================] - 953s 338ms/step - batch_loss: 0.4497\nEpoch 12/14\n2820/2820 [==============================] - 952s 337ms/step - batch_loss: 0.4415\nEpoch 13/14\n2820/2820 [==============================] - 962s 341ms/step - batch_loss: 0.4363\nEpoch 14/14\n2820/2820 [==============================] - 955s 339ms/step - batch_loss: 0.4364\n","output_type":"stream"},{"execution_count":85,"output_type":"execute_result","data":{"text/plain":""},"metadata":{}}]},{"cell_type":"code","source":"df = df.sample(frac = 1)\ninp=list(df['Sentence'])\ntarg=list(df['trr'])\nBUFFER_SIZE = len(inp)\nBATCH_SIZE = 128\ndataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\ndataset = dataset.batch(BATCH_SIZE)\ntrain_translator.fit(dataset, epochs=3,callbacks=[batch_loss])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T15:40:53.225716Z","iopub.execute_input":"2023-03-16T15:40:53.226111Z","iopub.status.idle":"2023-03-16T16:10:40.299151Z","shell.execute_reply.started":"2023-03-16T15:40:53.226076Z","shell.execute_reply":"2023-03-16T16:10:40.298062Z"},"trusted":true},"execution_count":101,"outputs":[{"name":"stdout","text":"Epoch 1/3\n1410/1410 [==============================] - 592s 420ms/step - batch_loss: 0.3202\nEpoch 2/3\n1410/1410 [==============================] - 575s 408ms/step - batch_loss: 0.2631\nEpoch 3/3\n1410/1410 [==============================] - 571s 405ms/step - batch_loss: 0.2652\n","output_type":"stream"},{"execution_count":101,"output_type":"execute_result","data":{"text/plain":""},"metadata":{}}]},{"cell_type":"code","source":"df = df.sample(frac = 1)\ninp=list(df['Sentence'])\ntarg=list(df['trr'])\nBUFFER_SIZE = len(inp)\nBATCH_SIZE = 256\ndataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\ndataset = dataset.batch(BATCH_SIZE)\ntrain_translator.fit(dataset, epochs=3,callbacks=[batch_loss])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T16:42:20.557205Z","iopub.execute_input":"2023-03-16T16:42:20.557606Z","iopub.status.idle":"2023-03-16T17:03:02.329620Z","shell.execute_reply.started":"2023-03-16T16:42:20.557572Z","shell.execute_reply":"2023-03-16T17:03:02.328512Z"},"trusted":true},"execution_count":118,"outputs":[{"name":"stdout","text":"Epoch 1/3\n705/705 [==============================] - 424s 600ms/step - batch_loss: 0.1966\nEpoch 2/3\n705/705 [==============================] - 409s 580ms/step - batch_loss: 0.1409\nEpoch 3/3\n705/705 [==============================] - 408s 579ms/step - batch_loss: 0.1234\n","output_type":"stream"},{"execution_count":118,"output_type":"execute_result","data":{"text/plain":""},"metadata":{}}]},{"cell_type":"code","source":"df = df.sample(frac = 1)\ninp=list(df['Sentence'])\ntarg=list(df['trr'])\nBUFFER_SIZE = len(inp)\nBATCH_SIZE = 512\ndataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\ndataset = dataset.batch(BATCH_SIZE)\ntrain_translator.fit(dataset, epochs=3,callbacks=[batch_loss])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:04:42.238316Z","iopub.execute_input":"2023-03-16T17:04:42.239043Z","iopub.status.idle":"2023-03-16T17:22:34.668307Z","shell.execute_reply.started":"2023-03-16T17:04:42.238994Z","shell.execute_reply":"2023-03-16T17:22:34.667065Z"},"trusted":true},"execution_count":131,"outputs":[{"name":"stdout","text":"Epoch 1/3\n353/353 [==============================] - 351s 993ms/step - batch_loss: 0.0902\nEpoch 2/3\n353/353 [==============================] - 339s 959ms/step - batch_loss: 0.0665\nEpoch 3/3\n353/353 [==============================] - 337s 953ms/step - batch_loss: 0.0593\n","output_type":"stream"},{"execution_count":131,"output_type":"execute_result","data":{"text/plain":""},"metadata":{}}]},{"cell_type":"code","source":"df = df.sample(frac = 1)\ninp=list(df['Sentence'])\ntarg=list(df['trr'])\nBUFFER_SIZE = len(inp)\nBATCH_SIZE = 1024\ndataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\ndataset = dataset.batch(BATCH_SIZE)\ntrain_translator.fit(dataset, epochs=3,callbacks=[batch_loss])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:23:44.856932Z","iopub.execute_input":"2023-03-16T17:23:44.857689Z","iopub.status.idle":"2023-03-16T17:39:40.716749Z","shell.execute_reply.started":"2023-03-16T17:23:44.857648Z","shell.execute_reply":"2023-03-16T17:39:40.715750Z"},"trusted":true},"execution_count":147,"outputs":[{"name":"stdout","text":"Epoch 1/3\n177/177 [==============================] - 326s 2s/step - batch_loss: 0.0480\nEpoch 2/3\n177/177 [==============================] - 314s 2s/step - batch_loss: 0.0421\nEpoch 3/3\n177/177 [==============================] - 314s 2s/step - batch_loss: 0.0405\n","output_type":"stream"},{"execution_count":147,"output_type":"execute_result","data":{"text/plain":""},"metadata":{}}]},{"cell_type":"code","source":"df = df.sample(frac = 1)\ninp=list(df['Sentence'])\ntarg=list(df['trr'])\nBUFFER_SIZE = len(inp)\nBATCH_SIZE = 1536\ndataset = tf.data.Dataset.from_tensor_slices((inp, targ)).shuffle(BUFFER_SIZE)\ndataset = dataset.batch(BATCH_SIZE)\ntrain_translator.fit(dataset, epochs=3,callbacks=[batch_loss])","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:43:00.227000Z","iopub.execute_input":"2023-03-16T17:43:00.227861Z","iopub.status.idle":"2023-03-16T17:43:28.536846Z","shell.execute_reply.started":"2023-03-16T17:43:00.227811Z","shell.execute_reply":"2023-03-16T17:43:28.534001Z"},"trusted":true},"execution_count":163,"outputs":[{"name":"stdout","text":"Epoch 1/3\n 6/118 [>.............................] - ETA: 4:50 - batch_loss: 0.0351","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mResourceExhaustedError\u001b[0m Traceback (most recent call last)","\u001b[0;32m/tmp/ipykernel_23/188685183.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_tensor_slices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBUFFER_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mtrain_translator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbatch_loss\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0;31m# To get the full stack trace, call:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0;31m# `tf.debugging.disable_traceback_filtering()`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 70\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfiltered_tb\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 71\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 72\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0mfiltered_tb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m 51\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 52\u001b[0m tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 53\u001b[0;31m inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m 54\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mResourceExhaustedError\u001b[0m: Graph execution error:\n\nDetected at node 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul' defined at (most recent call last):\n File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\n \"__main__\", mod_spec)\n File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\n exec(code, run_globals)\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py\", line 17, in \n app.launch_new_instance()\n File \"/opt/conda/lib/python3.7/site-packages/traitlets/config/application.py\", line 1041, in launch_instance\n app.start()\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelapp.py\", line 712, in start\n self.io_loop.start()\n File \"/opt/conda/lib/python3.7/site-packages/tornado/platform/asyncio.py\", line 199, in start\n self.asyncio_loop.run_forever()\n File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 541, in run_forever\n self._run_once()\n File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 1786, in _run_once\n handle._run()\n File \"/opt/conda/lib/python3.7/asyncio/events.py\", line 88, in _run\n self._context.run(self._callback, *self._args)\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 510, in dispatch_queue\n await self.process_one()\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 499, in process_one\n await dispatch(*args)\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 406, in dispatch_shell\n await result\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 730, in execute_request\n reply_content = await reply_content\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/ipkernel.py\", line 387, in do_execute\n cell_id=cell_id,\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/zmqshell.py\", line 528, in run_cell\n return super().run_cell(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2976, in run_cell\n raw_cell, store_history, silent, shell_futures, cell_id\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3030, in _run_cell\n return runner(coro)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/async_helpers.py\", line 78, in _pseudo_sync_runner\n coro.send(None)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3258, in run_cell_async\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3473, in run_ast_nodes\n if (await self.run_code(code, result, async_=asy)):\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3553, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"/tmp/ipykernel_23/2555411180.py\", line 2, in \n callbacks=[batch_loss])\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1650, in fit\n tmp_logs = self.train_function(iterator)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1249, in train_function\n return step_function(self, iterator)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1233, in step_function\n outputs = model.distribute_strategy.run(run_step, args=(data,))\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1222, in run_step\n outputs = model.train_step(data)\n File \"/tmp/ipykernel_23/3733470266.py\", line 24, in train_step\n return self._tf_train_step(inputs)\n File \"/tmp/ipykernel_23/626488378.py\", line 5, in _tf_train_step\n return self._train_step(inputs)\n File \"/tmp/ipykernel_23/4151822410.py\", line 22, in _train_step\n for t in tf.range(max_target_length-1):\n File \"/tmp/ipykernel_23/4151822410.py\", line 27, in _train_step\n step_loss, dec_state = self._loop_step(new_tokens, input_mask,\n File \"/tmp/ipykernel_23/431701214.py\", line 10, in _loop_step\n dec_result, dec_state = self.decoder(decoder_input, state=dec_state)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n outputs = call_fn(inputs, *args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n return fn(*args, **kwargs)\n File \"/tmp/ipykernel_23/2024488487.py\", line 24, in call\n context_vector, attention_weights = self.attention(\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n outputs = call_fn(inputs, *args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n return fn(*args, **kwargs)\n File \"/tmp/ipykernel_23/696840811.py\", line 17, in call\n w1_query = self.W1(query)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n outputs = call_fn(inputs, *args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/layers/core/dense.py\", line 244, in call\n outputs = tf.tensordot(inputs, self.kernel, [[rank - 1], [0]])\nNode: 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul'\nDetected at node 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul' defined at (most recent call last):\n File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\n \"__main__\", mod_spec)\n File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\n exec(code, run_globals)\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py\", line 17, in \n app.launch_new_instance()\n File \"/opt/conda/lib/python3.7/site-packages/traitlets/config/application.py\", line 1041, in launch_instance\n app.start()\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelapp.py\", line 712, in start\n self.io_loop.start()\n File \"/opt/conda/lib/python3.7/site-packages/tornado/platform/asyncio.py\", line 199, in start\n self.asyncio_loop.run_forever()\n File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 541, in run_forever\n self._run_once()\n File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 1786, in _run_once\n handle._run()\n File \"/opt/conda/lib/python3.7/asyncio/events.py\", line 88, in _run\n self._context.run(self._callback, *self._args)\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 510, in dispatch_queue\n await self.process_one()\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 499, in process_one\n await dispatch(*args)\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 406, in dispatch_shell\n await result\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 730, in execute_request\n reply_content = await reply_content\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/ipkernel.py\", line 387, in do_execute\n cell_id=cell_id,\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/zmqshell.py\", line 528, in run_cell\n return super().run_cell(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2976, in run_cell\n raw_cell, store_history, silent, shell_futures, cell_id\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3030, in _run_cell\n return runner(coro)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/async_helpers.py\", line 78, in _pseudo_sync_runner\n coro.send(None)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3258, in run_cell_async\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3473, in run_ast_nodes\n if (await self.run_code(code, result, async_=asy)):\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3553, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"/tmp/ipykernel_23/2555411180.py\", line 2, in \n callbacks=[batch_loss])\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1650, in fit\n tmp_logs = self.train_function(iterator)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1249, in train_function\n return step_function(self, iterator)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1233, in step_function\n outputs = model.distribute_strategy.run(run_step, args=(data,))\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1222, in run_step\n outputs = model.train_step(data)\n File \"/tmp/ipykernel_23/3733470266.py\", line 24, in train_step\n return self._tf_train_step(inputs)\n File \"/tmp/ipykernel_23/626488378.py\", line 5, in _tf_train_step\n return self._train_step(inputs)\n File \"/tmp/ipykernel_23/4151822410.py\", line 22, in _train_step\n for t in tf.range(max_target_length-1):\n File \"/tmp/ipykernel_23/4151822410.py\", line 27, in _train_step\n step_loss, dec_state = self._loop_step(new_tokens, input_mask,\n File \"/tmp/ipykernel_23/431701214.py\", line 10, in _loop_step\n dec_result, dec_state = self.decoder(decoder_input, state=dec_state)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n outputs = call_fn(inputs, *args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n return fn(*args, **kwargs)\n File \"/tmp/ipykernel_23/2024488487.py\", line 24, in call\n context_vector, attention_weights = self.attention(\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n outputs = call_fn(inputs, *args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n return fn(*args, **kwargs)\n File \"/tmp/ipykernel_23/696840811.py\", line 17, in call\n w1_query = self.W1(query)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n outputs = call_fn(inputs, *args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/layers/core/dense.py\", line 244, in call\n outputs = tf.tensordot(inputs, self.kernel, [[rank - 1], [0]])\nNode: 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul'\n2 root error(s) found.\n (0) RESOURCE_EXHAUSTED: OOM when allocating tensor with shape[1536,1024] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[{{node while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n\n\t [[StatefulPartitionedCall/while/loop_body_control/_609/_53]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n\n (1) RESOURCE_EXHAUSTED: OOM when allocating tensor with shape[1536,1024] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[{{node while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n\n0 successful operations.\n0 derived errors ignored. [Op:__inference_train_function_274694]"],"ename":"ResourceExhaustedError","evalue":"Graph execution error:\n\nDetected at node 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul' defined at (most recent call last):\n File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\n \"__main__\", mod_spec)\n File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\n exec(code, run_globals)\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py\", line 17, in \n app.launch_new_instance()\n File \"/opt/conda/lib/python3.7/site-packages/traitlets/config/application.py\", line 1041, in launch_instance\n app.start()\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelapp.py\", line 712, in start\n self.io_loop.start()\n File \"/opt/conda/lib/python3.7/site-packages/tornado/platform/asyncio.py\", line 199, in start\n self.asyncio_loop.run_forever()\n File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 541, in run_forever\n self._run_once()\n File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 1786, in _run_once\n handle._run()\n File \"/opt/conda/lib/python3.7/asyncio/events.py\", line 88, in _run\n self._context.run(self._callback, *self._args)\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 510, in dispatch_queue\n await self.process_one()\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 499, in process_one\n await dispatch(*args)\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 406, in dispatch_shell\n await result\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 730, in execute_request\n reply_content = await reply_content\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/ipkernel.py\", line 387, in do_execute\n cell_id=cell_id,\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/zmqshell.py\", line 528, in run_cell\n return super().run_cell(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2976, in run_cell\n raw_cell, store_history, silent, shell_futures, cell_id\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3030, in _run_cell\n return runner(coro)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/async_helpers.py\", line 78, in _pseudo_sync_runner\n coro.send(None)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3258, in run_cell_async\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3473, in run_ast_nodes\n if (await self.run_code(code, result, async_=asy)):\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3553, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"/tmp/ipykernel_23/2555411180.py\", line 2, in \n callbacks=[batch_loss])\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1650, in fit\n tmp_logs = self.train_function(iterator)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1249, in train_function\n return step_function(self, iterator)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1233, in step_function\n outputs = model.distribute_strategy.run(run_step, args=(data,))\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1222, in run_step\n outputs = model.train_step(data)\n File \"/tmp/ipykernel_23/3733470266.py\", line 24, in train_step\n return self._tf_train_step(inputs)\n File \"/tmp/ipykernel_23/626488378.py\", line 5, in _tf_train_step\n return self._train_step(inputs)\n File \"/tmp/ipykernel_23/4151822410.py\", line 22, in _train_step\n for t in tf.range(max_target_length-1):\n File \"/tmp/ipykernel_23/4151822410.py\", line 27, in _train_step\n step_loss, dec_state = self._loop_step(new_tokens, input_mask,\n File \"/tmp/ipykernel_23/431701214.py\", line 10, in _loop_step\n dec_result, dec_state = self.decoder(decoder_input, state=dec_state)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n outputs = call_fn(inputs, *args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n return fn(*args, **kwargs)\n File \"/tmp/ipykernel_23/2024488487.py\", line 24, in call\n context_vector, attention_weights = self.attention(\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n outputs = call_fn(inputs, *args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n return fn(*args, **kwargs)\n File \"/tmp/ipykernel_23/696840811.py\", line 17, in call\n w1_query = self.W1(query)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n outputs = call_fn(inputs, *args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/layers/core/dense.py\", line 244, in call\n outputs = tf.tensordot(inputs, self.kernel, [[rank - 1], [0]])\nNode: 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul'\nDetected at node 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul' defined at (most recent call last):\n File \"/opt/conda/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\n \"__main__\", mod_spec)\n File \"/opt/conda/lib/python3.7/runpy.py\", line 85, in _run_code\n exec(code, run_globals)\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py\", line 17, in \n app.launch_new_instance()\n File \"/opt/conda/lib/python3.7/site-packages/traitlets/config/application.py\", line 1041, in launch_instance\n app.start()\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelapp.py\", line 712, in start\n self.io_loop.start()\n File \"/opt/conda/lib/python3.7/site-packages/tornado/platform/asyncio.py\", line 199, in start\n self.asyncio_loop.run_forever()\n File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 541, in run_forever\n self._run_once()\n File \"/opt/conda/lib/python3.7/asyncio/base_events.py\", line 1786, in _run_once\n handle._run()\n File \"/opt/conda/lib/python3.7/asyncio/events.py\", line 88, in _run\n self._context.run(self._callback, *self._args)\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 510, in dispatch_queue\n await self.process_one()\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 499, in process_one\n await dispatch(*args)\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 406, in dispatch_shell\n await result\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 730, in execute_request\n reply_content = await reply_content\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/ipkernel.py\", line 387, in do_execute\n cell_id=cell_id,\n File \"/opt/conda/lib/python3.7/site-packages/ipykernel/zmqshell.py\", line 528, in run_cell\n return super().run_cell(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2976, in run_cell\n raw_cell, store_history, silent, shell_futures, cell_id\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3030, in _run_cell\n return runner(coro)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/async_helpers.py\", line 78, in _pseudo_sync_runner\n coro.send(None)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3258, in run_cell_async\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3473, in run_ast_nodes\n if (await self.run_code(code, result, async_=asy)):\n File \"/opt/conda/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3553, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"/tmp/ipykernel_23/2555411180.py\", line 2, in \n callbacks=[batch_loss])\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1650, in fit\n tmp_logs = self.train_function(iterator)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1249, in train_function\n return step_function(self, iterator)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1233, in step_function\n outputs = model.distribute_strategy.run(run_step, args=(data,))\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/training.py\", line 1222, in run_step\n outputs = model.train_step(data)\n File \"/tmp/ipykernel_23/3733470266.py\", line 24, in train_step\n return self._tf_train_step(inputs)\n File \"/tmp/ipykernel_23/626488378.py\", line 5, in _tf_train_step\n return self._train_step(inputs)\n File \"/tmp/ipykernel_23/4151822410.py\", line 22, in _train_step\n for t in tf.range(max_target_length-1):\n File \"/tmp/ipykernel_23/4151822410.py\", line 27, in _train_step\n step_loss, dec_state = self._loop_step(new_tokens, input_mask,\n File \"/tmp/ipykernel_23/431701214.py\", line 10, in _loop_step\n dec_result, dec_state = self.decoder(decoder_input, state=dec_state)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n outputs = call_fn(inputs, *args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n return fn(*args, **kwargs)\n File \"/tmp/ipykernel_23/2024488487.py\", line 24, in call\n context_vector, attention_weights = self.attention(\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n outputs = call_fn(inputs, *args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n return fn(*args, **kwargs)\n File \"/tmp/ipykernel_23/696840811.py\", line 17, in call\n w1_query = self.W1(query)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 65, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py\", line 1132, in __call__\n outputs = call_fn(inputs, *args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/utils/traceback_utils.py\", line 96, in error_handler\n return fn(*args, **kwargs)\n File \"/opt/conda/lib/python3.7/site-packages/keras/layers/core/dense.py\", line 244, in call\n outputs = tf.tensordot(inputs, self.kernel, [[rank - 1], [0]])\nNode: 'while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul'\n2 root error(s) found.\n (0) RESOURCE_EXHAUSTED: OOM when allocating tensor with shape[1536,1024] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[{{node while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n\n\t [[StatefulPartitionedCall/while/loop_body_control/_609/_53]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n\n (1) RESOURCE_EXHAUSTED: OOM when allocating tensor with shape[1536,1024] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[{{node while/decoder_2/bahdanau_attention_3/dense_10/Tensordot/MatMul}}]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.\n\n0 successful operations.\n0 derived errors ignored. [Op:__inference_train_function_274694]","output_type":"error"}]},{"cell_type":"code","source":"plt.plot(batch_loss.logs)\nplt.ylim([0, 5])\nplt.xlabel('Batch #')\nplt.ylabel('CE/token')","metadata":{"id":"38rLdlmtQHCm","outputId":"2c24e8f8-3ef3-4477-b845-34da28c8f96b","execution":{"iopub.status.busy":"2023-03-16T17:41:37.110859Z","iopub.execute_input":"2023-03-16T17:41:37.111920Z","iopub.status.idle":"2023-03-16T17:41:37.396141Z","shell.execute_reply.started":"2023-03-16T17:41:37.111881Z","shell.execute_reply":"2023-03-16T17:41:37.394978Z"},"trusted":true},"execution_count":162,"outputs":[{"execution_count":162,"output_type":"execute_result","data":{"text/plain":"Text(0, 0.5, 'CE/token')"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"
","image/png":"iVBORw0KGgoAAAANSUhEUgAAAioAAAG2CAYAAABRfK0WAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABLa0lEQVR4nO3dd1hT9/4H8HfYyIggsgRBxY0TF+5Vt62tHXqtP21v7dLeWturVeusLd7bW2uXtnbYoVXbOuve4B4oAg5coKAiInsFSM7vD2okECAJSc4Jeb+eh+eBk2/O+XhszZvvOjJBEAQQERERSZCN2AUQERERVYVBhYiIiCSLQYWIiIgki0GFiIiIJItBhYiIiCSLQYWIiIgki0GFiIiIJItBhYiIiCSLQYWIiIgki0GFiIiIJEvUoLJw4ULIZDKNL19fXzFLIiIiIgmxE7uAtm3bYv/+/eqfbW1tRayGiIiIpET0oGJnZ8deFCIiItJK9KBy7do1+Pv7w9HREd27d8fHH3+Mpk2bam2rUCigUCjUP6tUKmRkZKBBgwaQyWTmKpmIiIhqQRAE5Obmwt/fHzY21c9CkQmCIJiprkp27dqFgoICtGjRAvfv38eSJUtw5coVXLx4EQ0aNKjUfuHChVi0aJEIlRIREZGxJScnIyAgoNo2ogaVivLz89GsWTPMnDkTM2bMqPR6xR6V7OxsNG7cGMnJyXB3dzdqLaEL9mg9PndkK4zvFmTUaxEREVmTnJwcBAYGIisrC3K5vNq2og/9lOfi4oJ27drh2rVrWl93dHSEo6NjpePu7u5GDyo2jvW0Hnd2cTP6tYiIiKyRLtM2JLWPikKhwOXLl+Hn5yd2KURERCQBogaV9957D5GRkUhMTMSpU6fw7LPPIicnB5MmTRKzLCIiIpIIUYd+UlJSMH78eKSnp6Nhw4bo0aMHTp48iaAg6c4BuZNVCEEQuMqIiIjIDEQNKuvXrxfz8gb5NvIm4lKy8duUHmKXQkREVOdJao6KpTh+4yFuPywQuwwiIqI6j0HFQIkP88UugYiIqM5jUCEiIiLJYlAhIiIiyWJQISIiIsliUCEiIiLJYlAxEHdRISIiMj0GFSIiIpIsBhUiIiKSLAYVAwliF0BERGQFGFSIiIhIshhUqlC/nr3YJRAREVk9BpUqONnZVvu6IHDwh4iIyNQYVIiIiEiyGFSIiIhIshhUiIiISLIYVAzEGSpERESmx6BioB+PJqK4VCV2GURERHUag0oVAj2dq339yLV0fBt5w0zVEBERWScGlSqM69q4xjZfHrpuhkqIiIisF4NKFexsa34+skrFmSpERESmxKBSC4wpREREpsWgUgtK9qgQERGZFIMKERERSRaDShUCPKpf9UNERESmx6BShbAgT7FLICIisnoMKkRERCRZDCrVaO7tKnYJREREVo1BpRrPhgWIXQIREZFVY1Cpho2s5k3fiIiIyHQYVKrBnEJERCQuBpVqyJhUiIiIRMWgUg3GFCIiInExqFSDHSpERETiYlCphi45JU9RavI6iIiIrBWDSjVsbGqOKqnZRWaohIiIyDoxqFRDlx6VwcsikZFfbPJaiIiIrBGDSnV0nKRy8W62iQshIiKyTgwq1dB1Lq0gmLQMIiIiq8WgUg2u+iEiIhIXg0o1HGx5e4iIiMTET+JqjO7gL3YJREREVo1BpRpO9rY6teMUFSIiItNgUDECgbNpiYiITIJBhYiIiCSLQYWIiIgki0GFiIiIJItBxQj2XrqPtFw+84eIiMjYGFSM4LdTtzH0syixyyAiIqpzGFSMJLOgROwSiIiI6hwGlRqENnIXuwQiIiKrxaBSgx8mddW57fdHbmLB1njuq0JERGQkdmIXIHU+7k46t12y4zIA4OnOAegYWN9EFREREVkP9qiYQEFxqdglEBER1QkMKkRERCRZDCo66BAgF7sEIiIiq8SgoguZTOwKiIiIrBKDChEREUkWg4ouuNyYiIhIFAwqREREJFkMKkRERCRZDCo64MAPERGROBhUdCDTc9WPDFwlREREZAySCSoRERGQyWSYPn262KVUMrV/M7FLICIiskqSCCpnzpzBqlWr0L59e7FL0WpIW1+xSyAiIrJKogeVvLw8TJgwAd999x08PDzELoeIiIgkRPSgMnXqVIwcORKDBw+usa1CoUBOTo7GFxEREdVddmJefP369Th37hzOnDmjU/uIiAgsWrTIxFURERGRVIjWo5KcnIy3334ba9asgZOTk07vmT17NrKzs9VfycnJJq6SiIiIxCRaUImOjkZaWhrCwsJgZ2cHOzs7REZG4osvvoCdnR2USmWl9zg6OsLd3V3jy1xe7dvUbNciIiKiMqIN/QwaNAhxcXEax1566SW0atUKs2bNgq2trUiVaTd7eCusiropdhlERERWRbSg4ubmhtDQUI1jLi4uaNCgQaXjUqDPpm967g9HREREVRB91Q8RERFRVURd9VPR4cOHxS6BiIiIJIQ9KnpY/kJHsUsgIiKyKgwqehjTqZHYJRAREVkVBhUiIiKSLAYVE0hIzcX4VScRfStD7FKIiIgsGoOKCSzYdhEnbj7E2JUnxC6FiIjIojGoEBERkWQxqBAREZFkMagQERGRZDGoEBERkWQxqOhpyRjpPYeIiIiormJQ0ZOdDZ84SEREZC4MKnrik5GJiIjMh0FFTzIwqRAREZkLg4qePF0cxC6BiIjIajCo6KlLsIfYJRAREVkNBhU9ceiHiIjIfBhU9MWcQkREZDYMKkRERCRZDCp64vJkIiIi82FQISIiIsliUNETO1SIiIjMh0GFiIiIJItBhYiIiCSLQUVPMj1n02YVFOPXk7eQmV9sooqIiIjqLgYVE3tr3XnM2xKP19dEi10KERGRxWFQ0ZO+k2mPXEsHAJxKzDB+MURERHUcgwoRERFJFoMKERERSRaDip64My0REZH5MKjoyc6Gt4yIiMhc+KmrJwc7G3wxvpPYZRAREVkFBhUDjGznJ3YJREREVoFBxQA2nKdCRERkFgwqBtB3d1oiIiIyDIOKgV7t21TsEoiIiOo8BhUDBXo4i10CERFRncegYiBB7AKIiIisAIMKERERSRaDChEREUkWgwoRERFJFoMKERERSRaDioEEzqYlIiIyOQYVIiIikiwGFSIiIpIsBhUiIiKSLAYVIiIikiwGFQMJnE1LRERkcgwqREREJFkMKkRERCRZDCpmVlSihErFYSMiIiJdMKiYUXZhCVrN242nVx4XuxQiIiKLwKBiRkeuPQAAXEjOErcQIiIiC8GgYiAO3hAREZkegwoRERFJFoOKGckgE7sEIiIii8KgQkRERJLFoEJERESSxaBiIEN20L+XXWj8QoiIiOowBhUzWrLjstglEBERWRQGFSIiIpIsBhUiIiKSLAYVIiIikiwGFQO18XdXf9+5cX3xCiEiIqrDRA0qK1euRPv27eHu7g53d3eEh4dj165dYpaksx5NG2DlhM7Y+05f/PrP7mKXQ0REVCfZiXnxgIAALF26FCEhIQCAn3/+GU899RTOnz+Ptm3bilmaToa38xO7BCIiojpN1KAyevRojZ8/+ugjrFy5EidPnrSIoEJERESmJWpQKU+pVOKPP/5Afn4+wsPDtbZRKBRQKBTqn3NycsxVHhEREYlA9Mm0cXFxcHV1haOjI15//XVs3rwZbdq00do2IiICcrlc/RUYGGjmaomIiMicRA8qLVu2RExMDE6ePIk33ngDkyZNwqVLl7S2nT17NrKzs9VfycnJZq6WiIiIzEn0oR8HBwf1ZNouXbrgzJkz+Pzzz/Htt99Wauvo6AhHR0dzl0hEREQiEb1HpSJBEDTmoViKQE9nsUsgIiKqc0QNKnPmzMGRI0eQlJSEuLg4zJ07F4cPH8aECRPELMsgB9/tL3YJREREdY6oQz/379/HxIkTce/ePcjlcrRv3x67d+/GE088IWZZBrG3lVznFBERkcUTNaj88MMPYl6eiIiIJI7dAERERCRZDCpEREQkWQYN/eTn52Pp0qU4cOAA0tLSoFKpNF6/efOmUYojIiIi62ZQUHnllVcQGRmJiRMnws/PDzKZzNh1ERERERkWVHbt2oUdO3agV69exq6HiIiISM2gOSoeHh7w9PQ0di1WZWN0itglEBERSZ5BQeXDDz/E/PnzUVBQYOx6rMaCbRfFLoGIiEjyDBr6+fTTT3Hjxg34+PggODgY9vb2Gq+fO3fOKMURERGRdTMoqIwZM8bIZVifPEWp2CUQERFJnkFBZcGCBcauwyolpOaipa+b2GUQERFJlsEbvmVlZeH777/H7NmzkZGRAaBsyOfOnTtGK66uS87gHB8iIqLqGNSjEhsbi8GDB0MulyMpKQlTpkyBp6cnNm/ejFu3buGXX34xdp1ERERkhQzqUZkxYwYmT56Ma9euwcnJSX18+PDhiIqKMlpxREREZN0MCipnzpzBa6+9Vul4o0aNkJqaWuuiiIiIiAADg4qTkxNycnIqHU9ISEDDhg1rXRQRERERYGBQeeqpp7B48WKUlJQAAGQyGW7fvo33338fY8eONWqBREREZL0MCir/+9//8ODBA3h7e6OwsBD9+vVDSEgI3Nzc8NFHHxm7RiIiIrJSBq36cXd3x9GjR3Hw4EGcO3cOKpUKnTt3xuDBgyEIgrFrJCIiIitlUFCJiIjA7NmzMXDgQAwcOFB9XKlU4sUXX8S6deuMVmBdpmSoIyIiqpZBQz/Lly/HqlWrNI4plUqMGzcOMTExxqjLIgU3qKdX+9OJGSaqhIiIqG4wKKjs3LkTs2bNwu+//w4AKCkpwXPPPYeLFy/i0KFDRi3Qknw/qYte7VXsUSEiIqqWQUElLCwMmzdvxpQpU7B161aMHTsWCQkJOHToEHx9fY1do8UI8XbDnBGtdG6/+lgS3vvjggkrIiIismwGP+unf//++PXXX/Hss88iKSkJkZGR8PHxMWZtFun5LoF6tf8zOgVFJUoTVUNERGTZdJ5M+8wzz2g93rBhQ9SvXx+vvvqq+timTZtqXxkRERFZPZ2Dilwu13p86NChRiuGiIiIqDydg8rq1atNWUedYcj8WM6pJSIi0s6gfVQeefDgARISEiCTydCiRQs+5wcAMwcREZHxGDSZNj8/Hy+//DL8/PzQt29f9OnTB/7+/vjnP/+JgoICY9dIREREVsqgoDJjxgxERkbir7/+QlZWFrKysrB161ZERkbi3XffNXaNFsWQRwgI7IchIiLSyqChn40bN+LPP/9E//791cdGjBgBZ2dnPP/881i5cqWx6iMiIiIrZlCPSkFBgdY9U7y9vTn0Q0REREZjUFAJDw/HggULUFRUpD5WWFiIRYsWITw83GjFWSJDBnG46oeIiEg7g4Z+li9fjuHDhyMgIAAdOnSATCZDTEwMnJycsGfPHmPXSERERFbKoKDSrl07XLt2DWvWrMGVK1cgCALGjRuHCRMmwNnZ2dg1EhERkZUyKKhERUWhZ8+emDJlisbx0tJSREVFoW/fvkYpzhLJDHgPR36IiIi0M2iOyoABA5CRkVHpeHZ2NgYMGFDroiyZp4sDBrf2wRNt+IBGIiKi2jKoR0UQBMhklfsOHj58CBcXl1oXZclkMhm+n9QFABD8/g6d3mPI3itERETWQK+g8ugJyjKZDJMnT4ajo6P6NaVSidjYWPTs2dO4FRIREZHV0iuoPHqCsiAIcHNz05g46+DggB49elSat0JERERkKL2CypdffglXV1cEBwfjvffes/phHmNZefgG/j20pdbhNCIiImum12RaLy8vDB8+HN7e3sjOzjZVTVZnxeEb2BWfKnYZREREkqNXUElISMCIESOwceNGNGnSBF27dsWHH36I2NhYU9Vn0U7NGaRz29sZfPQAERFRRXoFlaCgILz11lvYv38/0tLSMGPGDFy8eBF9+/ZFkyZN8Pbbb+PgwYNQKpWmqtei+Lg76dyWgz5ERESVGbSPClA2sXb8+PFYv3490tPT8e2330KlUuGll15Cw4YNsXbtWmPWSURERFbIoH1UKp3Ezg5DhgzBkCFD8OWXX+L8+fMoLS01xqmtBufREhERVaZXj8p///tfFBYWqn+OioqCQqFQ/5ybm4s333wTnTp1QteuXY1XpRWQcfCHiIioEr2CyuzZs5Gbm6v+edSoUbhz547654KCAnz77bfGq86KsEeFiIioMr2CSsWt3rn1OxEREZmSwZNpybiSHuaLXQIREZHkMKhIxJqTt9Xff7TjEp766iiKSrjMm4iIrJveq36+//57uLq6AgBKS0vx008/wcvLCwA05q+Q/oZ+FoUtU3vhuyOJAICN51Lg4mCHfi0awsPFQeTqiIiIzE8m6DHRJDg4WKfn0SQmJtaqKF3l5ORALpcjOzsb7u7uZrmmvoLf36FX+4hn2mH2pjgAgL2tDCVKAaGN3LH9rT6mKI+IiMjs9Pn81qtHJSkpqTZ1kQ6Uqse5sURZ9n38nRyxyiEiIhKVXnNUDh48iDZt2iAnp/IHZ3Z2Ntq2bYsjR44Yrbi6wJNDNkRERAbTK6gsX74cU6ZM0dpNI5fL8dprr2HZsmVGK64uWDUxTOwSiIiILJZeQeXChQsYNmxYla8PGTIE0dHRtS6qLtF3I7cfj5pnfg8REZEl0Cuo3L9/H/b29lW+bmdnhwcPHtS6KGt2M537qRARET2iV1Bp1KgR4uLiqnw9NjYWfn5+tS6KiIiICNAzqIwYMQLz589HUVFRpdcKCwuxYMECjBo1ymjFERERkXXTa3nyBx98gE2bNqFFixaYNm0aWrZsCZlMhsuXL+Prr7+GUqnE3LlzTVUrERERWRm9goqPjw+OHz+ON954A7Nnz1Y/lFAmk2Ho0KFYsWIFfHx8TFIolcnIL8ath/no1NhD7FKIiIhMTu8t9IOCgrBz505kZmbi+vXrEAQBzZs3h4cHPzi18ZM7G/V8vZYeRGGJEr+90h09Q7yMem4iIiKpMfihhB4eHujatSu6devGkFIN//rOGNjK22jnK/z7QYWHr3J1FRER1X2iPj05IiICXbt2hZubG7y9vTFmzBgkJCSIWZJJ9GLPBxERkUFEDSqRkZGYOnUqTp48iX379qG0tBRDhgxBfn7d2kukY2B9sUsgIiKySHrPUTGm3bt3a/y8evVqeHt7Izo6Gn379hWpKuMLaeha63OMW3UCX/2js/pnPR56TUREZLFEDSoVZWdnAwA8PT21vq5QKKBQKNQ/a3s4ohQ52NW+4+rkzQwM/SzKCNUQERFZDlGHfsoTBAEzZsxA7969ERoaqrVNREQE5HK5+iswMNDMVRrG2cHWKOd5mF9slPMQERFZCskElWnTpiE2Nhbr1q2rss3s2bORnZ2t/kpOTjZjhdLCkR8iIrIGkhj6eeutt7Bt2zZERUUhICCgynaOjo5wdHQ0Y2VEREQkJlF7VARBwLRp07Bp0yYcPHgQTZo0EbMckwpuUE/sEoiIiCyOqD0qU6dOxW+//YatW7fCzc0NqampAAC5XA5nZ+Pu6FrXcOSHiIisgag9KitXrkR2djb69+8PPz8/9deGDRvELMskZDKZUc9X1RyV76JuYvLq01CUKo16PSIiIjGIPvSj7Wvy5MlilmURDiekaT3+0c7LOJzwAFvO31EfU5Qqse70bSRnFJirPCIiIqOQzKqfum7OiNZGPV9mgeZS5XxFKdJyitQ/FxY/7lH5+tANzN4Uh0GfRhq1BiIiIlOTxKofa/BEGx+jnq/8yE9KZgF6/+cQ6tez19r2+PV0AECxUmXUGoiIiEyNPSp1wJ/RKQCArIISra/HJGeZsRoiIiLjYVCxAqUqrhEiIiLLxKBiocr3nshg3BVFREREUsGgYsF2xt2r8rWcolIAQCnnpRARkQXjZFoL9ubac4hfNBTatmhZtu8qTtx4iBM3H5q/MCIiIiNhj4qFO3Gj6iDCkEJERJaOPSoWbsovZ7X2qFRFqRJga8M5LUREZBnYo1IHVLWdvjYdF+3FvexC0xVDRERkRAwqViZXUYrvohLFLoOIiEgnDCpm5GDL201ERKQPfnKaUcQz7cQugYiIyKIwqJhRv5YNxS6BiIjIojComJGXqyP+2buJ2GVAALfUJyIiy8CgYmYtfFzFLoGIiMhiMKiYmT5LiYmIiKwdgwoRERFJFoMKERERSRaDipnZ6LPfvYlw+ImIiCwFg4qZje7gj5Y+bqLWcCU1R9TrExER6YpBxcycHWyx552+otZQomSXChERWQYGFSIiIpIsBhUrJHCSChERWQgGFStUm5iiUgkMOkREZDYMKlbI0JxRqlRhyPIoPPfNCeMWREREVAU7sQsgy3EzPR/X0/K0vnYoIQ0QgAGtvM1cFRER1WUMKlbI0IGbqnpiCouVeGn1GQBA/KKhcHXkf1ZERGQcHPqxRkaeY6IoVaq/LyguNeq5iYjIujGoWCGDe1RqNQ2XiIhIfwwqVkhlylU7zDJERGREDCpWSFtOKSpRVj5YwfYL99Tfp+UUIfpWJgBABvGfX0RERHUTgwrhm8gbaDVvNw5dSauyTWxKFr46dF39c7ePD2DsyuM4dzvTHCUSEZGVYlCxQqoKPSpLd10BAMzcGFvle24+yNd6/JkVxzV+5sgPEREZE4OKFSq/s+zG6BSN1wqKSxF9KwP5Cs3VO9VNpL1c7mnM59nDQkRERsQNL6xQ+Tkqyw9cVX+fV1SKNvP3AABa+bph9/S+Wt9TkaJUpf5+zuZ4DAv1M16xRERk1dijIpJmDV1Eu3b53pHyAaSw3ITaK6m5SM9T6H3ujPxiLNl+qVb1ladUCbibVWi08xERkWVhUBHJb1N6iF0CgOp7SiJ2XtGpXUXfH0002oMLX/s1Gj2XHsS+S/eNcj4iIrIsDCoi8XF3Eu3aRSWqmhsBuHwvByXKmts+moxb3tMrjkNVcdauAfZfLgso3x25WetzERGR5WFQsUK3Mwp0anfpXg6eXnEM19NykVNUUmW7y/dyKh2LSc7C1bRcg2skIiICOJnW6tU0RBN/JweDl0UZdO7U7CK08nU36L1EREQAe1Ss3t3sIpOde/LfT1QmIiIyFIMKERERSRaDiohe7dtU1Osfv5Eu6vWJiIhqwqAiojkjWot6/X98d0rU6+uDjz0kIrJODCpW6mxShtglEBER1YhBRWQ+7o6iXPc/uyvvfUJERCQ1DCoiOzproCjX1XXTt9pfR1lzIyIioiowqIjM3lacv4K4O9lmuY7KSFvpExGRdWJQIZNiTiEiotpgUCGLIOOyHyIiq8SgQibFDhUiIqoNBhUiIiKSLD6UkCQn6uoDnEp8iOAGLupjJ29m4HpaLkK83TTaKkqVeGdDDHqHNMQ/ujc2d6lERGRiDCpkUjfS8tAhsD4A4PsjN/HziSRseDUc/vWdq3zP//14WuvxwcuikLR0pPrntNwidPvoAABgZ1wqgwoRUR3EoR8yqZyiEvX3S3ZcRnJGYa03m8suLMGPRxPx4fbLtS2PiIgkjj0qEuLj7oj7OQqxyzAqbcuTS1W1m2I7689Y7L6YWqtzEBGRZWCPioR0CKiP3iFeYpdhVFojSS2XAh24cr92JyAiIovBoCIBW6b2wriugfj4mXZilyJ5sSlZKFFy0TMRkbXg0I8EdAysj45/TzitawQtYz9CLbpUnvzqWG3KISIiC8OgIjF1bQdWfSJJdmEJFKV8iCERET3GoEKmVU1SyS4owcnEh2jr7w5BAPr895D56iIiIovAoEImVd0wz/jvTuLSvRwzVlOzguJSrD6WhKFtfSptLkdEROYn6mTaqKgojB49Gv7+/pDJZNiyZYuY5UhC/5beAAA3RzvMHNZS5GpqT9vy5J1xqbieliuJkFJUotSYR7Ns71V8sicBg5dFAQCW77+KWX/Gap1ro6ulu67gya+OoqiEw1pERPoSNajk5+ejQ4cO+Oqrr8QsQ1ImhQfh6390xv53++HN/iFil1Nrx64/xPnbmZWOPwoCYsrIL0arebvxj+9OqY+dq1Dr8v3XsOFsMi7ezUFaTpFB1/km8gZiU7KxNeZOreolIrJGog79DB8+HMOHDxezBMmxs7XByPZ+YpdhND8eS8SPxxI1tr6Xipl/xgIATtx8qD72IO/xhnvp5b7/394EHE54gFnDWuGN/s0Mup5SpVs7QRCw99J9FJUoMbKdH+xsuYsAEVkvi5qjolAooFA8/vDIyRF/6MDUnO1tUcghA5PYf1lz47i/LtxFckah+uecwsfb/x9OeAAA+M/uKwYHFV1XdG2NuYvpG2IAAHezigy+HhFRXWBRv6pFRERALpervwIDA8UuyeTOfDBY7BIshqqWW/N/sidB42dZFcni5oM8g86v68rz3fGPHw+w6VyKQdciIqorLCqozJ49G9nZ2eqv5ORksUsyOVdHi+r0kiylSkCeorTK16+n5eF2RoFO58ou19OiD117VMq3u5ZmWCgisjSluo6NktWxqKDi6OgId3d3jS+iR6rrT3n2m+MIXbAH97LLhnYqrsAZvCyy0nuqyhW74g17IKKshj6VUzcfYsHWeA71kdVZvv8qWs3bjUt36/5wPunPooKKtZoUHiR2CbWWlmvYihljOX87CwDw2q/RmPD9SbSat7vG91TVA7Iq6qZhRdTQo/LCqpP4+cQt9XwYImuxfP81lKoEfLzzstilkASJOq6Ql5eH69evq39OTExETEwMPD090bhxYxErk5ZFT4Xi5xO3xC6jVrp9dMDk1yjb66T6NBCbkq3z+VYfS6pdQRXUsacjEBGZhag9KmfPnkWnTp3QqVMnAMCMGTPQqVMnzJ8/X8yyyEIZ+5nKPx1PMur57ucUYemuK7iTVVhzYyIiAiByj0r//v1rteMnUU0OJaTB3cne6OctLlXBwU6/nP+/vVcBAHsvpuLge/0BAEevpSMxPQ8Tw4ONXKH0lCpVOJ2UgY6B9VHPgZPEqbLaPFmd6i7+a0F1RsXMezerEC+tPmOSa/18PAlT+jY16L030/NRWKzE5weu4ZvIGwCANv5yY5an4dj1dMzbEo+IZ9qhe9MGJrtOTb4+dAOf7b+KHk09sf7VcNHqICLLwsm0VGcIECAIAnbHp+L2wwKkGrjlvS6upeXW6v1fHHwcUgDj7ZcSfycbXx+6juLSx0s9J3x/CjfT8/HCqpNGuYahfjtdNs/q5M0MUesgIsvCoGIhfnm5G/o098ITbXyqbHNhwRAzViRNey6m4vU10ej7ySFJT16Nv6M5qXftqdtGOe+oL4/ikz0J+OFootbXFaVKbI+9i+tpuXjyq6NYe8p8k7Q5yktEhuDQj4Xo26Ih+rZoiIXbLlbZRu5sD1sbGZS13KHVUgmC5m/rVe0sa2ylShWSHuajWUNXna955Fq6SWu6kqp9P4ple6/i23LLq2NTsjGhu2mXv9/PKdJ5Mz1LU6JUwZ7PYjIahlnShv+H1TF7pvfBlD5NxC5DEszVo/LWuvMYvCzKaL0ixlDVP/jbY++ZtxAA3T8+gOe+OYG0XEXNjS3I+duZaD53F5btuyp2KUR1GoOKhSm/Sur6R8PR3NtV4/UQbzfMHdnG3GVJ0vEbD2tuZASPdqr9YEs85m+Nx/T15/HlgWsYtjzK4O32yxv06WGkZNbcI5Ga/XhOzuGENNzNKsShK2kabbQtjb5vwrk8plTdIxHMYfH2SwCALw5cM+p5U7OL8Py3J7BDhFBJJEUMKhbMztYGHQPri12GZCzYehEl5Z4X8p/dV0x2rRsP8rUe/+XELWyJuYtP913FldRcrIq6obWdvtdasr3mHTvLb72fU1SKnksP4qWfal711P3jA5UeKSB1n+5NQOiCPdh36X6l1x7mKZCRX2zWes4mGW+C8MJtF3E6MQNTfzuncVylEnD+diYKiy3r76oiQRBE36maLAuDCtUZG84mm234JfpWJgAg8mr1291/faj2QQUomwRblRm/x+Cpr4/Vam7S4YQHFrWn0ZcHy3a0XrjtItafvo11p8v+3otLVQhbsh+dP9ynsfLJ1J795oTRzpVRoD1krTl1C0+vOI5JP5422rXEMGdzHLp9dADbY+9Wes2C/hMkM2JQsTDPdA4AALTx0++BjD7ujqYox6oVlSjN9qFRXQbZdO4OLiRn4Uwtfqt/fU00ftHjMQ3JGQV45ecz6PvfQzhyrSysCYKArHIfsopSpcnDT35xKd7fFIfZm+KQXViiMdSWW1T7YTdRaLllgiBg/tayifSnjdh7I4Z1p8uees+5PaQrBhUL0yGwPk7MHoit03oB0H3b+HARN/qqqxRm/I1dl79nVS1DwYJqVpRVNG7VSey/nIbbGQWY+ENZWJu1MRYdF+9D1NUHSM4oQNv5ezDOxHu3lJT7O1CUKjXm6fxy4hZ+NvJjEMxB2+6sO+MMe2J3db6JvIFxq06IN+zH3hPSEZcnWyA/ubPe73m6cwC2xFTuaiXDLfpL9w/22qqqZ+Li3cf7sczdHG+WWtacvFVpUu7ZpAz8frZs07r/K9fLdCrR+L/955ebRKuxHFwARn91VP3j539Pch3TqRHkzsZ7jEJabhFe+zVa/URuY9P2V70jzvj/7y7dVTaHa8OZZEzqGWz089dE23/RZtpRgCwMe1TqqD9e19yi3NWRmdTYNp27I3YJGPnF0ZobGdkHWyoHImPO0agoX1GKjov3Ivj9Hfj+yE1M/OGU+rXyn2tV/YJefoK1Mfx3d4LeIeVQQho+2BKnU++Ftp4xU46gFVaoyVxzlRLT86GqMKZpykufuPEQhxPSam5IksOgUkd1DfbEvnf6qn92sudftSV79OFVVKJEWk4RBEFAQbG4y3Nro+IHVHV+OJqIrIKy+SZLdlzGufIhoVxSyS3Sfj9q+iU9M78Yi/66iEt3tW+SV1FeFdepbjLzS6vPYM3J2/jxmPYdgzXOL+Ky63vZhej+8QF8ZqT5I7EpWej730PYHa99qfWGs8l6nW/F4ev4ZI/+q/mUKgHjvzuJyavP4O3157HhjHT2PKKa8dPLwo3u4A8ACGpQr9JrzX3c8O+hLfFij8Z6T74laXn0m2anxfvQ7eMDaDN/D9rM3yNuUbWwI073PUIehRRtyoeQmX9e0N6mhvGED7bEY/WxJIz44ki17dLzFMhXlFY5PLE1puYetrta9rGp6Or9vErHjLEfT1XK92J8ceAa0nIV6mGz2nr1l2jczijA62vOaX39wOU0rD+tW2hQqQT8d3cCvj50Q6d9hcorHyK3xtzFrI1xer2fxMXxAAvXr0VD7J7eB409KwcVAJg6IMTMFZEpCAJw40Geupu+Yne9MR29lg5nB1uEBXmoj20+n4I/zqZgYCtvo1wjPU/3XWq1TS59JKdc78a5KoZjaupRKT/P55HtsXexdNcVrJwQhnYBcmTkF6PLkv0AgOGhvlrPY4qdd/dcTMXQtr5G37ywfI9W+Z658qFl7uY4vDWwOXzlTgZfp6iaZfUAsP/yfey//HgvnOr+rsu/outE9pWHbyBPUYK3B7XQqT1JE3tU6oBWvu6o58DMWZepBAGDPo00+XXS8xR48YdTGLvyuMbxdzZcwPEbD7FkR80bz+lCnzmTtZ23UNMEzfKnf7QvzrTfziMlsxCvr4kGAOwqN3TxaCfiitacrHl5d/k/S25RCT7Zc0XjuUwVh49e+zUa3T/eX+k8mbXc0K780uAvD17HmpO3UFyqwvozj4di1p66jR4RBwy+RtlydfGWiKtUAv6z+wq+PnRD647MZDkYVIgsgLlWcurT01Eb5npgJADIaohF5cPDpB9PqzfzA8oeOZBbVKLTiqqUTN0+DO9mFeJ6Wh4+3nkZXx+6gWHLHw85ffj3tvzl3c+p/Hfy5lrtQylVmfF7DN774/HQ2FeHrmu8/sGWeDz3rXEnRG+7YNpVhtmFJRj6WZTW+TSCICCr3HDZlvPiT3wnwzGoEFmA0yZY5luTRxu5mYKNluxw40Ee+vz3oMZEx6irD/BTLfdCkVXzr1xxqarSk50r9iZ1/nBfra5fngCg59KDGLwsElFXNZ+g/Wd0is5/1hM3tQ8FbYxOwaZzKRrH0vMU2HTuDv6MTql2rsuF5Cytx+PvVB4aq0mpUoW318doHDt45T5iU7RfQ1+CAPx8PAkJ93MrzacRBAEvrDqp8fdmrDk3JA6OFxCRVhN/OI05I1rhld5NjX7urw/dwITuQbCxkeHmgzxEXn2AbRfuIjmjELM2xqFpQ1e0D5Br7MliqOr6U3QZ2ihRGt6fdSghDZEJjwNf+d6b8sMRvf9zUOcemarkFJXg3b97TYaF+qqHg8sPJ526+VBj7pEu9lxMRWgjeZWvf33oOtJyirDwybbqnjJtw2Av/3S2xmtVN8xXcdm0tkckCIKARX9dEiXYk+kwqBCRWkyFCakf77yCvRcrP/ivtlJzirDm1C38X3gwBmqZe/OcEfdlqWqY6UJylskfXvjS6pofCgnoPmxUnfJ7tBSXqlDPoez7/+1JUB9/9ddovc9b0xyhT/4+/7hujdH679WFhk7+rdi7VW1dFQZEswtKEHWt9j1wJD0c+iEitfc3VV62ebbcnA1j2mimDfOq6lF59pvjVbxiSqaZbVRaYVO7uHLDNX9Ep1RsrpfoKv7+72QVajzhefjnR5Cep8DhhDTs1fJUa13cy676qcqac14EjQd+pucp0GHxXry17rzO17Kkh3BaOwYVIhKHmT4oqpq3W5shHUOZIpz9fDwJrebtxpnEx4Fi4g+nkVVQjF9PJNX6/FXNh5m+/jx2xGruh9NlyX5M1rEXSV8zfn88GbiwWDOYPVo6rg9t+9WQNDGokIZxXQMrHVs3pYcIlVBddyElW+fdYGuj/KqfwmIlNkanIPj9HSa5VmJ6Pnr/52CVS5W1zauorQXbLqJUJeDdP2I0jo9bdRLzthrneVQPchUY+cUR/Fruz5X0UL9N16TGlHsRkXFxjgpp+PjpdnjniRZIySzA2JVl8wQauDrodY5/9m6CH47WvFU4UU27wRpDWm4Rghq4IKeoBO0X7jXptaZviEFKZiE+2BKPF3sEmfRaFVXcg+VKaq7Rzt31o7Iei3lb4hHS0BXX0ox3bkP8a73uQzxVeWCCDfrINBhUSG315K6wsZHBx90JPu5OaOHjChdHO7TwccO+d/ritTXRuPkgv9pzyGTAByNbM6iQZPT75DDcnew0drE1lfJLfE3Va/NIUYkSNx48Hr4w11DW+O9OmvT87/1xAYpSFb4Y1xF/RKfg8r0czB/VRqNNYnr1/w7pQlHDrrkkHQwqpBbs5aLx8953+qm/b+7jhoPv9seluznq34I/fKqtumu5Q2B9LH2mHYIa1DPrZl5EujBHSDG3catOIqaKvU8s2Z9/T/6dM6IVZv4ZCwBoH1D18mhDVfVwSZIezlGxcmtf6a5X+zb+7tg9vQ+OzhqAieHB6uMNXR3R2o9b+ROZS10MKeWFRxxUf//OBu0PnKwNDv1YDn6qWLnwpg3U33vpOBellS+fxExERObBoGLFJvYo2xk06t8DUKJSwc3J3uBz9WzWoOZGtTR9cHNsu3AXSpWAWxa+4oCIiHTDoGKlpg0IwXtDWwIAGjeoZ/B5js4agDNJGRjd3t9YpQEAXu/XDOdvZ+JUua2wpw9ugemDW2D/pft45Zeat+MmIqqKivu9WQwGFSvUp7mXOqTUVoBHPQR4GB50qtLK1w3ZhSXqoLJ7eh/1awNaeePJDv5wtLPB+O6N8cwKMXYYJSJLpuLOtBaDk2lJVJve7Ik2fpXnvDRwddAYTio/L8bWRoYvxnfCJ891QOfG+j1gjYgIePwwgy3n72DprivcUl/C2KNCJvGvQc3xxYFr6BLkUe2zYgI8nLFtWi9cS8vDgm0XUd/ZHs28XdE7xAsA4GBnU+2TW4mIDLHl/B3MeKIFpm+IAVDW09zr7393Kvr9TDJWHL6OHyd3RdOGrmaskgAGFTKRGU+0wNQBzeBoZ4vvj9zE14euI7OgpFI7GWSws7VBaz93/P5aeKXXh7b1NUe5OhnbOQAbz9XuAW9EJA0Vn9ScVe7fp0GfHsaNB/nYMrUXOgbWx8yNZfu5zN0cj3Wv8pEi5sahHzIZRztbAMArfZri3Lwn0Ny78m8iUt0bLrRR5eEoqdZKRIZJy3n8tGabcv9/3/h7B+4xXx/TaF9QzE3ixMAeFTILmUyGDa+F48Dl+2ji5YJnvyl7jpBUP/u3v9UHey+m4tVfo9XHpForERmm28cH1N+/sfYcWvi44skOmisYn1nxOKxcSMk2W230GHtUyGw8XRzwXJdAja36jbHdfvlN66rzybPtEVRhKXbHwPro26KhxrHgv9sMaeuLpKUj1cft7fi/C1FddvV+Hv6396rGsXO3s0x+3aISJX46logZv8fg8j3TP1Hc0vBfXjK7Bi4O6N7EEz2aesKjnuGbzD3y6fMdMKyGuSzLX+iI57oEIvLfAzSOb5naC4Nbe6t//nJ8J2x+s5fWcwxo6a31uK4GtNQMRD+91LVW5yMi6Ym+lYF5W+KRU1R5Tl5VPtt3FQv/uoRN5+5g+OdHkJpdVPObrAiDCpmdTCbD+ld7YN2UHkbpUfGv74xvJoZpHDs1ZxCOvz9Q/fOgcmGkd4WZ/f/o1hj/HtoSW6b2wugO/vBw0f4oAXcnO0zo3tjgOle/1E39/WcvdEB/LcFnYo8g3Px4hMHXICLxRN/KwNiVJ/DryVtov3Avgt/fgQOX71fZ/mGeAjHJWfg26qbG8R4RB1BUwqc7P8KgYoWksF2ATCYz+lOW1/zz8QMWfdyd4Cd3woh2vniqo7/G4wHaVpgoa2drg6kDQtAxsH6NNS8ZE1qrGj98qi1GtvertJOvt5sjYuY/gcVPtYWNDWfDEFkCQRBw7X4uFKVKHLuejrErT1Rq88+fK++iLQgCZm+KRdiS/ZUm7D5y9Fq60eu1VJxMS3VGr5AGWPRkW/XqIplMhhUTwiq1mz6oBZztbTGkjW5Ln4eH+iLpYQE6N66vNVzNHdEaH+28rPW9U/o0wXdHEtHv73kwE8ODNZ46/ciknsGoX0+3h0LW5PScQRqTBG9+PAL/2X2l0m9txvLByNY4lJCGfi0a4uOdV0xyDSIpidh5GcVKFTo39sBb687X2F4QBI1/O66n5WHd6eRq3/PKL2c15shZMwYVqjNkMhkm9QyusZ2zgy2mD26h83lXvhhW6R+a8qb0baoOKq183fDj5K5Ycfg6JvdsguAG9TCotQ86BNTX+t5AT2ckZxRiaFsfnevR5vLiYWg9fzcAwMvVUeM1GxsZZo9ojTf6N0PHxftqdR1twoI88EqfpgCAiT2CAUBdyyMh3q64npZnlOt9Pq4jikqUmLUxDsNDfTGinZ9OHxZExnA3q1Ad+lcfS9LpPVkFJRpDyk98FmWK0uosBhUiHVQVUj59rgMAIOKZdvh071Use74j/Os7Y8mYduo2PapZlbTvnX7IKiiBr9xJpzqae7vimpYPfGcHW5yeOwh2NjawsZHh24lhWPzXJXwxvpO6Tf16Dmjt527QqoJnOjXCrvhUFJYbN5/YIwgFxUqNITNnh7K9c4Ib1EPSwwLELhwCN0c79f0Lfn+H3teuqH1AfTTxcsELXcvmCylVQqWgIpOZZ4hzeKgvnurYCK+via65MYmqjZ87Fj/VVr01gqF6Lj2o93s6fbgP3Zt44oORbTD6q6O1ur414hwVIj399kp3tPFzx9apvTA2LAAAML5bY5yZOwht/CtvFFcdJ3tbnUMKAOz4Vx8kRmhOtl3/906Z3m5O8Pz7t7ahbX1x7P2BCAvyqNT2x8ldcPaDwZg6oBn2z+hb4zUn9wzG0rHtMXdka/UxX3cnLHyyLT59voPWEHf43wOQtHQk3J3sDZqLNLmKnrE+zb3QpNzydqDs2U/fvKg5xHf9I9NMSJ4zohXOfjAYcmd7DGnjg5UvhsHe1nRzil7u1cRk564o0NPZpOd3trc16flrsviptkYbXjXEqcQMhhQDsUfFijz6LbNHU0+xS7FoPUO8sPPtPpWOG3tycEWN6jvDocJeLh0C5NX22FQkd7bHwFZlw0z/HtpKp/fMG9UGtjYyjO/WGF6ujmgfIEdDN0fYmnDS76OemfI6BMjxc7mVU+UNC30832jOiFZGr+3qkuHIV5Squ+/PzXtCvZNpKy0P1TSWqQOaYfaIVmg+d5dRz9uovjPuZBWqf57Spwnmjmyj7vF6sUdjrDl522jX+2FSFwxs5Y3/+/E0HuYVQ1GqVO/+agqDW3vj42faYd+l+3iqYyM42NrAwc4G19NyTXZNMh0GFSsS9e8BOHY9Hc90DhC7FKqBl6sj0vMUGscOvddf/f32t3rjh6OJeHeI7nNtqlLVcFLM/Cc0fgO1tZFpBAJT++P1cDxXrpt+y9ReOoXBsCD9g/iX4zuhtZ8bBi8rmzuw9pXumPD9KQDAv4e2hIOdDRzsNO/FI43qG94T8eX4Tjhx8yF+O3UbS8aE4sUeQQCAB7kK2NvK1Pd/x79647VfozG4tQ9+Op5k8PW++kcn9G/pDRnKtofv07whnu7USN0TGPFMO1y7n4d5o1rXKqi4ONgiv/jxMOGg1mXh+Ne/V+bdzSrE8v1X8fvZ2j07y9neFmc+GIzQBXsAlD2NvfwT1Sd0D9Jor5LAikfSH4OKFQn0rIdx3QzfB4TMp2KHQLcmnhq9KaGN5PjshY5Guda6V3vg0JU0jGrvjye/OopraXl4soO/qN3kgR710DXYE2tf6Y53NsRg6dh2OvdYGdKxNfrvbdNj5j8BNyd72NrIsOnNnjh4OQ2v9DHe8EsrXzek5ykQ6FkPbw9qjv4tvTGqvR/e6NcMgZ6Pd01u6KY5IbqtvxxHZ5XtC1SboDKq3LL4fTP6VXp9vIH/PjRwccDD/GIAZfO2BrX2xuivjiI5o1Bre//6zvjvsx0gd7aHna0NVh6+odf1ghvUwyfPdUDX4LJQuunNnrj1MF8jpGgjha0ZSH8MKkQSVP7D1tHOBqsmVl5mbSxero54rksgAGDbtN44dj29ysfdG8vIdn7oGuyBhX9dAgC4OdphzsjWWH8mGb1DGuD5LmW9fr1CvHB67mC9zv3o1r3Rv5nWD0BfdycsHdsOzRq6Ys7mOHQLftwDUz6cdW7sUeMH3yMznmiBZfuuan3N2d4WHQPrI8TbFR9q2YdHJpNphJTa6BrsgTNJmeqfl7/QEetO38apxAy9z1VxeKg680a1wdHr6bC3tVHP23o+LBCfVnFPHpk7sg0A4GxShkbd2pT/+zzwbn+NXi1d/64aG+k+m8uei6kY2tYXKpWAYzfS0dZfrp6HZk0YVIgkLmHJcLNdy9nBFoPb1G6ptC4aujlicq8m6qDSKcgD47s1Nvg3+vJC/t5HZ9awVnhvSEvY2sg0Vhvtm9FXvQHgr+U2CayNfw1qjl4hDTB25QlMHdAMT3ZohJa+btUuazeW3dP7YNjyIwAAdyd7dG/iiVOJGephkLvZhTiVmIF6Wub9VOfge/3Q8oPHy8y3v9UbzX1cce5WFi7ezcaSHWVL8je/2RMdA+tjTKdGGu/XZ+PCpWPbY9CnkZWON6rvjJ9f7oYAD2c42dtiVHs/+MmdDZ6D5Oxgiwvzh6DD4r0Gvd/cXvv7oahO9jYoKlEBgFXurcKgQiRBsjr+rOZHn93rX+2Bn48nYcHotrU+Z/yioSguVWnsQvzoA+35LgH4/WwKzswdrPG6MYUFeSIxYoRGMDFFSDkycwD2XEzFgctpaNLQBa18H0/mbeHrhveGtMTDfAW83cpWk03p0xT+cmd013MSvaOdLb77vy7YEXsXHz3dDi6OZR8X4c0aoGuwB7bE3EFTL1d0qqIn48XuQdhwJhnD29U8r6lZQ1f191MHNMM/ezdFckYBmjZ00dxV2l+u159BG3k9e3i5OiA9r7j6ds72yC7U/Xk9pvQopABlS/wr/ndW18kEwXJH7XJyciCXy5GdnQ13d9PNvCcytxGfH8Glv/c7qUu/QT3q2fhn7yaYN6qNyNXUHbEpWdhzMRXTBjTXumLKFHTpLdKnR+mvC3cRfStTvcrMlFIyCzDj9ws4XWFIrIGLA+T17DFrWCsMbeuL7MIS/HXhLj7YEm/SevS16+0+aG3C1WbmoM/nN3tUiCToy390wsw/YzFtYIjYpZhEK183sUuoU9oH1Ef7KnY/NhVdAog+v/WP7uCvntRsagEe9fD7a+EoKlFiV/w9FBarsPl8Cj5+uh2a+zz+b1PubI8XewQZFFT2z+irXkVWk/HdGuNBbhEauDhiw9nqt9YHgIJizQcW/nIiCfO3XsSF+UMgN8IT6aWGQYVIgpo1dMXGN3qKXYbR/TWtN84kZWAsl8iTBDjZ2+LpTmX/Lf6jFk9GL2/VxDD0bdFQ516hCwuGQO78OFzoElQEQcDNB3nYEnMXJ26kqycid1i8V6MH9kGuAkM+i8T80W3Uf05LxJ1pichs2gXI8XLvJnxCNFkUXXsA3Z3sMKStL5zsbaudZWZvK0NTLxesfqmrRkgBgP+MbVfFux5LySzEwE8j8cWBa1pXSwmCgK0xd9D1o/3ILCjBOxsuQGnBm8iwR4WIiKgaz3cJxOLtZSvUxnUNRG5RKS7ezcaWqb3w9aHr+O5IIgAget4T6vdoG/baP6MvQrzdoFIJVYb157sEYtbGuGrrmb4hpsrXikqUmL81vtJmes3m7LTY+W6cTEtERFQNpUrAtgt30NzbDaGNNFceqVQCbqbnoVlD10rh5O3155FZUIKfX+qq13ydM0kZePH7U1CUqmpurAcpBRV9Pr8ZVIiIiCTqX+vOIyWzAL9N6YFW83bX/IZqWGpQ4dAPERGRRH0xvpPRztV+4R58+nxHPGGGTR2NiZNpiYiIrEBOUSmm/HIWI784InYpemFQISIiskD1HGzR0scNoY30m/pw8W4Ogt/fgeD3d+CPv5dDlypVyMyvfrdesXCOChERkYXIV5SinoMtZDKZesnxiRsP8eIPpyq1jfx3f/T75LDe1zDHXBZOpiUiIrIi3x+5iZvp+egW7Ak7WxlGtS/b5VepEtBszk69zhXetAGm9G2C5IxCDG/nq35ulDExqBAREZHa2lO3MHez4c8sMnYviz6f36LPUVmxYgWaNGkCJycnhIWF4cgRy5rkQ0REJHUTugfh6pLhYpdhEFGDyoYNGzB9+nTMnTsX58+fR58+fTB8+HDcvn1bzLKIiIjqHAc7GyRGjMCFBUOQsGQYvnkxTP3a3nf6ilhZ9UQd+unevTs6d+6MlStXqo+1bt0aY8aMQURERI3v59APERGR8RQWK7H21C3kFJbgi4PX8a+BIegc5IH+Lb2Neh2L2PCtuLgY0dHReP/99zWODxkyBMePH9f6HoVCAYVCof45OzsbQNkfmIiIiGrv+Q5eAIBXevipjxn7c/bR+XTpKxEtqKSnp0OpVMLHR3OHPB8fH6Smpmp9T0REBBYtWlTpeGBgoElqJCIiItPJzc2FXC6vto3oW+hXfFCTIAhVPrxp9uzZmDFjhvpnlUqFjIwMNGjQQK8HPukiJycHgYGBSE5O5rCSmfHei4f3Xjy89+LhvTc/QRCQm5sLf3//GtuKFlS8vLxga2tbqfckLS2tUi/LI46OjnB0dNQ4Vr9+fVOVCABwd3fnf7gi4b0XD++9eHjvxcN7b1419aQ8ItqqHwcHB4SFhWHfvn0ax/ft24eePXuKVBURERFJiahDPzNmzMDEiRPRpUsXhIeHY9WqVbh9+zZef/11McsiIiIiiRA1qLzwwgt4+PAhFi9ejHv37iE0NBQ7d+5EUFCQmGUBKBtmWrBgQaWhJjI93nvx8N6Lh/dePLz30mbRW+gTERFR3Sb6FvpEREREVWFQISIiIsliUCEiIiLJYlAhIiIiyWJQ0WLFihVo0qQJnJycEBYWhiNHjohdkqRFRUVh9OjR8Pf3h0wmw5YtWzReFwQBCxcuhL+/P5ydndG/f39cvHhRo41CocBbb70FLy8vuLi44Mknn0RKSopGm8zMTEycOBFyuRxyuRwTJ05EVlaWRpvbt29j9OjRcHFxgZeXF/71r3+huLjYFH9sSYiIiEDXrl3h5uYGb29vjBkzBgkJCRpteP9NY+XKlWjfvr16k7Dw8HDs2rVL/Trvu/lERERAJpNh+vTp6mO8/3WIQBrWr18v2NvbC999951w6dIl4e233xZcXFyEW7duiV2aZO3cuVOYO3eusHHjRgGAsHnzZo3Xly5dKri5uQkbN24U4uLihBdeeEHw8/MTcnJy1G1ef/11oVGjRsK+ffuEc+fOCQMGDBA6dOgglJaWqtsMGzZMCA0NFY4fPy4cP35cCA0NFUaNGqV+vbS0VAgNDRUGDBggnDt3Tti3b5/g7+8vTJs2zeT3QCxDhw4VVq9eLcTHxwsxMTHCyJEjhcaNGwt5eXnqNrz/prFt2zZhx44dQkJCgpCQkCDMmTNHsLe3F+Lj4wVB4H03l9OnTwvBwcFC+/bthbffflt9nPe/7mBQqaBbt27C66+/rnGsVatWwvvvvy9SRZalYlBRqVSCr6+vsHTpUvWxoqIiQS6XC998840gCIKQlZUl2NvbC+vXr1e3uXPnjmBjYyPs3r1bEARBuHTpkgBAOHnypLrNiRMnBADClStXBEEoC0w2NjbCnTt31G3WrVsnODo6CtnZ2Sb580pNWlqaAECIjIwUBIH339w8PDyE77//nvfdTHJzc4XmzZsL+/btE/r166cOKrz/dQuHfsopLi5GdHQ0hgwZonF8yJAhOH78uEhVWbbExESkpqZq3FNHR0f069dPfU+jo6NRUlKi0cbf3x+hoaHqNidOnIBcLkf37t3VbXr06AG5XK7RJjQ0VOMhV0OHDoVCoUB0dLRJ/5xSkZ2dDQDw9PQEwPtvLkqlEuvXr0d+fj7Cw8N5381k6tSpGDlyJAYPHqxxnPe/bhH96clSkp6eDqVSWemhiD4+PpUenki6eXTftN3TW7duqds4ODjAw8OjUptH709NTYW3t3el83t7e2u0qXgdDw8PODg4WMXfnyAImDFjBnr37o3Q0FAAvP+mFhcXh/DwcBQVFcHV1RWbN29GmzZt1B9ivO+ms379epw7dw5nzpyp9Br/u69bGFS0kMlkGj8LglDpGOnHkHtasY229oa0qaumTZuG2NhYHD16tNJrvP+m0bJlS8TExCArKwsbN27EpEmTEBkZqX6d9900kpOT8fbbb2Pv3r1wcnKqsh3vf93AoZ9yvLy8YGtrWykFp6WlVUrMpBtfX18AqPae+vr6ori4GJmZmdW2uX//fqXzP3jwQKNNxetkZmaipKSkzv/9vfXWW9i2bRsOHTqEgIAA9XHef9NycHBASEgIunTpgoiICHTo0AGff/4577uJRUdHIy0tDWFhYbCzs4OdnR0iIyPxxRdfwM7OTv3n5v2vGxhUynFwcEBYWBj27duncXzfvn3o2bOnSFVZtiZNmsDX11fjnhYXFyMyMlJ9T8PCwmBvb6/R5t69e4iPj1e3CQ8PR3Z2Nk6fPq1uc+rUKWRnZ2u0iY+Px71799Rt9u7dC0dHR4SFhZn0zykWQRAwbdo0bNq0CQcPHkSTJk00Xuf9Ny9BEKBQKHjfTWzQoEGIi4tDTEyM+qtLly6YMGECYmJi0LRpU97/usS8c3el79Hy5B9++EG4dOmSMH36dMHFxUVISkoSuzTJys3NFc6fPy+cP39eACAsW7ZMOH/+vHpJ99KlSwW5XC5s2rRJiIuLE8aPH691mWBAQICwf/9+4dy5c8LAgQO1LhNs3769cOLECeHEiRNCu3bttC4THDRokHDu3Dlh//79QkBAQJ1eJvjGG28IcrlcOHz4sHDv3j31V0FBgboN779pzJ49W4iKihISExOF2NhYYc6cOYKNjY2wd+9eQRB4382t/KofQeD9r0sYVLT4+uuvhaCgIMHBwUHo3LmzeqknaXfo0CEBQKWvSZMmCYJQtlRwwYIFgq+vr+Do6Cj07dtXiIuL0zhHYWGhMG3aNMHT01NwdnYWRo0aJdy+fVujzcOHD4UJEyYIbm5ugpubmzBhwgQhMzNTo82tW7eEkSNHCs7OzoKnp6cwbdo0oaioyJR/fFFpu+8AhNWrV6vb8P6bxssvv6z+d6Jhw4bCoEGD1CFFEHjfza1iUOH9rztkgiAI4vTlEBEREVWPc1SIiIhIshhUiIiISLIYVIiIiEiyGFSIiIhIshhUiIiISLIYVIiIiEiyGFSIiIhIshhUiMhi/PTTT6hfv77YZRCRGTGoEJFeJk+eDJlMpv5q0KABhg0bhtjYWL3Os3DhQnTs2NE0RVZj6tSpmDNnDgDgo48+wssvv2z2GohIdwwqRKS3YcOG4d69e7h37x4OHDgAOzs7jBo1SuyydHLixAn06tULAHD06FH190QkTQwqRKQ3R0dH+Pr6wtfXFx07dsSsWbOQnJyMBw8eqNvMmjULLVq0QL169dC0aVPMmzcPJSUlAMqGcBYtWoQLFy6oe2Z++uknAEBWVhZeffVV+Pj4wMnJCaGhodi+fbvG9ffs2YPWrVvD1dVVHZp0kZ+fj/j4eISHh0OlUmmEFiKSJjuxCyAiy5aXl4e1a9ciJCQEDRo0UB93c3PDTz/9BH9/f8TFxWHKlClwc3PDzJkz8cILLyA+Ph67d+/G/v37AQByuRwqlQrDhw9Hbm4u1qxZg2bNmuHSpUuwtbVVn7egoAD/+9//8Ouvv8LGxgYvvvgi3nvvPaxdu7bKGt9880389ttvUKlUKCkpQdOmTSEIAnJyctCjRw8AQGxsLBo3bmyiu0REhmJQISK9bd++Ha6urgDKein8/Pywfft22Ng87qT94IMP1N8HBwfj3XffxYYNGzBz5kw4OzvD1dUVdnZ28PX1Vbfbu3cvTp8+jcuXL6NFixYAgKZNm2pcu6SkBN988w2aNWsGAJg2bRoWL15cbb2LFy/GzJkzsWTJEnVtq1atwpUrV7Bs2TIAgL+/v6G3g4hMiEGFiPQ2YMAArFy5EgCQkZGBFStWYPjw4Th9+jSCgoIAAH/++SeWL1+O69evIy8vD6WlpXB3d6/2vDExMQgICFCHFG3q1aunDikA4Ofnh7S0tGrP6+XlBS8vLxw/fhyff/45goODcebMGUyaNAnBwcE6/qmJSAyco0JEenNxcUFISAhCQkLQrVs3/PDDD8jPz8d3330HADh58iTGjRuH4cOHY/v27Th//jzmzp2L4uLias/r7Oxc47Xt7e01fpbJZBAEocr2a9euhaurK1xdXXH58mWMGTMGrq6uOHDgAF599VW4urpWO2xEROJijwoR1ZpMJoONjQ0KCwsBAMeOHUNQUBDmzp2rbnPr1i2N9zg4OECpVGoca9++PVJSUnD16tVqe1X08eSTT6J79+7YunUrNm3ahJ9//hnHjx/HkiVLsHPnTgCAj4+PUa5FRMbHoEJEelMoFEhNTQUAZGZm4quvvkJeXh5Gjx4NAAgJCcHt27exfv16dO3aFTt27MDmzZs1zhEcHIzExET1cI+bmxv69euHvn37YuzYsVi2bBlCQkJw5coVyGQyDBs2zKBa3dzc4ObmhmvXrmHw4MEICQnBb7/9hgEDBiAkJKR2N4KITI5DP0Skt927d8PPzw9+fn7o3r07zpw5gz/++AP9+/cHADz11FN45513MG3aNHTs2BHHjx/HvHnzNM4xduxYDBs2DAMGDEDDhg2xbt06AMDGjRvRtWtXjB8/Hm3atMHMmTMr9bwY4vDhw+jbty8AIDIyUv09EUmbTKhucJeIiIhIROxRISIiIsliUCEiIiLJYlAhIiIiyWJQISIiIsliUCEiIiLJYlAhIiIiyWJQISIiIsliUCEiIiLJYlAhIiIiyWJQISIiIsliUCEiIiLJYlAhIiIiyfp/VK70p39kYHoAAAAASUVORK5CYII=\n"},"metadata":{}}]},{"cell_type":"markdown","source":"Translation","metadata":{"id":"ygvrNyB-DvUF"}},{"cell_type":"code","source":"class Translator(tf.Module):\n\n def __init__(self, encoder, decoder, input_text_processor,\n output_text_processor):\n self.encoder = encoder\n self.decoder = decoder\n self.input_text_processor = input_text_processor\n self.output_text_processor = output_text_processor\n\n self.output_token_string_from_index = (\n tf.keras.layers.StringLookup(\n vocabulary=output_text_processor.get_vocabulary(),\n mask_token='',\n invert=True))\n\n # The output should never generate padding, unknown, or start.\n index_from_string = tf.keras.layers.StringLookup(\n vocabulary=output_text_processor.get_vocabulary(), mask_token='')\n token_mask_ids = index_from_string(['', '[UNK]', '[START]']).numpy()\n\n token_mask = np.zeros([index_from_string.vocabulary_size()], dtype=np.bool)\n token_mask[np.array(token_mask_ids)] = True\n self.token_mask = token_mask\n\n self.start_token = index_from_string(tf.constant('[START]'))\n self.end_token = index_from_string(tf.constant('[END]'))","metadata":{"id":"PO-CLL1LVBbM","execution":{"iopub.status.busy":"2023-03-16T17:39:56.628844Z","iopub.execute_input":"2023-03-16T17:39:56.629611Z","iopub.status.idle":"2023-03-16T17:39:56.637695Z","shell.execute_reply.started":"2023-03-16T17:39:56.629566Z","shell.execute_reply":"2023-03-16T17:39:56.636484Z"},"trusted":true},"execution_count":148,"outputs":[]},{"cell_type":"code","source":"translator = Translator(\n encoder=train_translator.encoder,\n decoder=train_translator.decoder,\n input_text_processor=input_text_processor,\n output_text_processor=output_text_processor,\n)","metadata":{"id":"iBQzFZ9uWU79","execution":{"iopub.status.busy":"2023-03-16T17:39:56.793148Z","iopub.execute_input":"2023-03-16T17:39:56.794470Z","iopub.status.idle":"2023-03-16T17:39:57.546904Z","shell.execute_reply.started":"2023-03-16T17:39:56.794408Z","shell.execute_reply":"2023-03-16T17:39:57.544201Z"},"trusted":true},"execution_count":149,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:21: DeprecationWarning: `np.bool` is a deprecated alias for the builtin `bool`. To silence this warning, use `bool` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.bool_` here.\nDeprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n","output_type":"stream"}]},{"cell_type":"code","source":"# converting token IDs to human readable text\ndef tokens_to_text(self, result_tokens):\n shape_checker = ShapeChecker()\n shape_checker(result_tokens, ('batch', 't'))\n result_text_tokens = self.output_token_string_from_index(result_tokens)\n shape_checker(result_text_tokens, ('batch', 't'))\n\n result_text = tf.strings.reduce_join(result_text_tokens,\n axis=1, separator=' ')\n shape_checker(result_text, ('batch'))\n\n result_text = tf.strings.strip(result_text)\n shape_checker(result_text, ('batch',))\n return result_text","metadata":{"id":"8IjwKTwtmdFf","execution":{"iopub.status.busy":"2023-03-16T17:39:57.549319Z","iopub.execute_input":"2023-03-16T17:39:57.549740Z","iopub.status.idle":"2023-03-16T17:39:57.557163Z","shell.execute_reply.started":"2023-03-16T17:39:57.549696Z","shell.execute_reply":"2023-03-16T17:39:57.555921Z"},"trusted":true},"execution_count":150,"outputs":[]},{"cell_type":"code","source":"Translator.tokens_to_text = tokens_to_text","metadata":{"id":"912aV0K7r90w","execution":{"iopub.status.busy":"2023-03-16T17:39:57.558702Z","iopub.execute_input":"2023-03-16T17:39:57.559697Z","iopub.status.idle":"2023-03-16T17:39:57.566470Z","shell.execute_reply.started":"2023-03-16T17:39:57.559651Z","shell.execute_reply":"2023-03-16T17:39:57.565321Z"},"trusted":true},"execution_count":151,"outputs":[]},{"cell_type":"code","source":"# inputting random token IDs to see what it generates\nexample_output_tokens = tf.random.uniform(\n shape=[5, 2], minval=0, dtype=tf.int64,\n maxval=output_text_processor.vocabulary_size())\ntranslator.tokens_to_text(example_output_tokens).numpy()","metadata":{"id":"cWCMHdoS32QN","outputId":"b49a9c58-b11b-4aa3-e127-3a7b4f5b98d8","execution":{"iopub.status.busy":"2023-03-16T17:39:57.568863Z","iopub.execute_input":"2023-03-16T17:39:57.569707Z","iopub.status.idle":"2023-03-16T17:39:57.619101Z","shell.execute_reply.started":"2023-03-16T17:39:57.569665Z","shell.execute_reply":"2023-03-16T17:39:57.618059Z"},"trusted":true},"execution_count":152,"outputs":[{"execution_count":152,"output_type":"execute_result","data":{"text/plain":"array([b'suncream priest', b'imupro heber', b'milles cgs', b'dudley br',\n b'handsome corvallis'], dtype=object)"},"metadata":{}}]},{"cell_type":"code","source":"# takes the decoder's logit outputs and samples token IDs from that distribution\ndef sample(self, logits, temperature):\n shape_checker = ShapeChecker()\n # 't' is usually 1 here.\n shape_checker(logits, ('batch', 't', 'vocab'))\n shape_checker(self.token_mask, ('vocab',))\n\n token_mask = self.token_mask[tf.newaxis, tf.newaxis, :]\n shape_checker(token_mask, ('batch', 't', 'vocab'), broadcast=True)\n\n # Set the logits for all masked tokens to -inf, so they are never chosen.\n logits = tf.where(self.token_mask, -np.inf, logits)\n\n if temperature == 0.0:\n new_tokens = tf.argmax(logits, axis=-1)\n else: \n logits = tf.squeeze(logits, axis=1)\n new_tokens = tf.random.categorical(logits/temperature,\n num_samples=1)\n \n shape_checker(new_tokens, ('batch', 't'))\n\n return new_tokens","metadata":{"id":"8lfuj3GcdD6e","execution":{"iopub.status.busy":"2023-03-16T17:39:57.621086Z","iopub.execute_input":"2023-03-16T17:39:57.621913Z","iopub.status.idle":"2023-03-16T17:39:57.630581Z","shell.execute_reply.started":"2023-03-16T17:39:57.621874Z","shell.execute_reply":"2023-03-16T17:39:57.629751Z"},"trusted":true},"execution_count":153,"outputs":[]},{"cell_type":"code","source":"Translator.sample = sample","metadata":{"id":"4DpDnBdBdL9_","execution":{"iopub.status.busy":"2023-03-16T17:39:57.789040Z","iopub.execute_input":"2023-03-16T17:39:57.789406Z","iopub.status.idle":"2023-03-16T17:39:57.796297Z","shell.execute_reply.started":"2023-03-16T17:39:57.789371Z","shell.execute_reply":"2023-03-16T17:39:57.794960Z"},"trusted":true},"execution_count":154,"outputs":[]},{"cell_type":"code","source":"# testing the function on random i/p\nexample_logits = tf.random.normal([5, 1, output_text_processor.vocabulary_size()])\nexample_output_tokens = translator.sample(example_logits, temperature=1.0)\nexample_output_tokens","metadata":{"id":"rwLT0nxXym80","outputId":"492baa1e-7a3f-4ee3-a0cb-4df725ece439","execution":{"iopub.status.busy":"2023-03-16T17:39:58.065949Z","iopub.execute_input":"2023-03-16T17:39:58.067146Z","iopub.status.idle":"2023-03-16T17:39:58.091672Z","shell.execute_reply.started":"2023-03-16T17:39:58.067091Z","shell.execute_reply":"2023-03-16T17:39:58.090659Z"},"trusted":true},"execution_count":155,"outputs":[{"execution_count":155,"output_type":"execute_result","data":{"text/plain":""},"metadata":{}}]},{"cell_type":"code","source":"# implementing text to text translation loop\ndef translate_unrolled(self,\n input_text, *,\n max_length=50,\n return_attention=True,\n temperature=1.0):\n batch_size = tf.shape(input_text)[0]\n input_tokens = self.input_text_processor(input_text)\n enc_output, enc_state = self.encoder(input_tokens)\n\n dec_state = enc_state\n new_tokens = tf.fill([batch_size, 1], self.start_token)\n\n result_tokens = []\n attention = []\n done = tf.zeros([batch_size, 1], dtype=tf.bool)\n\n for _ in range(max_length):\n dec_input = DecoderInput(new_tokens=new_tokens,\n enc_output=enc_output,\n mask=(input_tokens!=0))\n \n dec_result, dec_state = self.decoder(dec_input, state=dec_state)\n\n attention.append(dec_result.attention_weights)\n\n new_tokens = self.sample(dec_result.logits, temperature)\n\n # If a sequence produces an `end_token`, set it `done`\n done = done | (new_tokens == self.end_token)\n # Once a sequence is done it only produces 0-padding.\n new_tokens = tf.where(done, tf.constant(0, dtype=tf.int64), new_tokens)\n\n # Collect the generated tokens\n result_tokens.append(new_tokens)\n\n if tf.executing_eagerly() and tf.reduce_all(done):\n break\n\n # Convert the list of generates token ids to a list of strings.\n result_tokens = tf.concat(result_tokens, axis=-1)\n result_text = self.tokens_to_text(result_tokens)\n\n if return_attention:\n attention_stack = tf.concat(attention, axis=1)\n return {'text': result_text, 'attention': attention_stack}\n else:\n return {'text': result_text}\n","metadata":{"id":"ZmOvVrZmwAxg","execution":{"iopub.status.busy":"2023-03-16T17:39:58.289827Z","iopub.execute_input":"2023-03-16T17:39:58.291300Z","iopub.status.idle":"2023-03-16T17:39:58.320405Z","shell.execute_reply.started":"2023-03-16T17:39:58.291246Z","shell.execute_reply":"2023-03-16T17:39:58.319212Z"},"trusted":true},"execution_count":156,"outputs":[]},{"cell_type":"code","source":"Translator.translate = translate_unrolled","metadata":{"id":"JOmd8Y269MG3","execution":{"iopub.status.busy":"2023-03-16T17:39:58.570219Z","iopub.execute_input":"2023-03-16T17:39:58.571429Z","iopub.status.idle":"2023-03-16T17:39:58.577001Z","shell.execute_reply.started":"2023-03-16T17:39:58.571373Z","shell.execute_reply":"2023-03-16T17:39:58.575644Z"},"trusted":true},"execution_count":157,"outputs":[]},{"cell_type":"code","source":"change_vocab('tum kaise ho')","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:39:58.882755Z","iopub.execute_input":"2023-03-16T17:39:58.883159Z","iopub.status.idle":"2023-03-16T17:39:58.892384Z","shell.execute_reply.started":"2023-03-16T17:39:58.883123Z","shell.execute_reply":"2023-03-16T17:39:58.891112Z"},"trusted":true},"execution_count":158,"outputs":[{"execution_count":158,"output_type":"execute_result","data":{"text/plain":"'tum kassie hoo '"},"metadata":{}}]},{"cell_type":"code","source":"# TESTING\ninput_text = tf.constant([\n change_vocab('mera naam rohan hai'),\n change_vocab('kaha jana hai'),\n change_vocab('tumhare papa kya kaam karte hai '),\n change_vocab('tumhare papa ka naam kya hai'),\n change_vocab('maine khana kha liya hai')\n])\n\nresult = translator.translate(\n input_text = input_text)\n\nprint(result['text'][0].numpy().decode())\nprint(result['text'][1].numpy().decode())\nprint(result['text'][2].numpy().decode())\nprint(result['text'][3].numpy().decode())\nprint(result['text'][4].numpy().decode())\nprint()","metadata":{"id":"hd2rgyHwVVrv","outputId":"72a81edc-d510-4038-fc8b-527a5b824127","execution":{"iopub.status.busy":"2023-03-16T17:40:15.051171Z","iopub.execute_input":"2023-03-16T17:40:15.052400Z","iopub.status.idle":"2023-03-16T17:40:16.039222Z","shell.execute_reply.started":"2023-03-16T17:40:15.052348Z","shell.execute_reply":"2023-03-16T17:40:16.038061Z"},"trusted":true},"execution_count":161,"outputs":[{"name":"stdout","text":"my name is loud\nwhere is it ?\ndo you do your papa . ?\nis there a name of your father ?\ni have to take\n\n","output_type":"stream"}]},{"cell_type":"code","source":"@tf.function(input_signature=[tf.TensorSpec(dtype=tf.string, shape=[None])])\ndef tf_translate(self, input_text):\n return self.translate(input_text)\n\nTranslator.tf_translate = tf_translate","metadata":{"id":"_JhTZ5hOptO-","execution":{"iopub.status.busy":"2023-03-16T17:43:54.731553Z","iopub.execute_input":"2023-03-16T17:43:54.733482Z","iopub.status.idle":"2023-03-16T17:43:54.739930Z","shell.execute_reply.started":"2023-03-16T17:43:54.733439Z","shell.execute_reply":"2023-03-16T17:43:54.738640Z"},"trusted":true},"execution_count":164,"outputs":[]},{"cell_type":"code","source":"tf.saved_model.save(translator, 'translator1',\n signatures={'serving_default': translator.tf_translate})","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:43:55.442867Z","iopub.execute_input":"2023-03-16T17:43:55.443608Z","iopub.status.idle":"2023-03-16T17:44:17.215533Z","shell.execute_reply.started":"2023-03-16T17:43:55.443567Z","shell.execute_reply":"2023-03-16T17:44:17.214346Z"},"trusted":true},"execution_count":165,"outputs":[]},{"cell_type":"code","source":"%%time\nresult = translator.tf_translate(\n input_text = input_text)","metadata":{"id":"_NzrixLvVBjQ","outputId":"469e8d6c-c011-43b1-ead8-0f442e9c602f","execution":{"iopub.status.busy":"2023-03-16T11:27:34.224701Z","iopub.execute_input":"2023-03-16T11:27:34.225096Z","iopub.status.idle":"2023-03-16T11:27:54.830320Z","shell.execute_reply.started":"2023-03-16T11:27:34.225060Z","shell.execute_reply":"2023-03-16T11:27:54.829264Z"},"trusted":true},"execution_count":125,"outputs":[{"name":"stdout","text":"CPU times: user 20.5 s, sys: 90.8 ms, total: 20.6 s\nWall time: 20.6 s\n","output_type":"stream"}]},{"cell_type":"code","source":"%%time\nresult = translator.tf_translate(\n input_text = input_text)\n\nprint(result['text'][0].numpy().decode())\nprint(result['text'][1].numpy().decode())\nprint()","metadata":{"id":"USJdu00tVFbd","outputId":"9c40b1e8-a2a9-4ad1-9992-430dd580d8fb","execution":{"iopub.status.busy":"2023-03-16T17:46:33.012957Z","iopub.execute_input":"2023-03-16T17:46:33.013943Z","iopub.status.idle":"2023-03-16T17:46:53.566845Z","shell.execute_reply.started":"2023-03-16T17:46:33.013903Z","shell.execute_reply":"2023-03-16T17:46:53.565732Z"},"trusted":true},"execution_count":170,"outputs":[{"name":"stdout","text":"everyone is in my name ?\nwhere is it ?\n\nCPU times: user 20.3 s, sys: 243 ms, total: 20.5 s\nWall time: 20.5 s\n","output_type":"stream"}]},{"cell_type":"code","source":"#@title [Optional] Use a symbolic loop\ndef translate_symbolic(self,\n input_text,\n *,\n max_length=50,\n return_attention=True,\n temperature=1.0):\n shape_checker = ShapeChecker()\n shape_checker(input_text, ('batch',))\n\n batch_size = tf.shape(input_text)[0]\n\n # Encode the input\n input_tokens = self.input_text_processor(input_text)\n shape_checker(input_tokens, ('batch', 's'))\n\n enc_output, enc_state = self.encoder(input_tokens)\n shape_checker(enc_output, ('batch', 's', 'enc_units'))\n shape_checker(enc_state, ('batch', 'enc_units'))\n\n # Initialize the decoder\n dec_state = enc_state\n new_tokens = tf.fill([batch_size, 1], self.start_token)\n shape_checker(new_tokens, ('batch', 't1'))\n\n # Initialize the accumulators\n result_tokens = tf.TensorArray(tf.int64, size=1, dynamic_size=True)\n attention = tf.TensorArray(tf.float32, size=1, dynamic_size=True)\n done = tf.zeros([batch_size, 1], dtype=tf.bool)\n shape_checker(done, ('batch', 't1'))\n\n for t in tf.range(max_length):\n dec_input = DecoderInput(\n new_tokens=new_tokens, enc_output=enc_output, mask=(input_tokens != 0))\n\n dec_result, dec_state = self.decoder(dec_input, state=dec_state)\n\n shape_checker(dec_result.attention_weights, ('batch', 't1', 's'))\n attention = attention.write(t, dec_result.attention_weights)\n\n new_tokens = self.sample(dec_result.logits, temperature)\n shape_checker(dec_result.logits, ('batch', 't1', 'vocab'))\n shape_checker(new_tokens, ('batch', 't1'))\n\n # If a sequence produces an `end_token`, set it `done`\n done = done | (new_tokens == self.end_token)\n # Once a sequence is done it only produces 0-padding.\n new_tokens = tf.where(done, tf.constant(0, dtype=tf.int64), new_tokens)\n\n # Collect the generated tokens\n result_tokens = result_tokens.write(t, new_tokens)\n\n if tf.reduce_all(done):\n break\n\n # Convert the list of generated token ids to a list of strings.\n result_tokens = result_tokens.stack()\n shape_checker(result_tokens, ('t', 'batch', 't0'))\n result_tokens = tf.squeeze(result_tokens, -1)\n result_tokens = tf.transpose(result_tokens, [1, 0])\n shape_checker(result_tokens, ('batch', 't'))\n\n result_text = self.tokens_to_text(result_tokens)\n shape_checker(result_text, ('batch',))\n\n if return_attention:\n attention_stack = attention.stack()\n shape_checker(attention_stack, ('t', 'batch', 't1', 's'))\n\n attention_stack = tf.squeeze(attention_stack, 2)\n shape_checker(attention_stack, ('t', 'batch', 's'))\n\n attention_stack = tf.transpose(attention_stack, [1, 0, 2])\n shape_checker(attention_stack, ('batch', 't', 's'))\n\n return {'text': result_text, 'attention': attention_stack}\n else:\n return {'text': result_text}","metadata":{"id":"EbQpyYs13jF_","execution":{"iopub.status.busy":"2023-03-16T17:46:53.569825Z","iopub.execute_input":"2023-03-16T17:46:53.570556Z","iopub.status.idle":"2023-03-16T17:46:53.586068Z","shell.execute_reply.started":"2023-03-16T17:46:53.570509Z","shell.execute_reply":"2023-03-16T17:46:53.584852Z"},"trusted":true},"execution_count":171,"outputs":[]},{"cell_type":"code","source":"Translator.translate = translate_symbolic","metadata":{"id":"ngywxv1WYO_O","execution":{"iopub.status.busy":"2023-03-16T17:46:53.587484Z","iopub.execute_input":"2023-03-16T17:46:53.588398Z","iopub.status.idle":"2023-03-16T17:46:53.605490Z","shell.execute_reply.started":"2023-03-16T17:46:53.588358Z","shell.execute_reply":"2023-03-16T17:46:53.604434Z"},"trusted":true},"execution_count":172,"outputs":[]},{"cell_type":"markdown","source":"The initial implementation used python lists to collect the outputs. This uses `tf.range` as the loop iterator, allowing `tf.autograph` to convert the loop. The biggest change in this implementation is the use of `tf.TensorArray` instead of python `list` to accumulate tensors. `tf.TensorArray` is required to collect a variable number of tensors in graph mode. ","metadata":{"id":"lItV7qjEGsYc"}},{"cell_type":"markdown","source":"With eager execution this implementation performs on par with the original:","metadata":{"id":"AJ_NznOgZTxC"}},{"cell_type":"code","source":"# RUN THIS\nresult = translator.translate(\n input_text = input_text)\n\nprint(result['text'][0].numpy().decode())\nprint(result['text'][1].numpy().decode())\nprint()","metadata":{"id":"JRh66y-YYeBw","outputId":"8992e4bf-e4c7-47cb-84a4-0465f2153d83","execution":{"iopub.status.busy":"2023-03-16T17:46:53.608455Z","iopub.execute_input":"2023-03-16T17:46:53.608896Z","iopub.status.idle":"2023-03-16T17:46:54.641135Z","shell.execute_reply.started":"2023-03-16T17:46:53.608858Z","shell.execute_reply":"2023-03-16T17:46:54.640057Z"},"trusted":true},"execution_count":173,"outputs":[{"name":"stdout","text":"peace is as loud\nwhere is it ?\n\n","output_type":"stream"}]},{"cell_type":"markdown","source":"But when you wrap it in a `tf.function` you'll notice two differences.","metadata":{"id":"l6B8W4_MZdX0"}},{"cell_type":"code","source":"@tf.function(input_signature=[tf.TensorSpec(dtype=tf.string, shape=[None])])\ndef tf_translate(self, input_text):\n return self.translate(input_text)\n\nTranslator.tf_translate = tf_translate","metadata":{"id":"WX6EF8KtYh20","execution":{"iopub.status.busy":"2023-03-16T17:46:54.644456Z","iopub.execute_input":"2023-03-16T17:46:54.644768Z","iopub.status.idle":"2023-03-16T17:46:54.651847Z","shell.execute_reply.started":"2023-03-16T17:46:54.644737Z","shell.execute_reply":"2023-03-16T17:46:54.650594Z"},"trusted":true},"execution_count":174,"outputs":[]},{"cell_type":"markdown","source":"First: Graph creation is much faster (~10x), since it doesn't create `max_iterations` copies of the model.","metadata":{"id":"9S0kQ-bBZswZ"}},{"cell_type":"code","source":"%%time\nresult = translator.tf_translate(\n input_text = input_text)","metadata":{"id":"Eq8d40RKYoJa","outputId":"e8bfbaaa-9b1c-4eaa-d08e-78b76749add4","execution":{"iopub.status.busy":"2023-03-16T17:46:54.653377Z","iopub.execute_input":"2023-03-16T17:46:54.654207Z","iopub.status.idle":"2023-03-16T17:46:57.327429Z","shell.execute_reply.started":"2023-03-16T17:46:54.654122Z","shell.execute_reply":"2023-03-16T17:46:57.326288Z"},"trusted":true},"execution_count":175,"outputs":[{"name":"stdout","text":"CPU times: user 2.58 s, sys: 102 ms, total: 2.68 s\nWall time: 2.66 s\n","output_type":"stream"}]},{"cell_type":"markdown","source":"Second: The compiled function is much faster on small inputs (5x on this example), because it can break out of the loop.","metadata":{"id":"2ABEwtKIZ6eE"}},{"cell_type":"code","source":"%%time\nresult = translator.tf_translate(\n input_text = input_text)\n\nprint(result['text'][0].numpy().decode())\nprint(result['text'][1].numpy().decode())\nprint()","metadata":{"id":"d5VdCLxPYrpz","outputId":"b0e36859-576d-4d40-c6df-efb0b3f7a4c6","execution":{"iopub.status.busy":"2023-03-16T17:46:57.329281Z","iopub.execute_input":"2023-03-16T17:46:57.329764Z","iopub.status.idle":"2023-03-16T17:46:57.390001Z","shell.execute_reply.started":"2023-03-16T17:46:57.329722Z","shell.execute_reply":"2023-03-16T17:46:57.389026Z"},"trusted":true},"execution_count":176,"outputs":[{"name":"stdout","text":"my name is loud\nwhere is it ?\n\nCPU times: user 57 ms, sys: 16.7 ms, total: 73.8 ms\nWall time: 53.4 ms\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Visualize the process","metadata":{"id":"eo5sf4jZaO2l"}},{"cell_type":"markdown","source":"The attention weights returned by the `translate` method show where the model was \"looking\" when it generated each output token.\n\nSo the sum of the attention over the input should return all ones:","metadata":{"id":"FzZzC2cJacTv"}},{"cell_type":"code","source":"a = result['attention'][0]\n\nprint(np.sum(a, axis=-1))","metadata":{"id":"UEd2GljgqQ-0","outputId":"79eecf74-f06e-4d9a-ec44-5338c4df2c8c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here is the attention distribution for the first output step of the first example. Note how the attention is now much more focused than it was for the untrained model:","metadata":{"id":"k_HWQHcI2_h5"}},{"cell_type":"code","source":"_ = plt.bar(range(len(a[0, :])), a[0, :])","metadata":{"id":"M8BHdqQujALu","outputId":"31b6618c-07e0-4821-c953-efe35ddc95e5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since there is some rough alignment between the input and output words, you expect the attention to be focused near the diagonal:","metadata":{"id":"qB13OG472Z3V"}},{"cell_type":"code","source":"plt.imshow(np.array(a), vmin=0.0)","metadata":{"id":"xyeXuEYHd0kQ","outputId":"8e85df2f-bfc4-43db-abd2-412001afb29e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here is some code to make a better attention plot:","metadata":{"id":"mXECcNTn2mxN"}},{"cell_type":"code","source":"#@title Labeled attention plots\ndef plot_attention(attention, sentence, predicted_sentence):\n sentence = tf_lower_and_split_punct(sentence).numpy().decode().split()\n predicted_sentence = predicted_sentence.numpy().decode().split() + ['[END]']\n fig = plt.figure(figsize=(10, 10))\n ax = fig.add_subplot(1, 1, 1)\n\n attention = attention[:len(predicted_sentence), :len(sentence)]\n\n ax.matshow(attention, cmap='viridis', vmin=0.0)\n\n fontdict = {'fontsize': 14}\n\n ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n\n ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n\n ax.set_xlabel('Input text')\n ax.set_ylabel('Output text')\n plt.suptitle('Attention weights')","metadata":{"id":"s5hQWlbN3jGF","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=0\nplot_attention(result['attention'][i], input_text[i], result['text'][i])","metadata":{"id":"rrGawQv2eiA4","outputId":"a8ce33b3-cfaf-49f9-c04f-2f6e89d90eeb","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nthree_input_text = tf.constant([#'sabse best friend', \n #'That toh I know',\n #change_vocab('She was bhunnoing the masalas jub phone ki ghuntee bugee'),\n change_vocab('tum batao'),\n change_vocab('kya karna hai'),\n])\n\nresult = translator.tf_translate(three_input_text)\nprint(result['text'])\nfor tr in result['text']:\n print(tr.numpy().decode())\n\nprint()","metadata":{"id":"WrAM0FDomq3E","outputId":"8641124a-e81a-4aea-ff61-51425a2e0b42","execution":{"iopub.status.busy":"2023-03-16T17:46:57.391628Z","iopub.execute_input":"2023-03-16T17:46:57.392024Z","iopub.status.idle":"2023-03-16T17:46:57.421652Z","shell.execute_reply.started":"2023-03-16T17:46:57.391965Z","shell.execute_reply":"2023-03-16T17:46:57.420668Z"},"trusted":true},"execution_count":177,"outputs":[{"name":"stdout","text":"tf.Tensor([b'you know' b'is there any ?'], shape=(2,), dtype=string)\nyou know\nis there any ?\n\nCPU times: user 26.1 ms, sys: 4.04 ms, total: 30.1 ms\nWall time: 23.2 ms\n","output_type":"stream"}]},{"cell_type":"code","source":"result['text']","metadata":{"id":"-LjFp0AljOaZ","outputId":"8f3de323-4809-4d52-8092-bac9d1e1f8fc","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 0\nplot_attention(result['attention'][i], three_input_text[i], result['text'][i])","metadata":{"id":"v7QwIMrG-id2","outputId":"02b9f2a4-4ea0-4b45-a294-283b21e1cec9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 1\nplot_attention(result['attention'][i], three_input_text[i], result['text'][i])","metadata":{"id":"zYVoVf8P-lr-","outputId":"59fde2f6-58e3-4cfa-e9b1-f2bba1a872a4","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = 2\nplot_attention(result['attention'][i], three_input_text[i], result['text'][i])","metadata":{"id":"9sFvlZBk-me4","outputId":"7e25b713-c576-4c03-8614-c55bb3c7d83e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"long_input_text = tf.constant([inp[-1]])\n\nimport textwrap\nprint('Expected output:\\n', '\\n'.join(textwrap.wrap(targ[-1])))","metadata":{"id":"-FUHFLEvSMbG","outputId":"4ea7bcc6-e870-4959-dc40-dd0e6b1ff60a","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = translator.tf_translate(long_input_text)\n\ni = 0\nplot_attention(result['attention'][i], long_input_text[i], result['text'][i])\n_ = plt.suptitle('Graph')","metadata":{"id":"lDa_8NaN_RUy","outputId":"e8e9580b-e815-465d-dc0b-e42875d3399c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.saved_model.save(translator, 'translator1',\n signatures={'serving_default': translator.tf_translate})","metadata":{"id":"OyvxT5V0_X5B","outputId":"ced85551-7dd8-48b7-cf12-b37055831235","execution":{"iopub.status.busy":"2023-03-16T17:47:01.584098Z","iopub.execute_input":"2023-03-16T17:47:01.584497Z","iopub.status.idle":"2023-03-16T17:47:06.183442Z","shell.execute_reply.started":"2023-03-16T17:47:01.584463Z","shell.execute_reply":"2023-03-16T17:47:06.182299Z"},"trusted":true},"execution_count":178,"outputs":[]},{"cell_type":"code","source":"!cd /kaggle/working","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:47:06.190128Z","iopub.execute_input":"2023-03-16T17:47:06.190452Z","iopub.status.idle":"2023-03-16T17:47:07.219232Z","shell.execute_reply.started":"2023-03-16T17:47:06.190421Z","shell.execute_reply":"2023-03-16T17:47:07.217784Z"},"trusted":true},"execution_count":179,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:47:07.222283Z","iopub.execute_input":"2023-03-16T17:47:07.223150Z","iopub.status.idle":"2023-03-16T17:47:08.261595Z","shell.execute_reply.started":"2023-03-16T17:47:07.223102Z","shell.execute_reply":"2023-03-16T17:47:08.260165Z"},"trusted":true},"execution_count":180,"outputs":[{"name":"stdout","text":"__notebook_source__.ipynb translator.zip translator1\n","output_type":"stream"}]},{"cell_type":"code","source":"!zip -r translator.zip translator1","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:47:08.265384Z","iopub.execute_input":"2023-03-16T17:47:08.265863Z","iopub.status.idle":"2023-03-16T17:47:19.369936Z","shell.execute_reply.started":"2023-03-16T17:47:08.265818Z","shell.execute_reply":"2023-03-16T17:47:19.368688Z"},"trusted":true},"execution_count":181,"outputs":[{"name":"stdout","text":"updating: translator1/ (stored 0%)\nupdating: translator1/variables/ (stored 0%)\nupdating: translator1/variables/variables.index (deflated 56%)\nupdating: translator1/variables/variables.data-00000-of-00001 (deflated 7%)\nupdating: translator1/saved_model.pb (deflated 74%)\nupdating: translator1/assets/ (stored 0%)\nupdating: translator1/fingerprint.pb (stored 0%)\n","output_type":"stream"}]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'translator.zip')","metadata":{"execution":{"iopub.status.busy":"2023-03-16T17:47:19.372132Z","iopub.execute_input":"2023-03-16T17:47:19.372458Z","iopub.status.idle":"2023-03-16T17:47:19.381320Z","shell.execute_reply.started":"2023-03-16T17:47:19.372424Z","shell.execute_reply":"2023-03-16T17:47:19.380160Z"},"trusted":true},"execution_count":182,"outputs":[{"execution_count":182,"output_type":"execute_result","data":{"text/plain":"/kaggle/working/translator.zip","text/html":"translator.zip
"},"metadata":{}}]},{"cell_type":"code","source":"reloaded = tf.saved_model.load('translator1')\nresult = reloaded.tf_translate(input_text)","metadata":{"id":"-I0j3i3ekOba","execution":{"iopub.status.busy":"2023-03-16T11:27:57.379083Z","iopub.execute_input":"2023-03-16T11:27:57.379373Z","iopub.status.idle":"2023-03-16T11:28:12.032478Z","shell.execute_reply.started":"2023-03-16T11:27:57.379346Z","shell.execute_reply":"2023-03-16T11:28:12.031392Z"},"trusted":true},"execution_count":133,"outputs":[]},{"cell_type":"code","source":"%%time\nresult = reloaded.tf_translate(input_text)\n\nfor tr in result['text']:\n print(tr.numpy().decode())\n\nprint()","metadata":{"id":"GXZF__FZXJCm","outputId":"2473fa25-b3e1-460d-a647-76c1350f2c81","execution":{"iopub.status.busy":"2023-03-16T11:28:12.034087Z","iopub.execute_input":"2023-03-16T11:28:12.034457Z","iopub.status.idle":"2023-03-16T11:28:12.143606Z","shell.execute_reply.started":"2023-03-16T11:28:12.034419Z","shell.execute_reply":"2023-03-16T11:28:12.142646Z"},"trusted":true},"execution_count":134,"outputs":[{"name":"stdout","text":"where to take ?\nwhat are you doing this , whats love\ndoes your name have his sister\nhave food lost\n\nCPU times: user 127 ms, sys: 11 ms, total: 138 ms\nWall time: 104 ms\n","output_type":"stream"}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]} \ No newline at end of file