{ "cells": [ { "cell_type": "markdown", "id": "e28a25ec-df73-4876-a380-1dc12b0f1b64", "metadata": { "id": "e28a25ec-df73-4876-a380-1dc12b0f1b64" }, "source": [ "# Fine-tuning Bioinspired-Llama 3.1 8B\n", "\n", "#### Markus J. Buehler, MIT" ] }, { "cell_type": "code", "execution_count": 1, "id": "d72ae88a-eac6-48e7-8e4a-49b60da52eba", "metadata": { "id": "d72ae88a-eac6-48e7-8e4a-49b60da52eba" }, "outputs": [], "source": [ "!pip install -U -q accelerate peft bitsandbytes transformers trl wandb" ] }, { "cell_type": "code", "execution_count": 2, "id": "aa0a69fb-f742-4d71-9ca8-9afd6e612ca6", "metadata": { "id": "aa0a69fb-f742-4d71-9ca8-9afd6e612ca6" }, "outputs": [], "source": [ "import os\n", "\n", "#os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "\n", "import torch\n", "from tqdm.notebook import tqdm\n", "import copy\n", "\n", "from datasets import load_dataset, load_from_disk\n", "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, HfArgumentParser, TrainingArguments, pipeline\n", "from transformers import logging\n", "\n", "from trl import SFTTrainer\n", "from peft import LoraConfig, PeftModel\n" ] }, { "cell_type": "code", "execution_count": 3, "id": "b3bc138a-c08f-4448-b623-c8ac0468d4cb", "metadata": { "id": "b3bc138a-c08f-4448-b623-c8ac0468d4cb" }, "outputs": [], "source": [ "def generate_response (text_input=\"What is spider silk?\",\n", " system_prompt='',\n", " num_return_sequences=1,\n", " temperature=1., #the higher the temperature, the more creative the model becomes\n", " max_new_tokens=127,device='cuda',\n", " add_special_tokens = False, #since tokenizer.apply_chat_template adds <|begin_of_text|> template already, set to False\n", " num_beams=1,eos_token_id= [\n", " 128001,\n", " 128008,\n", " 128009\n", " ], verbatim=False,\n", " top_k = 50,\n", " top_p = 0.9,\n", " repetition_penalty=1.1,\n", " messages=[],\n", " ):\n", "\n", " if messages==[]: #start new messages dictionary\n", " if system_prompt != '': #include system prompt if provided\n", " messages.extend ([ {\"role\": \"system\", \"content\": system_prompt}, ])\n", " messages.extend ( [ {\"role\": \"user\", \"content\": text_input}, ])\n", "\n", " else: #if messages provided, will extend (make sure to add previous response as assistant message)\n", " messages.append ({\"role\": \"user\", \"content\": text_input})\n", "\n", " text_input = tokenizer.apply_chat_template(\n", " messages,\n", " tokenize=False,\n", " add_generation_prompt=True\n", " )\n", " inputs = tokenizer([text_input], add_special_tokens = add_special_tokens, return_tensors ='pt' ).to(device)\n", " if verbatim:\n", " print (inputs)\n", " with torch.no_grad():\n", " outputs = model.generate(**inputs,\n", " max_new_tokens=max_new_tokens,\n", " temperature=temperature,\n", " num_beams=num_beams,\n", " top_k = top_k,eos_token_id=eos_token_id,\n", " top_p =top_p,\n", " num_return_sequences = num_return_sequences,\n", " do_sample =True, repetition_penalty=repetition_penalty,\n", " )\n", " outputs=outputs[:, inputs[\"input_ids\"].shape[1]:]\n", " return tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True), messages" ] }, { "cell_type": "markdown", "id": "25675672-82ed-45a2-823f-802f9373ca8a", "metadata": { "id": "25675672-82ed-45a2-823f-802f9373ca8a" }, "source": [ "### Define parameters" ] }, { "cell_type": "code", "execution_count": null, "id": "aaaa8bbd-b4af-4474-a2dd-d484b85be81f", "metadata": { "id": "aaaa8bbd-b4af-4474-a2dd-d484b85be81f" }, "outputs": [], "source": [ "################################################################################\n", "# Base model to train from\n", "################################################################################\n", "\n", "# The model that you want to train from the Hugging Face hub\n", "base_model_name = \"lamm-mit/BioinspiredLlama-3-1-8B-128k\"\n", "\n", "# Fine-tuned model name\n", "new_model = \"protein_secondary_structure_predictor\"\n", "\n", "################################################################################\n", "# LoRA parameters\n", "################################################################################\n", "\n", "# LoRA target modules\n", "lora_target_modules=['q_proj','k_proj','v_proj','o_proj','down_proj',\n", " 'up_proj','gate_proj']\n", "\n", "# LoRA attention dimension\n", "lora_r = 64\n", "\n", "# Alpha parameter for LoRA scaling\n", "lora_alpha = 64\n", "\n", "# Dropout probability for LoRA layers\n", "lora_dropout = 0.1\n", "\n", "################################################################################\n", "# bitsandbytes parameters\n", "################################################################################\n", "\n", "# Activate 4-bit precision base model loading\n", "use_4bit = True\n", "\n", "# Compute dtype for 4-bit base models\n", "bnb_4bit_compute_dtype = \"bfloat16\"\n", "\n", "# Quantization type (fp4 or nf4)\n", "bnb_4bit_quant_type = \"nf4\"\n", "\n", "# Activate nested quantization for 4-bit base models, i.e. double quantization\n", "use_nested_quant = False\n", "\n", "################################################################################\n", "# TrainingArguments parameters\n", "################################################################################\n", "\n", "# Output directory where the model predictions and checkpoints will be stored\n", "output_dir = new_model\n", "\n", "# Number of training epochs\n", "num_train_epochs = 5\n", "\n", "# Enable fp16/bf16 training (set bf16 to True with an A100)\n", "fp16 = False\n", "bf16 = True\n", "\n", "# Batch size per GPU for training\n", "#per_device_train_batch_size = 12\n", "per_device_train_batch_size = 4 #on Colab, 16 GB GPU\n", "\n", "# Batch size per GPU for evaluation\n", "per_device_eval_batch_size = 6\n", "\n", "# Number of update steps to accumulate the gradients for\n", "gradient_accumulation_steps = 4\n", "\n", "# Enable gradient checkpointing\n", "gradient_checkpointing = True\n", "\n", "# Maximum gradient normal (gradient clipping)\n", "max_grad_norm = 0.5\n", "\n", "# Initial learning rate (AdamW optimizer)\n", "learning_rate = 2e-4\n", "\n", "# Weight decay to apply to all layers except bias/LayerNorm weights\n", "weight_decay = 0.001\n", "\n", "# Optimizer to use\n", "optim = \"paged_adamw_32bit\"\n", "\n", "# Learning rate schedule (constant, cosine, etc.)\n", "lr_scheduler_type = \"cosine\" # \"constant\"\n", "\n", "# Number of training steps (overrides num_train_epochs)\n", "max_steps = -1\n", "\n", "# Ratio of steps for a linear warmup (from 0 to learning rate)\n", "warmup_ratio = 0.03\n", "\n", "# Group sequences into batches with same length\n", "# Saves memory and speeds up training considerably\n", "group_by_length = True\n", "\n", "# Log every X updates steps\n", "logging_steps = 25\n", "save_strategy='epoch'\n", "eval_steps= 100\n", "evaluation_strategy = \"steps\" #\"steps\", \"epoch\", \"no\"\n", "\n", "################################################################################\n", "# SFT parameters\n", "################################################################################\n", "\n", "# Maximum sequence length to use\n", "max_seq_length = 128\n", "\n", "# Pack multiple short examples in the same input sequence to increase efficiency\n", "packing = True\n", "\n", "# Load the entire model on the GPU 0\n", "device_map = {\"\": 0}\n", "\n", "push_formatted_dataset_to_hub = False" ] }, { "cell_type": "markdown", "id": "1f989b7f-9e68-441e-ac61-f661dd1eddaa", "metadata": { "id": "1f989b7f-9e68-441e-ac61-f661dd1eddaa" }, "source": [ "### Prepare dataset\n", "\n", "We use a protein dataset here, with the goal to train a model that predicts the dominant secondary structure for a given protein" ] }, { "cell_type": "markdown", "id": "6caab12c-62cb-447c-aaa9-12439dbf3d2c", "metadata": { "id": "6caab12c-62cb-447c-aaa9-12439dbf3d2c" }, "source": [ "#### Chat template: Example\n", "\n", "In this case, we use the chat template so that the model learns to respond to a particular query" ] }, { "cell_type": "code", "execution_count": 5, "id": "a0592c35-28b1-4421-b8f8-20150f7a39f8", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 194 }, "id": "a0592c35-28b1-4421-b8f8-20150f7a39f8", "outputId": "26bd6578-8a61-4275-85b4-648c7109fef1" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n", "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", "You will be able to reuse this secret in all of your notebooks.\n", "Please note that authentication is recommended but still optional to access public models or datasets.\n", " warnings.warn(\n" ] }, { "data": { "application/vnd.google.colaboratory.intrinsic+json": { "type": "string" }, "text/plain": [ "'<|begin_of_text|><|start_header_id|>system<|end_header_id|>\\n\\nYou are an expert in protein mechanics.<|eot_id|><|start_header_id|>user<|end_header_id|>\\n\\nWhat is a protein?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n\\nA protein is a chain of amino acids...<|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n\\n'" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenizer = AutoTokenizer.from_pretrained(base_model_name, trust_remote_code=True,)\n", "\n", "messages = [\n", " {\"role\": \"system\", \"content\": \"You are an expert in protein mechanics.\"},\n", " {\"role\": \"user\", \"content\": \"What is a protein?\"},\n", " {\"role\": \"assistant\", \"content\": \"A protein is a chain of amino acids...\"},\n", "]\n", "\n", "prompt = tokenizer.apply_chat_template(\n", " messages,\n", " tokenize=False,\n", " add_generation_prompt=True\n", ")\n", "prompt" ] }, { "cell_type": "markdown", "id": "3433c8c7-088d-4722-9068-26e9d29371e1", "metadata": { "id": "3433c8c7-088d-4722-9068-26e9d29371e1" }, "source": [ "#### Build dataset for LLM training from 'raw sources'" ] }, { "cell_type": "code", "execution_count": 6, "id": "724ace7f-d157-44b2-aa5d-961cd64e5367", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 309, "referenced_widgets": [ "3d441c6fe11d4c9cbb1a4d29e165b484", "c653871a43bf430b954b02f5fd91cdda", "502240b743a648c1a0f452182d312135", "81ce8c7c00f5412c8ed82a1cb8633af3", "17f5eda8dce247659adf33f598f15bbe", "027b94630fd34e8eaf915592a51edd89", "e821b92ac54b4569971ac51d7106f178", "7d1afc79ef2b4f78abe7611e81043350", "3d8a58b93b104e1e892e4fedb2d02f56", "8254778ae23f4572ac58824fd6f1750d", "008d5138dc2241aa84ed7eed5693ccd0", "72bb45159c0449958822b38dedfbbb22", "0cb92a1263814fd993fb835c516336cc", "e4b25d01700d493ebab4b51b857203f7", "c0e56ab1984d4da49f130c5e24b101d1", "7973783a74944950aa0b4aa9e1c25216", "be13531d3aa94e838a7aea7d18bf8c4f", "d6dc16801dca4f7fb2ec2b7f2504b4a1", "5dded94f8d944cdcbf22d4126e97cef7", "02b792973a674343a2d2c5e2efaeacc5", "4b6388050be8496d869084d8673fb0f5", "e18a5564d1f441049f6606f40def1be7" ] }, "id": "724ace7f-d157-44b2-aa5d-961cd64e5367", "outputId": "86884659-798b-4ac0-82dc-2c29c5b692c4", "scrolled": true }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "3d441c6fe11d4c9cbb1a4d29e165b484", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Map: 0%| | 0/12921 [00:00<|start_header_id|>user<|end_header_id|>\\n\\nDominant secondary structure of < S S N A K I D Q L S S D V Q T L N A K V D Q L S N D V N A M R S D V Q A A K D D A A R A N Q R L D N M A T K Y R ><|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n\\nAH<|eot_id|>'}\n", "{'content': '<|begin_of_text|><|start_header_id|>user<|end_header_id|>\\n\\nDominant secondary structure of < K D W S F Y K D W S F Y K D W S F Y K D W S F Y ><|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n\\nUNSTRUCTURED<|eot_id|>'}\n" ] } ], "source": [ "from datasets import load_dataset, DatasetDict, concatenate_datasets\n", "\n", "# Define the preprocess function for conversations with a chat template\n", "def preprocess_conversation(samples):\n", " system_prompt = ''\n", " batch = []\n", " for question, answer in zip(samples[\"question\"], samples[\"answer\"]):\n", " messages=[]\n", " if system_prompt != '':\n", "\n", " messages.extend ([\n", " {\"role\": \"system\", \"content\": system_prompt},\n", "\n", " ])\n", " messages.extend ( [\n", " {\"role\": \"user\", \"content\": 'Dominant secondary structure of < '+question+' >'},\n", " {\"role\": \"assistant\", \"content\": answer}\n", " ])\n", " formatted_conv = tokenizer.apply_chat_template(\n", " messages,\n", " tokenize=False,\n", " add_generation_prompt=False\n", " )\n", " remove_gen_prompt='<|start_header_id|>assistant<|end_header_id|>\\n\\n'\n", " if formatted_conv.endswith(remove_gen_prompt):\n", " formatted_conv = formatted_conv[:-len(remove_gen_prompt)]\n", "\n", " formatted_conv\n", "\n", " batch.append(formatted_conv)\n", " return {\"content\": batch}\n", "\n", "\n", "# Load the original dataset\n", "dataset = load_dataset('lamm-mit/protein_secondary_structure_from_PDB')['train']\n", "#dataset = load_from_disk('protein_secondary_structure_from_PDB')\n", "\n", "dataset = dataset.filter(lambda example: example['Sequence_length'] < max_seq_length)\n", "\n", "# Rename columns\n", "dataset = dataset.rename_column('Sequence_spaced', 'question')\n", "dataset = dataset.rename_column('Primary_SS_Type', 'answer')\n", "\n", "dataset = dataset.train_test_split(test_size=0.1, seed=42)\n", "\n", "test_dataset = copy.deepcopy(dataset['test'])\n", "\n", "# Apply preprocessing to the original dataset\n", "dataset['train'] = dataset['train'].map(\n", " #preprocess,\n", " preprocess_conversation,\n", " batched=True,\n", " #This argument removes the original columns from the dataset after preprocessing. dataset[\"train\"].column_names retrieves the column names from the training subset of the dataset, and these columns will be removed from the resulting dataset.\n", " #remove_columns=dataset[\"train\"].column_names\n", " remove_columns=dataset['train'].column_names\n", ")\n", "dataset['test'] = dataset['test'].map(\n", " #preprocess,\n", " preprocess_conversation,\n", " batched=True,\n", " #This argument removes the original columns from the dataset after preprocessing. dataset[\"train\"].column_names retrieves the column names from the training subset of the dataset, and these columns will be removed from the resulting dataset.\n", " #remove_columns=dataset[\"train\"].column_names\n", " remove_columns=dataset['test'].column_names\n", ")\n", "# Shuffle the dataset\n", "dataset = dataset.shuffle(100)\n", "\n", "#dataset = dataset['train'].train_test_split(test_size=0.1)\n", "\n", "# Create a new with 'train' and 'test' splits\n", "new_dataset = DatasetDict({\n", " # \"train\": dataset[\"train\"],\n", " \"train\": concatenate_datasets([dataset['train'],\n", " # dataset_2['train'],\n", " ],\n", " ),\n", " \"test\": concatenate_datasets([dataset['test'],\n", " # dataset_2['train'],\n", " ],\n", " )\n", "})\n", "\n", "print(new_dataset)\n", "print(new_dataset[\"train\"][0])\n", "print(new_dataset[\"test\"][0])\n" ] }, { "cell_type": "code", "execution_count": 7, "id": "6bcae638-0809-484d-bee7-93ac7c936157", "metadata": { "id": "6bcae638-0809-484d-bee7-93ac7c936157" }, "outputs": [], "source": [ "if push_formatted_dataset_to_hub:\n", " dataset_name='formatted_data_for_training'\n", " # Push the modified dataset to the hub\n", " new_dataset.push_to_hub(f\"lamm-mit/{dataset_name}\", private=True,\n", " )" ] }, { "cell_type": "markdown", "id": "7a54d3b0-3c56-4097-a4d7-afdea45e9287", "metadata": { "id": "7a54d3b0-3c56-4097-a4d7-afdea45e9287" }, "source": [ "### Train the model" ] }, { "cell_type": "code", "execution_count": 8, "id": "ed7f5a39-fc23-4b92-9129-3d9d34ccc8f2", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 66, "referenced_widgets": [ "cf3545758b0f48f2b159e1bd42fffee3", "14a244af747f42b58b88a2e46611d8bf", "328b2d32af0747a2978790e62217149f", "73f508c080c542cca5b7fc09406c594f", "879ffffedc9342159a47918f914837d4", "0675a2178c4844ba8ead2f56311a9c62", "edfb829258b04e13854ca9df8ffa2135", "132c1952b3b54a7580155b1cd05220d6", "44df33c09a29490a8a965cd67b2c2688", "64b10175cd5d4650bcd4f7d553c6292d", "3895b2ed98184ecc84bb43d43cc6cf3a" ] }, "id": "ed7f5a39-fc23-4b92-9129-3d9d34ccc8f2", "outputId": "121f719f-0590-4e4f-e49e-498e620abbb7" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "cf3545758b0f48f2b159e1bd42fffee3", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Loading checkpoint shards: 0%| | 0/4 [00:00= 8:\n", " print(\"=\" * 80)\n", " print(\"GPU supports bfloat16: accelerate training with bf16=True\")\n", " print(\"=\" * 80)\n", "\n", "# Load base model\n", "model = AutoModelForCausalLM.from_pretrained(\n", " base_model_name,\n", " quantization_config=bnb_config,\n", " device_map=device_map\n", ")\n", "model.config.use_cache = False\n", "#model.config.pretraining_tp = 1\n", "\n", "# Load tokenizer\n", "tokenizer = AutoTokenizer.from_pretrained(base_model_name, trust_remote_code=True)\n", "tokenizer.padding_side = \"right\"\n", "\n", "'''\n", "The pad_token_id and eos_token_id values should not be identical when using SFFT.\n", "This is because of the way masking is implemented, and can result in the end token to be masked out.\n", "As a result, in the model continuously generating questions and answers without eos token.\n", "To avoid this, we set the pad_token_id to a different value.\"\n", "'''\n", "\n", "tokenizer.pad_token_id= tokenizer.encode('<|end_of_text|>')[1] #set to 128001\n", "tokenizer.pad_token_id" ] }, { "cell_type": "code", "execution_count": 9, "id": "5b73c3b6-ac7b-4474-abd2-4dee3f2d4517", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "5b73c3b6-ac7b-4474-abd2-4dee3f2d4517", "outputId": "656c87f9-9633-4992-96d2-7388f8a69b97" }, "outputs": [ { "data": { "text/plain": [ "('<|end_of_text|>', '<|eot_id|>')" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenizer.pad_token, tokenizer.eos_token" ] }, { "cell_type": "code", "execution_count": 10, "id": "2c5c9772-0982-495d-bf8e-c457b08dbb65", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "2c5c9772-0982-495d-bf8e-c457b08dbb65", "outputId": "21d62446-af29-4ecc-8bf7-71d892a7196f" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", " warnings.warn(\n" ] } ], "source": [ "# Load LoRA configuration\n", "peft_config = LoraConfig(\n", " lora_alpha=lora_alpha,\n", " lora_dropout=lora_dropout,\n", " r=lora_r,\n", " bias=\"none\",\n", " target_modules=lora_target_modules,\n", " task_type=\"CAUSAL_LM\",\n", ")\n", "\n", "# Set training parameters\n", "# check: https://huggingface.co/docs/transformers/v4.15.0/en/main_classes/trainer#transformers.TrainingArguments\n", "training_arguments = TrainingArguments(\n", " output_dir=output_dir,\n", " num_train_epochs=num_train_epochs,\n", " per_device_train_batch_size=per_device_train_batch_size,\n", " gradient_accumulation_steps=gradient_accumulation_steps,\n", " optim=optim,\n", " save_strategy=save_strategy,\n", " evaluation_strategy= evaluation_strategy,\n", " eval_steps=eval_steps,\n", " logging_steps=logging_steps,\n", " learning_rate=learning_rate,\n", " weight_decay=weight_decay,\n", " fp16=fp16,\n", " bf16=bf16,\n", " max_grad_norm=max_grad_norm,\n", " max_steps=max_steps,\n", " warmup_ratio=warmup_ratio,\n", " group_by_length=group_by_length,\n", " lr_scheduler_type=lr_scheduler_type,\n", " report_to=\"wandb\",\n", " #report_to=\"none\",\n", "\n", ")" ] }, { "cell_type": "markdown", "id": "b829e338-868c-49f8-80e7-24a7eae231a4", "metadata": { "id": "b829e338-868c-49f8-80e7-24a7eae231a4" }, "source": [ "#### Check how the model performs BEFORE training" ] }, { "cell_type": "code", "execution_count": 11, "id": "90f373bc-e286-429d-8b8e-6d01ac776b15", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "90f373bc-e286-429d-8b8e-6d01ac776b15", "outputId": "b1dcb6a7-20b9-4a84-901e-48991643be9a" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Answer LLM: The dominant secondary structure of the given protein sequence is a combination of alpha helices and beta sheets. The protein contains several alpha helical regions, including one that spans from amino acid positions 40 to 57 (FVVTDCIYK), another from positions 79 to 103 (CVEVC-PVNCF\n", "Correct answer: UNSTRUCTURED\n" ] } ], "source": [ "question=test_dataset[0]['question']\n", "corr_answer=test_dataset[0]['answer']\n", "\n", "corr_answer\n", "answer,_ = generate_response (text_input='Dominant secondary structure of < '+question+' >', max_new_tokens=64)\n", "\n", "print (f\"Answer LLM: {answer[0]}\\nCorrect answer: {corr_answer}\")" ] }, { "cell_type": "code", "execution_count": 12, "id": "54022820-5634-4ca3-9fc6-b810d2add921", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "54022820-5634-4ca3-9fc6-b810d2add921", "outputId": "d0054631-039d-49b7-fc19-76561d689137" }, "outputs": [ { "data": { "text/plain": [ "128009" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenizer.eos_token_id" ] }, { "cell_type": "code", "execution_count": 13, "id": "ee45fb17-152c-460b-8c63-241b5983e060", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "312932c72a38420db037e56c339db559", "4a35e2765bc44d42b63d7a823d9c0e28", "fb7fb94611a04098bfed8dd0bfb2497e", "6d2ffac7ab894099b8782175fdde56d9", "5baf0a5192544cf6bd3a138fb632f64e", "a5a7cba95a474882af91facd5d43a863", "43a3c01dc9434d6d9beda4cf96684596", "bc7eb5ead2204888b8837956185a853e", "a7a8ef45c40448a884528e135642af2d", "c7523ae0393c4cad8b3161142a381e0f", "bc0b398597574e8ea658b4060dc9865d", "7f8a13951c7e4ca7ae01fe99996ba9e5", "26cfb83cb26249c8989840637cdc1917", "dd7bb23a6f1a461ab2a5738356b8a1df", "3cb9714a03454ee491683f48c0197e95", "b292c34dd8a74df5ab8d566826720f88", "c76b2d301871420bbe3ddc6526332397", "52089ee51c5f4cc4b8360105bc994a03", "4cc376198019481d8c7c944972425a52", "b616c213da6346bfb62ab0e268a9fd5e", "f343263df351401d96549534ab288020", "79a4f2eeec274592a4cd7c19b759e60b" ] }, "id": "ee45fb17-152c-460b-8c63-241b5983e060", "outputId": "3754ad03-088e-491c-fb7b-2fdda171ca39", "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_deprecation.py:100: FutureWarning: Deprecated argument(s) used in '__init__': dataset_text_field, max_seq_length, packing, dataset_kwargs. Will not be supported from version '1.0.0'.\n", "\n", "Deprecated positional argument(s) used in SFTTrainer, please use the SFTConfig to set these arguments instead.\n", " warnings.warn(message, FutureWarning)\n", "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/trl/trainer/sft_trainer.py:192: UserWarning: You passed a `packing` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1525: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/trl/trainer/sft_trainer.py:280: UserWarning: You passed a `max_seq_length` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/trl/trainer/sft_trainer.py:318: UserWarning: You passed a `dataset_text_field` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/trl/trainer/sft_trainer.py:366: UserWarning: You passed a `dataset_kwargs` argument to the SFTTrainer, the value you passed will override the one in the `SFTConfig`.\n", " warnings.warn(\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "312932c72a38420db037e56c339db559", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Generating train split: 0 examples [00:00, ? examples/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "7f8a13951c7e4ca7ae01fe99996ba9e5", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Generating train split: 0 examples [00:00, ? examples/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.\n", "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmarkusjbuehler\u001b[0m (\u001b[33mlamm-mit\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n" ] }, { "data": { "text/html": [ "Tracking run with wandb version 0.17.5" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "Run data is saved locally in /content/wandb/run-20240728_101654-9smq4g2s" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "Syncing run protein_secondary_structure_predictor to Weights & Biases (docs)
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ " View project at https://wandb.ai/lamm-mit/huggingface" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ " View run at https://wandb.ai/lamm-mit/huggingface/runs/9smq4g2s" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "
\n", " \n", " \n", " [3300/3300 1:44:35, Epoch 5/5]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
StepTraining LossValidation Loss
1002.2921002.290651
2001.9892002.018708
3001.8224001.855575
4001.7684001.763817
5001.6784001.703792
6001.6302001.639292
7001.4010001.580074
8001.3859001.548995
9001.3621001.504345
10001.2882001.470629
11001.3043001.436050
12001.2073001.396687
13001.1660001.363970
14000.8800001.424378
15000.7941001.385330
16000.8453001.372946
17000.8654001.340954
18000.7876001.315560
19000.8510001.308754
20000.5236001.526696
21000.4091001.531994
22000.3722001.512631
23000.3970001.523684
24000.3983001.511530
25000.4141001.517029
26000.4033001.511221
27000.1999001.774590
28000.2006001.821994
29000.2010001.833357
30000.2048001.828533
31000.1946001.834447
32000.2060001.833463
33000.2082001.832923

" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Set supervised fine-tuning parameters\n", "trainer = SFTTrainer(\n", " model=model,\n", " tokenizer=tokenizer,\n", " train_dataset=new_dataset[\"train\"],\n", " eval_dataset=new_dataset[\"test\"],\n", " peft_config=peft_config,\n", " dataset_text_field=\"content\",\n", " max_seq_length=max_seq_length,\n", " args=training_arguments,\n", " packing=packing,\n", "\n", " dataset_kwargs={\n", " \"append_concat_token\": True, #If true, appends eos_token_id at the end of each sample being packed.\n", " },\n", ")\n", "\n", "# Train model\n", "trainer.train()\n", "\n", "# Save trained model\n", "trainer.model.save_pretrained(new_model)" ] }, { "cell_type": "code", "execution_count": 14, "id": "6bed4f75-7783-4db4-9bf5-5a98fa29250d", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 504 }, "id": "6bed4f75-7783-4db4-9bf5-5a98fa29250d", "outputId": "bc304f2c-48e7-4389-cc03-bee33b3cac14", "scrolled": true }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAAHWCAYAAABACtmGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACLMklEQVR4nOzdd3hUddrG8e/MJJn0XoEk9F4FpUlRqSoCNlQUsK6KXd91cXcVdXdZ266KBRUFe11ARRRQQZHeOwGkhJJCgPQ+c94/hgyMpFCSzCS5P9d1rsycNs+ZCTA3v3JMhmEYiIiIiIiISIXM7i5ARERERETE0yk4iYiIiIiIVEHBSUREREREpAoKTiIiIiIiIlVQcBIREREREamCgpOIiIiIiEgVFJxERERERESqoOAkIiIiIiJSBQUnERERERGRKig4iYh4kAkTJtC0adNzOnby5MmYTKbqLcjD7Nu3D5PJxMyZM91dSoWaNm3KhAkT3PLadeH9ERGpqxScRETOgMlkOqNl8eLF7i5VgMWLF1f6OX322WfuLvG8fPLJJ7z88svuLsPFhAkTCAwMdHcZIiI1xsvdBYiI1AUffvihy/MPPviAhQsXnra+Xbt25/U677zzDna7/ZyO/dvf/sZf/vKX83r9+uaBBx7gwgsvPG1979693VBN9fnkk0/YsmULDz30kMv6xMRECgoK8Pb2dk9hIiL1mIKTiMgZuPnmm12er1ixgoULF562/o/y8/Px9/c/49c5ny+8Xl5eeHnpr/VT9evXj2uvvdbdZdQak8mEr6+vu8sQEamX1FVPRKSaDBw4kI4dO7J27Vr69++Pv78/TzzxBABff/01V1xxBY0aNcJqtdKiRQueffZZbDabyzn+OMapbMzKiy++yNtvv02LFi2wWq1ceOGFrF692uXY8sY4mUwm7rvvPubMmUPHjh2xWq106NCBH3744bT6Fy9eTI8ePfD19aVFixa89dZbZzxuasmSJVx33XUkJCRgtVqJj4/n4YcfpqCg4LTrCwwM5NChQ4waNYrAwECioqJ47LHHTnsvMjMzmTBhAiEhIYSGhjJ+/HgyMzOrrOVsdOzYkUsuueS09Xa7ncaNG7uErhdffJE+ffoQERGBn58f3bt356uvvqryNSp6D2fOnInJZGLfvn3OdWfyezJw4EC+++479u/f7+x6WPY7U9EYp59//pl+/foREBBAaGgoI0eOZPv27eXWuXv3biZMmEBoaCghISHceuut5OfnV3mdZ+rLL7+ke/fu+Pn5ERkZyc0338yhQ4dc9klNTeXWW2+lSZMmWK1W4uLiGDlypMt7tWbNGoYOHUpkZCR+fn40a9aM2267rdrqFBH5I/3XpIhINTp69CjDhw/nhhtu4OabbyYmJgZwfEkODAzkkUceITAwkJ9//pknn3yS7OxsXnjhhSrP+8knn5CTk8Of/vQnTCYTzz//PFdffTV79uypspXqt99+Y9asWdx7770EBQXx6quvcs0115CcnExERAQA69evZ9iwYcTFxfH0009js9l45plniIqKOqPr/vLLL8nPz+eee+4hIiKCVatWMXXqVA4ePMiXX37psq/NZmPo0KH07NmTF198kR9//JGXXnqJFi1acM899wBgGAYjR47kt99+4+6776Zdu3bMnj2b8ePHn1E9ZXJycsjIyDhtfUREBCaTiTFjxjB58mRSU1OJjY11ec8OHz7MDTfc4Fz3yiuvcNVVVzF27FiKi4v57LPPuO6665g7dy5XXHHFWdVVkTP5PfnrX/9KVlYWBw8e5L///S9ApWOLfvzxR4YPH07z5s2ZPHkyBQUFTJ06lb59+7Ju3brTJiO5/vrradasGVOmTGHdunVMnz6d6OhonnvuuWq5vltvvZULL7yQKVOmkJaWxiuvvMLSpUtZv349oaGhAFxzzTVs3bqV+++/n6ZNm5Kens7ChQtJTk52Ph8yZAhRUVH85S9/ITQ0lH379jFr1qzzrlFEpEKGiIictYkTJxp//Ct0wIABBmBMmzbttP3z8/NPW/enP/3J8Pf3NwoLC53rxo8fbyQmJjqf79271wCMiIgI49ixY871X3/9tQEY3377rXPdU089dVpNgOHj42Ps3r3buW7jxo0GYEydOtW5bsSIEYa/v79x6NAh57pdu3YZXl5ep52zPOVd35QpUwyTyWTs37/f5foA45lnnnHZt1u3bkb37t2dz+fMmWMAxvPPP+9cV1paavTr188AjBkzZlRaz6JFiwygwiUlJcUwDMNISko67b0wDMO49957jcDAQJfr+uM1FhcXGx07djQuvfRSl/WJiYnG+PHjnc/L+1wMwzBmzJhhAMbevXsrfA3DKP/35IorrnD5PSlT9vty6vvTtWtXIzo62jh69Khz3caNGw2z2WyMGzfutDpvu+02l3OOHj3aiIiIOO21/mj8+PFGQEBAhduLi4uN6Ohoo2PHjkZBQYFz/dy5cw3AePLJJw3DMIzjx48bgPHCCy9UeK7Zs2cbgLF69eoq6xIRqS7qqiciUo2sViu33nrraev9/Pycj8taQfr160d+fj47duyo8rxjxowhLCzM+bxfv34A7Nmzp8pjBw0aRIsWLZzPO3fuTHBwsPNYm83Gjz/+yKhRo2jUqJFzv5YtWzJ8+PAqzw+u15eXl0dGRgZ9+vTBMAzWr19/2v533323y/N+/fq5XMu8efPw8vJytkABWCwW7r///jOqp8yTTz7JwoULT1vCw8MBaN26NV27duXzzz93HmOz2fjqq68YMWKEy3Wd+vj48eNkZWXRr18/1q1bd1Y1VeZ8f0/+KCUlhQ0bNjBhwgTnNYPjd2Dw4MHMmzfvtGPK+2yOHj1Kdnb2Wb/+qdasWUN6ejr33nuvyzisK664grZt2/Ldd98BjvfAx8eHxYsXc/z48XLPVdYyNXfuXEpKSs6rLhGRM6XgJCJSjRo3boyPj89p67du3cro0aMJCQkhODiYqKgo58QSWVlZVZ43ISHB5XlZiKroi2Vlx5YdX3Zseno6BQUFtGzZ8rT9yltXnuTkZOeX87JxSwMGDABOvz5fX9/TugCeWg/A/v37iYuLO60LWps2bc6onjKdOnVi0KBBpy2nfkZjxoxh6dKlznE2ixcvJj09nTFjxrica+7cufTq1QtfX1/Cw8OJiorizTffPKPP70yd7+/JH+3fvx8o/31r164dGRkZ5OXluaw/n9+1c62lbdu2zu1Wq5XnnnuO77//npiYGPr378/zzz9Pamqqc/8BAwZwzTXX8PTTTxMZGcnIkSOZMWMGRUVF51WjiEhlFJxERKrRqS0GZTIzMxkwYAAbN27kmWee4dtvv2XhwoXOMSNnMv24xWIpd71hGDV67Jmw2WwMHjyY7777jscff5w5c+awcOFC5wQFf7y+iupxlzFjxmAYhnMs1hdffEFISAjDhg1z7rNkyRKuuuoqfH19eeONN5g3bx4LFy7kpptuqvJ9rGhyjfImwzjf35PqUNO/L2fioYceYufOnUyZMgVfX1/+/ve/065dO2frpclk4quvvmL58uXcd999HDp0iNtuu43u3buTm5tba3WKSMOiySFERGrY4sWLOXr0KLNmzaJ///7O9Xv37nVjVSdFR0fj6+vL7t27T9tW3ro/2rx5Mzt37uT9999n3LhxzvULFy4855oSExP56aefyM3NdWl1SkpKOudzVqRZs2ZcdNFFfP7559x3333MmjWLUaNGYbVanfv873//w9fXl/nz57usnzFjRpXnL2uxyczMdHYxg5MtMGXO5vfkTGY6BMf7COW/bzt27CAyMpKAgIAzOtf5OrWWSy+91GVbUlKSc3uZFi1a8Oijj/Loo4+ya9cuunbtyksvvcRHH33k3KdXr1706tWLf/7zn3zyySeMHTuWzz77jDvuuKPmL0hEGhy1OImI1LCy/8E/9X/si4uLeeONN9xVkguLxcKgQYOYM2cOhw8fdq7fvXs333///RkdD67XZxgGr7zyyjnXdPnll1NaWsqbb77pXGez2Zg6deo5n7MyY8aMYcWKFbz33ntkZGSc1k3PYrFgMplcWon27dvHnDlzqjx32fiyX3/91bkuLy+P999//7TXgDP7PQkICDijrntxcXF07dqV999/32Uq9y1btrBgwQIuv/zyKs9RXXr06EF0dDTTpk1z6VL3/fffs337dufMhPn5+RQWFroc26JFC4KCgpzHHT9+/LQWsK5duwKou56I1Bi1OImI1LA+ffoQFhbG+PHjeeCBBzCZTHz44Ye12vWpKpMnT2bBggX07duXe+65B5vNxmuvvUbHjh3ZsGFDpce2bduWFi1a8Nhjj3Ho0CGCg4P53//+d15jYkaMGEHfvn35y1/+wr59+2jfvj2zZs0663E+S5YsOe1LODgmR+jcubPz+fXXX89jjz3GY489Rnh4OIMGDXLZ/4orruA///kPw4YN46abbiI9PZ3XX3+dli1bsmnTpkprGDJkCAkJCdx+++383//9HxaLhffee4+oqCiSk5Od+53N70n37t35/PPPeeSRR7jwwgsJDAxkxIgR5b7+Cy+8wPDhw+nduze33367czrykJAQJk+eXGntZ6ukpIR//OMfp60PDw/n3nvv5bnnnuPWW29lwIAB3Hjjjc7pyJs2bcrDDz8MwM6dO7nsssu4/vrrad++PV5eXsyePZu0tDTn9PDvv/8+b7zxBqNHj6ZFixbk5OTwzjvvEBwcXKthUEQaGLfM5SciUsdVNB15hw4dyt1/6dKlRq9evQw/Pz+jUaNGxp///Gdj/vz5BmAsWrTIuV9F05GXNzUzYDz11FPO5xVNRz5x4sTTjv3jlNmGYRg//fST0a1bN8PHx8do0aKFMX36dOPRRx81fH19K3gXTtq2bZsxaNAgIzAw0IiMjDTuvPNO57Tnp06NXdGU1eXVfvToUeOWW24xgoODjZCQEOOWW24x1q9fXy3TkZ/6vpXp27evARh33HFHued89913jVatWhlWq9Vo27atMWPGjHLrLu+9Xbt2rdGzZ0/Dx8fHSEhIMP7zn/+UOx35mf6e5ObmGjfddJMRGhpqAM7fmfKmIzcMw/jxxx+Nvn37Gn5+fkZwcLAxYsQIY9u2bS77lF3LkSNHXNaXV2d5yqaaL29p0aKFc7/PP//c6Natm2G1Wo3w8HBj7NixxsGDB53bMzIyjIkTJxpt27Y1AgICjJCQEKNnz57GF1984dxn3bp1xo033mgkJCQYVqvViI6ONq688kpjzZo1ldYoInI+TIbhQf/lKSIiHmXUqFFs3bqVXbt2ubsUERERt9IYJxERAaCgoMDl+a5du5g3bx4DBw50T0EiIiIeRC1OIiICOCYSmDBhAs2bN2f//v28+eabFBUVsX79elq1auXu8kRERNxKk0OIiAgAw4YN49NPPyU1NRWr1Urv3r3517/+pdAkIiKCWpxERERERESqpDFOIiIiIiIiVVBwEhERERERqUKDG+Nkt9s5fPgwQUFBmEwmd5cjIiIiIiJuYhgGOTk5NGrUCLO58jalBhecDh8+THx8vLvLEBERERERD3HgwAGaNGlS6T4NLjgFBQUBjjcnODjYzdWIiIiIiIi7ZGdnEx8f78wIlWlwwamse15wcLCCk4iIiIiInNEQHk0OISIiIiIiUgUFJxERERERkSooOImIiIiIiFShwY1xEhERERH3MgyD0tJSbDabu0uRBsDb2xuLxXLe51FwEhEREZFaU1xcTEpKCvn5+e4uRRoIk8lEkyZNCAwMPK/zKDiJiIiISK2w2+3s3bsXi8VCo0aN8PHxOaPZzETOlWEYHDlyhIMHD9KqVavzanlScBIRERGRWlFcXIzdbic+Ph5/f393lyMNRFRUFPv27aOkpOS8gpMmhxARERGRWmU26yuo1J7qatXUb62IiIiIiEgVFJxERERERESqoOAkIiIiIuIGTZs25eWXXz7j/RcvXozJZCIzM7PGapKKKTiJiIiIiFTCZDJVukyePPmczrt69WruuuuuM96/T58+pKSkEBISck6vd6YU0MqnWfVERERERCqRkpLifPz555/z5JNPkpSU5Fx36v2BDMPAZrPh5VX11+yoqKizqsPHx4fY2NizOkaqj1tbnN588006d+5McHAwwcHB9O7dm++//77C/WfOnHlawvf19a3FiqvX9CV7GPrfX5m+ZI+7SxERERFxC8MwyC8udctiGMYZ1RgbG+tcQkJCMJlMzuc7duwgKCiI77//nu7du2O1Wvntt9/4/fffGTlyJDExMQQGBnLhhRfy448/upz3j131TCYT06dPZ/To0fj7+9OqVSu++eYb5/Y/tgTNnDmT0NBQ5s+fT7t27QgMDGTYsGEuQa+0tJQHHniA0NBQIiIiePzxxxk/fjyjRo0658/s+PHjjBs3jrCwMPz9/Rk+fDi7du1ybt+/fz8jRowgLCyMgIAAOnTowLx585zHjh07lqioKPz8/GjVqhUzZsw451pqk1tbnJo0acK///1vWrVqhWEYvP/++4wcOZL169fToUOHco8JDg52Sfh1+aZpx/KKSUrL4XBmobtLEREREXGLghIb7Z+c75bX3vbMUPx9qufr8F/+8hdefPFFmjdvTlhYGAcOHODyyy/nn//8J1arlQ8++IARI0aQlJREQkJChed5+umnef7553nhhReYOnUqY8eOZf/+/YSHh5e7f35+Pi+++CIffvghZrOZm2++mccee4yPP/4YgOeee46PP/6YGTNm0K5dO1555RXmzJnDJZdccs7XOmHCBHbt2sU333xDcHAwjz/+OJdffjnbtm3D29ubiRMnUlxczK+//kpAQADbtm1ztsr9/e9/Z9u2bXz//fdERkaye/duCgoKzrmW2uTW4DRixAiX5//85z958803WbFiRYXBqSzhn6mioiKKioqcz7Ozs8+t2BoQYHW8/blFJW6uRERERETOxzPPPMPgwYOdz8PDw+nSpYvz+bPPPsvs2bP55ptvuO+++yo8z4QJE7jxxhsB+Ne//sWrr77KqlWrGDZsWLn7l5SUMG3aNFq0aAHAfffdxzPPPOPcPnXqVCZNmsTo0aMBeO2115ytP+eiLDAtXbqUPn36APDxxx8THx/PnDlzuO6660hOTuaaa66hU6dOADRv3tx5fHJyMt26daNHjx6Ao9WtrvCYMU42m40vv/ySvLw8evfuXeF+ubm5JCYmYrfbueCCC/jXv/5VYcgCmDJlCk8//XRNlHzegnzLglOpmysRERERcQ8/bwvbnhnqtteuLmVBoExubi6TJ0/mu+++IyUlhdLSUgoKCkhOTq70PJ07d3Y+DggIIDg4mPT09Ar39/f3d4YmgLi4OOf+WVlZpKWlcdFFFzm3WywWunfvjt1uP6vrK7N9+3a8vLzo2bOnc11ERARt2rRh+/btADzwwAPcc889LFiwgEGDBnHNNdc4r+uee+7hmmuuYd26dQwZMoRRo0Y5A5inc/useps3byYwMBCr1crdd9/N7Nmzad++fbn7tmnThvfee4+vv/6ajz76CLvdTp8+fTh48GCF5580aRJZWVnO5cCBAzV1KWct8ESLU06hgpOIiIg0TCaTCX8fL7cs1TnkIyAgwOX5Y489xuzZs/nXv/7FkiVL2LBhA506daK4uLjS83h7e5/2/lQWcsrb/0zHbtWUO+64gz179nDLLbewefNmevTowdSpUwEYPnw4+/fv5+GHH+bw4cNcdtllPPbYY26t90y5PTi1adOGDRs2sHLlSu655x7Gjx/Ptm3byt23d+/ejBs3jq5duzJgwABmzZpFVFQUb731VoXnt1qtzsknyhZPUdZVL08tTiIiIiL1ytKlS5kwYQKjR4+mU6dOxMbGsm/fvlqtISQkhJiYGFavXu1cZ7PZWLdu3Tmfs127dpSWlrJy5UrnuqNHj5KUlOTS+BEfH8/dd9/NrFmzePTRR3nnnXec26Kiohg/fjwfffQRL7/8Mm+//fY511Ob3N5Vz8fHh5YtWwLQvXt3Vq9ezSuvvFJpGCrj7e1Nt27d2L17d02XWSOCrOqqJyIiIlIftWrVilmzZjFixAhMJhN///vfz7l73Pm4//77mTJlCi1btqRt27ZMnTqV48ePn1Fr2+bNmwkKCnI+N5lMdOnShZEjR3LnnXfy1ltvERQUxF/+8hcaN27MyJEjAXjooYcYPnw4rVu35vjx4yxatIh27doB8OSTT9K9e3c6dOhAUVERc+fOdW7zdG4PTn9kt9tdJnOojM1mY/PmzVx++eU1XFXNCCwb46SueiIiIiL1yn/+8x9uu+02+vTpQ2RkJI8//rhbJil7/PHHSU1NZdy4cVgsFu666y6GDh2KxVL1+K7+/fu7PLdYLJSWljJjxgwefPBBrrzySoqLi+nfvz/z5s1zdhu02WxMnDiRgwcPEhwczLBhw/jvf/8LOBpNJk2axL59+/Dz86Nfv3589tln1X/hNcBkuLET5KRJkxg+fDgJCQnk5OTwySef8NxzzzF//nwGDx7MuHHjaNy4MVOmTAEcs5X06tWLli1bkpmZyQsvvMCcOXNYu3ZtheOi/ig7O5uQkBCysrLc3m3v9yO5XPbSLwT7erFpsnsGRYqIiIjUlsLCQvbu3UuzZs3q9L046zK73U67du24/vrrefbZZ91dTq2o7PfubLKBW1uc0tPTGTduHCkpKYSEhNC5c2dnaALHdIVm88lhWMePH+fOO+8kNTWVsLAwunfvzrJly844NHmaU7vqGYZRp+9JJSIiIiKeZ//+/SxYsIABAwZQVFTEa6+9xt69e7npppvcXVqd49bg9O6771a6ffHixS7P//vf/zqb+eqDsq56dsNx87fqugGbiIiIiAiA2Wxm5syZPPbYYxiGQceOHfnxxx/rzLgiT6Jv6m7k523BbHIEp9yiUgUnEREREalW8fHxLF261N1l1Atun468ITOZTM4pyTVBhIiIiIiI51JwcjNNSS4iIiIi4vkUnNxMU5KLiIiIiHg+BSc3C1CLk4iIiIiIx1NwcrNABScREREREY+n4ORmQb4KTiIiIiIink7Byc0CfBScRERERAT27duHyWRiw4YNNf5aM2fOJDQ0tMZfpz5RcHIzTQ4hIiIi4vkmTJiAyWQ6bRk2bJi7S6tS06ZNefnll13WjRkzhp07d9b4aw8cOJCHHnqoxl+nNuiOq26m6chFRERE6oZhw4YxY8YMl3VWq9VN1ZwfPz8//Pz83F1GnaIWJzcL1BgnERERacgMA4rz3LMYxlmVarVaiY2NdVnCwsIAuOmmmxgzZozL/iUlJURGRvLBBx8A8MMPP3DxxRcTGhpKREQEV155Jb///nuFr1ded7o5c+ZgMpmcz3///XdGjhxJTEwMgYGBXHjhhfz444/O7QMHDmT//v08/PDDzlayis795ptv0qJFC3x8fGjTpg0ffvihy3aTycT06dMZPXo0/v7+tGrVim+++ebM3rwK/O9//6NDhw5YrVaaNm3KSy+95LL9jTfeoFWrVvj6+hITE8O1117r3PbVV1/RqVMn/Pz8iIiIYNCgQeTl5Z1XPZVRi5ObOacjV1c9ERERaYhK8uFfjdzz2k8cBp+AajnV2LFjue6668jNzSUwMBCA+fPnk5+fz+jRowHIy8vjkUceoXPnzuTm5vLkk08yevRoNmzYgNl8bu0Zubm5XH755fzzn//EarXywQcfMGLECJKSkkhISGDWrFl06dKFu+66izvvvLPC88yePZsHH3yQl19+mUGDBjF37lxuvfVWmjRpwiWXXOLc7+mnn+b555/nhRdeYOrUqYwdO5b9+/cTHh5+1rWvXbuW66+/nsmTJzNmzBiWLVvGvffeS0REBBMmTGDNmjU88MADfPjhh/Tp04djx46xZMkSAFJSUrjxxht5/vnnGT16NDk5OSxZsgTjLMPw2VBwcjNNRy4iIiJSN8ydO9cZiso88cQTPPHEEwwdOpSAgABmz57NLbfcAsAnn3zCVVddRVBQEADXXHONy7HvvfceUVFRbNu2jY4dO55TTV26dKFLly7O588++yyzZ8/mm2++4b777iM8PByLxUJQUBCxsbEVnufFF19kwoQJ3HvvvQA88sgjrFixghdffNElOE2YMIEbb7wRgH/961+8+uqrrFq16pzGev3nP//hsssu4+9//zsArVu3Ztu2bbzwwgtMmDCB5ORkAgICuPLKKwkKCiIxMZFu3boBjuBUWlrK1VdfTWJiIgCdOnU66xrOhoKTm5VNR56n4CQiIiINkbe/o+XHXa99Fi655BLefPNNl3VlLS1eXl5cf/31fPzxx9xyyy3k5eXx9ddf89lnnzn33bVrF08++SQrV64kIyMDu90OQHJy8jkHp9zcXCZPnsx3333nDBMFBQUkJyef1Xm2b9/OXXfd5bKub9++vPLKKy7rOnfu7HwcEBBAcHAw6enp51T79u3bGTly5Gmv+fLLL2Oz2Rg8eDCJiYk0b96cYcOGMWzYMGc3wS5dunDZZZfRqVMnhg4dypAhQ7j22mudXSdrgoKTm5VNR56j4CQiIiINkclUbd3lalpAQAAtW7ascPvYsWMZMGAA6enpLFy4ED8/P5eWmBEjRpCYmMg777xDo0aNsNvtdOzYkeLi4nLPZzabT+t6VlJS4vL8scceY+HChbz44ou0bNkSPz8/rr322grPeb68vb1dnptMJmcArG5BQUGsW7eOxYsXs2DBAp588kkmT57M6tWrCQ0NZeHChSxbtowFCxYwdepU/vrXv7Jy5UqaNWtWI/Vocgg303TkIiIiIvVDnz59iI+P5/PPP+fjjz/muuuucwaNo0ePkpSUxN/+9jcuu+wy2rVrx/Hjxys9X1RUFDk5OS4THvzxHk9Lly5lwoQJjB49mk6dOhEbG8u+fftc9vHx8cFms1X6Wu3atWPp0qWnnbt9+/ZVXPW5q+g1W7dujcViARwteYMGDeL5559n06ZN7Nu3j59//hlwhLa+ffvy9NNPs379enx8fJg9e3aN1asWJzcLsjr+MGmMk4iIiIhnKyoqIjU11WWdl5cXkZGRzuc33XQT06ZNY+fOnSxatMi5PiwsjIiICN5++23i4uJITk7mL3/5S6Wv17NnT/z9/XniiSd44IEHWLlyJTNnznTZp1WrVsyaNYsRI0ZgMpn4+9//floLUNOmTfn111+54YYbsFqtLvWW+b//+z+uv/56unXrxqBBg/j222+ZNWuWywx95+rIkSOnBb64uDgeffRRLrzwQp599lnGjBnD8uXLee2113jjjTcAx5iyPXv20L9/f8LCwpg3bx52u502bdqwcuVKfvrpJ4YMGUJ0dDQrV67kyJEjtGvX7rzrrZDRwGRlZRmAkZWV5e5SDMMwjIycQiPx8blG4uNzjVKb3d3liIiIiNSYgoICY9u2bUZBQYG7Szlr48ePN4DTljZt2rjst23bNgMwEhMTDbvd9bvdwoULjXbt2hlWq9Xo3LmzsXjxYgMwZs+ebRiGYezdu9cAjPXr1zuPmT17ttGyZUvDz8/PuPLKK423337bOPUr/N69e41LLrnE8PPzM+Lj443XXnvNGDBggPHggw8691m+fLnRuXNnw2q1Oo+dMWOGERIS4lLfG2+8YTRv3tzw9vY2WrdubXzwwQcu20+ttUxISIgxY8aMCt+3AQMGlPu+Pfvss4ZhGMZXX31ltG/f3vD29jYSEhKMF154wXnskiVLjAEDBhhhYWGGn5+f0blzZ+Pzzz93vs9Dhw41oqKiDKvVarRu3dqYOnVquTVU9nt3NtnAdOJNaDCys7MJCQkhKyuL4OBgd5dDUamNNn/7AYBNk4cQ7OtdxREiIiIidVNhYSF79+6lWbNm+Pr6urscaSAq+707m2ygMU5uZvWy4GNxfAwa5yQiIiIi4pkUnDxAgNUx+E1TkouIiIiIeCYFJw9QNrOepiQXEREREfFMCk4eILBsZj111RMRERER8UgKTh4gyOpocVJXPREREWkIGtjcZOJm1fX7puDkAcrGOKmrnoiIiNRnZTeDzc/Pd3Ml0pAUFxcDOG+qe650A1wPEOirrnoiIiJS/1ksFkJDQ0lPTwfA398fk8nk5qqkPrPb7Rw5cgR/f3+8vM4v+ig4eYBAddUTERGRBiI2NhbAGZ5EaprZbCYhIeG8Q7qCkwcIPNFVL1fBSUREROo5k8lEXFwc0dHRlJSUuLscaQB8fHwwm89/hJKCkwcom1VPY5xERESkobBYLOc95kSkNmlyCA9Qdh8njXESEREREfFMCk4eoKyrnsY4iYiIiIh4JgUnD6CueiIiIiIink3ByQOoq56IiIiIiGdTcPIAzq56xQpOIiIiIiKeSMHJA5R11VOLk4iIiIiIZ1Jw8gBlXfU0xklERERExDMpOHmAQKsjOBWX2ikutbu5GhERERER+SMFJw8Q4HPy5m+aklxERERExPMoOHkAL4sZP29HeMpVcBIRERER8TgKTh7COc5JE0SIiIiIiHgcBScPUTbOSVOSi4iIiIh4HrcGpzfffJPOnTsTHBxMcHAwvXv35vvvv6/0mC+//JK2bdvi6+tLp06dmDdvXi1VW7PKgpOmJBcRERER8TxuDU5NmjTh3//+N2vXrmXNmjVceumljBw5kq1bt5a7/7Jly7jxxhu5/fbbWb9+PaNGjWLUqFFs2bKlliuvfmXBSVOSi4iIiIh4HpNhGIa7izhVeHg4L7zwArfffvtp28aMGUNeXh5z5851ruvVqxddu3Zl2rRpZ3T+7OxsQkJCyMrKIjg4uNrqPl93vL+GH7enMeXqTtx4UYK7yxERERERqffOJht4zBgnm83GZ599Rl5eHr179y53n+XLlzNo0CCXdUOHDmX58uUVnreoqIjs7GyXxRMF+aqrnoiIiIiIp3J7cNq8eTOBgYFYrVbuvvtuZs+eTfv27cvdNzU1lZiYGJd1MTExpKamVnj+KVOmEBIS4lzi4+Ortf7qoq56IiIiIiKey+3BqU2bNmzYsIGVK1dyzz33MH78eLZt21Zt5580aRJZWVnO5cCBA9V27upUNh25boArIiIiIuJ5vNxdgI+PDy1btgSge/furF69mldeeYW33nrrtH1jY2NJS0tzWZeWlkZsbGyF57darVit1uotugZoVj0REREREc/l9hanP7Lb7RQVFZW7rXfv3vz0008u6xYuXFjhmKi6xBmc1OIkIiIiIuJx3NriNGnSJIYPH05CQgI5OTl88sknLF68mPnz5wMwbtw4GjduzJQpUwB48MEHGTBgAC+99BJXXHEFn332GWvWrOHtt99252VUCwUnERERERHP5dbglJ6ezrhx40hJSSEkJITOnTszf/58Bg8eDEBycjJm88lGsT59+vDJJ5/wt7/9jSeeeIJWrVoxZ84cOnbs6K5LqDYBCk4iIiIiIh7L4+7jVNM89T5OS3dnMHb6StrEBDH/4f7uLkdEREREpN6rk/dxaujUVU9ERERExHMpOHkIddUTEREREfFcCk4eIsj3ZHBqYL0nRUREREQ8noKThyjrqmezGxSW2N1cjYiIiIiInErByUP4+1gwmRyP1V1PRERERMSzKDh5CJPJRKCPxjmJiIiIiHgiBScPElg2zqlQwUlERERExJMoOHkQTUkuIiIiIuKZFJw8iKYkFxERERHxTApOHuTklOQlbq5EREREREROpeDkQZxd9TTGSURERETEoyg4eZCTXfVsbq5EREREREROpeDkQU5ODqGueiIiIiIinkTByYMEaTpyERERERGPpODkQdRVT0RERETEMyk4eRB11RMRERER8UwKTh7k5HTk6qonIiIiIuJJFJw8SICPuuqJiIiIiHgiBScPEuicHEJd9UREREREPImCkwc5OcZJXfVERERERDyJgpMHKRvjlKeueiIiIiIiHkXByYMEnNLiZLcbbq5GRERERETKKDh5kLKuegB5xequJyIiIiLiKRScPIjVy4y3xQRonJOIiIiIiCdRcPIgJpPJ2V0vT8FJRERERMRjKDh5mLLuejmFCk4iIiIiIp5CwcnDaEpyERERERHPo+DkYQLVVU9ERERExOMoOHmYQF911RMRERER8TQKTh5GXfVERERERDyPgpOHCfJVVz0REREREU+j4ORhAnxOdNVTcBIRERER8RgKTh6mbIxTrsY4iYiIiIh4DAUnD6NZ9UREREREPI+Ck4fR5BAiIiIiIp5HwcnDaDpyERERERHPo+DkYdTiJCIiIiLieRScPIzGOImIiIiIeB4FJw8T5OsNQEZuMYUlNjdXIyIiIiIioODkcZpHBdA41I/colI+WrHf3eWIiIiIiAgKTh7H22LmgctaAvDm4t811klERERExAMoOHmgay5oQrPIAI7mFTPjt73uLkdEREREpMFza3CaMmUKF154IUFBQURHRzNq1CiSkpIqPWbmzJmYTCaXxdfXt5Yqrh1eFjMPDWoFwNtL9pCVX+LmikREREREGja3BqdffvmFiRMnsmLFChYuXEhJSQlDhgwhLy+v0uOCg4NJSUlxLvv317+xQCM6N6JtbBA5haW89evv7i5HRERERKRB83Lni//www8uz2fOnEl0dDRr166lf//+FR5nMpmIjY2t6fLcymw28cjg1tz14VpmLN3HrX2bERVkdXdZIiIiIiINkkeNccrKygIgPDy80v1yc3NJTEwkPj6ekSNHsnXr1gr3LSoqIjs722WpKwa3j6FLfCgFJTbeWLzb3eWIiIiIiDRYHhOc7HY7Dz30EH379qVjx44V7temTRvee+89vv76az766CPsdjt9+vTh4MGD5e4/ZcoUQkJCnEt8fHxNXUK1M5lMPDakNQAfr0jmUGaBmysSEREREWmYTIZhGO4uAuCee+7h+++/57fffqNJkyZnfFxJSQnt2rXjxhtv5Nlnnz1te1FREUVFRc7n2dnZxMfHk5WVRXBwcLXUXpMMw+CGt1ewcu8xbrgwnn9f09ndJYmIiIiI1AvZ2dmEhIScUTbwiBan++67j7lz57Jo0aKzCk0A3t7edOvWjd27y+/KZrVaCQ4OdlnqEpPJxP8NbQPAl2sPknw0380ViYiIiIg0PG4NToZhcN999zF79mx+/vlnmjVrdtbnsNlsbN68mbi4uBqo0DP0aBpOnxYR2OwG321OcXc5IiIiIiINjluD08SJE/noo4/45JNPCAoKIjU1ldTUVAoKTo7lGTduHJMmTXI+f+aZZ1iwYAF79uxh3bp13Hzzzezfv5877rjDHZdQa4Z1dMwiuGhHupsrERERERFpeNw6Hfmbb74JwMCBA13Wz5gxgwkTJgCQnJyM2Xwy3x0/fpw777yT1NRUwsLC6N69O8uWLaN9+/a1VbZbXNImGtjK2uTjZOWXEOLv7e6SREREREQaDI+ZHKK2nM0AME8z+D+/sCs9l1dv7MZVXRq5uxwRERERkTqtzk0OIWfm0rbRgLrriYiIiIjUNgWnOuSSE8FpcVI6NnuDaigUEREREXErBac6pHtiGEG+XhzPL2HDgUx3lyMiIiIi0mAoONUh3hYz/VtHAequJyIiIiJSmxSc6phL2zi66/2s4CQiIiIiUmsUnOqYgW2iMJlgW0o2qVmF7i5HRERERKRBUHCqYyICrXRpEgrAoiS1OomIiIiI1AYFpzqobFpyddcTEREREakdCk51UFlwWro7g6JSm5urERERERGp/xSc6qAOjYKJDrKSX2xj5Z5j7i5HRERERKTeU3Cqg0wmE5dodj0RERERkVqj4FRHXXKiu96ipHQMw3BzNSIiIiIi9ZuCUx11catIvC0m9h/NZ09GnrvLERERERGp1xSc6qhAqxf9mgbylNf7rFy13N3liIiIiIjUawpOddifzR9xq9d8uq98mP/O20Bxqd3dJYmIiIiI1EsKTnVY82ueJtsSThvzARote4pr3lzG7vRcd5clIiIiIlLvKDjVYT6hcQTf/D4GZsZ4LaZVyrdcOXUJHyzfpwkjRERERESqkYJTXdesP6ZLJgEwxTqDJqXJPPn1Vu75aB12u8KTiIiIiEh1UHCqD/o9Cs0HYjWK+DJ8GiFexfywNZVFSbrHk4iIiIhIdVBwqg/MFrh6OgTGEpa3hw9jvwBg2i+/u7kwEREREZH6QcGpvgiMgmvfBZOZzhnzuMHrV1bvO86afcfcXZmIiIiISJ2n4FSfNL0YLnkCgGe8Z9DadECtTiIiIiIi1UDBqb65+FFocRk+RhFveL/Csu3J7EzLcXdVIiIiIiJ1moJTfWM2w9VvQ1AcLc2H+Yf3e7y1WK1OIiIiIiLnQ8GpPgqIhGvfwzBZuNryGz6bP+ZQZoG7qxIRERERqbMUnOqrxD6YLv0bAE9ZZvDt/IVuLkhEREREpO5ScKrP+j7E0bgB+JpKGLrtzxw/rhn2RERERETOhYJTfWY2E37zexwxRdLMlEL6J3eDYbi7KhERERGROkfBqZ4zBUSy4+KXKTXMtDkyn+JV77m7JBERERGROkfBqQHoPfAK3va5GQDz/L9AyiY3VyQiIiIiUrcoODUAXhYzoZc9yk+2bnjZi7F/MR4Ks91dloiIiIhInaHg1EBcd2ECLwc9zCEjAvPxPfDtgxrvJCIiIiJyhhScGghvi5k7hvTg/uL7KcECW2fBmnfdXZaIiIiISJ2g4NSAjOjciLzo7jxXcoNjxQ+T4PAGt9YkIiIiIlIXKDg1IGaziUeHtGa67XJ+MnqArRi+nACFWe4uTURERETEoyk4NTCD28fQJT6Mh4vuItMnFo7vhW/u13gnEREREZFKKDg1MCaTiT8PbUM2gdyRfx+G2Ru2fQ2r3nF3aSIiIiIiHkvBqQHq2zKSPi0iWFPanLnRdztWzn8CDq1zb2EiIiIiIh5KwamBemxoGwAeSu5NbvPhYC9xjHcqyHRrXSIiIiIinkjBqYG6ICGMQe2isdnh8ZI/YYQmQOZ++HqixjuJiIiIiPyBglMD9tjQNnhbTHy3K5/3Gz8NZm/YMRdWTnN3aSIiIiIiHsWtwWnKlClceOGFBAUFER0dzahRo0hKSqryuC+//JK2bdvi6+tLp06dmDdvXi1UW/+0jQ3m31d3BmDyWitr2j7q2LDg73BwrRsrExERERHxLG4NTr/88gsTJ05kxYoVLFy4kJKSEoYMGUJeXl6Fxyxbtowbb7yR22+/nfXr1zNq1ChGjRrFli1barHy+uOa7k148LJWAIzZ0Jkj8UNPjnc6+rt7ixMRERER8RAmw/CcAS1HjhwhOjqaX375hf79+5e7z5gxY8jLy2Pu3LnOdb169aJr165Mm1Z1F7Ps7GxCQkLIysoiODi42mqvywzD4NEvNjJr/SFircX8GvIUPtn7wTcErnkXWg12d4kiIiIiItXubLKBR41xysrKAiA8PLzCfZYvX86gQYNc1g0dOpTly5eXu39RURHZ2dkui7gymUxMuaYTPZuFk1rkwzWFf6c4rjsUZsHH18GvL2rCCBERERFp0DwmONntdh566CH69u1Lx44dK9wvNTWVmJgYl3UxMTGkpqaWu/+UKVMICQlxLvHx8dVad31h9bLw9i09aB4VwOZsf64r/Bul3SYABvz8LHxxCxTluLtMERERERG38JjgNHHiRLZs2cJnn31WreedNGkSWVlZzuXAgQPVev76JMTfm5kTLiIiwIeNKQX8rfR2GPEKWHxg+7cwfRBk7HZ3mSIiIiIitc4jgtN9993H3LlzWbRoEU2aNKl039jYWNLS0lzWpaWlERsbW+7+VquV4OBgl0UqlhDhz9Qbu2EywWerD/C1ZTBMmAdBcXBkB7xzCST94O4yRURERERqlVuDk2EY3HfffcyePZuff/6ZZs2aVXlM7969+emnn1zWLVy4kN69e9dUmQ1On5aR3H+pY6a9J2ZtZo9vO7jrF4jvBUXZ8OkYWPwc2O1urlREREREpHacU3A6cOAABw8edD5ftWoVDz30EG+//fZZnWfixIl89NFHfPLJJwQFBZGamkpqaioFBQXOfcaNG8ekSZOczx988EF++OEHXnrpJXbs2MHkyZNZs2YN991337lcilTgwcta0bNZOHnFNu77ZD2FvpEw/lu48E7HDov/BZ/fDIWabENERERE6r9zCk433XQTixYtAhyTNQwePJhVq1bx17/+lWeeeeaMz/Pmm2+SlZXFwIEDiYuLcy6ff/65c5/k5GRSUlKcz/v06cMnn3zC22+/TZcuXfjqq6+YM2dOpRNKyNmzmE28emM3wgN82JaSzb/mbQcvH7jiRRj5OliskPQdvHMpHNnp7nJFRERERGrUOd3HKSwsjBUrVtCmTRteffVVPv/8c5YuXcqCBQu4++672bNnT03UWi10H6ezszgpnQkzVgPwxtgLuLxTnGPDobXw+S2QfQh8guDqt6DtFW6sVERERETk7NT4fZxKSkqwWq0A/Pjjj1x11VUAtG3b1qV1SOq+gW2iuXtACwAe/2oTu9NzHRsad3eMe0rsC8U58NlN8PM/wW7nWF4xmw9mubFqEREREZHqdU7BqUOHDkybNo0lS5awcOFChg0bBsDhw4eJiIio1gLF/R4d0pruiWHkFJVy5dQlTF+yB5vdgMAoGPc19LzbseOvz3Pwzau48oW5jHjtN77deNi9hYuIiIiIVJNzCk7PPfccb731FgMHDuTGG2+kS5cuAHzzzTdcdNFF1VqguJ+3xcybYy+gT4sICkvs/OO77Vz95jKSUnPA4g3DnyN5wH8pwocmR5bwkX0SLU0HeWF+EsWlmnlPREREROq+cxrjBGCz2cjOziYsLMy5bt++ffj7+xMdHV1tBVY3jXE6d4Zh8PnqA/zzu+3kFJXibTFx78CWZOYX88GK/bRnL+9Y/0sjMsjDl0eK76bviFsZ17upu0sXERERETlNjY9xKigooKioyBma9u/fz8svv0xSUpJHhyY5PyaTiRsuSmDhIwMY1C6GEpvBKz/t4v3l+zEMaNWlL973/ALN+hNAIW/5vEzpgqfJKyhyd+kiIiIiIuflnFqchgwZwtVXX83dd99NZmYmbdu2xdvbm4yMDP7zn/9wzz331ESt1UItTtXDMAy+25zC099uI8TPm6ev6kDflpGOjbZSbAuexLLydQD2h/Uh8a5PwC+skjOKiIiIiNSuGm9xWrduHf369QPgq6++IiYmhv379/PBBx/w6quvnssppY4xmUxc2bkRKyZdxsKH+58MTQAWLyzD/8Wa7s9TYPiQeHwZtmkDIW2r2+oVERERETkf5xSc8vPzCQoKAmDBggVcffXVmM1mevXqxf79+6u1QPFsFrMJk8lU7rYLrriLx4Jf5IA9CkvWPpg+CJa9BkU5tVukiIiIiMh5Oqfg1LJlS+bMmcOBAweYP38+Q4YMASA9PV3d38TJbDZx7ZXDGVH8D36zd4KSfFjwV/hPB1jwd8g65O4SRURERETOyDmNcfrqq6+46aabsNlsXHrppSxcuBCAKVOm8Ouvv/L9999Xe6HVRWOcapdhGIx5ewVr9mbwQvONXFM0G47udmw0e0GH0dD7PmjU1a11ioiISANmK4WibEevGOdy4rnd5rj9isXnxHKWj81n2E5ht0FpEZQWgq34xOMisBVBafGJ9UWnrD+x7tT97HYwAZjAZAKT+eTjKtdR+X6nbj+rx1S8T+th4O17Fh9U9TubbHDO05GnpqaSkpJCly5dMJ/4hVi1ahXBwcG0bdv2XE5ZKxScat/a/ce55s1lmE2w4KF+tMxcCstfh31LTu7UtJ8jQLUawsIdR/hsVTIPD25Nx8Yh7itcRERE6gZbCWTsgsJM1+BTWF4Y+uO6HEevmJpispQTqLwcNZcWOkKRrQjspTVXg6d6bBcEundG7loJTmUOHjwIQJMmTc7nNLVGwck97vxgDQu3pdG5SQgzb72I8AAfOLzeEaC2zALDBkBuUDP+ffxSviztR1BgELPv7UN8uL+bqxcRERGPUpAJB1dD8go4sBIOroHSgvM/r5cfWIPAN9jx0xrk6CFjK3G08NiKq35cHQHIZAYvX0fI8rI6FovVsc7L58TjsvU+J9ebLIABhnHiJ2DY/7DOOIN1lL+u3Men7F/hY8o/xw0fg3/4+b9f56HGg5Pdbucf//gHL730Erm5uQAEBQXx6KOP8te//tXZAuWJFJzcY8+RXK5+cxmZ+SU0jwrgw9t70jjUz7Ex6yCsfIuS1e/hXeL4fTpmBPGhbTC/hY5i+sTLCfHzdmP1IiIi4jaGAZnJJ0LSCkheCenbcH4xL2MNgcCok4HHGnxiCXJdfEP+sO6UfSzV8H3Dbgd7SRUhq9TR6uQMR76uIcjidf51yBmp8eA0adIk3n33XZ5++mn69u0LwG+//cbkyZO58847+ec//3luldcCBSf32Z2ew7h3V3E4q5DYYF8+uP0iWsc4ZmdctCOdhz9cwtUsYqLfAiJK0wAoMrxZFnAZfW95Cp+49u4sX0RERGqDrRTSNjuCUlmLUk7K6fuFN4eE3hDfExJ6QUSrMx9PJHJCjQenRo0aMW3aNK666iqX9V9//TX33nsvhw557mxpCk7udTizgHHvrWJ3ei4hft68N6EHeUU27vhgDcWldq7oHMcr13XEK2ku+b+8gv+RDc5jjZaDMfW5D5oNcB1sKCIiInVXYfYp3e5WwMG1UJLnuo/ZC+K6OgJSQi9HWHLz2BipH2o8OPn6+rJp0yZat27tsj4pKYmuXbtSUFANfUxriIKT+x3PK+a291ezPjkTX28zhgFFpXaGdojhtZsuwNty4n+LDIN1v/3AkQUvMdi8BrPpxK9qTCfoPRE6XuPozysiIiJ1R95R2LPoZItS+tYTY2lO4RviCEdlrUmNLgAfjXmW6lfjwalnz5707NmTV1991WX9/fffz6pVq1i5cuXZnrLWKDh5hvziUu79eB2Lk44AMKhdNG+M7Y6P1+lN7B8u38f0b37iVssP3OT9Kz5GIQDZ3pGsib6O4+3HMqp3RyzmiluhCopt/G3OFiICffjLsLaYK9lXREREasjR32H6ICg45ro+NNHR7S6hJ8T3gqi26nYntaLGg9Mvv/zCFVdcQUJCAr179wZg+fLlHDhwgHnz5tGvX79zq7wWKDh5jhKbnZd/3El+sY2/DG+L1ctS4b7/mLuN6b/tJYRcxlp+YrzXfGJMmQAUGV4cDOlOiz5XQ+shjj7Pp7DbDe7/dD3fbXb0j35sSGvuu7RVjV2XiIiIlKM4H94dDGlbIKwptB5+sutdUKy7q5MGqlamIz98+DCvv/46O3bsAKBdu3bcdddd/OMf/+Dtt98+l1PWCgWnusluN5iz4RB7M/IcM1jaiml7dCHdD31MXOFu150jW0OrIdB6KCT05uVFe3n5x12YTWA3HMOjPrjtIvq1inLLtYiIiDQ4hgGz74ZNn0FANPzpVwiOc3dVIrV7H6dTbdy4kQsuuACbzVZdp6x2Ck71jGHw6bwf2bNsFpdaNtDTshOzcfL+CSVegSwoas8iezf6Db+R5WkWPlt9gDB/b+Y+0O/klOgiIiJSc1ZPh+8eddxnaPw30PRid1ckAig4VUrBqf4xDIMnZm/h01XJRHoX8uXgQpod/Y3SpPl4FR512dcedwGfZrbls8z2WBp14fN7+lbaRVBERETO08E18N4wx72NBj8LfR9wd0UiTmeTDXR3LanzTCYTz47swOHMAn7ZeYTrfg3mrVv+yX3briamaBsTopIYGbAFU8pGzCnrGMs6xlohPSOULa/3ofvgG6HFJY4b34mIiEj1ycuAL8Y5QlO7q6DP/e6uSOScqcVJ6o3colKun7acbSnZznUtogKYPbEvwb7ekJMKuxbAzvmU7l6EV+kp94gwe0NiH8e4qFZDIbKlG65ARESkHrHb4KOrYc9ix81p7/wZfPXdSzxLjXXVu/rqqyvdnpmZyS+//KLgJG6TmlXI6DeWkpJVSIifN19P7EvTyIDTdywt4n+zvyBz43dcZllPU1Oq6/bw5tDmcmh7JcRfBGZ15xMRETkrPz0DS14Cb39HaIpu5+6KRE5TY8Hp1ltvPaP9ZsyYcaanrHUKTvXf7vQc3v51Dzf1TKRrfGiF+9ntBnd8sIafd6TT1JTC6ICt3BC6nehjazDZS07uGBAFbYZD2xHQfAB4WWv+IkREROqyHfPgsxsdj695Fzpd6956RCrgtskh6gIFJzlVic3OF2sO8OpPu0jLLgKgU5SZZzqk0TV/Kaad86Ew6+QBPoHQarCjJarVYMedzUVEROSko7/D25dAURb0vBuGP+fuikQqpOBUCQUnKU9hiY0Plu/jjcW/k5nvaG3q1yqSV67rSPiRVbBjLuz4DnJSTh5k9na0QLW90tGtLyjGTdWLiIh4iFNvchvfE8bPBS8fd1clUiEFp0ooOEllsgtLmP7rHt5esofCEjuNQnx5fewFdEsIA7sdDq+HHd/C9rlwdJfzOAMTpviLHCGq7RUQ0cKNVyEiIuIGhgFz7oGNnzq6uf/pVwhu5O6qRCql4FQJBSc5E0mpOdzz0Vr2ZOThbTHx5JXtublXIiaTCYBth7OZteBnrLvnMdi8mq7mPa4niG5/MkTFdYETx4mIiNRbq9+F7x5x3OR23NfQrJ+7KxKpkoJTJRSc5EzlFJbwf19u4oetjhn3RnVtxHU94pm+ZA+Lko449/P3sRBcnM51gZu4J2Y7/oeXg3HKzJIhCY4A1fYKSOgNFt0+TURE6pmDa2HGMLAVw+BnoO+D7q5I5IwoOFVCwUnOhmEYTF+yl3//sAOb/eQfFbMJLu8Uxz0DWxDs683491axJyOPYF8vZtzQiu5Fq2D7t7D7JygtOHlCvzBoOQhaDXH89A8/7TWzCkq456O1RARaefWGrs5WLhEREY+UdxTe6g/ZBx29LcZ8pJ4WUmcoOFVCwUnOxco9R7nv0/Vk5ZdwTfcm/Kl/c5f7Qx3LK+aO91ezLjkTHy8zr4zpyvBOcRxMy2D/6u/w3jWPNlm/EULuKWc1QZMLHSGq1WCI7UypAbfOXM2SXRkA/Pp/l5AQ4V/LVysiInKGTr3JbXgLuGuRZpyVOkXBqRIKTnKuCoptlNjtBPt6V7j9gc/Ws3BbGiYTxAX7cjir0Lndgo0LTLsYat3E2PAk/I5tdz1BYAzrfHrwTmpLfrN3Igd/nr+mM9dfGF+TlyUiInLufnoWlrzouMntHT9BTHt3VyRyVhScKqHgJDXJZjd46pstfLQiGQAvs4nOTULo1TyCC5uF88ai3azed5wQP2++uDGBNjkrYNdC+H0RlOQ5z1OKhTX21hyJ7c+IayY47raubg8iIuJJkr6HT29wPL56OnS+zr31iJwDBadKKDhJTTMMg0VJ6XiZzXRPDCPAenIyiNyiUm55dyXrkzMJ8/fm07t60TY2mCXbD/L2R5/Q37Sea4O3EZa/z/WkwU0c3flaD4Vm/cEnABEREbc5tgfeGui4ye1Fd8HlL7i7IpFzouBUCQUncbfswhJumb6SjQeziAjw4R+jOvLnrzaRU1TKdd2b8Py1nSlK/53nXptKf9YzwCcJs+1klz8sPtD04hNjo4bonlEiIlK7ivPh3SGQthmaXAQTvtNNbqXOUnCqhIKTeIKsghLGTl/BlkPZznUXNQ3nozt64uNlBuC6actYve84L45qybXh+2HXAtg1HzKTXU8W3hwad4eIVhDZEiJOLGqVEhGR6mYYMOde2PgJ+Ec6bnIb0tjdVYmcs7PJBrqhjIgbhPh589HtPbnpnZVsS8kmIdyfabd0d4YmgN7NI1i97zi/7cvn2l5DoPUQMF6AjF0nQtQC2L/M0V3i2J7TXyS4sSNARbY6JVS1gpB4MJtP319ERKQqa2c4QpPJDNe+p9AkDYpanETcKDO/mNnrDzG0QyyNQv1cti3bncFN01cSG+zL8kmXln8/p6Ic2PcbpG+Ho7/D0V2OYFVwrOIXtVgd3ftcQlUrx3O/0Oq9QBERqT8OrYX3TtzkdtBkuPhhd1ckct7U4iRSR4T6+3Br32blbrsgMQwfi5nU7EL2H813uW+UkzUI2gx3LKfKP+YIUGVB6uhux3JsD9iKIH2bY/mjgChHkIpoAVFtoUkPiOsC3n6n7ysiIg1H3lH4fJwjNLW9Evo+5O6KRGqdgpOIh/L1ttA1IZRVe4+xfM/R8oNTRfzDIaGnYzmVrRSykiFjt2uoytgFuamQd8SxJC87eYzZC2I7OW7W27iHI0yFN9f06CIiDYXdBv+7HbIPOm5yO+oN/RsgDZKCk4gH69U8glV7j7Fiz1FuvCjh/E9o8XKEnvDmwBDXbUU5J0LUiVCVugUOroa8dDi83rHwtmNfv3BHkGpyoSNINb5Ad4oXqY8yD0BxLpQWOVoabMV/eFzsaMUu93F5+xeBrQR8/CEwBgKjT/kZ63jsF6ZxmJ5m8b9hzyLw8oMxH+rve2mw3Bqcfv31V1544QXWrl1LSkoKs2fPZtSoURXuv3jxYi655JLT1qekpBAbG1uDlYq4R6/m4bz6E6zYcxTDMMof51RdrEHQqJtjKWMYkHXAEaAOrnH8TNnoGEO1a75jAcAEUW0cIaosUEW1BbOl5uoVkZpjt8Gce2DT57X/2mYvCIiGoJg/hKtTlxPrfPyrPp9hQHGe4z+HypbiHNfnRdlQlPuHdSfWmy1gDQbfYLCGOEKDb/Ap64L/sC7E8bOmp+e2208E0qKToRUTeFkdi8UKFu/zbxnaOR9+fd7x+KpXIabDeZcuUle5NTjl5eXRpUsXbrvtNq6++uozPi4pKcll8FZ0dHRNlCfidhckhOHjZSYtu4i9GXk0jwqs3QJMJghNcCwdr3GsKy062RpVtmTuhyM7HMv6jxz7+QQ6WqKcXfwuhMCo2q1fRM6eYcDchxyhyWR2tDBbfBxBwOLj+ELufHxi8bKe8vjEPi7HnLqPtyPI5KZBTprjZ26642fBMbCXQs5hx1IVa/DJEOUXBiX55YSfHMAN82B5+ZUTsE4EK59AR8vbqaHHVnKiRe6Udc6Wuz+uK3K8T2fCYgUv35Ofi/On9eRn4nVin1M/p7Jj1n3gOM+Fd0Ln62vu/RKpA9wanIYPH87w4cOr3vEPoqOjCQ0Nrf6CRDyMr7eFbvGhrNx7jBV7jtV+cCqPlxWadHcs3O1Yl3sEDq05GaQOrXN079n7q2MpE5roGC8V1daxRLd1TEbh7euWSxGRPzAMWPA3x5dlkxmumwntR9be65cWO7oHnxqmyn7mpJ6yLg1KC0+0FGU7uhlXxWRxtKxXtPhUsN5uc7xGYdbJn4UnXtf585R1xbknrqUAcgsctdYGi4/j87OXuK63nQhjRedx7iYXwtB/nVd5IvVBnRzj1LVrV4qKiujYsSOTJ0+mb9++Fe5bVFREUdHJvy2ys7Mr3FfEE/VqHnEiOB3lpp7VMM6pJgRGuc7uZ7c5Wp+crVJrHc8z9zuWHXNPHmsyQ1jTE2GqzclQFdn6zLrhiEj1+fVFWP6a4/FVU2s3NIGjhSOkiWOpjGE4WpLKQlRuGhQcd7TkWINO/rQGnwxA3n61M6HBqUHr1IDlDF4nwtUfW+hObQlytvx4l7POx7UrXtm6smsr68JXWug6xqy08A+Py1qwispp6TplnckCve6t+a6HInVAnQpOcXFxTJs2jR49elBUVMT06dMZOHAgK1eu5IILLij3mClTpvD000/XcqUi1ad3iwhe+WlX7Yxzqi5mi6MffEwH6D7Bsa4wyzHBRPr2E936khyPCzNP3sQ3ad4pJznRTbAsUEW3c/yMbANWD2h5E6lvVkyDRf9wPB72b+h2s3vrqYzJdKLbW7DjPnSexGxxdBv0C3PT65vB7KuWfJEa4DE3wDWZTFVODlGeAQMGkJCQwIcfflju9vJanOLj43UDXKkzCktsdH56AcWldn56dAAtPKG7XnUxDMf05+nbHUGqLFAd2QH5GRUfFxJ/SutUG4g6Eap89Wda5Jxs+MQxGQTAwCdg4OPurUdEpJY0qBvgXnTRRfz2228VbrdarVit1lqsSKR6+XpbuCAhlBV7HN316lVwMplODOyOhuYDXLflZZyccOLUUJWb5pjpL+sA7P7R9ZjgJo5xU1FtIbr9ycc+Z3EPLJGGZts38PVEx+NeE2HAn91bj4iIh6rzwWnDhg3ExcW5uwyRGtW7eeSJ4HSMsT0Tz/k8R3KKiAqqI/+REBAJARdD04td1+cfO7116sgOyElx3Jwx++DpgSo0wRGkotqe6PJ3oqXK26/2rkfEE+3+Cb66DQw7dLsFhv5TNzYVEamAW4NTbm4uu3efnAln7969bNiwgfDwcBISEpg0aRKHDh3igw8cU2G+/PLLNGvWjA4dOlBYWMj06dP5+eefWbBggbsuQaRW9GoeDsDy389tnJNhGPx1zhY+WZnMxEta8H9D29ZEmbXDPxwSezuWUxUch/QdcGT7KT+3O7oCZiY7lp0/nHKACcKbObr5Rbc9GawiWzkGW4vUd8kr4fObHbOwtR8FI15RaBIRqYRbg9OaNWtcbmj7yCOPADB+/HhmzpxJSkoKycnJzu3FxcU8+uijHDp0CH9/fzp37syPP/5Y7k1xReqTrgmhWL3MZOQWsTMtlzaxQWd1/BuLf+eTlY4/S68v+p3WMUGM7Nq4Jkp1H7+w8gNV3tGTIapsYor07Y77xTgnpfju5P4mC4Q3d7RMxXSEuM4Q1wWC4vSlUuqPlE3w8XWO+x61HARXv6MbVouIVMFjJoeoLWczAEzEk9zy7kqW7MogOsjKlKs7cVm7mDM67tuNh7n/0/UAXNQ0nFX7jmH1MvPV3X3o1CSkJkv2XM5JKba5tlKlb4eirPKP8Y90hKjYE0EqrguENXPMYFXfGAakbnLcGyeyjaNbo8Xb3VVJdcnYDe8NdUzAktAbbp6lqf9FpME6m2yg4CRSR+xOz+GuD9ey50geANdc0IQnR7QnxK/iL7Rr9x/jxndWUlxq5/aLm/HXy9txxwdr+HlHOnEhvnxz38V1Z8xTbTAMx1ipstap1M2OAHEkCQzb6fv7BEFsx1PCVGdHd7+6GDKK82DPYtg5H3YtcLwPZSxWRwtcWXCM7ey4bk26UfdkHoD3hjnGAsZ1gfHfgm8D/Q8UEREUnCql4CR1WWGJjf8s3Mk7S/ZgGBATbOXfV3fmkrbRp+27/2geo99YxrG8Yga3j2Hazd2xmE1kF5Yw6vWl7DmSR4/EMD65sxc+XvWw1aQ6lRRA2jZI3ejo4pS6CdK2Om4i+UcWnxMho8vJQBXTwTNDxvF9sHMB7JoPe5c4bnZZxjvAMfYrY5fjpp2nMUFEy1Na4U78DIisrerlbOWmO0LTsd8dN5i+9Xt9XiLS4Ck4VULBSeqDtfuP8diXm9ib4Wh9ah8XTPtGwbSNDaJ9XDCNQv247f3V7DmSR6fGIXz+p174+5wc0vj7kVxGvb6UnMJSbrwonn+N7lThhBOpWYUs3J7Gwm1prN57jJt6JvDXy9thNlfPeJ+s/BLyS0qJC6ljM9zZSiFjpyNEpWyClI2OFqryuvqZzBDR6kS46HRyEorQxNodV2IrhQMrHUFp53zHeK9ThSZC62HQeqhjNkMvK9jtkLnPcW1loTFlE+Smlv8aQY1OD1OhCRof5m4Fx2HmlZC2BUIS4LYfIKSejXMUETkHCk6VUHCS+qKg2MaLC5J4b+leKvpT3CjElzkT+xIdfPod5BclpXPbzNUYBlzdrTGNw/zw9/HC38eCn4+F9OxCFm5LY+PB04PAlZ3jeOn6Lli9zv1Lv91u8PHK/Tz3QxJFpTbeHteDS9qc3nJWpxiGoxXHJUxtctx7qjxevo5Wm8jWjnFEZT8jWlbfzH75xxzTs+/8wfGz8JTP02RxjHFpPcQRmCJbn3nAyU0/EaQ2ngxVx34vf1/fkJNd/OI6O25g7BvsWG8Ndiz1cayYpyjKhQ9Hw8FVEBjjaGmKaOHuqkREPIKCUyUUnKS+OZxZwKaDWexIzWZ7SjY7UnPYfzSfED9vPv9TL9rGVvx7Pu2X3/n39zsq3A6O79Hd4kMZ3D6WQKuFZ+Zuo8Rm0LdlBNNu7k6Q79mP59mZlsNf/reJdcmZznW+3mY+vqMX3RPDzvp8Hi8n7USY2ujo4pex09EF7tSucacymSGs6YmJGVqfnKAhslXV41EMwzHpxc4TrUoHVznu0VPGLwxaDXG0KrW41PG8uhTlQOqWk8ExdaNj0g17SRUHmsAadDJI+Ya4Bqtyn/9hm5evWrXKU1IIn45xjF/zDYVb5zm6joqICKDgVCkFJ2kIcotKMQEB1srvOGAYBnM3pbD1cDYFxaXkFdsoKLaRV1yK1cvMwDbRXNYumuigky1Wv+48wt0frSW/2EbHxsHMmHDRGU8wUVhi4/VFu5n2y++U2AwCfCw8NrQNi5OO8MvOI4T4efPl3b1pHXN2063XSXYbZO6HIzshI8n1Z0Uz+wEExv4hTLV2tB6kbXO0Ku1aAFkHXI+J6XgiLA2DJj1qt3tgabGjS2BZmErb4mitKsxyjJ0qb5zYufD2d7wPka1PLK1OvDctG+6Njm2l8OV42DHXMWZt/DeOz19ERJwUnCqh4CRy/jYdzOTWGas5mldMYoQ/H9x2EYkRlU9+kJ5dyA3vrHDOCjioXQzPjupAXIgf+cWl3Dx9JeuSM4kJtvLV3X2ID2+g0yMbhqNr35GkEy1TO08+PnWmu8p4+UKzAY4ueK2GQmh8zdZ8PkqLoDD7RJDKcvx0Ps92fV7euqJsoLJ/xkyO6/9joIpsDQFR9beVym6Hr++FjZ86ZkW8+Sto1t/dVYmIeBwFp0ooOIlUj70ZeYx7byUHjhUQG+zLDw/1I9Tfp8L9J368ju82pxAdZOWZkR0Y2iHWZUKKzPxirn9rOTvTcmkWGcCXd/cmMrDmp0rPLixhfXImfVpE4G3x8HE2hVmOLn5HklxbqY7vc0zKUDZWqWm/hnNfHrsdinMg9wgc3XUybJa9T4WZFR/rG1J+oAprWjenlC9jGPD9n2HV245xbGM+graXu7sqERGPpOBUCQUnkeqTnlPIDW+tYE9GHqO6NuLlG7qVu9/ipHQmzFiNxWzi2/supn2j8v/spWYVcs2byziUWUDHxsF8emevcxpDdabSsgu56Z0V/H4kj74tI3hjbPdK74vlsWylju539bX15FwZBuQfdQ1TZY+P76fCliqzF4Q3d4So8GaOWehCy5Z4x3gsT/bzP+DXFwATXP0OdL7O3RWJiHgsBadKKDiJVK8NBzK5+o2l2A14+5buDOkQ67K9sMTGkP/+SvKxfG6/uBl/v7J9pefbcySX66Yt52heMddc0ISXru9SI3UfyizgpndWsP9ovnNdy+hAZky4sOF2E2xISgodswD+MVBl7IKS/MqP9Qs7JUglOmYJPDVYufOGsktfhYV/dzy+4iW48A731SIiUgcoOFVCwUmk+v37+x1M++V3ooKsLHy4v0uXvf8sSOLVn3cTG+zLj48OILCKCSvAcZ+qa6ctxzDg0zt70btFRLXWm3w0nxvfWcGhzALiw/148soO/H3OFlKzC4kI8OGd8T24IKEezu4nVbPbIefwyRB1fL9jEo/MZMekGwXHqz6Hb8jJUBWa8IdglQB+oTVT+9qZ8O2DjseXPQX9HqmZ1xERqUcUnCqh4CRS/QpLbFw59Td2p+e6dNn7/Uguw19eQrHNzptjL2B4p7gzPuff5mzmoxXJNI8K4PsH+53XPaNOtedILje9s5LU7EKaRQbw8R09aRTqR2pWIbe/v5qth7Oxepn575iuXH4G9RqGwZGcIqKCrBXeRFjqkcJsR4DKTIbMAydDVdlScKzqc3gHOGb68/Zz3K/L68RP53Nfx+LtW862U5/7nlyO7XGMa8KAix+GQZNr+p0QEakXFJwqoeAkUjP+2GVvcPsYxk5fybLfjzKwTRQzJlx4VsEiq6CEy176hYzcIh4d3Jr7L2t1XvUZhsHWw9lMmLGajNwiWkYH8skdPV1uDpxXVMoDn67npx3pAIzpEc/lnePo3TwCHy/XiSPSswv5at1BvlxzkL0ZefxleFvuHqCbijZ4RbmnBKtylvyMmn39Hrc7uugpxIuInBEFp0ooOInUnLIue5GBVu6/tCVPfbMVq5eZhQ8PICHi7McNfb3hEA9+tgEfLzMLHupP08jTpzw3DINjecXkFTnuP5VfXEpekY2cwlL2Hc3j9/Rcfj+Sy+9H8sgtKgWgbWwQH93Rs9xZ+2x2g2fnbmPmsn3OdUFWLwa0iWJIh1isXma+XHOARUlHsNlP/vUZaPViyZ8vISyg4pkFRSjOg5xUx/2rSgsdY61KC6vheRG0Ge7oomf28NkhRUQ8iIJTJRScRGrOqV32yjw2pDX3XXpurUWGYXDLu6v4bXcG/VpF8sFtF7m0Wm1PyeavszezLjnzjM5nNkHflpG8ekO3KgPO0t0ZzN2Uwo/b0ziSU1TuPt0TwxjTI54Zy/axPSWbewa24PFhbc/4+kRERMS9FJwqoeAkUrNO7bJXHeOT9mbkMfTlXykutTP1xm6M6NKIgmIbr/y0i+lL9lB6otXH38eCv48XAdYTP30sJIT70yI6kBZRAbSICiQhwv+sa7HbDTYczGThtjQWbksjv6iUK7s04voeTWgZ7ZiWeuG2NO78YA1+3haWPH5Jrdx/SkRERM6fglMlFJxEat6bi3/n3d/2MO3m7vRoGn7e53vlx13898edRAVZefqqDkz5fjsHjhUAMLxjLE+N6EBsiG8VZ6k5hmEw6vWlbDyYxR0XN+NvVUy5LiIiIp5BwakSCk4idU9RqY3hLy9hT0aec12jEF+eGdmRQe1j3FjZSWU3+bV6mfn1z5cQE3zuQc4wDNbuP46vt4WOjd14TyAREZF67myygUaQiojHs3pZ+MeojoBjnNLtFzdj4SMDPCY0AQxoHUX3xDCKSu28sWj3OZ0jK7+Ed3/by2X/+YVrpy1n9BtL+f1IbtUHioiISI1Ti5OI1BkbDmQSaPWiZXSgu0sp17LdGdw0fSU+FjOL/m8gjUP9zui4TQcz+XD5fr7ddJjCErvLtis7x/HaTRfURLkiIiINnlqcRKRe6hof6rGhCaBPy0h6NQ+n2GbntZ8rb3UyDINlv2dw49sruOq1pXy59iCFJXbaxgbxz9Ed+eru3gDM3ZTC1sNZtVG+iIiIVELBSUSkGj06pA0AX645wIFj+adtNwyDxUnpXDdtOTe9s5Lle47ibTExqmsj/ndPb75/sB9jeybSo2k4V3VpBMBLC3bW6jWIiIjI6bzcXYCISH1yYdNw+rWKZMmuDO7+aC1tYoPwMpuwmM14mU1sPJjJpoOOFiQfLzM3XBjPnwa0KLdb38ODW/Pd5hR+3pHOmn3HqmWGQhERETk3Ck4iItXskcGtWbIrg62Hs9l6OPu07X7eFsb2TODO/s0rnX2vWWQA1/dowqerDvD8/CQ+v6uXyw2ARUREpPYoOImIVLNuCWF8cNtF7EzLwWY3KLUbzp/Bvl6M7taYiDO8Se79l7bif+sOsWrvMZbsyqB/66garl5ERETKo+AkIlID+reOqpaQ0yjUj1t6JfLub3t5YX4S/VpFqtVJRETEDTQ5hIiIh7t3YAsCfCxsPpTF/K2pLttKbHa2p2SzIzWbtOxCikptbqpSRESkflOLk4iIh4sItHL7xc149efdvLhgJ7lFNjYfzGTjwSy2pWRTXOp676cAHwthAT60iwtm6o3d8PW2uKlyERGR+kM3wBURqQOyC0vo//wiMvNLTtsW5OuF1cvM8fwSbHbXv9JnTLiQS9pG11aZIiIidcrZZAO1OImI1AHBvt787Yr2PP/DDppGBNC5SQidmoTQpUkoiRH+mEwm7HaDnKJSjucV84/vtvHj9nS2HMpScBIREakGCk4iInXEtd2bcG33JhVuN5tNhPh5E+LnTa/mEY7gdDirFisUERGpvzQ5hIhIPdShUQgAWw6dfh8pEREROXsKTiIi9VD7Ro5+2ocyCzieV+zmakREROo+BScRkXooxM+bxAh/AHXXExERqQYKTiIi9VTHxuquJyIiUl0UnERE6qmOZeOc1OIkIiJy3hScRETqqY6NHeOcthxScBIRETlfCk4iIvVUWYvT/qP5ZBWcfuNcEREROXMKTiIi9VRYgA+NQ/0A2HZY45xERETOh4KTiEg9VtZdb6vGOYmIiJwXBScRkXqsrLveZo1zEhEROS9uDU6//vorI0aMoFGjRphMJubMmVPlMYsXL+aCCy7AarXSsmVLZs6cWeN1iojUVR2blE1JruAkIiJyPtwanPLy8ujSpQuvv/76Ge2/d+9errjiCi655BI2bNjAQw89xB133MH8+fNruFIRkbqprMVpT0YeeUWlbq5GRESk7vJy54sPHz6c4cOHn/H+06ZNo1mzZrz00ksAtGvXjt9++43//ve/DB06tKbKFBGps6KCrMQEW0nLLmJ7SjY9moa7uyQREZE6qU6NcVq+fDmDBg1yWTd06FCWL19e4TFFRUVkZ2e7LCIiDYnGOYmIiJy/OhWcUlNTiYmJcVkXExNDdnY2BQUF5R4zZcoUQkJCnEt8fHxtlCoi4jE6Ni4b56T/OBIRETlXdSo4nYtJkyaRlZXlXA4cOODukkREalVZcNKU5CIiIufOrWOczlZsbCxpaWku69LS0ggODsbPz6/cY6xWK1artTbKExHxSGX3ctqVnkthiQ1fb4ubKxIREal76lSLU+/evfnpp59c1i1cuJDevXu7qSIREc8XG+xLZKAPNrvB9hR11xMRETkXbg1Oubm5bNiwgQ0bNgCO6cY3bNhAcnIy4OhmN27cOOf+d999N3v27OHPf/4zO3bs4I033uCLL77g4Ycfdkf5IiJ1gslkosOJCSK2HFZwEhERORduDU5r1qyhW7dudOvWDYBHHnmEbt268eSTTwKQkpLiDFEAzZo147vvvmPhwoV06dKFl156ienTp2sqchGRKpR119uqmfVERETOiVvHOA0cOBDDMCrcPnPmzHKPWb9+fQ1WJSJS/3R0tjgpOImIiJyLOjXGSUREzk3ZzHpJqTkUldrcXI2IiEjdo+AkItIANAnzI8TPmxKbwa60XHeXIyIiUucoOImINAAmk8k5zmnDgUz3FiMiIlIHKTiJiDQQZd31nvpmKw98up7NBz17vFNlY2BFRERqW526Aa6IiJy7cb2bsuVQFkt3H+WbjYf5ZuNhejUP567+zbm4ZRQlNjtFpXaKSm0Uldjx87EQE+xb63Xa7QZv/bqHab/8zmXtovnLsLZEu6EOERGRU5mMBvZfetnZ2YSEhJCVlUVwcLC7yxERqXVbDmUxfcke5m5KodRe8T8BJhNMH9eDy9rF1FptGblFPPLFRn7decS5LtDqxQOXtWRCn2b4eKmjhIiIVJ+zyQYKTiIiDdThzAJmLtvHpyuTySkqda73sZgxm6GwxE50kJWFjwwgxM+7xutZ9nsGD362gSM5Rfh6m7n/0lYs2JrKxhNdCltEBTD5qg70axVVo3UUlthYufcYv+48QlGpjfAAKxEBPoQH+BAR4ENsiC9NIwIwm001XkdBsY2wAJ8afR0RkYZMwakSCk4iIq6KS+0UFNuweptPhCYThSU2Ln9lCXsy8ri+RxOev7ZLucfa7QbPz08iJauAsT0TubBpGCbT2QUKm93glZ92MfXnXRgGtIoO5PWxF9A6Jgi73eCrtQd57ocdHM0rBuCiZuFc3DKSns3C6RIfiq+35bzfg/TsQhYlpfPT9nR+251BfnHlU7YHWb3o1CSEzk1C6Rrv+BkX4nvW116RrIISRr72G6nZhXx8R0+6J4ZXun9RqQ1vs7nGw5yISH2j4FQJBScRkTOzZt8xrntrOYYBH9x2Ef1bu7b0GIbBk19v5cMV+53rOjcJ4faLm3F5pzi8LVV3qysutfOnD9ewKMnRNW9Mj3gmX9UBPx/XMJRVUMIrP+7i/eX7sJ3SvdDHy0zX+FAubhnJ7Rc3I8B6dkN3j+YW8a95O5i1/iCn/msYE2zl0rbRRAVaycgr5lhuMcfyijmaV8ShzAIKS+ynnSvM35t2ccG0jQ2mXVwQ7eKCaRMbdEbvwx89/PkGZq8/BEBUkJW5919c4XiztfuPc+cHa0gI9+fjO3qe9XsgItKQKThVQsFJROTMTf5mKzOX7aNxqB/zH+5P4Clfyv+zcCev/rQLkwmGto9lUVI6RaWOQBEX4suEPk0Z36dphS1CdrvBQ59v4JuNh/HztvDvazoxsmvjSutJPprPLzvTWbH3GCv3HCMjt8i5rVPjEN6d0IPooKonkrDbDb5Yc4Ap3+8gq6AEgC7xoVzWNppL20bToVFwha1HpTY7O9Ny2Xgwk00HM9lwIIudaTkuga5Ms8gAPrqjJ41D/aqsqczcTYe575P1mE3QOMyPA8cK6JYQymd39cLq5fpebjmUxY3vrCCn0NHVcnjHWN4Ye0G1tXyJiNR3Ck6VUHASETlz+cWlDH35Vw4cK+CWXok8O6ojADOW7uXpb7cB8OyojtzSK5GjuUV8vDKZD5bvIyPX0a2uY+Ngpt3cnSZh/i7nNQyDp7/dxsxl+/Aym3h3woUMaH12Y5cMw2BvRh7L9xzlpQU7OZZXTONQP2beeiGtYoIqPG5HajZ/m72FNfuPA9A2Noh/ju5E98Sws3r9UxWW2NiVlsv21Gy2p2SzIyWHLYeyyCkqpVlkAJ//qdcZBbrUrEKGvvwrWQUl3H9pS665oAlXvfYb2YWl3HhRAlOu7uTcd2daDmPeWs7x/BLaxwWzKz2HEpvBY0Nac9+lrc649jX7jvG3OVsI8vXi87t6q7ufiDQoCk6VUHASETk7S3dnMHb6SgA+u6sXhzMLeOSLjQA8Org191/m+iW9sMTGNxsO8+8fdnAsr5gwf2+m3ngBF7eKdO7z2s+7eHHBTgBeuaFrlS1NVdmXkcetM1ezNyOPIF8v3rqlO31anHw9wzBYl5zJrHUH+Xz1AUrtBv4+Fh4Z3JoJfZridQ7d6apyOLOA66Yt51BmAW1igvjsrl6VTvRgGAbj3lvFkl0ZdGocwqx7++BtMbM4KZ1bZ67GMOBfoztxU88E9mbkcf1byzmSU0TnJiF8fEdPvtuUwl9mbT7j2RALim28MD+JGcv2Orsp/vJ/A0mMCKjOt0FExKMpOFVCwUlE5OxNmrWZT1clEx1k5WheMTa7wW19m/H3K9tV2C3sUGYB93y0lk0HszCb4M/D2vKn/s35dNUBnpi9GYCnRrTn1r7NqqXGY3nF3PXBGtbsP463xcTz13amfVwIX284xDcbD3PweIFz36EdYnhqRAcanUUXunOx/2ge101bTnpOEZ0ah/DxnT0J9i1/hsIPlu/jya+3YvUy890DF9My+mSr2euLdvPC/CS8LSZeur4r/563ncNZhbSNdQSyUH9HIPvbnM18tCKZIKsXc+7rS4uowHJfa+Weozz+v03sO5oPOMaKFZfamT6uB4Pa19708yIi7qbgVAkFJxGRs5ddWMLQ//5KSlYhAFdf0JgXr+1SZbeuwhIbT369hS/WHAQcM+Kt2XcMuwH3XdKSx4a2qdY6C0tsPPrFRr7bnHLaNn8fC0M7xHJt9yb0bRlZztE1Y1daDmPeXsGxvGJ6JIbxwe0X4e/jOoHD7vRcrpy6hMISO5NHtGfCH8KkYRhM/GQd8zanOtc1jwrgiz/1JjLQ6lxXXGpn7PQVrN53nOZRAcyZ2NcZ1DLzi9mWks33m1OdE3rEBvsy5ZpOzFl/iK83HObPw9pw78CWNfVWiIh4HAWnSig4iYicmyW7jnD3h2u5pG00/x3T9YxnizMMg09XHeCpb7ZQYnP8k3PjRfH8a3SnGpnEwG43eG7+Dt76ZQ/eFhMD20QzsmsjLmsbc9psfbVly6EsbnpnBdmFpVzULJxL2kRjNwwMw8BuwLzNKexIzaFfq0jev/WicgNpXlEpo99Yys60XOLD/fjyT32IDTl93NSRnCJGTHVMZX5BQijhAT5sO5zN4ROht8wNF8bzxBXtCPb1drZoje7WmP+O6VpTb4OIiMdRcKqEgpOIyLkrKrWdNrPbmVqXfJy/zt5C1/gQ/jGqE5YanoRgV1oO0UG+hPjX/M17z8S65OPcPH1lhfeICvHzZv5D/csNQ2XSsgv537qDjO7WmLiQirsZbjyQyXVvLae41HXa9PhwP9rHBXNzr0SXGwkv2JrKXR+upWPjYObe3+8sr0xEpO5ScKqEgpOIiLjLpoOZfLoqmVKbgdlkwmwGk8mExWRiZNdG9Gha+Y1uz8bipHR+3J5Gq2jHPaXaxgVVOL5qb0Yel7y4GF9vM9ueHqaZ9USkwTibbKC75ImIiNSSzk1C6dwktFZea2CbaAa2iT6jfRPC/fHxMlNYYufA8XzNrCciUo7qn39VRERE6hSL2eScgW9XWq6bqxER8UwKTiIiIkLrGEdw2pme4+ZKREQ8k4KTiIiI0DrGcd8otTiJiJRPwUlERERoGX2iq55anEREyqXgJCIiIs4Wp93pudjtDWrCXRGRM6LgJCIiIqfNrCciIq4UnEREREQz64mIVEHBSURERADNrCciUhkFJxEREQGgVbRanEREKqLgJCIiIgC0KpuSXC1OIiKnUXASERERQDPriYhURsFJREREAM2sJyJSGQUnERERATSznohIZRScRERExKlsggjNrCci4krBSURERJzKpiTfrRYnEREXCk4iIiLiVDaznlqcRERcKTiJiIiIU1lXPc2sJyLiSsFJREREnBIjApwz6x08XuDuckREPIaCk4iIiDidOrPezjR11xMRKaPgJCIiIi40s56IyOkUnERERMSFZtYTETmdgpOIiIi4aBmtmfVERP5IwUlERERcOFucNLOeiIiTgpOIiIi4SAj318x6IiJ/4BHB6fXXX6dp06b4+vrSs2dPVq1aVeG+M2fOxGQyuSy+vr61WK2IiEj95mUx0zwyANDMeiIiZdwenD7//HMeeeQRnnrqKdatW0eXLl0YOnQo6enpFR4THBxMSkqKc9m/f38tViwiIlL/tW8UDMBPOyr+91hEpCFxe3D6z3/+w5133smtt95K+/btmTZtGv7+/rz33nsVHmMymYiNjXUuMTExtVixiIhI/Xd9j3gAZq8/yPG8YjdXIyLifm4NTsXFxaxdu5ZBgwY515nNZgYNGsTy5csrPC43N5fExETi4+MZOXIkW7durXDfoqIisrOzXRYRERGpXM9m4bSPC6awxM4nq5LdXY6IiNu5NThlZGRgs9lOazGKiYkhNTW13GPatGnDe++9x9dff81HH32E3W6nT58+HDx4sNz9p0yZQkhIiHOJj4+v9usQERGpb0wmE7df3AyAD5bvo7jU7uaKRETcy+1d9c5W7969GTduHF27dmXAgAHMmjWLqKgo3nrrrXL3nzRpEllZWc7lwIEDtVyxiIhI3XRllziigqykZRfx/ZYUd5cjIuJWbg1OkZGRWCwW0tLSXNanpaURGxt7Rufw9vamW7du7N69u9ztVquV4OBgl0VERESqZvWycEuvRADe/W0vhqF7OolIw+XW4OTj40P37t356aefnOvsdjs//fQTvXv3PqNz2Gw2Nm/eTFxcXE2VKSIi0mCN7ZmAj5eZTQezWLv/uLvLERFxG7d31XvkkUd45513eP/999m+fTv33HMPeXl53HrrrQCMGzeOSZMmOfd/5plnWLBgAXv27GHdunXcfPPN7N+/nzvuuMNdlyAiIlJvRQRaubpbY8DR6iQi0lB5ubuAMWPGcOTIEZ588klSU1Pp2rUrP/zwg3PCiOTkZMzmk/nu+PHj3HnnnaSmphIWFkb37t1ZtmwZ7du3d9cliIiI1Gu39m3GZ6sPMH9rKgeO5RMf7u/ukkREap3JaGAdlrOzswkJCSErK0vjnURERM7QLe+uZMmuDO64uBl/u1L/WVkXFJXayCuyER7g4+5SRDzW2WQDt7c4iYiIiOe77eJmLNmVweerD/DQ4NYEWmv/K4RhGKTnFBHi542vt6Vazrn1cBav/bybX3YeoWezcB4c1Jqu8aHVcu5zdSyvmO+3pDB3YwrFNjuPDm5Nn5aRlR6TnlPIhuRMklJz2JGWw87UHPZk5GGzG3RsHMzlneK4olMciREBtXQVIvWPWpxERESkSna7waD//sKeI3lc0iaKAKsXR3KKOJJT5Awzdw9ozg0XJeBtKX8I9f6jeXy8MplSm8EFiaFckBBGo1C/cvfNKihh/9E8dqTksD01m+0p2exIzSEzv4Qgqxf/HdOVQe1jyj0WHCFrzf7jlNoM2sQGndbqsvFAJlN/3sWP29NPO7Z/6ygevKwV3RPDKj1/Rm4xe47ksjcjj+Rj+QT5etM4zI/GoX40CfMjKtCK2Wyq8Bynniszv4Qft6fx7aYUlu7OwGZ3/Xo2vncijw9vi7+Pa2A9nlfM1J938+GKfZTYqv5K16GRI0SF+nuTmV9CVkEJmfnFZOaXYDJB41B/Goc56m8c6kdEoA+HMwvYm5HPvow89h7N4+CxfIZ0iGXiJS2rfD0RT3c22UDBSURERM7IRyv287c5WyrdJzHCn0eHtOHKTnHO0LDtcDZv/vI73206zB/yAHEhvlyQEEZihD+HMgvYfzSf/UfzOJ5fUunrmEzw2JA23DuwBSaTazg5nFnAE7M3szjpiHNdZKCVNrGBtI4JYs+RPH7ZecR5nis7N+L6Hk34ZsNhZq0/5AwtF7eMZGCbKLILS8kuKHEshSUcySliT0YeOYWlldboYzETFWQlxM+bED9vQv29na1lR/OKScsu5EhOEWnZheQX21yO7dQ4hCs7x5F8LJ+PVyYD0DTCn5eu70L3xHAKim28t3Qv0xb/Tk6Ro442MUF0aBRMm9ggWscG0TY2CB+Lmflb05i3OYXle46eFsjOlckEPz0ygOZRgdVyPhF3UXCqhIKTiIjIuSmx2Xlj0e/kl5QSFWglOtiXqEArUUFWlv2ewas/7SIjtxhwtGyM79OUH7ak8vOOk606A9tEkRjuz7rkTLalZFf6RT4y0IfWMUG0iwumbazjZ9PIAP79/XY+WuEIEyO6NOL5azrj52PBMAy+WHOAf8zdTk5RKT5eZqKDrBw8XnDauS1mEyO7NmLiJS1pccqX/+Sj+by+aDf/W3eQ0ipChskETcL8aBYZSGK4P7lFpRw6XsChzAJSswvPOqS0iQliRJc4ruzciKaRJ7vU/brzCI//bxMpWYWYTHB1tyYs3Z1BanYhAO3igpk0vC39W0dVev6juUUs2JbGohOfR6i/N6H+Ps5QV2ozOJRZwMHj+Rw6XsDB4wUcyy8mLtiXppEBNI0MoFlEAD9uT2Pl3mNc170JL1zX5ayuUcTTKDhVQsFJRESkZuQVlfLub3t5+9c95BadbI0xm+DyTnHcM7AFHRqFONfnF5ey8UAW65KPk5JVQJMwfxLD/UmMCCAhwr/ScVQfr9zPU19vpfTEGJ7JIzrwyk+7WLIrA4Cu8aG8eF1nWkYHkVdUyq70XHam5ZCUmoOX2cTYnokkRFQ8O+CBY/nMXLaPIzlFBPt5EeLnTbCvo8UoLMCHZpEBJIT7VzjWqtRmJzW7kIzcYjLzi8kqKOsWV0J+sY3IQB9ign1PLFaig3zx86l43FZWQQnPfLuN/6076FzXONSPx4a2ZmSXxmfUJfBc2O3Gaedel3ycq99YhpfZxC9/voTGFXS3FKkLFJwqoeAkIiJSs47lFfP6ot0s2pFOz+bh/Kl/C5cWlOqycs9R7vl4Hcfyip3rrF5mHh3Smtsvbo6lhsKEOy3clsaMpXu5pE00t/ROrLZJMs7WTe+sYNnvRxnfO5GnR3Z0Sw0i1UHBqRIKTiIiIvXHgWP53PnBGnak5tAjMYznr+2scTe1YNnuDG6avhKrl5nfHr+UqCCru0sSOSeajlxEREQahPhwf+ZM7MuO1Bw6NQ6pl61Mnqh3iwi6JYSyPjmT6b/tYdLwdu4uSaTGlT9fqIiIiEgd4ettoWt8qEJTLTKZTNx3Yjryj5bvJzO/uIojROo+BScREREROWuXto2mbWwQecU2Zi7b5+5yRGqcgpOIiIiInDWTyeS8Ce6MpftcZlIUqY8UnERERETknFzeKY7mkQFkFZTw8Yr97i5HpEYpOImIiIjIObGYTdw9sAUA7yzZS2GJzc0VidQcBScREREROWejuzWmcagfGblFfLwy2d3liNQYBScREREROWfeFjP3XeoY6/TGot3kaayT1FMKTiIiIiJyXq7t3oTECH+O5hVrhj2ptxScREREROS8eFvMPDyoNQBv/fI7WQUlbq5IpPopOImIiIjIeRvRpRGtYwLJLixl+pI97i5HpNopOImIiIjIebOYTTwyuA0A7/22l6O5RW6uSKR6KTiJiIiISLUY2iGGzk1CyCu28ebi391djki1UnASERERkWphMpl4dIij1emDFftJzSp0c0Ui1UfBSURERESqTf9WkVzUNJziUjtTf97l7nJEqo2XuwsQERERkfrD0erUmjFvr+Dz1Qe4qksj2sQGEeLnjclkcu5nsxvsP5rHjtQcdqTmcOh4AVZvM37eFsfiY8HX20KLqAAuSAwj2Ne7ytc2DIODxwvYmeY45660HLILS7HZDeyGgc3uWPx9LHRLCKNH0zC6xofi71P1V+ISm53sghKyC0vJLy4lwMeLIF8vAn29sHpZzui9sdsNMvKKSM8uoqjURlGpneKyxWbHYjLh623B6m3G19uCr5eFIF8vYkN88bacW3tHcakdH69zOzYjt4jNh7KICPAhIdz/tM+wjGEY5BaVUmIzCA/wqfSchzML+GFLKlsPZ/PS9V3OqS53UXASERERkWrVs3kE/VpFsmRXBmPeXgGAr7eZuBA/YoN9ySsuZWdaDoUl9jM6n8kEbWKC6J7oCDuNQvxIyykiLauQ1GzHcuh4AbvTc8k9wxvwLko6AoCX2USHxiFckBCK2WQiq6DEuWSfWLIKSsgrtlV4Lh+LmUBfLwKtJxZfL4JO/LTZDVKzCknJKiQtu5BSu3FG9Z3KYjbRONSPhHB/4sP9SQj3JzzAmyBfbwKtjgAX5OtFVoHjfU1KdSw703I4mldMkK8XcSG+xIb4ERfsS2yI74nnvo7PJMSXYF8vikrtrNp7jN92Z7BkVwbbU7Jd6gjy9XLUEOaPyQRHcopIzyniSE4RBSWO9ych3J+LmoVzUbNwejYLJyHcn31H8/lhSyo/bElh48Es5/nuvaQFLaICz/r9cBeTYRhn/+nVYdnZ2YSEhJCVlUVwcLC7yxERERGpl3an5/DYl5tIPpbPsbzicvfx9TbTJiaINrFBJEYEYLMbFJTYKCh2LDlFJWw5lE3ysfwzfl1vi4kWUYG0iQ2idUwQUYFWzGYTZpMjgJhNJo7lFbNm/3FW7z1GavbZjcMK8LHg5+NFQXFppWGqIiYTRARY8fex4ONlxsdidvz0MmMYBoUldgpLbBSW2igssZNVUEJx6ZkFzPPh72Oh1G6c9lotowPJKSwhLfvcZkkM8fN2ua+XyQQ9EsMY1jGO0d0aV9lCVdPOJhsoOImIiIhIjSossZGW7Wh1Sc0qxOplpm1cMAnh/ljMp3f9+qP07ELW7j/OmhNLVn4x0cG+xJ5oPYkJdrSgtIwOpFlkwBl3azMMg0OZBazZd5zNh7LwspgI9vUmxO/kEnzqY18vvE45t81ukFdcSm5hKblFpeSc+Ol4XkJukSNYxZ7SyhMVZD2rbnd2u8GR3CL2H80n+Vg+yUfzOHC8gKyCEnILS8kuLHG+tp+3hdaxQbSJCaT1iUDaJMyfY3nFpGQVON9/x88Tz7MLycw/GWziQny5uGUkF7eKpG/LSCIDrc7P8ODxshocQTY62HE90UFWooKs2OwGa/cfZ9XeY6zae4yNBzMpsRlYzCb6tIhgaIdYhnSIITrI94yvv6YpOFVCwUlERERE5KSCYhup2YWYgMQI/3LHMZ2LwhIbSak5JEb4E+rv3palipxNNtAYJxERERGRBszPx0KzyIBqP6+vt4Uu8aHVfl530XTkIiIiIiIiVVBwEhERERERqYKCk4iIiIiISBUUnERERERERKqg4CQiIiIiIlIFBScREREREZEqKDiJiIiIiIhUQcFJRERERESkCgpOIiIiIiIiVVBwEhERERERqYKCk4iIiIiISBUUnERERERERKqg4CQiIiIiIlIFBScREREREZEqeLm7gNpmGAYA2dnZbq5ERERERETcqSwTlGWEyjS44JSTkwNAfHy8mysRERERERFPkJOTQ0hISKX7mIwziVf1iN1u5/DhwwQFBWEymWrtdbOzs4mPj+fAgQMEBwfX2uvKudHnVXfos6pb9HnVHfqs6hZ9XnWLPi/PYRgGOTk5NGrUCLO58lFMDa7FyWw206RJE7e9fnBwsP6A1CH6vOoOfVZ1iz6vukOfVd2iz6tu0eflGapqaSqjySFERERERESqoOAkIiIiIiJSBQWnWmK1WnnqqaewWq3uLkXOgD6vukOfVd2iz6vu0GdVt+jzqlv0edVNDW5yCBERERERkbOlFicREREREZEqKDiJiIiIiIhUQcFJRERERESkCgpOIiIiIiIiVVBwqiWvv/46TZs2xdfXl549e7Jq1Sp3l9TgTJ48GZPJ5LK0bdvWub2wsJCJEycSERFBYGAg11xzDWlpaS7nSE5O5oorrsDf35/o6Gj+7//+j9LS0tq+lHrn119/ZcSIETRq1AiTycScOXNcthuGwZNPPklcXBx+fn4MGjSIXbt2uexz7Ngxxo4dS3BwMKGhodx+++3k5ua67LNp0yb69euHr68v8fHxPP/88zV9afVSVZ/XhAkTTvuzNmzYMJd99HnVjilTpnDhhRcSFBREdHQ0o0aNIikpyWWf6vq7b/HixVxwwQVYrVZatmzJzJkza/ry6pUz+awGDhx42p+tu+++22UffVa1480336Rz587OG9j27t2b77//3rldf67qKUNq3GeffWb4+PgY7733nrF161bjzjvvNEJDQ420tDR3l9agPPXUU0aHDh2MlJQU53LkyBHn9rvvvtuIj483fvrpJ2PNmjVGr169jD59+ji3l5aWGh07djQGDRpkrF+/3pg3b54RGRlpTJo0yR2XU6/MmzfP+Otf/2rMmjXLAIzZs2e7bP/3v/9thISEGHPmzDE2btxoXHXVVUazZs2MgoIC5z7Dhg0zunTpYqxYscJYsmSJ0bJlS+PGG290bs/KyjJiYmKMsWPHGlu2bDE+/fRTw8/Pz3jrrbdq6zLrjao+r/HjxxvDhg1z+bN27Ngxl330edWOoUOHGjNmzDC2bNlibNiwwbj88suNhIQEIzc317lPdfzdt2fPHsPf39945JFHjG3bthlTp041LBaL8cMPP9Tq9dZlZ/JZDRgwwLjzzjtd/mxlZWU5t+uzqj3ffPON8d133xk7d+40kpKSjCeeeMLw9vY2tmzZYhiG/lzVVwpOteCiiy4yJk6c6Hxus9mMRo0aGVOmTHFjVQ3PU089ZXTp0qXcbZmZmYa3t7fx5ZdfOtdt377dAIzly5cbhuH4smg2m43U1FTnPm+++aYRHBxsFBUV1WjtDckfv4jb7XYjNjbWeOGFF5zrMjMzDavVanz66aeGYRjGtm3bDMBYvXq1c5/vv//eMJlMxqFDhwzDMIw33njDCAsLc/msHn/8caNNmzY1fEX1W0XBaeTIkRUeo8/LfdLT0w3A+OWXXwzDqL6/+/785z8bHTp0cHmtMWPGGEOHDq3pS6q3/vhZGYYjOD344IMVHqPPyr3CwsKM6dOn689VPaauejWsuLiYtWvXMmjQIOc6s9nMoEGDWL58uRsra5h27dpFo0aNaN68OWPHjiU5ORmAtWvXUlJS4vI5tW3bloSEBOfntHz5cjp16kRMTIxzn6FDh5Kdnc3WrVtr90IakL1795Kamury2YSEhNCzZ0+XzyY0NJQePXo49xk0aBBms5mVK1c69+nfvz8+Pj7OfYYOHUpSUhLHjx+vpatpOBYvXkx0dDRt2rThnnvu4ejRo85t+rzcJysrC4Dw8HCg+v7uW758ucs5yvbRv3Pn7o+fVZmPP/6YyMhIOnbsyKRJk8jPz3du02flHjabjc8++4y8vDx69+6tP1f1mJe7C6jvMjIysNlsLn8wAGJiYtixY4ebqmqYevbsycyZM2nTpg0pKSk8/fTT9OvXjy1btpCamoqPjw+hoaEux8TExJCamgpAampquZ9j2TapGWXvbXnv/amfTXR0tMt2Ly8vwsPDXfZp1qzZaeco2xYWFlYj9TdEw4YN4+qrr6ZZs2b8/vvvPPHEEwwfPpzly5djsVj0ebmJ3W7noYceom/fvnTs2BGg2v7uq2if7OxsCgoK8PPzq4lLqrfK+6wAbrrpJhITE2nUqBGbNm3i8ccfJykpiVmzZgH6rGrb5s2b6d27N4WFhQQGBjJ79mzat2/Phg0b9OeqnlJwkgZj+PDhzsedO3emZ8+eJCYm8sUXX+gvH5FqdMMNNzgfd+rUic6dO9OiRQsWL17MZZdd5sbKGraJEyeyZcsWfvvtN3eXIlWo6LO66667nI87depE3P+3d/+hVdV/HMdfd7Z7p1zXjA3vZbXZdbbaH5Z3pl0SI2aaEJrMmhGlSYuUYaAZGqzYKkwho6IkJJyguFLRiYio+2FmTV3sTk0ZblyTcGhMVrMtN9v7+4d5vt+L2s0v7l43nw+4f9z7+ZxzP+e++Zx73/dz7vv6/SooKFBra6tGjx4d72He8XJzcxUOh/Xbb79py5Ytmjt3rvbv35/oYaEfcaleP0tPT9eQIUOuqaRy7tw5+Xy+BI0KkpSWlqYHHnhALS0t8vl86unpUUdHR1Sf/42Tz+e7bhyvtqF/XH1t/2kO+Xw+nT9/Pqr98uXLunDhAvG7DQQCAaWnp6ulpUUS8UqEkpIS7dy5U7W1tbr33nudx2/Vue9GfVJTU/li6ibdKFbXM3HiREmKmlvEKn7cbrdycnKUn5+vFStW6OGHH9Ynn3zCvBrESJz6mdvtVn5+vqqrq53H+vr6VF1drVAolMCR4eLFi2ptbZXf71d+fr6Sk5Oj4tTc3KwzZ844cQqFQjp27FjUB769e/cqNTVVeXl5cR//neL++++Xz+eLis3vv/+uQ4cORcWmo6NDP/74o9OnpqZGfX19zgeLUCikb7/9Vr29vU6fvXv3Kjc3l8u++tkvv/yi9vZ2+f1+ScQrnsxMJSUl2rZtm2pqaq65/PFWnftCoVDUPq724X3u34sVq+sJh8OSFDW3iFXi9PX16dKlS8yrwSzR1SnuBJWVlebxeKyiosJOnDhhr732mqWlpUVVUkH/W7JkidXV1VkkErGDBw/alClTLD093c6fP29mV0qHZmVlWU1NjTU0NFgoFLJQKORsf7V06NSpUy0cDtvu3bstIyODcuS3QGdnpzU2NlpjY6NJstWrV1tjY6P9/PPPZnalHHlaWppVVVXZ0aNHbebMmdctRz5u3Dg7dOiQfffddzZmzJio8tYdHR02cuRIe+mll+z48eNWWVlpw4YNo7z1/+Gf4tXZ2Wlvvvmm/fDDDxaJRGzfvn0WDAZtzJgx9ueffzr7IF7xsWDBArv77rutrq4uqoR1V1eX0+dWnPuulk1eunSpnTx50j7//HPKJt+kWLFqaWmx8vJya2hosEgkYlVVVRYIBGzy5MnOPohV/Cxbtsz2799vkUjEjh49asuWLTOXy2V79uwxM+bVYEXiFCefffaZZWVlmdvttgkTJlh9fX2ih3THKSoqMr/fb2632zIzM62oqMhaWlqc9u7ublu4cKGNGDHChg0bZrNmzbK2traofZw+fdqmT59uQ4cOtfT0dFuyZIn19vbG+1AGndraWpN0zW3u3LlmdqUkeWlpqY0cOdI8Ho8VFBRYc3Nz1D7a29vthRdeMK/Xa6mpqfbKK69YZ2dnVJ+mpiabNGmSeTwey8zMtA8//DBehzio/FO8urq6bOrUqZaRkWHJycmWnZ1txcXF13xRRLzi43pxkmTr1q1z+tyqc19tba098sgj5na7LRAIRD0HYosVqzNnztjkyZPtnnvuMY/HYzk5ObZ06dKo/3EyI1bxMn/+fMvOzja3220ZGRlWUFDgJE1mzKvBymVmFr/1LQAAAAAYePiNEwAAAADEQOIEAAAAADGQOAEAAABADCROAAAAABADiRMAAAAAxEDiBAAAAAAxkDgBAAAAQAwkTgAAAAAQA4kTAAAAAMRA4gQAGHB+/fVXLViwQFlZWfJ4PPL5fJo2bZoOHjwoSXK5XNq+fXtiBwkAGFTuSvQAAAC4WYWFherp6dH69esVCAR07tw5VVdXq729PdFDAwAMUqw4AQAGlI6ODh04cEArV67Uk08+qezsbE2YMEHLly/XjBkzNGrUKEnSrFmz5HK5nPuSVFVVpWAwqJSUFAUCAZWVleny5ctOu8vl0po1azR9+nQNHTpUgUBAW7Zscdp7enpUUlIiv9+vlJQUZWdna8WKFfE6dABAApE4AQAGFK/XK6/Xq+3bt+vSpUvXtB85ckSStG7dOrW1tTn3Dxw4oJdffllvvPGGTpw4oS+//FIVFRX64IMPorYvLS1VYWGhmpqa9OKLL2rOnDk6efKkJOnTTz/Vjh079M0336i5uVkbN26MSswAAIOXy8ws0YMAAOBmbN26VcXFxeru7lYwGNQTTzyhOXPmaOzYsZKurBxt27ZNzz77rLPNlClTVFBQoOXLlzuPbdiwQW+99ZbOnj3rbPf6669rzZo1Tp/HHntMwWBQX3zxhRYtWqSffvpJ+/btk8vlis/BAgBuC6w4AQAGnMLCQp09e1Y7duzQ008/rbq6OgWDQVVUVNxwm6amJpWXlzsrVl6vV8XFxWpra1NXV5fTLxQKRW0XCoWcFad58+YpHA4rNzdXixYt0p49e/rl+AAAtx8SJwDAgJSSkqKnnnpKpaWl+v777zVv3jy9++67N+x/8eJFlZWVKRwOO7djx47p1KlTSklJ+VfPGQwGFYlE9N5776m7u1vPP/+8Zs+efasOCQBwGyNxAgAMCnl5efrjjz8kScnJyfrrr7+i2oPBoJqbm5WTk3PNLSnpv2+H9fX1UdvV19froYcecu6npqaqqKhIa9eu1ddff62tW7fqwoUL/XhkAIDbAeXIAQADSnt7u5577jnNnz9fY8eO1fDhw9XQ0KBVq1Zp5syZkqRRo0apurpajz/+uDwej0aMGKF33nlHzzzzjLKysjR79mwlJSWpqalJx48f1/vvv+/sf/PmzRo/frwmTZqkjRs36vDhw/rqq68kSatXr5bf79e4ceOUlJSkzZs3y+fzKS0tLREvBQAgjkicAAADitfr1cSJE/Xxxx+rtbVVvb29uu+++1RcXKy3335bkvTRRx9p8eLFWrt2rTIzM3X69GlNmzZNO3fuVHl5uVauXKnk5GQ9+OCDevXVV6P2X1ZWpsrKSi1cuFB+v1+bNm1SXl6eJGn48OFatWqVTp06pSFDhujRRx/Vrl27olasAACDE1X1AAD42/Wq8QEAIPEbJwAAAACIicQJAAAAAGLgN04AAPyNq9cBADfCihMAAAAAxEDiBAAAAAAxkDgBAAAAQAwkTgAAAAAQA4kTAAAAAMRA4gQAAAAAMZA4AQAAAEAMJE4AAAAAEMN/AIUzj5Rx9csbAAAAAElFTkSuQmCC", "text/plain": [ "

" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "The training loss at the step closest to the lowest evaluation loss (1.3087536096572876) is 0.851 at step 1900\n" ] } ], "source": [ "import matplotlib.pyplot as plt\n", "\n", "# Assuming 'trainer' is your trained Hugging Face Trainer object\n", "# Extract the training and evaluation loss from the Trainer's state\n", "training_loss = trainer.state.log_history\n", "\n", "# Initialize lists to store the steps, training loss, and evaluation loss\n", "steps = []\n", "l_steps = []\n", "train_loss = []\n", "eval_loss = []\n", "\n", "\n", "# Extract the relevant data\n", "for entry in training_loss:\n", " if 'loss' in entry:\n", " steps.append(entry['step'])\n", " train_loss.append(entry['loss'])\n", " if 'eval_loss' in entry:\n", " eval_loss.append(entry['eval_loss'])\n", " l_steps.append(entry['step'])\n", "\n", "# Plot the training and evaluation loss\n", "plt.figure(figsize=(10, 5))\n", "plt.plot(steps, train_loss, label='Training Loss')\n", "if eval_loss:\n", " plt.plot(l_steps, eval_loss, label='Evaluation Loss')\n", "\n", "# Adding labels and title\n", "plt.xlabel('Steps')\n", "plt.ylabel('Loss')\n", "plt.title('Training and Evaluation Loss')\n", "plt.legend()\n", "\n", "# Save the plot as an SVG file\n", "plt.savefig('training_evaluation_loss.svg')\n", "\n", "# Show the plot\n", "plt.show()\n", "\n", "\n", "# Identify the training loss checkpoint closest to the lowest eval loss\n", "min_eval_loss = min(eval_loss)\n", "min_eval_loss_step = l_steps[eval_loss.index(min_eval_loss)]\n", "\n", "# Find the closest training loss step\n", "closest_train_loss_step = min(steps, key=lambda x: abs(x - min_eval_loss_step))\n", "closest_train_loss = train_loss[steps.index(closest_train_loss_step)]\n", "\n", "print(f'The training loss at the step closest to the lowest evaluation loss ({min_eval_loss}) is {closest_train_loss} at step {closest_train_loss_step}')" ] }, { "cell_type": "markdown", "id": "ba9d4035-e79d-4502-9806-5e704dada48b", "metadata": { "id": "ba9d4035-e79d-4502-9806-5e704dada48b" }, "source": [ "### Load and test the fine-tuned model" ] }, { "cell_type": "code", "execution_count": 16, "id": "2ceca2f2-42b5-4f95-9c90-44f13d041185", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 84, "referenced_widgets": [ "c386ef03c3db48189d857e4a86125b05", "a4c4d886c5d540a6a6d683fa2c576240", "4000cf23d2a444dcb84a8254b947502c", "c07a3936cd08420f8583c34b73072028", "c26fe456208f454aba0ce3872eb74751", "45ea9cec348d450b88b9fdc3acc5541e", "0814ceb877c94bcc9fce074adc49cea8", "0ca3ec39ec5046768b77786c45d4a1ad", "708beaf047f546c68ba58b73cfcda51c", "64272d0e01fb46d38ef21d22a6d49b3d", "75de7b33bed94aef9ce3fca10b1ff09e" ] }, "id": "2ceca2f2-42b5-4f95-9c90-44f13d041185", "outputId": "82c410f0-e712-404c-cdec-9d80c9c68949", "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Unused kwargs: ['use_nested_quant']. These kwargs are not used in .\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "lamm-mit/BioinspiredLlama-3-1-8B-128k protein_secondary_structure_predictor/checkpoint-1980\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "c386ef03c3db48189d857e4a86125b05", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Loading checkpoint shards: 0%| | 0/4 [00:00', max_new_tokens=16, temperature=0.1, messages=[], system_prompt='')\n", "\n", "print (f\"Answer LLM: {answer[0]}\\nCorrect answer: {corr_answer}\")" ] }, { "cell_type": "code", "execution_count": 20, "id": "28700cde-6489-4fa7-a57b-af8233e3f2aa", "metadata": { "id": "28700cde-6489-4fa7-a57b-af8233e3f2aa" }, "outputs": [], "source": [ "def calculate_accuracy(model, test_dataset, N_test):\n", " \"\"\"\n", " Calculate the accuracy of the model on the test dataset.\n", "\n", " Parameters:\n", " model (object): The model to use for predictions.\n", " test_dataset (list): The list of test samples, each containing a 'question' and 'answer'.\n", " N_test (int): The number of test samples to evaluate.\n", "\n", " Returns:\n", " float: The accuracy of the model on the test dataset.\n", " \"\"\"\n", " correct_predictions = 0\n", "\n", " for i in tqdm(range(N_test)):\n", " question = test_dataset[i]['question']\n", " correct_answer = test_dataset[i]['answer']\n", "\n", " # Generate the model's response\n", " answer, _ = generate_response(\n", " text_input='Dominant secondary structure of < ' + question + ' >',\n", " max_new_tokens=16,\n", " temperature=0.01, messages=[], system_prompt='',\n", " )\n", "\n", " # Check if the model's answer matches the correct answer\n", " if answer[0].strip() == correct_answer.strip():\n", " correct_predictions += 1\n", "\n", " else:\n", " print(f\"Answer LLM: {answer[0]}\\nWrong answer: {correct_answer}\")\n", "\n", " # Calculate accuracy\n", " accuracy = correct_predictions / N_test\n", " return accuracy\n" ] }, { "cell_type": "code", "execution_count": 21, "id": "a55e1a0c-87af-42b1-ba3c-15c2d8865215", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "ed008dac238848349eaf427fc764b4c8", "fa1ff9dfeef6453c9eae1b4d55ad6bca", "20bebd52648447ef9c947b196b271520", "31d66ef3d1c94745a7ed52d748cc7fcc", "fa7bd2fceb844f1da76a081303991874", "597d5a90a55e4d8f8da0a4d9d4661173", "8cda86c26f4b4213b98fd75f49500f47", "e10ac0a882174a00a908514cfd0f6f63", "6c8fc499373541c0856970a565961938", "dc4868d3c0e540dd980d282e1ee2ef59", "c26d3ecdf8314508aac6ba467315dc76" ] }, "id": "a55e1a0c-87af-42b1-ba3c-15c2d8865215", "outputId": "35eabea8-ba13-4d29-99ed-cee8ce8cf9bf", "scrolled": true }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ed008dac238848349eaf427fc764b4c8", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/1436 [00:00