Training in progress, step 1000

cmd.txt

@@ -0,0 +1 @@
+/var/spool/slurmd/job116988/slurm_script 04-06_01-01

commit.txt

@@ -0,0 +1,5 @@
+commit c4fe47d125efdcc428a5dd46500d754dc07f4a94
+Author: Shteyman <[email protected]>
+Date:   Sun Jun 2 08:25:22 2024 -0700
+
+    clean version of run_clm.py

events.out.tfevents.1717488111.isl-gpu33.2434801.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f45ed4aeee7fe305f565aefaa0438afb9c9eb197a3d37c0f97c5f88ba132713
+size 5696

experiment_code/config/config1.yaml

@@ -0,0 +1,26 @@
+config_name: "JackFram/llama-68m"
+tokenizer_name: "JackFram/llama-68m"
+validation_split_percentage: 2
+train_file: "/home/dshteyma/shareGPT_data/ShareGPT_V3_unfiltered_cleaned_split.json"
+dataset_name_hub: "anon8231489123/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json"
+dataset_name_local: "ShareGPT"
+# max_train_samples: 1000
+# max_eval_samples: 10
+do_train: True
+do_eval: True
+output_dir: "/home/dshteyma/target_draft_coupling_code/target_draft_training/training_outputs"
+overwrite_output_dir: True
+per_device_train_batch_size: 4
+gradient_accumulation_steps: 1
+report_to: "tensorboard"
+logging_dir: "/home/dshteyma/target_draft_coupling_code/target_draft_training/training_outputs"
+logging_steps: 500
+save_steps: 1000
+eval_strategy: "steps"
+eval_steps: 1000
+learning_rate: 0.0001
+gradient_accumulation_steps: 1
+weight_decay: 0.01
+warmup_ratio: 0.05
+push_to_hub: True
+hub_model_id: "DorinSht/recreate_llama_68M_vanilla"

experiment_code/config/config_redpajama.yaml

@@ -0,0 +1,27 @@
+config_name: "JackFram/llama-68m"
+tokenizer_name: "JackFram/llama-68m"
+validation_split_percentage: 2
+train_file: "/home/dshteyma/target_draft_coupling_code/dataset_dict.json"
+dataset_name_local: "RedPajama"
+dataset_name: "togethercomputer/RedPajama-Data-1T-Sample"
+dataset_name_hub: "togethercomputer/RedPajama-Data-1T-Sample"
+# max_train_samples: 1000
+# max_eval_samples: 10
+do_train: True
+do_eval: True
+output_dir: "/home/dshteyma/target_draft_coupling_code/target_draft_training/training_outputs"
+overwrite_output_dir: True
+per_device_train_batch_size: 4
+gradient_accumulation_steps: 3
+report_to: "tensorboard"
+logging_dir: "/home/dshteyma/target_draft_coupling_code/target_draft_training/training_outputs"
+logging_steps: 10000
+save_steps: 10000
+eval_strategy: "steps"
+eval_steps: 10000
+learning_rate: 0.0001
+weight_decay: 0.01
+warmup_ratio: 0.05
+push_to_hub: False
+hub_model_id: "DorinSht/llama_68M_redpajama"
+hub_strategy: "all_checkpoints"

experiment_code/prepare_sharegpt.py

@@ -0,0 +1,44 @@
+"""
+This script is largely copied from the Vicuna repo: https://github.com/lm-sys/FastChat/blob/main/fastchat/data/split_long_conversation.py
+We fixed a bug in `split_one_sample`, which previously includes long conversations in the processed data. Now we skip these long conversations.
+"""
+import argparse
+from concurrent.futures import ProcessPoolExecutor
+import json
+import transformers
+from tqdm import tqdm
+
+def shareGPT_pipeline(tokenizer, raw_datasets, overwrite_cache):
+    
+    def preprocess_conversation(convo):
+        key_mapping = {"role" : "from", "content" : "value"}
+        value_mapping = {"user" : "user", "human" : "user", "gpt" : "assistant", 'system': 'assitant', 'bing': 'assitant', 'chatgpt': 'assitant', 'bard': 'assitant'}
+        # mapping = {"human" : "user", "gpt" : "assitant"}
+        if value_mapping[convo[0][key_mapping['role']]] != 'user': 
+            convo = convo[1:]
+        preproc_convos_user = [{"role": 'user', "content": convo_elem[key_mapping['content']]} for i, convo_elem in enumerate(convo) if (i % 2 == 0 and value_mapping[convo_elem[key_mapping['role']]] == 'user')]
+        preproc_convos_assistant = [{"role": 'assistant', "content": convo_elem[key_mapping['content']]} for i, convo_elem in enumerate(convo) if (i % 2 == 1 and value_mapping[convo_elem[key_mapping['role']]] == 'assistant')]
+        if len(preproc_convos_user) != len(preproc_convos_assistant):
+            return []
+        preproc_convos = [conv_elem for pair in zip(preproc_convos_user, preproc_convos_assistant) for conv_elem in pair]
+        return preproc_convos
+
+    def filter_incorrect_conversations(examples):
+        convos = examples["conversations"]
+        ids_to_remove = [True if preprocess_conversation(convo) == [] else False for convo in convos]
+        return { "ids_to_remove" : ids_to_remove, }
+
+    def formatting_prompts_func(examples):
+        convos = examples["conversations"]
+        # preproc_convos = [convo for convo in convos if (convo[0]['from'] == 'human' or convo[0]['from'] == 'user')]
+        preproc_convos = [preprocess_conversation(convo) for convo in convos]
+        # preproc_convos2 = [preproc_convo for preproc_convo in preproc_convos if preproc_convo[0]['role'] == 'user']
+        texts = [tokenizer.apply_chat_template(convo, tokenize = False, add_generation_prompt = False) for i, convo in enumerate(preproc_convos)]
+        return { "text" : texts,}
+
+    filtered_datasets = raw_datasets.filter(lambda example: example['conversations'] != [], load_from_cache_file=not overwrite_cache,)
+    dataset = filtered_datasets.map(filter_incorrect_conversations, batched = True, load_from_cache_file=not overwrite_cache,)
+    filtered_datasets2 = dataset.filter(lambda example: example['ids_to_remove'] == False, load_from_cache_file=not overwrite_cache,)
+    raw_datasets_preprocessed = filtered_datasets2.map(formatting_prompts_func, batched = True, load_from_cache_file=not overwrite_cache,)    
+    
+    return raw_datasets_preprocessed

experiment_code/requirements.txt

@@ -0,0 +1,2 @@
+huggingface-hub==0.22.2
+-e git+https://github.com/huggingface/transformers.git@bbaa8ceff696c479aecdb4575b2deb1349efd3aa#egg=transformers

experiment_code/run_clm.py

@@ -0,0 +1,754 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2020 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset.
+
+Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
+https://huggingface.co/models?filter=text-generation
+"""
+# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments.
+import random
+import logging
+import math
+import os
+from datetime import datetime
+import sys
+import warnings
+from dataclasses import dataclass, field
+from itertools import chain
+from typing import Optional
+import datasets
+import evaluate
+import torch
+from datasets import load_dataset
+import argparse
+import transformers
+from prepare_sharegpt import shareGPT_pipeline
+from transformers import (
+    CONFIG_MAPPING,
+    MODEL_FOR_CAUSAL_LM_MAPPING,
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    HfArgumentParser,
+    Trainer,
+    TrainingArguments,
+    default_data_collator,
+    set_seed,
+)
+from transformers.testing_utils import CaptureLogger
+from transformers.trainer_utils import get_last_checkpoint
+from transformers.utils import check_min_version, send_example_telemetry
+from transformers.utils.versions import require_version
+from functools import partial
+
+from omegaconf import DictConfig, OmegaConf
+import hydra 
+
+# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
+check_min_version("4.41.0.dev0")
+
+require_version("datasets>=2.14.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt")
+
+logger = logging.getLogger(__name__)
+
+MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+
+random.seed(42)
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
+    """
+
+    model_name_or_path: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The model checkpoint for weights initialization. Don't set if you want to train a model from scratch."
+            )
+        },
+    )
+    model_type: Optional[str] = field(
+        default=None,
+        metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)},
+    )
+    padding_side: str = field(
+        default="right", metadata={"help": "The padding side in tokenizer"}
+    ) 
+    config_overrides: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Override some existing default config settings when a model is trained from scratch. Example: "
+                "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"
+            )
+        },
+    )
+    config_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
+    )
+    tokenizer_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
+    )
+    cache_dir: Optional[str] = field(
+        default=None,
+        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+    )
+    use_fast_tokenizer: bool = field(
+        default=True,
+        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
+    )
+    model_revision: str = field(
+        default="main",
+        metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
+    )
+    token: str = field(
+        default=None,
+        metadata={
+            "help": (
+                "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token "
+                "generated when running `huggingface-cli login` (stored in `~/.huggingface`)."
+            )
+        },
+    )
+    use_auth_token: bool = field(
+        default=None,
+        metadata={
+            "help": "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token` instead."
+        },
+    )
+    trust_remote_code: bool = field(
+        default=True,
+        metadata={
+            "help": (
+                "Whether or not to allow for custom models defined on the Hub in their own modeling files. This option "
+                "should only be set to `True` for repositories you trust and in which you have read the code, as it will "
+                "execute code present on the Hub on your local machine."
+            )
+        },
+    )
+    torch_dtype: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Override the default `torch.dtype` and load the model under this dtype. If `auto` is passed, the "
+                "dtype will be automatically derived from the model's weights."
+            ),
+            "choices": ["auto", "bfloat16", "float16", "float32"],
+        },
+    )
+    low_cpu_mem_usage: bool = field(
+        default=False,
+        metadata={
+            "help": (
+                "It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded. "
+                "set True will benefit LLM loading time and RAM consumption."
+            )
+        },
+    )
+
+    def __post_init__(self):
+        if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):
+            raise ValueError(
+                "--config_overrides can't be used in combination with --config_name or --model_name_or_path"
+            )
+            
+
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+    """
+    dataset_name: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
+    )
+    dataset_name_hub: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
+    )
+    dataset_name_local: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset for identification."}
+    )
+    dataset_config_name: Optional[str] = field(
+        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
+    )
+    train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
+    validation_file: Optional[str] = field(
+        default=None,
+        metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
+    )
+    max_train_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "For debugging purposes or quicker training, truncate the number of training examples to this "
+                "value if set."
+            )
+        },
+    )
+    max_eval_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
+                "value if set."
+            )
+        },
+    )
+    streaming: bool = field(default=False, metadata={"help": "Enable streaming mode"})
+    block_size: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Optional input sequence length after tokenization. "
+                "The training dataset will be truncated in block of this size for training. "
+                "Default to the model max input length for single sentence inputs (take into account special tokens)."
+            )
+        },
+    )
+    overwrite_cache: bool = field(
+        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
+    )
+    validation_split_percentage: Optional[int] = field(
+        default=5,
+        metadata={
+            "help": "The percentage of the train set used as validation set in case there's no validation split"
+        },
+    )
+    preprocessing_num_workers: Optional[int] = field(
+        default=None,
+        metadata={"help": "The number of processes to use for the preprocessing."},
+    )
+    keep_linebreaks: bool = field(
+        default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."}
+    )
+    lazy_preprocess: bool = False
+
+    def __post_init__(self):
+        if self.streaming:
+            require_version("datasets>=2.0.0", "The streaming feature requires `datasets>=2.0.0`")
+
+        if self.dataset_name is None and self.train_file is None and self.validation_file is None:
+            raise ValueError("Need either a dataset name or a training/validation file.")
+        else:
+            if self.train_file is not None:
+                extension = self.train_file.split(".")[-1]
+                assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file."
+            if self.validation_file is not None:
+                extension = self.validation_file.split(".")[-1]
+                assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file."   
+
+# @dataclass
+# class TrainingArguments(transformers.TrainingArguments):
+#     cache_dir: Optional[str] = field(default=None)
+#     optim: str = field(default="adamw_torch")
+#     model_max_length: int = field(
+#         default=2048,
+#         metadata={
+#             "help": "Maximum sequence length. Sequences will be right padded (and possibly truncated)."
+#         },
+#     )
+
+def create_output_directory(dir_root_path):
+    # Get the current date and time
+    current_time = datetime.now()
+    # Format the date and time as a string
+    # Example format: YYYYMMDD_HHMMSS
+    formatted_time = current_time.strftime("%Y%m%d_%H%M%S")
+    # Define the directory name with the formatted time
+    directory_full_path = os.path.join(dir_root_path, f"training_outputs_{formatted_time}")
+    # Create the directory
+    os.makedirs(directory_full_path)
+    print(f"Directory '{directory_full_path}' created successfully.")
+    return directory_full_path
+
+def main():
+    # See all possible arguments in src/transformers/training_args.py
+    # or by passing the --help flag to this script.
+    # We now keep distinct sets of args, for a cleaner separation of concerns.
+    parser = argparse.ArgumentParser(description="parser for arguments from .py script call")
+    parser.add_argument('--output_dir', type=str, help='Path for training_args.output_dir')
+    parser.add_argument('--logging_dir', type=str, help='Path for training_args.logging_dir')
+    parser.add_argument('--config_file', type=str, help='An additional required option.')
+    args = parser.parse_args()
+    
+    parser_hf = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
+    if args.config_file is not None and args.output_dir is not None and args.output_dir is not None:
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser_hf.parse_yaml_file(args.config_file)
+        training_args.output_dir = args.output_dir
+        training_args.logging_dir = args.logging_dir
+    else:
+        # use the preset config file defined in the slurm .sh script
+        # model_args, data_args, training_args = parser_hf.parse_yaml_file(os.getenv("DEFAULT_CONFIG_FILE"))
+        model_args, data_args, training_args = parser_hf.parse_yaml_file('./config/config1.yaml')
+
+
+    if model_args.use_auth_token is not None:
+        warnings.warn(
+            "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token` instead.",
+            FutureWarning,
+        )
+        if model_args.token is not None:
+            raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.")
+        model_args.token = model_args.use_auth_token
+
+    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
+    # information sent is the one passed as arguments along with your Python/PyTorch versions.
+    send_example_telemetry("run_clm", model_args, data_args)
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        handlers=[logging.StreamHandler(sys.stdout)],
+    )
+
+    if training_args.should_log:
+        # The default of training_args.log_level is passive, so we set log level at info here to have that default.
+        transformers.utils.logging.set_verbosity_info()
+
+    log_level = training_args.get_process_log_level()
+    logger.setLevel(log_level)
+    datasets.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.enable_default_handler()
+    transformers.utils.logging.enable_explicit_format()
+
+    # Log on each process the small summary:
+    logger.warning(
+        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, "
+        + f"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}"
+    )
+    logger.info(f"Training/evaluation parameters {training_args}")
+
+    # Detecting last checkpoint.
+    last_checkpoint = None
+    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
+        last_checkpoint = get_last_checkpoint(training_args.output_dir)
+        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
+            raise ValueError(
+                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
+                "Use --overwrite_output_dir to overcome."
+            )
+        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
+            logger.info(
+                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
+                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
+            )
+
+    # Set seed before initializing model.
+    set_seed(training_args.seed)
+
+    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
+    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
+    # (the dataset will be downloaded automatically from the datasets Hub).
+    #
+    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
+    # 'text' is found. You can easily tweak this behavior (see below).
+    #
+    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
+    # download the dataset.
+    if data_args.dataset_name is not None:
+        # Downloading and loading a dataset from the hub.
+        raw_datasets = load_dataset(
+            data_args.dataset_name,
+            data_args.dataset_config_name,
+            cache_dir=model_args.cache_dir,
+            token=model_args.token,
+            streaming=data_args.streaming,
+        )
+        if "validation" not in raw_datasets.keys():
+            raw_datasets["validation"] = load_dataset(
+                data_args.dataset_name,
+                data_args.dataset_config_name,
+                split=f"train[:{data_args.validation_split_percentage}%]",
+                cache_dir=model_args.cache_dir,
+                token=model_args.token,
+                streaming=data_args.streaming,
+            )
+            raw_datasets["train"] = load_dataset(
+                data_args.dataset_name,
+                data_args.dataset_config_name,
+                split=f"train[{data_args.validation_split_percentage}%:]",
+                cache_dir=model_args.cache_dir,
+                token=model_args.token,
+                streaming=data_args.streaming,
+            )
+    else:
+        data_files = {}
+        dataset_args = {}
+        if data_args.train_file is not None:
+            data_files["train"] = data_args.train_file
+        if data_args.validation_file is not None:
+            data_files["validation"] = data_args.validation_file
+        extension = (
+            data_args.train_file.split(".")[-1]
+            if data_args.train_file is not None
+            else data_args.validation_file.split(".")[-1]
+        )
+        if extension == "txt":
+            extension = "text"
+            dataset_args["keep_linebreaks"] = data_args.keep_linebreaks
+        raw_datasets = load_dataset(
+            extension,
+            data_files=data_files,
+            cache_dir=model_args.cache_dir,
+            token=model_args.token,
+            **dataset_args,
+        )
+        # If no validation data is there, validation_split_percentage will be used to divide the dataset.
+        if "validation" not in raw_datasets.keys():
+            raw_datasets["validation"] = load_dataset(
+                extension,
+                data_files=data_files,
+                split=f"train[:{data_args.validation_split_percentage}%]",
+                cache_dir=model_args.cache_dir,
+                token=model_args.token,
+                **dataset_args,
+            )
+            raw_datasets["train"] = load_dataset(
+                extension,
+                data_files=data_files,
+                split=f"train[{data_args.validation_split_percentage}%:]",
+                cache_dir=model_args.cache_dir,
+                token=model_args.token,
+                **dataset_args,
+            )
+    
+    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
+    # https://huggingface.co/docs/datasets/loading_datasets.
+
+    # Load pretrained model and tokenizer
+    #
+    # Distributed training:
+    # The .from_pretrained methods guarantee that only one local process can concurrently
+    # download model & vocab.
+
+    config_kwargs = {
+        "cache_dir": model_args.cache_dir,
+        "revision": model_args.model_revision,
+        "token": model_args.token,
+        "trust_remote_code": model_args.trust_remote_code,
+    }
+    if model_args.config_name:
+        config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs)
+    elif model_args.model_name_or_path:
+        config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)
+    else:
+        config = CONFIG_MAPPING[model_args.model_type]()
+        logger.warning("You are instantiating a new config instance from scratch.")
+        if model_args.config_overrides is not None:
+            logger.info(f"Overriding config: {model_args.config_overrides}")
+            config.update_from_string(model_args.config_overrides)
+            logger.info(f"New config: {config}")
+
+    tokenizer_kwargs = {
+        "cache_dir": model_args.cache_dir,
+        "use_fast": model_args.use_fast_tokenizer,
+        "revision": model_args.model_revision,
+        "token": model_args.token,
+        "padding": 'max_length',
+        "trust_remote_code": model_args.trust_remote_code,
+        "model_max_length": config.max_position_embeddings,
+        "return_tensors":'pt'
+    }
+    if model_args.tokenizer_name:
+        tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs)
+    elif model_args.model_name_or_path:
+        tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, **tokenizer_kwargs)
+    else:
+        raise ValueError(
+            "You are instantiating a new tokenizer from scratch. This is not supported by this script. "
+            "You can do it from another script, save it, and load it from here, using --tokenizer_name."
+        )
+    if tokenizer.pad_token != tokenizer.unk_token:
+        tokenizer.pad_token = tokenizer.unk_token
+
+    if model_args.model_name_or_path:
+        torch_dtype = (
+            model_args.torch_dtype
+            if model_args.torch_dtype in ["auto", None]
+            else getattr(torch, model_args.torch_dtype)
+        )
+        model = AutoModelForCausalLM.from_pretrained(
+            model_args.model_name_or_path,
+            from_tf=bool(".ckpt" in model_args.model_name_or_path),
+            config=config,
+            cache_dir=model_args.cache_dir,
+            revision=model_args.model_revision,
+            token=model_args.token,
+            trust_remote_code=model_args.trust_remote_code,
+            torch_dtype=torch_dtype,
+            low_cpu_mem_usage=model_args.low_cpu_mem_usage,
+        )
+    else:
+        model = AutoModelForCausalLM.from_config(config, trust_remote_code=model_args.trust_remote_code)
+        n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values())
+        logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params")
+
+    # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch
+    # on a small vocab and want a smaller embedding size, remove this test.
+    embedding_size = model.get_input_embeddings().weight.shape[0]
+    if len(tokenizer) > embedding_size:
+        model.resize_token_embeddings(len(tokenizer))    
+    
+    if "ShareGPT" == data_args.dataset_name_local:
+        raw_datasets_preprocessed = shareGPT_pipeline(tokenizer=tokenizer, raw_datasets=raw_datasets, overwrite_cache=data_args.overwrite_cache) 
+    if "RedPajama" == data_args.dataset_name_local:
+        raw_datasets_preprocessed = raw_datasets
+    
+### HEREE
+    # Preprocessing the datasets.
+    # First we tokenize all the texts.
+    if training_args.do_train:
+        column_names = list(raw_datasets_preprocessed["train"].features)
+    else:
+        column_names = list(raw_datasets_preprocessed["validation"].features)
+    text_column_name = "text"
+
+
+    # since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function
+    tok_logger = transformers.utils.logging.get_logger("transformers.tokenization_utils_base")
+    
+    def tokenize_function(examples):
+        with CaptureLogger(tok_logger) as cl:
+            # print(tokenizer(examples[text_column_name]))
+            # output = tokenizer(examples[text_column_name])
+            output = tokenizer(
+                            examples[text_column_name],
+                            return_tensors="pt",
+                            padding="max_length",
+                            max_length=tokenizer.model_max_length,
+                            truncation=True,
+                        )
+            # output = input_ids.clone() 
+        # clm input could be much much longer than block_size
+        if "Token indices sequence length is longer than the" in cl.out:
+            tok_logger.warning(
+                "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits"
+                " before being passed to the model."
+            )
+        return output
+    
+    with training_args.main_process_first(desc="dataset map tokenization"):
+        if not data_args.streaming:
+            tokenized_datasets = raw_datasets_preprocessed.map(
+                tokenize_function,
+                batched=True,
+                num_proc=data_args.preprocessing_num_workers,
+                remove_columns=column_names,
+                load_from_cache_file=not data_args.overwrite_cache,
+                desc="Running tokenizer on dataset",
+            )
+        else:
+            tokenized_datasets = raw_datasets_preprocessed.map(
+                tokenize_function,
+                batched=True,
+                remove_columns=column_names,
+                load_from_cache_file=not data_args.overwrite_cache,
+            )
+    if hasattr(config, "max_position_embeddings"):
+        max_pos_embeddings = config.max_position_embeddings
+    else:
+        # Define a default value if the attribute is missing in the config.
+        max_pos_embeddings = 1024
+
+    if data_args.block_size is None:
+        block_size = tokenizer.model_max_length
+        if block_size > max_pos_embeddings:
+            logger.warning(
+                f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). "
+                f"Using block_size={min(1024, max_pos_embeddings)} instead. You can change that default value by passing --block_size xxx."
+            )
+            if max_pos_embeddings > 0:
+                block_size = min(1024, max_pos_embeddings)
+            else:
+                block_size = 1024
+    else:
+        if data_args.block_size > tokenizer.model_max_length:
+            logger.warning(
+                f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model "
+                f"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}."
+            )
+        block_size = min(data_args.block_size, tokenizer.model_max_length)
+
+    # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.
+    def group_texts(examples):
+        # Concatenate all texts.
+        concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()}
+        total_length = len(concatenated_examples[list(examples.keys())[0]])
+        # We drop the small remainder, and if the total_length < block_size  we exclude this batch and return an empty dict.
+        # We could add padding if the model supported it instead of this drop, you can customize this part to your needs.
+        total_length = (total_length // block_size) * block_size
+        # Split by chunks of max_len.
+        result = {
+            k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
+            for k, t in concatenated_examples.items()
+        }
+        result["labels"] = result["input_ids"].copy()
+        return result
+
+    # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder
+    # for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower
+    # to preprocess.
+    #
+    # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:
+    # https://huggingface.co/docs/datasets/process#map
+
+    with training_args.main_process_first(desc="grouping texts together"):
+        if not data_args.streaming:
+            lm_datasets = tokenized_datasets.map(
+                group_texts,
+                batched=True,
+                num_proc=data_args.preprocessing_num_workers,
+                load_from_cache_file=not data_args.overwrite_cache,
+                desc=f"Grouping texts in chunks of {block_size}",
+            )
+        else:
+            lm_datasets = tokenized_datasets.map(
+                group_texts,
+                batched=True,
+                load_from_cache_file=not data_args.overwrite_cache,
+            )
+
+    if training_args.do_train:
+        if "train" not in tokenized_datasets:
+            raise ValueError("--do_train requires a train dataset")
+        train_dataset = lm_datasets["train"]
+        if data_args.max_train_samples is not None:
+            max_train_samples = min(len(train_dataset), data_args.max_train_samples)
+            train_dataset = train_dataset.select(range(max_train_samples))
+
+    if training_args.do_eval:
+        if "validation" not in tokenized_datasets:
+            raise ValueError("--do_eval requires a validation dataset")
+        eval_dataset = lm_datasets["validation"]
+        if data_args.max_eval_samples is not None:
+            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)
+            eval_dataset = eval_dataset.select(range(max_eval_samples))
+
+        def preprocess_logits_for_metrics(logits, labels):
+            if isinstance(logits, tuple):
+                # Depending on the model and config, logits may contain extra tensors,
+                # like past_key_values, but logits always come first
+                logits = logits[0]
+            return logits.argmax(dim=-1)
+
+        
+        def compute_metrics(eval_preds):
+            accuracy = evaluate.load("accuracy", cache_dir=model_args.cache_dir)
+            perplexity = evaluate.load("perplexity", module_type="metric")
+            preds, labels = eval_preds
+            # preds have the same shape as the labels, after the argmax(-1) has been calculated
+            # by preprocess_logits_for_metrics but we need to shift the labels
+            labels = labels[:, 1:].reshape(-1)
+            preds = preds[:, :-1].reshape(-1)
+            accuracy = accuracy.compute(predictions=preds, references=labels)
+            # perplexity = perplexity.compute(predictions=preds, model_id='llama')
+            return accuracy
+
+    # Initialize the optimizer
+    optimizer = torch.optim.AdamW(model.parameters(), lr=training_args.learning_rate, weight_decay=training_args.weight_decay)
+    # Calculate the number of training steps
+    train_steps = (len(train_dataset) // (training_args.per_device_train_batch_size * training_args._n_gpu)) * training_args.num_train_epochs
+
+    # Initialize the scheduler
+    linear_scheduler = transformers.get_linear_schedule_with_warmup(
+        optimizer,
+        num_warmup_steps=train_steps*training_args.warmup_ratio,
+        num_training_steps=train_steps
+    )
+        
+    # Initialize our Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset if training_args.do_train else None,
+        eval_dataset=eval_dataset if training_args.do_eval else None,
+        tokenizer=tokenizer,
+        optimizers=(optimizer, linear_scheduler),
+        # Data collator will default to DataCollatorWithPadding, so we change it.
+        data_collator=default_data_collator,
+        compute_metrics=compute_metrics if training_args.do_eval else None,
+        preprocess_logits_for_metrics=preprocess_logits_for_metrics
+        if training_args.do_eval else None,
+    )
+
+    # Training
+    if training_args.do_train:
+        checkpoint = None
+        if training_args.resume_from_checkpoint is not None:
+            checkpoint = training_args.resume_from_checkpoint
+        elif last_checkpoint is not None:
+            checkpoint = last_checkpoint
+        train_result = trainer.train(resume_from_checkpoint=checkpoint)
+        trainer.save_model()  # Saves the tokenizer too for easy upload
+
+        metrics = train_result.metrics
+
+        max_train_samples = (
+            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)
+        )
+        metrics["train_samples"] = min(max_train_samples, len(train_dataset))
+
+        trainer.log_metrics("train", metrics)
+        trainer.save_metrics("train", metrics)
+        trainer.save_state()
+        try:
+            torch.save([vars(a) for a in [training_args, data_args, model_args]], os.path.join(training_args.output_dir, "args.bin"))
+        except:
+            logger.info("Failed to save arguments")
+
+    # Evaluation
+    if training_args.do_eval:
+        logger.info("*** Evaluate ***")
+
+        metrics = trainer.evaluate()
+
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
+        try:
+            perplexity = math.exp(metrics["eval_loss"])
+        except OverflowError:
+            perplexity = float("inf")
+        metrics["perplexity"] = perplexity
+
+        trainer.log_metrics("eval", metrics)
+        trainer.save_metrics("eval", metrics)
+
+    kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "text-generation"}
+    if data_args.dataset_name is not None:
+        kwargs["dataset_tags"] = data_args.dataset_name
+        if data_args.dataset_config_name is not None:
+            kwargs["dataset_args"] = data_args.dataset_config_name
+            kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}"
+        else:
+            kwargs["dataset"] = data_args.dataset_name
+    elif data_args.dataset_name_hub is not None:
+        kwargs["dataset"] = data_args.dataset_name_hub
+
+    if training_args.push_to_hub:
+        trainer.push_to_hub(**kwargs)
+    else:
+        trainer.create_model_card(**kwargs)
+
+if __name__ == "__main__":
+    main()

experiment_code/submit_job.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+#SBATCH -p g48
+#SBATCH --job-name=myjob_shareGPT
+#SBATCH --qos=high
+#SBATCH --nodes=1                 # Number of nodes
+#SBATCH --ntasks=1         # Number of tasks (one for each script)
+#SBATCH --cpus-per-task=60
+#SBATCH --gres=gpu:6
+#SBATCH --array=1-2                      # Array range
+# #SBATCH --output=./slurm_outputs/run_clm_job_%A_task_%a.out  # Standard output
+#SBATCH --output=/dev/null     # Discard standard output  # Because we write to the log.txt file
+
+# # Get the current date and time
+current_time=$(date +"%d-%m_%H-%M")
+OUTPUT_DIR="./training_outputs_job_${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}_${current_time}"
+
+while test $# -gt 0; do
+    echo $1
+    case "$1" in 
+        --output_dir)
+            shift
+            OUTPUT_DIR=$1
+            shift
+            ;;
+    esac
+done
+
+mkdir_is_exists() {
+    if [ -d "$1" ]; then
+        echo "Directory '$1' already exists."
+    else
+        mkdir -p "$1"
+        echo "Directory '$1' created."
+    fi
+}
+
+
+mkdir_is_exists $OUTPUT_DIR
+mkdir_is_exists $OUTPUT_DIR/experiment_code
+git log -n 1 > $OUTPUT_DIR/commit.txt
+pip freeze > $OUTPUT_DIR/pip_freeze.txt
+echo $0 $ARGS $current_time > $OUTPUT_DIR/cmd.txt
+cp -r ./run_clm.py $OUTPUT_DIR/experiment_code
+cp -r ./prepare_sharegpt.py $OUTPUT_DIR/experiment_code
+cp -r config $OUTPUT_DIR/experiment_code
+cp -r ./submit_job.sh $OUTPUT_DIR/experiment_code
+cp -r ./requirements.txt $OUTPUT_DIR/experiment_code
+
+# Define the Python scripts and their corresponding input files
+declare -A scripts_and_inputs=(
+        ["1"]="./config/config1.yaml"
+    	["2"]="./config/config_redpajama.yaml"
+    # ["3"]="./config/config1.yaml"
+    # ["4"]="./config/config1.yaml"
+    # ["5"]="./config/config1.yaml"
+    # ["6"]="./config/config1.yaml"
+    # ["7"]="./config/config1.yaml"
+    # ["8"]="./config/config1.yaml"
+    # ["9"]="./config/config1.yaml"
+    # ["10"]="./config/config1.yaml"
+    # ["11"]="./config/config1.yaml"
+    # ["12"]="./config/config1.yaml"
+    # ["13"]="./config/config1.yaml"
+    # ["14"]="./config/config1.yaml"
+    # ["15"]="./config/config1.yaml"
+    # ["16"]="./config/config1.yaml"
+    # ["17"]="./config/config1.yaml"
+    # ["18"]="./config/config1.yaml"
+    # ["19"]="./config/config1.yaml"
+    # ["20"]="./config/config1.yaml"
+)
+
+# Launch each script with its corresponding input file as a separate task
+echo "Starting job array task: $SLURM_ARRAY_TASK_ID"
+
+INPUT_DIR="${scripts_and_inputs[$SLURM_ARRAY_TASK_ID]}"
+export DEFAULT_CONFIG_FILE="./config/config1.yaml"
+srun --exclusive python run_clm.py --output_dir $OUTPUT_DIR --logging_dir $OUTPUT_DIR --config_file $INPUT_DIR 2>&1 | tee $OUTPUT_DIR/log.txt
+
+
+# Wait for all background jobs to complete
+wait
+
+# Print a message indicating completion
+echo "All Python scripts have been executed."
+
+
+# mv ./slurm_outputs/run_clm_job_$SLURM_ARRAY_JOB_ID*$SLURM_ARRAY_TASK_ID* "$output_dir/"
+
+
+# python -m torch.distributed.launch ~/target_draft_coupling_code/target_draft_training/run_clm.py --multirun task=1,2 

log.txt

@@ -0,0 +1,307 @@
+2024-06-04 01:01:39.001875: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
+2024-06-04 01:01:39.054364: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
+2024-06-04 01:01:39.054415: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
+2024-06-04 01:01:39.055904: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
+2024-06-04 01:01:39.063872: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
+To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
+2024-06-04 01:01:41.050561: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT
+06/04/2024 01:01:46 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 6, distributed training: False, 16-bits training: False
+06/04/2024 01:01:46 - INFO - __main__ - Training/evaluation parameters TrainingArguments(
+_n_gpu=6,
+accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None},
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=False,
+bf16_full_eval=False,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+dataloader_prefetch_factor=None,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=None,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=True,
+do_predict=False,
+do_train=True,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_do_concat_batches=True,
+eval_steps=1000,
+eval_strategy=steps,
+evaluation_strategy=None,
+fp16=False,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=1,
+gradient_checkpointing=False,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=DorinSht/recreate_llama_68M_vanilla,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=0.0001,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=./training_outputs_job_116987_1_04-06_01-01,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=500,
+logging_strategy=steps,
+lr_scheduler_kwargs={},
+lr_scheduler_type=linear,
+max_grad_norm=1.0,
+max_steps=-1,
+metric_for_best_model=None,
+mp_parameters=,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=3.0,
+optim=adamw_torch,
+optim_args=None,
+optim_target_modules=None,
+output_dir=./training_outputs_job_116987_1_04-06_01-01,
+overwrite_output_dir=True,
+past_index=-1,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=4,
+prediction_loss_only=False,
+push_to_hub=True,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+ray_scope=last,
+remove_unused_columns=True,
+report_to=['tensorboard'],
+restore_callback_states_from_checkpoint=False,
+resume_from_checkpoint=None,
+run_name=/home/dshteyma/target_draft_coupling_code/target_draft_training/training_outputs,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=1000,
+save_strategy=steps,
+save_total_limit=None,
+seed=42,
+skip_memory_metrics=True,
+split_batches=None,
+tf32=None,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.05,
+warmup_steps=0,
+weight_decay=0.01,
+)
+06/04/2024 01:01:47 - INFO - datasets.builder - Using custom data configuration default-afe4b27d28cbdcb1
+Using custom data configuration default-afe4b27d28cbdcb1
+Loading Dataset Infos from /home/dshteyma/miniconda3/lib/python3.9/site-packages/datasets/packaged_modules/json
+06/04/2024 01:01:47 - INFO - datasets.info - Loading Dataset Infos from /home/dshteyma/miniconda3/lib/python3.9/site-packages/datasets/packaged_modules/json
+06/04/2024 01:01:47 - INFO - datasets.builder - Overwrite dataset info from restored data version if exists.
+Overwrite dataset info from restored data version if exists.
+Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/04/2024 01:01:47 - INFO - datasets.info - Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/04/2024 01:01:47 - INFO - datasets.builder - Found cached dataset json (/home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7)
+Found cached dataset json (/home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7)
+Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/04/2024 01:01:47 - INFO - datasets.info - Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/04/2024 01:01:47 - INFO - datasets.builder - Using custom data configuration default-afe4b27d28cbdcb1
+Using custom data configuration default-afe4b27d28cbdcb1
+Loading Dataset Infos from /home/dshteyma/miniconda3/lib/python3.9/site-packages/datasets/packaged_modules/json
+06/04/2024 01:01:47 - INFO - datasets.info - Loading Dataset Infos from /home/dshteyma/miniconda3/lib/python3.9/site-packages/datasets/packaged_modules/json
+06/04/2024 01:01:47 - INFO - datasets.builder - Overwrite dataset info from restored data version if exists.
+Overwrite dataset info from restored data version if exists.
+Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/04/2024 01:01:47 - INFO - datasets.info - Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/04/2024 01:01:47 - INFO - datasets.builder - Found cached dataset json (/home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7)
+Found cached dataset json (/home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7)
+Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/04/2024 01:01:47 - INFO - datasets.info - Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/04/2024 01:01:48 - INFO - datasets.builder - Using custom data configuration default-afe4b27d28cbdcb1
+Using custom data configuration default-afe4b27d28cbdcb1
+Loading Dataset Infos from /home/dshteyma/miniconda3/lib/python3.9/site-packages/datasets/packaged_modules/json
+06/04/2024 01:01:48 - INFO - datasets.info - Loading Dataset Infos from /home/dshteyma/miniconda3/lib/python3.9/site-packages/datasets/packaged_modules/json
+06/04/2024 01:01:48 - INFO - datasets.builder - Overwrite dataset info from restored data version if exists.
+Overwrite dataset info from restored data version if exists.
+Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/04/2024 01:01:48 - INFO - datasets.info - Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/04/2024 01:01:48 - INFO - datasets.builder - Found cached dataset json (/home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7)
+Found cached dataset json (/home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7)
+Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+06/04/2024 01:01:48 - INFO - datasets.info - Loading Dataset info from /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7
+[INFO|configuration_utils.py:726] 2024-06-04 01:01:48,499 >> loading configuration file config.json from cache at /home/dshteyma/.cache/huggingface/hub/models--JackFram--llama-68m/snapshots/964a5d77df908b69f8d6476fb70e940425b04cb5/config.json
+[INFO|configuration_utils.py:789] 2024-06-04 01:01:48,501 >> Model config LlamaConfig {
+  "_name_or_path": "JackFram/llama-68m",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 2048,
+  "model_type": "llama",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 12,
+  "pad_token_id": 1,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.41.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+[INFO|tokenization_utils_base.py:2102] 2024-06-04 01:01:48,634 >> loading file tokenizer.model from cache at /home/dshteyma/.cache/huggingface/hub/models--JackFram--llama-68m/snapshots/964a5d77df908b69f8d6476fb70e940425b04cb5/tokenizer.model
+[INFO|tokenization_utils_base.py:2102] 2024-06-04 01:01:48,635 >> loading file tokenizer.json from cache at None
+[INFO|tokenization_utils_base.py:2102] 2024-06-04 01:01:48,635 >> loading file added_tokens.json from cache at None
+[INFO|tokenization_utils_base.py:2102] 2024-06-04 01:01:48,635 >> loading file special_tokens_map.json from cache at /home/dshteyma/.cache/huggingface/hub/models--JackFram--llama-68m/snapshots/964a5d77df908b69f8d6476fb70e940425b04cb5/special_tokens_map.json
+[INFO|tokenization_utils_base.py:2102] 2024-06-04 01:01:48,635 >> loading file tokenizer_config.json from cache at /home/dshteyma/.cache/huggingface/hub/models--JackFram--llama-68m/snapshots/964a5d77df908b69f8d6476fb70e940425b04cb5/tokenizer_config.json
+[WARNING|logging.py:329] 2024-06-04 01:01:48,636 >> You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+[WARNING|logging.py:329] 2024-06-04 01:01:48,716 >> You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama_fast.LlamaTokenizerFast'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+[INFO|configuration_utils.py:936] 2024-06-04 01:01:49,214 >> Generate config GenerationConfig {
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "pad_token_id": 1
+}
+
+06/04/2024 01:01:50 - INFO - __main__ - Training new model from scratch - Total size=64.88M params
+06/04/2024 01:01:50 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-988d048fea8d2473.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-988d048fea8d2473.arrow
+06/04/2024 01:01:50 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-4e281c930893bca9.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-4e281c930893bca9.arrow
+06/04/2024 01:01:50 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-3fe350bccdda6078.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-3fe350bccdda6078.arrow
+06/04/2024 01:01:50 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-35d09b588a0c62b9.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-35d09b588a0c62b9.arrow
+06/04/2024 01:01:50 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-4e5279ee31a5d8d3.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-4e5279ee31a5d8d3.arrow
+06/04/2024 01:01:50 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-63d56456928edd43.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-63d56456928edd43.arrow
+06/04/2024 01:01:50 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-6a784a78d9818240.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-6a784a78d9818240.arrow
+06/04/2024 01:01:50 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-46540f58a00a92bf.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-46540f58a00a92bf.arrow
+06/04/2024 01:01:50 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-73605724efaea9d2.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-73605724efaea9d2.arrow
+06/04/2024 01:01:50 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-83d3df87e1b82021.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-83d3df87e1b82021.arrow
+06/04/2024 01:01:50 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-efdbb02491aa6344.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-efdbb02491aa6344.arrow
+06/04/2024 01:01:50 - INFO - datasets.arrow_dataset - Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-0cf2ae38fef927f3.arrow
+Loading cached processed dataset at /home/dshteyma/.cache/huggingface/datasets/json/default-afe4b27d28cbdcb1/0.0.0/c8d2d9508a2a2067ab02cd118834ecef34c3700d143b31835ec4235bf10109f7/cache-0cf2ae38fef927f3.arrow
+06/04/2024 01:01:50 - WARNING - accelerate.utils.other - Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.
+[INFO|trainer.py:2068] 2024-06-04 01:01:51,038 >> ***** Running training *****
+[INFO|trainer.py:2069] 2024-06-04 01:01:51,038 >>   Num examples = 90,745
+[INFO|trainer.py:2070] 2024-06-04 01:01:51,038 >>   Num Epochs = 3
+[INFO|trainer.py:2071] 2024-06-04 01:01:51,038 >>   Instantaneous batch size per device = 4
+[INFO|trainer.py:2073] 2024-06-04 01:01:51,038 >>   Training with DataParallel so batch size has been adjusted to: 24
+[INFO|trainer.py:2074] 2024-06-04 01:01:51,038 >>   Total train batch size (w. parallel, distributed & accumulation) = 24
+[INFO|trainer.py:2075] 2024-06-04 01:01:51,038 >>   Gradient Accumulation steps = 1
+[INFO|trainer.py:2076] 2024-06-04 01:01:51,038 >>   Total optimization steps = 11,346
+[INFO|trainer.py:2077] 2024-06-04 01:01:51,038 >>   Number of trainable parameters = 68,030,208
+
  0%|          | 0/11346 [00:00<?, ?it/s]/home/dshteyma/miniconda3/lib/python3.9/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
+  warnings.warn('Was asked to gather along dimension 0, but all '
+
  0%|          | 1/11346 [00:04<15:05:26,  4.79s/it]
  0%|          | 2/11346 [00:06<9:21:06,  2.97s/it] 
  0%|          | 3/11346 [00:08<7:30:15,  2.38s/it]
  0%|          | 4/11346 [00:09<6:37:25,  2.10s/it]
  0%|          | 5/11346 [00:11<6:07:24,  1.94s/it]
  0%|          | 6/11346 [00:13<5:48:09,  1.84s/it]
  0%|          | 7/11346 [00:14<5:35:53,  1.78s/it]
  0%|          | 8/11346 [00:16<5:30:15,  1.75s/it]
  0%|          | 9/11346 [00:18<5:24:26,  1.72s/it]
  0%|          | 10/11346 [00:19<5:20:12,  1.69s/it]
  0%|          | 11/11346 [00:21<5:19:03,  1.69s/it]
  0%|          | 12/11346 [00:23<5:16:19,  1.67s/it]
  0%|          | 13/11346 [00:24<5:14:47,  1.67s/it]
  0%|          | 14/11346 [00:26<5:15:45,  1.67s/it]
  0%|          | 15/11346 [00:28<5:15:55,  1.67s/it]
  0%|          | 16/11346 [00:29<5:16:01,  1.67s/it]
  0%|          | 17/11346 [00:31<5:14:59,  1.67s/it]
  0%|          | 18/11346 [00:33<5:13:43,  1.66s/it]
  0%|          | 19/11346 [00:34<5:12:42,  1.66s/it]
  0%|          | 20/11346 [00:36<5:11:52,  1.65s/it]
  0%|          | 21/11346 [00:37<5:11:11,  1.65s/it]
  0%|          | 22/11346 [00:39<5:10:54,  1.65s/it]
  0%|          | 23/11346 [00:41<5:10:37,  1.65s/it]
  0%|          | 24/11346 [00:42<5:10:31,  1.65s/it]
  0%|          | 25/11346 [00:44<5:10:24,  1.65s/it]
  0%|          | 26/11346 [00:46<5:10:12,  1.64s/it]
  0%|          | 27/11346 [00:47<5:11:36,  1.65s/it]
  0%|          | 28/11346 [00:49<5:11:16,  1.65s/it]
  0%|          | 29/11346 [00:51<5:10:53,  1.65s/it]
  0%|          | 30/11346 [00:52<5:10:39,  1.65s/it]
  0%|          | 31/11346 [00:54<5:10:26,  1.65s/it]
  0%|          | 32/11346 [00:56<5:10:17,  1.65s/it]
  0%|          | 33/11346 [00:57<5:10:09,  1.64s/it]
  0%|          | 34/11346 [00:59<5:10:07,  1.64s/it]
  0%|          | 35/11346 [01:01<5:09:56,  1.64s/it]
  0%|          | 36/11346 [01:02<5:09:53,  1.64s/it]
  0%|          | 37/11346 [01:04<5:09:51,  1.64s/it]
  0%|          | 38/11346 [01:05<5:09:44,  1.64s/it]
  0%|          | 39/11346 [01:07<5:09:40,  1.64s/it]
  0%|          | 40/11346 [01:09<5:09:35,  1.64s/it]
  0%|          | 41/11346 [01:10<5:09:32,  1.64s/it]
  0%|          | 42/11346 [01:12<5:09:25,  1.64s/it]
  0%|          | 43/11346 [01:14<5:09:27,  1.64s/it]
  0%|          | 44/11346 [01:15<5:09:44,  1.64s/it]
  0%|          | 45/11346 [01:17<5:09:45,  1.64s/it]
  0%|          | 46/11346 [01:19<5:09:40,  1.64s/it]
  0%|          | 47/11346 [01:20<5:09:36,  1.64s/it]
  0%|          | 48/11346 [01:22<5:09:32,  1.64s/it]
  0%|          | 49/11346 [01:24<5:09:30,  1.64s/it]
  0%|          | 50/11346 [01:25<5:09:30,  1.64s/it]
  0%|          | 51/11346 [01:27<5:09:30,  1.64s/it]
  0%|          | 52/11346 [01:28<5:09:24,  1.64s/it]
  0%|          | 53/11346 [01:30<5:09:22,  1.64s/it]
  0%|          | 54/11346 [01:32<5:09:22,  1.64s/it]
  0%|          | 55/11346 [01:33<5:09:26,  1.64s/it]
  0%|          | 56/11346 [01:35<5:09:35,  1.65s/it]
  1%|          | 57/11346 [01:37<5:09:41,  1.65s/it]
  1%|          | 58/11346 [01:38<5:09:36,  1.65s/it]
  1%|          | 59/11346 [01:40<5:09:28,  1.65s/it]
  1%|          | 60/11346 [01:42<5:09:21,  1.64s/it]
  1%|          | 61/11346 [01:43<5:09:45,  1.65s/it]
  1%|          | 62/11346 [01:45<5:09:30,  1.65s/it]
  1%|          | 63/11346 [01:47<5:09:21,  1.65s/it]
  1%|          | 64/11346 [01:48<5:09:16,  1.64s/it]
  1%|          | 65/11346 [01:50<5:09:11,  1.64s/it]
  1%|          | 66/11346 [01:52<5:09:06,  1.64s/it]
  1%|          | 67/11346 [01:53<5:09:05,  1.64s/it]
  1%|          | 68/11346 [01:55<5:10:50,  1.65s/it]
  1%|          | 69/11346 [01:57<5:11:57,  1.66s/it]
  1%|          | 70/11346 [01:58<5:11:27,  1.66s/it]
  1%|          | 71/11346 [02:00<5:10:43,  1.65s/it]
  1%|          | 72/11346 [02:01<5:10:09,  1.65s/it]
  1%|          | 73/11346 [02:03<5:09:46,  1.65s/it]
  1%|          | 74/11346 [02:05<5:09:27,  1.65s/it]
  1%|          | 75/11346 [02:06<5:09:07,  1.65s/it]
  1%|          | 76/11346 [02:08<5:08:57,  1.64s/it]
  1%|          | 77/11346 [02:10<5:08:54,  1.64s/it]
  1%|          | 78/11346 [02:11<5:08:52,  1.64s/it]
  1%|          | 79/11346 [02:13<5:08:52,  1.64s/it]
  1%|          | 80/11346 [02:15<5:08:52,  1.64s/it]
  1%|          | 81/11346 [02:16<5:08:47,  1.64s/it]
  1%|          | 82/11346 [02:18<5:08:40,  1.64s/it]
  1%|          | 83/11346 [02:20<5:08:40,  1.64s/it]
  1%|          | 84/11346 [02:21<5:08:33,  1.64s/it]
  1%|          | 85/11346 [02:23<5:08:32,  1.64s/it]
  1%|          | 86/11346 [02:25<5:12:04,  1.66s/it]
  1%|          | 87/11346 [02:26<5:13:39,  1.67s/it]
  1%|          | 88/11346 [02:28<5:15:04,  1.68s/it]
  1%|          | 89/11346 [02:30<5:14:52,  1.68s/it]
  1%|          | 90/11346 [02:31<5:15:18,  1.68s/it]
  1%|          | 91/11346 [02:33<5:15:25,  1.68s/it]
  1%|          | 92/11346 [02:35<5:16:00,  1.68s/it]
  1%|          | 93/11346 [02:36<5:16:22,  1.69s/it]
  1%|          | 94/11346 [02:38<5:15:59,  1.68s/it]
  1%|          | 95/11346 [02:40<5:16:37,  1.69s/it]
  1%|          | 96/11346 [02:41<5:16:52,  1.69s/it]
  1%|          | 97/11346 [02:43<5:16:53,  1.69s/it]
  1%|          | 98/11346 [02:45<5:16:50,  1.69s/it]
  1%|          | 99/11346 [02:46<5:16:42,  1.69s/it]
  1%|          | 100/11346 [02:48<5:16:32,  1.69s/it]
  1%|          | 101/11346 [02:50<5:16:02,  1.69s/it]
  1%|          | 102/11346 [02:52<5:16:21,  1.69s/it]
  1%|          | 103/11346 [02:53<5:16:22,  1.69s/it]
  1%|          | 104/11346 [02:55<5:16:56,  1.69s/it]
  1%|          | 105/11346 [02:57<5:17:05,  1.69s/it]
  1%|          | 106/11346 [02:58<5:16:58,  1.69s/it]
  1%|          | 107/11346 [03:00<5:17:08,  1.69s/it]
  1%|          | 108/11346 [03:02<5:17:07,  1.69s/it]
  1%|          | 109/11346 [03:03<5:16:39,  1.69s/it]
  1%|          | 110/11346 [03:05<5:16:25,  1.69s/it]
  1%|          | 111/11346 [03:07<5:16:27,  1.69s/it]
  1%|          | 112/11346 [03:08<5:16:08,  1.69s/it]
  1%|          | 113/11346 [03:10<5:15:47,  1.69s/it]
  1%|          | 114/11346 [03:12<5:15:59,  1.69s/it]
  1%|          | 115/11346 [03:14<5:16:09,  1.69s/it]
  1%|          | 116/11346 [03:15<5:15:54,  1.69s/it]
  1%|          | 117/11346 [03:17<5:16:14,  1.69s/it]
  1%|          | 118/11346 [03:19<5:16:21,  1.69s/it]
  1%|          | 119/11346 [03:20<5:16:37,  1.69s/it]
  1%|          | 120/11346 [03:22<5:15:42,  1.69s/it]
  1%|          | 121/11346 [03:24<5:15:53,  1.69s/it]
  1%|          | 122/11346 [03:25<5:15:52,  1.69s/it]
  1%|          | 123/11346 [03:27<5:15:37,  1.69s/it]
  1%|          | 124/11346 [03:29<5:15:21,  1.69s/it]
  1%|          | 125/11346 [03:30<5:15:20,  1.69s/it]
  1%|          | 126/11346 [03:32<5:15:08,  1.69s/it]
  1%|          | 127/11346 [03:34<5:15:16,  1.69s/it]
  1%|          | 128/11346 [03:35<5:15:28,  1.69s/it]
  1%|          | 129/11346 [03:37<5:15:06,  1.69s/it]
  1%|          | 130/11346 [03:39<5:15:41,  1.69s/it]
  1%|          | 131/11346 [03:41<5:15:22,  1.69s/it]
  1%|          | 132/11346 [03:42<5:15:06,  1.69s/it]
  1%|          | 133/11346 [03:44<5:15:00,  1.69s/it]
  1%|          | 134/11346 [03:46<5:15:08,  1.69s/it]
  1%|          | 135/11346 [03:47<5:15:12,  1.69s/it]
  1%|          | 136/11346 [03:49<5:14:45,  1.68s/it]
  1%|          | 137/11346 [03:51<5:13:39,  1.68s/it]
  1%|          | 138/11346 [03:52<5:13:47,  1.68s/it]
  1%|          | 139/11346 [03:54<5:13:33,  1.68s/it]
  1%|          | 140/11346 [03:56<5:13:47,  1.68s/it]
  1%|          | 141/11346 [03:57<5:13:56,  1.68s/it]
  1%|▏         | 142/11346 [03:59<5:14:16,  1.68s/it]
  1%|▏         | 143/11346 [04:01<5:13:54,  1.68s/it]
  1%|▏         | 144/11346 [04:02<5:13:57,  1.68s/it]
  1%|▏         | 145/11346 [04:04<5:13:51,  1.68s/it]
  1%|▏         | 146/11346 [04:06<5:14:21,  1.68s/it]
  1%|▏         | 147/11346 [04:07<5:15:46,  1.69s/it]
  1%|▏         | 148/11346 [04:09<5:16:13,  1.69s/it]
  1%|▏         | 149/11346 [04:11<5:15:22,  1.69s/it]
  1%|▏         | 150/11346 [04:13<5:16:59,  1.70s/it]
  1%|▏         | 151/11346 [04:14<5:17:05,  1.70s/it]
  1%|▏         | 152/11346 [04:16<5:16:55,  1.70s/it]
  1%|▏         | 153/11346 [04:18<5:17:29,  1.70s/it]
  1%|▏         | 154/11346 [04:19<5:17:43,  1.70s/it]
  1%|▏         | 155/11346 [04:21<5:16:45,  1.70s/it]
  1%|▏         | 156/11346 [04:23<5:17:07,  1.70s/it]
  1%|▏         | 157/11346 [04:24<5:17:52,  1.70s/it]
  1%|▏         | 158/11346 [04:26<5:16:59,  1.70s/it]
  1%|▏         | 159/11346 [04:28<5:17:13,  1.70s/it]
  1%|▏         | 160/11346 [04:30<5:17:42,  1.70s/it]
  1%|▏         | 161/11346 [04:31<5:17:20,  1.70s/it]
  1%|▏         | 162/11346 [04:33<5:16:54,  1.70s/it]
  1%|▏         | 163/11346 [04:35<5:15:58,  1.70s/it]
  1%|▏         | 164/11346 [04:36<5:16:33,  1.70s/it]
  1%|▏         | 165/11346 [04:38<5:16:46,  1.70s/it]
  1%|▏         | 166/11346 [04:40<5:16:18,  1.70s/it]
  1%|▏         | 167/11346 [04:41<5:15:53,  1.70s/it]
  1%|▏         | 168/11346 [04:43<5:15:33,  1.69s/it]
  1%|▏         | 169/11346 [04:45<5:16:19,  1.70s/it]
  1%|▏         | 170/11346 [04:47<5:15:30,  1.69s/it]
  2%|▏         | 171/11346 [04:48<5:15:01,  1.69s/it]
  2%|▏         | 172/11346 [04:50<5:15:21,  1.69s/it]
  2%|▏         | 173/11346 [04:52<5:14:48,  1.69s/it]
  2%|▏         | 174/11346 [04:53<5:16:03,  1.70s/it]
  2%|▏         | 175/11346 [04:55<5:15:36,  1.70s/it]
  2%|▏         | 176/11346 [04:57<5:15:10,  1.69s/it]
  2%|▏         | 177/11346 [04:58<5:15:37,  1.70s/it]
  2%|▏         | 178/11346 [05:00<5:15:06,  1.69s/it]
  2%|▏         | 179/11346 [05:02<5:14:42,  1.69s/it]
  2%|▏         | 180/11346 [05:03<5:15:07,  1.69s/it]
  2%|▏         | 181/11346 [05:05<5:15:02,  1.69s/it]
  2%|▏         | 182/11346 [05:07<5:14:59,  1.69s/it]
  2%|▏         | 183/11346 [05:09<5:14:40,  1.69s/it]
  2%|▏         | 184/11346 [05:10<5:15:53,  1.70s/it]
  2%|▏         | 185/11346 [05:12<5:15:11,  1.69s/it]
  2%|▏         | 186/11346 [05:14<5:14:53,  1.69s/it]
  2%|▏         | 187/11346 [05:15<5:15:43,  1.70s/it]
  2%|▏         | 188/11346 [05:17<5:15:29,  1.70s/it]
  2%|▏         | 189/11346 [05:19<5:15:17,  1.70s/it]
  2%|▏         | 190/11346 [05:20<5:15:03,  1.69s/it]
  2%|▏         | 191/11346 [05:22<5:14:08,  1.69s/it]
  2%|▏         | 192/11346 [05:24<5:14:18,  1.69s/it]
  2%|▏         | 193/11346 [05:25<5:14:10,  1.69s/it]
  2%|▏         | 194/11346 [05:27<5:13:51,  1.69s/it]
  2%|▏         | 195/11346 [05:29<5:13:31,  1.69s/it]
  2%|▏         | 196/11346 [05:31<5:14:23,  1.69s/it]
  2%|▏         | 197/11346 [05:32<5:15:01,  1.70s/it]
  2%|▏         | 198/11346 [05:34<5:15:03,  1.70s/it]
  2%|▏         | 199/11346 [05:36<5:14:37,  1.69s/it]
  2%|▏         | 200/11346 [05:37<5:14:54,  1.70s/it]
  2%|▏         | 201/11346 [05:39<5:14:20,  1.69s/it]
  2%|▏         | 202/11346 [05:41<5:14:27,  1.69s/it]
  2%|▏         | 203/11346 [05:42<5:13:49,  1.69s/it]
  2%|▏         | 204/11346 [05:44<5:13:47,  1.69s/it]
  2%|▏         | 205/11346 [05:46<5:13:39,  1.69s/it]
  2%|▏         | 206/11346 [05:47<5:13:03,  1.69s/it]
  2%|▏         | 207/11346 [05:49<5:13:46,  1.69s/it]
  2%|▏         | 208/11346 [05:51<5:13:31,  1.69s/it]
  2%|▏         | 209/11346 [05:53<5:13:34,  1.69s/it]
  2%|▏         | 210/11346 [05:54<5:13:27,  1.69s/it]
  2%|▏         | 211/11346 [05:56<5:12:29,  1.68s/it]
  2%|▏         | 212/11346 [05:58<5:13:18,  1.69s/it]
  2%|▏         | 213/11346 [05:59<5:12:47,  1.69s/it]
  2%|▏         | 214/11346 [06:01<5:12:54,  1.69s/it]
  2%|▏         | 215/11346 [06:03<5:12:27,  1.68s/it]
  2%|▏         | 216/11346 [06:04<5:13:25,  1.69s/it]
  2%|▏         | 217/11346 [06:06<5:13:28,  1.69s/it]
  2%|▏         | 218/11346 [06:08<5:13:45,  1.69s/it]
  2%|▏         | 219/11346 [06:09<5:12:47,  1.69s/it]
  2%|▏         | 220/11346 [06:11<5:13:05,  1.69s/it]
  2%|▏         | 221/11346 [06:13<5:13:41,  1.69s/it]
  2%|▏         | 222/11346 [06:14<5:13:08,  1.69s/it]
  2%|▏         | 223/11346 [06:16<5:12:37,  1.69s/it]
  2%|▏         | 224/11346 [06:18<5:12:56,  1.69s/it]
  2%|▏         | 225/11346 [06:20<5:12:02,  1.68s/it]
  2%|▏         | 226/11346 [06:21<5:12:05,  1.68s/it]
  2%|▏         | 227/11346 [06:23<5:13:03,  1.69s/it]
  2%|▏         | 228/11346 [06:25<5:12:56,  1.69s/it]
  2%|▏         | 229/11346 [06:26<5:12:24,  1.69s/it]
  2%|▏         | 230/11346 [06:28<5:12:29,  1.69s/it]
  2%|▏         | 231/11346 [06:30<5:12:28,  1.69s/it]
  2%|▏         | 232/11346 [06:31<5:12:35,  1.69s/it]
  2%|▏         | 233/11346 [06:33<5:13:07,  1.69s/it]
  2%|▏         | 234/11346 [06:35<5:12:34,  1.69s/it]
  2%|▏         | 235/11346 [06:36<5:12:19,  1.69s/it]
  2%|▏         | 236/11346 [06:38<5:12:52,  1.69s/it]
  2%|▏         | 237/11346 [06:40<5:12:53,  1.69s/it]
  2%|▏         | 238/11346 [06:41<5:13:25,  1.69s/it]
  2%|▏         | 239/11346 [06:43<5:13:20,  1.69s/it]
  2%|▏         | 240/11346 [06:45<5:11:49,  1.68s/it]
  2%|▏         | 241/11346 [06:47<5:12:48,  1.69s/it]
  2%|▏         | 242/11346 [06:48<5:12:47,  1.69s/it]
  2%|▏         | 243/11346 [06:50<5:13:03,  1.69s/it]
  2%|▏         | 244/11346 [06:52<5:13:03,  1.69s/it]
  2%|▏         | 245/11346 [06:53<5:12:45,  1.69s/it]
  2%|▏         | 246/11346 [06:55<5:12:53,  1.69s/it]
  2%|▏         | 247/11346 [06:57<5:12:42,  1.69s/it]
  2%|▏         | 248/11346 [06:58<5:11:35,  1.68s/it]
  2%|▏         | 249/11346 [07:00<5:11:17,  1.68s/it]
  2%|▏         | 250/11346 [07:02<5:11:09,  1.68s/it]
  2%|▏         | 251/11346 [07:03<5:11:38,  1.69s/it]
  2%|▏         | 252/11346 [07:05<5:11:13,  1.68s/it]
  2%|▏         | 253/11346 [07:07<5:11:38,  1.69s/it]
  2%|▏         | 254/11346 [07:08<5:10:30,  1.68s/it]
  2%|▏         | 255/11346 [07:10<5:10:14,  1.68s/it]
  2%|▏         | 256/11346 [07:12<5:10:15,  1.68s/it]
  2%|▏         | 257/11346 [07:13<5:09:47,  1.68s/it]
  2%|▏         | 258/11346 [07:15<5:10:30,  1.68s/it]
  2%|▏         | 259/11346 [07:17<5:11:11,  1.68s/it]
  2%|▏         | 260/11346 [07:19<5:10:54,  1.68s/it]
  2%|▏         | 261/11346 [07:20<5:10:47,  1.68s/it]
  2%|▏         | 262/11346 [07:22<5:10:19,  1.68s/it]
  2%|▏         | 263/11346 [07:24<5:10:42,  1.68s/it]
  2%|▏         | 264/11346 [07:25<5:10:37,  1.68s/it]
  2%|▏         | 265/11346 [07:27<5:10:42,  1.68s/it]
  2%|▏         | 266/11346 [07:29<5:11:00,  1.68s/it]
  2%|▏         | 267/11346 [07:30<5:11:00,  1.68s/it]
  2%|▏         | 268/11346 [07:32<5:11:15,  1.69s/it]
  2%|▏         | 269/11346 [07:34<5:10:59,  1.68s/it]
  2%|▏         | 270/11346 [07:35<5:11:20,  1.69s/it]
  2%|▏         | 271/11346 [07:37<5:11:27,  1.69s/it]
  2%|▏         | 272/11346 [07:39<5:11:38,  1.69s/it]
  2%|▏         | 273/11346 [07:40<5:11:37,  1.69s/it]
  2%|▏         | 274/11346 [07:42<5:11:42,  1.69s/it]
  2%|▏         | 275/11346 [07:44<5:11:02,  1.69s/it]
  2%|▏         | 276/11346 [07:46<5:10:47,  1.68s/it]
  2%|▏         | 277/11346 [07:47<5:11:04,  1.69s/it]
  2%|▏         | 278/11346 [07:49<5:11:01,  1.69s/it]
  2%|▏         | 279/11346 [07:51<5:10:57,  1.69s/it]
  2%|▏         | 280/11346 [07:52<5:10:04,  1.68s/it]
  2%|▏         | 281/11346 [07:54<5:09:03,  1.68s/it]
  2%|▏         | 282/11346 [07:56<5:10:14,  1.68s/it]
  2%|▏         | 283/11346 [07:57<5:10:30,  1.68s/it]
  3%|▎         | 284/11346 [07:59<5:09:50,  1.68s/it]
  3%|▎         | 285/11346 [08:01<5:09:45,  1.68s/it]
  3%|▎         | 286/11346 [08:02<5:11:25,  1.69s/it]
  3%|▎         | 287/11346 [08:04<5:11:07,  1.69s/it]
  3%|▎         | 288/11346 [08:06<5:10:45,  1.69s/it]
  3%|▎         | 289/11346 [08:07<5:10:22,  1.68s/it]
  3%|▎         | 290/11346 [08:09<5:09:34,  1.68s/it]
  3%|▎         | 291/11346 [08:11<5:09:15,  1.68s/it]
  3%|▎         | 292/11346 [08:12<5:08:31,  1.67s/it]
  3%|▎         | 293/11346 [08:14<5:09:04,  1.68s/it]
  3%|▎         | 294/11346 [08:16<5:09:19,  1.68s/it]
  3%|▎         | 295/11346 [08:17<5:09:08,  1.68s/it]
  3%|▎         | 296/11346 [08:19<5:09:44,  1.68s/it]
  3%|▎         | 297/11346 [08:21<5:10:02,  1.68s/it]
  3%|▎         | 298/11346 [08:23<5:10:14,  1.68s/it]
  3%|▎         | 299/11346 [08:24<5:10:42,  1.69s/it]
  3%|▎         | 300/11346 [08:26<5:10:02,  1.68s/it]
  3%|▎         | 301/11346 [08:28<5:09:29,  1.68s/it]
  3%|▎         | 302/11346 [08:29<5:09:15,  1.68s/it]
  3%|▎         | 303/11346 [08:31<5:09:55,  1.68s/it]
  3%|▎         | 304/11346 [08:33<5:09:50,  1.68s/it]
  3%|▎         | 305/11346 [08:34<5:08:58,  1.68s/it]
  3%|▎         | 306/11346 [08:36<5:08:28,  1.68s/it]
  3%|▎         | 307/11346 [08:38<5:08:26,  1.68s/it]
  3%|▎         | 308/11346 [08:39<5:09:05,  1.68s/it]
  3%|▎         | 309/11346 [08:41<5:09:25,  1.68s/it]
  3%|▎         | 310/11346 [08:43<5:09:32,  1.68s/it]
  3%|▎         | 311/11346 [08:44<5:09:56,  1.69s/it]
  3%|▎         | 312/11346 [08:46<5:09:44,  1.68s/it]
  3%|▎         | 313/11346 [08:48<5:09:14,  1.68s/it]
  3%|▎         | 314/11346 [08:49<5:09:10,  1.68s/it]
  3%|▎         | 315/11346 [08:51<5:08:30,  1.68s/it]
  3%|▎         | 316/11346 [08:53<5:08:03,  1.68s/it]
  3%|▎         | 317/11346 [08:54<5:08:00,  1.68s/it]
  3%|▎         | 318/11346 [08:56<5:07:26,  1.67s/it]
  3%|▎         | 319/11346 [08:58<5:07:48,  1.67s/it]
  3%|▎         | 320/11346 [08:59<5:07:47,  1.67s/it]
  3%|▎         | 321/11346 [09:01<5:07:54,  1.68s/it]
  3%|▎         | 322/11346 [09:03<5:08:15,  1.68s/it]
  3%|▎         | 323/11346 [09:05<5:08:35,  1.68s/it]
  3%|▎         | 324/11346 [09:06<5:07:48,  1.68s/it]
  3%|▎         | 325/11346 [09:08<5:07:33,  1.67s/it]
  3%|▎         | 326/11346 [09:10<5:07:48,  1.68s/it]
  3%|▎         | 327/11346 [09:11<5:08:02,  1.68s/it]
  3%|▎         | 328/11346 [09:13<5:08:07,  1.68s/it]
  3%|▎         | 329/11346 [09:15<5:08:07,  1.68s/it]
  3%|▎         | 330/11346 [09:16<5:07:30,  1.67s/it]
  3%|▎         | 331/11346 [09:18<5:07:48,  1.68s/it]
  3%|▎         | 332/11346 [09:20<5:07:49,  1.68s/it]
  3%|▎         | 333/11346 [09:21<5:07:30,  1.68s/it]
  3%|▎         | 334/11346 [09:23<5:07:12,  1.67s/it]
  3%|▎         | 335/11346 [09:25<5:07:25,  1.68s/it]
  3%|▎         | 336/11346 [09:26<5:08:11,  1.68s/it]
  3%|▎         | 337/11346 [09:28<5:08:37,  1.68s/it]
  3%|▎         | 338/11346 [09:30<5:08:59,  1.68s/it]
  3%|▎         | 339/11346 [09:31<5:09:20,  1.69s/it]
  3%|▎         | 340/11346 [09:33<5:08:50,  1.68s/it]
  3%|▎         | 341/11346 [09:35<5:08:26,  1.68s/it]
  3%|▎         | 342/11346 [09:36<5:08:35,  1.68s/it]
  3%|▎         | 343/11346 [09:38<5:07:49,  1.68s/it]
  3%|▎         | 344/11346 [09:40<5:08:33,  1.68s/it]
  3%|▎         | 345/11346 [09:41<5:08:50,  1.68s/it]
  3%|▎         | 346/11346 [09:43<5:07:25,  1.68s/it]
  3%|▎         | 347/11346 [09:45<5:07:46,  1.68s/it]
  3%|▎         | 348/11346 [09:46<5:07:39,  1.68s/it]
  3%|▎         | 349/11346 [09:48<5:07:15,  1.68s/it]
  3%|▎         | 350/11346 [09:50<5:07:38,  1.68s/it]
  3%|▎         | 351/11346 [09:51<5:06:52,  1.67s/it]
  3%|▎         | 352/11346 [09:53<5:07:16,  1.68s/it]
  3%|▎         | 353/11346 [09:55<5:07:09,  1.68s/it]
  3%|▎         | 354/11346 [09:57<5:06:57,  1.68s/it]
  3%|▎         | 355/11346 [09:58<5:07:25,  1.68s/it]
  3%|▎         | 356/11346 [10:00<5:07:07,  1.68s/it]
  3%|▎         | 357/11346 [10:02<5:07:48,  1.68s/it]
  3%|▎         | 358/11346 [10:03<5:07:20,  1.68s/it]
  3%|▎         | 359/11346 [10:05<5:07:33,  1.68s/it]
  3%|▎         | 360/11346 [10:07<5:08:02,  1.68s/it]
  3%|▎         | 361/11346 [10:08<5:07:04,  1.68s/it]
  3%|▎         | 362/11346 [10:10<5:07:14,  1.68s/it]
  3%|▎         | 363/11346 [10:12<5:06:59,  1.68s/it]
  3%|▎         | 364/11346 [10:13<5:07:16,  1.68s/it]
  3%|▎         | 365/11346 [10:15<5:06:41,  1.68s/it]
  3%|▎         | 366/11346 [10:17<5:06:39,  1.68s/it]
  3%|▎         | 367/11346 [10:18<5:06:27,  1.67s/it]
  3%|▎         | 368/11346 [10:20<5:06:38,  1.68s/it]
  3%|▎         | 369/11346 [10:22<5:06:19,  1.67s/it]
  3%|▎         | 370/11346 [10:23<5:06:04,  1.67s/it]
  3%|▎         | 371/11346 [10:25<5:06:24,  1.68s/it]
  3%|▎         | 372/11346 [10:27<5:06:36,  1.68s/it]
  3%|▎         | 373/11346 [10:28<5:06:09,  1.67s/it]
  3%|▎         | 374/11346 [10:30<5:06:08,  1.67s/it]
  3%|▎         | 375/11346 [10:32<5:06:13,  1.67s/it]
  3%|▎         | 376/11346 [10:33<5:06:27,  1.68s/it]
  3%|▎         | 377/11346 [10:35<5:06:12,  1.67s/it]
  3%|▎         | 378/11346 [10:37<5:05:59,  1.67s/it]
  3%|▎         | 379/11346 [10:38<5:06:33,  1.68s/it]
  3%|▎         | 380/11346 [10:40<5:06:22,  1.68s/it]
  3%|▎         | 381/11346 [10:42<5:05:56,  1.67s/it]
  3%|▎         | 382/11346 [10:43<5:05:40,  1.67s/it]
  3%|▎         | 383/11346 [10:45<5:04:57,  1.67s/it]
  3%|▎         | 384/11346 [10:47<5:05:19,  1.67s/it]
  3%|▎         | 385/11346 [10:48<5:06:05,  1.68s/it]
  3%|▎         | 386/11346 [10:50<5:05:20,  1.67s/it]
  3%|▎         | 387/11346 [10:52<5:05:16,  1.67s/it]
  3%|▎         | 388/11346 [10:53<5:05:16,  1.67s/it]
  3%|▎         | 389/11346 [10:55<5:05:18,  1.67s/it]
  3%|▎         | 390/11346 [10:57<5:05:54,  1.68s/it]
  3%|▎         | 391/11346 [10:59<5:06:06,  1.68s/it]
  3%|▎         | 392/11346 [11:00<5:05:30,  1.67s/it]
  3%|▎         | 393/11346 [11:02<5:05:10,  1.67s/it]
  3%|▎         | 394/11346 [11:04<5:05:04,  1.67s/it]
  3%|▎         | 395/11346 [11:05<5:05:07,  1.67s/it]
  3%|▎         | 396/11346 [11:07<5:04:45,  1.67s/it]
  3%|▎         | 397/11346 [11:09<5:04:28,  1.67s/it]
  4%|▎         | 398/11346 [11:10<5:04:42,  1.67s/it]
  4%|▎         | 399/11346 [11:12<5:05:08,  1.67s/it]
  4%|▎         | 400/11346 [11:14<5:05:40,  1.68s/it]
  4%|▎         | 401/11346 [11:15<5:05:43,  1.68s/it]
  4%|▎         | 402/11346 [11:17<5:05:13,  1.67s/it]
  4%|▎         | 403/11346 [11:19<5:05:09,  1.67s/it]
  4%|▎         | 404/11346 [11:20<5:05:18,  1.67s/it]
  4%|▎         | 405/11346 [11:22<5:05:18,  1.67s/it]
  4%|▎         | 406/11346 [11:24<5:04:55,  1.67s/it]
  4%|▎         | 407/11346 [11:25<5:04:46,  1.67s/it]
  4%|▎         | 408/11346 [11:27<5:05:44,  1.68s/it]
  4%|▎         | 409/11346 [11:29<5:05:17,  1.67s/it]
  4%|▎         | 410/11346 [11:30<5:05:06,  1.67s/it]
  4%|▎         | 411/11346 [11:32<5:04:43,  1.67s/it]
  4%|▎         | 412/11346 [11:34<5:04:45,  1.67s/it]
  4%|▎         | 413/11346 [11:35<5:04:46,  1.67s/it]
  4%|▎         | 414/11346 [11:37<5:04:21,  1.67s/it]
  4%|▎         | 415/11346 [11:39<5:04:18,  1.67s/it]
  4%|▎         | 416/11346 [11:40<5:04:29,  1.67s/it]
  4%|▎         | 417/11346 [11:42<5:21:42,  1.77s/it]
  4%|▎         | 418/11346 [11:44<5:16:59,  1.74s/it]
  4%|▎         | 419/11346 [11:46<5:13:55,  1.72s/it]
  4%|▎         | 420/11346 [11:47<5:10:47,  1.71s/it]
  4%|▎         | 421/11346 [11:49<5:08:47,  1.70s/it]
  4%|▎         | 422/11346 [11:51<5:07:36,  1.69s/it]
  4%|▎         | 423/11346 [11:52<5:06:24,  1.68s/it]
  4%|▎         | 424/11346 [11:54<5:05:53,  1.68s/it]
  4%|▎         | 425/11346 [11:56<5:04:52,  1.68s/it]
  4%|▍         | 426/11346 [11:57<5:04:15,  1.67s/it]
  4%|▍         | 427/11346 [11:59<5:04:23,  1.67s/it]
  4%|▍         | 428/11346 [12:01<5:04:02,  1.67s/it]
  4%|▍         | 429/11346 [12:02<5:03:42,  1.67s/it]
  4%|▍         | 430/11346 [12:04<5:03:38,  1.67s/it]
  4%|▍         | 431/11346 [12:06<5:03:47,  1.67s/it]
  4%|▍         | 432/11346 [12:07<5:03:50,  1.67s/it]
  4%|▍         | 433/11346 [12:09<5:03:43,  1.67s/it]
  4%|▍         | 434/11346 [12:11<5:03:07,  1.67s/it]
  4%|▍         | 435/11346 [12:12<5:03:32,  1.67s/it]
  4%|▍         | 436/11346 [12:14<5:04:27,  1.67s/it]
  4%|▍         | 437/11346 [12:16<5:03:56,  1.67s/it]
  4%|▍         | 438/11346 [12:17<5:03:46,  1.67s/it]
  4%|▍         | 439/11346 [12:19<5:03:49,  1.67s/it]
  4%|▍         | 440/11346 [12:21<5:03:23,  1.67s/it]
  4%|▍         | 441/11346 [12:22<5:03:29,  1.67s/it]
  4%|▍         | 442/11346 [12:24<5:03:01,  1.67s/it]
  4%|▍         | 443/11346 [12:26<5:02:27,  1.66s/it]
  4%|▍         | 444/11346 [12:27<5:02:31,  1.67s/it]
  4%|▍         | 445/11346 [12:29<5:02:39,  1.67s/it]
  4%|▍         | 446/11346 [12:31<5:02:30,  1.67s/it]
  4%|▍         | 447/11346 [12:32<5:03:08,  1.67s/it]
  4%|▍         | 448/11346 [12:34<5:03:54,  1.67s/it]
  4%|▍         | 449/11346 [12:36<5:04:07,  1.67s/it]
  4%|▍         | 450/11346 [12:37<5:04:10,  1.67s/it]
  4%|▍         | 451/11346 [12:39<5:03:38,  1.67s/it]
  4%|▍         | 452/11346 [12:41<5:03:29,  1.67s/it]
  4%|▍         | 453/11346 [12:42<5:03:48,  1.67s/it]
  4%|▍         | 454/11346 [12:44<5:03:09,  1.67s/it]
  4%|▍         | 455/11346 [12:46<5:02:57,  1.67s/it]
  4%|▍         | 456/11346 [12:47<5:02:57,  1.67s/it]
  4%|▍         | 457/11346 [12:49<5:04:03,  1.68s/it]
  4%|▍         | 458/11346 [12:51<5:03:59,  1.68s/it]
  4%|▍         | 459/11346 [12:53<5:21:15,  1.77s/it]
  4%|▍         | 460/11346 [12:55<5:17:49,  1.75s/it]
  4%|▍         | 461/11346 [12:56<5:13:41,  1.73s/it]
  4%|▍         | 462/11346 [12:58<5:13:35,  1.73s/it]
  4%|▍         | 463/11346 [13:00<5:10:36,  1.71s/it]
  4%|▍         | 464/11346 [13:01<5:08:09,  1.70s/it]
  4%|▍         | 465/11346 [13:03<5:06:18,  1.69s/it]
  4%|▍         | 466/11346 [13:05<5:04:55,  1.68s/it]
  4%|▍         | 467/11346 [13:06<5:04:22,  1.68s/it]
  4%|▍         | 468/11346 [13:08<5:04:22,  1.68s/it]
  4%|▍         | 469/11346 [13:10<5:04:45,  1.68s/it]
  4%|▍         | 470/11346 [13:11<5:04:50,  1.68s/it]
  4%|▍         | 471/11346 [13:13<5:04:24,  1.68s/it]
  4%|▍         | 472/11346 [13:15<5:04:33,  1.68s/it]
  4%|▍         | 473/11346 [13:16<5:04:15,  1.68s/it]
  4%|▍         | 474/11346 [13:18<5:03:50,  1.68s/it]
  4%|▍         | 475/11346 [13:20<5:02:43,  1.67s/it]
  4%|▍         | 476/11346 [13:21<5:03:09,  1.67s/it]
  4%|▍         | 477/11346 [13:23<5:02:54,  1.67s/it]
  4%|▍         | 478/11346 [13:25<5:02:49,  1.67s/it]
  4%|▍         | 479/11346 [13:26<5:02:38,  1.67s/it]
  4%|▍         | 480/11346 [13:28<5:02:39,  1.67s/it]
  4%|▍         | 481/11346 [13:30<5:02:47,  1.67s/it]
  4%|▍         | 482/11346 [13:31<5:01:53,  1.67s/it]
  4%|▍         | 483/11346 [13:33<5:02:10,  1.67s/it]
  4%|▍         | 484/11346 [13:35<5:02:28,  1.67s/it]
  4%|▍         | 485/11346 [13:36<5:02:10,  1.67s/it]
  4%|▍         | 486/11346 [13:38<5:02:15,  1.67s/it]
  4%|▍         | 487/11346 [13:40<5:02:22,  1.67s/it]
  4%|▍         | 488/11346 [13:41<5:02:32,  1.67s/it]
  4%|▍         | 489/11346 [13:43<5:03:04,  1.67s/it]
  4%|▍         | 490/11346 [13:45<5:02:53,  1.67s/it]
  4%|▍         | 491/11346 [13:46<5:02:19,  1.67s/it]
  4%|▍         | 492/11346 [13:48<5:02:10,  1.67s/it]
  4%|▍         | 493/11346 [13:50<5:01:41,  1.67s/it]
  4%|▍         | 494/11346 [13:51<5:02:02,  1.67s/it]
  4%|▍         | 495/11346 [13:53<5:01:21,  1.67s/it]
  4%|▍         | 496/11346 [13:55<5:01:26,  1.67s/it]
  4%|▍         | 497/11346 [13:56<5:01:15,  1.67s/it]
  4%|▍         | 498/11346 [13:58<5:01:42,  1.67s/it]
  4%|▍         | 499/11346 [14:00<5:01:44,  1.67s/it]
  4%|▍         | 500/11346 [14:01<5:02:14,  1.67s/it]
                                                     
  4%|▍         | 500/11346 [14:01<5:02:14,  1.67s/it]
  4%|▍         | 501/11346 [14:03<5:02:32,  1.67s/it]
  4%|▍         | 502/11346 [14:05<5:02:02,  1.67s/it]
  4%|▍         | 503/11346 [14:06<5:01:34,  1.67s/it]
  4%|▍         | 504/11346 [14:08<5:02:24,  1.67s/it]
  4%|▍         | 505/11346 [14:10<5:02:05,  1.67s/it]
  4%|▍         | 506/11346 [14:11<5:02:21,  1.67s/it]
  4%|▍         | 507/11346 [14:13<5:01:52,  1.67s/it]
  4%|▍         | 508/11346 [14:15<5:01:57,  1.67s/it]
  4%|▍         | 509/11346 [14:16<5:02:53,  1.68s/it]
  4%|▍         | 510/11346 [14:18<5:02:34,  1.68s/it]
  5%|▍         | 511/11346 [14:20<5:02:36,  1.68s/it]
  5%|▍         | 512/11346 [14:22<5:02:55,  1.68s/it]
  5%|▍         | 513/11346 [14:23<5:03:14,  1.68s/it]
  5%|▍         | 514/11346 [14:25<5:02:12,  1.67s/it]
  5%|▍         | 515/11346 [14:27<5:02:19,  1.67s/it]
  5%|▍         | 516/11346 [14:28<5:01:39,  1.67s/it]
  5%|▍         | 517/11346 [14:30<5:01:15,  1.67s/it]
  5%|▍         | 518/11346 [14:32<5:01:18,  1.67s/it]
  5%|▍         | 519/11346 [14:33<5:00:55,  1.67s/it]
  5%|▍         | 520/11346 [14:35<5:02:20,  1.68s/it]
  5%|▍         | 521/11346 [14:37<5:03:12,  1.68s/it]
  5%|▍         | 522/11346 [14:38<5:03:18,  1.68s/it]
  5%|▍         | 523/11346 [14:40<5:03:23,  1.68s/it]
  5%|▍         | 524/11346 [14:42<5:03:28,  1.68s/it]
  5%|▍         | 525/11346 [14:43<5:03:05,  1.68s/it]
  5%|▍         | 526/11346 [14:45<5:03:17,  1.68s/it]
  5%|▍         | 527/11346 [14:47<5:03:29,  1.68s/it]
  5%|▍         | 528/11346 [14:48<5:03:48,  1.69s/it]
  5%|▍         | 529/11346 [14:50<5:03:55,  1.69s/it]
  5%|▍         | 530/11346 [14:52<5:04:50,  1.69s/it]
  5%|▍         | 531/11346 [14:53<5:04:48,  1.69s/it]
  5%|▍         | 532/11346 [14:55<5:04:26,  1.69s/it]
  5%|▍         | 533/11346 [14:57<5:04:27,  1.69s/it]
  5%|▍         | 534/11346 [14:59<5:04:52,  1.69s/it]
  5%|▍         | 535/11346 [15:00<5:04:59,  1.69s/it]
  5%|▍         | 536/11346 [15:02<5:04:05,  1.69s/it]
  5%|▍         | 537/11346 [15:04<5:03:08,  1.68s/it]
  5%|▍         | 538/11346 [15:05<5:02:24,  1.68s/it]
  5%|▍         | 539/11346 [15:07<5:01:50,  1.68s/it]
  5%|▍         | 540/11346 [15:09<5:01:27,  1.67s/it]
  5%|▍         | 541/11346 [15:10<5:01:09,  1.67s/it]
  5%|▍         | 542/11346 [15:12<5:00:10,  1.67s/it]
  5%|▍         | 543/11346 [15:14<5:00:16,  1.67s/it]
  5%|▍         | 544/11346 [15:15<5:00:51,  1.67s/it]
  5%|▍         | 545/11346 [15:17<5:01:04,  1.67s/it]
  5%|▍         | 546/11346 [15:19<5:01:08,  1.67s/it]
  5%|▍         | 547/11346 [15:20<5:00:45,  1.67s/it]
  5%|▍         | 548/11346 [15:22<5:01:03,  1.67s/it]
  5%|▍         | 549/11346 [15:24<5:00:43,  1.67s/it]
  5%|▍         | 550/11346 [15:25<5:00:52,  1.67s/it]
  5%|▍         | 551/11346 [15:27<5:00:40,  1.67s/it]
  5%|▍         | 552/11346 [15:29<5:00:01,  1.67s/it]
  5%|▍         | 553/11346 [15:30<4:59:21,  1.66s/it]
  5%|▍         | 554/11346 [15:32<4:59:19,  1.66s/it]
  5%|▍         | 555/11346 [15:34<4:59:11,  1.66s/it]
  5%|▍         | 556/11346 [15:35<4:59:44,  1.67s/it]
  5%|▍         | 557/11346 [15:37<4:59:52,  1.67s/it]
  5%|▍         | 558/11346 [15:39<4:59:58,  1.67s/it]
  5%|▍         | 559/11346 [15:40<4:59:58,  1.67s/it]
  5%|▍         | 560/11346 [15:42<4:59:56,  1.67s/it]
  5%|▍         | 561/11346 [15:44<4:59:27,  1.67s/it]
  5%|▍         | 562/11346 [15:45<4:59:50,  1.67s/it]
  5%|▍         | 563/11346 [15:47<4:59:42,  1.67s/it]
  5%|▍         | 564/11346 [15:49<4:59:51,  1.67s/it]
  5%|▍         | 565/11346 [15:50<4:59:37,  1.67s/it]
  5%|▍         | 566/11346 [15:52<4:59:35,  1.67s/it]
  5%|▍         | 567/11346 [15:54<4:59:40,  1.67s/it]
  5%|▌         | 568/11346 [15:55<4:59:11,  1.67s/it]
  5%|▌         | 569/11346 [15:57<4:58:53,  1.66s/it]
  5%|▌         | 570/11346 [15:59<4:58:52,  1.66s/it]
  5%|▌         | 571/11346 [16:00<4:58:37,  1.66s/it]
  5%|▌         | 572/11346 [16:02<4:58:21,  1.66s/it]
  5%|▌         | 573/11346 [16:04<4:58:34,  1.66s/it]
  5%|▌         | 574/11346 [16:05<4:58:30,  1.66s/it]
  5%|▌         | 575/11346 [16:07<4:58:31,  1.66s/it]
  5%|▌         | 576/11346 [16:09<4:58:09,  1.66s/it]
  5%|▌         | 577/11346 [16:10<4:58:28,  1.66s/it]
  5%|▌         | 578/11346 [16:12<4:59:12,  1.67s/it]
  5%|▌         | 579/11346 [16:14<4:58:45,  1.66s/it]
  5%|▌         | 580/11346 [16:15<4:58:44,  1.66s/it]
  5%|▌         | 581/11346 [16:17<4:58:33,  1.66s/it]
  5%|▌         | 582/11346 [16:19<4:58:39,  1.66s/it]
  5%|▌         | 583/11346 [16:20<4:58:37,  1.66s/it]
  5%|▌         | 584/11346 [16:22<4:59:01,  1.67s/it]
  5%|▌         | 585/11346 [16:24<4:59:00,  1.67s/it]
  5%|▌         | 586/11346 [16:25<4:58:48,  1.67s/it]
  5%|▌         | 587/11346 [16:27<4:59:31,  1.67s/it]
  5%|▌         | 588/11346 [16:29<4:59:17,  1.67s/it]
  5%|▌         | 589/11346 [16:30<4:59:01,  1.67s/it]
  5%|▌         | 590/11346 [16:32<4:59:00,  1.67s/it]
  5%|▌         | 591/11346 [16:34<4:58:32,  1.67s/it]
  5%|▌         | 592/11346 [16:35<4:58:14,  1.66s/it]
  5%|▌         | 593/11346 [16:37<4:58:02,  1.66s/it]
  5%|▌         | 594/11346 [16:39<4:58:02,  1.66s/it]
  5%|▌         | 595/11346 [16:40<4:58:33,  1.67s/it]
  5%|▌         | 596/11346 [16:42<4:58:51,  1.67s/it]
  5%|▌         | 597/11346 [16:44<4:59:06,  1.67s/it]
  5%|▌         | 598/11346 [16:45<4:59:38,  1.67s/it]
  5%|▌         | 599/11346 [16:47<4:59:16,  1.67s/it]
  5%|▌         | 600/11346 [16:49<4:58:32,  1.67s/it]
  5%|▌         | 601/11346 [16:50<4:58:21,  1.67s/it]
  5%|▌         | 602/11346 [16:52<4:58:30,  1.67s/it]
  5%|▌         | 603/11346 [16:54<4:58:01,  1.66s/it]
  5%|▌         | 604/11346 [16:55<4:57:49,  1.66s/it]
  5%|▌         | 605/11346 [16:57<4:58:10,  1.67s/it]
  5%|▌         | 606/11346 [16:59<4:57:58,  1.66s/it]
  5%|▌         | 607/11346 [17:00<4:57:37,  1.66s/it]
  5%|▌         | 608/11346 [17:02<4:58:11,  1.67s/it]
  5%|▌         | 609/11346 [17:04<4:57:44,  1.66s/it]
  5%|▌         | 610/11346 [17:05<4:58:14,  1.67s/it]
  5%|▌         | 611/11346 [17:07<4:58:20,  1.67s/it]
  5%|▌         | 612/11346 [17:09<4:58:21,  1.67s/it]
  5%|▌         | 613/11346 [17:10<4:57:54,  1.67s/it]
  5%|▌         | 614/11346 [17:12<4:57:44,  1.66s/it]
  5%|▌         | 615/11346 [17:14<4:57:21,  1.66s/it]
  5%|▌         | 616/11346 [17:15<4:57:10,  1.66s/it]
  5%|▌         | 617/11346 [17:17<4:57:06,  1.66s/it]
  5%|▌         | 618/11346 [17:19<4:57:04,  1.66s/it]
  5%|▌         | 619/11346 [17:20<4:56:53,  1.66s/it]
  5%|▌         | 620/11346 [17:22<4:56:52,  1.66s/it]
  5%|▌         | 621/11346 [17:24<4:57:12,  1.66s/it]
  5%|▌         | 622/11346 [17:25<4:57:38,  1.67s/it]
  5%|▌         | 623/11346 [17:27<4:57:06,  1.66s/it]
  5%|▌         | 624/11346 [17:29<4:57:06,  1.66s/it]
  6%|▌         | 625/11346 [17:30<4:57:03,  1.66s/it]
  6%|▌         | 626/11346 [17:32<4:57:12,  1.66s/it]
  6%|▌         | 627/11346 [17:34<4:57:32,  1.67s/it]
  6%|▌         | 628/11346 [17:35<4:57:31,  1.67s/it]
  6%|▌         | 629/11346 [17:37<4:57:00,  1.66s/it]
  6%|▌         | 630/11346 [17:39<4:57:02,  1.66s/it]
  6%|▌         | 631/11346 [17:40<4:57:57,  1.67s/it]
  6%|▌         | 632/11346 [17:42<4:59:03,  1.67s/it]
  6%|▌         | 633/11346 [17:44<4:59:56,  1.68s/it]
  6%|▌         | 634/11346 [17:45<5:00:30,  1.68s/it]
  6%|▌         | 635/11346 [17:47<5:01:20,  1.69s/it]
  6%|▌         | 636/11346 [17:49<5:01:53,  1.69s/it]
  6%|▌         | 637/11346 [17:50<5:02:01,  1.69s/it]
  6%|▌         | 638/11346 [17:52<5:01:55,  1.69s/it]
  6%|▌         | 639/11346 [17:54<5:02:05,  1.69s/it]
  6%|▌         | 640/11346 [17:55<5:01:57,  1.69s/it]
  6%|▌         | 641/11346 [17:57<5:02:03,  1.69s/it]
  6%|▌         | 642/11346 [17:59<5:01:59,  1.69s/it]
  6%|▌         | 643/11346 [18:01<5:01:45,  1.69s/it]
  6%|▌         | 644/11346 [18:02<5:01:34,  1.69s/it]
  6%|▌         | 645/11346 [18:04<5:01:09,  1.69s/it]
  6%|▌         | 646/11346 [18:06<5:01:04,  1.69s/it]
  6%|▌         | 647/11346 [18:07<5:00:31,  1.69s/it]
  6%|▌         | 648/11346 [18:09<4:59:12,  1.68s/it]
  6%|▌         | 649/11346 [18:11<4:58:50,  1.68s/it]
  6%|▌         | 650/11346 [18:12<4:57:53,  1.67s/it]
  6%|▌         | 651/11346 [18:14<4:57:24,  1.67s/it]
  6%|▌         | 652/11346 [18:16<4:57:12,  1.67s/it]
  6%|▌         | 653/11346 [18:17<4:56:21,  1.66s/it]
  6%|▌         | 654/11346 [18:19<4:57:55,  1.67s/it]
  6%|▌         | 655/11346 [18:21<4:58:01,  1.67s/it]
  6%|▌         | 656/11346 [18:22<4:58:46,  1.68s/it]
  6%|▌         | 657/11346 [18:24<4:59:30,  1.68s/it]
  6%|▌         | 658/11346 [18:26<5:00:04,  1.68s/it]
  6%|▌         | 659/11346 [18:27<5:00:31,  1.69s/it]
  6%|▌         | 660/11346 [18:29<5:00:33,  1.69s/it]
  6%|▌         | 661/11346 [18:31<5:00:11,  1.69s/it]
  6%|▌         | 662/11346 [18:32<5:00:16,  1.69s/it]
  6%|▌         | 663/11346 [18:34<5:00:07,  1.69s/it]
  6%|▌         | 664/11346 [18:36<4:59:52,  1.68s/it]
  6%|▌         | 665/11346 [18:37<4:59:42,  1.68s/it]
  6%|▌         | 666/11346 [18:39<5:00:08,  1.69s/it]
  6%|▌         | 667/11346 [18:41<4:59:57,  1.69s/it]
  6%|▌         | 668/11346 [18:43<5:00:15,  1.69s/it]
  6%|▌         | 669/11346 [18:44<5:00:14,  1.69s/it]
  6%|▌         | 670/11346 [18:46<5:00:31,  1.69s/it]
  6%|▌         | 671/11346 [18:48<5:00:12,  1.69s/it]
  6%|▌         | 672/11346 [18:49<4:59:53,  1.69s/it]
  6%|▌         | 673/11346 [18:51<5:00:01,  1.69s/it]
  6%|▌         | 674/11346 [18:53<5:00:30,  1.69s/it]
  6%|▌         | 675/11346 [18:54<5:00:06,  1.69s/it]
  6%|▌         | 676/11346 [18:56<4:59:33,  1.68s/it]
  6%|▌         | 677/11346 [18:58<4:59:41,  1.69s/it]
  6%|▌         | 678/11346 [18:59<4:59:27,  1.68s/it]
  6%|▌         | 679/11346 [19:01<4:59:22,  1.68s/it]
  6%|▌         | 680/11346 [19:03<4:58:49,  1.68s/it]
  6%|▌         | 681/11346 [19:04<4:58:44,  1.68s/it]
  6%|▌         | 682/11346 [19:06<4:59:19,  1.68s/it]
  6%|▌         | 683/11346 [19:08<5:00:28,  1.69s/it]
  6%|▌         | 684/11346 [19:10<5:00:22,  1.69s/it]
  6%|▌         | 685/11346 [19:11<5:00:50,  1.69s/it]
  6%|▌         | 686/11346 [19:13<5:00:45,  1.69s/it]
  6%|▌         | 687/11346 [19:15<5:00:53,  1.69s/it]
  6%|▌         | 688/11346 [19:16<5:00:21,  1.69s/it]
  6%|▌         | 689/11346 [19:18<4:59:46,  1.69s/it]
  6%|▌         | 690/11346 [19:20<4:59:40,  1.69s/it]
  6%|▌         | 691/11346 [19:21<4:59:23,  1.69s/it]
  6%|▌         | 692/11346 [19:23<4:59:26,  1.69s/it]
  6%|▌         | 693/11346 [19:25<4:59:19,  1.69s/it]
  6%|▌         | 694/11346 [19:26<4:58:49,  1.68s/it]
  6%|▌         | 695/11346 [19:28<4:58:41,  1.68s/it]
  6%|▌         | 696/11346 [19:30<4:58:39,  1.68s/it]
  6%|▌         | 697/11346 [19:31<4:58:06,  1.68s/it]
  6%|▌         | 698/11346 [19:33<4:58:06,  1.68s/it]
  6%|▌         | 699/11346 [19:35<4:57:56,  1.68s/it]
  6%|▌         | 700/11346 [19:36<4:58:00,  1.68s/it]
  6%|▌         | 701/11346 [19:38<4:58:51,  1.68s/it]
  6%|▌         | 702/11346 [19:40<4:59:12,  1.69s/it]
  6%|▌         | 703/11346 [19:42<4:59:37,  1.69s/it]
  6%|▌         | 704/11346 [19:43<5:00:02,  1.69s/it]
  6%|▌         | 705/11346 [19:45<4:59:14,  1.69s/it]
  6%|▌         | 706/11346 [19:47<4:58:56,  1.69s/it]
  6%|▌         | 707/11346 [19:48<4:58:31,  1.68s/it]
  6%|▌         | 708/11346 [19:50<4:57:54,  1.68s/it]
  6%|▌         | 709/11346 [19:52<4:58:07,  1.68s/it]
  6%|▋         | 710/11346 [19:53<4:57:46,  1.68s/it]
  6%|▋         | 711/11346 [19:55<4:57:57,  1.68s/it]
  6%|▋         | 712/11346 [19:57<4:58:10,  1.68s/it]
  6%|▋         | 713/11346 [19:58<4:57:58,  1.68s/it]
  6%|▋         | 714/11346 [20:00<4:57:53,  1.68s/it]
  6%|▋         | 715/11346 [20:02<4:57:37,  1.68s/it]
  6%|▋         | 716/11346 [20:03<4:57:45,  1.68s/it]
  6%|▋         | 717/11346 [20:05<4:57:19,  1.68s/it]
  6%|▋         | 718/11346 [20:07<4:57:44,  1.68s/it]
  6%|▋         | 719/11346 [20:08<4:56:51,  1.68s/it]
  6%|▋         | 720/11346 [20:10<4:56:15,  1.67s/it]
  6%|▋         | 721/11346 [20:12<4:55:17,  1.67s/it]
  6%|▋         | 722/11346 [20:13<4:54:35,  1.66s/it]
  6%|▋         | 723/11346 [20:15<4:54:33,  1.66s/it]
  6%|▋         | 724/11346 [20:17<4:54:14,  1.66s/it]
  6%|▋         | 725/11346 [20:18<4:54:05,  1.66s/it]
  6%|▋         | 726/11346 [20:20<4:55:30,  1.67s/it]
  6%|▋         | 727/11346 [20:22<4:56:16,  1.67s/it]
  6%|▋         | 728/11346 [20:23<4:56:08,  1.67s/it]
  6%|▋         | 729/11346 [20:25<4:56:47,  1.68s/it]
  6%|▋         | 730/11346 [20:27<4:57:13,  1.68s/it]
  6%|▋         | 731/11346 [20:28<4:56:30,  1.68s/it]
  6%|▋         | 732/11346 [20:30<4:56:32,  1.68s/it]
  6%|▋         | 733/11346 [20:32<4:55:43,  1.67s/it]
  6%|▋         | 734/11346 [20:33<4:55:09,  1.67s/it]
  6%|▋         | 735/11346 [20:35<4:54:32,  1.67s/it]
  6%|▋         | 736/11346 [20:37<4:54:17,  1.66s/it]
  6%|▋         | 737/11346 [20:38<4:53:46,  1.66s/it]
  7%|▋         | 738/11346 [20:40<4:53:33,  1.66s/it]
  7%|▋         | 739/11346 [20:42<4:54:24,  1.67s/it]
  7%|▋         | 740/11346 [20:43<4:54:11,  1.66s/it]
  7%|▋         | 741/11346 [20:45<4:54:15,  1.66s/it]
  7%|▋         | 742/11346 [20:47<4:53:57,  1.66s/it]
  7%|▋         | 743/11346 [20:48<4:53:34,  1.66s/it]
  7%|▋         | 744/11346 [20:50<4:53:25,  1.66s/it]
  7%|▋         | 745/11346 [20:52<4:53:35,  1.66s/it]
  7%|▋         | 746/11346 [20:53<4:53:32,  1.66s/it]
  7%|▋         | 747/11346 [20:55<4:53:38,  1.66s/it]
  7%|▋         | 748/11346 [20:57<4:53:14,  1.66s/it]
  7%|▋         | 749/11346 [20:58<4:53:01,  1.66s/it]
  7%|▋         | 750/11346 [21:00<4:53:11,  1.66s/it]
  7%|▋         | 751/11346 [21:02<4:53:15,  1.66s/it]
  7%|▋         | 752/11346 [21:03<4:52:58,  1.66s/it]
  7%|▋         | 753/11346 [21:05<4:54:12,  1.67s/it]
  7%|▋         | 754/11346 [21:07<4:55:08,  1.67s/it]
  7%|▋         | 755/11346 [21:08<4:54:37,  1.67s/it]
  7%|▋         | 756/11346 [21:10<4:54:09,  1.67s/it]
  7%|▋         | 757/11346 [21:12<4:53:54,  1.67s/it]
  7%|▋         | 758/11346 [21:13<4:53:53,  1.67s/it]
  7%|▋         | 759/11346 [21:15<4:53:47,  1.66s/it]
  7%|▋         | 760/11346 [21:17<4:53:31,  1.66s/it]
  7%|▋         | 761/11346 [21:18<4:53:03,  1.66s/it]
  7%|▋         | 762/11346 [21:20<4:53:05,  1.66s/it]
  7%|▋         | 763/11346 [21:22<4:54:17,  1.67s/it]
  7%|▋         | 764/11346 [21:23<4:54:43,  1.67s/it]
  7%|▋         | 765/11346 [21:25<4:55:05,  1.67s/it]
  7%|▋         | 766/11346 [21:27<4:55:33,  1.68s/it]
  7%|▋         | 767/11346 [21:28<4:55:35,  1.68s/it]
  7%|▋         | 768/11346 [21:30<4:55:51,  1.68s/it]
  7%|▋         | 769/11346 [21:32<4:56:18,  1.68s/it]
  7%|▋         | 770/11346 [21:33<4:55:49,  1.68s/it]
  7%|▋         | 771/11346 [21:35<4:55:55,  1.68s/it]
  7%|▋         | 772/11346 [21:37<4:55:46,  1.68s/it]
  7%|▋         | 773/11346 [21:38<4:55:42,  1.68s/it]
  7%|▋         | 774/11346 [21:40<4:55:06,  1.67s/it]
  7%|▋         | 775/11346 [21:42<4:54:05,  1.67s/it]
  7%|▋         | 776/11346 [21:43<4:53:21,  1.67s/it]
  7%|▋         | 777/11346 [21:45<4:54:19,  1.67s/it]
  7%|▋         | 778/11346 [21:47<4:54:38,  1.67s/it]
  7%|▋         | 779/11346 [21:48<4:54:30,  1.67s/it]
  7%|▋         | 780/11346 [21:50<4:55:19,  1.68s/it]
  7%|▋         | 781/11346 [21:52<4:55:11,  1.68s/it]
  7%|▋         | 782/11346 [21:54<4:54:34,  1.67s/it]
  7%|▋         | 783/11346 [21:55<4:54:44,  1.67s/it]
  7%|▋         | 784/11346 [21:57<4:54:42,  1.67s/it]
  7%|▋         | 785/11346 [21:59<4:54:49,  1.68s/it]
  7%|▋         | 786/11346 [22:00<4:55:14,  1.68s/it]
  7%|▋         | 787/11346 [22:02<4:55:26,  1.68s/it]
  7%|▋         | 788/11346 [22:04<4:55:16,  1.68s/it]
  7%|▋         | 789/11346 [22:05<4:54:52,  1.68s/it]
  7%|▋         | 790/11346 [22:07<4:54:51,  1.68s/it]
  7%|▋         | 791/11346 [22:09<4:55:31,  1.68s/it]
  7%|▋         | 792/11346 [22:10<4:55:13,  1.68s/it]
  7%|▋         | 793/11346 [22:12<4:54:55,  1.68s/it]
  7%|▋         | 794/11346 [22:14<4:54:53,  1.68s/it]
  7%|▋         | 795/11346 [22:15<4:54:41,  1.68s/it]
  7%|▋         | 796/11346 [22:17<4:54:32,  1.68s/it]
  7%|▋         | 797/11346 [22:19<4:54:57,  1.68s/it]
  7%|▋         | 798/11346 [22:20<4:55:02,  1.68s/it]
  7%|▋         | 799/11346 [22:22<4:55:15,  1.68s/it]
  7%|▋         | 800/11346 [22:24<4:55:11,  1.68s/it]
  7%|▋         | 801/11346 [22:25<4:54:19,  1.67s/it]
  7%|▋         | 802/11346 [22:27<4:53:24,  1.67s/it]
  7%|▋         | 803/11346 [22:29<4:52:46,  1.67s/it]
  7%|▋         | 804/11346 [22:30<4:52:32,  1.67s/it]
  7%|▋         | 805/11346 [22:32<4:52:00,  1.66s/it]
  7%|▋         | 806/11346 [22:34<4:52:02,  1.66s/it]
  7%|▋         | 807/11346 [22:35<4:53:44,  1.67s/it]
  7%|▋         | 808/11346 [22:37<4:54:03,  1.67s/it]
  7%|▋         | 809/11346 [22:39<4:54:23,  1.68s/it]
  7%|▋         | 810/11346 [22:40<4:53:49,  1.67s/it]
  7%|▋         | 811/11346 [22:42<4:53:01,  1.67s/it]
  7%|▋         | 812/11346 [22:44<4:52:27,  1.67s/it]
  7%|▋         | 813/11346 [22:45<4:52:18,  1.67s/it]
  7%|▋         | 814/11346 [22:47<4:52:02,  1.66s/it]
  7%|▋         | 815/11346 [22:49<4:51:27,  1.66s/it]
  7%|▋         | 816/11346 [22:50<4:51:19,  1.66s/it]
  7%|▋         | 817/11346 [22:52<4:51:09,  1.66s/it]
  7%|▋         | 818/11346 [22:54<4:51:02,  1.66s/it]
  7%|▋         | 819/11346 [22:55<4:51:01,  1.66s/it]
  7%|▋         | 820/11346 [22:57<4:51:02,  1.66s/it]
  7%|▋         | 821/11346 [22:59<5:08:01,  1.76s/it]
  7%|▋         | 822/11346 [23:01<5:02:52,  1.73s/it]
  7%|▋         | 823/11346 [23:02<4:59:30,  1.71s/it]
  7%|▋         | 824/11346 [23:04<4:56:55,  1.69s/it]
  7%|▋         | 825/11346 [23:06<4:55:07,  1.68s/it]
  7%|▋         | 826/11346 [23:07<4:53:39,  1.67s/it]
  7%|▋         | 827/11346 [23:09<4:52:41,  1.67s/it]
  7%|▋         | 828/11346 [23:11<4:51:52,  1.66s/it]
  7%|▋         | 829/11346 [23:12<4:51:17,  1.66s/it]
  7%|▋         | 830/11346 [23:14<4:52:05,  1.67s/it]
  7%|▋         | 831/11346 [23:16<4:52:35,  1.67s/it]
  7%|▋         | 832/11346 [23:17<4:52:52,  1.67s/it]
  7%|▋         | 833/11346 [23:19<4:52:25,  1.67s/it]
  7%|▋         | 834/11346 [23:21<4:52:15,  1.67s/it]
  7%|▋         | 835/11346 [23:22<4:51:22,  1.66s/it]
  7%|▋         | 836/11346 [23:24<4:50:47,  1.66s/it]
  7%|▋         | 837/11346 [23:26<4:50:29,  1.66s/it]
  7%|▋         | 838/11346 [23:27<4:50:16,  1.66s/it]
  7%|▋         | 839/11346 [23:29<4:49:57,  1.66s/it]
  7%|▋         | 840/11346 [23:31<4:50:23,  1.66s/it]
  7%|▋         | 841/11346 [23:32<4:50:18,  1.66s/it]
  7%|▋         | 842/11346 [23:34<4:50:20,  1.66s/it]
  7%|▋         | 843/11346 [23:35<4:49:52,  1.66s/it]
  7%|▋         | 844/11346 [23:37<4:49:43,  1.66s/it]
  7%|▋         | 845/11346 [23:39<4:49:51,  1.66s/it]
  7%|▋         | 846/11346 [23:40<4:49:52,  1.66s/it]
  7%|▋         | 847/11346 [23:42<4:49:52,  1.66s/it]
  7%|▋         | 848/11346 [23:44<4:50:00,  1.66s/it]
  7%|▋         | 849/11346 [23:45<4:50:11,  1.66s/it]
  7%|▋         | 850/11346 [23:47<4:50:22,  1.66s/it]
  8%|▊         | 851/11346 [23:49<4:49:56,  1.66s/it]
  8%|▊         | 852/11346 [23:50<4:49:53,  1.66s/it]
  8%|▊         | 853/11346 [23:52<4:50:04,  1.66s/it]
  8%|▊         | 854/11346 [23:54<4:50:03,  1.66s/it]
  8%|▊         | 855/11346 [23:55<4:49:53,  1.66s/it]
  8%|▊         | 856/11346 [23:57<4:49:48,  1.66s/it]
  8%|▊         | 857/11346 [23:59<4:49:27,  1.66s/it]
  8%|▊         | 858/11346 [24:00<4:49:06,  1.65s/it]
  8%|▊         | 859/11346 [24:02<4:49:20,  1.66s/it]
  8%|▊         | 860/11346 [24:04<4:49:13,  1.65s/it]
  8%|▊         | 861/11346 [24:05<4:49:15,  1.66s/it]
  8%|▊         | 862/11346 [24:07<4:49:24,  1.66s/it]
  8%|▊         | 863/11346 [24:09<4:49:17,  1.66s/it]
  8%|▊         | 864/11346 [24:10<4:49:44,  1.66s/it]
  8%|▊         | 865/11346 [24:12<4:49:21,  1.66s/it]
  8%|▊         | 866/11346 [24:14<4:49:42,  1.66s/it]
  8%|▊         | 867/11346 [24:15<4:52:05,  1.67s/it]
  8%|▊         | 868/11346 [24:17<4:51:23,  1.67s/it]
  8%|▊         | 869/11346 [24:19<4:50:54,  1.67s/it]
  8%|▊         | 870/11346 [24:20<4:50:04,  1.66s/it]
  8%|▊         | 871/11346 [24:22<4:49:53,  1.66s/it]
  8%|▊         | 872/11346 [24:24<4:49:23,  1.66s/it]
  8%|▊         | 873/11346 [24:25<4:49:25,  1.66s/it]
  8%|▊         | 874/11346 [24:27<4:49:14,  1.66s/it]
  8%|▊         | 875/11346 [24:29<4:50:23,  1.66s/it]
  8%|▊         | 876/11346 [24:30<4:51:25,  1.67s/it]
  8%|▊         | 877/11346 [24:32<4:52:09,  1.67s/it]
  8%|▊         | 878/11346 [24:34<4:52:06,  1.67s/it]
  8%|▊         | 879/11346 [24:35<4:52:55,  1.68s/it]
  8%|▊         | 880/11346 [24:37<4:52:37,  1.68s/it]
  8%|▊         | 881/11346 [24:39<4:52:20,  1.68s/it]
  8%|▊         | 882/11346 [24:40<4:52:30,  1.68s/it]
  8%|▊         | 883/11346 [24:42<4:52:46,  1.68s/it]
  8%|▊         | 884/11346 [24:44<4:52:39,  1.68s/it]
  8%|▊         | 885/11346 [24:45<4:52:46,  1.68s/it]
  8%|▊         | 886/11346 [24:47<4:52:38,  1.68s/it]
  8%|▊         | 887/11346 [24:49<4:52:23,  1.68s/it]
  8%|▊         | 888/11346 [24:50<4:52:35,  1.68s/it]
  8%|▊         | 889/11346 [24:52<4:53:13,  1.68s/it]
  8%|▊         | 890/11346 [24:54<4:53:15,  1.68s/it]
  8%|▊         | 891/11346 [24:55<4:53:01,  1.68s/it]
  8%|▊         | 892/11346 [24:57<4:52:58,  1.68s/it]
  8%|▊         | 893/11346 [24:59<4:52:40,  1.68s/it]
  8%|▊         | 894/11346 [25:01<4:52:50,  1.68s/it]
  8%|▊         | 895/11346 [25:02<4:52:28,  1.68s/it]
  8%|▊         | 896/11346 [25:04<4:52:07,  1.68s/it]
  8%|▊         | 897/11346 [25:06<4:52:10,  1.68s/it]
  8%|▊         | 898/11346 [25:07<4:51:51,  1.68s/it]
  8%|▊         | 899/11346 [25:09<4:51:58,  1.68s/it]
  8%|▊         | 900/11346 [25:11<4:51:59,  1.68s/it]
  8%|▊         | 901/11346 [25:12<4:51:45,  1.68s/it]
  8%|▊         | 902/11346 [25:14<4:51:35,  1.68s/it]
  8%|▊         | 903/11346 [25:16<4:51:44,  1.68s/it]
  8%|▊         | 904/11346 [25:17<4:51:24,  1.67s/it]
  8%|▊         | 905/11346 [25:19<4:51:19,  1.67s/it]
  8%|▊         | 906/11346 [25:21<4:51:39,  1.68s/it]
  8%|▊         | 907/11346 [25:22<4:51:33,  1.68s/it]
  8%|▊         | 908/11346 [25:24<4:51:36,  1.68s/it]
  8%|▊         | 909/11346 [25:26<4:51:44,  1.68s/it]
  8%|▊         | 910/11346 [25:27<4:51:53,  1.68s/it]
  8%|▊         | 911/11346 [25:29<4:51:47,  1.68s/it]
  8%|▊         | 912/11346 [25:31<4:51:38,  1.68s/it]
  8%|▊         | 913/11346 [25:32<4:51:39,  1.68s/it]
  8%|▊         | 914/11346 [25:34<4:51:36,  1.68s/it]
  8%|▊         | 915/11346 [25:36<4:51:14,  1.68s/it]
  8%|▊         | 916/11346 [25:37<4:51:21,  1.68s/it]
  8%|▊         | 917/11346 [25:39<4:51:13,  1.68s/it]
  8%|▊         | 918/11346 [25:41<4:51:13,  1.68s/it]
  8%|▊         | 919/11346 [25:42<4:51:11,  1.68s/it]
  8%|▊         | 920/11346 [25:44<4:50:58,  1.67s/it]
  8%|▊         | 921/11346 [25:46<4:50:41,  1.67s/it]
  8%|▊         | 922/11346 [25:47<4:50:43,  1.67s/it]
  8%|▊         | 923/11346 [25:49<4:50:50,  1.67s/it]
  8%|▊         | 924/11346 [25:51<4:50:00,  1.67s/it]
  8%|▊         | 925/11346 [25:52<4:49:06,  1.66s/it]
  8%|▊         | 926/11346 [25:54<4:48:28,  1.66s/it]
  8%|▊         | 927/11346 [25:56<4:49:21,  1.67s/it]
  8%|▊         | 928/11346 [25:57<4:49:41,  1.67s/it]
  8%|▊         | 929/11346 [25:59<4:49:47,  1.67s/it]
  8%|▊         | 930/11346 [26:01<4:50:23,  1.67s/it]
  8%|▊         | 931/11346 [26:02<4:50:44,  1.67s/it]
  8%|▊         | 932/11346 [26:04<4:50:59,  1.68s/it]
  8%|▊         | 933/11346 [26:06<4:51:13,  1.68s/it]
  8%|▊         | 934/11346 [26:08<4:51:35,  1.68s/it]
  8%|▊         | 935/11346 [26:09<4:51:26,  1.68s/it]
  8%|▊         | 936/11346 [26:11<4:51:53,  1.68s/it]
  8%|▊         | 937/11346 [26:13<4:51:13,  1.68s/it]
  8%|▊         | 938/11346 [26:14<4:51:23,  1.68s/it]
  8%|▊         | 939/11346 [26:16<4:50:59,  1.68s/it]
  8%|▊         | 940/11346 [26:18<4:51:04,  1.68s/it]
  8%|▊         | 941/11346 [26:19<4:50:42,  1.68s/it]
  8%|▊         | 942/11346 [26:21<4:50:25,  1.67s/it]
  8%|▊         | 943/11346 [26:23<4:50:25,  1.68s/it]
  8%|▊         | 944/11346 [26:24<4:50:40,  1.68s/it]
  8%|▊         | 945/11346 [26:26<4:50:35,  1.68s/it]
  8%|▊         | 946/11346 [26:28<4:50:44,  1.68s/it]
  8%|▊         | 947/11346 [26:29<4:50:50,  1.68s/it]
  8%|▊         | 948/11346 [26:31<4:50:59,  1.68s/it]
  8%|▊         | 949/11346 [26:33<4:50:40,  1.68s/it]
  8%|▊         | 950/11346 [26:34<4:50:36,  1.68s/it]
  8%|▊         | 951/11346 [26:36<4:50:42,  1.68s/it]
  8%|▊         | 952/11346 [26:38<4:50:17,  1.68s/it]
  8%|▊         | 953/11346 [26:39<4:50:12,  1.68s/it]
  8%|▊         | 954/11346 [26:41<4:50:11,  1.68s/it]
  8%|▊         | 955/11346 [26:43<4:50:10,  1.68s/it]
  8%|▊         | 956/11346 [26:44<4:50:10,  1.68s/it]
  8%|▊         | 957/11346 [26:46<4:50:18,  1.68s/it]
  8%|▊         | 958/11346 [26:48<4:50:12,  1.68s/it]
  8%|▊         | 959/11346 [26:49<4:49:16,  1.67s/it]
  8%|▊         | 960/11346 [26:51<4:48:29,  1.67s/it]
  8%|▊         | 961/11346 [26:53<4:48:08,  1.66s/it]
  8%|▊         | 962/11346 [26:54<4:47:46,  1.66s/it]
  8%|▊         | 963/11346 [26:56<4:47:19,  1.66s/it]
  8%|▊         | 964/11346 [26:58<4:46:57,  1.66s/it]
  9%|▊         | 965/11346 [26:59<4:46:44,  1.66s/it]
  9%|▊         | 966/11346 [27:01<4:46:35,  1.66s/it]
  9%|▊         | 967/11346 [27:03<4:46:30,  1.66s/it]
  9%|▊         | 968/11346 [27:04<4:46:48,  1.66s/it]
  9%|▊         | 969/11346 [27:06<4:47:47,  1.66s/it]
  9%|▊         | 970/11346 [27:08<4:48:03,  1.67s/it]
  9%|▊         | 971/11346 [27:09<4:48:29,  1.67s/it]
  9%|▊         | 972/11346 [27:11<4:48:49,  1.67s/it]
  9%|▊         | 973/11346 [27:13<4:49:01,  1.67s/it]
  9%|▊         | 974/11346 [27:14<4:49:08,  1.67s/it]
  9%|▊         | 975/11346 [27:16<4:49:18,  1.67s/it]
  9%|▊         | 976/11346 [27:18<4:49:31,  1.68s/it]
  9%|▊         | 977/11346 [27:19<4:49:28,  1.68s/it]
  9%|▊         | 978/11346 [27:21<4:49:20,  1.67s/it]
  9%|▊         | 979/11346 [27:23<4:49:30,  1.68s/it]
  9%|▊         | 980/11346 [27:24<4:49:38,  1.68s/it]
  9%|▊         | 981/11346 [27:26<4:49:21,  1.68s/it]
  9%|▊         | 982/11346 [27:28<4:49:27,  1.68s/it]
  9%|▊         | 983/11346 [27:29<4:49:37,  1.68s/it]
  9%|▊         | 984/11346 [27:31<4:49:31,  1.68s/it]
  9%|▊         | 985/11346 [27:33<4:49:24,  1.68s/it]
  9%|▊         | 986/11346 [27:34<4:49:08,  1.67s/it]
  9%|▊         | 987/11346 [27:36<4:49:02,  1.67s/it]
  9%|▊         | 988/11346 [27:38<4:49:15,  1.68s/it]
  9%|▊         | 989/11346 [27:39<4:48:55,  1.67s/it]
  9%|▊         | 990/11346 [27:41<4:48:56,  1.67s/it]
  9%|▊         | 991/11346 [27:43<4:48:55,  1.67s/it]
  9%|▊         | 992/11346 [27:45<4:49:00,  1.67s/it]
  9%|▉         | 993/11346 [27:46<4:49:17,  1.68s/it]
  9%|▉         | 994/11346 [27:48<4:49:04,  1.68s/it]
  9%|▉         | 995/11346 [27:50<4:49:10,  1.68s/it]
  9%|▉         | 996/11346 [27:51<4:49:07,  1.68s/it]
  9%|▉         | 997/11346 [27:53<4:49:17,  1.68s/it]
  9%|▉         | 998/11346 [27:55<4:49:20,  1.68s/it]
  9%|▉         | 999/11346 [27:56<4:49:15,  1.68s/it]
  9%|▉         | 1000/11346 [27:58<4:49:43,  1.68s/it]
                                                      
  9%|▉         | 1000/11346 [27:58<4:49:43,  1.68s/it][INFO|trainer.py:3662] 2024-06-04 01:29:49,519 >> ***** Running Evaluation *****
+[INFO|trainer.py:3664] 2024-06-04 01:29:49,519 >>   Num examples = 1840
+[INFO|trainer.py:3667] 2024-06-04 01:29:49,519 >>   Batch size = 48
+{'loss': 5.1118, 'grad_norm': 0.8546377420425415, 'learning_rate': 8.816009873931059e-05, 'epoch': 0.13}
+{'loss': 3.406, 'grad_norm': 0.8593683838844299, 'learning_rate': 9.59831475011252e-05, 'epoch': 0.26}
+
+
  0%|          | 0/39 [00:00<?, ?it/s]
+
  5%|▌         | 2/39 [00:02<00:54,  1.48s/it]
+
  8%|▊         | 3/39 [00:05<01:15,  2.10s/it]
+
 10%|█         | 4/39 [00:08<01:24,  2.42s/it]
+
 13%|█▎        | 5/39 [00:11<01:28,  2.61s/it]
+
 15%|█▌        | 6/39 [00:14<01:29,  2.72s/it]
+
 18%|█▊        | 7/39 [00:17<01:29,  2.80s/it]
+
 21%|██        | 8/39 [00:20<01:28,  2.85s/it]
+
 23%|██▎       | 9/39 [00:23<01:26,  2.88s/it]
+
 26%|██▌       | 10/39 [00:26<01:24,  2.91s/it]
+
 28%|██▊       | 11/39 [00:29<01:21,  2.92s/it]
+
 31%|███       | 12/39 [00:32<01:19,  2.93s/it]
+
 33%|███▎      | 13/39 [00:35<01:16,  2.94s/it]
+
 36%|███▌      | 14/39 [00:38<01:13,  2.95s/it]
+
 38%|███▊      | 15/39 [00:41<01:10,  2.95s/it]
+
 41%|████      | 16/39 [00:44<01:07,  2.95s/it]
+
 44%|████▎     | 17/39 [00:47<01:04,  2.95s/it]
+
 46%|████▌     | 18/39 [00:50<01:01,  2.94s/it]
+
 49%|████▊     | 19/39 [00:53<00:58,  2.94s/it]
+
 51%|█████▏    | 20/39 [00:56<00:55,  2.94s/it]
+
 54%|█████▍    | 21/39 [00:59<00:52,  2.94s/it]
+
 56%|█████▋    | 22/39 [01:02<00:50,  2.94s/it]
+
 59%|█████▉    | 23/39 [01:04<00:47,  2.94s/it]
+
 62%|██████▏   | 24/39 [01:07<00:44,  2.94s/it]
+
 64%|██████▍   | 25/39 [01:10<00:41,  2.94s/it]
+
 67%|██████▋   | 26/39 [01:13<00:38,  2.94s/it]
+
 69%|██████▉   | 27/39 [01:16<00:35,  2.94s/it]
+
 72%|███████▏  | 28/39 [01:19<00:32,  2.94s/it]
+
 74%|███████▍  | 29/39 [01:22<00:29,  2.94s/it]
+
 77%|███████▋  | 30/39 [01:25<00:26,  2.94s/it]
+
 79%|███████▉  | 31/39 [01:28<00:23,  2.94s/it]
+
 82%|████████▏ | 32/39 [01:31<00:20,  2.95s/it]
+
 85%|████████▍ | 33/39 [01:34<00:17,  2.95s/it]
+
 87%|████████▋ | 34/39 [01:37<00:14,  2.96s/it]
+
 90%|████████▉ | 35/39 [01:40<00:11,  2.96s/it]
+
 92%|█████████▏| 36/39 [01:43<00:08,  2.96s/it]
+
 95%|█████████▍| 37/39 [01:46<00:05,  2.96s/it]
+
 97%|█████████▋| 38/39 [01:49<00:02,  2.93s/it]
+
+
                                               
                                                      
+
  9%|▉         | 1000/11346 [30:07<4:49:43,  1.68s/it]
+
                                               [INFO|trainer.py:3353] 2024-06-04 01:31:58,493 >> Saving model checkpoint to ./training_outputs_job_116987_1_04-06_01-01/checkpoint-1000
+[INFO|configuration_utils.py:471] 2024-06-04 01:31:58,504 >> Configuration saved in ./training_outputs_job_116987_1_04-06_01-01/checkpoint-1000/config.json
+[INFO|configuration_utils.py:705] 2024-06-04 01:31:58,509 >> Configuration saved in ./training_outputs_job_116987_1_04-06_01-01/checkpoint-1000/generation_config.json
+[INFO|modeling_utils.py:2592] 2024-06-04 01:31:59,417 >> Model weights saved in ./training_outputs_job_116987_1_04-06_01-01/checkpoint-1000/model.safetensors
+[INFO|tokenization_utils_base.py:2503] 2024-06-04 01:31:59,430 >> tokenizer config file saved in ./training_outputs_job_116987_1_04-06_01-01/checkpoint-1000/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2512] 2024-06-04 01:31:59,434 >> Special tokens file saved in ./training_outputs_job_116987_1_04-06_01-01/checkpoint-1000/special_tokens_map.json
+[INFO|tokenization_utils_base.py:2503] 2024-06-04 01:32:01,652 >> tokenizer config file saved in ./training_outputs_job_116987_1_04-06_01-01/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2512] 2024-06-04 01:32:01,656 >> Special tokens file saved in ./training_outputs_job_116987_1_04-06_01-01/special_tokens_map.json
+/home/dshteyma/miniconda3/lib/python3.9/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
+  warnings.warn('Was asked to gather along dimension 0, but all '

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b29f07972e09b7e99ddb0e5808e0d88a17501a11c554772a8caa6699722d71ee
+oid sha256:85a2564ab57b7b37bcdcfd2e43c48a69cbb78c4a211d0fbf6d340595efdecd4d
 size 272123144

pip_freeze.txt

@@ -0,0 +1,330 @@
+absl-py==2.1.0
+accelerate==0.26.1
+aiofiles==23.2.1
+aiohttp==3.8.6
+aiosignal==1.3.1
+altair==5.3.0
+annotated-types==0.6.0
+antlr4-python3-runtime==4.9.3
+anyio==4.0.0
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==2.4.0
+astunparse==1.6.3
+async-lru==2.0.4
+async-timeout==4.0.3
+attrs==23.1.0
+auto-gptq==0.6.0
+Babel==2.13.0
+backcall @ file:///home/ktietz/src/ci/backcall_1611930011877/work
+beartype==0.17.2
+beautifulsoup4==4.12.2
+bitsandbytes==0.43.1
+bleach==6.1.0
+blis==0.7.11
+brotlipy==0.7.0
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2023.7.22
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.0
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+colorama @ file:///tmp/build/80754af9/colorama_1607707115595/work
+coloredlogs==15.0.1
+comm==0.1.4
+conda==4.12.0
+conda-content-trust @ file:///tmp/build/80754af9/conda-content-trust_1617045594566/work
+conda-package-handling @ file:///tmp/build/80754af9/conda-package-handling_1649105784853/work
+confection==0.1.4
+contextlib2==21.6.0
+contexttimer==0.3.3
+contourpy==1.1.1
+cryptography @ file:///tmp/build/80754af9/cryptography_1639414572950/work
+cycler==0.12.1
+cymem==2.0.8
+dataclasses-json==0.6.4
+DataProperty==1.0.1
+datasets==2.19.1
+debugpy==1.8.0
+decorator @ file:///opt/conda/conda-bld/decorator_1643638310831/work
+defusedxml==0.7.1
+dill==0.3.7
+dnspython==2.6.1
+docstring_parser==0.16
+dos2unix==1
+einops==0.8.0
+eval_type_backport==0.2.0
+evaluate==0.4.1
+exceptiongroup==1.1.3
+executing==2.0.0
+fastapi==0.111.0
+fastapi-cli==0.0.2
+fastchat==0.1.0
+fastjsonschema==2.18.1
+ffmpy==0.3.2
+filelock==3.12.4
+fire==0.5.0
+flash-attn==2.5.8
+flatbuffers==23.5.26
+fonttools==4.43.1
+fqdn==1.5.1
+frozenlist==1.4.0
+fschat==0.2.36
+fsspec==2023.6.0
+gast==0.5.4
+gekko==1.0.6
+globals==0.3.36
+google-auth==2.27.0
+google-auth-oauthlib==1.2.0
+google-pasta==0.2.0
+gradio==4.29.0
+gradio_client==0.16.1
+greenlet==3.0.3
+grpcio==1.60.1
+h11==0.14.0
+h5py==3.10.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.22.2
+humanfriendly==10.0
+hydra-core==1.3.2
+hydra-joblib-launcher==1.2.0
+hydra-submitit-launcher==1.2.0
+idna==3.4
+importlib-metadata==6.8.0
+importlib-resources==6.1.0
+ipykernel==6.25.2
+ipython==8.18.1
+isoduration==20.11.0
+jedi==0.19.1
+Jinja2==3.1.2
+joblib==1.3.2
+json5==0.9.14
+jsonlines==4.0.0
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.19.1
+jsonschema-specifications==2023.7.1
+jupyter-events==0.7.0
+jupyter-lsp==2.2.0
+jupyter_client==8.3.1
+jupyter_core==5.3.2
+jupyter_server==2.7.3
+jupyter_server_terminals==0.4.4
+jupyterlab==4.0.6
+jupyterlab-pygments==0.2.2
+jupyterlab_server==2.25.0
+keras==2.15.0
+kiwisolver==1.4.5
+langchain==0.1.8
+langchain-community==0.0.21
+langchain-core==0.1.25
+langcodes==3.3.0
+langdetect==1.0.9
+langsmith==0.1.5
+libclang==16.0.6
+lxml==5.1.0
+Markdown==3.5.2
+markdown-it-py==3.0.0
+markdown2==2.4.13
+MarkupSafe==2.1.5
+marshmallow==3.20.2
+matplotlib==3.8.0
+matplotlib-inline @ file:///opt/conda/conda-bld/matplotlib-inline_1662014470464/work
+mbstrdecoder==1.1.3
+mdurl==0.1.2
+mistune==3.0.2
+ml-collections==0.1.1
+ml-dtypes==0.2.0
+more-itertools==10.2.0
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.15
+murmurhash==1.0.10
+mypy-extensions==1.0.0
+nbclient==0.8.0
+nbconvert==7.9.2
+nbformat==5.9.2
+nest-asyncio==1.5.8
+networkx==3.1
+nh3==0.2.17
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==7.0.4
+notebook_shim==0.2.3
+numexpr==2.9.0
+numpy==1.26.0
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-ml-py3==7.352.0
+nvidia-nccl-cu12==2.18.1
+nvidia-nvjitlink-cu12==12.2.140
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+opt-einsum==3.3.0
+optimum==1.16.2
+orjson==3.10.3
+overrides==7.4.0
+packaging==23.2
+pandas==2.1.1
+pandocfilters==1.5.0
+parso @ file:///opt/conda/conda-bld/parso_1641458642106/work
+pathvalidate==3.2.0
+patsy==0.5.3
+peft==0.8.2
+pexpect @ file:///tmp/build/80754af9/pexpect_1605563209008/work
+pickleshare @ file:///tmp/build/80754af9/pickleshare_1606932040724/work
+Pillow==10.0.1
+platformdirs==3.11.0
+plotly==5.17.0
+plotly-express==0.4.1
+portalocker==2.8.2
+preshed==3.0.9
+prometheus-client==0.17.1
+prompt-toolkit==3.0.43
+protobuf==3.20.3
+psutil==5.9.5
+ptyprocess @ file:///tmp/build/80754af9/ptyprocess_1609355006118/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl
+pure-eval @ file:///opt/conda/conda-bld/pure_eval_1646925070566/work
+pyarrow==13.0.0
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
+pyasn1-modules==0.3.0
+pybind11==2.11.1
+pycosat==0.6.3
+pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work
+pydantic==2.6.1
+pydantic_core==2.16.2
+pydub==0.25.1
+Pygments==2.16.1
+pyOpenSSL @ file:///opt/conda/conda-bld/pyopenssl_1643788558760/work
+pyparsing==3.1.1
+PySocks @ file:///tmp/build/80754af9/pysocks_1605305812635/work
+pytablewriter==1.2.0
+python-dateutil==2.8.2
+python-dotenv==1.0.1
+python-helper==0.3.74
+python-json-logger==2.0.7
+python-multipart==0.0.9
+pytz==2023.3.post1
+PyYAML==6.0.1
+pyzmq==25.1.1
+referencing==0.30.2
+regex==2023.10.3
+requests==2.31.0
+requests-oauthlib==1.3.1
+responses==0.18.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.7.1
+rotary-embedding-torch==0.5.3
+rouge==1.0.1
+rouge-score==0.1.2
+rpds-py==0.10.4
+rsa==4.9
+ruamel-yaml-conda @ file:///tmp/build/80754af9/ruamel_yaml_1616016711199/work
+ruff==0.4.3
+sacrebleu==2.4.0
+safetensors==0.4.3
+scikit-learn==1.4.1.post1
+scipy==1.11.3
+seaborn==0.13.0
+semantic-version==2.10.0
+Send2Trash==1.8.2
+sentencepiece==0.2.0
+shellingham==1.5.4
+shortuuid==1.0.13
+shtab==1.7.1
+six @ file:///tmp/build/80754af9/six_1644875935023/work
+smart-open==6.4.0
+sniffio==1.3.0
+soupsieve==2.5
+spacy==3.7.4
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+speculative-decoding==0.1.2
+SQLAlchemy==2.0.27
+sqlitedict==2.1.0
+srsly==2.4.8
+stack-data==0.6.3
+starlette==0.37.2
+statsmodels==0.14.0
+submitit==1.5.1
+svgwrite==1.4.3
+sympy==1.12
+tabledata==1.3.3
+tabulate==0.9.0
+tcolorpy==0.1.4
+tenacity==8.2.3
+tensorboard==2.15.1
+tensorboard-data-server==0.7.2
+tensorflow==2.15.0.post1
+tensorflow-estimator==2.15.0
+tensorflow-io-gcs-filesystem==0.35.0
+tensorrt==8.6.1.post1
+tensorrt-bindings==8.6.1
+tensorrt-libs==8.6.1
+termcolor==2.4.0
+terminado==0.17.1
+thinc==8.2.3
+threadpoolctl==3.3.0
+tiktoken==0.6.0
+tinycss2==1.2.1
+tk==0.1.0
+tokenizers==0.19.1
+tomli==2.0.1
+tomlkit==0.12.0
+toolz==0.12.1
+torch==2.1.0
+torchaudio==2.1.0
+torchvision==0.16.0
+tornado==6.3.3
+tqdm==4.66.1
+tqdm-multiprocess==0.0.11
+traitlets==5.11.2
+-e git+https://github.com/huggingface/transformers.git@bbaa8ceff696c479aecdb4575b2deb1349efd3aa#egg=transformers
+triton==2.1.0
+trl==0.8.6
+typepy==1.3.2
+typer==0.12.3
+types-python-dateutil==2.8.19.14
+typing-inspect==0.9.0
+typing_extensions==4.8.0
+tyro==0.8.3
+tzdata==2023.3
+ujson==5.9.0
+unsloth @ git+https://github.com/unslothai/unsloth.git@4211cc01409e3ced4f7abebaf68e244193b46e2c
+uri-template==1.3.0
+urllib3==2.0.6
+uvicorn==0.29.0
+uvloop==0.19.0
+wasabi==1.1.2
+watchfiles==0.21.0
+wavedrom==2.0.3.post3
+wcwidth==0.2.8
+weasel==0.3.4
+webcolors==1.13
+webencodings==0.5.1
+websocket-client==1.6.4
+websockets==11.0.3
+Werkzeug==3.0.1
+word2number==1.1
+wrapt==1.14.1
+xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp39-cp39-manylinux2014_x86_64.whl
+xxhash==3.4.1
+yarl==1.9.2
+zipp==3.17.0
+zstandard==0.22.0

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b997f16f1f5bfcc99bd4bc2fc9d1821e67341b4093297f45f67ca5d81966f1ba
+oid sha256:989480cea002c6608e056242fe1a9579f9b0ce114baac8ebf59373c8a1228b15
 size 5240