Upload 8 files

config.json

@@ -0,0 +1,28 @@
+{
+  "architectures": [
+    "DeciLMForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_decilm.DeciLMConfig",
+    "AutoModelForCausalLM": "modeling_decilm.DeciLMForCausalLM"
+  },
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads_per_layer": [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 4],
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {"type": "dynamic", "factor": 2.0},
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "use_bfloat16": true,
+  "transformers_version": "4.31.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

configuration_decilm.py

@@ -0,0 +1,46 @@
+from packaging import version
+import transformers
+if version.parse(transformers.__version__) < version.parse("4.31.0"):
+    raise ImportError(
+        f"You are using transformers=={transformers.__version__}, but transformers>=4.31.0 is required to use DeciLM. Please upgrade transformers."
+    )
+from transformers.models.llama.configuration_llama import LlamaConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
+
+
+class DeciLMConfig(LlamaConfig):
+    r"""
+
+    Args:
+        num_key_value_heads_per_layer (`List[int]`):
+            The number of key-value heads per layer.
+        naive_attention_prefill (`bool`, *optional*, defaults to False):
+            Whether to use naive matmul or scaled dot product attention during prefill.
+        naive_attention_decode_batched (`bool`, *optional*, defaults to True):
+            Whether to use naive matmul or scaled dot product attention during decode for batch_size > 1.
+        naive_attention_decode_single (`bool`, *optional*, defaults to False):
+            Whether to use naive matmul or scaled dot product attention during decode for batch_size == 1.
+       
+
+       ```"""
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        num_key_value_heads_per_layer: list = None,
+        naive_attention_prefill: bool = False,
+        naive_attention_decode_batched: bool = False,
+        naive_attention_decode_single: bool = False,
+        **kwargs,
+    ):
+        self.num_key_value_heads_per_layer = num_key_value_heads_per_layer
+        self.naive_attention_prefill = naive_attention_prefill
+        self.naive_attention_decode_batched = naive_attention_decode_batched
+        self.naive_attention_decode_single = naive_attention_decode_single
+        super().__init__(**kwargs, )
+

hf_benchmark_example.py

@@ -0,0 +1,214 @@
+"""
+cmd example
+You need a file called "sample.txt" (default path) with text to take tokens for prompts or supply --text_file "path/to/text.txt" as an argument to a text file.
+You can use our attached "sample.txt" file with one of Deci's blogs as a prompt.
+
+# Run this and record tokens per second (652 tokens per second on A10 for DeciLM-6b)
+python hf_benchmark_example.py --model Deci/DeciLM-6b
+
+# Run this and record tokens per second (136 tokens per second on A10 for meta-llama/Llama-2-7b-hf), CUDA OOM above batch size 8
+python hf_benchmark_example.py --model meta-llama/Llama-2-7b-hf --batch_size 8
+"""
+
+import json
+
+import datasets
+import torch
+import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser
+from argparse import ArgumentParser
+
+
+def parse_args():
+    parser = ArgumentParser()
+
+    parser.add_argument(
+        "--model",
+        required=True,
+        help="Model to evaluate, provide a repo name in Hugging Face hub or a local path",
+    )
+    parser.add_argument(
+        "--temperature",
+        default=0.2,
+        type=float
+    )
+    parser.add_argument(
+        "--top_p",
+        default=0.95,
+        type=float
+    )
+    parser.add_argument(
+        "--top_k",
+        default=0,
+        type=float
+    )
+
+    parser.add_argument(
+        "--revision",
+        default=None,
+        help="Model revision to use",
+    )
+    parser.add_argument(
+        "--iterations",
+        type=int,
+        default=6,
+        help="Model revision to use",
+    )
+    parser.add_argument(
+        "--batch_size",
+        type=int,
+        default=64,
+        help="Batch size for evaluation on each worker, can be larger for HumanEval",
+    
+    )
+    parser.add_argument(
+        "--prompt_length",
+        type=int,
+        default=512,
+    )
+    parser.add_argument(
+        "--max_new_tokens",
+        type=int,
+        default=512,
+        help="Maximum length of generated sequence (prompt+generation)",
+    )
+    parser.add_argument(
+        "--precision",
+        type=str,
+        default="bf16",
+        help="Model precision, from: fp32, fp16 or bf16",
+    )
+    parser.add_argument(
+        "--text_file",
+	type=str,
+        default="sample.txt",
+        help="text file that will be used to generate tokens for prompts",
+    )
+    parser.add_argument(
+        "--load_in_8bit",
+        action="store_true",
+        help="Load model in 8bit",
+    )
+    parser.add_argument(
+        "--load_in_4bit",
+        action="store_true",
+        help="Load model in 4bit",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    transformers.logging.set_verbosity_error()
+    datasets.logging.set_verbosity_error()
+
+
+    results = {}
+    dict_precisions = {
+        "fp32": torch.float32,
+        "fp16": torch.float16,
+        "bf16": torch.bfloat16,
+    }
+    if args.precision not in dict_precisions:
+        raise ValueError(
+            f"Non valid precision {args.precision}, choose from: fp16, fp32, bf16"
+        )
+    if args.load_in_8bit:
+        print("Loading model in 8bit")
+        # the model needs to fit in one GPU
+        model = AutoModelForCausalLM.from_pretrained(
+            args.model,
+            revision=args.revision,
+            load_in_8bit=args.load_in_8bit,
+            trust_remote_code=args.trust_remote_code,
+            use_auth_token=args.use_auth_token,
+            device_map={"": 'cuda'},
+        )
+    elif args.load_in_4bit:
+        print("Loading model in 4bit")
+        # the model needs to fit in one GPU
+        model = AutoModelForCausalLM.from_pretrained(
+            args.model,
+            revision=args.revision,
+            load_in_4bit=args.load_in_4bit,
+            trust_remote_code=args.trust_remote_code,
+            use_auth_token=args.use_auth_token,
+            device_map={"": 'cuda'},
+        )
+    else:
+        print(f"Loading model in {args.precision}")
+        model = AutoModelForCausalLM.from_pretrained(
+            args.model,
+            torch_dtype=torch.bfloat16,
+            trust_remote_code=True,
+            use_auth_token=True
+        )
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        args.model,
+        revision=args.revision,
+        trust_remote_code=True,
+        use_auth_token=True,
+    )
+
+    starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
+    model.cuda()
+    model.eval()
+    
+    with open(args.text_file, "r") as f:
+        prompt = f.read()
+
+    prompt = torch.tensor(tokenizer.encode(prompt))[:args.prompt_length].cuda()    
+    
+    results = {'prefill': [], 'gen': [], 'max_new_tokens': args.max_new_tokens, 'prompt_length': args.prompt_length, 'model': args.model, 'batch_size': args.batch_size}
+    inputs = prompt.repeat(args.batch_size, 1)
+
+    #warmup
+    print('start warmup')
+    for _ in range(10):
+        with torch.no_grad():
+            _ = model.generate(
+                                input_ids=inputs,
+                                max_new_tokens=1,
+                        do_sample=False,
+                    )
+    print('finish warmup')
+    torch.cuda.synchronize()
+            
+    for prefill_iter in range(args.iterations):
+        starter.record()
+        with torch.no_grad():
+            _ = model.generate(
+                                input_ids=inputs,
+                                max_new_tokens=1,
+                        do_sample=False,
+                    )
+        ender.record()
+        torch.cuda.synchronize()
+        t = starter.elapsed_time(ender) / 1000
+        results['prefill'].append(t)
+        print(f'{args.batch_size} prefill iter {prefill_iter} took: {t}')
+
+    
+    for gen_iter in range(args.iterations):
+        starter.record()
+        with torch.no_grad():
+            _ = model.generate(
+                                input_ids=inputs,
+                                max_new_tokens=args.max_new_tokens,
+                        do_sample=False,
+                    )
+        ender.record()
+        torch.cuda.synchronize()
+        t = starter.elapsed_time(ender) / 1000
+        results['gen'].append(t)
+
+        print(f'{args.batch_size} total generation iter {gen_iter} took: {t}')
+        print(f'{args.batch_size * args.max_new_tokens / t} tokens per seconds')
+    model_str = args.model.split('/')[-1]
+    with open(f'timing_{model_str}_{args.batch_size}.json', 'w') as f:
+       json.dump(results, f)
+    
+
+if __name__ == "__main__":
+    main()

modeling_decilm.py

@@ -0,0 +1,254 @@
+# coding=utf-8
+# Copyright and license here
+""" PyTorch DeciLM model."""
+import math
+from typing import Optional, Tuple
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from packaging import version
+import transformers
+if version.parse(transformers.__version__) < version.parse("4.31.0"):
+    raise ImportError(
+        f"You are using transformers=={transformers.__version__}, but transformers>=4.31.0 is required to use DeciLM. Please upgrade transformers."
+    )
+from transformers.models.llama.modeling_llama import LlamaMLP, LlamaRMSNorm, LlamaAttention, apply_rotary_pos_emb, \
+    repeat_kv, LlamaPreTrainedModel, LLAMA_START_DOCSTRING, LlamaDecoderLayer, LlamaForCausalLM, LlamaModel
+from transformers.utils import add_start_docstrings
+
+from .configuration_decilm import DeciLMConfig
+
+_CONFIG_FOR_DOC = "DeciLMConfig"
+
+
+class DeciLMAttention(LlamaAttention):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, config: DeciLMConfig, layer_idx: int):
+        nn.Module.__init__(self)
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.hidden_size // self.num_heads
+        self.layer_idx = layer_idx
+        self.num_key_value_heads = config.num_key_value_heads_per_layer[layer_idx]
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.pretraining_tp = config.pretraining_tp
+        self.max_position_embeddings = config.max_position_embeddings
+        self.rope_theta = getattr(config, 'rope_theta', None)
+
+        if (self.head_dim * self.num_heads) != self.hidden_size:
+            raise ValueError(
+                f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
+                f" and `num_heads`: {self.num_heads})."
+            )
+        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
+
+        self.naive_attention_prefill = config.naive_attention_prefill
+        self.naive_attention_decode_batched = config.naive_attention_decode_batched
+        self.naive_attention_decode_single = config.naive_attention_decode_single
+        self._init_rope()
+
+    def forward(
+            self,
+            hidden_states: torch.Tensor,
+            attention_mask: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            past_key_value: Optional[Tuple[torch.Tensor]] = None,
+            output_attentions: bool = False,
+            use_cache: bool = False,
+            padding_mask: Optional[torch.LongTensor] = None,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        bsz, q_len, _ = hidden_states.size()
+        if past_key_value is None:
+            is_decode = False
+        else:
+            is_decode = True
+        if self.pretraining_tp > 1:
+            key_value_slicing = (self.num_key_value_heads * self.head_dim) // self.pretraining_tp
+            query_slices = self.q_proj.weight.split((self.num_heads * self.head_dim) // self.pretraining_tp, dim=0)
+            key_slices = self.k_proj.weight.split(key_value_slicing, dim=0)
+            value_slices = self.v_proj.weight.split(key_value_slicing, dim=0)
+
+            query_states = [F.linear(hidden_states, query_slices[i]) for i in range(self.pretraining_tp)]
+            query_states = torch.cat(query_states, dim=-1)
+
+            key_states = [F.linear(hidden_states, key_slices[i]) for i in range(self.pretraining_tp)]
+            key_states = torch.cat(key_states, dim=-1)
+
+            value_states = [F.linear(hidden_states, value_slices[i]) for i in range(self.pretraining_tp)]
+            value_states = torch.cat(value_states, dim=-1)
+
+        else:
+            query_states = self.q_proj(hidden_states)
+            key_states = self.k_proj(hidden_states)
+            value_states = self.v_proj(hidden_states)
+
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value[0].shape[-2]
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+
+        query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+
+        if past_key_value is not None:
+            # reuse k, v, self_attention
+            key_states = torch.cat([past_key_value[0], key_states], dim=2)
+            value_states = torch.cat([past_key_value[1], value_states], dim=2)
+
+        past_key_value = (key_states, value_states) if use_cache else None
+
+        # repeat k/v heads if n_kv_heads < n_heads
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+        if is_decode:
+            if self.naive_attention_decode_batched and bsz > 1 or self.naive_attention_decode_single and bsz == 1:
+                attn_weights = (query_states @ key_states.transpose(-2, -1)) / math.sqrt(key_states.size(-1))
+                attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+                if attention_mask is not None:
+                    if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                        raise ValueError(
+                            f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                        )
+                    attn_weights = attn_weights + attention_mask
+
+                attn_output = torch.matmul(attn_weights, value_states)
+            else:
+                attn_output = F.scaled_dot_product_attention(query_states, key_states, value_states, is_causal=False,
+                                                             dropout_p=0.0)
+            attn_output = attn_output.contiguous().view(bsz, q_len, self.hidden_size)
+
+        else:
+            if not self.naive_attention_prefill:
+                with torch.backends.cuda.sdp_kernel(enable_math=True, enable_flash=False, enable_mem_efficient=False):
+                    attn_output = F.scaled_dot_product_attention(query_states, key_states, value_states, is_causal=True,
+                                                                 dropout_p=0.0)
+            else:
+                attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
+                if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+                    raise ValueError(
+                        f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
+                        f" {attn_weights.size()}"
+                    )
+
+                if attention_mask is not None:
+                    if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                        raise ValueError(
+                            f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                        )
+                    attn_weights = attn_weights + attention_mask
+
+                # upcast attention to fp32
+                attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+                attn_output = torch.matmul(attn_weights, value_states)
+
+            if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+                raise ValueError(
+                    f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+                    f" {attn_output.size()}"
+                )
+
+            attn_output = attn_output.transpose(1, 2).contiguous().view(bsz, q_len, self.hidden_size)
+
+        if self.pretraining_tp > 1:
+            attn_output = attn_output.split(self.hidden_size // self.pretraining_tp, dim=2)
+            o_proj_slices = self.o_proj.weight.split(self.hidden_size // self.pretraining_tp, dim=1)
+            attn_output = sum([F.linear(attn_output[i], o_proj_slices[i]) for i in range(self.pretraining_tp)])
+        else:
+            attn_output = self.o_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+
+class DeciLMDecoderLayer(LlamaDecoderLayer):
+    def __init__(self, config: DeciLMConfig, layer_idx: int):
+        nn.Module.__init__(self)
+        self.hidden_size = config.hidden_size
+        self.layer_idx = layer_idx
+        self.self_attn = DeciLMAttention(config=config, layer_idx=layer_idx)
+        self.mlp = LlamaMLP(config)
+        self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+
+@add_start_docstrings(
+    "The bare DeciLM Model outputting raw hidden-states without any specific head on top.",
+    LLAMA_START_DOCSTRING,
+)
+class DeciLMPreTrainedModel(LlamaPreTrainedModel):
+    config_class = DeciLMConfig
+    _no_split_modules = ["DeciLMDecoderLayer"]
+    _keys_to_ignore_on_load_missing = ["self_attn.rotary_emb.inv_freq"]
+
+
+@add_start_docstrings(
+    "The bare DeciLM Model outputting raw hidden-states without any specific head on top.",
+    LLAMA_START_DOCSTRING,
+)
+class DeciLMModel(LlamaModel, DeciLMPreTrainedModel):
+    """
+    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`DeciLMDecoderLayer`]
+
+    Args:
+        config: DeciLMConfig
+    """
+
+    def __init__(self, config: DeciLMConfig):
+        DeciLMPreTrainedModel.__init__(self, config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList([DeciLMDecoderLayer(config, layer_idx) for layer_idx
+                                     in range(config.num_hidden_layers)])
+        self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+        self.gradient_checkpointing = False
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length):
+        self._validate_config_supports_attention_mask(attention_mask, input_shape, past_key_values_length)
+        return LlamaModel._prepare_decoder_attention_mask(
+            self, attention_mask, input_shape, inputs_embeds, past_key_values_length)
+
+    def _validate_config_supports_attention_mask(self, attention_mask, input_shape, past_key_values_length):
+        is_decode = past_key_values_length > 0
+        if not torch.all(torch.eq(attention_mask, 1)).item():
+            if is_decode:
+                if input_shape[0] == 1 and not self.config.naive_attention_decode_single:
+                    raise ValueError(
+                        "For support of custom attention masks please set naive_attention_decode_single to True in the "
+                        "config")
+                elif input_shape[0] > 1 and not self.config.naive_attention_decode_batched:
+                    raise ValueError(
+                        "For support of custom attention masks please set naive_attention_decode_batched to True in the"
+                        "config")
+            else:
+                if not self.config.naive_attention_prefill:
+                    raise ValueError("For support of custom attention masks please set naive_attention_prefill to "
+                                     "True in the config")
+
+
+class DeciLMForCausalLM(LlamaForCausalLM, DeciLMPreTrainedModel):
+    def __init__(self, config):
+        DeciLMPreTrainedModel.__init__(self, config)
+        self.model = DeciLMModel(config)
+        self.pretraining_tp = config.pretraining_tp
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()

sample.txt

@@ -0,0 +1,363 @@
+The network latency is one of the more crucial aspects of deploying a deep network into a production environment. Most real-world applications require blazingly fast inference time, varying anywhere from a few milliseconds to one second. But the task of correctly and meaningfully measuring the inference time, or latency, of a neural network requires profound understanding. Even experienced programmers often make common mistakes that lead to inaccurate latency measurements. The impact of these mistakes has the potential to trigger bad decisions and unnecessary expenditures.
+
+In this post, we review some of the main issues that should be addressed to measure latency time correctly. We review the main processes that make GPU execution unique, including asynchronous execution and GPU warm up. We then share code samples for measuring time correctly on a GPU. Finally, we review some of the common mistakes people make when quantifying inference time on GPUs.
+Asynchronous execution
+
+We begin by discussing the GPU execution mechanism. In multithreaded or multi-device programming, two blocks of code that are independent can be executed in parallel; this means that the second block may be executed before the first is finished. This process is referred to as asynchronous execution. In the deep learning context, we often use this execution because the GPU operations are asynchronous by default. More specifically, when calling a function using a GPU, the operations are enqueued to the specific device, but not necessarily to other devices. This allows us to execute computations in parallel on the CPU or another GPU.
+
+Figure 1. Asynchronous execution. Left: Synchronous process where process A waits for a response from process B before it can continue working. Right: Asynchronous process A continues working without waiting for process B to finish.
+
+Asynchronous execution offers huge advantages for deep learning, such as the ability to decrease run-time by a large factor. For example, at the inference of multiple batches, the second batch can be preprocessed on the CPU while the first batch is fed forward through the network on the GPU. Clearly, it would be beneficial to use asynchronism whenever possible at inference time.
+
+The effect of asynchronous execution is invisible to the user; but, when it comes to time measurements, it can be the cause of many headaches. When you calculate time with the “time” library in Python, the measurements are performed on the CPU device. Due to the asynchronous nature of the GPU, the line of code that stops the timing will be executed before the GPU process finishes. As a result, the timing will be inaccurate or irrelevant to the actual inference time. Keeping in mind that we want to use asynchronism, later in this post we explain how to correctly measure time despite the asynchronous processes.
+GPU warm-up
+
+A modern GPU device can exist in one of several different power states. When the GPU is not being used for any purpose and persistence mode (i.e., which keeps the GPU on) is not enabled, the GPU will automatically reduce its power state to a very low level, sometimes even a complete shutdown. In lower power state, the GPU shuts down different pieces of hardware, including memory subsystems, internal subsystems, or even compute cores and caches.
+
+The invocation of any program that attempts to interact with the GPU will cause the driver to load and/or initialize the GPU. This driver load behavior is noteworthy. Applications that trigger GPU initialization can incur up to 3 seconds of latency, due to the scrubbing behavior of the error correcting code. For instance, if we measure time for a network that takes 10 milliseconds for one example, running over 1000 examples may result in most of our running time being wasted on initializing the GPU. Naturally, we don’t want to measure such side effects because the timing is not accurate. Nor does it reflect a production environment where usually the GPU is already initialized or working in persistence mode.
+
+Since, we want to enable the GPU power-saving mode whenever possible, let’s look at how to overcome the initialization of the GPU while measuring time.
+The correct way to measure inference time
+
+The PyTorch code snippet below shows how to measure time correctly. Here we use Efficient-net-b0 but you can use any other network. In the code, we deal with the two caveats described above. Before we make any time measurements, we run some dummy examples through the network to do a ‘GPU warm-up.’ This will automatically initialize the GPU and prevent it from going into power-saving mode when we measure time. Next, we use tr.cuda.event to measure time on the GPU. It is crucial here to use torch.cuda.synchronize(). This line of code performs synchronization between the host and device (i.e., GPU and CPU), so the time recording takes place only after the process running on the GPU is finished. This overcomes the issue of unsynchronized execution.
+
+model = EfficientNet.from_pretrained('efficientnet-b0')
+device = torch.device("cuda")
+model.to(device)
+dummy_input = torch.randn(1, 3,224,224, dtype=torch.float).to(device)
+
+# INIT LOGGERS
+starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
+repetitions = 300
+timings=np.zeros((repetitions,1))
+#GPU-WARM-UP
+for _ in range(10):
+    _ = model(dummy_input)
+# MEASURE PERFORMANCE
+with torch.no_grad():
+    for rep in range(repetitions):
+        starter.record()
+        _ = model(dummy_input)
+        ender.record()
+        # WAIT FOR GPU SYNC
+        torch.cuda.synchronize()
+        curr_time = starter.elapsed_time(ender)
+        timings[rep] = curr_time
+
+mean_syn = np.sum(timings) / repetitions
+std_syn = np.std(timings)
+print(mean_syn)
+
+Common mistakes when measuring time
+
+When we measure the latency of a network, our goal is to measure only the feed-forward of the network, not more and not less. Often, even experts will make certain common mistakes in their measurements. Here are some of them, along with their consequences:
+
+1. Transferring data between the host and the device. The point of view of this post is to measure only the inference time of a neural network. Under this point of view, one of the most common mistakes involves the transfer of data between the CPU and GPU while taking time measurements. This is usually done unintentionally when a tensor is created on the CPU and inference is then performed on the GPU. This memory allocation takes a considerable amount of time, which subsequently enlarges the time for inference. The effect of this mistake over the mean and variance of the measurements can be seen below:
+
+Figure 2: Impact of transferring between CPU and GPU while measuring time. Left: The correct measurements for mean and standard deviation (bar). Right: The mean and standard deviation when the input tensor is transferred between CPU and GPU at each call for the network. The X-axis is the timing method and the Y-axis is the time in milliseconds.
+
+2. Not using GPU warm-up. As mentioned above, the first run on the GPU prompts its initialization. GPU initialization can take up to 3 seconds, which makes a huge difference when the timing is in terms of milliseconds.
+
+3. Using standard CPU timing. The most common mistake made is to measure time without synchronization. Even experienced programmers have been known to use the following piece of code.
+
+ s = time.time()
+ _ = model(dummy_input)
+curr_time = (time.time()-s )*1000
+
+This of course completely ignores the asynchronous execution mentioned earlier and hence outputs incorrect times. The impact of this mistake on the mean and variance of the measurements are shown below:
+
+Figure 3: Impact of measuring time on CPU. Left: The correct measurements for mean and standard deviation (bar). Right: The mean and standard deviation when processes are not synchronized. The X-axis is the timing method and the Y-axis is the time in milliseconds.
+
+4. Taking one sample. Like many processes in computer science, feed forward of the neural network has a (small) stochastic component. The variance of the run-time can be significant, especially when measuring a low latency network. To this end, it is essential to run the network over several examples and then average the results (300 examples can be a good number). A common mistake is to use one sample and refer to it as the run-time. This, of course, won’t represent the true run-time.
+Measuring Throughput
+
+The throughput of a neural network is defined as the maximal number of input instances the network can process in time a unit (e.g., a second). Unlike latency, which involves the processing of a single instance, to achieve maximal throughput we would like to process in parallel as many instances as possible. The effective parallelism is obviously data-, model-, and device-dependent. Thus, to correctly measure throughput we perform the following two steps: (1) we estimate the optimal batch size that allows for maximum parallelism; and (2), given this optimal batch size, we measure the number of instances the network can process in one second.
+To find the optimal batch size, a good rule of thumb is to reach the memory limit of our GPU for the given data type. This size of course depends on the hardware type and the size of the network. The quickest way to find this maximal batch size is by performing a binary search. When time is of no concern a simple sequential search is sufficient. To this end, using a for loop we increase by one the batch size until Run Time error is achieved, this identifies the largest batch size the GPU can process, for our neural network model and the input data it processes.
+After finding the optimal batch size, we calculate the actual throughput. To this end, we would like to process many batches (100 batches will be a sufficient number) and then use the following formula:
+
+(number of batches X batch size)/(total time in seconds)
+
+This formula gives the number of examples our network can process in one second. The code below provides a simple way to perform the above calculation (given the optimal batch size):
+
+model = EfficientNet.from_pretrained('efficientnet-b0')
+device = torch.device("cuda")
+model.to(device)
+dummy_input = torch.randn(optimal_batch_size, 3,224,224, dtype=torch.float).to(device)
+
+repetitions=100
+total_time = 0
+with torch.no_grad():
+    for rep in range(repetitions):
+        starter, ender = torch.cuda.Event(enable_timing=True),   torch.cuda.Event(enable_timing=True)
+        starter.record()
+        _ = model(dummy_input)
+        ender.record()
+        torch.cuda.synchronize()
+        curr_time = starter.elapsed_time(ender)/1000
+        total_time += curr_time
+Throughput =   (repetitions*optimal_batch_size)/total_time
+print('Final Throughput:',Throughput)
+
+Conclusion
+
+To accurately measure inference time of neural networks is not as trivial as it sounds. We detailed several issues that deep learning practitioners should be aware of, such as asynchronous execution and GPU power-saving modes. The PyTorch code presented here demonstrates how to correctly measure the timing in neural networks, despite the aforementioned caveats. Finally, we mentioned some common mistakes that cause people to measure inference time incorrectly. In future posts, we will dive even deeper into this topic and explain existing deep learning profilers which enable us to achieve even more accurate time measurements of networks. If you are interested in how to reduce the latency of the network without compromising its accuracy you are invited to book a demo with one of our experts, or  read more about this topic in Deci’s Guide to Inference Acceleration.
+
+The network latency is one of the more crucial aspects of deploying a deep network into a production environment. Most real-world applications require blazingly fast inference time, varying anywhere from a few milliseconds to one second. But the task of correctly and meaningfully measuring the inference time, or latency, of a neural network requires profound understanding. Even experienced programmers often make common mistakes that lead to inaccurate latency measurements. The impact of these mistakes has the potential to trigger bad decisions and unnecessary expenditures.
+
+In this post, we review some of the main issues that should be addressed to measure latency time correctly. We review the main processes that make GPU execution unique, including asynchronous execution and GPU warm up. We then share code samples for measuring time correctly on a GPU. Finally, we review some of the common mistakes people make when quantifying inference time on GPUs.
+Asynchronous execution
+
+We begin by discussing the GPU execution mechanism. In multithreaded or multi-device programming, two blocks of code that are independent can be executed in parallel; this means that the second block may be executed before the first is finished. This process is referred to as asynchronous execution. In the deep learning context, we often use this execution because the GPU operations are asynchronous by default. More specifically, when calling a function using a GPU, the operations are enqueued to the specific device, but not necessarily to other devices. This allows us to execute computations in parallel on the CPU or another GPU.
+
+Figure 1. Asynchronous execution. Left: Synchronous process where process A waits for a response from process B before it can continue working. Right: Asynchronous process A continues working without waiting for process B to finish.
+
+Asynchronous execution offers huge advantages for deep learning, such as the ability to decrease run-time by a large factor. For example, at the inference of multiple batches, the second batch can be preprocessed on the CPU while the first batch is fed forward through the network on the GPU. Clearly, it would be beneficial to use asynchronism whenever possible at inference time.
+
+The effect of asynchronous execution is invisible to the user; but, when it comes to time measurements, it can be the cause of many headaches. When you calculate time with the ?~@~\time?~@~] library in Python, the measurements are performed on the CPU device. Due to the asynchronous nature of the GPU, the line of code that stops the timing will be executed before the GPU process finishes. As a result, the timing will be inaccurate or irrelevant to the actual inference time. Keeping in mind that we want to use asynchronism, later in this post we explain how to correctly measure time despite the asynchronous processes.
+GPU warm-up
+
+A modern GPU device can exist in one of several different power states. When the GPU is not being used for any purpose and persistence mode (i.e., which keeps the GPU on) is not enabled, the GPU will automatically reduce its power state to a very low level, sometimes even a complete shutdown. In lower power state, the GPU shuts down different pieces of hardware, including memory subsystems, internal subsystems, or even compute cores and caches.
+
+The invocation of any program that attempts to interact with the GPU will cause the driver to load and/or initialize the GPU. This driver load behavior is noteworthy. Applications that trigger GPU initialization can incur up to 3 seconds of latency, due to the scrubbing behavior of the error correcting code. For instance, if we measure time for a network that takes 10 milliseconds for one example, running over 1000 examples may result in most of our running time being wasted on initializing the GPU. Naturally, we don?~@~Yt want to measure such side effects because the timing is not accurate. Nor does it reflect a production environment where usually the GPU is already initialized or working in persistence mode.
+
+Since, we want to enable the GPU power-saving mode whenever possible, let?~@~Ys look at how to overcome the initialization of the GPU while measuring time.
+The correct way to measure inference time
+
+The PyTorch code snippet below shows how to measure time correctly. Here we use Efficient-net-b0 but you can use any other network. In the code, we deal with the two caveats described above. Before we make any time measurements, we run some dummy examples through the network to do a ?~@~XGPU warm-up.?~@~Y This will automatically initialize the GPU and prevent it from going into power-saving mode when we measure time. Next, we use tr.cuda.event to measure time on the GPU. It is crucial here to use torch.cuda.synchronize(). This line of code performs synchronization between the host and device (i.e., GPU and CPU), so the time recording takes place only after the process running on the GPU is finished. This overcomes the issue of unsynchronized execution.
+
+model = EfficientNet.from_pretrained('efficientnet-b0')
+device = torch.device("cuda")
+model.to(device)
+dummy_input = torch.randn(1, 3,224,224, dtype=torch.float).to(device)
+
+# INIT LOGGERS
+starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
+repetitions = 300
+timings=np.zeros((repetitions,1))
+#GPU-WARM-UP
+for _ in range(10):
+    _ = model(dummy_input)
+# MEASURE PERFORMANCE
+with torch.no_grad():
+    for rep in range(repetitions):
+        starter.record()
+        _ = model(dummy_input)
+        ender.record()
+        # WAIT FOR GPU SYNC
+        torch.cuda.synchronize()
+        curr_time = starter.elapsed_time(ender)
+        timings[rep] = curr_time
+
+mean_syn = np.sum(timings) / repetitions
+std_syn = np.std(timings)
+print(mean_syn)
+
+Common mistakes when measuring time
+
+When we measure the latency of a network, our goal is to measure only the feed-forward of the network, not more and not less. Often, even experts will make certain common mistakes in their measurements. Here are some of them, along with their consequences:
+
+1. Transferring data between the host and the device. The point of view of this post is to measure only the inference time of a neural network. Under this point of view, one of the most common mistakes involves the transfer of data between the CPU and GPU while taking time measurements. This is usually done unintentionally when a tensor is created on the CPU and inference is then performed on the GPU. This memory allocation takes a considerable amount of time, which subsequently enlarges the time for inference. The effect of this mistake over the mean and variance of the measurements can be seen below:
+
+Figure 2: Impact of transferring between CPU and GPU while measuring time. Left: The correct measurements for mean and standard deviation (bar). Right: The mean and standard deviation when the input tensor is transferred between CPU and GPU at each call for the network. The X-axis is the timing method and the Y-axis is the time in milliseconds.
+
+2. Not using GPU warm-up. As mentioned above, the first run on the GPU prompts its initialization. GPU initialization can take up to 3 seconds, which makes a huge difference when the timing is in terms of milliseconds.
+
+3. Using standard CPU timing. The most common mistake made is to measure time without synchronization. Even experienced programmers have been known to use the following piece of code.
+
+ s = time.time()
+ _ = model(dummy_input)
+curr_time = (time.time()-s )*1000
+
+This of course completely ignores the asynchronous execution mentioned earlier and hence outputs incorrect times. The impact of this mistake on the mean and variance of the measurements are shown below:
+
+Figure 3: Impact of measuring time on CPU. Left: The correct measurements for mean and standard deviation (bar). Right: The mean and standard deviation when processes are not synchronized. The X-axis is the timing method and the Y-axis is the time in milliseconds.
+
+4. Taking one sample. Like many processes in computer science, feed forward of the neural network has a (small) stochastic component. The variance of the run-time can be significant, especially when measuring a low latency network. To this end, it is essential to run the network over several examples and then average the results (300 examples can be a good number). A common mistake is to use one sample and refer to it as the run-time. This, of course, won?~@~Yt represent the true run-time.
+Measuring Throughput
+
+The throughput of a neural network is defined as the maximal number of input instances the network can process in time a unit (e.g., a second). Unlike latency, which involves the processing of a single instance, to achieve maximal throughput we would like to process in parallel as many instances as possible. The effective parallelism is obviously data-, model-, and device-dependent. Thus, to correctly measure throughput we perform the following two steps: (1) we estimate the optimal batch size that allows for maximum parallelism; and (2), given this optimal batch size, we measure the number of instances the network can process in one second.
+To find the optimal batch size, a good rule of thumb is to reach the memory limit of our GPU for the given data type. This size of course depends on the hardware type and the size of the network. The quickest way to find this maximal batch size is by performing a binary search. When time is of no concern a simple sequential search is sufficient. To this end, using a for loop we increase by one the batch size until Run Time error is achieved, this identifies the largest batch size the GPU can process, for our neural network model and the input data it processes.
+After finding the optimal batch size, we calculate the actual throughput. To this end, we would like to process many batches (100 batches will be a sufficient number) and then use the following formula:
+
+(number of batches X batch size)/(total time in seconds)
+
+This formula gives the number of examples our network can process in one second. The code below provides a simple way to perform the above calculation (given the optimal batch size):
+
+model = EfficientNet.from_pretrained('efficientnet-b0')
+device = torch.device("cuda")
+model.to(device)
+dummy_input = torch.randn(optimal_batch_size, 3,224,224, dtype=torch.float).to(device)
+
+The network latency is one of the more crucial aspects of deploying a deep network into a production environment. Most real-world applications require blazingly fast inference time, varying anywhere from a few milliseconds to one second. But the task of correctly and meaningfully measuring the inference time, or latency, of a neural network requires profound understanding. Even experienced programmers often make common mistakes that lead to inaccurate latency measurements. The impact of these mistakes has the potential to trigger bad decisions and unnecessary expenditures.
+
+In this post, we review some of the main issues that should be addressed to measure latency time correctly. We review the main processes that make GPU execution unique, including asynchronous execution and GPU warm up. We then share code samples for measuring time correctly on a GPU. Finally, we review some of the common mistakes people make when quantifying inference time on GPUs.
+Asynchronous execution
+
+We begin by discussing the GPU execution mechanism. In multithreaded or multi-device programming, two blocks of code that are independent can be executed in parallel; this means that the second block may be executed before the first is finished. This process is referred to as asynchronous execution. In the deep learning context, we often use this execution because the GPU operations are asynchronous by default. More specifically, when calling a function using a GPU, the operations are enqueued to the specific device, but not necessarily to other devices. This allows us to execute computations in parallel on the CPU or another GPU.
+
+Figure 1. Asynchronous execution. Left: Synchronous process where process A waits for a response from process B before it can continue working. Right: Asynchronous process A continues working without waiting for process B to finish.
+
+Asynchronous execution offers huge advantages for deep learning, such as the ability to decrease run-time by a large factor. For example, at the inference of multiple batches, the second batch can be preprocessed on the CPU while the first batch is fed forward through the network on the GPU. Clearly, it would be beneficial to use asynchronism whenever possible at inference time.
+
+The effect of asynchronous execution is invisible to the user; but, when it comes to time measurements, it can be the cause of many headaches. When you calculate time with the ?~@~\time?~@~] library in Python, the measurements are performed on the CPU device. Due to the asynchronous nature of the GPU, the line of code that stops the timing will be executed before the GPU process finishes. As a result, the timing will be inaccurate or irrelevant to the actual inference time. Keeping in mind that we want to use asynchronism, later in this post we explain how to correctly measure time despite the asynchronous processes.
+GPU warm-up
+
+A modern GPU device can exist in one of several different power states. When the GPU is not being used for any purpose and persistence mode (i.e., which keeps the GPU on) is not enabled, the GPU will automatically reduce its power state to a very low level, sometimes even a complete shutdown. In lower power state, the GPU shuts down different pieces of hardware, including memory subsystems, internal subsystems, or even compute cores and caches.
+
+The invocation of any program that attempts to interact with the GPU will cause the driver to load and/or initialize the GPU. This driver load behavior is noteworthy. Applications that trigger GPU initialization can incur up to 3 seconds of latency, due to the scrubbing behavior of the error correcting code. For instance, if we measure time for a network that takes 10 milliseconds for one example, running over 1000 examples may result in most of our running time being wasted on initializing the GPU. Naturally, we don?~@~Yt want to measure such side effects because the timing is not accurate. Nor does it reflect a production environment where usually the GPU is already initialized or working in persistence mode.
+
+Since, we want to enable the GPU power-saving mode whenever possible, let?~@~Ys look at how to overcome the initialization of the GPU while measuring time.
+The correct way to measure inference time
+
+The PyTorch code snippet below shows how to measure time correctly. Here we use Efficient-net-b0 but you can use any other network. In the code, we deal with the two caveats described above. Before we make any time measurements, we run some dummy examples through the network to do a ?~@~XGPU warm-up.?~@~Y This will automatically initialize the GPU and prevent it from going into power-saving mode when we measure time. Next, we use tr.cuda.event to measure time on the GPU. It is crucial here to use torch.cuda.synchronize(). This line of code performs synchronization between the host and device (i.e., GPU and CPU), so the time recording takes place only after the process running on the GPU is finished. This overcomes the issue of unsynchronized execution.
+
+model = EfficientNet.from_pretrained('efficientnet-b0')
+device = torch.device("cuda")
+model.to(device)
+dummy_input = torch.randn(1, 3,224,224, dtype=torch.float).to(device)
+
+# INIT LOGGERS
+starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
+repetitions = 300
+timings=np.zeros((repetitions,1))
+#GPU-WARM-UP
+for _ in range(10):
+    _ = model(dummy_input)
+# MEASURE PERFORMANCE
+with torch.no_grad():
+    for rep in range(repetitions):
+        starter.record()
+        _ = model(dummy_input)
+        ender.record()
+                                                                                                                                                                                          1,1           Top
+
+
+The network latency is one of the more crucial aspects of deploying a deep network into a production environment. Most real-world applications require blazingly fast inference time, varying anywhere from a few milliseconds to one second. But the task of correctly and meaningfully measuring the inference time, or latency, of a neural network requires profound understanding. Even experienced programmers often make common mistakes that lead to inaccurate latency measurements. The impact of these mistakes has the potential to trigger bad decisions and unnecessary expenditures.
+
+In this post, we review some of the main issues that should be addressed to measure latency time correctly. We review the main processes that make GPU execution unique, including asynchronous execution and GPU warm up. We then share code samples for measuring time correctly on a GPU. Finally, we review some of the common mistakes people make when quantifying inference time on GPUs.
+Asynchronous execution
+
+We begin by discussing the GPU execution mechanism. In multithreaded or multi-device programming, two blocks of code that are independent can be executed in parallel; this means that the second block may be executed before the first is finished. This process is referred to as asynchronous execution. In the deep learning context, we often use this execution because the GPU operations are asynchronous by default. More specifically, when calling a function using a GPU, the operations are enqueued to the specific device, but not necessarily to other devices. This allows us to execute computations in parallel on the CPU or another GPU.
+
+Figure 1. Asynchronous execution. Left: Synchronous process where process A waits for a response from process B before it can continue working. Right: Asynchronous process A continues working without waiting for process B to finish.
+
+Asynchronous execution offers huge advantages for deep learning, such as the ability to decrease run-time by a large factor. For example, at the inference of multiple batches, the second batch can be preprocessed on the CPU while the first batch is fed forward through the network on the GPU. Clearly, it would be beneficial to use asynchronism whenever possible at inference time.
+
+The effect of asynchronous execution is invisible to the user; but, when it comes to time measurements, it can be the cause of many headaches. When you calculate time with the ?~@~\time?~@~] library in Python, the measurements are performed on the CPU device. Due to the asynchronous nature of the GPU, the line of code that stops the timing will be executed before the GPU process finishes. As a result, the timing will be inaccurate or irrelevant to the actual inference time. Keeping in mind that we want to use asynchronism, later in this post we explain how to correctly measure time despite the asynchronous processes.
+GPU warm-up
+
+A modern GPU device can exist in one of several different power states. When the GPU is not being used for any purpose and persistence mode (i.e., which keeps the GPU on) is not enabled, the GPU will automatically reduce its power state to a very low level, sometimes even a complete shutdown. In lower power state, the GPU shuts down different pieces of hardware, including memory subsystems, internal subsystems, or even compute cores and caches.
+
+The invocation of any program that attempts to interact with the GPU will cause the driver to load and/or initialize the GPU. This driver load behavior is noteworthy. Applications that trigger GPU initialization can incur up to 3 seconds of latency, due to the scrubbing behavior of the error correcting code. For instance, if we measure time for a network that takes 10 milliseconds for one example, running over 1000 examples may result in most of our running time being wasted on initializing the GPU. Naturally, we don?~@~Yt want to measure such side effects because the timing is not accurate. Nor does it reflect a production environment where usually the GPU is already initialized or working in persistence mode.
+
+Since, we want to enable the GPU power-saving mode whenever possible, let?~@~Ys look at how to overcome the initialization of the GPU while measuring time.
+The correct way to measure inference time
+
+The PyTorch code snippet below shows how to measure time correctly. Here we use Efficient-net-b0 but you can use any other network. In the code, we deal with the two caveats described above. Before we make any time measurements, we run some dummy examples through the network to do a ?~@~XGPU warm-up.?~@~Y This will automatically initialize the GPU and prevent it from going into power-saving mode when we measure time. Next, we use tr.cuda.event to measure time on the GPU. It is crucial here to use torch.cuda.synchronize(). This line of code performs synchronization between the host and device (i.e., GPU and CPU), so the time recording takes place only after the process running on the GPU is finished. This overcomes the issue of unsynchronized execution.
+
+model = EfficientNet.from_pretrained('efficientnet-b0')
+device = torch.device("cuda")
+model.to(device)
+dummy_input = torch.randn(1, 3,224,224, dtype=torch.float).to(device)
+
+# INIT LOGGERS
+starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
+repetitions = 300
+timings=np.zeros((repetitions,1))
+#GPU-WARM-UP
+for _ in range(10):
+    _ = model(dummy_input)
+# MEASURE PERFORMANCE
+with torch.no_grad():
+    for rep in range(repetitions):
+        starter.record()
+        _ = model(dummy_input)
+        ender.record()
+                                                                                                                                                                                          1,1           Top
+
+The network latency is one of the more crucial aspects of deploying a deep network into a production environment. Most real-world applications require blazingly fast inference time, varying anywhere from a few milliseconds to one second. But the task of correctly and meaningfully measuring the inference time, or latency, of a neural network requires profound understanding. Even experienced programmers often make common mistakes that lead to inaccurate latency measurements. The impact of these mistakes has the potential to trigger bad decisions and unnecessary expenditures.
+
+In this post, we review some of the main issues that should be addressed to measure latency time correctly. We review the main processes that make GPU execution unique, including asynchronous execution and GPU warm up. We then share code samples for measuring time correctly on a GPU. Finally, we review some of the common mistakes people make when quantifying inference time on GPUs.
+Asynchronous execution
+
+We begin by discussing the GPU execution mechanism. In multithreaded or multi-device programming, two blocks of code that are independent can be executed in parallel; this means that the second block may be executed before the first is finished. This process is referred to as asynchronous execution. In the deep learning context, we often use this execution because the GPU operations are asynchronous by default. More specifically, when calling a function using a GPU, the operations are enqueued to the specific device, but not necessarily to other devices. This allows us to execute computations in parallel on the CPU or another GPU.
+
+Figure 1. Asynchronous execution. Left: Synchronous process where process A waits for a response from process B before it can continue working. Right: Asynchronous process A continues working without waiting for process B to finish.
+
+Asynchronous execution offers huge advantages for deep learning, such as the ability to decrease run-time by a large factor. For example, at the inference of multiple batches, the second batch can be preprocessed on the CPU while the first batch is fed forward through the network on the GPU. Clearly, it would be beneficial to use asynchronism whenever possible at inference time.
+
+The effect of asynchronous execution is invisible to the user; but, when it comes to time measurements, it can be the cause of many headaches. When you calculate time with the ?~@~\time?~@~] library in Python, the measurements are performed on the CPU device. Due to the asynchronous nature of the GPU, the line of code that stops the timing will be executed before the GPU process finishes. As a result, the timing will be inaccurate or irrelevant to the actual inference time. Keeping in mind that we want to use asynchronism, later in this post we explain how to correctly measure time despite the asynchronous processes.
+GPU warm-up
+
+A modern GPU device can exist in one of several different power states. When the GPU is not being used for any purpose and persistence mode (i.e., which keeps the GPU on) is not enabled, the GPU will automatically reduce its power state to a very low level, sometimes even a complete shutdown. In lower power state, the GPU shuts down different pieces of hardware, including memory subsystems, internal subsystems, or even compute cores and caches.
+
+The invocation of any program that attempts to interact with the GPU will cause the driver to load and/or initialize the GPU. This driver load behavior is noteworthy. Applications that trigger GPU initialization can incur up to 3 seconds of latency, due to the scrubbing behavior of the error correcting code. For instance, if we measure time for a network that takes 10 milliseconds for one example, running over 1000 examples may result in most of our running time being wasted on initializing the GPU. Naturally, we don?~@~Yt want to measure such side effects because the timing is not accurate. Nor does it reflect a production environment where usually the GPU is already initialized or working in persistence mode.
+
+Since, we want to enable the GPU power-saving mode whenever possible, let?~@~Ys look at how to overcome the initialization of the GPU while measuring time.
+The correct way to measure inference time
+
+The PyTorch code snippet below shows how to measure time correctly. Here we use Efficient-net-b0 but you can use any other network. In the code, we deal with the two caveats described above. Before we make any time measurements, we run some dummy examples through the network to do a ?~@~XGPU warm-up.?~@~Y This will automatically initialize the GPU and prevent it from going into power-saving mode when we measure time. Next, we use tr.cuda.event to measure time on the GPU. It is crucial here to use torch.cuda.synchronize(). This line of code performs synchronization between the host and device (i.e., GPU and CPU), so the time recording takes place only after the process running on the GPU is finished. This overcomes the issue of unsynchronized execution.
+
+model = EfficientNet.from_pretrained('efficientnet-b0')
+device = torch.device("cuda")
+model.to(device)
+dummy_input = torch.randn(1, 3,224,224, dtype=torch.float).to(device)
+
+# INIT LOGGERS
+starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
+repetitions = 300
+timings=np.zeros((repetitions,1))
+#GPU-WARM-UP
+for _ in range(10):
+    _ = model(dummy_input)
+# MEASURE PERFORMANCE
+with torch.no_grad():
+    for rep in range(repetitions):
+        starter.record()
+        _ = model(dummy_input)
+        ender.record()
+                                                                                                                                                                                          1,1           Top
+
+The network latency is one of the more crucial aspects of deploying a deep network into a production environment. Most real-world applications require blazingly fast inference time, varying anywhere from a few milliseconds to one second. But the task of correctly and meaningfully measuring the inference time, or latency, of a neural network requires profound understanding. Even experienced programmers often make common mistakes that lead to inaccurate latency measurements. The impact of these mistakes has the potential to trigger bad decisions and unnecessary expenditures.
+
+In this post, we review some of the main issues that should be addressed to measure latency time correctly. We review the main processes that make GPU execution unique, including asynchronous execution and GPU warm up. We then share code samples for measuring time correctly on a GPU. Finally, we review some of the common mistakes people make when quantifying inference time on GPUs.
+Asynchronous execution
+
+We begin by discussing the GPU execution mechanism. In multithreaded or multi-device programming, two blocks of code that are independent can be executed in parallel; this means that the second block may be executed before the first is finished. This process is referred to as asynchronous execution. In the deep learning context, we often use this execution because the GPU operations are asynchronous by default. More specifically, when calling a function using a GPU, the operations are enqueued to the specific device, but not necessarily to other devices. This allows us to execute computations in parallel on the CPU or another GPU.
+
+Figure 1. Asynchronous execution. Left: Synchronous process where process A waits for a response from process B before it can continue working. Right: Asynchronous process A continues working without waiting for process B to finish.
+
+Asynchronous execution offers huge advantages for deep learning, such as the ability to decrease run-time by a large factor. For example, at the inference of multiple batches, the second batch can be preprocessed on the CPU while the first batch is fed forward through the network on the GPU. Clearly, it would be beneficial to use asynchronism whenever possible at inference time.
+
+The effect of asynchronous execution is invisible to the user; but, when it comes to time measurements, it can be the cause of many headaches. When you calculate time with the ?~@~\time?~@~] library in Python, the measurements are performed on the CPU device. Due to the asynchronous nature of the GPU, the line of code that stops the timing will be executed before the GPU process finishes. As a result, the timing will be inaccurate or irrelevant to the actual inference time. Keeping in mind that we want to use asynchronism, later in this post we explain how to correctly measure time despite the asynchronous processes.
+GPU warm-up
+
+A modern GPU device can exist in one of several different power states. When the GPU is not being used for any purpose and persistence mode (i.e., which keeps the GPU on) is not enabled, the GPU will automatically reduce its power state to a very low level, sometimes even a complete shutdown. In lower power state, the GPU shuts down different pieces of hardware, including memory subsystems, internal subsystems, or even compute cores and caches.
+
+The invocation of any program that attempts to interact with the GPU will cause the driver to load and/or initialize the GPU. This driver load behavior is noteworthy. Applications that trigger GPU initialization can incur up to 3 seconds of latency, due to the scrubbing behavior of the error correcting code. For instance, if we measure time for a network that takes 10 milliseconds for one example, running over 1000 examples may result in most of our running time being wasted on initializing the GPU. Naturally, we don?~@~Yt want to measure such side effects because the timing is not accurate. Nor does it reflect a production environment where usually the GPU is already initialized or working in persistence mode.
+
+Since, we want to enable the GPU power-saving mode whenever possible, let?~@~Ys look at how to overcome the initialization of the GPU while measuring time.
+The correct way to measure inference time
+
+The PyTorch code snippet below shows how to measure time correctly. Here we use Efficient-net-b0 but you can use any other network. In the code, we deal with the two caveats described above. Before we make any time measurements, we run some dummy examples through the network to do a ?~@~XGPU warm-up.?~@~Y This will automatically initialize the GPU and prevent it from going into power-saving mode when we measure time. Next, we use tr.cuda.event to measure time on the GPU. It is crucial here to use torch.cuda.synchronize(). This line of code performs synchronization between the host and device (i.e., GPU and CPU), so the time recording takes place only after the process running on the GPU is finished. This overcomes the issue of unsynchronized execution.
+
+model = EfficientNet.from_pretrained('efficientnet-b0')
+device = torch.device("cuda")
+model.to(device)
+dummy_input = torch.randn(1, 3,224,224, dtype=torch.float).to(device)
+
+# INIT LOGGERS
+starter, ender = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True)
+repetitions = 300
+timings=np.zeros((repetitions,1))
+#GPU-WARM-UP
+for _ in range(10):
+    _ = model(dummy_input)
+# MEASURE PERFORMANCE
+with torch.no_grad():
+    for rep in range(repetitions):
+        starter.record()
+        _ = model(dummy_input)
+        ender.record()
+                                                                                                                                                                                          1,1           Top
+                                                                                                                                                                                    

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,35 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}