parameters: (ort_session,
                                               inputs: Gpt2Inputs,
                                               output_buffers: Dict[str, torch.Tensor],
                                               output_shapes: Dict[str, List[int]],
                                               total_runs: int = 0,
                                               return_numpy: bool = True,
                                               include_copy_output_latency: bool = False)
return_statement: return ort_outputs, average_latency
docstring: Inference with IO binding. Returns outputs, and optional latency when total_runs > 0.
docstring_summary: Inference with IO binding. Returns outputs, and optional latency when total_runs > 0.
docstring_tokens: Inference with IO binding . Returns outputs and optional latency when total_runs > 0 .
function: def onnxruntime_inference_with_binded_io(ort_session,
                                               inputs: Gpt2Inputs,
                                               output_buffers: Dict[str, torch.Tensor],
                                               output_shapes: Dict[str, List[int]],
                                               total_runs: int = 0,
                                               return_numpy: bool = True,
                                               include_copy_output_latency: bool = False):
          """ Inference with IO binding. Returns outputs, and optional latency when total_runs > 0.
          """
          logger.debug(f"start onnxruntime_inference_with_binded_io")
  
          # Bind inputs and outputs to onnxruntime session
          io_binding = Gpt2Helper.prepare_io_binding(ort_session, inputs.input_ids, inputs.position_ids,
                                                     inputs.attention_mask, inputs.past, output_buffers, output_shapes)
  
          # Run onnxruntime with io binding
          ort_session.run_with_iobinding(io_binding)
  
          # Copy results to cpu for verification
          ort_outputs = Gpt2Helper.get_outputs_from_io_binding_buffer(ort_session, output_buffers, output_shapes,
                                                                      return_numpy)
  
          if total_runs == 0:
              return ort_outputs
  
          latency = []
          for _ in range(total_runs):
              start = time.time()
              # Run onnxruntime with io binding
              ort_session.run_with_iobinding(io_binding)
              if include_copy_output_latency:
                  _ = Gpt2Helper.get_outputs_from_io_binding_buffer(ort_session, output_buffers, output_shapes,
                                                                    return_numpy)
              latency.append(time.time() - start)
  
          average_latency = sum(latency) * 1000 / len(latency)
          logger.debug("OnnxRuntime with IO binding inference time = {} ms".format(format(average_latency, '.2f')))
  
          return ort_outputs, average_latency
function_tokens: def onnxruntime_inference_with_binded_io ( ort_session , inputs : Gpt2Inputs , output_buffers : Dict [ str , torch . Tensor ] , output_shapes : Dict [ str , List [ int ] ] , total_runs : int = 0 , return_numpy : bool = True , include_copy_output_latency : bool = False ) : logger . debug ( f"start onnxruntime_inference_with_binded_io" ) # Bind inputs and outputs to onnxruntime session io_binding = Gpt2Helper . prepare_io_binding ( ort_session , inputs . input_ids , inputs . position_ids , inputs . attention_mask , inputs . past , output_buffers , output_shapes ) # Run onnxruntime with io binding ort_session . run_with_iobinding ( io_binding ) # Copy results to cpu for verification ort_outputs = Gpt2Helper . get_outputs_from_io_binding_buffer ( ort_session , output_buffers , output_shapes , return_numpy ) if total_runs == 0 : return ort_outputs latency = [ ] for _ in range ( total_runs ) : start = time . time ( ) # Run onnxruntime with io binding ort_session . run_with_iobinding ( io_binding ) if include_copy_output_latency : _ = Gpt2Helper . get_outputs_from_io_binding_buffer ( ort_session , output_buffers , output_shapes , return_numpy ) latency . append ( time . time ( ) - start ) average_latency = sum ( latency ) * 1000 / len ( latency ) logger . debug ( "OnnxRuntime with IO binding inference time = {} ms" . format ( format ( average_latency , '.2f' ) ) ) return ort_outputs , average_latency