| gpu,task,model_name,dtype,offload,Greedy,Assisted | |
| 3090,OPT: Open Text Generation,1.3B,FP32,0,11.64,10.01 | |
| 3090,OPT: Open Text Generation,6.7B,FP32,1,428.47,114.99 | |
| 3090,OPT: Open Text Generation,6.7B,FP16,0,19.62,12.44 | |
| 3090,OPT: Open Text Generation,6.7B,INT8,0,104.43,40.33 | |
| 3090,OPT: Open Text Generation,30B,FP16,1,2616,1099 | |
| 3090,OPT: Summarization,1.3B,FP32,0,13.16,10.89 | |
| 3090,OPT: Summarization,6.7B,FP32,1,587.8,114.53 | |
| 3090,OPT: Summarization,6.7B,FP16,0,25.14,14.56 | |
| 3090,OPT: Summarization,30B,FP16,1,2732,331.2 | |
| 3090,Whisper: ARS,large-v2,FP32,0,24.81,12.55 | |
| 3090,CodeGen: Code Generation,2B,FP32,0,28.90,28.36 | |
| 3090,CodeGen: Code Generation,6B,FP32,1,544.11,110.42 | |
| 3090,CodeGen: Code Generation,6B,FP16,0,34.36,31.84 | |
| 3090,CodeGen: Code Generation,16B,FP16,1,808.69,161.50 | |
| 3090,CodeGen: Code Generation,16B,INT8,0,66.69,41.47 | |
| 3090,Flan-T5: Summarization,large,FP32,0,21.27,15.76 | |
| 3090,Flan-T5: Summarization,xl,FP32,0,25.60,18.94 | |
| 3090,Flan-T5: Summarization,xxl,FP32,1,1326.22,580.10 | |
| 3090,Flan-T5: Summarization,xxl,FP16,1,52.52,36.07 | |
| 3090,Flan-T5: Summarization,xxl,INT8,0,67.13,38.92 | |
| 3090,Flan-T5: Summarization,ul2,FP16,1,1185.25,480.11 | |
| T4,OPT: Open Text Generation,1.3B,FP32,0,24.74,22.37 | |
| T4,OPT: Open Text Generation,6.7B,FP32,1,2863.57,733.32 | |
| T4,OPT: Open Text Generation,6.7B,FP16,0,62.04,29.67 | |
| T4,OPT: Open Text Generation,6.7B,INT8,0,180.59,66.12 | |
| T4,OPT: Summarization,1.3B,FP32,0,32.50,26.58 | |
| T4,OPT: Summarization,6.7B,FP16,1,499.00,67.33 | |
| T4,OPT: Summarization,6.7B,INT8,0,182.98,37.89 | |
| T4,Whisper: ARS,large-v2,FP32,0,62.68,40.74 | |
| T4,CodeGen: Code Generation,2B,FP32,0,73.88,67.62 | |
| T4,CodeGen: Code Generation,6B,FP16,1,682.94,135.99 | |
| T4,CodeGen: Code Generation,6B,INT8,0,117.91,72.40 | |
| T4,Flan-T5: Summarization,large,FP32,0,43.67,36.26 | |
| T4,Flan-T5: Summarization,xl,FP16,0,53.54,42.27 | |
| T4,Flan-T5: Summarization,xxl,FP16,1,2814,1177 | |
| T4 *2,OPT: Open Text Generation,6.7B,FP32,0,118.42,55.42 | |
| T4 *2,OPT: Open Text Generation,6.7B,FP16,0,61.30,34.76 | |
| T4 *2,OPT: Summarization,6.7B,FP32,1,1238.59,339.34 | |
| T4 *2,OPT: Summarization,6.7B,FP16,0,94.62,34.37 | |
| T4 *2,CodeGen: Code Generation,6B,FP16,0,116.34,72.09 | |
| T4 *2,CodeGen: Code Generation,6B,INT8,0,119.14,79.01 | |
| T4 *2,CodeGen: Code Generation,16B,FP16,1,1509.05,693.01 | |
| T4 *2,CodeGen: Code Generation,16B,INT8,0,200.79,99.00 | |
| T4 *2,Flan-T5: Summarization,xl,FP32,0,59.27,68.70 | |
| T4 *2,Flan-T5: Summarization,xl,FP16,0,51.59,50.56 | |
| T4 *2,Flan-T5: Summarization,xxl,FP16,1,797.7,534.3 | |
| T4 *2,Flan-T5: Summarization,xxl,INT8,0,243.3,143.38 | |
| A100 (80GB),OPT: Open Text Generation,6.7B,FP32,0,35.34,30.00 | |
| A100 (80GB),OPT: Open Text Generation,30B,FP16,0,54.57,38.27 | |
| A100 (80GB),OPT: Open Text Generation,30B,INT8,0,290.82,135.77 | |
| A100 (80GB),OPT: Open Text Generation,66B,INT8,0,398.49,146.04 | |
| A100 (80GB),OPT: Summarization,6.7B,FP32,0,43.64,27.03 | |
| A100 (80GB),OPT: Summarization,30B,FP16,0,54.94,28.87 | |
| A100 (80GB),OPT: Summarization,30B,INT8,0,291.57,49.42 | |
| A100 (80GB),OPT: Summarization,66B,INT8,0,392.34,82.29 | |
| A100 (80GB),CodeGen: Code Generation,16B,FP32,0,75.56,80.44 | |
| A100 (80GB),CodeGen: Code Generation,16B,FP16,0,70.51,74.79 | |
| A100 (80GB),CodeGen: Code Generation,16B,INT8,0,130.77,90.28 | |
| A100 (80GB),Flan-T5: Summarization,ul2,FP32,0,87.40,59.26 | |
| A100 (80GB),Flan-T5: Summarization,ul2,FP16,0,78.13,42.95 | |
| A100 (80GB),Flan-T5: Summarization,ul2,INT8,0,187.66,81.72 | |