| quantizations: | |
| - IQ2_XS | |
| - IQ3_M | |
| - IQ3_S | |
| - IQ3_XS | |
| - IQ3_XXS | |
| - IQ4_NL | |
| - IQ4_XS | |
| - Q3_K_L | |
| - Q3_K_M | |
| - Q3_K_S | |
| - Q4_K_M | |
| - Q4_K_S | |
| - Q5_K_M | |
| - Q5_K_S | |
| - Q6_K | |
| - Q8_0 | |
| - TQ1_0 | |
| - TQ2_0 | |
| allowed_quantization_types: | |
| - name: Q4_0 | |
| size: 4.34G | |
| ppl: +0.4685 | |
| details: Llama-3-8B | |
| - name: Q4_1 | |
| size: 4.78G | |
| ppl: +0.4511 | |
| details: Llama-3-8B | |
| - name: Q5_0 | |
| size: 5.21G | |
| ppl: +0.1316 | |
| details: Llama-3-8B | |
| - name: Q5_1 | |
| size: 5.65G | |
| ppl: +0.1062 | |
| details: Llama-3-8B | |
| - name: IQ2_XXS | |
| size: "2.06 bpw" | |
| type: quantization | |
| - name: IQ2_XS | |
| size: "2.31 bpw" | |
| type: quantization | |
| - name: IQ2_S | |
| size: "2.5 bpw" | |
| type: quantization | |
| - name: IQ2_M | |
| size: "2.7 bpw" | |
| type: quantization | |
| - name: IQ1_S | |
| size: "1.56 bpw" | |
| type: quantization | |
| - name: IQ1_M | |
| size: "1.75 bpw" | |
| type: quantization | |
| - name: TQ1_0 | |
| size: "1.69 bpw" | |
| type: ternarization | |
| - name: TQ2_0 | |
| size: "2.06 bpw" | |
| type: ternarization | |
| - name: Q2_K | |
| size: 2.96G | |
| ppl: +3.5199 | |
| details: Llama-3-8B | |
| - name: Q2_K_S | |
| size: 2.96G | |
| ppl: +3.1836 | |
| details: Llama-3-8B | |
| - name: IQ3_XXS | |
| size: "3.06 bpw" | |
| type: quantization | |
| - name: IQ3_S | |
| size: "3.44 bpw" | |
| type: quantization | |
| - name: IQ3_M | |
| size: "3.66 bpw" | |
| type: quantization mix | |
| - name: Q3_K | |
| alias: Q3_K_M | |
| - name: IQ3_XS | |
| size: "3.3 bpw" | |
| type: quantization | |
| - name: Q3_K_S | |
| size: 3.41G | |
| ppl: +1.6321 | |
| details: Llama-3-8B | |
| - name: Q3_K_M | |
| size: 3.74G | |
| ppl: +0.6569 | |
| details: Llama-3-8B | |
| - name: Q3_K_L | |
| size: 4.03G | |
| ppl: +0.5562 | |
| details: Llama-3-8B | |
| - name: IQ4_NL | |
| size: "4.50 bpw" | |
| type: non-linear quantization | |
| - name: IQ4_XS | |
| size: "4.25 bpw" | |
| type: non-linear quantization | |
| - name: Q4_K | |
| alias: Q4_K_M | |
| - name: Q4_K_S | |
| size: 4.37G | |
| ppl: +0.2689 | |
| details: Llama-3-8B | |
| - name: Q4_K_M | |
| size: 4.58G | |
| ppl: +0.1754 | |
| details: Llama-3-8B | |
| - name: Q5_K | |
| alias: Q5_K_M | |
| - name: Q5_K_S | |
| size: 5.21G | |
| ppl: +0.1049 | |
| details: Llama-3-8B | |
| - name: Q5_K_M | |
| size: 5.33G | |
| ppl: +0.0569 | |
| details: Llama-3-8B | |
| - name: Q6_K | |
| size: 6.14G | |
| ppl: +0.0217 | |
| details: Llama-3-8B | |
| - name: Q8_0 | |
| size: 7.96G | |
| ppl: +0.0026 | |
| details: Llama-3-8B | |
| - name: F16 | |
| size: 14.00G | |
| ppl: +0.0020 | |
| details: Mistral-7B | |
| - name: BF16 | |
| size: 14.00G | |
| ppl: -0.0050 | |
| details: Mistral-7B | |
| - name: F32 | |
| size: 26.00G | |
| details: 7B | |
| - name: COPY | |
| description: Only copy tensors, no quantizing | |