File size: 711 Bytes
d9113e7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
{
"architectures": [
"GPTNeoCodebookModel"
],
"codebook_at": [
"attn",
"mlp"
],
"codebook_kwargs": {},
"codebook_type": [
"vanilla",
"vanilla"
],
"k_codebook": [
16,
16
],
"kmeans_init": false,
"kmeans_init_examples": 1000,
"kmeans_kwargs": {
"batch_size": 24576,
"n_init": "auto"
},
"kmeans_path": "/.cache/cb_volume/huggingface/kmeans_embeddings.pt",
"layers_to_snap": [
0
],
"loss": "aeloss",
"model_type": "codebook",
"num_codebooks": [
1,
1
],
"num_codes": [
10000,
10000
],
"replace_codes": false,
"similarity_metric": "inner_product",
"torch_dtype": "float32",
"transformers_version": "4.35.2"
}
|