Hugging Face's logo

tangledgroup
/
tangled-llama-q-128k-base-v0.1

Text Generation
Transformers
llama
litgpt
litdata
conversational
Inference Endpoints
Model card Files Community
tangled-llama-q-128k-base-v0.1
File size: 1,329 Bytes 
b85057c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
model_name: Llama-3.2-1B
model_config:
  name: ''
  hf_config: {}
  scale_embeddings: false
  block_size: 8192
  vocab_size: 38400
  padding_multiple: 512
  padded_vocab_size: 38400
  n_layer: 8
  n_head: 32
  head_size: 64
  n_embd: 512
  rotary_percentage: 1.0
  parallel_residual: false
  bias: false
  lm_head_bias: false
  n_query_groups: 8
  shared_attention_norm: false
  norm_class_name: RMSNorm
  post_attention_norm: false
  post_mlp_norm: false
  norm_eps: 1.0e-05
  mlp_class_name: LLaMAMLP
  gelu_approximate: none
  intermediate_size: 2048
  rope_condense_ratio: 1
  rope_base: 500000
  n_expert: 0
  n_expert_per_token: 0
out_dir: ../out/pretrain
precision: bf16-true
resume: auto
data:
  class_path: litgpt.data.LitData
  init_args:
    data_path: ../pretrain-data/
    seed: 42
    num_workers: 32
train:
  save_interval: 200
  log_interval: 1
  global_batch_size: 512
  micro_batch_size: 15
  lr_warmup_steps: 2000
  max_tokens: 16698858240
  max_seq_length: 2049
  tie_embeddings: true
  max_norm: 1.0
  min_lr: 4.0e-05
eval:
  interval: 100
  max_iters: 100
  initial_validation: false
  final_validation: true
optimizer:
  class_path: grokadamw.GrokAdamW
  init_args:
    lr: 0.01
    weight_decay: 0.1
    betas:
    - 0.9
    - 0.95
devices: auto
num_nodes: 1
tokenizer_dir: ..
logger_name: wandb
seed: 42