Fix cache max_seq_len

app.py

@@ -414,7 +414,7 @@ def build_app():
                             label="Maximum tokens per batch, 0 means no limit",
                             minimum=0,
                             maximum=2048,
-                            value=1024,  # 0 means no limit
+                            value=0,  # 0 means no limit
                             step=8,
                         )
 
@@ -640,7 +640,7 @@ if __name__ == "__main__":
             reference_audio=None,
             reference_text="",
             max_new_tokens=0,
-            chunk_length=100,
+            chunk_length=200,
             top_p=0.7,
             repetition_penalty=1.2,
             temperature=0.7,

tools/llama/generate.py

@@ -250,9 +250,11 @@ def generate(
     device, dtype = prompt.device, prompt.dtype
     with torch.device(device):
         model.setup_caches(
-            max_batch_size=1, max_seq_len=T_new, dtype=next(model.parameters()).dtype
+                max_batch_size=1,
+                max_seq_len=model.config.max_seq_len,
+                dtype=next(model.parameters()).dtype,
         )
-
+        
     codebook_dim = 1 + model.config.num_codebooks
     # create an empty tensor of the expected final shape and fill in the current tokens
     empty = torch.empty((codebook_dim, T_new), dtype=dtype, device=device)