compress pretrain dataset
Browse files- scripts/requirements.in +2 -1
scripts/requirements.in
CHANGED
|
@@ -9,4 +9,5 @@ wandb
|
|
| 9 |
litgpt[all] @ git+https://github.com/Lightning-AI/litgpt.git
|
| 10 |
litdata
|
| 11 |
grokadamw
|
| 12 |
-
# bitsandbytes
|
|
|
|
|
|
| 9 |
litgpt[all] @ git+https://github.com/Lightning-AI/litgpt.git
|
| 10 |
litdata
|
| 11 |
grokadamw
|
| 12 |
+
# bitsandbytes
|
| 13 |
+
pyzstd
|