compress pretrain dataset
Browse files- scripts/requirements.in +2 -1
scripts/requirements.in
CHANGED
|
@@ -10,4 +10,5 @@ litgpt[all] @ git+https://github.com/Lightning-AI/litgpt.git
|
|
| 10 |
litdata
|
| 11 |
grokadamw
|
| 12 |
# bitsandbytes
|
| 13 |
-
pyzstd
|
|
|
|
|
|
| 10 |
litdata
|
| 11 |
grokadamw
|
| 12 |
# bitsandbytes
|
| 13 |
+
# pyzstd
|
| 14 |
+
zstd
|