Spaces:
Runtime error
Runtime error
Disabling parallelism to tring avoid deadlocks in the hf tokenizer
Browse files
modules/module_languageModel.py
CHANGED
|
@@ -1,4 +1,7 @@
|
|
| 1 |
from transformers import AutoTokenizer, AutoModelForMaskedLM
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
class LanguageModel:
|
| 4 |
def __init__(
|
|
|
|
| 1 |
from transformers import AutoTokenizer, AutoModelForMaskedLM
|
| 2 |
+
import os
|
| 3 |
+
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
| 4 |
+
|
| 5 |
|
| 6 |
class LanguageModel:
|
| 7 |
def __init__(
|