Spaces:
Sleeping
Sleeping
jonathanjordan21
commited on
Commit
•
3e08cc6
1
Parent(s):
a8d0d52
Update app.py
Browse files
app.py
CHANGED
@@ -11,7 +11,8 @@ def load_tok_and_data(lan):
|
|
11 |
tokenizer._src_lang = _Tokens[lan]
|
12 |
tokenizer.cur_lang_code_id = tokenizer.convert_tokens_to_ids(_Tokens[lan])
|
13 |
tokenizer.set_src_lang_special_tokens(_Tokens[lan])
|
14 |
-
dataset = load_dataset('Babelscape/SREDFM', lan, split="test", streaming=True)
|
|
|
15 |
dataset = [example for example in dataset.take(1001)]
|
16 |
return (tokenizer, dataset)
|
17 |
|
|
|
11 |
tokenizer._src_lang = _Tokens[lan]
|
12 |
tokenizer.cur_lang_code_id = tokenizer.convert_tokens_to_ids(_Tokens[lan])
|
13 |
tokenizer.set_src_lang_special_tokens(_Tokens[lan])
|
14 |
+
dataset = load_dataset('Babelscape/SREDFM', lan, split="test", streaming=True, trust_remote_code=True)
|
15 |
+
|
16 |
dataset = [example for example in dataset.take(1001)]
|
17 |
return (tokenizer, dataset)
|
18 |
|