altryne commited on
Commit
2f63725
1 Parent(s): a179016

change hebrew code from iw (deprecated) to he

Browse files

Signed-off-by: Alex Volkov <[email protected]>

README.md CHANGED
@@ -20,7 +20,7 @@ language:
20
  - hi
21
  - fi
22
  - vi
23
- - iw
24
  - uk
25
  - el
26
  - ms
 
20
  - hi
21
  - fi
22
  - vi
23
+ - he
24
  - uk
25
  - el
26
  - ms
added_tokens.json CHANGED
@@ -29,6 +29,7 @@
29
  "<|gu|>": 50333,
30
  "<|haw|>": 50352,
31
  "<|ha|>": 50354,
 
32
  "<|hi|>": 50276,
33
  "<|hr|>": 50291,
34
  "<|ht|>": 50339,
@@ -37,7 +38,6 @@
37
  "<|id|>": 50275,
38
  "<|is|>": 50311,
39
  "<|it|>": 50274,
40
- "<|iw|>": 50279,
41
  "<|ja|>": 50266,
42
  "<|jw|>": 50356,
43
  "<|ka|>": 50329,
 
29
  "<|gu|>": 50333,
30
  "<|haw|>": 50352,
31
  "<|ha|>": 50354,
32
+ "<|he|>": 50279,
33
  "<|hi|>": 50276,
34
  "<|hr|>": 50291,
35
  "<|ht|>": 50339,
 
38
  "<|id|>": 50275,
39
  "<|is|>": 50311,
40
  "<|it|>": 50274,
 
41
  "<|ja|>": 50266,
42
  "<|jw|>": 50356,
43
  "<|ka|>": 50329,
generation_config.json CHANGED
@@ -49,6 +49,7 @@
49
  "<|gu|>": 50333,
50
  "<|haw|>": 50352,
51
  "<|ha|>": 50354,
 
52
  "<|hi|>": 50276,
53
  "<|hr|>": 50291,
54
  "<|ht|>": 50339,
@@ -57,7 +58,6 @@
57
  "<|id|>": 50275,
58
  "<|is|>": 50311,
59
  "<|it|>": 50274,
60
- "<|iw|>": 50279,
61
  "<|ja|>": 50266,
62
  "<|jw|>": 50356,
63
  "<|ka|>": 50329,
 
49
  "<|gu|>": 50333,
50
  "<|haw|>": 50352,
51
  "<|ha|>": 50354,
52
+ "<|he|>": 50279,
53
  "<|hi|>": 50276,
54
  "<|hr|>": 50291,
55
  "<|ht|>": 50339,
 
58
  "<|id|>": 50275,
59
  "<|is|>": 50311,
60
  "<|it|>": 50274,
 
61
  "<|ja|>": 50266,
62
  "<|jw|>": 50356,
63
  "<|ka|>": 50329,
special_tokens_map.json CHANGED
@@ -22,7 +22,7 @@
22
  "<|hi|>",
23
  "<|fi|>",
24
  "<|vi|>",
25
- "<|iw|>",
26
  "<|uk|>",
27
  "<|el|>",
28
  "<|ms|>",
 
22
  "<|hi|>",
23
  "<|fi|>",
24
  "<|vi|>",
25
+ "<|he|>",
26
  "<|uk|>",
27
  "<|el|>",
28
  "<|ms|>",
tokenizer.json CHANGED
@@ -203,7 +203,7 @@
203
  },
204
  {
205
  "id": 50279,
206
- "content": "<|iw|>",
207
  "single_word": false,
208
  "lstrip": false,
209
  "rstrip": false,
 
203
  },
204
  {
205
  "id": 50279,
206
+ "content": "<|he|>",
207
  "single_word": false,
208
  "lstrip": false,
209
  "rstrip": false,