ylacombe HF staff Xenova HF staff commited on
Commit
c6bd200
1 Parent(s): c817153

Upload tokenizer.json with huggingface_hub (#2)

Browse files

- Upload tokenizer.json with huggingface_hub (a5f8b3e4962b7ee43afbca28697aec17fd0ead88)


Co-authored-by: Joshua <[email protected]>

Files changed (1) hide show
  1. tokenizer.json +116 -0
tokenizer.json ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 58,
8
+ "content": "<unk>",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ }
15
+ ],
16
+ "normalizer": {
17
+ "type": "Sequence",
18
+ "normalizers": [
19
+ {
20
+ "type": "Lowercase"
21
+ },
22
+ {
23
+ "type": "Replace",
24
+ "pattern": {
25
+ "Regex": "[^3\u0b85\u0b88\u0b89_\u0ba8\u0b95 \u0bc7\u0bbe\u0bc6\u0b8a\u0bc0'7a\u0b8f\u0bb5\u0bcd\u0bb9\u0bc2\u0bb2\u0b9f2\u0bb4\u0bb0\u0bae\u0b92\u0bb1\u0b9e\u0b9a9\u0bbf\u0bb8\u0bc8\u0ba96\u0b8e\u0b87\u0baf\u0bcc\u0ba4\u0bcb\u0bb31\u0b86\u0baa0\u0ba3\u0b93\u0b99\u0bc1\u0b90\u0b9c4\u0bb7\u0bca5]"
26
+ },
27
+ "content": ""
28
+ },
29
+ {
30
+ "type": "Strip",
31
+ "strip_left": true,
32
+ "strip_right": true
33
+ },
34
+ {
35
+ "type": "Replace",
36
+ "pattern": {
37
+ "Regex": "(?=.)|(?<!^)$"
38
+ },
39
+ "content": "3"
40
+ }
41
+ ]
42
+ },
43
+ "pre_tokenizer": {
44
+ "type": "Split",
45
+ "pattern": {
46
+ "Regex": ""
47
+ },
48
+ "behavior": "Isolated",
49
+ "invert": false
50
+ },
51
+ "post_processor": null,
52
+ "decoder": null,
53
+ "model": {
54
+ "vocab": {
55
+ "3": 0,
56
+ "\u0b85": 1,
57
+ "\u0b88": 2,
58
+ "\u0b89": 3,
59
+ "_": 4,
60
+ "\u0ba8": 5,
61
+ "\u0b95": 6,
62
+ " ": 7,
63
+ "\u0bc7": 8,
64
+ "\u0bbe": 9,
65
+ "\u0bc6": 10,
66
+ "\u0b8a": 11,
67
+ "\u0bc0": 12,
68
+ "'": 13,
69
+ "7": 14,
70
+ "a": 15,
71
+ "\u0b8f": 16,
72
+ "\u0bb5": 17,
73
+ "\u0bcd": 18,
74
+ "\u0bb9": 19,
75
+ "\u0bc2": 20,
76
+ "\u0bb2": 21,
77
+ "\u0b9f": 22,
78
+ "2": 23,
79
+ "\u0bb4": 24,
80
+ "\u0bb0": 25,
81
+ "\u0bae": 26,
82
+ "\u0b92": 27,
83
+ "\u0bb1": 28,
84
+ "\u0b9e": 29,
85
+ "\u0b9a": 30,
86
+ "9": 31,
87
+ "\u0bbf": 32,
88
+ "\u0bb8": 33,
89
+ "\u0bc8": 34,
90
+ "\u0ba9": 35,
91
+ "6": 36,
92
+ "\u0b8e": 37,
93
+ "\u0b87": 38,
94
+ "\u0baf": 39,
95
+ "\u0bcc": 40,
96
+ "\u0ba4": 41,
97
+ "\u0bcb": 42,
98
+ "\u0bb3": 43,
99
+ "1": 44,
100
+ "\u0b86": 45,
101
+ "\u0baa": 46,
102
+ "0": 47,
103
+ "\u0ba3": 48,
104
+ "\u0b93": 49,
105
+ "\u0b99": 50,
106
+ "\u0bc1": 51,
107
+ "\u0b90": 52,
108
+ "\u0b9c": 53,
109
+ "4": 54,
110
+ "\u0bb7": 55,
111
+ "\u0bca": 56,
112
+ "5": 57,
113
+ "<unk>": 58
114
+ }
115
+ }
116
+ }