File size: 923 Bytes
6c3b177
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
{
	"bos_token": {
		"content": "<s>",
		"single_word": false,
		"lstrip": false,
		"rstrip": false,
		"normalized": true
	},
	"eos_token": {
		"content": "</s>",
		"single_word": false,
		"lstrip": false,
		"rstrip": false,
		"normalized": true
	},
	"unk_token": {
		"content": "<unk>",
		"single_word": false,
		"lstrip": false,
		"rstrip": false,
		"normalized": true
	},
	"sep_token": {
		"content": "</s>",
		"single_word": false,
		"lstrip": false,
		"rstrip": false,
		"normalized": true
	},
	"pad_token": {
		"content": "<pad>",
		"single_word": false,
		"lstrip": false,
		"rstrip": false,
		"normalized": true
	},
	"cls_token": {
		"content": "<s>",
		"single_word": false,
		"lstrip": false,
		"rstrip": false,
		"normalized": true
	},
	"mask_token": {
		"content": "<mask>",
		"single_word": false,
		"lstrip": true,
		"rstrip": false,
		"normalized": true
	},
	"additional_special_tokens": ["<ent>", "<ent2>"]
}