shainaraza commited on
Commit
f069dac
1 Parent(s): 0f3a69a

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +4 -0
  2. special_tokens_map.json +4 -0
  3. tokenizer.json +18 -0
added_tokens.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "<BIASED>": 30522,
3
+ "<NON_BIASED>": 30523
4
+ }
special_tokens_map.json CHANGED
@@ -1,4 +1,8 @@
1
  {
 
 
 
 
2
  "cls_token": "[CLS]",
3
  "mask_token": "[MASK]",
4
  "pad_token": "[PAD]",
 
1
  {
2
+ "additional_special_tokens": [
3
+ "<BIASED>",
4
+ "<NON_BIASED>"
5
+ ],
6
  "cls_token": "[CLS]",
7
  "mask_token": "[MASK]",
8
  "pad_token": "[PAD]",
tokenizer.json CHANGED
@@ -59,6 +59,24 @@
59
  "rstrip": false,
60
  "normalized": false,
61
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
  }
63
  ],
64
  "normalizer": {
 
59
  "rstrip": false,
60
  "normalized": false,
61
  "special": true
62
+ },
63
+ {
64
+ "id": 30522,
65
+ "content": "<BIASED>",
66
+ "single_word": false,
67
+ "lstrip": false,
68
+ "rstrip": false,
69
+ "normalized": true,
70
+ "special": false
71
+ },
72
+ {
73
+ "id": 30523,
74
+ "content": "<NON_BIASED>",
75
+ "single_word": false,
76
+ "lstrip": false,
77
+ "rstrip": false,
78
+ "normalized": true,
79
+ "special": false
80
  }
81
  ],
82
  "normalizer": {