added the tokenizer

README.md

@@ -1,4 +1,17 @@
 ---
+widget:
+- text: Great New World Equity LLC
+- text: Jerome J Glaser DBA International Curtain Call
+- text: PICKERING INTERNATIONAL, INC.
+- text: R & S BEVERAGE COMPANY, A CALIFORNIA LIMITED PARTNERSHIP
+- text: Kehilla Community Synagogue and School
+- text: Temarry Recycling, Inc.
+- text: Jacobus De Groot Dairy
+- text: United Brands Products Design Development and Marketing, Inc.
+- text: GREEN VALLEY COMMUNITY CHURCH
+- text: Sierra Central Credit Union
+- text: Tri Counties Bank
+- text: Lowkey Community LLC
 model-index:
 - name: Sociovestix/US-CA
   results:
@@ -20,19 +33,6 @@ model-index:
       name: f1 macro
       args:
         average: macro
-widget:
-- text: "Great New World Equity LLC"
-- text: "Jerome J Glaser DBA International Curtain Call"
-- text: "PICKERING INTERNATIONAL, INC."
-- text: "R & S BEVERAGE COMPANY, A CALIFORNIA LIMITED PARTNERSHIP"
-- text: "Kehilla Community Synagogue and School"
-- text: "Temarry Recycling, Inc."
-- text: "Jacobus De Groot Dairy"
-- text: "United Brands Products Design Development and Marketing, Inc."
-- text: "GREEN VALLEY COMMUNITY CHURCH"
-- text: "Sierra Central Credit Union"
-- text: "Tri Counties Bank"
-- text: "Lowkey Community LLC"
 ---
 
 # LENU - Legal Entity Name Understanding for US California

tokenizer_config.json

@@ -1,14 +1,53 @@
 {
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
-  "do_basic_tokenize": true,
   "do_lower_case": true,
   "mask_token": "[MASK]",
-  "model_max_length": 1000000000000000019884624838656,
-  "name_or_path": "yiyanghkust/finbert-pretrain",
-  "never_split": null,
+  "model_max_length": 512,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
-  "special_tokens_map_file": null,
   "strip_accents": null,
   "tokenize_chinese_chars": true,
   "tokenizer_class": "BertTokenizer",