add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"[PAD]": 50257, "[QUES]": 50262, "<|text|>": 50259, "[eot]": 50265, "<|question|>": 50258, "[ASPECT_SEP]": 50266, "[TEXT]": 50263, "<|answer_sep|>": 50261, "[ANSW]": 50264, "<|answer|>": 50260}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>", "pad_token": "[PAD]", "additional_special_tokens": ["<|question|>", "<|text|>", "<|answer|>", "<|answer_sep|>", "[QUES]", "[TEXT]", "[ANSW]", "[eot]", "[ASPECT_SEP]"]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "use_fast": true, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "/Users/stefanhg/Documents/UMich/Research/Zeroshot Text Classification/models-upload/2022-06-19_13-09-36_NVIDIA-GPT2-gpt2-medium-implicit-aspect-norm/trained", "tokenizer_class": "ZsGPT2Tokenizer"}