Hugging Face's logo

suriya7
/
ChatGPT-2.V2

Text Generation
Transformers
Safetensors
gpt2
conversational-ai
fine-tuning
causal-lm
chatbots
text-generation-inference
Inference Endpoints
Model card Files Community
suriya7 commited on
Commit
c63f016
•
1 Parent(s): 14c9f5f

Upload tokenizer

Browse files
Files changed (4) hide show
  1. special_tokens_map.json +3 -32
  2. tokenizer.json +2 -41
  3. tokenizer_config.json +1 -34
  4. vocab.json +0 -0
special_tokens_map.json CHANGED
@@ -1,34 +1,5 @@
1
  {
2
- "additional_special_tokens": [
3
- "<|im_start|>",
4
- "<|im_end|>"
5
- ],
6
- "bos_token": {
7
- "content": "<|endoftext|>",
8
- "lstrip": false,
9
- "normalized": true,
10
- "rstrip": false,
11
- "single_word": false
12
- },
13
- "eos_token": {
14
- "content": "<|endoftext|>",
15
- "lstrip": false,
16
- "normalized": true,
17
- "rstrip": false,
18
- "single_word": false
19
- },
20
- "pad_token": {
21
- "content": "<|endoftext|>",
22
- "lstrip": false,
23
- "normalized": true,
24
- "rstrip": false,
25
- "single_word": false
26
- },
27
- "unk_token": {
28
- "content": "<|endoftext|>",
29
- "lstrip": false,
30
- "normalized": true,
31
- "rstrip": false,
32
- "single_word": false
33
- }
34
  }
 
1
  {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "unk_token": "<|endoftext|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  }
tokenizer.json CHANGED
@@ -1,19 +1,7 @@
1
  {
2
  "version": "1.0",
3
- "truncation": {
4
- "direction": "Right",
5
- "max_length": 512,
6
- "strategy": "LongestFirst",
7
- "stride": 0
8
- },
9
- "padding": {
10
- "strategy": "BatchLongest",
11
- "direction": "Right",
12
- "pad_to_multiple_of": null,
13
- "pad_id": 50256,
14
- "pad_type_id": 0,
15
- "pad_token": "<|endoftext|>"
16
- },
17
  "added_tokens": [
18
  {
19
  "id": 50256,
@@ -23,33 +11,6 @@
23
  "rstrip": false,
24
  "normalized": true,
25
  "special": true
26
- },
27
- {
28
- "id": 50257,
29
- "content": "[PAD]",
30
- "single_word": false,
31
- "lstrip": false,
32
- "rstrip": false,
33
- "normalized": false,
34
- "special": true
35
- },
36
- {
37
- "id": 50258,
38
- "content": "<|im_start|>",
39
- "single_word": false,
40
- "lstrip": false,
41
- "rstrip": false,
42
- "normalized": false,
43
- "special": true
44
- },
45
- {
46
- "id": 50259,
47
- "content": "<|im_end|>",
48
- "single_word": false,
49
- "lstrip": false,
50
- "rstrip": false,
51
- "normalized": false,
52
- "special": true
53
  }
54
  ],
55
  "normalizer": null,
 
1
  {
2
  "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
 
 
 
 
 
 
 
 
 
 
 
 
5
  "added_tokens": [
6
  {
7
  "id": 50256,
 
11
  "rstrip": false,
12
  "normalized": true,
13
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  }
15
  ],
16
  "normalizer": null,
tokenizer_config.json CHANGED
@@ -1,6 +1,4 @@
1
  {
2
- "add_bos_token": false,
3
- "add_eos_token": true,
4
  "add_prefix_space": false,
5
  "added_tokens_decoder": {
6
  "50256": {
@@ -10,43 +8,12 @@
10
  "rstrip": false,
11
  "single_word": false,
12
  "special": true
13
- },
14
- "50257": {
15
- "content": "[PAD]",
16
- "lstrip": false,
17
- "normalized": false,
18
- "rstrip": false,
19
- "single_word": false,
20
- "special": true
21
- },
22
- "50258": {
23
- "content": "<|im_start|>",
24
- "lstrip": false,
25
- "normalized": false,
26
- "rstrip": false,
27
- "single_word": false,
28
- "special": true
29
- },
30
- "50259": {
31
- "content": "<|im_end|>",
32
- "lstrip": false,
33
- "normalized": false,
34
- "rstrip": false,
35
- "single_word": false,
36
- "special": true
37
  }
38
  },
39
- "additional_special_tokens": [
40
- "<|im_start|>",
41
- "<|im_end|>"
42
- ],
43
  "bos_token": "<|endoftext|>",
44
  "clean_up_tokenization_spaces": false,
45
  "eos_token": "<|endoftext|>",
46
- "errors": "replace",
47
- "model_max_length": 512,
48
- "pad_token": "<|endoftext|>",
49
- "padding_side": "right",
50
  "tokenizer_class": "GPT2Tokenizer",
51
  "unk_token": "<|endoftext|>"
52
  }
 
1
  {
 
 
2
  "add_prefix_space": false,
3
  "added_tokens_decoder": {
4
  "50256": {
 
8
  "rstrip": false,
9
  "single_word": false,
10
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  }
12
  },
 
 
 
 
13
  "bos_token": "<|endoftext|>",
14
  "clean_up_tokenization_spaces": false,
15
  "eos_token": "<|endoftext|>",
16
+ "model_max_length": 1024,
 
 
 
17
  "tokenizer_class": "GPT2Tokenizer",
18
  "unk_token": "<|endoftext|>"
19
  }
vocab.json CHANGED
The diff for this file is too large to render. See raw diff