MaMaL-Com / added_tokens.json
hanbin's picture
Upload 7 files
39d3867
raw
history blame
5.72 kB
{"<pad>": 50257, "<s>": 50258, "</s>": 50259, "<EOL>": 50260, "<STR_LIT>": 50261, "<NUM_LIT>": 50262, "<CHAR_LIT>": 50263, "<STR_LIT:name>": 50264, "<STR_LIT:\\n>": 50265, "<STR_LIT:U+0020>": 50266, "<STR_LIT:True>": 50267, "<STR_LIT:a>": 50268, "<STR_LIT:type>": 50269, "<STR_LIT:__main__>": 50270, "<STR_LIT:id>": 50271, "<STR_LIT:/>": 50272, "<STR_LIT:.>": 50273, "<STR_LIT:w>": 50274, "<STR_LIT:utf-8>": 50275, "<STR_LIT:b>": 50276, "<STR_LIT:foo>": 50277, "<STR_LIT:1>": 50278, "<STR_LIT:r>": 50279, "<STR_LIT:default>": 50280, "<STR_LIT:U+002C>": 50281, "<STR_LIT:->": 50282, "<STR_LIT:data>": 50283, "<STR_LIT:0>": 50284, "<STR_LIT:%s>": 50285, "<STR_LIT:x>": 50286, "<STR_LIT:blank>": 50287, "<STR_LIT::>": 50288, "<STR_LIT:value>": 50289, "<STR_LIT:max_length>": 50290, "<STR_LIT:c>": 50291, "<STR_LIT:_>": 50292, "<STR_LIT:test>": 50293, "<STR_LIT:A>": 50294, "<STR_LIT:store_true>": 50295, "<STR_LIT:description>": 50296, "<STR_LIT:*>": 50297, "<STR_LIT:null>": 50298, "<STR_LIT:title>": 50299, "<STR_LIT:y>": 50300, "<STR_LIT:U+002CU+0020>": 50301, "<STR_LIT:true>": 50302, "<STR_LIT:key>": 50303, "<STR_LIT:bar>": 50304, "<STR_LIT:=>": 50305, "<STR_LIT:string>": 50306, "<STR_LIT:status>": 50307, "<STR_LIT:2>": 50308, "<STR_LIT:rb>": 50309, "<STR_LIT:to>": 50310, "<STR_LIT:B>": 50311, "<STR_LIT:url>": 50312, "<STR_LIT:d>": 50313, "<STR_LIT:\">": 50314, "<STR_LIT:version>": 50315, "<STR_LIT:str>": 50316, "<STR_LIT:abc>": 50317, "<STR_LIT:int>": 50318, "<STR_LIT:GET>": 50319, "<STR_LIT:text>": 50320, "<STR_LIT:error>": 50321, "<STR_LIT:user>": 50322, "<STR_LIT:C>": 50323, "<STR_LIT:)>": 50324, "<STR_LIT:wb>": 50325, "<STR_LIT:\\t>": 50326, "<STR_LIT:+>": 50327, "<STR_LIT:\\\\>": 50328, "<STR_LIT:#>": 50329, "<STR_LIT:>>": 50330, "<STR_LIT:f>": 50331, "<STR_LIT:;>": 50332, "<STR_LIT:path>": 50333, "<STR_LIT:ascii>": 50334, "<STR_LIT:POST>": 50335, "<STR_LIT:?>": 50336, "<STR_LIT:3>": 50337, "<STR_LIT:False>": 50338, "<STR_LIT:D>": 50339, "<STR_LIT:|>": 50340, "<STR_LIT:label>": 50341, "<STR_LIT:s>": 50342, "<STR_LIT:(>": 50343, "<STR_LIT:message>": 50344, "<STR_LIT:'>": 50345, "<STR_LIT:i>": 50346, "<STR_LIT:primary_key>": 50347, "<STR_LIT:<>": 50348, "<STR_LIT:Meta>": 50349, "<STR_LIT:N>": 50350, "<STR_LIT:e>": 50351, "<STR_LIT:object_name>": 50352, "<STR_LIT:password>": 50353, "<STR_LIT:index>": 50354, "<STR_LIT:..>": 50355, "<STR_LIT:code>": 50356, "<STR_LIT:class>": 50357, "<STR_LIT:username>": 50358, "<STR_LIT:F>": 50359, "<STR_LIT:size>": 50360, "<STR_LIT:}>": 50361, "<STR_LIT:n>": 50362, "<STR_LIT:X>": 50363, "<STR_LIT:Y>": 50364, "<STR_LIT:file>": 50365, "<STR_LIT:z>": 50366, "<STR_LIT:[>": 50367, "<STR_LIT:store>": 50368, "<STR_LIT:none>": 50369, "<STR_LIT:html>": 50370, "<STR_LIT:]>": 50371, "<STR_LIT:utf8>": 50372, "<STR_LIT:p>": 50373, "<STR_LIT:strict>": 50374, "<STR_LIT:\\r\\n>": 50375, "<STR_LIT:end>": 50376, "<STR_LIT:all>": 50377, "<STR_LIT:Name>": 50378, "<STR_LIT:state>": 50379, "<STR_LIT:produces>": 50380, "<STR_LIT:m>": 50381, "<STR_LIT:t>": 50382, "<STR_LIT:I>": 50383, "<STR_LIT:result>": 50384, "<STR_LIT:bool>": 50385, "<STR_LIT:date>": 50386, "<STR_LIT:{}>": 50387, "<STR_LIT:\\x00>": 50388, "<STR_LIT:hello>": 50389, "<STR_LIT:start>": 50390, "<STR_LIT:left>": 50391, "<STR_LIT:email>": 50392, "<STR_LIT:ignore>": 50393, "<STR_LIT:S>": 50394, "<STR_LIT:@>": 50395, "<STR_LIT:L>": 50396, "<STR_LIT:false>": 50397, "<STR_LIT:action>": 50398, "<STR_LIT:{>": 50399, "<STR_LIT:related_name>": 50400, "<STR_LIT:time>": 50401, "<STR_LIT:count>": 50402, "<STR_LIT:v>": 50403, "<STR_LIT:4>": 50404, "<STR_LIT:relu>": 50405, "<STR_LIT:win32>": 50406, "<STR_LIT:port>": 50407, "<STR_LIT:h>": 50408, "<STR_LIT:#pop>": 50409, "<STR_LIT:source>": 50410, "<STR_LIT:Content-Type>": 50411, "<STR_LIT:application/json>": 50412, "<STR_LIT:&>": 50413, "<STR_LIT:src>": 50414, "<STR_LIT:O>": 50415, "<STR_LIT:-c>": 50416, "<STR_LIT:H>": 50417, "<STR_LIT:o>": 50418, "<STR_LIT:g>": 50419, "<STR_LIT:None>": 50420, "<STR_LIT:E>": 50421, "<STR_LIT:M>": 50422, "<STR_LIT:host>": 50423, "<STR_LIT:body>": 50424, "<STR_LIT:%>": 50425, "<STR_LIT:info>": 50426, "<STR_LIT:image>": 50427, "<STR_LIT:float>": 50428, "<STR_LIT:root>": 50429, "<STR_LIT:l>": 50430, "<STR_LIT:T>": 50431, "<STR_LIT:replace>": 50432, "<STR_LIT:k>": 50433, "<STR_LIT:\\ufffe>": 50434, "<STR_LIT:100>": 50435, "<STR_LIT:5>": 50436, "<STR_LIT:$>": 50437, "<STR_LIT:right>": 50438, "<STR_LIT:content>": 50439, "<STR_LIT:q>": 50440, "<STR_LIT:args>": 50441, "<STR_LIT:u>": 50442, "<STR_LIT:address>": 50443, "<STR_LIT:\\r>": 50444, "<STR_LIT:list>": 50445, "<STR_LIT:target>": 50446, "<STR_LIT:http>": 50447, "<STR_LIT:P>": 50448, "<STR_LIT:object>": 50449, "<STR_LIT:yes>": 50450, "<STR_LIT:1.0>": 50451, "<STR_LIT:R>": 50452, "<STR_LIT:width>": 50453, "<STR_LIT:localhost>": 50454, "<STR_LIT:127.0.0.1>": 50455, "<STR_LIT:train>": 50456, "<STR_LIT:[?]>": 50457, "<STR_LIT:OK>": 50458, "<STR_LIT:location>": 50459, "<STR_LIT:filename>": 50460, "<STR_LIT:!>": 50461, "<STR_LIT:success>": 50462, "<STR_LIT:input>": 50463, "<NUM_LIT:0>": 50464, "<NUM_LIT:1>": 50465, "<NUM_LIT:2>": 50466, "<NUM_LIT:3>": 50467, "<NUM_LIT:4>": 50468, "<NUM_LIT:5>": 50469, "<NUM_LIT:10>": 50470, "<NUM_LIT:6>": 50471, "<NUM_LIT:8>": 50472, "<NUM_LIT:1.0>": 50473, "<NUM_LIT:0.0>": 50474, "<NUM_LIT:7>": 50475, "<NUM_LIT:100>": 50476, "<NUM_LIT:9>": 50477, "<NUM_LIT:255>": 50478, "<NUM_LIT:16>": 50479, "<NUM_LIT:0.5>": 50480, "<NUM_LIT:20>": 50481, "<NUM_LIT:12>": 50482, "<NUM_LIT:32>": 50483, "<NUM_LIT:11>": 50484, "<NUM_LIT:200>": 50485, "<NUM_LIT:1000>": 50486, "<NUM_LIT:30>": 50487, "<NUM_LIT:1.>": 50488, "<NUM_LIT:15>": 50489, "<NUM_LIT:64>": 50490, "<NUM_LIT:0.1>": 50491, "<NUM_LIT:50>": 50492, "<NUM_LIT:0.>": 50493}