bigcode_starcoder2-3b_10_ft_clm / added_tokens.json
yanghuattt's picture
Upload tokenizer
81ab989 verified
{
"</s>": 49153,
"<CHAR_LIT:\">": 49403,
"<CHAR_LIT:(>": 49411,
"<CHAR_LIT:)>": 49404,
"<CHAR_LIT:->": 49400,
"<CHAR_LIT:.>": 49391,
"<CHAR_LIT:/>": 49393,
"<CHAR_LIT:0>": 49394,
"<CHAR_LIT:1>": 49410,
"<CHAR_LIT:9>": 49406,
"<CHAR_LIT::>": 49397,
"<CHAR_LIT:;>": 49412,
"<CHAR_LIT:=>": 49405,
"<CHAR_LIT:>>": 49417,
"<CHAR_LIT:A>": 49401,
"<CHAR_LIT:U+0020>": 49390,
"<CHAR_LIT:U+002C>": 49399,
"<CHAR_LIT:Z>": 49418,
"<CHAR_LIT:[>": 49408,
"<CHAR_LIT:\\'>": 49395,
"<CHAR_LIT:\\\\>": 49402,
"<CHAR_LIT:\\n>": 49392,
"<CHAR_LIT:\\r>": 49407,
"<CHAR_LIT:\\t>": 49414,
"<CHAR_LIT:]>": 49409,
"<CHAR_LIT:_>": 49413,
"<CHAR_LIT:a>": 49396,
"<CHAR_LIT:b>": 49419,
"<CHAR_LIT:c>": 49416,
"<CHAR_LIT:e>": 49415,
"<CHAR_LIT:}>": 49398,
"<CHAR_LIT>": 49159,
"<EOL>": 49155,
"<NUM_LIT:0.0>": 49379,
"<NUM_LIT:0.0f>": 49381,
"<NUM_LIT:0>": 49360,
"<NUM_LIT:0F>": 49377,
"<NUM_LIT:0x00>": 49384,
"<NUM_LIT:1.0>": 49385,
"<NUM_LIT:1.0f>": 49389,
"<NUM_LIT:1000>": 49373,
"<NUM_LIT:100>": 49371,
"<NUM_LIT:10>": 49366,
"<NUM_LIT:11>": 49378,
"<NUM_LIT:12>": 49374,
"<NUM_LIT:15>": 49382,
"<NUM_LIT:16>": 49370,
"<NUM_LIT:1>": 49361,
"<NUM_LIT:1L>": 49375,
"<NUM_LIT:20>": 49376,
"<NUM_LIT:24>": 49388,
"<NUM_LIT:255>": 49383,
"<NUM_LIT:2>": 49362,
"<NUM_LIT:30>": 49386,
"<NUM_LIT:31>": 49387,
"<NUM_LIT:32>": 49380,
"<NUM_LIT:3>": 49363,
"<NUM_LIT:4>": 49364,
"<NUM_LIT:5>": 49365,
"<NUM_LIT:6>": 49368,
"<NUM_LIT:7>": 49369,
"<NUM_LIT:8>": 49367,
"<NUM_LIT:9>": 49372,
"<NUM_LIT>": 49158,
"<STR_LIT:!>": 49307,
"<STR_LIT:#>": 49216,
"<STR_LIT:$>": 49300,
"<STR_LIT:%>": 49214,
"<STR_LIT:%s>": 49215,
"<STR_LIT:&>": 49265,
"<STR_LIT:'>": 49179,
"<STR_LIT:'{0}'>": 49251,
"<STR_LIT:(>": 49193,
"<STR_LIT:)>": 49167,
"<STR_LIT:*>": 49200,
"<STR_LIT:+>": 49276,
"<STR_LIT:-->": 49341,
"<STR_LIT:-1>": 49309,
"<STR_LIT:->": 49183,
"<STR_LIT:...>": 49297,
"<STR_LIT:.>": 49165,
"<STR_LIT:.class>": 49330,
"<STR_LIT:/>": 49162,
"<STR_LIT:0>": 49177,
"<STR_LIT:1.0>": 49268,
"<STR_LIT:10>": 49266,
"<STR_LIT:123>": 49282,
"<STR_LIT:127.0.0.1>": 49283,
"<STR_LIT:1>": 49168,
"<STR_LIT:2.0>": 49201,
"<STR_LIT:2>": 49188,
"<STR_LIT:3>": 49203,
"<STR_LIT:4>": 49230,
"<STR_LIT:5>": 49250,
"<STR_LIT:6>": 49317,
"<STR_LIT::>": 49172,
"<STR_LIT::U+0020>": 49186,
"<STR_LIT:;>": 49199,
"<STR_LIT:;U+0020>": 49274,
"<STR_LIT:<>": 49246,
"<STR_LIT:=>": 49195,
"<STR_LIT:>>": 49209,
"<STR_LIT:?>": 49239,
"<STR_LIT:@>": 49255,
"<STR_LIT:A>": 49192,
"<STR_LIT:B>": 49210,
"<STR_LIT:C>": 49212,
"<STR_LIT:Content-Type>": 49273,
"<STR_LIT:D>": 49269,
"<STR_LIT:Doppelganger>": 49204,
"<STR_LIT:E>": 49298,
"<STR_LIT:F>": 49323,
"<STR_LIT:GET>": 49237,
"<STR_LIT:GMT>": 49359,
"<STR_LIT:Hello>": 49343,
"<STR_LIT:I>": 49328,
"<STR_LIT:ID>": 49287,
"<STR_LIT:M>": 49351,
"<STR_LIT:N>": 49338,
"<STR_LIT:Name>": 49291,
"<STR_LIT:NotU+0020supportedU+0020yet.>": 49219,
"<STR_LIT:OK>": 49257,
"<STR_LIT:POST>": 49264,
"<STR_LIT:S>": 49320,
"<STR_LIT:String>": 49342,
"<STR_LIT:T>": 49279,
"<STR_LIT:Test>": 49278,
"<STR_LIT:U+0020(>": 49220,
"<STR_LIT:U+0020-U+0020>": 49262,
"<STR_LIT:U+0020:U+0020>": 49263,
"<STR_LIT:U+0020=U+0020>": 49272,
"<STR_LIT:U+0020>": 49160,
"<STR_LIT:U+0020U+0020>": 49224,
"<STR_LIT:U+0020U+0020U+0020U+0020>": 49259,
"<STR_LIT:U+0020U+0020U+0020U+0020}\\n>": 49295,
"<STR_LIT:U+0020U+0020}>": 49350,
"<STR_LIT:U+0020U+0020}\\n>": 49293,
"<STR_LIT:U+0020andU+0020>": 49337,
"<STR_LIT:U+0020toU+0020>": 49284,
"<STR_LIT:U+002C>": 49164,
"<STR_LIT:U+002CU+0020>": 49169,
"<STR_LIT:UTF-8>": 49184,
"<STR_LIT:X>": 49299,
"<STR_LIT:Y>": 49288,
"<STR_LIT:Z>": 49316,
"<STR_LIT:[>": 49196,
"<STR_LIT:[]>": 49356,
"<STR_LIT:\\\">": 49180,
"<STR_LIT:\\\\.>": 49335,
"<STR_LIT:\\\\>": 49314,
"<STR_LIT:\\n>": 49161,
"<STR_LIT:\\r\\n>": 49243,
"<STR_LIT:\\t>": 49202,
"<STR_LIT:]>": 49175,
"<STR_LIT:_>": 49190,
"<STR_LIT:_cvp_Core.PatientShort>": 49245,
"<STR_LIT:_cvp_Naes.Referral>": 49290,
"<STR_LIT:a>": 49171,
"<STR_LIT:abc>": 49225,
"<STR_LIT:address>": 49332,
"<STR_LIT:admin>": 49304,
"<STR_LIT:all>": 49222,
"<STR_LIT:application/json>": 49238,
"<STR_LIT:b>": 49181,
"<STR_LIT:bar>": 49194,
"<STR_LIT:baz>": 49345,
"<STR_LIT:body>": 49334,
"<STR_LIT:boolean>": 49311,
"<STR_LIT:c>": 49198,
"<STR_LIT:cast>": 49354,
"<STR_LIT:class>": 49241,
"<STR_LIT:code>": 49289,
"<STR_LIT:content>": 49275,
"<STR_LIT:count>": 49294,
"<STR_LIT:d>": 49223,
"<STR_LIT:data>": 49227,
"<STR_LIT:date>": 49306,
"<STR_LIT:default>": 49236,
"<STR_LIT:deprecation>": 49211,
"<STR_LIT:description>": 49221,
"<STR_LIT:double>": 49296,
"<STR_LIT:e>": 49271,
"<STR_LIT:email>": 49336,
"<STR_LIT:en>": 49260,
"<STR_LIT:error>": 49249,
"<STR_LIT:f>": 49258,
"<STR_LIT:false>": 49191,
"<STR_LIT:field>": 49310,
"<STR_LIT:file>": 49253,
"<STR_LIT:float>": 49357,
"<STR_LIT:foo>": 49173,
"<STR_LIT:get>": 49286,
"<STR_LIT:hello>": 49234,
"<STR_LIT:http://>": 49326,
"<STR_LIT:http>": 49280,
"<STR_LIT:i>": 49302,
"<STR_LIT:id>": 49170,
"<STR_LIT:index>": 49319,
"<STR_LIT:int>": 49244,
"<STR_LIT:key1>": 49352,
"<STR_LIT:key>": 49208,
"<STR_LIT:label>": 49321,
"<STR_LIT:line.separator>": 49305,
"<STR_LIT:list>": 49303,
"<STR_LIT:localhost>": 49231,
"<STR_LIT:long>": 49301,
"<STR_LIT:m>": 49277,
"<STR_LIT:message>": 49232,
"<STR_LIT:msg=>": 49213,
"<STR_LIT:n>": 49349,
"<STR_LIT:name>": 49166,
"<STR_LIT:none>": 49267,
"<STR_LIT:null>": 49189,
"<STR_LIT:number>": 49325,
"<STR_LIT:one>": 49254,
"<STR_LIT:p>": 49270,
"<STR_LIT:password>": 49226,
"<STR_LIT:path>": 49285,
"<STR_LIT:q>": 49347,
"<STR_LIT:query>": 49358,
"<STR_LIT:r>": 49315,
"<STR_LIT:rawtypes>": 49205,
"<STR_LIT:result>": 49318,
"<STR_LIT:root>": 49322,
"<STR_LIT:s>": 49248,
"<STR_LIT:serial>": 49206,
"<STR_LIT:size>": 49348,
"<STR_LIT:source>": 49327,
"<STR_LIT:src>": 49329,
"<STR_LIT:start>": 49324,
"<STR_LIT:state>": 49331,
"<STR_LIT:status>": 49235,
"<STR_LIT:string>": 49228,
"<STR_LIT:success>": 49312,
"<STR_LIT:t>": 49339,
"<STR_LIT:target>": 49333,
"<STR_LIT:test1>": 49355,
"<STR_LIT:test>": 49174,
"<STR_LIT:text/html>": 49346,
"<STR_LIT:text/plain>": 49281,
"<STR_LIT:text>": 49233,
"<STR_LIT:title>": 49218,
"<STR_LIT:true>": 49178,
"<STR_LIT:two>": 49292,
"<STR_LIT:type>": 49185,
"<STR_LIT:unchecked>": 49163,
"<STR_LIT:unknown>": 49313,
"<STR_LIT:unused>": 49187,
"<STR_LIT:url>": 49256,
"<STR_LIT:user>": 49229,
"<STR_LIT:userId>": 49353,
"<STR_LIT:username>": 49261,
"<STR_LIT:utf-8>": 49308,
"<STR_LIT:value1>": 49340,
"<STR_LIT:value>": 49182,
"<STR_LIT:version>": 49247,
"<STR_LIT:x>": 49197,
"<STR_LIT:y>": 49242,
"<STR_LIT:yes>": 49344,
"<STR_LIT:{>": 49207,
"<STR_LIT:{}>": 49240,
"<STR_LIT:|>": 49252,
"<STR_LIT:}>": 49176,
"<STR_LIT:}\\n>": 49217,
"<STR_LIT>": 49157,
"<pad>": 49156,
"<s>": 49152,
"<|UNKNOWN|>": 49154
}