upload

ALTo.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:320057d9b06177a73b3cc7cdba8a090149a7e41b2071f5eb3f799578bec706e6
+size 3853850831

README.md

@@ -1,3 +1,7 @@
 ---
 license: apache-2.0
+base_model:
+- OpenGVLab/InternVL2_5-8B
+pipeline_tag: mask-generation
 ---
+<div align="center">

added_tokens.json

@@ -0,0 +1,1037 @@
+{
+  "</box>": 92552,
+  "</img>": 92545,
+  "</quad>": 92548,
+  "</ref>": 92550,
+  "<ALTo_End>": 92554,
+  "<ALTo_Start>": 92553,
+  "<IMG_CONTEXT>": 92546,
+  "<TOK_0>": 92555,
+  "<TOK_1000>": 93555,
+  "<TOK_1001>": 93556,
+  "<TOK_1002>": 93557,
+  "<TOK_1003>": 93558,
+  "<TOK_1004>": 93559,
+  "<TOK_1005>": 93560,
+  "<TOK_1006>": 93561,
+  "<TOK_1007>": 93562,
+  "<TOK_1008>": 93563,
+  "<TOK_1009>": 93564,
+  "<TOK_100>": 92655,
+  "<TOK_1010>": 93565,
+  "<TOK_1011>": 93566,
+  "<TOK_1012>": 93567,
+  "<TOK_1013>": 93568,
+  "<TOK_1014>": 93569,
+  "<TOK_1015>": 93570,
+  "<TOK_1016>": 93571,
+  "<TOK_1017>": 93572,
+  "<TOK_1018>": 93573,
+  "<TOK_1019>": 93574,
+  "<TOK_101>": 92656,
+  "<TOK_1020>": 93575,
+  "<TOK_1021>": 93576,
+  "<TOK_1022>": 93577,
+  "<TOK_1023>": 93578,
+  "<TOK_102>": 92657,
+  "<TOK_103>": 92658,
+  "<TOK_104>": 92659,
+  "<TOK_105>": 92660,
+  "<TOK_106>": 92661,
+  "<TOK_107>": 92662,
+  "<TOK_108>": 92663,
+  "<TOK_109>": 92664,
+  "<TOK_10>": 92565,
+  "<TOK_110>": 92665,
+  "<TOK_111>": 92666,
+  "<TOK_112>": 92667,
+  "<TOK_113>": 92668,
+  "<TOK_114>": 92669,
+  "<TOK_115>": 92670,
+  "<TOK_116>": 92671,
+  "<TOK_117>": 92672,
+  "<TOK_118>": 92673,
+  "<TOK_119>": 92674,
+  "<TOK_11>": 92566,
+  "<TOK_120>": 92675,
+  "<TOK_121>": 92676,
+  "<TOK_122>": 92677,
+  "<TOK_123>": 92678,
+  "<TOK_124>": 92679,
+  "<TOK_125>": 92680,
+  "<TOK_126>": 92681,
+  "<TOK_127>": 92682,
+  "<TOK_128>": 92683,
+  "<TOK_129>": 92684,
+  "<TOK_12>": 92567,
+  "<TOK_130>": 92685,
+  "<TOK_131>": 92686,
+  "<TOK_132>": 92687,
+  "<TOK_133>": 92688,
+  "<TOK_134>": 92689,
+  "<TOK_135>": 92690,
+  "<TOK_136>": 92691,
+  "<TOK_137>": 92692,
+  "<TOK_138>": 92693,
+  "<TOK_139>": 92694,
+  "<TOK_13>": 92568,
+  "<TOK_140>": 92695,
+  "<TOK_141>": 92696,
+  "<TOK_142>": 92697,
+  "<TOK_143>": 92698,
+  "<TOK_144>": 92699,
+  "<TOK_145>": 92700,
+  "<TOK_146>": 92701,
+  "<TOK_147>": 92702,
+  "<TOK_148>": 92703,
+  "<TOK_149>": 92704,
+  "<TOK_14>": 92569,
+  "<TOK_150>": 92705,
+  "<TOK_151>": 92706,
+  "<TOK_152>": 92707,
+  "<TOK_153>": 92708,
+  "<TOK_154>": 92709,
+  "<TOK_155>": 92710,
+  "<TOK_156>": 92711,
+  "<TOK_157>": 92712,
+  "<TOK_158>": 92713,
+  "<TOK_159>": 92714,
+  "<TOK_15>": 92570,
+  "<TOK_160>": 92715,
+  "<TOK_161>": 92716,
+  "<TOK_162>": 92717,
+  "<TOK_163>": 92718,
+  "<TOK_164>": 92719,
+  "<TOK_165>": 92720,
+  "<TOK_166>": 92721,
+  "<TOK_167>": 92722,
+  "<TOK_168>": 92723,
+  "<TOK_169>": 92724,
+  "<TOK_16>": 92571,
+  "<TOK_170>": 92725,
+  "<TOK_171>": 92726,
+  "<TOK_172>": 92727,
+  "<TOK_173>": 92728,
+  "<TOK_174>": 92729,
+  "<TOK_175>": 92730,
+  "<TOK_176>": 92731,
+  "<TOK_177>": 92732,
+  "<TOK_178>": 92733,
+  "<TOK_179>": 92734,
+  "<TOK_17>": 92572,
+  "<TOK_180>": 92735,
+  "<TOK_181>": 92736,
+  "<TOK_182>": 92737,
+  "<TOK_183>": 92738,
+  "<TOK_184>": 92739,
+  "<TOK_185>": 92740,
+  "<TOK_186>": 92741,
+  "<TOK_187>": 92742,
+  "<TOK_188>": 92743,
+  "<TOK_189>": 92744,
+  "<TOK_18>": 92573,
+  "<TOK_190>": 92745,
+  "<TOK_191>": 92746,
+  "<TOK_192>": 92747,
+  "<TOK_193>": 92748,
+  "<TOK_194>": 92749,
+  "<TOK_195>": 92750,
+  "<TOK_196>": 92751,
+  "<TOK_197>": 92752,
+  "<TOK_198>": 92753,
+  "<TOK_199>": 92754,
+  "<TOK_19>": 92574,
+  "<TOK_1>": 92556,
+  "<TOK_200>": 92755,
+  "<TOK_201>": 92756,
+  "<TOK_202>": 92757,
+  "<TOK_203>": 92758,
+  "<TOK_204>": 92759,
+  "<TOK_205>": 92760,
+  "<TOK_206>": 92761,
+  "<TOK_207>": 92762,
+  "<TOK_208>": 92763,
+  "<TOK_209>": 92764,
+  "<TOK_20>": 92575,
+  "<TOK_210>": 92765,
+  "<TOK_211>": 92766,
+  "<TOK_212>": 92767,
+  "<TOK_213>": 92768,
+  "<TOK_214>": 92769,
+  "<TOK_215>": 92770,
+  "<TOK_216>": 92771,
+  "<TOK_217>": 92772,
+  "<TOK_218>": 92773,
+  "<TOK_219>": 92774,
+  "<TOK_21>": 92576,
+  "<TOK_220>": 92775,
+  "<TOK_221>": 92776,
+  "<TOK_222>": 92777,
+  "<TOK_223>": 92778,
+  "<TOK_224>": 92779,
+  "<TOK_225>": 92780,
+  "<TOK_226>": 92781,
+  "<TOK_227>": 92782,
+  "<TOK_228>": 92783,
+  "<TOK_229>": 92784,
+  "<TOK_22>": 92577,
+  "<TOK_230>": 92785,
+  "<TOK_231>": 92786,
+  "<TOK_232>": 92787,
+  "<TOK_233>": 92788,
+  "<TOK_234>": 92789,
+  "<TOK_235>": 92790,
+  "<TOK_236>": 92791,
+  "<TOK_237>": 92792,
+  "<TOK_238>": 92793,
+  "<TOK_239>": 92794,
+  "<TOK_23>": 92578,
+  "<TOK_240>": 92795,
+  "<TOK_241>": 92796,
+  "<TOK_242>": 92797,
+  "<TOK_243>": 92798,
+  "<TOK_244>": 92799,
+  "<TOK_245>": 92800,
+  "<TOK_246>": 92801,
+  "<TOK_247>": 92802,
+  "<TOK_248>": 92803,
+  "<TOK_249>": 92804,
+  "<TOK_24>": 92579,
+  "<TOK_250>": 92805,
+  "<TOK_251>": 92806,
+  "<TOK_252>": 92807,
+  "<TOK_253>": 92808,
+  "<TOK_254>": 92809,
+  "<TOK_255>": 92810,
+  "<TOK_256>": 92811,
+  "<TOK_257>": 92812,
+  "<TOK_258>": 92813,
+  "<TOK_259>": 92814,
+  "<TOK_25>": 92580,
+  "<TOK_260>": 92815,
+  "<TOK_261>": 92816,
+  "<TOK_262>": 92817,
+  "<TOK_263>": 92818,
+  "<TOK_264>": 92819,
+  "<TOK_265>": 92820,
+  "<TOK_266>": 92821,
+  "<TOK_267>": 92822,
+  "<TOK_268>": 92823,
+  "<TOK_269>": 92824,
+  "<TOK_26>": 92581,
+  "<TOK_270>": 92825,
+  "<TOK_271>": 92826,
+  "<TOK_272>": 92827,
+  "<TOK_273>": 92828,
+  "<TOK_274>": 92829,
+  "<TOK_275>": 92830,
+  "<TOK_276>": 92831,
+  "<TOK_277>": 92832,
+  "<TOK_278>": 92833,
+  "<TOK_279>": 92834,
+  "<TOK_27>": 92582,
+  "<TOK_280>": 92835,
+  "<TOK_281>": 92836,
+  "<TOK_282>": 92837,
+  "<TOK_283>": 92838,
+  "<TOK_284>": 92839,
+  "<TOK_285>": 92840,
+  "<TOK_286>": 92841,
+  "<TOK_287>": 92842,
+  "<TOK_288>": 92843,
+  "<TOK_289>": 92844,
+  "<TOK_28>": 92583,
+  "<TOK_290>": 92845,
+  "<TOK_291>": 92846,
+  "<TOK_292>": 92847,
+  "<TOK_293>": 92848,
+  "<TOK_294>": 92849,
+  "<TOK_295>": 92850,
+  "<TOK_296>": 92851,
+  "<TOK_297>": 92852,
+  "<TOK_298>": 92853,
+  "<TOK_299>": 92854,
+  "<TOK_29>": 92584,
+  "<TOK_2>": 92557,
+  "<TOK_300>": 92855,
+  "<TOK_301>": 92856,
+  "<TOK_302>": 92857,
+  "<TOK_303>": 92858,
+  "<TOK_304>": 92859,
+  "<TOK_305>": 92860,
+  "<TOK_306>": 92861,
+  "<TOK_307>": 92862,
+  "<TOK_308>": 92863,
+  "<TOK_309>": 92864,
+  "<TOK_30>": 92585,
+  "<TOK_310>": 92865,
+  "<TOK_311>": 92866,
+  "<TOK_312>": 92867,
+  "<TOK_313>": 92868,
+  "<TOK_314>": 92869,
+  "<TOK_315>": 92870,
+  "<TOK_316>": 92871,
+  "<TOK_317>": 92872,
+  "<TOK_318>": 92873,
+  "<TOK_319>": 92874,
+  "<TOK_31>": 92586,
+  "<TOK_320>": 92875,
+  "<TOK_321>": 92876,
+  "<TOK_322>": 92877,
+  "<TOK_323>": 92878,
+  "<TOK_324>": 92879,
+  "<TOK_325>": 92880,
+  "<TOK_326>": 92881,
+  "<TOK_327>": 92882,
+  "<TOK_328>": 92883,
+  "<TOK_329>": 92884,
+  "<TOK_32>": 92587,
+  "<TOK_330>": 92885,
+  "<TOK_331>": 92886,
+  "<TOK_332>": 92887,
+  "<TOK_333>": 92888,
+  "<TOK_334>": 92889,
+  "<TOK_335>": 92890,
+  "<TOK_336>": 92891,
+  "<TOK_337>": 92892,
+  "<TOK_338>": 92893,
+  "<TOK_339>": 92894,
+  "<TOK_33>": 92588,
+  "<TOK_340>": 92895,
+  "<TOK_341>": 92896,
+  "<TOK_342>": 92897,
+  "<TOK_343>": 92898,
+  "<TOK_344>": 92899,
+  "<TOK_345>": 92900,
+  "<TOK_346>": 92901,
+  "<TOK_347>": 92902,
+  "<TOK_348>": 92903,
+  "<TOK_349>": 92904,
+  "<TOK_34>": 92589,
+  "<TOK_350>": 92905,
+  "<TOK_351>": 92906,
+  "<TOK_352>": 92907,
+  "<TOK_353>": 92908,
+  "<TOK_354>": 92909,
+  "<TOK_355>": 92910,
+  "<TOK_356>": 92911,
+  "<TOK_357>": 92912,
+  "<TOK_358>": 92913,
+  "<TOK_359>": 92914,
+  "<TOK_35>": 92590,
+  "<TOK_360>": 92915,
+  "<TOK_361>": 92916,
+  "<TOK_362>": 92917,
+  "<TOK_363>": 92918,
+  "<TOK_364>": 92919,
+  "<TOK_365>": 92920,
+  "<TOK_366>": 92921,
+  "<TOK_367>": 92922,
+  "<TOK_368>": 92923,
+  "<TOK_369>": 92924,
+  "<TOK_36>": 92591,
+  "<TOK_370>": 92925,
+  "<TOK_371>": 92926,
+  "<TOK_372>": 92927,
+  "<TOK_373>": 92928,
+  "<TOK_374>": 92929,
+  "<TOK_375>": 92930,
+  "<TOK_376>": 92931,
+  "<TOK_377>": 92932,
+  "<TOK_378>": 92933,
+  "<TOK_379>": 92934,
+  "<TOK_37>": 92592,
+  "<TOK_380>": 92935,
+  "<TOK_381>": 92936,
+  "<TOK_382>": 92937,
+  "<TOK_383>": 92938,
+  "<TOK_384>": 92939,
+  "<TOK_385>": 92940,
+  "<TOK_386>": 92941,
+  "<TOK_387>": 92942,
+  "<TOK_388>": 92943,
+  "<TOK_389>": 92944,
+  "<TOK_38>": 92593,
+  "<TOK_390>": 92945,
+  "<TOK_391>": 92946,
+  "<TOK_392>": 92947,
+  "<TOK_393>": 92948,
+  "<TOK_394>": 92949,
+  "<TOK_395>": 92950,
+  "<TOK_396>": 92951,
+  "<TOK_397>": 92952,
+  "<TOK_398>": 92953,
+  "<TOK_399>": 92954,
+  "<TOK_39>": 92594,
+  "<TOK_3>": 92558,
+  "<TOK_400>": 92955,
+  "<TOK_401>": 92956,
+  "<TOK_402>": 92957,
+  "<TOK_403>": 92958,
+  "<TOK_404>": 92959,
+  "<TOK_405>": 92960,
+  "<TOK_406>": 92961,
+  "<TOK_407>": 92962,
+  "<TOK_408>": 92963,
+  "<TOK_409>": 92964,
+  "<TOK_40>": 92595,
+  "<TOK_410>": 92965,
+  "<TOK_411>": 92966,
+  "<TOK_412>": 92967,
+  "<TOK_413>": 92968,
+  "<TOK_414>": 92969,
+  "<TOK_415>": 92970,
+  "<TOK_416>": 92971,
+  "<TOK_417>": 92972,
+  "<TOK_418>": 92973,
+  "<TOK_419>": 92974,
+  "<TOK_41>": 92596,
+  "<TOK_420>": 92975,
+  "<TOK_421>": 92976,
+  "<TOK_422>": 92977,
+  "<TOK_423>": 92978,
+  "<TOK_424>": 92979,
+  "<TOK_425>": 92980,
+  "<TOK_426>": 92981,
+  "<TOK_427>": 92982,
+  "<TOK_428>": 92983,
+  "<TOK_429>": 92984,
+  "<TOK_42>": 92597,
+  "<TOK_430>": 92985,
+  "<TOK_431>": 92986,
+  "<TOK_432>": 92987,
+  "<TOK_433>": 92988,
+  "<TOK_434>": 92989,
+  "<TOK_435>": 92990,
+  "<TOK_436>": 92991,
+  "<TOK_437>": 92992,
+  "<TOK_438>": 92993,
+  "<TOK_439>": 92994,
+  "<TOK_43>": 92598,
+  "<TOK_440>": 92995,
+  "<TOK_441>": 92996,
+  "<TOK_442>": 92997,
+  "<TOK_443>": 92998,
+  "<TOK_444>": 92999,
+  "<TOK_445>": 93000,
+  "<TOK_446>": 93001,
+  "<TOK_447>": 93002,
+  "<TOK_448>": 93003,
+  "<TOK_449>": 93004,
+  "<TOK_44>": 92599,
+  "<TOK_450>": 93005,
+  "<TOK_451>": 93006,
+  "<TOK_452>": 93007,
+  "<TOK_453>": 93008,
+  "<TOK_454>": 93009,
+  "<TOK_455>": 93010,
+  "<TOK_456>": 93011,
+  "<TOK_457>": 93012,
+  "<TOK_458>": 93013,
+  "<TOK_459>": 93014,
+  "<TOK_45>": 92600,
+  "<TOK_460>": 93015,
+  "<TOK_461>": 93016,
+  "<TOK_462>": 93017,
+  "<TOK_463>": 93018,
+  "<TOK_464>": 93019,
+  "<TOK_465>": 93020,
+  "<TOK_466>": 93021,
+  "<TOK_467>": 93022,
+  "<TOK_468>": 93023,
+  "<TOK_469>": 93024,
+  "<TOK_46>": 92601,
+  "<TOK_470>": 93025,
+  "<TOK_471>": 93026,
+  "<TOK_472>": 93027,
+  "<TOK_473>": 93028,
+  "<TOK_474>": 93029,
+  "<TOK_475>": 93030,
+  "<TOK_476>": 93031,
+  "<TOK_477>": 93032,
+  "<TOK_478>": 93033,
+  "<TOK_479>": 93034,
+  "<TOK_47>": 92602,
+  "<TOK_480>": 93035,
+  "<TOK_481>": 93036,
+  "<TOK_482>": 93037,
+  "<TOK_483>": 93038,
+  "<TOK_484>": 93039,
+  "<TOK_485>": 93040,
+  "<TOK_486>": 93041,
+  "<TOK_487>": 93042,
+  "<TOK_488>": 93043,
+  "<TOK_489>": 93044,
+  "<TOK_48>": 92603,
+  "<TOK_490>": 93045,
+  "<TOK_491>": 93046,
+  "<TOK_492>": 93047,
+  "<TOK_493>": 93048,
+  "<TOK_494>": 93049,
+  "<TOK_495>": 93050,
+  "<TOK_496>": 93051,
+  "<TOK_497>": 93052,
+  "<TOK_498>": 93053,
+  "<TOK_499>": 93054,
+  "<TOK_49>": 92604,
+  "<TOK_4>": 92559,
+  "<TOK_500>": 93055,
+  "<TOK_501>": 93056,
+  "<TOK_502>": 93057,
+  "<TOK_503>": 93058,
+  "<TOK_504>": 93059,
+  "<TOK_505>": 93060,
+  "<TOK_506>": 93061,
+  "<TOK_507>": 93062,
+  "<TOK_508>": 93063,
+  "<TOK_509>": 93064,
+  "<TOK_50>": 92605,
+  "<TOK_510>": 93065,
+  "<TOK_511>": 93066,
+  "<TOK_512>": 93067,
+  "<TOK_513>": 93068,
+  "<TOK_514>": 93069,
+  "<TOK_515>": 93070,
+  "<TOK_516>": 93071,
+  "<TOK_517>": 93072,
+  "<TOK_518>": 93073,
+  "<TOK_519>": 93074,
+  "<TOK_51>": 92606,
+  "<TOK_520>": 93075,
+  "<TOK_521>": 93076,
+  "<TOK_522>": 93077,
+  "<TOK_523>": 93078,
+  "<TOK_524>": 93079,
+  "<TOK_525>": 93080,
+  "<TOK_526>": 93081,
+  "<TOK_527>": 93082,
+  "<TOK_528>": 93083,
+  "<TOK_529>": 93084,
+  "<TOK_52>": 92607,
+  "<TOK_530>": 93085,
+  "<TOK_531>": 93086,
+  "<TOK_532>": 93087,
+  "<TOK_533>": 93088,
+  "<TOK_534>": 93089,
+  "<TOK_535>": 93090,
+  "<TOK_536>": 93091,
+  "<TOK_537>": 93092,
+  "<TOK_538>": 93093,
+  "<TOK_539>": 93094,
+  "<TOK_53>": 92608,
+  "<TOK_540>": 93095,
+  "<TOK_541>": 93096,
+  "<TOK_542>": 93097,
+  "<TOK_543>": 93098,
+  "<TOK_544>": 93099,
+  "<TOK_545>": 93100,
+  "<TOK_546>": 93101,
+  "<TOK_547>": 93102,
+  "<TOK_548>": 93103,
+  "<TOK_549>": 93104,
+  "<TOK_54>": 92609,
+  "<TOK_550>": 93105,
+  "<TOK_551>": 93106,
+  "<TOK_552>": 93107,
+  "<TOK_553>": 93108,
+  "<TOK_554>": 93109,
+  "<TOK_555>": 93110,
+  "<TOK_556>": 93111,
+  "<TOK_557>": 93112,
+  "<TOK_558>": 93113,
+  "<TOK_559>": 93114,
+  "<TOK_55>": 92610,
+  "<TOK_560>": 93115,
+  "<TOK_561>": 93116,
+  "<TOK_562>": 93117,
+  "<TOK_563>": 93118,
+  "<TOK_564>": 93119,
+  "<TOK_565>": 93120,
+  "<TOK_566>": 93121,
+  "<TOK_567>": 93122,
+  "<TOK_568>": 93123,
+  "<TOK_569>": 93124,
+  "<TOK_56>": 92611,
+  "<TOK_570>": 93125,
+  "<TOK_571>": 93126,
+  "<TOK_572>": 93127,
+  "<TOK_573>": 93128,
+  "<TOK_574>": 93129,
+  "<TOK_575>": 93130,
+  "<TOK_576>": 93131,
+  "<TOK_577>": 93132,
+  "<TOK_578>": 93133,
+  "<TOK_579>": 93134,
+  "<TOK_57>": 92612,
+  "<TOK_580>": 93135,
+  "<TOK_581>": 93136,
+  "<TOK_582>": 93137,
+  "<TOK_583>": 93138,
+  "<TOK_584>": 93139,
+  "<TOK_585>": 93140,
+  "<TOK_586>": 93141,
+  "<TOK_587>": 93142,
+  "<TOK_588>": 93143,
+  "<TOK_589>": 93144,
+  "<TOK_58>": 92613,
+  "<TOK_590>": 93145,
+  "<TOK_591>": 93146,
+  "<TOK_592>": 93147,
+  "<TOK_593>": 93148,
+  "<TOK_594>": 93149,
+  "<TOK_595>": 93150,
+  "<TOK_596>": 93151,
+  "<TOK_597>": 93152,
+  "<TOK_598>": 93153,
+  "<TOK_599>": 93154,
+  "<TOK_59>": 92614,
+  "<TOK_5>": 92560,
+  "<TOK_600>": 93155,
+  "<TOK_601>": 93156,
+  "<TOK_602>": 93157,
+  "<TOK_603>": 93158,
+  "<TOK_604>": 93159,
+  "<TOK_605>": 93160,
+  "<TOK_606>": 93161,
+  "<TOK_607>": 93162,
+  "<TOK_608>": 93163,
+  "<TOK_609>": 93164,
+  "<TOK_60>": 92615,
+  "<TOK_610>": 93165,
+  "<TOK_611>": 93166,
+  "<TOK_612>": 93167,
+  "<TOK_613>": 93168,
+  "<TOK_614>": 93169,
+  "<TOK_615>": 93170,
+  "<TOK_616>": 93171,
+  "<TOK_617>": 93172,
+  "<TOK_618>": 93173,
+  "<TOK_619>": 93174,
+  "<TOK_61>": 92616,
+  "<TOK_620>": 93175,
+  "<TOK_621>": 93176,
+  "<TOK_622>": 93177,
+  "<TOK_623>": 93178,
+  "<TOK_624>": 93179,
+  "<TOK_625>": 93180,
+  "<TOK_626>": 93181,
+  "<TOK_627>": 93182,
+  "<TOK_628>": 93183,
+  "<TOK_629>": 93184,
+  "<TOK_62>": 92617,
+  "<TOK_630>": 93185,
+  "<TOK_631>": 93186,
+  "<TOK_632>": 93187,
+  "<TOK_633>": 93188,
+  "<TOK_634>": 93189,
+  "<TOK_635>": 93190,
+  "<TOK_636>": 93191,
+  "<TOK_637>": 93192,
+  "<TOK_638>": 93193,
+  "<TOK_639>": 93194,
+  "<TOK_63>": 92618,
+  "<TOK_640>": 93195,
+  "<TOK_641>": 93196,
+  "<TOK_642>": 93197,
+  "<TOK_643>": 93198,
+  "<TOK_644>": 93199,
+  "<TOK_645>": 93200,
+  "<TOK_646>": 93201,
+  "<TOK_647>": 93202,
+  "<TOK_648>": 93203,
+  "<TOK_649>": 93204,
+  "<TOK_64>": 92619,
+  "<TOK_650>": 93205,
+  "<TOK_651>": 93206,
+  "<TOK_652>": 93207,
+  "<TOK_653>": 93208,
+  "<TOK_654>": 93209,
+  "<TOK_655>": 93210,
+  "<TOK_656>": 93211,
+  "<TOK_657>": 93212,
+  "<TOK_658>": 93213,
+  "<TOK_659>": 93214,
+  "<TOK_65>": 92620,
+  "<TOK_660>": 93215,
+  "<TOK_661>": 93216,
+  "<TOK_662>": 93217,
+  "<TOK_663>": 93218,
+  "<TOK_664>": 93219,
+  "<TOK_665>": 93220,
+  "<TOK_666>": 93221,
+  "<TOK_667>": 93222,
+  "<TOK_668>": 93223,
+  "<TOK_669>": 93224,
+  "<TOK_66>": 92621,
+  "<TOK_670>": 93225,
+  "<TOK_671>": 93226,
+  "<TOK_672>": 93227,
+  "<TOK_673>": 93228,
+  "<TOK_674>": 93229,
+  "<TOK_675>": 93230,
+  "<TOK_676>": 93231,
+  "<TOK_677>": 93232,
+  "<TOK_678>": 93233,
+  "<TOK_679>": 93234,
+  "<TOK_67>": 92622,
+  "<TOK_680>": 93235,
+  "<TOK_681>": 93236,
+  "<TOK_682>": 93237,
+  "<TOK_683>": 93238,
+  "<TOK_684>": 93239,
+  "<TOK_685>": 93240,
+  "<TOK_686>": 93241,
+  "<TOK_687>": 93242,
+  "<TOK_688>": 93243,
+  "<TOK_689>": 93244,
+  "<TOK_68>": 92623,
+  "<TOK_690>": 93245,
+  "<TOK_691>": 93246,
+  "<TOK_692>": 93247,
+  "<TOK_693>": 93248,
+  "<TOK_694>": 93249,
+  "<TOK_695>": 93250,
+  "<TOK_696>": 93251,
+  "<TOK_697>": 93252,
+  "<TOK_698>": 93253,
+  "<TOK_699>": 93254,
+  "<TOK_69>": 92624,
+  "<TOK_6>": 92561,
+  "<TOK_700>": 93255,
+  "<TOK_701>": 93256,
+  "<TOK_702>": 93257,
+  "<TOK_703>": 93258,
+  "<TOK_704>": 93259,
+  "<TOK_705>": 93260,
+  "<TOK_706>": 93261,
+  "<TOK_707>": 93262,
+  "<TOK_708>": 93263,
+  "<TOK_709>": 93264,
+  "<TOK_70>": 92625,
+  "<TOK_710>": 93265,
+  "<TOK_711>": 93266,
+  "<TOK_712>": 93267,
+  "<TOK_713>": 93268,
+  "<TOK_714>": 93269,
+  "<TOK_715>": 93270,
+  "<TOK_716>": 93271,
+  "<TOK_717>": 93272,
+  "<TOK_718>": 93273,
+  "<TOK_719>": 93274,
+  "<TOK_71>": 92626,
+  "<TOK_720>": 93275,
+  "<TOK_721>": 93276,
+  "<TOK_722>": 93277,
+  "<TOK_723>": 93278,
+  "<TOK_724>": 93279,
+  "<TOK_725>": 93280,
+  "<TOK_726>": 93281,
+  "<TOK_727>": 93282,
+  "<TOK_728>": 93283,
+  "<TOK_729>": 93284,
+  "<TOK_72>": 92627,
+  "<TOK_730>": 93285,
+  "<TOK_731>": 93286,
+  "<TOK_732>": 93287,
+  "<TOK_733>": 93288,
+  "<TOK_734>": 93289,
+  "<TOK_735>": 93290,
+  "<TOK_736>": 93291,
+  "<TOK_737>": 93292,
+  "<TOK_738>": 93293,
+  "<TOK_739>": 93294,
+  "<TOK_73>": 92628,
+  "<TOK_740>": 93295,
+  "<TOK_741>": 93296,
+  "<TOK_742>": 93297,
+  "<TOK_743>": 93298,
+  "<TOK_744>": 93299,
+  "<TOK_745>": 93300,
+  "<TOK_746>": 93301,
+  "<TOK_747>": 93302,
+  "<TOK_748>": 93303,
+  "<TOK_749>": 93304,
+  "<TOK_74>": 92629,
+  "<TOK_750>": 93305,
+  "<TOK_751>": 93306,
+  "<TOK_752>": 93307,
+  "<TOK_753>": 93308,
+  "<TOK_754>": 93309,
+  "<TOK_755>": 93310,
+  "<TOK_756>": 93311,
+  "<TOK_757>": 93312,
+  "<TOK_758>": 93313,
+  "<TOK_759>": 93314,
+  "<TOK_75>": 92630,
+  "<TOK_760>": 93315,
+  "<TOK_761>": 93316,
+  "<TOK_762>": 93317,
+  "<TOK_763>": 93318,
+  "<TOK_764>": 93319,
+  "<TOK_765>": 93320,
+  "<TOK_766>": 93321,
+  "<TOK_767>": 93322,
+  "<TOK_768>": 93323,
+  "<TOK_769>": 93324,
+  "<TOK_76>": 92631,
+  "<TOK_770>": 93325,
+  "<TOK_771>": 93326,
+  "<TOK_772>": 93327,
+  "<TOK_773>": 93328,
+  "<TOK_774>": 93329,
+  "<TOK_775>": 93330,
+  "<TOK_776>": 93331,
+  "<TOK_777>": 93332,
+  "<TOK_778>": 93333,
+  "<TOK_779>": 93334,
+  "<TOK_77>": 92632,
+  "<TOK_780>": 93335,
+  "<TOK_781>": 93336,
+  "<TOK_782>": 93337,
+  "<TOK_783>": 93338,
+  "<TOK_784>": 93339,
+  "<TOK_785>": 93340,
+  "<TOK_786>": 93341,
+  "<TOK_787>": 93342,
+  "<TOK_788>": 93343,
+  "<TOK_789>": 93344,
+  "<TOK_78>": 92633,
+  "<TOK_790>": 93345,
+  "<TOK_791>": 93346,
+  "<TOK_792>": 93347,
+  "<TOK_793>": 93348,
+  "<TOK_794>": 93349,
+  "<TOK_795>": 93350,
+  "<TOK_796>": 93351,
+  "<TOK_797>": 93352,
+  "<TOK_798>": 93353,
+  "<TOK_799>": 93354,
+  "<TOK_79>": 92634,
+  "<TOK_7>": 92562,
+  "<TOK_800>": 93355,
+  "<TOK_801>": 93356,
+  "<TOK_802>": 93357,
+  "<TOK_803>": 93358,
+  "<TOK_804>": 93359,
+  "<TOK_805>": 93360,
+  "<TOK_806>": 93361,
+  "<TOK_807>": 93362,
+  "<TOK_808>": 93363,
+  "<TOK_809>": 93364,
+  "<TOK_80>": 92635,
+  "<TOK_810>": 93365,
+  "<TOK_811>": 93366,
+  "<TOK_812>": 93367,
+  "<TOK_813>": 93368,
+  "<TOK_814>": 93369,
+  "<TOK_815>": 93370,
+  "<TOK_816>": 93371,
+  "<TOK_817>": 93372,
+  "<TOK_818>": 93373,
+  "<TOK_819>": 93374,
+  "<TOK_81>": 92636,
+  "<TOK_820>": 93375,
+  "<TOK_821>": 93376,
+  "<TOK_822>": 93377,
+  "<TOK_823>": 93378,
+  "<TOK_824>": 93379,
+  "<TOK_825>": 93380,
+  "<TOK_826>": 93381,
+  "<TOK_827>": 93382,
+  "<TOK_828>": 93383,
+  "<TOK_829>": 93384,
+  "<TOK_82>": 92637,
+  "<TOK_830>": 93385,
+  "<TOK_831>": 93386,
+  "<TOK_832>": 93387,
+  "<TOK_833>": 93388,
+  "<TOK_834>": 93389,
+  "<TOK_835>": 93390,
+  "<TOK_836>": 93391,
+  "<TOK_837>": 93392,
+  "<TOK_838>": 93393,
+  "<TOK_839>": 93394,
+  "<TOK_83>": 92638,
+  "<TOK_840>": 93395,
+  "<TOK_841>": 93396,
+  "<TOK_842>": 93397,
+  "<TOK_843>": 93398,
+  "<TOK_844>": 93399,
+  "<TOK_845>": 93400,
+  "<TOK_846>": 93401,
+  "<TOK_847>": 93402,
+  "<TOK_848>": 93403,
+  "<TOK_849>": 93404,
+  "<TOK_84>": 92639,
+  "<TOK_850>": 93405,
+  "<TOK_851>": 93406,
+  "<TOK_852>": 93407,
+  "<TOK_853>": 93408,
+  "<TOK_854>": 93409,
+  "<TOK_855>": 93410,
+  "<TOK_856>": 93411,
+  "<TOK_857>": 93412,
+  "<TOK_858>": 93413,
+  "<TOK_859>": 93414,
+  "<TOK_85>": 92640,
+  "<TOK_860>": 93415,
+  "<TOK_861>": 93416,
+  "<TOK_862>": 93417,
+  "<TOK_863>": 93418,
+  "<TOK_864>": 93419,
+  "<TOK_865>": 93420,
+  "<TOK_866>": 93421,
+  "<TOK_867>": 93422,
+  "<TOK_868>": 93423,
+  "<TOK_869>": 93424,
+  "<TOK_86>": 92641,
+  "<TOK_870>": 93425,
+  "<TOK_871>": 93426,
+  "<TOK_872>": 93427,
+  "<TOK_873>": 93428,
+  "<TOK_874>": 93429,
+  "<TOK_875>": 93430,
+  "<TOK_876>": 93431,
+  "<TOK_877>": 93432,
+  "<TOK_878>": 93433,
+  "<TOK_879>": 93434,
+  "<TOK_87>": 92642,
+  "<TOK_880>": 93435,
+  "<TOK_881>": 93436,
+  "<TOK_882>": 93437,
+  "<TOK_883>": 93438,
+  "<TOK_884>": 93439,
+  "<TOK_885>": 93440,
+  "<TOK_886>": 93441,
+  "<TOK_887>": 93442,
+  "<TOK_888>": 93443,
+  "<TOK_889>": 93444,
+  "<TOK_88>": 92643,
+  "<TOK_890>": 93445,
+  "<TOK_891>": 93446,
+  "<TOK_892>": 93447,
+  "<TOK_893>": 93448,
+  "<TOK_894>": 93449,
+  "<TOK_895>": 93450,
+  "<TOK_896>": 93451,
+  "<TOK_897>": 93452,
+  "<TOK_898>": 93453,
+  "<TOK_899>": 93454,
+  "<TOK_89>": 92644,
+  "<TOK_8>": 92563,
+  "<TOK_900>": 93455,
+  "<TOK_901>": 93456,
+  "<TOK_902>": 93457,
+  "<TOK_903>": 93458,
+  "<TOK_904>": 93459,
+  "<TOK_905>": 93460,
+  "<TOK_906>": 93461,
+  "<TOK_907>": 93462,
+  "<TOK_908>": 93463,
+  "<TOK_909>": 93464,
+  "<TOK_90>": 92645,
+  "<TOK_910>": 93465,
+  "<TOK_911>": 93466,
+  "<TOK_912>": 93467,
+  "<TOK_913>": 93468,
+  "<TOK_914>": 93469,
+  "<TOK_915>": 93470,
+  "<TOK_916>": 93471,
+  "<TOK_917>": 93472,
+  "<TOK_918>": 93473,
+  "<TOK_919>": 93474,
+  "<TOK_91>": 92646,
+  "<TOK_920>": 93475,
+  "<TOK_921>": 93476,
+  "<TOK_922>": 93477,
+  "<TOK_923>": 93478,
+  "<TOK_924>": 93479,
+  "<TOK_925>": 93480,
+  "<TOK_926>": 93481,
+  "<TOK_927>": 93482,
+  "<TOK_928>": 93483,
+  "<TOK_929>": 93484,
+  "<TOK_92>": 92647,
+  "<TOK_930>": 93485,
+  "<TOK_931>": 93486,
+  "<TOK_932>": 93487,
+  "<TOK_933>": 93488,
+  "<TOK_934>": 93489,
+  "<TOK_935>": 93490,
+  "<TOK_936>": 93491,
+  "<TOK_937>": 93492,
+  "<TOK_938>": 93493,
+  "<TOK_939>": 93494,
+  "<TOK_93>": 92648,
+  "<TOK_940>": 93495,
+  "<TOK_941>": 93496,
+  "<TOK_942>": 93497,
+  "<TOK_943>": 93498,
+  "<TOK_944>": 93499,
+  "<TOK_945>": 93500,
+  "<TOK_946>": 93501,
+  "<TOK_947>": 93502,
+  "<TOK_948>": 93503,
+  "<TOK_949>": 93504,
+  "<TOK_94>": 92649,
+  "<TOK_950>": 93505,
+  "<TOK_951>": 93506,
+  "<TOK_952>": 93507,
+  "<TOK_953>": 93508,
+  "<TOK_954>": 93509,
+  "<TOK_955>": 93510,
+  "<TOK_956>": 93511,
+  "<TOK_957>": 93512,
+  "<TOK_958>": 93513,
+  "<TOK_959>": 93514,
+  "<TOK_95>": 92650,
+  "<TOK_960>": 93515,
+  "<TOK_961>": 93516,
+  "<TOK_962>": 93517,
+  "<TOK_963>": 93518,
+  "<TOK_964>": 93519,
+  "<TOK_965>": 93520,
+  "<TOK_966>": 93521,
+  "<TOK_967>": 93522,
+  "<TOK_968>": 93523,
+  "<TOK_969>": 93524,
+  "<TOK_96>": 92651,
+  "<TOK_970>": 93525,
+  "<TOK_971>": 93526,
+  "<TOK_972>": 93527,
+  "<TOK_973>": 93528,
+  "<TOK_974>": 93529,
+  "<TOK_975>": 93530,
+  "<TOK_976>": 93531,
+  "<TOK_977>": 93532,
+  "<TOK_978>": 93533,
+  "<TOK_979>": 93534,
+  "<TOK_97>": 92652,
+  "<TOK_980>": 93535,
+  "<TOK_981>": 93536,
+  "<TOK_982>": 93537,
+  "<TOK_983>": 93538,
+  "<TOK_984>": 93539,
+  "<TOK_985>": 93540,
+  "<TOK_986>": 93541,
+  "<TOK_987>": 93542,
+  "<TOK_988>": 93543,
+  "<TOK_989>": 93544,
+  "<TOK_98>": 92653,
+  "<TOK_990>": 93545,
+  "<TOK_991>": 93546,
+  "<TOK_992>": 93547,
+  "<TOK_993>": 93548,
+  "<TOK_994>": 93549,
+  "<TOK_995>": 93550,
+  "<TOK_996>": 93551,
+  "<TOK_997>": 93552,
+  "<TOK_998>": 93553,
+  "<TOK_999>": 93554,
+  "<TOK_99>": 92654,
+  "<TOK_9>": 92564,
+  "<box>": 92551,
+  "<img>": 92544,
+  "<quad>": 92547,
+  "<ref>": 92549
+}

config.json

@@ -0,0 +1,206 @@
+{
+  "_commit_hash": null,
+  "_name_or_path": "/mnt/checkpoints/internvl_seg/0506_internvl_himtokv2_adaptive_exp_joint_train_ov/checkpoint-28000",
+  "architectures": [
+    "InternVLChatModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_internvl_chat.InternVLChatConfig",
+    "AutoModel": "modeling_internvl_chat.InternVLChatModel",
+    "AutoModelForCausalLM": "modeling_internvl_chat.InternVLChatModel"
+  },
+  "downsample_ratio": 0.5,
+  "dynamic_image_size": true,
+  "force_image_size": 448,
+  "hidden_size": 4096,
+  "llm_config": {
+    "_attn_implementation_autoset": true,
+    "_name_or_path": "internlm/internlm2_5-7b-chat",
+    "add_cross_attention": false,
+    "architectures": [
+      "InternLM2ForCausalLM"
+    ],
+    "attn_implementation": "flash_attention_2",
+    "auto_map": {
+      "AutoConfig": "configuration_internlm2.InternLM2Config",
+      "AutoModel": "modeling_internlm2.InternLM2ForCausalLM",
+      "AutoModelForCausalLM": "modeling_internlm2.InternLM2ForCausalLM",
+      "AutoModelForSequenceClassification": "modeling_internlm2.InternLM2ForSequenceClassification"
+    },
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bias": false,
+    "bos_token_id": 1,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 14336,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 32768,
+    "min_length": 0,
+    "model_type": "internlm2",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 32,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 8,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 2,
+    "prefix": null,
+    "pretraining_tp": 1,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": {
+      "factor": 2.0,
+      "type": "dynamic"
+    },
+    "rope_theta": 1000000,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "transformers_version": "4.46.3",
+    "typical_p": 1.0,
+    "use_bfloat16": true,
+    "use_cache": false,
+    "vocab_size": 93579
+  },
+  "max_dynamic_patch": 1,
+  "min_dynamic_patch": 1,
+  "model_type": "internvl_chat",
+  "pad2square": false,
+  "ps_version": "v2",
+  "select_layer": -1,
+  "template": "internvl2_5",
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": null,
+  "use_backbone_lora": 0,
+  "use_llm_lora": 0,
+  "use_thumbnail": true,
+  "vision_config": {
+    "_attn_implementation_autoset": true,
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": [
+      "InternVisionModel"
+    ],
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "drop_path_rate": 0.1,
+    "dropout": 0.0,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu",
+    "hidden_size": 1024,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 448,
+    "initializer_factor": 1.0,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-06,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "intern_vit_6b",
+    "no_repeat_ngram_size": 0,
+    "norm_type": "layer_norm",
+    "num_attention_heads": 16,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_hidden_layers": 24,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "qk_normalization": false,
+    "qkv_bias": true,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "transformers_version": "4.46.3",
+    "typical_p": 1.0,
+    "use_bfloat16": true,
+    "use_flash_attn": true
+  }
+}

generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": [
+    92542,
+    92543
+  ],
+  "transformers_version": "4.46.3"
+}

model-00001-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f28282abf82a558e8f6e319610e68f676731b626da62fd0687f32205f36d5e98
+size 3992013192

model-00002-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b64d9e9875ce6006ac2559b15df439b8cdeeb2c4c57172f1fb9111da3cadf0a5
+size 3976355888

model-00003-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad03429cdddba87f6a79a22503a8accf7948e315e1a33bb3300f0f76895df11b
+size 3959578704

model-00004-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdcef2063296f11ba3b082db83f6af7b3c59938ebf304203e3365607a1ae5de2
+size 3405921192

model-00005-of-00005.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4af41d52432d8be12907d0ff351814f6dd0d4727371a2d00d58c610262fdc00
+size 2748025063

preprocessor_config.json

@@ -0,0 +1,19 @@
+{
+  "crop_size": 448,
+  "do_center_crop": true,
+  "do_normalize": true,
+  "do_resize": true,
+  "feature_extractor_type": "CLIPFeatureExtractor",
+  "image_mean": [
+    0.485,
+    0.456,
+    0.406
+  ],
+  "image_std": [
+    0.229,
+    0.224,
+    0.225
+  ],
+  "resample": 3,
+  "size": 448
+}

special_tokens_map.json

@@ -0,0 +1,47 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|action_start|>",
+    "<|action_end|>",
+    "<|interpreter|>",
+    "<|plugin|>",
+    "<img>",
+    "</img>",
+    "<IMG_CONTEXT>",
+    "<quad>",
+    "</quad>",
+    "<ref>",
+    "</ref>",
+    "<box>",
+    "</box>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenization_internlm2.py

@@ -0,0 +1,235 @@
+# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on transformers/src/transformers/models/llama/tokenization_llama.py
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tokenization classes for InternLM."""
+import os
+from shutil import copyfile
+from typing import Any, Dict, List, Optional, Tuple
+
+import sentencepiece as spm
+from transformers.tokenization_utils import PreTrainedTokenizer
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {'vocab_file': './tokenizer.model'}
+
+PRETRAINED_VOCAB_FILES_MAP = {}
+
+
+# Modified from transformers.model.llama.tokenization_llama.LlamaTokenizer
+class InternLM2Tokenizer(PreTrainedTokenizer):
+    """
+    Construct a InternLM2 tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    model_input_names = ['input_ids', 'attention_mask']
+    _auto_class = 'AutoTokenizer'
+
+    def __init__(
+        self,
+        vocab_file,
+        unk_token='<unk>',
+        bos_token='<s>',
+        eos_token='</s>',
+        pad_token='</s>',
+        sp_model_kwargs: Optional[Dict[str, Any]] = None,
+        add_bos_token=True,
+        add_eos_token=False,
+        decode_with_prefix_space=False,
+        clean_up_tokenization_spaces=False,
+        **kwargs,
+    ):
+        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
+        self.vocab_file = vocab_file
+        self.add_bos_token = add_bos_token
+        self.add_eos_token = add_eos_token
+        self.decode_with_prefix_space = decode_with_prefix_space
+        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.sp_model.Load(vocab_file)
+        self._no_prefix_space_tokens = None
+        super().__init__(
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            pad_token=pad_token,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs,
+        )
+
+    @property
+    def no_prefix_space_tokens(self):
+        if self._no_prefix_space_tokens is None:
+            vocab = self.convert_ids_to_tokens(list(range(self.vocab_size)))
+            self._no_prefix_space_tokens = {i for i, tok in enumerate(vocab) if not tok.startswith('▁')}
+        return self._no_prefix_space_tokens
+
+    @property
+    def vocab_size(self):
+        """Returns vocab size"""
+        return self.sp_model.get_piece_size()
+
+    @property
+    def bos_token_id(self) -> Optional[int]:
+        return self.sp_model.bos_id()
+
+    @property
+    def eos_token_id(self) -> Optional[int]:
+        return self.sp_model.eos_id()
+
+    def get_vocab(self):
+        """Returns vocab as a dict"""
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def _tokenize(self, text):
+        """Returns a tokenized string."""
+        return self.sp_model.encode(text, out_type=str)
+
+    def _convert_token_to_id(self, token):
+        """Converts a token (str) in an id using the vocab."""
+        return self.sp_model.piece_to_id(token)
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        token = self.sp_model.IdToPiece(index)
+        return token
+
+    def _maybe_add_prefix_space(self, tokens, decoded):
+        if tokens and tokens[0] not in self.no_prefix_space_tokens:
+            return ' ' + decoded
+        else:
+            return decoded
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        current_sub_tokens = []
+        out_string = ''
+        prev_is_special = False
+        for token in tokens:
+            # make sure that special tokens are not decoded using sentencepiece model
+            if token in self.all_special_tokens:
+                if not prev_is_special:
+                    out_string += ' '
+                out_string += self.sp_model.decode(current_sub_tokens) + token
+                prev_is_special = True
+                current_sub_tokens = []
+            else:
+                current_sub_tokens.append(token)
+                prev_is_special = False
+        out_string += self.sp_model.decode(current_sub_tokens)
+        out_string = self.clean_up_tokenization(out_string)
+        out_string = self._maybe_add_prefix_space(tokens=tokens, decoded=out_string)
+        return out_string[1:]
+
+    def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        """
+        Save the vocabulary and special tokens file to a directory.
+
+        Args:
+            save_directory (`str`):
+                The directory in which to save the vocabulary.
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        if not os.path.isdir(save_directory):
+            logger.error(f'Vocabulary path ({save_directory}) should be a directory')
+            return
+        out_vocab_file = os.path.join(
+            save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']
+        )
+
+        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
+            copyfile(self.vocab_file, out_vocab_file)
+        elif not os.path.isfile(self.vocab_file):
+            with open(out_vocab_file, 'wb') as fi:
+                content_spiece_model = self.sp_model.serialized_model_proto()
+                fi.write(content_spiece_model)
+
+        return (out_vocab_file,)
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        if self.add_bos_token:
+            bos_token_ids = [self.bos_token_id]
+        else:
+            bos_token_ids = []
+
+        output = bos_token_ids + token_ids_0
+
+        if token_ids_1 is not None:
+            output = output + token_ids_1
+
+        if self.add_eos_token:
+            output = output + [self.eos_token_id]
+
+        return output
+
+    def get_special_tokens_mask(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
+    ) -> List[int]:
+        """
+        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer `prepare_for_model` method.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+                Whether or not the token list is already formatted with special tokens for the model.
+
+        Returns:
+            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+        """
+        if already_has_special_tokens:
+            return super().get_special_tokens_mask(
+                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+            )
+
+        if token_ids_1 is None:
+            return [1] + ([0] * len(token_ids_0)) + [1]
+        return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]
+
+    def create_token_type_ids_from_sequences(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make
+        use of token type ids, therefore a list of zeros is returned.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of zeros.
+        """
+        eos = [self.eos_token_id]
+
+        if token_ids_1 is None:
+            return len(token_ids_0 + eos) * [0]
+        return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f868398fc4e05ee1e8aeba95ddf18ddcc45b8bce55d5093bead5bbf80429b48b
+size 1477754