| { | |
| "prefill": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 3326.472192, | |
| "max_vram": 6551.724032, | |
| "max_reserved": 6192.889856, | |
| "max_allocated": 5914.272768 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.3276853132247926, | |
| "stdev": 0.0001833999146052767, | |
| "values": [ | |
| 0.3276174011230469, | |
| 0.32867181396484374, | |
| 0.32760861206054687, | |
| 0.32757742309570315, | |
| 0.3276591796875, | |
| 0.32761468505859376, | |
| 0.3276138916015625, | |
| 0.32770590209960937, | |
| 0.32760491943359377, | |
| 0.32756829833984374, | |
| 0.32763742065429685, | |
| 0.32761996459960935, | |
| 0.3276365966796875, | |
| 0.32764349365234374, | |
| 0.327709716796875, | |
| 0.32766012573242187, | |
| 0.3276359558105469, | |
| 0.32768429565429685, | |
| 0.32769998168945313, | |
| 0.3276470031738281, | |
| 0.32775726318359377, | |
| 0.3277359619140625, | |
| 0.3275996398925781, | |
| 0.32766220092773435, | |
| 0.3277169189453125, | |
| 0.32775115966796875, | |
| 0.32764764404296876, | |
| 0.32769949340820315, | |
| 0.32765707397460936, | |
| 0.32764907836914064, | |
| 0.327616455078125, | |
| 0.3276204528808594 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 1562.474664980689 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "decode": { | |
| "memory": { | |
| "unit": "MB", | |
| "max_ram": 3327.54944, | |
| "max_vram": 6618.8288, | |
| "max_reserved": 6259.99872, | |
| "max_allocated": 6171.117568 | |
| }, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 2.8039431738853438, | |
| "stdev": 0.0071516740717089255, | |
| "values": [ | |
| 2.815587160110471, | |
| 2.803603824615478, | |
| 2.7998705368041987, | |
| 2.7967111740112296 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 181.173429166212 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| }, | |
| "per_token": { | |
| "memory": null, | |
| "latency": { | |
| "unit": "s", | |
| "mean": 0.022078292707758632, | |
| "stdev": 0.00020362764334839968, | |
| "values": [ | |
| 0.02250587272644043, | |
| 0.02186347198486328, | |
| 0.021771791458129883, | |
| 0.021881872177124024, | |
| 0.021755311965942385, | |
| 0.021843791961669922, | |
| 0.021809871673583985, | |
| 0.021867792129516603, | |
| 0.02194219207763672, | |
| 0.021840431213378905, | |
| 0.021806991577148438, | |
| 0.021799631118774414, | |
| 0.021739152908325194, | |
| 0.021758672714233398, | |
| 0.021798351287841796, | |
| 0.02180843162536621, | |
| 0.02195787239074707, | |
| 0.021928112030029298, | |
| 0.021863632202148438, | |
| 0.0218482723236084, | |
| 0.021785871505737305, | |
| 0.021875791549682617, | |
| 0.02192011260986328, | |
| 0.02188283157348633, | |
| 0.021788591384887694, | |
| 0.021833232879638672, | |
| 0.021875791549682617, | |
| 0.021866352081298827, | |
| 0.02187179183959961, | |
| 0.021864751815795898, | |
| 0.02191307258605957, | |
| 0.021891151428222656, | |
| 0.021964752197265625, | |
| 0.021964111328125, | |
| 0.02188747215270996, | |
| 0.02185755157470703, | |
| 0.02187499237060547, | |
| 0.021745872497558594, | |
| 0.021839311599731445, | |
| 0.021856752395629883, | |
| 0.021882192611694336, | |
| 0.021824911117553712, | |
| 0.022031471252441405, | |
| 0.022068592071533204, | |
| 0.022063631057739258, | |
| 0.021949392318725586, | |
| 0.022044431686401366, | |
| 0.021936912536621094, | |
| 0.022140752792358397, | |
| 0.022066192626953127, | |
| 0.02201915168762207, | |
| 0.02199611282348633, | |
| 0.022025232315063478, | |
| 0.02185291290283203, | |
| 0.02201019287109375, | |
| 0.021972431182861327, | |
| 0.02187803268432617, | |
| 0.02206907272338867, | |
| 0.02208955192565918, | |
| 0.022294511795043947, | |
| 0.022291471481323242, | |
| 0.022222511291503905, | |
| 0.022308591842651368, | |
| 0.022245712280273437, | |
| 0.022381071090698243, | |
| 0.02234859275817871, | |
| 0.022368751525878906, | |
| 0.022469711303710938, | |
| 0.022372432708740234, | |
| 0.022433551788330078, | |
| 0.02239179229736328, | |
| 0.022424911499023436, | |
| 0.022390029907226563, | |
| 0.022470191955566408, | |
| 0.02241867256164551, | |
| 0.0224058723449707, | |
| 0.022462991714477538, | |
| 0.022332592010498045, | |
| 0.022465871810913085, | |
| 0.02234459114074707, | |
| 0.02235403251647949, | |
| 0.02237803268432617, | |
| 0.02232411193847656, | |
| 0.02249307060241699, | |
| 0.022487632751464842, | |
| 0.02245467185974121, | |
| 0.02240363121032715, | |
| 0.022560909271240236, | |
| 0.02243275260925293, | |
| 0.02252571105957031, | |
| 0.022423791885375976, | |
| 0.022586509704589842, | |
| 0.022435632705688477, | |
| 0.022485071182250977, | |
| 0.02246107292175293, | |
| 0.022455469131469726, | |
| 0.02252203178405762, | |
| 0.022583629608154295, | |
| 0.02243579292297363, | |
| 0.02250107192993164, | |
| 0.022488592147827148, | |
| 0.022521869659423828, | |
| 0.022406991958618165, | |
| 0.02258363151550293, | |
| 0.022417232513427733, | |
| 0.02247067070007324, | |
| 0.022341072082519533, | |
| 0.022482831954956053, | |
| 0.0224543514251709, | |
| 0.022464912414550783, | |
| 0.022441072463989256, | |
| 0.02271002960205078, | |
| 0.022395631790161134, | |
| 0.022248111724853515, | |
| 0.021988271713256836, | |
| 0.022166351318359374, | |
| 0.022313711166381835, | |
| 0.02242923164367676, | |
| 0.022342031478881835, | |
| 0.022845550537109374, | |
| 0.02248523139953613, | |
| 0.022435951232910156, | |
| 0.022028432846069337, | |
| 0.022121391296386718, | |
| 0.02207899284362793, | |
| 0.02210523223876953, | |
| 0.02199787139892578, | |
| 0.022596590042114256, | |
| 0.022396591186523436, | |
| 0.022208112716674805, | |
| 0.02226411247253418, | |
| 0.022019952774047852, | |
| 0.02222923278808594, | |
| 0.02229643249511719, | |
| 0.022334991455078126, | |
| 0.022219951629638673, | |
| 0.022143951416015627, | |
| 0.022292112350463866, | |
| 0.02228443145751953, | |
| 0.022219472885131836, | |
| 0.022264432907104493, | |
| 0.02230299186706543, | |
| 0.022273391723632813, | |
| 0.022191152572631836, | |
| 0.021997072219848632, | |
| 0.021929872512817382, | |
| 0.022081071853637696, | |
| 0.021884111404418944, | |
| 0.021947471618652344, | |
| 0.021899471282958983, | |
| 0.02205611228942871, | |
| 0.021859952926635742, | |
| 0.0219573917388916, | |
| 0.02201019287109375, | |
| 0.02201883125305176, | |
| 0.021958671569824218, | |
| 0.02195195198059082, | |
| 0.02202555274963379, | |
| 0.021934192657470705, | |
| 0.02198075294494629, | |
| 0.022266672134399413, | |
| 0.02229899215698242, | |
| 0.022193071365356445, | |
| 0.022345232009887694, | |
| 0.022344432830810546, | |
| 0.022257232666015626, | |
| 0.021975471496582032, | |
| 0.02198651123046875, | |
| 0.021977231979370116, | |
| 0.0220393123626709, | |
| 0.021977712631225586, | |
| 0.02205659294128418, | |
| 0.022006351470947264, | |
| 0.0221036319732666, | |
| 0.022057552337646485, | |
| 0.022074512481689453, | |
| 0.02202699279785156, | |
| 0.021975631713867187, | |
| 0.022073232650756835, | |
| 0.021991151809692383, | |
| 0.022099952697753906, | |
| 0.021990032196044923, | |
| 0.02200971221923828, | |
| 0.02196459197998047, | |
| 0.022037391662597657, | |
| 0.021905712127685547, | |
| 0.02196843147277832, | |
| 0.021946992874145508, | |
| 0.021975631713867187, | |
| 0.021917552947998048, | |
| 0.021992271423339843, | |
| 0.021899471282958983, | |
| 0.02206715202331543, | |
| 0.02191707229614258, | |
| 0.022063472747802734, | |
| 0.021920431137084962, | |
| 0.021896432876586915, | |
| 0.021892431259155274, | |
| 0.021973392486572266, | |
| 0.021875152587890623, | |
| 0.021917552947998048, | |
| 0.02194331169128418, | |
| 0.022065391540527345, | |
| 0.021958032608032228, | |
| 0.021986991882324218, | |
| 0.021906351089477537, | |
| 0.021984432220458986, | |
| 0.02202203178405762, | |
| 0.022033872604370116, | |
| 0.022234352111816406, | |
| 0.022291791915893554, | |
| 0.02254458999633789, | |
| 0.0223111515045166, | |
| 0.022194992065429686, | |
| 0.022181072235107423, | |
| 0.022146671295166016, | |
| 0.022020751953125, | |
| 0.021979631423950194, | |
| 0.021949071884155274, | |
| 0.022034351348876952, | |
| 0.02200811195373535, | |
| 0.02202299118041992, | |
| 0.022334192276000978, | |
| 0.022065231323242188, | |
| 0.022102832794189453, | |
| 0.02202155113220215, | |
| 0.022015951156616212, | |
| 0.022033872604370116, | |
| 0.02204715156555176, | |
| 0.02202827262878418, | |
| 0.022565711975097658, | |
| 0.022052431106567384, | |
| 0.02208203125, | |
| 0.021861072540283202, | |
| 0.022007471084594728, | |
| 0.021879791259765625, | |
| 0.022010351181030275, | |
| 0.02199675178527832, | |
| 0.02262539291381836, | |
| 0.022059152603149414, | |
| 0.02203755187988281, | |
| 0.022059152603149414, | |
| 0.021998512268066407, | |
| 0.02195243263244629, | |
| 0.021986991882324218, | |
| 0.021900272369384764, | |
| 0.02285803031921387, | |
| 0.02200779151916504, | |
| 0.022025232315063478, | |
| 0.021956111907958983, | |
| 0.02198539161682129, | |
| 0.021954992294311523, | |
| 0.022010032653808593, | |
| 0.02197323226928711, | |
| 0.022328592300415038, | |
| 0.022033712387084962, | |
| 0.0219125919342041, | |
| 0.021933391571044923, | |
| 0.021865871429443358, | |
| 0.02200923156738281, | |
| 0.021856271743774414, | |
| 0.022008752822875975, | |
| 0.021903472900390623, | |
| 0.021971311569213867, | |
| 0.021878511428833007, | |
| 0.02206443214416504, | |
| 0.021794191360473635, | |
| 0.02195355224609375, | |
| 0.021825071334838866, | |
| 0.021933071136474608, | |
| 0.021960912704467775, | |
| 0.021962032318115235, | |
| 0.021780431747436522, | |
| 0.022022192001342773, | |
| 0.021760751724243164, | |
| 0.022062992095947264, | |
| 0.02182459259033203, | |
| 0.022029552459716797, | |
| 0.02191371154785156, | |
| 0.022080751419067384, | |
| 0.02191499137878418, | |
| 0.022058832168579103, | |
| 0.021859952926635742, | |
| 0.02204987144470215, | |
| 0.0218199520111084, | |
| 0.022072912216186524, | |
| 0.021930992126464845, | |
| 0.022016752243041993, | |
| 0.021973072052001955, | |
| 0.022000591278076173, | |
| 0.021992752075195313, | |
| 0.021958831787109376, | |
| 0.021967151641845702, | |
| 0.02197915267944336, | |
| 0.0219826717376709, | |
| 0.021965232849121094, | |
| 0.021981231689453123, | |
| 0.02206987190246582, | |
| 0.021961231231689454, | |
| 0.022060592651367186, | |
| 0.022099311828613282, | |
| 0.022021072387695313, | |
| 0.02202123260498047, | |
| 0.022062511444091798, | |
| 0.022020111083984373, | |
| 0.021982831954956053, | |
| 0.022001392364501954, | |
| 0.02201131248474121, | |
| 0.022097551345825195, | |
| 0.022068592071533204, | |
| 0.022059791564941405, | |
| 0.021986032485961916, | |
| 0.021960752487182617, | |
| 0.021999631881713867, | |
| 0.021929231643676758, | |
| 0.021948591232299804, | |
| 0.021918031692504884, | |
| 0.021991952896118164, | |
| 0.021826992034912108, | |
| 0.021979312896728515, | |
| 0.021957231521606446, | |
| 0.021931791305541994, | |
| 0.022042671203613282, | |
| 0.022018991470336913, | |
| 0.02207707214355469, | |
| 0.02201915168762207, | |
| 0.02202427291870117, | |
| 0.022037391662597657, | |
| 0.02191307258605957, | |
| 0.022036752700805663, | |
| 0.021976112365722656, | |
| 0.022057392120361327, | |
| 0.02203963279724121, | |
| 0.02203835105895996, | |
| 0.02204587173461914, | |
| 0.021910032272338866, | |
| 0.02212059211730957, | |
| 0.02208171272277832, | |
| 0.02205259132385254, | |
| 0.022110031127929686, | |
| 0.022194992065429686, | |
| 0.02222667121887207, | |
| 0.02220267105102539, | |
| 0.02223019218444824, | |
| 0.02237019157409668, | |
| 0.02234699249267578, | |
| 0.022475151062011717, | |
| 0.022305072784423827, | |
| 0.022449871063232422, | |
| 0.022356592178344726, | |
| 0.02208443260192871, | |
| 0.022066511154174805, | |
| 0.021978031158447264, | |
| 0.022011951446533205, | |
| 0.02204043197631836, | |
| 0.02199083137512207, | |
| 0.022059152603149414, | |
| 0.02255035209655762, | |
| 0.022002992630004884, | |
| 0.02206603240966797, | |
| 0.022015312194824218, | |
| 0.022029071807861327, | |
| 0.022063791275024416, | |
| 0.022011632919311522, | |
| 0.022140592575073243, | |
| 0.022710992813110353, | |
| 0.022164911270141602, | |
| 0.022124752044677735, | |
| 0.02200667190551758, | |
| 0.022048112869262695, | |
| 0.022124591827392577, | |
| 0.02213083267211914, | |
| 0.022087312698364257, | |
| 0.02288186836242676, | |
| 0.022104272842407225, | |
| 0.022044591903686524, | |
| 0.02208363151550293, | |
| 0.022128271102905273, | |
| 0.022027471542358398, | |
| 0.02203755187988281, | |
| 0.02213163185119629, | |
| 0.02227947235107422, | |
| 0.022184911727905272, | |
| 0.02183675193786621, | |
| 0.02196299171447754, | |
| 0.021805871963500978, | |
| 0.02210539245605469, | |
| 0.021823471069335937, | |
| 0.022016271591186524, | |
| 0.02188443183898926, | |
| 0.022007951736450194, | |
| 0.0218580322265625, | |
| 0.02192523193359375, | |
| 0.021844911575317382, | |
| 0.02198315238952637, | |
| 0.02180427169799805, | |
| 0.021981231689453123, | |
| 0.02187563133239746, | |
| 0.022151952743530275, | |
| 0.021775632858276366, | |
| 0.021969871520996095, | |
| 0.02178251266479492, | |
| 0.02205131149291992, | |
| 0.021931951522827148, | |
| 0.022111631393432616, | |
| 0.021881551742553712, | |
| 0.022056432723999025, | |
| 0.021822032928466798, | |
| 0.022114032745361327, | |
| 0.021835792541503907, | |
| 0.02201291275024414, | |
| 0.02184395217895508, | |
| 0.022004112243652344, | |
| 0.021970191955566407, | |
| 0.02193435287475586, | |
| 0.021921072006225586, | |
| 0.021936431884765625, | |
| 0.021928272247314452, | |
| 0.02192731285095215, | |
| 0.0220109920501709, | |
| 0.021985231399536134, | |
| 0.021894512176513673, | |
| 0.02195707130432129, | |
| 0.022084272384643555, | |
| 0.02205131149291992, | |
| 0.02208795166015625, | |
| 0.0220042724609375, | |
| 0.022103952407836913, | |
| 0.02200155258178711, | |
| 0.021983631134033205, | |
| 0.022027151107788086, | |
| 0.02197867202758789, | |
| 0.02197995185852051, | |
| 0.022031471252441405, | |
| 0.02198539161682129, | |
| 0.022004751205444335, | |
| 0.022059152603149414, | |
| 0.021975631713867187, | |
| 0.02207067108154297, | |
| 0.02201019287109375, | |
| 0.022032432556152344, | |
| 0.021992431640625, | |
| 0.02196571159362793, | |
| 0.02200267219543457, | |
| 0.02202939224243164, | |
| 0.021873552322387694, | |
| 0.02204747200012207, | |
| 0.0220089111328125, | |
| 0.021990032196044923, | |
| 0.021938671112060548, | |
| 0.022052112579345702, | |
| 0.021936431884765625, | |
| 0.022045711517333984, | |
| 0.021954992294311523, | |
| 0.021959632873535157, | |
| 0.02197483253479004, | |
| 0.022019952774047852, | |
| 0.02201131248474121, | |
| 0.0220228328704834, | |
| 0.022025232315063478, | |
| 0.021995311737060548, | |
| 0.021957712173461916, | |
| 0.021810991287231445, | |
| 0.022035152435302734, | |
| 0.022013551712036134, | |
| 0.021950191497802734, | |
| 0.022020751953125, | |
| 0.02207691192626953, | |
| 0.02221659278869629, | |
| 0.02211851119995117, | |
| 0.022118192672729492, | |
| 0.02207067108154297, | |
| 0.02190123176574707, | |
| 0.022146991729736328, | |
| 0.022068592071533204, | |
| 0.022080432891845702, | |
| 0.02234619140625, | |
| 0.02202043151855469, | |
| 0.02202827262878418, | |
| 0.022045711517333984, | |
| 0.022033712387084962, | |
| 0.02198459243774414, | |
| 0.022075151443481444, | |
| 0.022007471084594728, | |
| 0.022570669174194338, | |
| 0.022097711563110353, | |
| 0.02217323112487793, | |
| 0.022009552001953123, | |
| 0.022065711975097657, | |
| 0.022066831588745117, | |
| 0.022029071807861327, | |
| 0.022106191635131837, | |
| 0.02266539192199707, | |
| 0.022094671249389648, | |
| 0.022126672744750976, | |
| 0.02197675132751465, | |
| 0.022089712142944334, | |
| 0.022086191177368163, | |
| 0.02201083183288574, | |
| 0.021974191665649415, | |
| 0.022857868194580078, | |
| 0.02200587272644043, | |
| 0.02202827262878418, | |
| 0.02203771209716797, | |
| 0.02202427291870117, | |
| 0.021985551834106446, | |
| 0.022129552841186523, | |
| 0.02205994987487793 | |
| ] | |
| }, | |
| "throughput": { | |
| "unit": "tokens/s", | |
| "value": 181.17342916621183 | |
| }, | |
| "energy": null, | |
| "efficiency": null | |
| } | |
| } |