Spaces:
Sleeping
Sleeping
import asyncio | |
import random | |
from itertools import combinations | |
from time import time | |
from types import SimpleNamespace | |
from typing import Dict, List, Literal, Tuple | |
from varco_arena_core.prompts import ComparisonPromptBase | |
from .match import Match | |
async def limited_coro(coro, semaphore): | |
async with semaphore: | |
return await coro | |
class League: | |
""" | |
Full grid matches of the model outputs (all-play-all) | |
""" | |
def __init__(self, participants, evaluation_model): | |
self.participants = participants | |
self.evaluation_model = evaluation_model | |
async def async_run( | |
self, | |
prompt_obj: ComparisonPromptBase = None, | |
semaphore: asyncio.Semaphore = None, | |
) -> List[Dict]: | |
async def get_match_results(a, b) -> List[Dict]: | |
match = Match(A=a, B=b, eval_model=self.evaluation_model) | |
if self.evaluation_model == "debug": | |
winner, match_result = await match.async_dbg_eval() | |
else: | |
winner, match_result = await match.async_comp_eval( | |
comp_prompt=prompt_obj, | |
) | |
if winner is not None: | |
now_time = time() | |
# A vs B, B vs A | |
result = [ | |
{ # 여기서 *_a, *_b 는 prompt내의 position에 관한 것이다. Match.A, Match.B 가 아니다. | |
"task": a.task, # participant_pair[1].task | |
"model_a": a.model_id, | |
"model_b": b.model_id, | |
"winner": "A" | |
if match_result[0]["A"] > match_result[0]["B"] | |
else "B", | |
"prob_a": match_result[0]["A"], | |
"prob_b": match_result[0]["B"], | |
"evaluation_model": self.evaluation_model, | |
"instruction": a.instruction, # participant_pair[1].instruction, | |
"source": a.source, # participant_pair[1].source, | |
"generated_a": a.generated, | |
"generated_b": b.generated, | |
"round": "league", | |
"match_order_in_round": "league", | |
"tstamp": now_time, | |
"api_call_kwargs": match_result[0]["api_call_kwargs"], | |
"actual_response_text": match_result[0]["actual_response_text"], | |
}, | |
] | |
else: | |
result = None | |
return result | |
match_jobs = [] | |
for participant_pair in combinations(self.participants, 2): | |
match_jobs.append( | |
limited_coro( | |
get_match_results(participant_pair[0], participant_pair[1]), | |
semaphore, | |
) | |
) | |
# match_jobs.append( get_match_results(participant_pair[0], participant_pair[1]) ) | |
result = await asyncio.gather(*match_jobs) | |
result = [ | |
_result | |
for result_pair in result | |
if result_pair is not None | |
for _result in result_pair | |
] | |
return result # returning only match result, not the winner's SimpleNamespace (because it would never be used.) | |