change to lighteval's latest

README.md

@@ -4,7 +4,7 @@ emoji: 🥇
 colorFrom: green
 colorTo: indigo
 sdk: gradio
-sdk_version: 4.26.0
+sdk_version: 5.1.0
 app_file: app.py
 pinned: true
 license: apache-2.0

requirements.txt

@@ -9,7 +9,7 @@ accelerate>=0.26.0
 sentencepiece
 
 # Evaluation suites
-lighteval
+lighteval>=0.5.0
 lm_eval==0.4.3
 
 # Log Visualizer

src/backend/run_eval_suite_lighteval.py

@@ -3,7 +3,12 @@ import argparse
 import logging
 from datetime import datetime
 
-from lighteval.main_accelerate import main, EnvConfig, create_model_config, load_model
+import lighteval
+from lighteval.logging.evaluation_tracker import EvaluationTracker
+from lighteval.models.model_config import InferenceEndpointModelConfig
+from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters
+
+from lighteval.main_accelerate import main, EnvConfig, create_model_config
 
 from src.envs import RESULTS_REPO, CACHE_PATH, TOKEN
 from src.backend.manage_requests import EvalRequest
@@ -32,57 +37,55 @@ def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int,
     if limit:
         logger.info("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
 
-    args_dict = {
-            # Endpoint parameters
-            "endpoint_model_name":eval_request.model,
-            "accelerator": accelerator,
-            "vendor": vendor,
-            "region": region,
-            "instance_size": instance_size,
-            "instance_type": instance_type,
-            "reuse_existing": False,
-            "model_dtype": eval_request.precision,
-            "revision": eval_request.revision,
-            # Save parameters
-            "push_results_to_hub": True,
-            "save_details": True,
-            "push_details_to_hub": True,
-            "public_run": False,
-            "cache_dir": CACHE_PATH,
-            "results_org": RESULTS_REPO,
-            "output_dir": local_dir,
-            "job_id": str(datetime.now()),
-            # Experiment parameters
-            "override_batch_size": batch_size,
-            "custom_tasks": "custom_tasks.py",
-            "tasks": task_names,
-            "max_samples": limit,
-            "use_chat_template": False,
-            "system_prompt": None,
-            # Parameters which would be set to things by the kwargs if actually using argparse
-            "inference_server_address": None,
-            "model_args": None,
-            "num_fewshot_seeds": None,
-            "delta_weights": False,
-            "adapter_weights": False
-    }
-    args = argparse.Namespace(**args_dict)
+    evaluation_tracker = EvaluationTracker(
+        output_dir="./results",
+        save_details = True,
+        push_to_hub = True,
+        push_to_tensorboard = False,
+        hub_results_org= RESULTS_REPO,
+        public = False,
+    )
 
-    try:
-        results = main(args)
+    pipeline_params = PipelineParameters(
+        launcher_type=ParallelismManager.ACCELERATE,
+        override_batch_size=batch_size,
+        max_samples=limit,
+        use_chat_template=False,
+        system_prompt=None,
+        custom_tasks_directory="custom_tasks.py", # if using a custom task
+    )
+
+    model_config = InferenceEndpointModelConfig(
+        # Endpoint parameters
+        name = eval_request.model.replace(".", "-").lower(), 
+        repository = eval_request.model,
+        accelerator =  accelerator,
+        vendor= vendor,
+        region= region,
+        instance_size= instance_size,
+        instance_type= instance_type,
+        should_reuse_existing= False,
+        model_dtype= eval_request.precision,
+        revision= eval_request.revision,
+    )
+
+    pipeline = Pipeline(
+        tasks=task_names,
+        pipeline_parameters=pipeline_params,
+        evaluation_tracker=evaluation_tracker,
+        model_config=model_config,
+    )
 
-        results["config"]["model_dtype"] = eval_request.precision
-        results["config"]["model_name"] = eval_request.model
-        results["config"]["model_sha"] = eval_request.revision
+    try:
+        pipeline.evaluate()
+        pipeline.show_results()
+        pipeline.save_and_push_results()
+        results = pipeline.get_results()
 
         dumped = json.dumps(results, indent=2)
         logger.info(dumped)
-    except Exception as e: # if eval failed, we force a cleanup
-        env_config = EnvConfig(token=TOKEN, cache_dir=args.cache_dir)
-
-        model_config = create_model_config(args=args, accelerator=accelerator)
-        model, _ = load_model(config=model_config, env_config=env_config)
-        model.cleanup()
 
+    except Exception as e: # if eval failed, we force a cleanup
+        pipeline.model.cleanup()
 
     return results