feat: update about session

.gitignore

@@ -11,3 +11,5 @@ eval-results/
 eval-queue-bk/
 eval-results-bk/
 logs/
+
+

app.py

@@ -1,4 +1,5 @@
 import json
+import os
 import time
 from pathlib import Path
 
@@ -23,6 +24,8 @@ from src.submission.check_validity import (
 )
 from src.submission.submit import update_dataset_with_scores
 
+STATIC_DIR = str(Path(__file__).parent / "src" / "static")
+
 # Global state for leaderboard data
 current_leaderboard_df = None
 
@@ -102,6 +105,7 @@ def process_submission(
 
 # Create the Gradio interface
 demo = gr.Blocks().queue()
+demo.static_dir = STATIC_DIR
 
 with demo:
     gr.HTML(TITLE)

src/about.py

@@ -14,51 +14,71 @@ NUM_FEWSHOT = 0  # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center">🧪 ChemBench Leaderboard</h1>"""
+TITLE = """<h1 align="center">👩‍🔬 MaCBench Leaderboard </h1>"""
 
-TITLE_MARKDOWN_DESCRIPTION = """Welcome to ChemBench, a comprehensive benchmark for evaluating language models on Chemical Reasoning, Knowledge and Intuition."""
+TITLE_MARKDOWN_DESCRIPTION = """Can VLMs assist in scientific discovery?"""
 
 ABOUT_TEXT = """
-# Welcome to *ChemBench* 🧪
-*ChemBench* is a comprehensive toolkit for benchmarking language and multimodal models in chemistry tasks. *ChemBench* is designed to be modular and extensible, allowing users to easily add new datasets, models, and evaluation metrics.
-## Key Features 🔑
-- **Standardized Benchmarking**: Built on the proven Big Bench format
-- **Universal Model Support**: Seamless integration with LiteLLM backend
-- **Scientifically sound parsing and evaluation**: *ChemBench* goes beyond simple MCQ evaluation, supports floating point answers (also in scientific notation). We also support different extraction techniques, including LLM-based extraction.
-- **Extensible**: *ChemBench* is designed to be easily extensible. Prompts, models, extraction, and evaluation methods can be easily added and customized via a pipeline system.
-- **Easy to use**: *ChemBench* is designed to be easy to use. We provide a simple interface you can use to get started testing your models with just a few lines of code.
-- **Caching and parallelization**: *ChemBench* can cache results and can parallelize inference to speed up benchmarking.
-- **Refusal detection**: *ChemBench* can detect when a model refuses to answer a question, handle it, and report it in the final results.
-## Getting Started 🚀
-Start benchmarking your models with just a few lines of code. Here's how you can get started:
+<h1 align="center">
+  Can VLMs assist in scientific discovery?
+</h1>
+
+*MaCBench* is a benchmark for multimodal language models in chemistry and materials research. We manually curated a benchmark of tasks across the entire scientific lifecycle from data extraction, experiment execution to data analysis. 
+
+This repository contains the reports and is used to collect new contributions. 
+The benchmark dataset itself is hosted on [HuggingFace](https://huggingface.co/datasets/jablonkagroup/MaCBench) and can be run using the [*ChemBench* benchmark engine](https://github.com/lamalab-org/chembench). You can find more details on how to do so in the [*ChemBench* documentation](https://lamalab-org.github.io/chembench/).
+
+
+## Running *MaCBench* Evaluation 🚀
+
+*MaCBench* can be run using [*ChemBench*](https://github.com/lamalab-org/chembench) pipeline. For that, the only thing needed is to create the running script. The following script is an example of how to run the benchmark using the `gpt-4o` model.
+
 ```python
-from chembench.evaluate import ChemBenchmark
-from chembench.prompter import PrompterBuilder
-from chembench.utilis import enable_logging
+from chembench.evaluate import ChemBenchmark, save_topic_reports
+from chembench.prompter import PrompterBuilder, PrompterPipeline
+from chembench.utils import (
+    enable_caching,
+    enable_logging,
+)
 from dotenv import load_dotenv
-load_dotenv(".env")
+
+# Load environment variables
+load_dotenv("../../.env", override=True)
+
+# Enable logging and caching
 enable_logging()
-benchmark = ChemBenchmark.from_huggingface()
-model = "openai/gpt-4"  
+enable_caching()
+
+# Define the prompter object specifying the model and "multimodal_instruction" prompt type
 prompter = PrompterBuilder.from_model_object(
-    model=model,
+    "openai/gpt-4o-2024-08-06", prompt_type="multimodal_instruction"
 )
-results = benchmark.bench(
-    prompter,
-)
-benchmark.submit(results) 
+
+# Load benchmark from HuggingFace
+benchmark = ChemBenchmark.from_huggingface("jablonkagroup/MaCBench")
+
+# Run benchmark with batch processing equal to 1
+result = benchmark.bench(prompter, batch_size=1, model_kwargs={"temperature": 0})
+
+benchmark.submit(results) # submit results to leaderboard
 ```
-For more details, check out our [documentation](https://lamalab-org.github.io/chembench/getting_started/)
+
+
+For more details, check out our [documentation](https://lamalab-org.github.io/chembench/getting_started/#how-to-benchmark-on-multi-modal-tasks)
+
 ## Scientific Foundation 📚
-*ChemBench*  has been validated through peer review and extensive testing. For detailed methodology, check our papers:
+For detailed methodology, check our papers:
+- [*MaCBench* Paper](https://arxiv.org/pdf/2411.16955)
 - [*ChemBench*  Paper](https://arxiv.org/abs/2404.01475)
-- [Multimodal Paper](https://arxiv.org/pdf/2411.16955)
+
+
 Ready to elevate your chemistry AI research?  🧬
-*ChemBench* is open-source software licensed under the MIT license. 
+
+*MaCBench* is open-source software licensed under the MIT license. 
 Its development is led by the [Lab for AI for Materials (LamaLab)](https://jablonkagroup.uni-jena.de/) at the [University of Jena](https://www.uni-jena.de/) and the [Helmholtz Institute for Polymers in Energy Applications Jena](https://www.hipole-jena.de/).
 ## Connect With Us 🔗
 <div class="social-links">
-<a href="https://github.com/lamalab-org/chembench.git" target="_blank">
+<a href="https://github.com/lamalab-org/macbench.git" target="_blank">
     <img src="https://raw.githubusercontent.com/FortAwesome/Font-Awesome/master/svgs/brands/github.svg" width="20" height="20" alt="GitHub"/> GitHub
 </a>
 <a href="https://twitter.com/jablonkagroup" target="_blank">
@@ -93,10 +113,13 @@ Its development is led by the [Lab for AI for Materials (LamaLab)](https://jablo
 
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""@article{mirza2024large,
-  title   = {Are large language models superhuman chemists?},
-  author  = {Adrian Mirza and Nawaf Alampara and Sreekanth Kunchapu and Benedict Emoekabu and Aswanth Krishnan and Mara Wilhelmi and Macjonathan Okereke and Juliane Eberhardt and Amir Mohammad Elahi and Maximilian Greiner and Caroline T. Holick and Tanya Gupta and Mehrdad Asgari and Christina Glaubitz and Lea C. Klepsch and Yannik Köster and Jakob Meyer and Santiago Miret and Tim Hoffmann and Fabian Alexander Kreth and Michael Ringleb and Nicole Roesner and Ulrich S. Schubert and Leanne M. Stafast and Dinga Wonanke and Michael Pieler and Philippe Schwaller and Kevin Maik Jablonka},
-  year    = {2024},
-  journal = {arXiv preprint arXiv: 2404.01475}
+CITATION_BUTTON_TEXT = r"""@misc{alampara2024probinglimitationsmultimodallanguage,
+      title={Probing the limitations of multimodal language models for chemistry and materials research}, 
+      author={Nawaf Alampara and Mara Schilling-Wilhelmi and Martiño Ríos-García and Indrajeet Mandal and Pranav Khetarpal and Hargun Singh Grover and N. M. Anoop Krishnan and Kevin Maik Jablonka},
+      year={2024},
+      eprint={2411.16955},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG},
+      url={https://arxiv.org/abs/2411.16955}, 
 }
 """