Paper cuts (#64)

about.py

@@ -3,76 +3,127 @@ import gradio as gr
 
 def build_page():
     with gr.Column(elem_id="about-page-content-wrapper"):
-        gr.Markdown(
+        # --- Section 1: About AstaBench ---
+        gr.HTML(
             """
-        ## About AstaBench
-        AstaBench is a novel AI agents evaluation framework, providing a challenging new test for AI agents: the first benchmark challenge that evaluates agents’ scientific abilities on a broad spectrum of research skills, including literature understanding, data analysis, planning, tool use, coding, and search. Asta’s set of standard tools makes it easy to build general-purpose science agents and to compare their performance in an apples-to-apples manner.
-        """)
-        gr.Markdown("---",elem_classes="divider-line")
+            <h2>About AstaBench</h2>
+            <p>
+                AstaBench is a novel AI agents evaluation framework, providing a challenging new test for AI agents: the first benchmark challenge that evaluates agents’ scientific abilities on a broad spectrum of research skills, including literature understanding, data analysis, planning, tool use, coding, and search. Asta’s set of standard tools makes it easy to build general-purpose science agents and to compare their performance in an apples-to-apples manner.
+            </p>
+            """
+        )
+        gr.Markdown("---", elem_classes="divider-line")
 
-        gr.Markdown(""" ## Why AstaBench?
-        Most current benchmarks test agentic AI and isolated aspects of scientific reasoning, but rarely evaluate AI agentic behavior rigorously or capture the full skill set scientific research requires. Agents can appear effective despite inconsistent results and high compute use, often outperforming others by consuming more resources. Advancing scientific AI requires evaluations that emphasize reproducibility, efficiency, and the real complexity of research.
-        
-        AstaBench fills this gap: an agents evaluation framework and suite of open benchmarks for evaluating scientific AI assistants on core scientific tasks that require novel reasoning. AstaBench helps scientists identify which agents best support their needs through task-relevant leaderboards, while giving AI developers a standard execution environment and tools to test the scientific reasoning capabilities of their agents compared to well-known baselines from the literature, including both open and closed LLM foundation models.
-        """)
-        gr.Markdown("---",elem_classes="divider-line")
+        # --- Section 2: Why AstaBench? ---
+        gr.HTML(
+            """
+            <h2>Why AstaBench?</h2>
+            <p>
+                Most current benchmarks test agentic AI and isolated aspects of scientific reasoning, but rarely evaluate AI agentic behavior rigorously or capture the full skill set scientific research requires. Agents can appear effective despite inconsistent results and high compute use, often outperforming others by consuming more resources. Advancing scientific AI requires evaluations that emphasize reproducibility, efficiency, and the real complexity of research.
+            </p>
+            <br>
+            <p>
+                AstaBench fills this gap: an agents evaluation framework and suite of open benchmarks for evaluating scientific AI assistants on core scientific tasks that require novel reasoning. AstaBench helps scientists identify which agents best support their needs through task-relevant leaderboards, while giving AI developers a standard execution environment and tools to test the scientific reasoning capabilities of their agents compared to well-known baselines from the literature, including both open and closed LLM foundation models.
+            </p>
+            """
+        )
+        gr.Markdown("---", elem_classes="divider-line")
 
-        gr.Markdown("""
-        ## What Does AstaBench Include?
-        AstaBench includes a rigorous agents evaluation framework and a suite of benchmarks consisting of over 2,400 problems across 11 benchmarks, organized into four core categories:
-        - Literature Understanding
-        - Code & Execution
-        - Data Analysis
-        - End-to-End Discovery
-        Plus: a large suite of integrated agents and leaderboards with results from extensive evaluation of agents and models.
-        
-        🔍 Learn more in the AstaBench technical blog post
-        """)
-        gr.Markdown("---",elem_classes="divider-line")
+        # --- Section 3: What Does AstaBench Include? ---
+        gr.HTML(
+            """
+            <h2>What Does AstaBench Include?</h2>
+            <p>
+                AstaBench includes a rigorous agents evaluation framework and a suite of benchmarks consisting of over 2,400 problems across 11 benchmarks, organized into four core categories:
+            </p>
+            <ul class="info-list">
+                <li>Literature Understanding</li>
+                <li>Code & Execution</li>
+                <li>Data Analysis</li>
+                <li>End-to-End Discovery</li>
+            </ul>
+            <p>
+                Plus: a large suite of integrated agents and leaderboards with results from extensive evaluation of agents and models.
+            </p>
+            <p>
+                🔍 Learn more in the AstaBench technical blog post
+            </p>
+            """
+        )
+        gr.Markdown("---", elem_classes="divider-line")
 
-        gr.Markdown("""
-        ## Understanding the Leaderboards
-        The AstaBench Overall Leaderboard provides a high-level view of overall agent performance and efficiency:
-        - Overall score: A macro-average of the four category-level averages (equal weighting)
-        - Overall cost: Average cost per task, aggregated only across benchmarks with reported cost
-        
-        Each category leaderboard provides:
-        - Average score and cost for that category (macro-averaged across the benchmarks in the category)
-        - A breakdown by individual benchmarks
-        """)
-        gr.Markdown("---",elem_classes="divider-line")
+        # --- Section 4: Understanding the Leaderboards ---
+        gr.HTML(
+            """
+            <h2>Understanding the Leaderboards</h2>
+            <p>
+                The AstaBench Overall Leaderboard provides a high-level view of overall agent performance and efficiency:
+            </p>
+            <ul class="info-list">
+                <li>Overall score: A macro-average of the four category-level averages (equal weighting)</li>
+                <li>Overall cost: Average cost per task, aggregated only across benchmarks with reported cost</li>
+            </ul>
+            <p>
+                Each category leaderboard provides:
+            </p>
+            <ul class="info-list">
+                <li>Average score and cost for that category (macro-averaged across the benchmarks in the category)</li>
+                <li>A breakdown by individual benchmarks</li>
+            </ul>
+            """
+        )
+        gr.Markdown("---", elem_classes="divider-line")
 
-        gr.Markdown("""
-        ## Scoring & Aggregation
-        AstaBench encourages careful, transparent evaluation. Here's how we handle scoring, cost, and partial results:
-        
-        **Scores**
-        - Each benchmark returns an average score based on per-problem scores
-        - All scores are aggregated upward using macro-averaging
-        - Partial completions are included (even with poor performance)
-        
-        **Cost**
-        - Costs are reported in USD per task.
-        - Benchmarks without cost data are excluded from cost averages
-        - In scatter plots, agents without cost are plotted to the far right and clearly marked. 
-        
-        Note: Cost values reflect pricing and infrastructure conditions at the time of each submission. We recognize that compute costs may change over time, and are actively working on methods to normalize cost data across submissions for fairer longitudinal comparisons.
-        
-        **Coverage**
-        - Main leaderboard: category coverage (X/4)
-        - Category view: benchmark coverage (X/Y)
-        - Incomplete coverage is flagged visually
-        
-        These design choices ensure fair comparison while penalizing cherry-picking and omissions.
-        """)
-        gr.Markdown("---",elem_classes="divider-line")
+        # --- Section 5: Scoring & Aggregation ---
+        gr.HTML(
+            """
+            <h2>Scoring & Aggregation</h2>
+            <p>
+                AstaBench encourages careful, transparent evaluation. Here's how we handle scoring, cost, and partial results:
+            </p>
+            
+            <h3>Scores</h3>
+            <ul class="info-list">
+                <li>Each benchmark returns an average score based on per-problem scores</li>
+                <li>All scores are aggregated upward using macro-averaging</li>
+                <li>Partial completions are included (even with poor performance)</li>
+            </ul>
+            
+            <h3>Cost</h3>
+            <ul class="info-list">
+                <li>Costs are reported in USD per task.</li>
+                <li>Benchmarks without cost data are excluded from cost averages</li>
+                <li>In scatter plots, agents without cost are plotted to the far right and clearly marked.</li>
+            </ul>
+            
+            <p>
+                <em>Note: Cost values reflect pricing and infrastructure conditions at the time of each submission. We recognize that compute costs may change over time, and are actively working on methods to normalize cost data across submissions for fairer longitudinal comparisons.</em>
+            </p>
+            
+            <h3>Coverage</h3>
+            <ul class="info-list">
+                <li>Main leaderboard: category coverage (X/4)</li>
+                <li>Category view: benchmark coverage (X/Y)</li>
+                <li>Incomplete coverage is flagged visually</li>
+            </ul>
+            
+            <p>
+                These design choices ensure fair comparison while penalizing cherry-picking and omissions.
+            </p>
+            """
+        )
+        gr.Markdown("---", elem_classes="divider-line")
 
-        gr.Markdown("""
-        ## Learn More
-        - AstaBench technical blog post
-        - FAQ and submission guide
-        """
-            )
+        # --- Section 6: Learn More ---
+        gr.HTML(
+            """
+            <h2>Learn More</h2>
+            <ul class="info-list">
+                <li>AstaBench technical blog post</li>
+                <li>FAQ and submission guide</li>
+            </ul>
+            """
+        )
 
         # Floating feedback button
         floating_feedback_button_html = """

category_page_builder.py

@@ -6,6 +6,7 @@ from ui_components import create_leaderboard_display, create_benchmark_details_d
 
 def build_category_page(CATEGORY_NAME, PAGE_DESCRIPTION):
     with gr.Column(elem_id="page-content-wrapper"):
+        gr.HTML(f'<h2>AstaBench {CATEGORY_NAME} Leaderboard <span style="font-weight: normal; color: inherit;">(Aggregate)</span></h2>', elem_id="main-header")
         validation_df, validation_tag_map = get_full_leaderboard_data("validation")
         test_df, test_tag_map = get_full_leaderboard_data("test")
         with gr.Column(elem_id="validation_nav_container", visible=False) as validation_nav_container:
@@ -13,7 +14,6 @@ def build_category_page(CATEGORY_NAME, PAGE_DESCRIPTION):
 
         with gr.Column(elem_id="test_nav_container", visible=True) as test_nav_container:
             create_sub_navigation_bar(test_tag_map, CATEGORY_NAME)
-        gr.HTML(f'<h2>AstaBench {CATEGORY_NAME} Leaderboard <span style="font-weight: normal; color: inherit;">(Aggregate)</span></h2>', elem_id="main-header")
         gr.Markdown(PAGE_DESCRIPTION, elem_id="category-intro")
         # --- This page now has two main sections: Validation and Test ---
         with gr.Tabs():

content.py

@@ -1,48 +1,45 @@
 TITLE = """<h1 align="left" id="space-title">AstaBench Leaderboard</h1>"""
 
 INTRO_PARAGRAPH = """
-Newer benchmarks may test agentic AI and isolated aspects of scientific reasoning, but none rigorously measure agentic AI or capture the full range of skills research demands. Agents can appear effective by simply retrying tasks—often at high computational cost and with inconsistent results. Scientific AI needs evaluations that reflect the real complexity of research.
+<p>
+    Newer benchmarks may test agentic AI and isolated aspects of scientific reasoning, but none rigorously measure agentic AI or capture the full range of skills research demands. Agents can appear effective by simply retrying tasks—often at high computational cost and with inconsistent results. Scientific AI needs evaluations that reflect the real complexity of research.
+</p>
 <br>
+<p>
+    AstaBench fills that gap: a suite of open benchmarks for evaluating scientific AI assistants on core scientific tasks that require novel reasoning. The suite includes over 8,000 tasks across 11 benchmarks, organized into four core categories: Literature Understanding, Code & Execution, Data Analysis, and End-to-End Discovery.
+</p>
 <br>
-AstaBench fills that gap: a suite of open benchmarks for evaluating scientific AI assistants on core scientific tasks that require novel reasoning. The suite includes over 8,000 tasks across 11 benchmarks, organized into four core categories: Literature Understanding, Code & Execution, Data Analysis, and End-to-End Discovery.
+<p>
+    The <strong>AstaBench Leaderboard</strong> below provides a high-level summary of agent performance and efficiency. It includes:
+</p>
+<ul class="info-list">
+    <li>
+        An <strong>overall score</strong>, computed as a macro average of the four category-level macro averages, ensuring each domain contributes equally—regardless of how many benchmarks each category includes. This provides a fair and balanced comparison across agents with varying capabilities.
+    </li>
+    <li>
+        An <strong>overall average cost per task</strong>, consistently aggregated across all categories, to reflect the real efficiency of each agent under comparable conditions.
+    </li>
+</ul>
 <br>
+<p>
+    To support domain-specific insight, AstaBench also provides per-category leaderboards:
+</p>
+<ul class="info-list">
+    <li>Literature Understanding</li>
+    <li>Code & Execution</li>
+    <li>Data Analysis</li>
+    <li>End-to-End Discovery</li>
+</ul>
 <br>
-The **AstaBench Leaderboard** below provides a high-level summary of agent performance and efficiency. It includes:
-<br>
-<br>
-- An **overall score**, computed as a macro average of the four category-level macro averages, ensuring each domain contributes equally—regardless of how many benchmarks each category includes. This provides a fair and balanced comparison across agents with varying capabilities.
-- An **overall average cost per task**, consistently aggregated across all categories, to reflect the real efficiency of each agent under comparable conditions.
-<br>
-To support domain-specific insight, AstaBench also provides per-category leaderboards:
-<br>
-<br>
-- Literature Understanding
-<br>
-- Code & Execution
-<br>
-- Data Analysis
-<br>
-- End-to-End Discovery
-<br>
-<br>
-Each category page includes a summary table (average score and cost per problem for that domain), as well as per-benchmark leaderboards for detailed comparisons on specific tasks.
-<br>
-<br>
-🔍 Learn more in the AstaBench technical blog post
+<p>
+    Each category page includes a summary table (average score and cost per problem for that domain), as well as per-benchmark leaderboards for detailed comparisons on specific tasks.
+</p>
+<p>
+    🔍 Learn more in the AstaBench technical blog post
+</p>
 """
 SCATTER_DISCLAIMER = """
-**Note:** Agents without cost data are displayed to the right of the vertical divider line. <span class="tooltip-icon" data-tooltip="Missing Cost Dashed Line: Max Cost + (MaxCost/10) Missing Cost Datapoints/No Cost Data = Max Cost + (MaxCost/5)">ⓘ</span>
-"""
-scatter_disclaimer_html = """
-<div class="disclaimer-text">
-    <b>Note:</b> Agents without cost data are displayed to the right of the vertical divider line.
-    <span class="tooltip-icon" data-tooltip="Missing Cost Dashed Line: 
-    Max Cost + (MaxCost/10)
-    Missing Cost Datapoints / No Cost Data: 
-    Max Cost + (MaxCost/5)">
-        ⓘ
-    </span>
-</div>
+**Note:** Agents without cost data are displayed to the right of the vertical divider line.
 """
 PARETO_DISCLAIMER = """
 Agents names that are green are Pareto optimal, meaning they achieve the best performance for their cost. 
@@ -246,7 +243,7 @@ nav.svelte-ti537g.svelte-ti537g {
     line-height: 1.2 !important;
     vertical-align: top !important;
     font-size: 12px !important;
-    
+    font-family: 'Manrope';
 }
 .wrap-header-df th {
     height: auto !important;
@@ -539,11 +536,11 @@ span.wrap[tabindex="0"][role="button"][data-editable="false"] {
     margin-top: 40px;
     opacity: 85%; 
 }
-.divider-line {
-    opacity: 40%;
-}
 #leaderboard-accordion table {
     width: auto !important;
     margin-right: auto !important;
 }
+.info-list {
+    padding-left: 20px;
+}
 """

leaderboard_transformer.py

@@ -511,7 +511,7 @@ def _plot_scatter_plotly(
             marker=dict(
                 color=color_map.get(category, 'black'),
                 symbol=group['shape_symbol'],
-                size=10,
+                size=15,
                 opacity=0.8,
                 line=dict(width=1, color='deeppink')
             )
@@ -561,7 +561,7 @@ def _plot_scatter_plotly(
 
     fig.update_layout(
         template="plotly_white",
-        title=f"Astabench {name} Leaderboard",
+        title=f"AstaBench {name} Leaderboard",
         xaxis=xaxis_config, # Use the updated config
         yaxis=dict(title="Average (mean) score", rangemode="tozero"),
         legend=dict(

main_page.py

@@ -16,11 +16,11 @@ CACHED_VIEWERS = {}
 CACHED_TAG_MAPS = {}
 
 def build_page():
-    gr.Markdown(INTRO_PARAGRAPH, elem_id="intro-paragraph")
+    gr.HTML(INTRO_PARAGRAPH, elem_id="intro-paragraph")
     # --- Leaderboard Display Section ---
     gr.Markdown("---")
     CATEGORY_NAME = "Overall"
-    gr.Markdown(f"## Astabench {CATEGORY_NAME} Leaderboard")
+    gr.Markdown(f"## AstaBench {CATEGORY_NAME} Leaderboard")
 
     with gr.Tabs() as tabs:
         with gr.Tab("Results: Test Set") as test_tab:

ui_components.py

@@ -27,12 +27,12 @@ from config import (
     RESULTS_DATASET,
 )
 from content import (
-    scatter_disclaimer_html,
     format_error,
     format_log,
     format_warning,
     hf_uri_to_web_url,
     hyperlink,
+    SCATTER_DISCLAIMER,
 )
 
 api = HfApi()
@@ -167,6 +167,8 @@ def build_openness_tooltip_content() -> str:
 
 def build_pareto_tooltip_content() -> str:
     """Generates the inner HTML for the Pareto tooltip card with final copy."""
+    trophy_uri = get_svg_as_data_uri("assets/trophy.svg")
+    trophy_icon_html = f'<img src="{trophy_uri}" style="width: 25px; height: 25px; vertical-align: middle;">'
     return f"""
         <h3>On Pareto Frontier</h3>
         <p class="tooltip-description">The Pareto frontier represents the best balance between score and cost.</p>
@@ -175,7 +177,10 @@ def build_pareto_tooltip_content() -> str:
             <li>Offer the lowest cost for a given performance, or</li>
             <li>Deliver the best performance at a given cost.</li>
         </ul>
-        <p class="tooltip-description" style="margin-top: 12px;">These agents are marked with this icon: 🏆</p>
+        <div class="tooltip-description" style="margin-top: 12px; display: flex; align-items: center;">
+            <span>These agents are marked with this icon:</span>
+            <span>{trophy_icon_html}</span>
+        </div>
     """
 
 def build_tooling_tooltip_content() -> str:
@@ -295,6 +300,7 @@ def create_legend_markdown(which_table: str) -> str:
     This is used in the main leaderboard display.
     """
     descriptions_tooltip_content = build_descriptions_tooltip_content(which_table)
+    trophy_uri = get_svg_as_data_uri("assets/trophy.svg")
     legend_markdown = f"""
     <div style="display: flex; flex-wrap: wrap; align-items: flex-start; gap: 10px; font-size: 14px; padding-bottom: 8px;">
             
@@ -304,7 +310,10 @@ def create_legend_markdown(which_table: str) -> str:
                 ⓘ
                 <span class="tooltip-card">{pareto_tooltip_content}</span>
             </span>
-            <div style="margin-top: 8px;"><span>🏆 On frontier</span></div>
+            <div style="margin-top: 8px; display: flex; align-items: center; gap: 6px;">
+            <img src="{trophy_uri}" alt="On frontier" style="width: 25px; height: 25px;">
+            <span>On frontier</span>
+        </div>
         </div>
     
         <div> <!-- Container for the Openness section -->
@@ -424,12 +433,14 @@ def create_leaderboard_display(
     df_view, plots_dict = transformer.view(tag=category_name, use_plotly=True)
     pareto_df = get_pareto_df(df_view)
     # Get the list of agents on the frontier. We'll use this list later.
+    trophy_uri = get_svg_as_data_uri("assets/trophy.svg")
+    trophy_icon_html = f'<img src="{trophy_uri}" alt="On Pareto Frontier" title="On Pareto Frontier" style="width:25px; height:25px;">'
     if not pareto_df.empty and 'id' in pareto_df.columns:
         pareto_agent_names = pareto_df['id'].tolist()
     else:
         pareto_agent_names = []
     df_view['Pareto'] = df_view.apply(
-        lambda row: '🏆' if row['id'] in pareto_agent_names else '',
+        lambda row: trophy_icon_html if row['id'] in pareto_agent_names else '',
         axis=1
     )
     # Create mapping for Openness / tooling
@@ -472,7 +483,7 @@ def create_leaderboard_display(
     for col in df_headers:
         if col == "Logs" or "Cost" in col or "Score" in col:
             df_datatypes.append("markdown")
-        elif col in ["Agent","Icon","LLM Base"]:
+        elif col in ["Agent","Icon","LLM Base", "Pareto"]:
             df_datatypes.append("html")
         else:
             df_datatypes.append("str")
@@ -499,7 +510,7 @@ def create_leaderboard_display(
         value=scatter_plot,
         show_label=False
     )
-    gr.HTML(value=scatter_disclaimer_html, elem_id="scatter-disclaimer")
+    gr.Markdown(value=SCATTER_DISCLAIMER, elem_id="scatter-disclaimer")
     # Put table and key into an accordion
     with gr.Accordion("Show / Hide Table View", open=True, elem_id="leaderboard-accordion"):
         dataframe_component = gr.DataFrame(
@@ -539,8 +550,8 @@ def create_benchmark_details_display(
         gr.Markdown(f"No detailed benchmarks found for the category: {category_name}")
         return
 
+    gr.HTML(f'<h2 style="padding-top: 120px;">{category_name} Detailed Benchmark Results</h2>')
     gr.Markdown("---")
-    gr.Markdown("## Detailed Benchmark Results")
     # 2. Loop through each benchmark and create its UI components
     for benchmark_name in benchmark_names:
         with gr.Row(elem_classes=["benchmark-header"]):
@@ -573,12 +584,14 @@ def create_benchmark_details_display(
         benchmark_table_df = full_df[existing_table_cols].copy()
         pareto_df = get_pareto_df(benchmark_table_df)
         # Get the list of agents on the frontier. We'll use this list later.
+        trophy_uri = get_svg_as_data_uri("assets/trophy.svg")
+        trophy_icon_html = f'<img src="{trophy_uri}" alt="On Pareto Frontier" title="On Pareto Frontier" style="width:25px; height:25px;">'
         if not pareto_df.empty and 'id' in pareto_df.columns:
             pareto_agent_names = pareto_df['id'].tolist()
         else:
             pareto_agent_names = []
         benchmark_table_df['Pareto'] = benchmark_table_df.apply(
-            lambda row: ' 🏆' if row['id'] in pareto_agent_names else '',
+            lambda row: trophy_icon_html if row['id'] in pareto_agent_names else '',
             axis=1
         )
 
@@ -643,7 +656,7 @@ def create_benchmark_details_display(
         for col in df_headers:
             if "Logs" in col or "Cost" in col or "Score" in col:
                 df_datatypes.append("markdown")
-            elif col in ["Agent","Icon", "LLM Base"]:
+            elif col in ["Agent", "Icon", "LLM Base", "Pareto"]:
                 df_datatypes.append("html")
             else:
                 df_datatypes.append("str")
@@ -662,7 +675,7 @@ def create_benchmark_details_display(
             name=benchmark_name
         )
         gr.Plot(value=benchmark_plot, show_label=False)
-        gr.HTML(value=scatter_disclaimer_html, elem_id="scatter-disclaimer")
+        gr.Markdown(value=SCATTER_DISCLAIMER, elem_id="scatter-disclaimer")
         # Put table and key into an accordion
         with gr.Accordion("Show / Hide Table View", open=True, elem_id="leaderboard-accordion"):
             gr.DataFrame(
@@ -755,7 +768,7 @@ def create_sub_navigation_bar(tag_map: dict, category_name: str, validation: boo
     # This container will be our flexbox row.
     full_html = f"""
         <div class="sub-nav-bar-container">
-            <span class="sub-nav-label">Benchmarks:</span>
+            <span class="sub-nav-label">Benchmarks in this category:</span>
             {''.join(html_buttons)}
         </div>
     """