Create test.ipynb

test.ipynb

@@ -0,0 +1,951 @@
+# GAIA Dataset - Final Working Implementation for 300 Questions
+# Based on actual GAIA dataset structure: 2023_all config with test split
+
+import os
+import json
+import pandas as pd
+import numpy as np
+import time
+from typing import Dict, List, Tuple
+import requests
+from datetime import datetime
+import matplotlib.pyplot as plt
+import seaborn as sns
+from tqdm import tqdm
+import warnings
+warnings.filterwarnings('ignore')
+
+# Install required packages
+print("📦 Installing required packages...")
+import subprocess
+import sys
+
+packages = [
+    "langchain-community", "langchain-core", "langchain-google-genai", 
+    "langchain-groq", "langchain-huggingface", "langgraph", "supabase", 
+    "sentence-transformers", "tavily-python", "wikipedia", "arxiv", 
+    "python-dotenv", "gradio", "datasets", "huggingface_hub"
+]
+
+for package in packages:
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "-q", package])
+
+print("✅ All packages installed!")
+
+# Import libraries
+from dotenv import load_dotenv
+from langchain_core.messages import HumanMessage, SystemMessage
+from langchain_groq import ChatGroq
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_community.tools.tavily_search import TavilySearchResults
+from langchain_community.document_loaders import WikipediaLoader, ArxivLoader
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import SupabaseVectorStore
+from langchain_core.tools import tool
+from langchain.tools.retriever import create_retriever_tool
+from langgraph.graph import START, StateGraph, MessagesState
+from langgraph.prebuilt import tools_condition, ToolNode
+from supabase.client import Client, create_client
+from datasets import load_dataset
+from huggingface_hub import login, hf_hub_download
+
+# ================================
+# API KEYS SETUP
+# ================================
+
+print("🔑 Setting up API keys...")
+
+API_KEYS = {
+    'GROQ_API_KEY': "",
+    'TAVILY_API_KEY': "",
+    'SUPABASE_URL': '',
+    'SUPABASE_SERVICE_KEY': '',
+    'GOOGLE_API_KEY': '',
+    'HUGGINGFACEHUB_API_TOKEN': 'h'
+}
+
+for key, value in API_KEYS.items():
+    os.environ[key] = value
+
+print(" API keys configured!")
+
+# HuggingFace login
+try:
+    login(token=API_KEYS['HUGGINGFACEHUB_API_TOKEN'])
+    print("HuggingFace authentication successful!")
+except Exception as e:
+    print(f"HuggingFace login warning: {e}")
+
+# ================================
+# FINAL GAIA DATASET LOADER
+# ================================
+
+class FinalGAIALoader:
+    """Final GAIA dataset loader using correct configuration"""
+    
+    def __init__(self):
+        self.dataset = None
+        self.test_data = []
+        
+    def load_gaia_dataset(self):
+        """Load GAIA dataset using the correct configuration"""
+        print("Loading GAIA dataset with correct configuration...")
+        
+        try:
+            # Based on the dataset code, use "2023_all" config
+            print("Loading with config '2023_all'...")
+            
+            self.dataset = load_dataset(
+                "gaia-benchmark/GAIA", 
+                "2023_all",
+                token=True,
+                trust_remote_code=True  # Important for custom dataset scripts
+            )
+            
+            print(f"Dataset loaded successfully!")
+            print(f"Available splits: {list(self.dataset.keys())}")
+            
+            if 'test' in self.dataset:
+                self.test_data = list(self.dataset['test'])
+                print(f"Loaded {len(self.test_data)} test questions")
+                
+                # Analyze the data structure
+                self._analyze_data_structure()
+                return True
+            else:
+                print("No test split found")
+                return False
+                
+        except Exception as e:
+            print(f"Primary method failed: {e}")
+            
+            # Fallback: Try without trust_remote_code
+            try:
+                print("Trying fallback method...")
+                self.dataset = load_dataset(
+                    "gaia-benchmark/GAIA", 
+                    "2023_all",
+                    token=True
+                )
+                
+                if 'test' in self.dataset:
+                    self.test_data = list(self.dataset['test'])
+                    print(f"Fallback successful! Loaded {len(self.test_data)} questions")
+                    self._analyze_data_structure()
+                    return True
+                    
+            except Exception as e2:
+                print(f"Fallback also failed: {e2}")
+                
+                # Final fallback: Manual download
+                return self._manual_download_fallback()
+    
+    def _manual_download_fallback(self):
+        """Manual download fallback method"""
+        print("Attempting manual download...")
+        
+        try:
+            # Download the metadata file directly
+            metadata_file = hf_hub_download(
+                repo_id="gaia-benchmark/GAIA",
+                filename="2023/test/metadata.jsonl",
+                repo_type="dataset",
+                token=API_KEYS['HUGGINGFACEHUB_API_TOKEN']
+            )
+            
+            # Read the JSONL file
+            test_questions = []
+            with open(metadata_file, 'r', encoding='utf-8') as f:
+                for line in f:
+                    if line.strip():
+                        test_questions.append(json.loads(line))
+            
+            self.test_data = test_questions
+            print(f"Manual download successful! Loaded {len(test_questions)} questions")
+            self._analyze_data_structure()
+            return True
+            
+        except Exception as e:
+            print(f"Manual download failed: {e}")
+            return False
+    
+    def _analyze_data_structure(self):
+        """Analyze the structure of loaded GAIA data"""
+        if not self.test_data:
+            return
+        
+        print(f"\n GAIA Data Analysis:")
+        print(f"Total questions: {len(self.test_data)}")
+        
+        # Show sample item structure
+        if self.test_data:
+            sample = self.test_data[0]
+            print(f"Sample item keys: {list(sample.keys())}")
+            
+            # Level distribution
+            levels = [item.get('Level') for item in self.test_data]
+            level_counts = pd.Series(levels).value_counts().sort_index()
+            print(f"\nLevel distribution:")
+            for level, count in level_counts.items():
+                print(f"  Level {level}: {count} questions")
+            
+            # Sample questions
+            print(f"\nSample questions by level:")
+            for level in sorted(level_counts.index):
+                level_questions = [item for item in self.test_data if item.get('Level') == level]
+                if level_questions:
+                    sample_q = level_questions[0]['Question'][:100]
+                    print(f"  Level {level}: {sample_q}...")
+    
+    def filter_300_questions(self, level1_ratio=0.6, level2_ratio=0.25, level3_ratio=0.15):
+        """Filter exactly 300 questions with specified ratios"""
+        
+        if not self.test_data:
+            print("No test data available")
+            return []
+        
+        total_questions = 300
+        level1_count = int(total_questions * level1_ratio)  # 180
+        level2_count = int(total_questions * level2_ratio)  # 75
+        level3_count = total_questions - level1_count - level2_count  # 45
+        
+        print(f"Filtering {total_questions} questions:")
+        print(f"  Level 1: {level1_count} questions ({level1_ratio*100:.0f}%)")
+        print(f"  Level 2: {level2_count} questions ({level2_ratio*100:.0f}%)")
+        print(f"  Level 3: {level3_count} questions ({level3_ratio*100:.0f}%)")
+        
+        # Group by level
+        level_groups = {1: [], 2: [], 3: []}
+        for item in self.test_data:
+            level = item.get('Level')
+            if level in level_groups:
+                level_groups[level].append(item)
+        
+        # Check availability
+        print(f"\nAvailable questions by level:")
+        for level in [1, 2, 3]:
+            available = len(level_groups[level])
+            print(f"  Level {level}: {available} available")
+        
+        # Sample questions
+        filtered_questions = []
+        np.random.seed(42)  # For reproducibility
+        
+        for level, target_count in [(1, level1_count), (2, level2_count), (3, level3_count)]:
+            available = len(level_groups[level])
+            
+            if available >= target_count:
+                # Random sample
+                sampled_indices = np.random.choice(available, size=target_count, replace=False)
+                sampled = [level_groups[level][i] for i in sampled_indices]
+                filtered_questions.extend(sampled)
+                print(f"Level {level}: Selected {target_count} from {available}")
+            else:
+                # Take all available
+                filtered_questions.extend(level_groups[level])
+                print(f"Level {level}: Only {available} available (needed {target_count})")
+        
+        print(f"\n📊 Total filtered: {len(filtered_questions)} questions")
+        
+        # Verify final distribution
+        final_levels = [q['Level'] for q in filtered_questions]
+        final_dist = pd.Series(final_levels).value_counts().sort_index()
+        print(f"Final distribution:")
+        for level, count in final_dist.items():
+            percentage = (count / len(filtered_questions)) * 100
+            print(f"  Level {level}: {count} questions ({percentage:.1f}%)")
+        
+        return filtered_questions
+    
+    def create_dataframe(self, questions):
+        """Create DataFrame from filtered questions"""
+        if not questions:
+            return pd.DataFrame()
+        
+        data = []
+        for i, item in enumerate(questions):
+            data.append({
+                'id': i,
+                'task_id': item.get('task_id', f'gaia_{i}'),
+                'question': item.get('Question', ''),
+                'level': item.get('Level', 1),
+                'final_answer': item.get('Final answer', ''),
+                'file_name': item.get('file_name', ''),
+                'annotator_metadata': item.get('Annotator Metadata', {})
+            })
+        
+        df = pd.DataFrame(data)
+        print(f"📋 Created DataFrame with {len(df)} questions")
+        return df
+
+# ================================
+# ENHANCED SYSTEM PROMPT
+# ================================
+
+SYSTEM_PROMPT = """
+You are an expert research assistant designed to answer complex, multi-step questions from the GAIA benchmark.
+You have access to powerful tools for web search, Wikipedia, arXiv research, and mathematical calculations.
+
+Key Instructions:
+1. Read the question carefully and identify what information you need
+2. Use tools strategically - web search for current info, Wikipedia for general knowledge, arXiv for research
+3. For mathematical questions, use the calculation tools
+4. Think step-by-step and show your reasoning
+5. Be precise and accurate in your final answer
+
+Answer Format:
+- Provide clear reasoning and methodology
+- Show any calculations or research steps
+- End with: FINAL ANSWER: [exact answer]
+- Keep the final answer concise and precise
+- Match the format requested (number, name, list, etc.)
+
+Examples:
+Question: What is the population of Tokyo according to the latest census?
+I need to search for the most recent population data for Tokyo.
+[uses web_search("Tokyo population latest census")]
+Based on the search results, the latest census shows...
+FINAL ANSWER: 13960000
+
+Question: What is 157 * 238?
+I need to calculate 157 multiplied by 238.
+[uses multiply(157, 238)]
+FINAL ANSWER: 37366
+"""
+
+# ================================
+# ENHANCED TOOLS
+# ================================
+
+@tool
+def multiply(a: float, b: float) -> float:
+    """Multiply two numbers."""
+    return a * b
+
+@tool
+def add(a: float, b: float) -> float:
+    """Add two numbers."""
+    return a + b
+
+@tool
+def subtract(a: float, b: float) -> float:
+    """Subtract two numbers."""
+    return a - b
+
+@tool
+def divide(a: float, b: float) -> float:
+    """Divide two numbers."""
+    if abs(b) < 1e-10:
+        raise ValueError("Cannot divide by zero.")
+    return a / b
+
+@tool
+def power(a: float, b: float) -> float:
+    """Calculate a raised to the power of b."""
+    return a ** b
+
+@tool
+def square_root(a: float) -> float:
+    """Calculate the square root of a number."""
+    if a < 0:
+        raise ValueError("Cannot calculate square root of negative number.")
+    return a ** 0.5
+
+@tool
+def modulo(a: float, b: float) -> float:
+    """Calculate a modulo b (remainder after division)."""
+    return a % b
+
+@tool
+def absolute_value(a: float) -> float:
+    """Calculate the absolute value of a number."""
+    return abs(a)
+
+@tool
+def wiki_search(query: str) -> str:
+    """Search Wikipedia for comprehensive information on a topic."""
+    try:
+        search_docs = WikipediaLoader(query=query, load_max_docs=3).load()
+        if not search_docs:
+            return f"No Wikipedia results found for: {query}"
+        
+        # Return the most relevant result with more content
+        result = search_docs[0]
+        title = result.metadata.get('title', 'Unknown')
+        content = result.page_content[:1500]
+        
+        return f"Wikipedia article '{title}':\n{content}..."
+    except Exception as e:
+        return f"Wikipedia search failed: {str(e)}"
+
+@tool
+def web_search(query: str) -> str:
+    """Search the web for current, factual information."""
+    try:
+        search_tool = TavilySearchResults(max_results=4)
+        results = search_tool.invoke(query)
+        
+        if not results:
+            return f"No web results found for: {query}"
+        
+        formatted_results = f"Web search results for '{query}':\n\n"
+        for i, result in enumerate(results, 1):
+            url = result.get('url', 'Unknown URL')
+            content = result.get('content', 'No content')[:800]
+            formatted_results += f"{i}. Source: {url}\n{content}...\n\n"
+        
+        return formatted_results
+    except Exception as e:
+        return f"Web search failed: {str(e)}"
+
+@tool
+def arxiv_search(query: str) -> str:
+    """Search arXiv for academic papers and research."""
+    try:
+        search_docs = ArxivLoader(query=query, load_max_docs=3).load()
+        if not search_docs:
+            return f"No arXiv papers found for: {query}"
+        
+        formatted_results = f"arXiv search results for '{query}':\n\n"
+        for i, doc in enumerate(search_docs, 1):
+            title = doc.metadata.get('Title', 'Unknown Title')
+            authors = doc.metadata.get('Authors', 'Unknown Authors')
+            content = doc.page_content[:1000]
+            formatted_results += f"{i}. Title: {title}\nAuthors: {authors}\nAbstract: {content}...\n\n"
+        
+        return formatted_results
+    except Exception as e:
+        return f"arXiv search failed: {str(e)}"
+
+# All tools
+tools = [
+    multiply, add, subtract, divide, power, square_root, modulo, absolute_value,
+    wiki_search, web_search, arxiv_search
+]
+
+# ================================
+# ENHANCED GAIA AGENT
+# ================================
+
+class EnhancedGAIAAgent:
+    """Enhanced GAIA agent optimized for benchmark performance"""
+    
+    def __init__(self, provider="groq", model="llama3-70b-8192"):
+        self.provider = provider
+        self.model = model
+        self.graph = self._build_graph()
+        
+    def _build_graph(self):
+        """Build optimized agent graph"""
+        print(f"Building enhanced agent: {self.provider} {self.model}")
+        
+        if self.provider == "groq":
+            llm = ChatGroq(
+                model=self.model, 
+                temperature=0,
+                max_retries=3,
+                timeout=120
+            )
+        elif self.provider == "google":
+            llm = ChatGoogleGenerativeAI(
+                model="gemini-pro", 
+                temperature=0,
+                max_retries=3
+            )
+        else:
+            raise ValueError("Choose 'groq' or 'google'")
+        
+        llm_with_tools = llm.bind_tools(tools)
+        
+        def assistant_node(state: MessagesState):
+            """Enhanced assistant node with better error handling"""
+            try:
+                messages = state["messages"]
+                
+                # Add system message if not present
+                if not any(isinstance(msg, SystemMessage) for msg in messages):
+                    messages = [SystemMessage(content=SYSTEM_PROMPT)] + messages
+                
+                # Invoke LLM
+                response = llm_with_tools.invoke(messages)
+                return {"messages": [response]}
+                
+            except Exception as e:
+                error_msg = f"Assistant error: {str(e)}"
+                print(f" {error_msg}")
+                return {"messages": [HumanMessage(content=f"Error: {error_msg}")]}
+        
+        # Build graph
+        builder = StateGraph(MessagesState)
+        builder.add_node("assistant", assistant_node)
+        builder.add_node("tools", ToolNode(tools))
+        
+        builder.add_edge(START, "assistant")
+        builder.add_conditional_edges("assistant", tools_condition)
+        builder.add_edge("tools", "assistant")
+        
+        return builder.compile()
+    
+    def process_question(self, question: str, question_id: str = None) -> Dict:
+        """Process a single GAIA question with enhanced handling"""
+        start_time = time.time()
+        
+        try:
+            # Create message and invoke graph
+            messages = [HumanMessage(content=question)]
+            result = self.graph.invoke({"messages": messages})
+            
+            # Extract final response
+            final_response = result["messages"][-1].content
+            
+            # Extract final answer more robustly
+            answer = self._extract_final_answer(final_response)
+            
+            processing_time = time.time() - start_time
+            
+            return {
+                'question_id': question_id,
+                'question': question,
+                'full_response': final_response,
+                'final_answer': answer,
+                'processing_time': processing_time,
+                'status': 'success',
+                'error': None
+            }
+            
+        except Exception as e:
+            processing_time = time.time() - start_time
+            error_msg = str(e)
+            
+            return {
+                'question_id': question_id,
+                'question': question,
+                'full_response': f"Error: {error_msg}",
+                'final_answer': f"ERROR: {error_msg}",
+                'processing_time': processing_time,
+                'status': 'error',
+                'error': error_msg
+            }
+    
+    def _extract_final_answer(self, response: str) -> str:
+        """Extract final answer more robustly"""
+        if "FINAL ANSWER:" in response:
+            answer = response.split("FINAL ANSWER:")[-1].strip()
+            # Clean up the answer
+            answer = answer.split('\n')[0].strip()  # Take first line only
+            return answer
+        else:
+            # Fallback: take last substantial line
+            lines = [line.strip() for line in response.split('\n') if line.strip()]
+            return lines[-1] if lines else response.strip()
+    
+    def evaluate_300_questions(self, questions_df: pd.DataFrame) -> pd.DataFrame:
+        """Evaluate 300 GAIA questions with comprehensive tracking"""
+        print(f"Starting GAIA evaluation: {len(questions_df)} questions")
+        print(f" Estimated time: {len(questions_df) * 15 / 60:.0f}-{len(questions_df) * 25 / 60:.0f} minutes")
+        
+        results = []
+        
+        # Progress tracking
+        with tqdm(total=len(questions_df), desc="GAIA Evaluation Progress") as pbar:
+            for idx, row in questions_df.iterrows():
+                question_id = row.get('task_id', f'gaia_{idx}')
+                question = row['question']
+                level = row['level']
+                expected = row.get('final_answer', '')
+                
+                print(f"\n Question {idx+1}/{len(questions_df)} - Level {level}")
+                print(f"ID: {question_id}")
+                print(f"Q: {question[:120]}...")
+                
+                # Process question
+                result = self.process_question(question, question_id)
+                
+                # Add metadata
+                result.update({
+                    'level': level,
+                    'expected_answer': expected,
+                    'question_index': idx
+                })
+                
+                results.append(result)
+                
+                # Show result
+                status_emoji = "" if result['status'] == 'success' else ""
+                print(f"A: {result['final_answer'][:120]}...")
+                print(f"{status_emoji} Time: {result['processing_time']:.2f}s")
+                
+                # Update progress bar
+                pbar.update(1)
+                pbar.set_postfix({
+                    'Success Rate': f"{len([r for r in results if r['status'] == 'success'])/len(results):.1%}",
+                    'Avg Time': f"{np.mean([r['processing_time'] for r in results]):.1f}s"
+                })
+                
+                # Save progress every 25 questions
+                if (idx + 1) % 25 == 0:
+                    temp_df = pd.DataFrame(results)
+                    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+                    temp_df.to_csv(f'gaia_progress_{timestamp}.csv', index=False)
+                    print(f"💾 Progress saved at question {idx+1}")
+                
+                # Rate limiting
+                time.sleep(1.5)
+        
+        results_df = pd.DataFrame(results)
+        print(f"\n GAIA evaluation completed!")
+        return results_df
+
+# ================================
+# ANALYSIS FUNCTIONS
+# ================================
+
+def analyze_gaia_results(results_df: pd.DataFrame) -> Dict:
+    """Comprehensive analysis of GAIA results"""
+    
+    if results_df.empty:
+        return {}
+    
+    # Basic metrics
+    total = len(results_df)
+    successful = len(results_df[results_df['status'] == 'success'])
+    success_rate = successful / total
+    error_rate = 1 - success_rate
+    
+    metrics = {
+        'total_questions': total,
+        'successful_runs': successful,
+        'success_rate': success_rate,
+        'error_rate': error_rate,
+        'avg_processing_time': results_df['processing_time'].mean(),
+        'median_processing_time': results_df['processing_time'].median(),
+        'total_processing_time': results_df['processing_time'].sum()
+    }
+    
+    # Level-wise analysis
+    level_metrics = {}
+    for level in sorted(results_df['level'].unique()):
+        level_data = results_df[results_df['level'] == level]
+        level_success = len(level_data[level_data['status'] == 'success'])
+        
+        level_metrics[f'level_{level}'] = {
+            'count': len(level_data),
+            'success_count': level_success,
+            'success_rate': level_success / len(level_data),
+            'error_rate': 1 - (level_success / len(level_data)),
+            'avg_time': level_data['processing_time'].mean()
+        }
+    
+    metrics['by_level'] = level_metrics
+    
+    # Print comprehensive analysis
+    print(f"\n GAIA Benchmark Results Analysis")
+    print(f"=" * 60)
+    print(f" Total Questions: {total}")
+    print(f" Successful: {successful}")
+    print(f" Overall Success Rate: {success_rate:.2%}")
+    print(f" Error Rate: {error_rate:.2%}")
+    print(f" Average Time: {metrics['avg_processing_time']:.2f}s")
+    print(f" Total Time: {metrics['total_processing_time']/60:.1f} minutes")
+    
+    print(f"\n Performance by Difficulty Level:")
+    for level_key, level_data in level_metrics.items():
+        level_num = level_key.split('_')[1]
+        print(f"Level {level_num}:")
+        print(f"   Questions: {level_data['count']}")
+        print(f"   Success: {level_data['success_count']}/{level_data['count']} ({level_data['success_rate']:.1%})")
+        print(f"   Avg Time: {level_data['avg_time']:.1f}s")
+    
+    return metrics
+
+def save_gaia_results(results_df: pd.DataFrame, metrics: Dict):
+    """Save comprehensive GAIA results"""
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    
+    # Save detailed results
+    results_filename = f'gaia_300_final_results_{timestamp}.csv'
+    results_df.to_csv(results_filename, index=False)
+    print(f" Detailed results: {results_filename}")
+    
+    # Save metrics
+    metrics_filename = f'gaia_300_metrics_{timestamp}.json'
+    with open(metrics_filename, 'w') as f:
+        json.dump(metrics, f, indent=2, default=str)
+    print(f" Metrics: {metrics_filename}")
+    
+    # Create summary report
+    report = f"""
+# GAIA Benchmark - 300 Questions Evaluation Report
+Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
+
+## Overall Performance
+- Total Questions: {metrics['total_questions']}
+- Success Rate: {metrics['success_rate']:.2%}
+- Average Time: {metrics['avg_processing_time']:.2f}s
+- Total Time: {metrics['total_processing_time']/60:.1f} minutes
+
+## Level Performance
+"""
+    
+    for level_key, level_data in metrics['by_level'].items():
+        level_num = level_key.split('_')[1]
+        report += f"""
+### Level {level_num}
+- Questions: {level_data['count']}
+- Success Rate: {level_data['success_rate']:.2%}
+- Average Time: {level_data['avg_time']:.2f}s
+"""
+    
+    report_filename = f'gaia_300_report_{timestamp}.txt'
+    with open(report_filename, 'w') as f:
+        f.write(report)
+    print(f"💾 Report: {report_filename}")
+    
+    return timestamp
+
+# ================================
+# MAIN EXECUTION
+# ================================
+
+def main_gaia_300():
+    """Main function for 300 GAIA questions evaluation"""
+    print(" GAIA Benchmark - 300 Questions Evaluation")
+    print("=" * 60)
+    
+    try:
+        # Step 1: Load GAIA dataset
+        print("\n Step 1: Loading GAIA Dataset")
+        loader = FinalGAIALoader()
+        
+        if not loader.load_gaia_dataset():
+            print(" Failed to load GAIA dataset")
+            return None, None
+        
+        # Step 2: Filter 300 questions
+        print(f"\n Step 2: Filtering 300 Questions")
+        filtered_questions = loader.filter_300_questions()
+        
+        if len(filtered_questions) < 250:  # Allow some flexibility
+            print(f" Only {len(filtered_questions)} questions available")
+            proceed = input("Proceed with available questions? (y/n): ")
+            if proceed.lower() != 'y':
+                return None, None
+        
+        # Create DataFrame
+        questions_df = loader.create_dataframe(filtered_questions)
+        
+        # Step 3: Confirm evaluation
+        print(f"\n Step 3: Ready for Evaluation")
+        print(f" Questions to evaluate: {len(questions_df)}")
+        
+        level_dist = questions_df['level'].value_counts().sort_index()
+        for level, count in level_dist.items():
+            percentage = (count / len(questions_df)) * 100
+            print(f"  Level {level}: {count} questions ({percentage:.1f}%)")
+        
+        estimated_time = len(questions_df) * 18 / 60  # 18 seconds average per question
+        print(f" Estimated time: {estimated_time:.0f} minutes")
+        
+        # Confirm with user
+        proceed = input(f"\nProceed with {len(questions_df)} questions evaluation? (y/n): ")
+        if proceed.lower() != 'y':
+            print("Evaluation cancelled.")
+            return None, None
+        
+        # Step 4: Initialize enhanced agent
+        print(f"\n Step 4: Initializing Enhanced GAIA Agent")
+        agent = EnhancedGAIAAgent(provider="groq", model="llama3-70b-8192")
+        
+        # Step 5: Run evaluation
+        print(f"\n Step 5: Running GAIA Evaluation")
+        results_df = agent.evaluate_300_questions(questions_df)
+        
+        # Step 6: Analyze results
+        print(f"\n Step 6: Analyzing Results")
+        metrics = analyze_gaia_results(results_df)
+        
+        # Step 7: Save results
+        print(f"\n Step 7: Saving Results")
+        timestamp = save_gaia_results(results_df, metrics)
+        
+        # Final summary
+        print(f"\n GAIA Evaluation Completed Successfully!")
+        print(f" Success Rate: {metrics['success_rate']:.2%}")
+        print(f" Total Time: {metrics['total_processing_time']/60:.1f} minutes")
+        print(f" Files saved with timestamp: {timestamp}")
+        
+        # Performance insights
+        best_level = max(metrics['by_level'].items(), key=lambda x: x[1]['success_rate'])
+        worst_level = min(metrics['by_level'].items(), key=lambda x: x[1]['success_rate'])
+        
+        print(f"\n Key Insights:")
+        print(f" Best Performance: Level {best_level[0].split('_')[1]} ({best_level[1]['success_rate']:.1%})")
+        print(f" Most Challenge: Level {worst_level[0].split('_')[1]} ({worst_level[1]['success_rate']:.1%})")
+        
+        if metrics['success_rate'] >= 0.85:
+            print(" Excellent performance on GAIA benchmark!")
+        elif metrics['success_rate'] >= 0.70:
+            print(" Good performance on GAIA benchmark!")
+        else:
+            print(" Room for improvement on GAIA benchmark.")
+        
+        return results_df, metrics
+        
+    except Exception as e:
+        print(f" Error in main execution: {e}")
+        import traceback
+        traceback.print_exc()
+        return None, None
+
+def test_gaia_single():
+    """Test with a single GAIA question"""
+    print(" Testing Single GAIA Question")
+    print("=" * 40)
+    
+    try:
+        # Load dataset
+        loader = FinalGAIALoader()
+        if not loader.load_gaia_dataset():
+            print(" Failed to load dataset")
+            return None
+        
+        # Get one question of each level
+        sample_questions = []
+        for level in [1, 2, 3]:
+            level_questions = [q for q in loader.test_data if q.get('Level') == level]
+            if level_questions:
+                sample_questions.append(level_questions[0])
+        
+        if not sample_questions:
+            print(" No sample questions found")
+            return None
+        
+        # Test with agent
+        agent = EnhancedGAIAAgent()
+        
+        for i, question_data in enumerate(sample_questions):
+            question = question_data['Question']
+            level = question_data['Level']
+            
+            print(f"\n Test {i+1} - Level {level}")
+            print(f"Q: {question[:150]}...")
+            
+            result = agent.process_question(question, f"test_{level}")
+            
+            print(f"A: {result['final_answer']}")
+            print(f" Time: {result['processing_time']:.2f}s")
+            print(f"Status: {result['status']}")
+        
+        return True
+        
+    except Exception as e:
+        print(f" Test failed: {e}")
+        return None
+
+def quick_gaia_test():
+    """Quick test with 10 questions"""
+    print(" Quick GAIA Test - 10 Questions")
+    print("=" * 40)
+    
+    try:
+        # Load and filter
+        loader = FinalGAIALoader()
+        if not loader.load_gaia_dataset():
+            return None, None
+        
+        # Get 10 questions (3/4/3 distribution)
+        level_groups = {1: [], 2: [], 3: []}
+        for item in loader.test_data:
+            level = item.get('Level')
+            if level in level_groups:
+                level_groups[level].append(item)
+        
+        quick_questions = []
+        quick_questions.extend(level_groups[1][:3])  # 3 Level 1
+        quick_questions.extend(level_groups[2][:4])  # 4 Level 2  
+        quick_questions.extend(level_groups[3][:3])  # 3 Level 3
+        
+        questions_df = loader.create_dataframe(quick_questions)
+        
+        # Run evaluation
+        agent = EnhancedGAIAAgent()
+        results_df = agent.evaluate_300_questions(questions_df)
+        
+        # Quick analysis
+        metrics = analyze_gaia_results(results_df)
+        
+        return results_df, metrics
+        
+    except Exception as e:
+        print(f" Quick test failed: {e}")
+        return None, None
+
+# ================================
+# EXECUTION
+# ================================
+
+if __name__ == "__main__":
+    print(" GAIA Benchmark Evaluation System")
+    print("Using correct dataset configuration: 2023_all")
+    print("=" * 60)
+    
+    # Test API connectivity
+    print("\n Testing API Connectivity...")
+    try:
+        # Test Groq
+        test_llm = ChatGroq(model="llama3-8b-8192", temperature=0)
+        test_response = test_llm.invoke([HumanMessage(content="Hello")])
+        print(" Groq API working")
+        
+        # Test Tavily
+        test_search = TavilySearchResults(max_results=1)
+        test_results = test_search.invoke("test")
+        print(" Tavily API working")
+        
+    except Exception as e:
+        print(f" API test warning: {e}")
+    
+    # Main menu
+    print(f"\n🎮 Choose evaluation option:")
+    print("1. Test single GAIA questions (5 minutes)")
+    print("2. Quick test with 10 questions (15 minutes)")
+    print("3. Full 300 questions evaluation (75-90 minutes)")
+    print("4. Auto-run full evaluation")
+    
+    choice = input("\nEnter choice (1-4): ").strip()
+    
+    if choice == "1":
+        print(" Running single question tests...")
+        test_gaia_single()
+        
+    elif choice == "2":
+        print(" Running quick 10-question test...")
+        results_df, metrics = quick_gaia_test()
+        if results_df is not None:
+            print(f"\n Quick Test Results:")
+            print(f" Success Rate: {metrics['success_rate']:.2%}")
+            print(f"⏱️ Average Time: {metrics['avg_processing_time']:.1f}s")
+        
+    elif choice == "3" or choice == "4":
+        print(" Starting full GAIA 300 questions evaluation...")
+        results_df, metrics = main_gaia_300()
+        
+        if results_df is not None:
+            print(f"\n Final GAIA Benchmark Results:")
+            print(f" Overall Success Rate: {metrics['success_rate']:.2%}")
+            print(f" Questions Completed: {len(results_df)}")
+            print(f"⏱️ Total Evaluation Time: {metrics['total_processing_time']/60:.1f} minutes")
+            
+            # Level breakdown
+            for level_key, level_data in metrics['by_level'].items():
+                level_num = level_key.split('_')[1]
+                print(f" Level {level_num}: {level_data['success_rate']:.1%} success rate")
+                
+        else:
+            print(" Evaluation failed")
+            
+    else:
+        print("Invalid choice. Running single question test...")
+        test_gaia_single()
+
+print("\n���� GAIA Evaluation System Complete!")