Upload 8 files

app.py

@@ -1,12 +1,13 @@
+import re
 import os
 import numpy as np
 import matplotlib.pyplot as plt
 import gradio as gr
 from typing import Dict, List, Any, Optional, Tuple
-import cv2 
-from PIL import Image 
-import tempfile 
-import uuid 
+import cv2
+from PIL import Image
+import tempfile
+import uuid
 import spaces
 
 from detection_model import DetectionModel
@@ -15,10 +16,11 @@ from evaluation_metrics import EvaluationMetrics
 from style import Style
 from image_processor import ImageProcessor
 from video_processor import VideoProcessor
+from llm_enhancer import LLMEnhancer
 
-# Initialize Processors 
-image_processor = ImageProcessor()
-video_processor = VideoProcessor(image_processor) 
+# Initialize Processors with LLM support
+image_processor = ImageProcessor(use_llm=True, llm_model_path="meta-llama/Llama-3.2-3B-Instruct")
+video_processor = VideoProcessor(image_processor)
 
 # Helper Function
 def get_all_classes():
@@ -56,10 +58,15 @@ def get_all_classes():
     return sorted(default_classes.items())
 
 @spaces.GPU
-def handle_image_upload(image, model_name, confidence_threshold, filter_classes=None):
+def handle_image_upload(image, model_name, confidence_threshold, filter_classes=None, use_llm=True):
     """Processes a single uploaded image."""
-    print(f"Processing image with model: {model_name}, confidence: {confidence_threshold}")
+    print(f"Processing image with model: {model_name}, confidence: {confidence_threshold}, use_llm: {use_llm}")
     try:
+        image_processor.use_llm = use_llm
+        if hasattr(image_processor, 'scene_analyzer'):
+            image_processor.scene_analyzer.use_llm = use_llm
+            print(f"Updated existing scene_analyzer use_llm setting to: {use_llm}")
+
         class_ids_to_filter = None
         if filter_classes:
             class_ids_to_filter = []
@@ -118,8 +125,127 @@ def handle_image_upload(image, model_name, confidence_threshold, filter_classes=
         scene_desc = scene_analysis.get("description", "Scene analysis requires detected objects.")
         # Ensure scene_desc is a string before adding HTML
         if not isinstance(scene_desc, str):
-             scene_desc = str(scene_desc)
-        scene_desc_html = f"<div style='padding:10px; font-family:Arial, sans-serif; line-height:1.7;'>{scene_desc}</div>"
+            scene_desc = str(scene_desc)
+
+        def clean_description(desc):
+            if not desc:
+                return ""
+
+            # 先過濾問答格式
+            if "Questions:" in desc:
+                desc = desc.split("Questions:")[0].strip()
+            if "Answers:" in desc:
+                desc = desc.split("Answers:")[0].strip()
+
+            # 然後按行過濾代碼和其他非敘述內容
+            lines = desc.split('\n')
+            clean_lines = []
+            skip_block = False
+
+            for line in lines:
+                # 檢測問題格式
+                if re.match(r'^\d+\.\s+(What|How|Why|When|Where|Who|The)', line):
+                    continue
+
+                # 檢查需要跳過的行
+                if line.strip().startswith(':param') or line.strip().startswith('"""'):
+                    continue
+                if line.strip().startswith("Exercise") or "class SceneDescriptionSystem" in line:
+                    skip_block = True
+                    continue
+                if ('def generate_scene_description' in line or
+                    'def enhance_scene_descriptions' in line or
+                    'def __init__' in line):
+                    skip_block = True
+                    continue
+                if line.strip().startswith('#TEST'):
+                    skip_block = True
+                    continue
+
+                if skip_block and line.strip() == "":
+                    skip_block = False
+
+                # 如果不需要跳過
+                if not skip_block:
+                    clean_lines.append(line)
+
+            cleaned_text = '\n'.join(clean_lines)
+
+            # 如果清理後為空，返回原始描述的第一段作為保險
+            if not cleaned_text.strip():
+                paragraphs = [p.strip() for p in desc.split('\n\n') if p.strip()]
+                if paragraphs:
+                    return paragraphs[0]
+                return desc
+
+            return cleaned_text
+
+        # 獲取和處理場景描述
+        scene_analysis = stats.get("scene_analysis", {})
+        print("Processing scene_analysis:", scene_analysis.keys())
+
+        # 獲取原始描述
+        scene_desc = scene_analysis.get("description", "Scene analysis requires detected objects.")
+        if not isinstance(scene_desc, str):
+            scene_desc = str(scene_desc)
+
+        print(f"Original scene description (first 50 chars): {scene_desc[:50]}...")
+
+        # 確保使用的是有效的描述
+        clean_scene_desc = clean_description(scene_desc)
+        print(f"Cleaned scene description (first 50 chars): {clean_scene_desc[:50]}...")
+
+        # 即使清理後為空也確保顯示原始內容
+        if not clean_scene_desc.strip():
+            clean_scene_desc = scene_desc
+
+        # 創建原始描述的HTML
+        scene_desc_html = f"<div>{clean_scene_desc}</div>"
+
+        # 獲取LLM增強描述並且確保設置默認值為空字符串而非 None，不然會有None type Error
+        enhanced_description = scene_analysis.get("enhanced_description", "")
+        if enhanced_description is None:
+            enhanced_description = ""
+
+        if not enhanced_description or not enhanced_description.strip():
+            print("WARNING: LLM enhanced description is empty!")
+
+        # 準備徽章和描述標籤
+        llm_badge = ""
+        description_to_show = ""
+
+        if use_llm and enhanced_description:
+            llm_badge = '<span style="display:inline-block; margin-left:8px; padding:3px 10px; border-radius:12px; background: linear-gradient(90deg, #38b2ac, #4299e1); color:white; font-size:0.7rem; font-weight:bold; box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); border: 1px solid rgba(255, 255, 255, 0.2);">LLM Enhanced</span>'
+            description_to_show = enhanced_description
+            # 在 Original Scene Analysis 折疊區顯示原始的描述
+        else:
+            llm_badge = '<span style="display:inline-block; margin-left:8px; padding:3px 10px; border-radius:12px; background-color:#718096; color:white; font-size:0.7rem; font-weight:bold;">Basic</span>'
+            description_to_show = clean_scene_desc
+            # 不使用 LLM 時，折疊區不顯示內容
+
+        # 使用LLM敘述時會有徽章標籤在標題上
+        scene_description_html = f'''
+        <div>
+            <div class="section-heading" style="font-size:1.2rem; margin-top:15px;">Scene Description {llm_badge}
+                <span style="font-size:0.8rem; color:#666; font-weight:normal; display:block; margin-top:2px;">
+                    {('(Enhanced by AI language model)' if use_llm and enhanced_description else '(Based on object detection)')}
+                </span>
+            </div>
+            <div style="padding:15px; background-color:#ffffff; border-radius:8px; border:1px solid #e2e8f0; margin-bottom:20px; box-shadow:0 1px 3px rgba(0,0,0,0.05);">
+                {description_to_show}
+            </div>
+        </div>
+        '''
+
+        # 原始描述只在使用 LLM 且有增強描述時在折疊區顯示
+        original_desc_visibility = "block" if use_llm and enhanced_description else "none"
+        original_desc_html = f'''
+        <div id="original_scene_analysis_accordion" style="display: {original_desc_visibility};">
+            <div style="padding:15px; background-color:#f0f0f0; border-radius:8px; border:1px solid #e2e8f0;">
+                {clean_scene_desc}
+            </div>
+        </div>
+        '''
 
         # Prepare activities list
         activities_list = scene_analysis.get("possible_activities", [])
@@ -138,8 +264,15 @@ def handle_image_upload(image, model_name, confidence_threshold, filter_classes=
         zones = scene_analysis.get("functional_zones", {})
         lighting = scene_analysis.get("lighting_conditions", {"time_of_day": "unknown", "confidence": 0})
 
+        # 如果描述為空，記錄警告
+        if not clean_scene_desc.strip():
+            print("WARNING: Scene description is empty after cleaning!")
+        if not enhanced_description.strip():
+            print("WARNING: LLM enhanced description is empty!")
+
         return (result_image, result_text, formatted_stats, plot_figure,
-                scene_desc_html, activities_list_data, safety_data, zones, lighting)
+            scene_description_html, original_desc_html, 
+            activities_list_data, safety_data, zones, lighting)
 
     except Exception as e:
         print(f"Error in handle_image_upload: {e}")
@@ -149,8 +282,8 @@ def handle_image_upload(image, model_name, confidence_threshold, filter_classes=
         ax.text(0.5, 0.5, "Processing Error", color="red", ha="center", va="center")
         ax.axis('off')
         # Ensure return structure matches outputs even on error
-        return (None, error_msg, {}, fig, f"<div>Error: {str(e)}</div>",
-                [["Error"]], [["Error"]], {}, {"time_of_day": "error", "confidence": 0})
+        return (None, error_msg, {}, fig, f"<div>Error: {str(e)}</div>", "Error",
+            [["Error"]], [["Error"]], {}, {"time_of_day": "error", "confidence": 0})
 
 def download_video_from_url(video_url, max_duration_minutes=10):
     """
@@ -273,7 +406,7 @@ def handle_video_upload(video_input, video_url, input_type, model_name, confiden
         return None, error_html, {"error": str(e)}
 
 
-# Create Gradio Interface 
+# Create Gradio Interface
 def create_interface():
     """Creates the Gradio interface with Tabs."""
     css = Style.get_css()
@@ -283,7 +416,7 @@ def create_interface():
 
     with gr.Blocks(css=css, theme=gr.themes.Soft(primary_hue="teal", secondary_hue="blue")) as demo:
 
-        # Header 
+        # Header
         with gr.Group(elem_classes="app-header"):
               gr.HTML("""
                     <div style="text-align: center; width: 100%; padding: 2rem 0 3rem 0; background: linear-gradient(135deg, #f0f9ff, #e1f5fe);">
@@ -303,10 +436,10 @@ def create_interface():
                     </div>
                 """)
 
-        # Main Content with Tabs 
+        # Main Content with Tabs
         with gr.Tabs(elem_classes="tabs"):
 
-            # Tab 1: Image Processing 
+            # Tab 1: Image Processing
             with gr.Tab("Image Processing"):
                 current_image_model = gr.State("yolov8m.pt") # State for image model selection
                 with gr.Row(equal_height=False): # Allow columns to have different heights
@@ -331,6 +464,13 @@ def create_interface():
                                     label="Confidence Threshold",
                                     info="Minimum confidence for displaying a detected object"
                                 )
+
+                                use_llm = gr.Checkbox(
+                                    label="Use LLM for enhanced scene descriptions",
+                                    value=True,
+                                    info="Provides more detailed and natural language descriptions (may increase processing time)"
+                                )
+
                                 with gr.Accordion("Filter Classes", open=False):
                                      gr.HTML('<div class="section-heading" style="font-size: 1rem;">Common Categories</div>')
                                      with gr.Row():
@@ -350,11 +490,12 @@ def create_interface():
                         with gr.Group(elem_classes="how-to-use"):
                              gr.HTML('<div class="section-heading">How to Use (Image)</div>')
                              gr.Markdown("""
-                                1. Upload an image or use the camera
-                                2. (Optional) Adjust settings like confidence threshold or model size (n, m, x)
-                                3. Optionally filter to specific object classes
-                                4. Click **Detect Objects** button
-                             """)
+                                    1. Upload an image or use the camera
+                                    2. (Optional) Adjust settings like confidence threshold or model size (n, m=balanced, x=accurate)
+                                    3. In Analysis Settings, you can uncheck "Use LLM for enhanced scene descriptions" if you prefer faster processing
+                                    4. Optionally filter to specific object classes
+                                    5. Click **Detect Objects** button
+                                """)
                         # Image Examples
                         gr.Examples(
                             examples=[
@@ -392,8 +533,18 @@ def create_interface():
                                     </details>
                                 """)
 
-                                # Wrap HTML description for potential styling
-                                image_scene_description_html = gr.HTML(label=None, elem_id="scene_analysis_description_text")
+                                gr.HTML('''
+                                    <div style="margin-top: 5px; padding: 6px 10px; background-color: #f0f9ff; border-radius: 4px; border-left: 3px solid #63b3ed; font-size: 12px; margin-bottom: 10px;">
+                                        <p style="margin: 0; color: #4a5568;">
+                                            <b>Note:</b> AI descriptions may vary slightly with each generation, reflecting the creative nature of AI. This is similar to how a person might use different words each time they describe the same image. Processing time may be longer during first use or when analyzing complex scenes, as the LLM enhancement requires additional computational resources.
+                                        </p>
+                                    </div>
+                                    ''')
+                                image_scene_description_html = gr.HTML(label=None, elem_id="scene_analysis_description_text")  
+                                
+                                # 使用LLM增強敘述時也會顯示原本敘述內容
+                                with gr.Accordion("Original Scene Analysis", open=False, elem_id="original_scene_analysis_accordion"):
+                                    image_llm_description = gr.HTML(label=None, elem_id="original_scene_description_text")
 
                                 with gr.Row():
                                      with gr.Column(scale=1):
@@ -419,7 +570,7 @@ def create_interface():
                                         gr.HTML('<div class="section-heading">Detection Statistics</div>')
                                         image_stats_json = gr.JSON(label=None, elem_classes="enhanced-json-display")
 
-            # Tab 2: Video Processing 
+            # Tab 2: Video Processing
             with gr.Tab("Video Processing"):
                 with gr.Row(equal_height=False):
                     # Left Column: Video Input & Controls
@@ -525,7 +676,7 @@ def create_interface():
                         gr.HTML('<div class="section-heading">Aggregated Statistics</div>')
                         video_stats_json = gr.JSON(label=None, elem_classes="video-stats-display") # Display statistics
 
-        # Event Listeners 
+        # Event Listeners
         # Image Model Change Handler
         image_model_dropdown.change(
             fn=lambda model: (model, DetectionModel.get_model_description(model)),
@@ -556,13 +707,12 @@ def create_interface():
             outputs=[video_input, video_url_input]
         )
 
-        # Image Processing Button Click
         image_detect_btn.click(
             fn=handle_image_upload,
-            inputs=[image_input, image_model_dropdown, image_confidence, image_class_filter],
+            inputs=[image_input, image_model_dropdown, image_confidence, image_class_filter, use_llm],
             outputs=[
                 image_result_image, image_result_text, image_stats_json, image_plot_output,
-                image_scene_description_html, image_activities_list, image_safety_list, image_zones_json,
+                image_scene_description_html, image_llm_description, image_activities_list, image_safety_list, image_zones_json,
                 image_lighting_info
             ]
         )
@@ -584,7 +734,7 @@ def create_interface():
         gr.HTML("""
              <div class="footer" style="padding: 25px 0; text-align: center; background: linear-gradient(to right, #f5f9fc, #e1f5fe); border-top: 1px solid #e2e8f0; margin-top: 30px;">
                  <div style="margin-bottom: 15px;">
-                     <p style="font-size: 14px; color: #4A5568; margin: 5px 0;">Powered by YOLOv8, CLIP and Ultralytics • Created with Gradio</p>
+                     <p style="font-size: 14px; color: #4A5568; margin: 5px 0;">Powered by YOLOv8, CLIP, Meta Llama3.2 and Ultralytics • Created with Gradio</p>
                  </div>
                  <div style="display: flex; align-items: center; justify-content: center; gap: 20px; margin-top: 15px;">
                      <p style="font-family: 'Arial', sans-serif; font-size: 14px; font-weight: 500; letter-spacing: 2px; background: linear-gradient(90deg, #38b2ac, #4299e1); -webkit-background-clip: text; -webkit-text-fill-color: transparent; margin: 0; text-transform: uppercase; display: inline-block;">EXPLORE THE CODE →</p>

enhance_scene_describer.py

@@ -164,8 +164,8 @@ class EnhancedSceneDescriber:
             "elevated_threshold": 0.6,  # Objects mostly in middle/bottom
             "elevated_top_threshold": 0.3  # Few objects at top of frame
         }
- 
-    
+
+
     def generate_description(self,
                         scene_type: str,
                         detected_objects: List[Dict],
@@ -193,7 +193,7 @@ class EnhancedSceneDescriber:
             return self._format_final_description(self._generate_generic_description(detected_objects, lighting_info))
 
         # Detect viewpoint
-        viewpoint = self._detect_viewpoint(detected_objects) 
+        viewpoint = self._detect_viewpoint(detected_objects)
 
         # Process aerial viewpoint scene types
         if viewpoint == "aerial":
@@ -326,7 +326,7 @@ class EnhancedSceneDescriber:
                 r"with \d+ people",
                 r"with \d+ person"
             ]
-            
+
             # Check and remove each pattern
             filtered_description = description
             for pattern in small_people_patterns:
@@ -358,40 +358,72 @@ class EnhancedSceneDescriber:
         # Final formatting to ensure correct punctuation and capitalization
         description = self._format_final_description(description)
 
-        return description
+        description_lines = description.split('\n')
+        clean_description = []
+        skip_block = False  # 添加這個變數的定義
+
+        for line in description_lines:
+            # 檢查是否需要跳過這行
+            if line.strip().startswith(':param') or line.strip().startswith('"""'):
+                continue
+            if line.strip().startswith("Exercise") or "class SceneDescriptionSystem" in line:
+                skip_block = True
+                continue
+            if ('def generate_scene_description' in line or
+                'def enhance_scene_descriptions' in line or
+                'def __init__' in line):
+                skip_block = True
+                continue
+            if line.strip().startswith('#TEST'):
+                skip_block = True
+                continue
+
+            # 空行結束跳過模式
+            if skip_block and line.strip() == "":
+                skip_block = False
+
+            # 如果不需要跳過，添加這行到結果
+            if not skip_block:
+                clean_description.append(line)
+
+        # 如果過濾後的描述為空，返回原始描述
+        if not clean_description:
+            return description
+        else:
+            return '\n'.join(clean_description)
 
     def _smart_append(self, current_text: str, new_fragment: str) -> str:
         """
         Intelligently append a new text fragment to the current text,
         handling punctuation and capitalization correctly.
-        
+
         Args:
             current_text: The existing text to append to
             new_fragment: The new text fragment to append
-            
+
         Returns:
             str: The combined text with proper formatting
         """
         # Handle empty cases
         if not new_fragment:
             return current_text
-            
+
         if not current_text:
             # Ensure first character is uppercase for the first fragment
             return new_fragment[0].upper() + new_fragment[1:] if new_fragment else ""
-        
+
         # Clean up existing text
         current_text = current_text.rstrip()
-        
+
         # Check for ending punctuation
         ends_with_sentence = current_text.endswith(('.', '!', '?'))
         ends_with_comma = current_text.endswith(',')
-        
+
         # Specifically handle the "A xxx A yyy" pattern that's causing issues
         if (current_text.startswith("A ") or current_text.startswith("An ")) and \
         (new_fragment.startswith("A ") or new_fragment.startswith("An ")):
             return current_text + ". " + new_fragment
-        
+
         # Decide how to join the texts
         if ends_with_sentence:
             # After a sentence, start with uppercase and add proper spacing
@@ -406,7 +438,7 @@ class EnhancedSceneDescriber:
             # When adding a new sentence about the scene, use a period
             joined_text = current_text + ". " + new_fragment
         else:
-            # For other cases, decide based on the content 
+            # For other cases, decide based on the content
             if self._is_related_phrases(current_text, new_fragment):
                 if new_fragment.startswith(('I ', 'I\'', 'A ', 'An ', 'The ')) or new_fragment[0].isupper():
                     joined_text = current_text + ", " + new_fragment
@@ -415,18 +447,18 @@ class EnhancedSceneDescriber:
             else:
                 # Use period for unrelated phrases
                 joined_text = current_text + ". " + (new_fragment[0].upper() + new_fragment[1:])
-        
+
         return joined_text
 
     def _is_related_phrases(self, text1: str, text2: str) -> bool:
         """
         Determine if two phrases are related and should be connected with a comma
         rather than separated with a period.
-        
+
         Args:
             text1: The first text fragment
             text2: The second text fragment to be appended
-            
+
         Returns:
             bool: Whether the phrases appear to be related
         """
@@ -434,61 +466,61 @@ class EnhancedSceneDescriber:
         if (text1.startswith("A ") or text1.startswith("An ")) and \
         (text2.startswith("A ") or text2.startswith("An ")):
             return False  # These are separate descriptions, not related phrases
-        
+
         # Check if the second phrase starts with a connecting word
-        connecting_words = ["which", "where", "who", "whom", "whose", "with", "without", 
+        connecting_words = ["which", "where", "who", "whom", "whose", "with", "without",
                         "this", "these", "that", "those", "and", "or", "but"]
-        
+
         first_word = text2.split()[0].lower() if text2 else ""
         if first_word in connecting_words:
             return True
-        
+
         # Check if the first phrase ends with something that suggests continuity
-        ending_patterns = ["such as", "including", "like", "especially", "particularly", 
+        ending_patterns = ["such as", "including", "like", "especially", "particularly",
                         "for example", "for instance", "namely", "specifically"]
-        
+
         for pattern in ending_patterns:
             if text1.lower().endswith(pattern):
                 return True
-        
+
         # Check if both phrases are about the scene
         if "scene" in text1.lower() and "scene" in text2.lower():
             return False  # Separate statements about the scene should be separate sentences
-        
+
         return False
 
     def _format_final_description(self, text: str) -> str:
         """
         Format the final description text to ensure correct punctuation,
         capitalization, and spacing.
-        
+
         Args:
             text: The text to format
-            
+
         Returns:
             str: The properly formatted text
         """
         import re
-        
+
         if not text:
             return ""
-        
+
         # 1. 特別處理連續以"A"開頭的片段 (這是一個常見問題)
         text = re.sub(r'(A\s[^.!?]+?)\s+(A\s)', r'\1. \2', text, flags=re.IGNORECASE)
         text = re.sub(r'(An\s[^.!?]+?)\s+(An?\s)', r'\1. \2', text, flags=re.IGNORECASE)
-        
+
         # 2. 確保第一個字母大寫
         text = text[0].upper() + text[1:] if text else ""
-        
+
         # 3. 修正詞之間的空格問題
         text = re.sub(r'\s{2,}', ' ', text)  # 多個空格改為一個
         text = re.sub(r'([a-z])([A-Z])', r'\1 \2', text)  # 小寫後大寫間加空格
-        
+
         # 4. 修正詞連接問題
         text = re.sub(r'([a-zA-Z])and', r'\1 and', text)  # "xxx"和"and"間加空格
         text = re.sub(r'([a-zA-Z])with', r'\1 with', text)  # "xxx"和"with"間加空格
         text = re.sub(r'plants(and|with|or)', r'plants \1', text)  # 修正"plantsand"這類問題
-        
+
         # 5. 修正標點符號後的大小寫問題
         text = re.sub(r'\.(\s+)([a-z])', lambda m: f'.{m.group(1)}{m.group(2).upper()}', text)  # 句號後大寫
 
@@ -498,46 +530,46 @@ class EnhancedSceneDescriber:
             # 例外情況：保留專有名詞、人稱代詞等的大寫
             if word in ["I", "I'm", "I've", "I'd", "I'll"]:
                 return match.group(0)  # 保持原樣
-            
+
             # 保留月份、星期、地名等專有名詞的大寫
-            proper_nouns = ["January", "February", "March", "April", "May", "June", "July", 
+            proper_nouns = ["January", "February", "March", "April", "May", "June", "July",
                             "August", "September", "October", "November", "December",
                             "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
             if word in proper_nouns:
                 return match.group(0)  # 保持原樣
-                
+
             # 其他情況：將首字母改為小寫
             return match.group(1) + word[0].lower() + word[1:]
-        
+
         # 匹配逗號後接空格再接大寫單詞的模式
         text = re.sub(r'(,\s+)([A-Z][a-zA-Z]*)', fix_capitalization_after_comma, text)
-    
-    
+
+
         common_phrases = [
             (r'Social or seating area', r'social or seating area'),
             (r'Sleeping area', r'sleeping area'),
             (r'Dining area', r'dining area'),
             (r'Living space', r'living space')
         ]
-        
+
         for phrase, replacement in common_phrases:
             # 只修改句中的術語，保留句首的大寫
             text = re.sub(r'(?<=[.!?]\s)' + phrase, replacement, text)
             # 修改句中的術語，但保留句首的大寫
             text = re.sub(r'(?<=,\s)' + phrase, replacement, text)
-        
+
         # 7. 確保標點符號後有空格
         text = re.sub(r'\s+([.,;:!?])', r'\1', text)  # 標點符號前不要空格
         text = re.sub(r'([.,;:!?])([a-zA-Z0-9])', r'\1 \2', text)  # 標點符號後要有空格
-        
+
         # 8. 修正重複標點符號
         text = re.sub(r'\.{2,}', '.', text)  # 多個句號變一個
         text = re.sub(r',{2,}', ',', text)  # 多個逗號變一個
-        
+
         # 9. 確保文本以標點結束
         if text and not text[-1] in '.!?':
             text += '.'
-        
+
         return text
 
     def _is_intersection(self, detected_objects: List[Dict]) -> bool:

image_processor.py

@@ -20,11 +20,13 @@ class ImageProcessor:
     Separates processing logic from UI components
     """
 
-    def __init__(self):
+    def __init__(self, use_llm=True, llm_model_path=None):
         """Initialize the image processor with required components"""
         self.color_mapper = ColorMapper()
         self.model_instances = {}
         self.lighting_analyzer = LightingAnalyzer()
+        self.use_llm = use_llm
+        self.llm_model_path = llm_model_path
 
     def get_model_instance(self, model_name: str, confidence: float = 0.25, iou: float = 0.25) -> DetectionModel:
         """
@@ -65,7 +67,11 @@ class ImageProcessor:
         try:
             # Initialize scene analyzer if not already done
             if not hasattr(self, 'scene_analyzer'):
-                self.scene_analyzer = SceneAnalyzer(class_names=detection_result.names)
+                self.scene_analyzer = SceneAnalyzer(
+                    class_names=detection_result.names,
+                    use_llm=self.use_llm,
+                    llm_model_path=self.llm_model_path
+                )
 
             # 確保類名正確更新
             if self.scene_analyzer.class_names is None:

llm_enhancer.py

@@ -0,0 +1,1218 @@
+import re
+import os
+import torch
+from typing import Dict, List, Tuple, Any, Optional
+import logging
+
+class LLMEnhancer:
+    """
+    負責使用LLM (Large Language Model) 增強場景理解和描述。
+    未來可以再整合Llama或其他LLM模型進行場景描述的生成和豐富化。
+    """
+
+    def __init__(self,
+                model_path: Optional[str] = None,
+                tokenizer_path: Optional[str] = None,
+                device: Optional[str] = None,
+                max_length: int = 2048,
+                temperature: float = 0.3,
+                top_p: float = 0.85):
+        """
+        初始化LLM增強器
+
+        Args:
+            model_path: LLM模型的路徑或HuggingFace log in，默認使用Llama 3.2
+            tokenizer_path: token處理器的路徑，通常與model_path相同
+            device: 設備檢查 ('cpu'或'cuda')
+            max_length: 生成文本的最大長度
+            temperature: 生成文本的溫度（較高比較有創意，較低會偏保守）
+            top_p: 生成文本時的核心採樣機率閾值
+        """
+        self.logger = logging.getLogger("LLMEnhancer")
+        self.logger.setLevel(logging.INFO)
+        handler = logging.StreamHandler()
+        handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
+        self.logger.addHandler(handler)
+
+        # 設置默認模型路徑就是用Llama3.2
+        self.model_path = model_path or "meta-llama/Llama-3.2-3B-Instruct"
+        self.tokenizer_path = tokenizer_path or self.model_path
+
+        # 確定運行設備
+        self.device = device or ('cuda' if torch.cuda.is_available() else 'cpu')
+        self.logger.info(f"Using device: {self.device}")
+
+        # create parameters
+        self.max_length = max_length
+        self.temperature = temperature
+        self.top_p = top_p
+
+        self.model = None
+        self.tokenizer = None
+
+        # 計數器，用來追蹤模型調用次數
+        self.call_count = 0
+
+        self._initialize_prompts()
+
+        # 只在需要時加載模型
+        self._model_loaded = False
+
+        try:
+            self.hf_token = os.environ.get("HF_TOKEN")
+            if self.hf_token:
+                self.logger.info("Logging in to Hugging Face with token")
+                from huggingface_hub import login
+                login(token=self.hf_token)
+            else:
+                self.logger.warning("HF_TOKEN not found in environment variables. Access to gated models may be limited.")
+        except Exception as e:
+            self.logger.error(f"Error during Hugging Face login: {e}")
+
+    def _load_model(self):
+        """懶加載模型 - 僅在首次需要時加載，使用 8 位量化以節省記憶體"""
+        if self._model_loaded:
+            return
+
+        try:
+            self.logger.info(f"Loading LLM model from {self.model_path} with 8-bit quantization")
+            import torch
+            from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+            torch.cuda.empty_cache()
+
+            # 打印可用 GPU 記憶體
+            if torch.cuda.is_available():
+                free_in_GB = torch.cuda.get_device_properties(0).total_memory / 1024**3
+                print(f"Total GPU memory: {free_in_GB:.2f} GB")
+
+            # 設置 8 位元量化配置
+            quantization_config = BitsAndBytesConfig(
+                load_in_8bit=True,
+                llm_int8_enable_fp32_cpu_offload=True
+            )
+
+            # 加載詞元處理器
+            self.tokenizer = AutoTokenizer.from_pretrained(
+                self.tokenizer_path,
+                padding_side="left",
+                use_fast=False,
+                token=self.hf_token
+            )
+
+            # 設置特殊標記
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+
+            # 加載 8 位量化模型
+            self.model = AutoModelForCausalLM.from_pretrained(
+                self.model_path,
+                quantization_config=quantization_config,
+                device_map="auto",  
+                low_cpu_mem_usage=True,
+                token=self.hf_token
+            )
+
+            self.logger.info("Model loaded successfully with 8-bit quantization")
+            self._model_loaded = True
+
+        except Exception as e:
+            self.logger.error(f"Error loading LLM model: {e}")
+            import traceback
+            traceback.print_exc()
+            raise
+
+    def _initialize_prompts(self):
+        """Return an optimized prompt template specifically for Zephyr model"""
+        # the prompt for the model
+        self.enhance_description_template = """
+            <|system|>
+            You are an expert visual analyst. Your task is to improve the readability and fluency of scene descriptions using STRICT factual accuracy.
+
+            Your **top priority is to avoid hallucination** or fabrication. You are working in a computer vision pipeline using object detection (YOLO) and image embeddings. You MUST treat the input object list as a whitelist. Do not speculate beyond this list.
+
+            </|system|>
+
+            <|user|>
+            Rewrite the following scene description to be fluent and clear. DO NOT add any objects, events, or spatial relationships that are not explicitly present in the original or object list.
+
+            ORIGINAL:
+            {original_description}
+
+            CRITICAL RULES:
+            1. NEVER assume room type, object function, or scene purpose unless directly stated.
+            2. NEVER invent object types. You are limited to: {object_list}
+            3. NEVER speculate on object quantity. If the description says "10 people" , DO NOT say "dozens" or "many". Maintain the original quantity unless specified.
+            4. Use terms like "in the scene", "visible in the background", or "positioned in the lower left" instead of assuming direction or layout logic.
+            5. You MAY describe confirmed materials, colors, and composition style if visually obvious and non-speculative.
+            6. Write 2–4 complete, well-structured sentences with punctuation.
+            7. Final output MUST be a single fluent paragraph of 60–200 words (not longer).
+            8. NEVER include explanations, reasoning, or tags. ONLY provide the enhanced description.
+            9. Do not repeat any sentence structure or phrase more than once.
+            </|user|>
+
+            <|assistant|>
+            """
+
+
+        # 錯誤檢測提示
+        self.verify_detection_template = """
+            Task: You are an advanced vision system that verifies computer vision detections for accuracy.
+
+            Analyze the following detection results and identify any potential errors or inconsistencies:
+
+            SCENE TYPE: {scene_type}
+            SCENE NAME: {scene_name}
+            CONFIDENCE: {confidence:.2f}
+
+            DETECTED OBJECTS: {detected_objects}
+
+            CLIP ANALYSIS RESULTS:
+            {clip_analysis}
+
+            Possible Errors to Check:
+            1. Objects misidentified (e.g., architectural elements labeled as vehicles)
+            2. Cultural elements misunderstood (e.g., Asian temple structures labeled as boats)
+            3. Objects that seem out of place for this type of scene
+            4. Inconsistencies between different detection systems
+
+            If you find potential errors, list them clearly with explanations. If the detections seem reasonable, state that they appear accurate.
+
+            Verification Results:
+            """
+
+        # 無檢測處理提示
+        self.no_detection_template = """
+            Task: You are an advanced scene understanding system analyzing an image where standard object detection failed to identify specific objects.
+
+            Based on advanced image embeddings (CLIP analysis), we have the following information:
+
+            MOST LIKELY SCENE: {top_scene} (confidence: {top_confidence:.2f})
+            VIEWPOINT: {viewpoint}
+            LIGHTING: {lighting_condition}
+
+            CULTURAL ANALYSIS: {cultural_analysis}
+
+            Create a detailed description of what might be in this scene, considering:
+            1. The most likely type of location or setting
+            2. Possible architectural or natural elements present
+            3. The lighting and atmosphere
+            4. Potential cultural or regional characteristics
+
+            Your description should be natural, flowing, and offer insights into what the image likely contains despite the lack of specific object detection.
+
+            Scene Description:
+            """
+
+    def _clean_llama_response(self, response: str) -> str:
+        """處理 Llama 模型特有的輸出格式問題"""
+        # 首先應用通用清理
+        response = self._clean_model_response(response)
+
+        # 移除 Llama 常見的前綴短語
+        prefixes_to_remove = [
+            "Here's the enhanced description:",
+            "Enhanced description:",
+            "Here is the enhanced scene description:",
+            "I've enhanced the description while preserving all factual details:"
+        ]
+
+        for prefix in prefixes_to_remove:
+            if response.lower().startswith(prefix.lower()):
+                response = response[len(prefix):].strip()
+
+        # 移除可能的後綴說明
+        suffixes_to_remove = [
+            "I've maintained all the key factual elements",
+            "I've preserved all the factual details",
+            "All factual elements have been maintained"
+        ]
+
+        for suffix in suffixes_to_remove:
+            if response.lower().endswith(suffix.lower()):
+                response = response[:response.rfind(suffix)].strip()
+
+        return response
+
+    def _detect_scene_type(self, detected_objects: List[Dict]) -> str:
+        """
+        Detect scene type based on object distribution and patterns
+        """
+        # Default scene type
+        scene_type = "intersection"
+
+        # Count objects by class
+        object_counts = {}
+        for obj in detected_objects:
+            class_name = obj.get("class_name", "")
+            if class_name not in object_counts:
+                object_counts[class_name] = 0
+            object_counts[class_name] += 1
+
+        # 辨識人
+        people_count = object_counts.get("person", 0)
+
+        # 交通工具的
+        car_count = object_counts.get("car", 0)
+        bus_count = object_counts.get("bus", 0)
+        truck_count = object_counts.get("truck", 0)
+        total_vehicles = car_count + bus_count + truck_count
+
+        # Simple scene type detection logic
+        if people_count > 8 and total_vehicles < 2:
+            scene_type = "pedestrian_crossing"
+        elif people_count > 5 and total_vehicles > 2:
+            scene_type = "busy_intersection"
+        elif people_count < 3 and total_vehicles > 3:
+            scene_type = "traffic_junction"
+
+        return scene_type
+
+    def _clean_scene_type(self, scene_type: str) -> str:
+        """清理場景類型，使其更適合用於提示詞"""
+        if not scene_type:
+            return "scene"
+
+        # replace underline to space or sometime capital letter
+        if '_' in scene_type:
+            return ' '.join(word.capitalize() for word in scene_type.split('_'))
+
+        return scene_type
+
+    def _clean_model_response(self, response: str) -> str:
+        """清理模型回應以移除常見的標記和前綴"""
+        # 移除任何可能殘留的系統樣式標記
+        response = re.sub(r'<\|.*?\|>', '', response)
+
+        # 移除任何 "This european_plaza" 或類似前綴
+        response = re.sub(r'^This [a-z_]+\s+', '', response)
+
+        # 確保響應以大寫字母開頭
+        if response and not response[0].isupper():
+            response = response[0].upper() + response[1:]
+
+        return response.strip()
+
+    def _validate_scene_facts(self, enhanced_desc: str, original_desc: str, people_count: int) -> str:
+        """Validate key facts in enhanced description"""
+        # Check if people count is preserved
+        if people_count > 0:
+            people_pattern = re.compile(r'(\d+)\s+(?:people|persons|pedestrians|individuals)', re.IGNORECASE)
+            people_match = people_pattern.search(enhanced_desc)
+
+            if not people_match or int(people_match.group(1)) != people_count:
+                # Replace incorrect count or add if missing
+                if people_match:
+                    enhanced_desc = people_pattern.sub(f"{people_count} people", enhanced_desc)
+                else:
+                    enhanced_desc = f"The scene shows {people_count} people. " + enhanced_desc
+
+        # Ensure aerial perspective is mentioned
+        if "aerial" in original_desc.lower() and "aerial" not in enhanced_desc.lower():
+            enhanced_desc = "From an aerial perspective, " + enhanced_desc[0].lower() + enhanced_desc[1:]
+
+        return enhanced_desc
+
+    def reset_context(self):
+        """在處理新圖像前重置模型上下文"""
+        if self._model_loaded:
+            # 清除 GPU 緩存
+            torch.cuda.empty_cache()
+            self.logger.info("Model context reset")
+        else:
+            self.logger.info("Model not loaded, no context to reset")
+
+    def _remove_introduction_sentences(self, response: str) -> str:
+        """移除生成文本中可能的介紹性句子"""
+        # 識別常見的介紹性模式
+        intro_patterns = [
+            r'^Here is the (?:rewritten|enhanced) .*?description:',
+            r'^The (?:rewritten|enhanced) description:',
+            r'^Here\'s the (?:rewritten|enhanced) description of .*?:'
+        ]
+
+        for pattern in intro_patterns:
+            if re.match(pattern, response, re.IGNORECASE):
+                # 找到冒號後的內容
+                parts = re.split(r':', response, 1)
+                if len(parts) > 1:
+                    return parts[1].strip()
+
+        return response
+
+    def enhance_description(self, scene_data: Dict[str, Any]) -> str:
+        """改進的場景描述增強器，處理各種場景類型並保留視角與光照資訊，並作為總窗口可運用於其他class"""
+        try:
+            # 重置上下文
+            self.reset_context()
+
+            # 確保模型已加載
+            if not self._model_loaded:
+                self._load_model()
+
+            # extract original description
+            original_desc = scene_data.get("original_description", "")
+            if not original_desc:
+                return "No original description provided."
+
+            # 獲取scene type 並標準化
+            scene_type = scene_data.get("scene_type", "unknown scene")
+            scene_type = self._clean_scene_type(scene_type)
+
+            # 提取檢測到的物件並過濾低置信度物件
+            detected_objects = scene_data.get("detected_objects", [])
+            filtered_objects = []
+
+            # 高置信度閾值，嚴格過濾物件
+            high_confidence_threshold = 0.65
+
+            for obj in detected_objects:
+                confidence = obj.get("confidence", 0)
+                class_name = obj.get("class_name", "")
+
+                # 為特殊類別設置更高閾值
+                special_classes = ["airplane", "helicopter", "boat"]
+                if class_name in special_classes:
+                    if confidence < 0.75:  # 為這些類別設置更高閾值
+                        continue
+
+                # 僅保留高置信度物件
+                if confidence >= high_confidence_threshold:
+                    filtered_objects.append(obj)
+
+            # 計算物件列表和數量 - 僅使用過濾後的高置信度物件
+            object_counts = {}
+            for obj in filtered_objects:
+                class_name = obj.get("class_name", "")
+                if class_name not in object_counts:
+                    object_counts[class_name] = 0
+                object_counts[class_name] += 1
+
+            # 將高置信度物件格式化為清單
+            high_confidence_objects = ", ".join([f"{count} {obj}" for obj, count in object_counts.items()])
+
+            # 如果沒有高置信度物件，回退到使用原始描述中的關鍵詞
+            if not high_confidence_objects:
+                # 從原始描述中提取物件提及
+                object_keywords = self._extract_objects_from_description(original_desc)
+                high_confidence_objects = ", ".join(object_keywords) if object_keywords else "objects visible in the scene"
+
+            # 保留原始描述中的關鍵視角信息
+            perspective = self._extract_perspective_from_description(original_desc)
+
+            # 提取光照資訊
+            lighting_description = "unknown lighting"
+            if "lighting_info" in scene_data:
+                lighting_info = scene_data.get("lighting_info", {})
+                time_of_day = lighting_info.get("time_of_day", "unknown")
+                is_indoor = lighting_info.get("is_indoor", False)
+                lighting_description = f"{'indoor' if is_indoor else 'outdoor'} {time_of_day} lighting"
+
+            # 構建提示詞，整合所有關鍵資訊
+            prompt = self.enhance_description_template.format(
+                scene_type=scene_type,
+                object_list=high_confidence_objects,
+                original_description=original_desc,
+                perspective=perspective,
+                lighting_description=lighting_description
+            )
+
+            # 生成增強描述
+            self.logger.info("Generating LLM response...")
+            response = self._generate_llm_response(prompt)
+
+            # 檢查回應完整性的更嚴格標準
+            is_incomplete = (
+                len(response) < 100 or  # 太短
+                (len(response) < 200 and "." not in response[-30:]) or  # 結尾沒有適當標點
+                any(response.endswith(phrase) for phrase in ["in the", "with the", "and the"])  # 以不完整短語結尾
+            )
+
+            max_retries = 3
+            attempts = 0
+            while attempts < max_retries and is_incomplete:
+                self.logger.warning(f"Generated incomplete response, retrying... Attempt {attempts+1}/{max_retries}")
+                # 重新生成
+                response = self._generate_llm_response(prompt)
+                attempts += 1
+
+                # 重新檢查完整性
+                is_incomplete = (len(response) < 100 or
+                                (len(response) < 200 and "." not in response[-30:]) or
+                                any(response.endswith(phrase) for phrase in ["in the", "with the", "and the"]))
+
+            # 確保響應不為空
+            if not response or len(response.strip()) < 10:
+                self.logger.warning("Generated response was empty or too short, returning original description")
+                return original_desc
+
+            # 清理響應 - 使用與模型相符的清理方法
+            if "llama" in self.model_path.lower():
+                result = self._clean_llama_response(response)
+            else:
+                result = self._clean_model_response(response)
+
+            # 移除介紹性句子
+            result = self._remove_introduction_sentences(result)
+
+            # 移除解釋性注釋
+            result = self._remove_explanatory_notes(result)
+
+            # 進行事實準確性檢查
+            result = self._verify_factual_accuracy(original_desc, result, high_confidence_objects)
+
+            # 確保場景類型和視角一致性
+            result = self._ensure_scene_type_consistency(result, scene_type, original_desc)
+            if perspective and perspective.lower() not in result.lower():
+                result = f"{perspective}, {result[0].lower()}{result[1:]}"
+
+            return str(result)
+
+        except Exception as e:
+            self.logger.error(f"Enhancement failed: {str(e)}")
+            import traceback
+            self.logger.error(traceback.format_exc())
+            return original_desc  # 發生任何錯誤時返回原始描述
+
+    def _verify_factual_accuracy(self, original: str, generated: str, object_list: str) -> str:
+        """驗證生成的描述不包含原始描述或物體列表中沒有的信息"""
+
+        # 將原始描述和物體列表合併為授權詞彙源
+        authorized_content = original.lower() + " " + object_list.lower()
+
+        # 提取生成描述中具有實質意義的名詞
+        # 創建常見地點、文化和地域詞彙的列表
+        location_terms = ["plaza", "square", "market", "mall", "avenue", "boulevard"]
+        cultural_terms = ["european", "asian", "american", "african", "western", "eastern"]
+
+        # 檢查生成文本中的每個詞
+        for term in location_terms + cultural_terms:
+            # 僅當該詞出現在生成文本但不在授權內容中時進行替換
+            if term in generated.lower() and term not in authorized_content:
+                # 根據詞語類型選擇適當的替換詞
+                if term in location_terms:
+                    replacement = "area"
+                else:
+                    replacement = "scene"
+
+                # 使用正則表達式進行完整詞匹配替換
+                pattern = re.compile(r'\b' + term + r'\b', re.IGNORECASE)
+                generated = pattern.sub(replacement, generated)
+
+        return generated
+
+
+    def verify_detection(self,
+                       detected_objects: List[Dict],
+                       clip_analysis: Dict[str, Any],
+                       scene_type: str,
+                       scene_name: str,
+                       confidence: float) -> Dict[str, Any]:
+        """
+        驗證並可能修正YOLO的檢測結果
+
+        Args:
+            detected_objects: YOLO檢測到的物體列表
+            clip_analysis: CLIP分析結果
+            scene_type: 識別的場景類型
+            scene_name: 場景名稱
+            confidence: 場景分類的信心度
+
+        Returns:
+            Dict: 包含驗證結果和建議的字典
+        """
+        # 確保模型已加載
+        self._load_model()
+
+        # 格式化數據
+        objects_str = self._format_objects_for_prompt(detected_objects)
+        clip_str = self._format_clip_results(clip_analysis)
+
+        # 構建提示
+        prompt = self.verify_detection_template.format(
+            scene_type=scene_type,
+            scene_name=scene_name,
+            confidence=confidence,
+            detected_objects=objects_str,
+            clip_analysis=clip_str
+        )
+
+        # 調用LLM進行驗證
+        verification_result = self._generate_llm_response(prompt)
+
+        # 解析驗證結果
+        result = {
+            "verification_text": verification_result,
+            "has_errors": "appear accurate" not in verification_result.lower(),
+            "corrected_objects": None  # 可能在未來版本實現詳細錯誤修正
+        }
+
+        return result
+
+    def _validate_content_consistency(self, original_desc: str, enhanced_desc: str) -> str:
+        """驗證增強描述的內容與原始描述一致"""
+        # 提取原始描述中的關鍵數值
+        people_count_match = re.search(r'(\d+)\s+people', original_desc, re.IGNORECASE)
+        people_count = int(people_count_match.group(1)) if people_count_match else None
+
+        # 驗證人數一致性
+        if people_count:
+            enhanced_count_match = re.search(r'(\d+)\s+people', enhanced_desc, re.IGNORECASE)
+            if not enhanced_count_match or int(enhanced_count_match.group(1)) != people_count:
+                # 保留原始人數
+                if enhanced_count_match:
+                    enhanced_desc = re.sub(r'\b\d+\s+people\b', f"{people_count} people", enhanced_desc, flags=re.IGNORECASE)
+                elif "people" in enhanced_desc.lower():
+                    enhanced_desc = re.sub(r'\bpeople\b', f"{people_count} people", enhanced_desc, flags=re.IGNORECASE)
+
+        # 驗證視角/透視一致性
+        perspective_terms = ["aerial", "bird's-eye", "overhead", "ground level", "eye level"]
+
+        for term in perspective_terms:
+            if term in original_desc.lower() and term not in enhanced_desc.lower():
+                # 添加缺失的視角信息
+                if enhanced_desc[0].isupper():
+                    enhanced_desc = f"From {term} view, {enhanced_desc[0].lower()}{enhanced_desc[1:]}"
+                else:
+                    enhanced_desc = f"From {term} view, {enhanced_desc}"
+                break
+
+        return enhanced_desc
+
+    def _remove_explanatory_notes(self, response: str) -> str:
+        """移除解釋性注釋、說明和其他非描述性內容"""
+
+        # 識別常見的注釋和解釋模式
+        note_patterns = [
+            r'(?:^|\n)Note:.*?(?:\n|$)',
+            r'(?:^|\n)I have (?:followed|adhered to|ensured).*?(?:\n|$)',
+            r'(?:^|\n)This description (?:follows|adheres to|maintains).*?(?:\n|$)',
+            r'(?:^|\n)The enhanced description (?:maintains|preserves).*?(?:\n|$)'
+        ]
+
+        # 尋找第一段完整的描述內容
+        paragraphs = [p.strip() for p in response.split('\n\n') if p.strip()]
+
+        # 如果只有一個段落，檢查並清理它
+        if len(paragraphs) == 1:
+            for pattern in note_patterns:
+                paragraphs[0] = re.sub(pattern, '', paragraphs[0], flags=re.IGNORECASE)
+            return paragraphs[0].strip()
+
+        # 如果有多個段落，識別並移除注釋段落
+        content_paragraphs = []
+        for paragraph in paragraphs:
+            is_note = False
+            for pattern in note_patterns:
+                if re.search(pattern, paragraph, flags=re.IGNORECASE):
+                    is_note = True
+                    break
+
+            # 檢查段落是否以常見的注釋詞開頭
+            if paragraph.lower().startswith(('note:', 'please note:', 'remember:')):
+                is_note = True
+
+            if not is_note:
+                content_paragraphs.append(paragraph)
+
+        # 返回清理後的內容
+        return '\n\n'.join(content_paragraphs).strip()
+
+    def handle_no_detection(self, clip_analysis: Dict[str, Any]) -> str:
+        """
+        處理YOLO未檢測到物體的情況
+
+        Args:
+            clip_analysis: CLIP分析結果
+
+        Returns:
+            str: 生成的場景描述
+        """
+        # 確保模型已加載
+        self._load_model()
+
+        # 提取CLIP結果
+        top_scene, top_confidence = clip_analysis.get("top_scene", ("unknown", 0))
+        viewpoint = clip_analysis.get("viewpoint", ("standard", 0))[0]
+        lighting = clip_analysis.get("lighting_condition", ("unknown", 0))[0]
+
+        # 格式化文化分析
+        cultural_str = self._format_cultural_analysis(clip_analysis.get("cultural_analysis", {}))
+
+        # 構建提示
+        prompt = self.no_detection_template.format(
+            top_scene=top_scene,
+            top_confidence=top_confidence,
+            viewpoint=viewpoint,
+            lighting_condition=lighting,
+            cultural_analysis=cultural_str
+        )
+
+        # 調用LLM生成描述
+        description = self._generate_llm_response(prompt)
+
+        # 優化輸出
+        return self._clean_llm_response(description)
+
+    def _clean_input_text(self, text: str) -> str:
+        """
+        對輸入文本進行通用的格式清理，處理常見的格式問題。
+
+        Args:
+            text: 輸入文本
+
+        Returns:
+            清理後的文本
+        """
+        if not text:
+            return ""
+
+        # 清理格式的問題
+        # 1. 處理連續標點符號問題
+        text = re.sub(r'([.,;:!?])\1+', r'\1', text)
+
+        # 2. 修復不完整句子的標點（如 "Something," 後沒有繼續句子）
+        text = re.sub(r',\s*$', '.', text)
+
+        # 3. 修復如 "word." 後未加空格即接下一句的問題
+        text = re.sub(r'([.!?])([A-Z])', r'\1 \2', text)
+
+        # 4. 移除多餘空格
+        text = re.sub(r'\s+', ' ', text).strip()
+
+        # 5. 確保句子正確結束（句尾加句號）
+        if text and not text[-1] in '.!?':
+            text += '.'
+
+        return text
+
+    def _fact_check_description(self, original_desc: str, enhanced_desc: str, scene_type: str, detected_objects: List[str]) -> str:
+        """
+        驗證並可能修正增強後的描述，確保其保持事實準確性，針對普遍事實而非特定場景。
+
+        Args:
+            original_desc: 原始場景描述
+            enhanced_desc: 增強後的描述待驗證
+            scene_type: 場景類型
+            detected_objects: 檢測到的物體名稱列表
+
+        Returns:
+            經過事實檢查的描述
+        """
+        # 如果增強描述為空或太短，返回原始描述
+        if not enhanced_desc or len(enhanced_desc) < 30:
+            return original_desc
+
+        # 1. 檢查數值一致性（如人數、物體數量等）
+        # 從原始描述中提取數字和相關名詞
+        number_patterns = [
+            (r'(\d+)\s+(people|person|pedestrians|individuals)', r'\1', r'\2'), # 人數
+            (r'(\d+)\s+(cars|vehicles|automobiles)', r'\1', r'\2'),            # 車輛數
+            (r'(\d+)\s+(buildings|structures)', r'\1', r'\2')                  # 建築數
+        ]
+
+        # 檢查原始描述中的每個數字
+        for pattern, num_group, word_group in number_patterns:
+            original_matches = re.finditer(pattern, original_desc, re.IGNORECASE)
+            for match in original_matches:
+                number = match.group(1)
+                noun = match.group(2)
+
+                # 檢查增強描述中是否保留了這個數字
+                # 創建一個更通用的模式來檢查增強描述中是否包含此數字和對象類別
+                enhanced_pattern = r'(\d+)\s+(' + re.escape(noun) + r'|' + re.escape(noun.rstrip('s')) + r'|' + re.escape(noun + 's') + r')'
+                enhanced_matches = list(re.finditer(enhanced_pattern, enhanced_desc, re.IGNORECASE))
+
+                if not enhanced_matches:
+                    # 數字+名詞未在增強描述中找到
+                    plural_form = noun if noun.endswith('s') or number == '1' else noun + 's'
+                    if enhanced_desc.startswith("This") or enhanced_desc.startswith("The"):
+                        enhanced_desc = enhanced_desc.replace("This ", f"This scene with {number} {plural_form} ", 1)
+                        enhanced_desc = enhanced_desc.replace("The ", f"The scene with {number} {plural_form} ", 1)
+                    else:
+                        enhanced_desc = f"The scene includes {number} {plural_form}. " + enhanced_desc
+                elif enhanced_matches and match.group(1) != number:
+                    # 存在但數字不一致，就要更正數字
+                    for ematch in enhanced_matches:
+                        wrong_number = ematch.group(1)
+                        enhanced_desc = enhanced_desc.replace(f"{wrong_number} {ematch.group(2)}", f"{number} {ematch.group(2)}")
+
+        # 2. 檢查視角的一致性
+        perspective_terms = {
+            "aerial": ["aerial", "bird's-eye", "overhead", "top-down", "above", "looking down"],
+            "ground": ["street-level", "ground level", "eye-level", "standing"],
+            "indoor": ["inside", "interior", "indoor", "within"],
+            "close-up": ["close-up", "detailed view", "close shot"]
+        }
+
+        # 確定原始視角
+        original_perspective = None
+        for persp, terms in perspective_terms.items():
+            if any(term in original_desc.lower() for term in terms):
+                original_perspective = persp
+                break
+
+        # 檢查是否保留了視角方面
+        if original_perspective:
+            enhanced_has_perspective = any(term in enhanced_desc.lower() for term in perspective_terms[original_perspective])
+
+            if not enhanced_has_perspective:
+                # 添加之前缺的視角方面
+                perspective_prefixes = {
+                    "aerial": "From an aerial perspective, ",
+                    "ground": "From street level, ",
+                    "indoor": "In this indoor setting, ",
+                    "close-up": "In this close-up view, "
+                }
+
+                prefix = perspective_prefixes.get(original_perspective, "")
+                if prefix:
+                    if enhanced_desc[0].isupper():
+                        enhanced_desc = prefix + enhanced_desc[0].lower() + enhanced_desc[1:]
+                    else:
+                        enhanced_desc = prefix + enhanced_desc
+
+        # 3. 檢查場景類型一致性
+        if scene_type and scene_type.lower() != "unknown" and scene_type.lower() not in enhanced_desc.lower():
+            # 優雅地添加場景類型
+            if enhanced_desc.startswith("This ") or enhanced_desc.startswith("The "):
+                # 避免產生 "This scene" 和 "This intersection" 的重複
+                if "scene" in enhanced_desc[:15].lower():
+                    fixed_type = scene_type.lower()
+                    enhanced_desc = enhanced_desc.replace("scene", fixed_type, 1)
+                else:
+                    enhanced_desc = enhanced_desc.replace("This ", f"This {scene_type} ", 1)
+                    enhanced_desc = enhanced_desc.replace("The ", f"The {scene_type} ", 1)
+            else:
+                enhanced_desc = f"This {scene_type} " + enhanced_desc
+
+        # 4. 確保文字長度適當，這邊的限制要與prompt相同,否則會產生矛盾
+        words = enhanced_desc.split()
+        if len(words) > 200:
+            # 找尋接近字數限制的句子結束處
+            truncated = ' '.join(words[:200])
+            last_period = max(truncated.rfind('.'), truncated.rfind('!'), truncated.rfind('?'))
+
+            if last_period > 0:
+                enhanced_desc = truncated[:last_period+1]
+            else:
+                enhanced_desc = truncated + '.'
+
+        return enhanced_desc
+
+    def _extract_perspective_from_description(self, description: str) -> str:
+        """從原始描述中提取視角/透視信息"""
+        perspective_terms = {
+            "aerial": ["aerial perspective", "aerial view", "bird's-eye view", "overhead view", "from above"],
+            "ground": ["ground level", "eye level", "street level"],
+            "indoor": ["indoor setting", "inside", "interior"]
+        }
+
+        for persp_type, terms in perspective_terms.items():
+            for term in terms:
+                if term.lower() in description.lower():
+                    return term
+
+        return ""
+
+    def _extract_objects_from_description(self, description: str) -> List[str]:
+        """從原始描述中提取物件提及"""
+        # 常見物件正則表達式模式
+        object_patterns = [
+            r'(\d+)\s+(people|persons|pedestrians|individuals)',
+            r'(\d+)\s+(cars|vehicles|automobiles)',
+            r'(\d+)\s+(buildings|structures)',
+            r'(\d+)\s+(plants|potted plants|flowers)',
+            r'(\d+)\s+(beds|furniture|tables|chairs)'
+        ]
+
+        extracted_objects = []
+
+        for pattern in object_patterns:
+            matches = re.finditer(pattern, description, re.IGNORECASE)
+            for match in matches:
+                number = match.group(1)
+                object_type = match.group(2)
+                extracted_objects.append(f"{number} {object_type}")
+
+        return extracted_objects
+
+    def _ensure_scene_type_consistency(self, description: str, scene_type: str, original_desc: str) -> str:
+        """確保描述中的場景類型與指定的場景類型一致"""
+        # 禁止使用的錯誤場景詞列表
+        prohibited_scene_words = ["plaza", "square", "european", "asian", "american"]
+
+        # 檢查是否包含禁止的場景詞
+        for word in prohibited_scene_words:
+            if word in description.lower() and word not in original_desc.lower() and word not in scene_type.lower():
+                # 替換錯誤場景詞為正確場景類型
+                pattern = re.compile(r'\b' + word + r'\b', re.IGNORECASE)
+                description = pattern.sub(scene_type, description)
+
+        # 確保場景類型在描述中被提及
+        if scene_type.lower() not in description.lower():
+            # 尋找通用場景詞並替換
+            for general_term in ["scene", "area", "place", "location"]:
+                if general_term in description.lower():
+                    pattern = re.compile(r'\b' + general_term + r'\b', re.IGNORECASE)
+                    description = pattern.sub(scene_type, description, count=1)
+                    break
+            else:
+                # 如果沒有找到通用詞，在開頭添加場景類型
+                if description.startswith("The "):
+                    description = description.replace("The ", f"The {scene_type} ", 1)
+                elif description.startswith("This "):
+                    description = description.replace("This ", f"This {scene_type} ", 1)
+                else:
+                    description = f"This {scene_type} " + description
+
+        return description
+
+    def _generate_llm_response(self, prompt: str) -> str:
+        """生成 LLM 的回應"""
+        self._load_model()
+
+        try:
+            self.call_count += 1
+            self.logger.info(f"LLM call #{self.call_count}")
+
+            # 清除 GPU 緩存
+            torch.cuda.empty_cache()
+
+            # 設置固定種子以提高一致性
+            torch.manual_seed(42)
+
+            # 準備輸入
+            inputs = self.tokenizer(prompt, return_tensors="pt", truncation=True, max_length=self.max_length).to(self.device)
+
+            # 根據模型類型調整參數
+            generation_params = {
+                "max_new_tokens": 120,
+                "pad_token_id": self.tokenizer.eos_token_id,
+                "attention_mask": inputs.attention_mask,
+                "use_cache": True,
+            }
+
+            # 為 Llama 模型設置特定參數
+            if "llama" in self.model_path.lower():
+                generation_params.update({
+                    "temperature": 0.4,        # 不要太高, 否則模型可能會太有主觀意見
+                    "max_new_tokens": 600,      
+                    "do_sample": True,          
+                    "top_p": 0.8,              
+                    "repetition_penalty": 1.2,  # 重複的懲罰權重,可避免掉重複字
+                    "num_beams": 4 ,            
+                    "length_penalty": 1.2,      
+                })
+
+            else:
+                # 如果用其他模型的參數
+                generation_params.update({
+                    "temperature": 0.6,
+                    "max_new_tokens": 300,
+                    "top_p": 0.9,
+                    "do_sample": True,
+                    "num_beams": 1,
+                    "repetition_penalty": 1.05
+                })
+
+            # 生成回應
+            with torch.no_grad():
+                outputs = self.model.generate(inputs.input_ids, **generation_params)
+
+            # 解碼完整輸出
+            full_response = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+
+            # 提取生成的響應部分
+            assistant_tag = "<|assistant|>"
+            if assistant_tag in full_response:
+                response = full_response.split(assistant_tag)[-1].strip()
+
+                # 檢查是否有未閉合的 <|assistant|> 
+                user_tag = "<|user|>"
+                if user_tag in response:
+                    response = response.split(user_tag)[0].strip()
+            else:
+                # 移除輸入提示
+                input_text = self.tokenizer.decode(inputs.input_ids[0], skip_special_tokens=True)
+                response = full_response
+                if response.startswith(input_text):
+                    response = response[len(input_text):].strip()
+
+            # 確保不返回空響應
+            if not response or len(response.strip()) < 10:
+                self.logger.warning("生成的回應為空的或太短，返回默認回應")
+                return "No detailed description could be generated."
+
+            return response
+
+        except Exception as e:
+            self.logger.error(f"生成 LLM 響應時出錯: {str(e)}")
+            import traceback
+            self.logger.error(traceback.format_exc())
+            return "Unable to generate enhanced description."
+
+    def _clean_llm_response(self, response: str) -> str:
+        """
+        Clean the LLM response to ensure the output contains only clean descriptive text.
+        Sometimes it will not only display the description but display tags, notes...etc
+
+        Args:
+            response: Original response from the LLM
+
+        Returns:
+            Cleaned description text
+        """
+        if not response:
+            return ""
+
+        # Save original response as backup
+        original_response = response
+
+        # 1. Extract content between markers (if present)
+        output_start = response.find("[OUTPUT_START]")
+        output_end = response.find("[OUTPUT_END]")
+        if output_start != -1 and output_end != -1 and output_end > output_start:
+            response = response[output_start + len("[OUTPUT_START]"):output_end].strip()
+
+        # 2. Remove all remaining section markers and instructions
+        section_markers = [
+            r'\[.*?\]',                      # [any text]
+            r'OUTPUT_START\s*:|OUTPUT_END\s*:',  # OUTPUT_START: or OUTPUT_END:
+            r'ENHANCED DESCRIPTION\s*:',      # ENHANCED DESCRIPTION:
+            r'Scene Type\s*:.*?(?=\n|$)',    # Scene Type: text
+            r'Original Description\s*:.*?(?=\n|$)', # Original Description: text
+            r'GOOD\s*:|BAD\s*:',             # GOOD: or BAD:
+            r'PROBLEM\s*:.*?(?=\n|$)',       # PROBLEM: text
+            r'</?\|(?:assistant|system|user)\|>',  # Dialog markers
+            r'\(Note:.*?\)',                 # Notes in parentheses
+            r'\(.*?I\'ve.*?\)',              # Common explanatory content
+            r'\(.*?as per your request.*?\)' # References to instructions
+        ]
+
+        for marker in section_markers:
+            response = re.sub(marker, '', response, flags=re.IGNORECASE)
+
+        # 3. Remove common prefixes and suffixes
+        prefixes_to_remove = [
+            "Enhanced Description:",
+            "Scene Description:",
+            "Description:",
+            "Here is the enhanced description:",
+            "Here's the enhanced description:"
+        ]
+
+        for prefix in prefixes_to_remove:
+            if response.lower().startswith(prefix.lower()):
+                response = response[len(prefix):].strip()
+
+        # 4. Remove any Context tags or text containing Context
+        response = re.sub(r'<\s*Context:.*?>', '', response)
+        response = re.sub(r'Context:.*?(?=\n|$)', '', response)
+        response = re.sub(r'Note:.*?(?=\n|$)', '', response, flags=re.IGNORECASE)
+
+        # 5. Clean improper scene type references
+        scene_type_pattern = r'This ([a-zA-Z_]+) (features|shows|displays|contains)'
+        match = re.search(scene_type_pattern, response)
+        if match and '_' in match.group(1):
+            fixed_text = f"This scene {match.group(2)}"
+            response = re.sub(scene_type_pattern, fixed_text, response)
+
+        # 6. Reduce dash usage for more natural punctuation
+        response = re.sub(r'—', ', ', response)
+        response = re.sub(r' - ', ', ', response)
+
+        # 7. Remove excess whitespace and line breaks
+        response = response.replace('\r', ' ')
+        response = re.sub(r'\n+', ' ', response)  # 將所有換行符替換為空格
+        response = re.sub(r'\s{2,}', ' ', response)  # 將多個空格替換為單個空格
+
+        # 8. Remove Markdown formatting
+        response = re.sub(r'\*\*|\*|__|\|', '', response)  # Remove Markdown indicators
+
+        # 9. Detect and remove sentence duplicates
+        sentences = re.split(r'(?<=[.!?])\s+', response)
+        unique_sentences = []
+        seen_content = set()
+
+        for sentence in sentences:
+            # Skip empty sentences
+            if not sentence.strip():
+                continue
+
+            # Create simplified version for comparison (lowercase, no punctuation)
+            simplified = re.sub(r'[^\w\s]', '', sentence.lower())
+            simplified = ' '.join(simplified.split())  # Standardize whitespace
+
+            # Check if we've seen a similar sentence
+            is_duplicate = False
+            for existing in seen_content:
+                if len(simplified) > 10 and (existing in simplified or simplified in existing):
+                    is_duplicate = True
+                    break
+
+            if not is_duplicate and simplified:
+                unique_sentences.append(sentence)
+                seen_content.add(simplified)
+
+        # Recombine unique sentences
+        response = ' '.join(unique_sentences)
+
+        # 10. Ensure word count is within limits (50-150 words)
+        words = response.split()
+        if len(words) > 200:
+            # Find sentence ending near the word limit
+            truncated = ' '.join(words[:200])
+            last_period = max(truncated.rfind('.'), truncated.rfind('!'), truncated.rfind('?'))
+
+            if last_period > 0:
+                response = truncated[:last_period+1]
+            else:
+                response = truncated + "."
+
+        # 11. Check sentence completeness
+        if response and not response.strip()[-1] in ['.', '!', '?']:
+            # Find the last preposition or conjunction
+            common_prepositions = ["into", "onto", "about", "above", "across", "after", "along", "around", "at", "before", "behind", "below", "beneath", "beside", "between", "beyond", "by", "down", "during", "except", "for", "from", "in", "inside", "near", "of", "off", "on", "over", "through", "to", "toward", "under", "up", "upon", "with", "within"]
+
+            # Check if ending with preposition or conjunction
+            last_word = response.strip().split()[-1].lower() if response.strip().split() else ""
+            if last_word in common_prepositions or last_word in ["and", "or", "but"]:
+                # Find the last complete sentence
+                last_period = max(response.rfind('.'), response.rfind('!'), response.rfind('?'))
+                if last_period > 0:
+                    response = response[:last_period+1]
+                else:
+                    # If no complete sentence found, modify the ending
+                    words = response.strip().split()
+                    if words:
+                        # Remove the last preposition or conjunction
+                        response = " ".join(words[:-1]) + "."
+
+        # 12. Ensure haven't over-filtered
+        if not response or len(response) < 40:
+            # Try to get the first meaningful paragraph from the original response
+            paragraphs = [p for p in original_response.split('\n\n') if p.strip()]
+            if paragraphs:
+                # Choose the longest paragraph as it's most likely the actual description
+                best_para = max(paragraphs, key=len)
+                # Clean it using a subset of the above rules
+                best_para = re.sub(r'\[.*?\]', '', best_para)  # Remove [SECTION] markers
+                best_para = re.sub(r'\s{2,}', ' ', best_para).strip()  # Clean whitespace
+
+                if len(best_para) >= 40:
+                    return best_para
+
+            # If still no good content, return a simple message
+            return "Unable to generate a valid enhanced description."
+
+        # 13. Final cleaning - catch any missed special cases
+        response = re.sub(r'</?\|.*?\|>', '', response)  # Any remaining tags
+        response = re.sub(r'\(.*?\)', '', response)  # Any remaining parenthetical content
+        response = re.sub(r'Note:.*?(?=\n|$)', '', response, flags=re.IGNORECASE)  # Any remaining notes
+
+        # Ensure proper spacing after punctuation
+        response = re.sub(r'([.!?])([A-Z])', r'\1 \2', response)
+
+        # Ensure first letter is capitalized
+        if response and response[0].islower():
+            response = response[0].upper() + response[1:]
+
+        # 14. 統一格式 - 確保輸出始終是單一段落
+        response = re.sub(r'\s*\n\s*', ' ', response)  # 將所有換行符替換為空格
+        response = ' '.join(response.split())  
+
+        return response.strip()
+
+    def _format_objects_for_prompt(self, objects: List[Dict]) -> str:
+        """格式化物體列表以用於提示"""
+        if not objects:
+            return "No objects detected"
+
+        formatted = []
+        for obj in objects:
+            formatted.append(f"{obj['class_name']} (confidence: {obj['confidence']:.2f})")
+
+        return "\n- " + "\n- ".join(formatted)
+
+    def _format_lighting(self, lighting_info: Dict) -> str:
+        """格式化光照信息以用於提示"""
+        if not lighting_info:
+            return "Unknown lighting conditions"
+
+        time = lighting_info.get("time_of_day", "unknown")
+        conf = lighting_info.get("confidence", 0)
+        is_indoor = lighting_info.get("is_indoor", False)
+
+        base_info = f"{'Indoor' if is_indoor else 'Outdoor'} {time} (confidence: {conf:.2f})"
+
+        # 添加更詳細的診斷信息
+        diagnostics = lighting_info.get("diagnostics", {})
+        if diagnostics:
+            diag_str = "\nAdditional lighting diagnostics:"
+            for key, value in diagnostics.items():
+                diag_str += f"\n- {key}: {value}"
+            base_info += diag_str
+
+        return base_info
+
+    def _format_zones(self, zones: Dict) -> str:
+        """格式化功能區域以用於提示"""
+        if not zones:
+            return "No distinct functional zones identified"
+
+        formatted = ["Identified functional zones:"]
+        for zone_name, zone_data in zones.items():
+            desc = zone_data.get("description", "")
+            objects = zone_data.get("objects", [])
+
+            zone_str = f"- {zone_name}: {desc}"
+            if objects:
+                zone_str += f" (Contains: {', '.join(objects)})"
+
+            formatted.append(zone_str)
+
+        return "\n".join(formatted)
+
+    def _format_clip_results(self, clip_analysis: Dict) -> str:
+        """格式化CLIP分析結果以用於提示"""
+        if not clip_analysis or "error" in clip_analysis:
+            return "No CLIP analysis available"
+
+        parts = ["CLIP Analysis Results:"]
+
+        # 加上頂級場景
+        top_scene, confidence = clip_analysis.get("top_scene", ("unknown", 0))
+        parts.append(f"- Most likely scene: {top_scene} (confidence: {confidence:.2f})")
+
+        # 加上視角
+        viewpoint, vp_conf = clip_analysis.get("viewpoint", ("standard", 0))
+        parts.append(f"- Camera viewpoint: {viewpoint} (confidence: {vp_conf:.2f})")
+
+        # 加上物體組合
+        if "object_combinations" in clip_analysis:
+            combos = []
+            for combo, score in clip_analysis["object_combinations"][:3]:
+                combos.append(f"{combo} ({score:.2f})")
+            parts.append(f"- Object combinations: {', '.join(combos)}")
+
+        # 加上文化分析
+        if "cultural_analysis" in clip_analysis:
+            parts.append("- Cultural analysis:")
+            for culture_type, data in clip_analysis["cultural_analysis"].items():
+                best_desc = data.get("best_description", "")
+                desc_conf = data.get("confidence", 0)
+                parts.append(f"  * {culture_type}: {best_desc} ({desc_conf:.2f})")
+
+        return "\n".join(parts)
+
+    def _format_cultural_analysis(self, cultural_analysis: Dict) -> str:
+        """格式化文化分析結果"""
+        if not cultural_analysis:
+            return "No specific cultural elements detected"
+
+        parts = []
+        for culture_type, data in cultural_analysis.items():
+            best_desc = data.get("best_description", "")
+            desc_conf = data.get("confidence", 0)
+            parts.append(f"{culture_type}: {best_desc} (confidence: {desc_conf:.2f})")
+
+        return "\n".join(parts)

requirements.txt

@@ -9,3 +9,8 @@ gradio>=3.32.0
 git+https://github.com/openai/CLIP.git
 yt-dlp>=2023.3.4
 requests>=2.28.1
+transformers
+accelerate
+bitsandbytes
+sentencepiece
+huggingface_hub>=0.19.0

scene_analyzer.py

@@ -6,6 +6,7 @@ from spatial_analyzer import SpatialAnalyzer
 from scene_description import SceneDescriptor
 from enhance_scene_describer import EnhancedSceneDescriber
 from clip_analyzer import CLIPAnalyzer
+from llm_enhancer import LLMEnhancer
 from scene_type import SCENE_TYPES
 from object_categories import OBJECT_CATEGORIES
 
@@ -14,7 +15,7 @@ class SceneAnalyzer:
     Core class for scene analysis and understanding based on object detection results.
     Analyzes detected objects, their relationships, and infers the scene type.
     """
-    def __init__(self, class_names: Dict[int, str] = None):
+    def __init__(self, class_names: Dict[int, str] = None, use_llm: bool = True, llm_model_path: str = None):
         """
         Initialize the scene analyzer with optional class name mappings.
         Args:
@@ -40,6 +41,18 @@ class SceneAnalyzer:
             print("Scene analysis will proceed without CLIP. Install CLIP with 'pip install clip' for enhanced scene understanding.")
             self.use_clip = False
 
+        # 初始化LLM Model
+        self.use_llm = use_llm
+        if use_llm:
+            try:
+                # from llm_enhancer import LLMEnhancer
+                self.llm_enhancer = LLMEnhancer(model_path=llm_model_path)
+                print(f"LLM enhancer initialized successfully.")
+            except Exception as e:
+                print(f"Warning: Could not initialize LLM enhancer: {e}")
+                print("Scene analysis will proceed without LLM. Make sure required packages are installed.")
+                self.use_llm = False
+
     def generate_scene_description(self,
                              scene_type,
                              detected_objects,
@@ -106,8 +119,31 @@ class SceneAnalyzer:
         Returns:
             Dictionary with scene analysis results
         """
-        # If no result or no detections, return empty analysis
+        # If no result or no detections, handle with LLM if possible
         if detection_result is None or len(detection_result.boxes) == 0:
+            if self.use_llm and self.use_clip and detection_result is not None:
+                # 使用CLIP和LLM分析無物體檢測的情況
+                try:
+                    original_image = detection_result.orig_img
+                    clip_analysis = self.clip_analyzer.analyze_image(original_image)
+                    llm_description = self.llm_enhancer.handle_no_detection(clip_analysis)
+
+                    return {
+                        "scene_type": "llm_inferred",
+                        "confidence": clip_analysis.get("top_scene", ("unknown", 0))[1],
+                        "description": "No objects detected by standard detection.",
+                        "enhanced_description": llm_description,
+                        "objects_present": [],
+                        "object_count": 0,
+                        "regions": {},
+                        "possible_activities": [],
+                        "safety_concerns": [],
+                        "lighting_conditions": lighting_info or {"time_of_day": "unknown", "confidence": 0}
+                    }
+                except Exception as e:
+                    print(f"Error in LLM no-detection handling: {e}")
+
+            # 如果無法使用LLM/CLIP或處理失敗，返回原始的無檢測結果
             return {
                 "scene_type": "unknown",
                 "confidence": 0,
@@ -226,6 +262,53 @@ class SceneAnalyzer:
             functional_zones=functional_zones
         )
 
+        # 使用LLM進行增強處理
+        enhanced_description = None
+        llm_verification = None
+
+        if self.use_llm:
+            try:
+                # 準備用於LLM的場景數據
+                scene_data = {
+                    "original_description": scene_description,
+                    "scene_type": best_scene,
+                    "scene_name": self.SCENE_TYPES.get(best_scene, {}).get("name", "Unknown"),
+                    "detected_objects": detected_objects,
+                    "confidence": scene_confidence,
+                    "lighting_info": lighting_info,
+                    "functional_zones": functional_zones,
+                    "activities": activities,
+                    "safety_concerns": safety_concerns,
+                    "clip_analysis": clip_analysis
+                }
+
+                # 如果CLIP和YOLO結果之間存在顯著差異，使用LLM進行驗證
+                if self.use_clip and clip_analysis and "top_scene" in clip_analysis:
+                    clip_top_scene = clip_analysis["top_scene"][0]
+                    clip_confidence = clip_analysis["top_scene"][1]
+
+                    # 如果CLIP和YOLO的場景預測不同且都有較高的置信度，進行驗證
+                    if clip_top_scene != best_scene and clip_confidence > 0.4 and scene_confidence > 0.4:
+                        llm_verification = self.llm_enhancer.verify_detection(
+                            detected_objects,
+                            clip_analysis,
+                            best_scene,
+                            self.SCENE_TYPES.get(best_scene, {}).get("name", "Unknown"),
+                            scene_confidence
+                        )
+
+                        # 將驗證結果添加到場景數據中
+                        scene_data["verification_result"] = llm_verification.get("verification_text", "")
+
+                # 使用LLM生成增強描述
+                enhanced_description = self.llm_enhancer.enhance_description(scene_data)
+
+            except Exception as e:
+                print(f"Error in LLM enhancement: {e}")
+                import traceback
+                traceback.print_exc()
+                enhanced_description = None
+
         # Return comprehensive analysis
         result = {
             "scene_type": best_scene if scene_confidence >= scene_confidence_threshold else "unknown",
@@ -233,6 +316,7 @@ class SceneAnalyzer:
                         if scene_confidence >= scene_confidence_threshold else "Unknown Scene",
             "confidence": scene_confidence,
             "description": scene_description,
+            "enhanced_description": enhanced_description,  # 添加LLM增強的描述
             "objects_present": [
                 {"class_id": obj["class_id"],
                 "class_name": obj["class_name"],
@@ -248,6 +332,12 @@ class SceneAnalyzer:
             "lighting_conditions": lighting_info or {"time_of_day": "unknown", "confidence": 0}
         }
 
+        # 如果有LLM驗證結果，添加到輸出中
+        if llm_verification:
+            result["llm_verification"] = llm_verification.get("verification_text")
+            if llm_verification.get("has_errors", False):
+                result["detection_warnings"] = "LLM detected potential issues with object recognition"
+
         # 添加 CLIP 特定的結果
         if clip_analysis and "error" not in clip_analysis:
             result["clip_analysis"] = {

scene_type.py

@@ -282,13 +282,6 @@ SCENE_TYPES = {
         "minimum_required": 1,
         "description": "A traditional Asian temple complex with visitors and cultural elements"
     },
-    "european_plaza": {
-        "name": "European Plaza",
-        "required_objects": [0],  # person
-        "optional_objects": [1, 2, 4, 9, 24, 26, 67],  # bicycle, car, airplane, traffic light, backpack, handbag, cell phone
-        "minimum_required": 1,
-        "description": "A European-style city plaza with historic architecture and pedestrian activity"
-    },
 
     # specific time item
     "nighttime_street": {

style.py

@@ -289,13 +289,13 @@ class Style:
             padding: 15px !important;             /* 內邊距，讓文字和邊框有點空間 */
             border-radius: 8px !important;        /* 圓角 */
             margin: 10px 0 20px 0 !important;     /* 其他元素的間距，特別是上下的part */
-            display: block !important;            
-            width: 100% !important;               
-            box-sizing: border-box !important;    
+            display: block !important;
+            width: 100% !important;
+            box-sizing: border-box !important;
         }
 
         #scene_analysis_description_text p {
-            margin: 0 !important; 
+            margin: 0 !important;
             color: #2D3748 !important; /* 確保文字顏色 */
             font-family: Arial, sans-serif !important;
             font-size: 16px !important; /* 你可以調整文字大小 */
@@ -485,6 +485,37 @@ class Style:
             max-width: 100% !important;
         }
 
+        /* LLM 增強描述樣式 */
+        #llm_enhanced_description_text {
+            padding: 15px !important;
+            background-color: #ffffff !important;
+            border-radius: 8px !important;
+            border: 1px solid #e2e8f0 !important;
+            margin-bottom: 20px !important;
+            box-shadow: 0 1px 3px rgba(0,0,0,0.05) !important;
+            font-family: Arial, sans-serif !important;
+            line-height: 1.7 !important;
+            color: #2D3748 !important;
+            font-size: 16px !important;
+            width: 100% !important;
+            box-sizing: border-box !important;
+            min-height: 200px !important;
+        }
+
+        /* 原始描述折疊區域樣式 */
+        #original_scene_analysis_accordion {
+            margin-top: 10px !important;
+            margin-bottom: 20px !important;
+            background-color: #f8f9fa !important;
+            border-radius: 8px !important;
+            border: 1px solid #e2e8f0 !important;
+        }
+
+        /* 確保折疊區域內容與頁面樣式協調 */
+        #original_scene_analysis_accordion > div:nth-child(2) {
+            padding: 15px !important;
+        }
+
         /* 動畫效果, 增加互動感 */
         @keyframes fadeIn {
             from { opacity: 0; }