Update main.py

main.py

@@ -1,4 +1,4 @@
-# main.py (Corrected SyntaxError at line 424 - Now with Gemini 2.0 as primary)
+# main.py (Modified to add crawl4ai and adjust fetching logic)
 import os
 import re
 import logging
@@ -52,6 +52,15 @@ except ImportError:
     _gemini_available = False
     # logger will be defined later, log warning after logger setup
 
+# --- Crawl4AI (New Primary Web Scraper) ---
+try:
+    from crawl4ai import AsyncWebCrawler
+    _crawl4ai_available = True
+except ImportError:
+    AsyncWebCrawler = None
+    _crawl4ai_available = False
+    # logger will be defined later
+
 
 # --- Logging Setup ---
 logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO )
@@ -63,9 +72,14 @@ logging.getLogger('gunicorn.error').setLevel(logging.INFO)
 logging.getLogger('uvicorn').setLevel(logging.INFO)
 logging.getLogger('starlette').setLevel(logging.INFO)
 if _gemini_available: logging.getLogger("google.ai.generativelanguage").setLevel(logging.WARNING)
+# Reduce crawl4ai logging noise if needed
+if _crawl4ai_available:
+    logging.getLogger("crawl4ai").setLevel(logging.WARNING) # Or INFO for more detail
+    logging.getLogger("playwright").setLevel(logging.WARNING)
 logger = logging.getLogger(__name__)
 logger.info(f"Logging configured. Using BS4 parser: {DEFAULT_PARSER}")
 if not _gemini_available: logger.warning("google-generativeai library not found. Gemini functionality disabled.")
+if not _crawl4ai_available: logger.warning("crawl4ai library not found. Primary website scraping will be disabled.")
 
 
 # --- Global variable for PTB app ---
@@ -81,16 +95,16 @@ def get_secret(secret_name):
 
 TELEGRAM_TOKEN = get_secret('TELEGRAM_TOKEN')
 OPENROUTER_API_KEY = get_secret('OPENROUTER_API_KEY') # Now Fallback
-URLTOTEXT_API_KEY = get_secret('URLTOTEXT_API_KEY')
-SUPADATA_API_KEY = get_secret('SUPADATA_API_KEY')
-APIFY_API_TOKEN = get_secret('APIFY_API_TOKEN')
+URLTOTEXT_API_KEY = get_secret('URLTOTEXT_API_KEY') # Fallback Web 2
+SUPADATA_API_KEY = get_secret('SUPADATA_API_KEY') # Fallback YT 1
+APIFY_API_TOKEN = get_secret('APIFY_API_TOKEN') # Fallback YT 2
 WEBHOOK_SECRET = get_secret('WEBHOOK_SECRET')
-GEMINI_API_KEY = get_secret('GEMINI_API_KEY') # Now Primary
+GEMINI_API_KEY = get_secret('GEMINI_API_KEY') # Primary Summarizer
 
 # Models (User can still configure via env vars)
-OPENROUTER_MODEL = os.environ.get("OPENROUTER_MODEL", "deepseek/deepseek-chat-v3-0324:free") # Fallback Model
+OPENROUTER_MODEL = os.environ.get("OPENROUTER_MODEL", "deepseek/deepseek-chat-v3-0324:free") # Fallback Summarizer Model
 APIFY_ACTOR_ID = os.environ.get("APIFY_ACTOR_ID", "karamelo~youtube-transcripts")
-GEMINI_MODEL = os.environ.get("GEMINI_MODEL", "gemini-2.0-flash-001") # Primary Model
+GEMINI_MODEL = os.environ.get("GEMINI_MODEL", "gemini-2.0-flash-001") # Primary Summarizer Model
 
 if not TELEGRAM_TOKEN: logger.critical("❌ FATAL: TELEGRAM_TOKEN not found."); raise RuntimeError("Exiting: Telegram token missing.")
 if not GEMINI_API_KEY: logger.error("❌ ERROR: GEMINI_API_KEY not found. Primary summarization (Gemini) will fail.")
@@ -101,19 +115,29 @@ if not _gemini_available: logger.warning("⚠️ WARNING: google-generativeai li
 elif not GEMINI_API_KEY: logger.warning("⚠️ WARNING: GEMINI_API_KEY not found or empty. Gemini disabled.")
 
 _openrouter_fallback_enabled = bool(OPENROUTER_API_KEY)
-if not _openrouter_fallback_enabled: logger.warning("⚠️ WARNING: OPENROUTER_API_KEY not found. Fallback disabled.")
+if not _openrouter_fallback_enabled: logger.warning("⚠️ WARNING: OPENROUTER_API_KEY not found. Fallback summarization disabled.")
 
+_crawl4ai_primary_web_enabled = _crawl4ai_available
+if not _crawl4ai_primary_web_enabled: logger.warning("⚠️ WARNING: crawl4ai library not found. Primary web scraping disabled.")
+
+_bs4_fallback_web_enabled = True # Assumes bs4 is always available
+_urltotext_fallback_web_enabled = bool(URLTOTEXT_API_KEY)
+if not _urltotext_fallback_web_enabled: logger.info("ℹ️ INFO: URLTOTEXT_API_KEY not found. Secondary web fallback disabled.")
+
+# Fallback YT checks
+if not SUPADATA_API_KEY: logger.info("ℹ️ INFO: SUPADATA_API_KEY not found. First YT fallback disabled.")
+_apify_token_exists = bool(APIFY_API_TOKEN)
+if not _apify_token_exists: logger.info("ℹ️ INFO: APIFY_API_TOKEN not found. Second YT fallback disabled.")
 
-if not URLTOTEXT_API_KEY: pass
-if not SUPADATA_API_KEY: pass
-if not APIFY_API_TOKEN: pass
 if not WEBHOOK_SECRET: logger.info("Optional secret 'WEBHOOK_SECRET' not found. Webhook security disabled.")
 
 logger.info("Secret loading and configuration check finished.")
-logger.info(f"Using Gemini Model (Primary): {GEMINI_MODEL if _gemini_primary_enabled else 'DISABLED'}")
-logger.info(f"Using OpenRouter Model (Fallback): {OPENROUTER_MODEL if _openrouter_fallback_enabled else 'DISABLED'}")
+logger.info(f"Primary Web Scraper: {'Crawl4AI' if _crawl4ai_primary_web_enabled else 'DISABLED'}")
+logger.info(f"Fallback Web Scraper 1: {'BeautifulSoup' if _bs4_fallback_web_enabled else 'DISABLED'}")
+logger.info(f"Fallback Web Scraper 2: {'urltotext.com API' if _urltotext_fallback_web_enabled else 'DISABLED'}")
+logger.info(f"Using Gemini Model (Primary Summarizer): {GEMINI_MODEL if _gemini_primary_enabled else 'DISABLED'}")
+logger.info(f"Using OpenRouter Model (Fallback Summarizer): {OPENROUTER_MODEL if _openrouter_fallback_enabled else 'DISABLED'}")
 logger.info(f"Using Apify Actor (via REST): {APIFY_ACTOR_ID}")
-_apify_token_exists = bool(APIFY_API_TOKEN)
 
 
 if _gemini_primary_enabled:
@@ -143,26 +167,8 @@ def extract_youtube_id(url):
 
 
 # --- Content Fetching Functions ---
-# (fetch_url_content_for_scrape, get_transcript_via_supadata, get_transcript_via_apify, get_youtube_transcript, get_website_content, get_website_content_via_api remain the same as previous version)
-async def fetch_url_content_for_scrape(url: str, timeout: int = 25) -> Optional[str]:
-    headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Connection': 'keep-alive', 'DNT': '1', 'Upgrade-Insecure-Requests': '1' }
-    try:
-        async with httpx.AsyncClient(follow_redirects=True, timeout=timeout, headers=headers) as client:
-            logger.debug(f"[Web Scrape] Sending request to {url}")
-            response = await client.get(url)
-            logger.debug(f"[Web Scrape] Received response {response.status_code} from {url}")
-            response.raise_for_status()
-            content_type = response.headers.get('content-type', '').lower()
-            if 'html' not in content_type: logger.warning(f"[Web Scrape] Non-HTML content type from {url}: {content_type}"); return None
-            try: return response.text
-            except Exception as e: logger.error(f"[Web Scrape] Error decoding response for {url}: {e}"); return None
-    except httpx.HTTPStatusError as e: logger.error(f"[Web Scrape] HTTP error {e.response.status_code} fetching {url}: {e}")
-    except httpx.TimeoutException: logger.error(f"[Web Scrape] Timeout error fetching {url}")
-    except httpx.TooManyRedirects: logger.error(f"[Web Scrape] Too many redirects fetching {url}")
-    except httpx.RequestError as e: logger.error(f"[Web Scrape] Request error fetching {url}: {e}")
-    except Exception as e: logger.error(f"[Web Scrape] Unexpected error fetching {url}: {e}", exc_info=True)
-    return None
 
+# --- YouTube Fetching (Unchanged) ---
 async def get_transcript_via_supadata(video_id: str, api_key: str) -> Optional[str]:
     if not video_id: logger.error("[Supadata] No video_id provided"); return None
     if not api_key: logger.error("[Supadata] API key missing."); return None
@@ -278,55 +284,128 @@ async def get_youtube_transcript(video_id: str, video_url: str) -> Optional[str]
     if transcript_text is None: logger.error(f"All methods failed for YT transcript: {video_id}"); return None
     return transcript_text
 
-async def get_website_content(url: str) -> Optional[str]:
-    if not url: logger.error("get_website_content: No URL"); return None
-    logger.info(f"[Primary Web] Fetching website content for: {url}")
-    html_content = await fetch_url_content_for_scrape(url)
-    if not html_content: return None
+# --- Website Content Fetching ---
+
+# NEW: Primary Method using Crawl4AI
+async def get_website_content_via_crawl4ai(url: str) -> Optional[str]:
+    """Fetches website content using Crawl4AI. Returns Markdown."""
+    global _crawl4ai_primary_web_enabled
+    if not _crawl4ai_primary_web_enabled:
+        logger.warning("[Crawl4AI Primary] Called but disabled/unavailable.")
+        return None
+    if not url: logger.error("[Crawl4AI Primary] No URL provided"); return None
+
+    logger.info(f"[Crawl4AI Primary] Attempting fetch for: {url}")
+    try:
+        # Using async with for proper resource cleanup
+        async with AsyncWebCrawler(headless=True) as crawler: # Headless is generally preferred for server environments
+            # Timeout can be added here if needed: crawler_params={"timeout": 60000} # milliseconds
+            result = await crawler.arun(url=url)
+            if result and result.markdown:
+                logger.info(f"[Crawl4AI Primary] Success via Crawl4AI for {url}. Markdown length: {len(result.markdown)}")
+                return result.markdown.strip()
+            elif result and not result.markdown:
+                logger.warning(f"[Crawl4AI Primary] Crawl4AI ran but returned empty markdown for {url}.")
+                return None
+            else:
+                logger.warning(f"[Crawl4AI Primary] Crawl4AI returned no result object for {url}.")
+                return None
+    except ImportError:
+        logger.error("[Crawl4AI Primary] Import Error - library might be missing.")
+        _crawl4ai_primary_web_enabled = False # Disable if import fails at runtime
+        return None
+    except Exception as e:
+        # Catch potential Playwright errors, timeouts, or other issues
+        logger.error(f"[Crawl4AI Primary] Error during Crawl4AI execution for {url}: {e}", exc_info=True)
+        return None
+
+# HELPER: Used by Fallback 1 (BS4) - no changes needed here
+async def fetch_url_content_for_scrape(url: str, timeout: int = 25) -> Optional[str]:
+    headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Connection': 'keep-alive', 'DNT': '1', 'Upgrade-Insecure-Requests': '1' }
+    try:
+        async with httpx.AsyncClient(follow_redirects=True, timeout=timeout, headers=headers) as client:
+            logger.debug(f"[Web Scrape Helper] Sending request to {url}")
+            response = await client.get(url)
+            logger.debug(f"[Web Scrape Helper] Received response {response.status_code} from {url}")
+            response.raise_for_status()
+            content_type = response.headers.get('content-type', '').lower()
+            if 'html' not in content_type: logger.warning(f"[Web Scrape Helper] Non-HTML content type from {url}: {content_type}"); return None
+            try: return response.text
+            except Exception as e: logger.error(f"[Web Scrape Helper] Error decoding response for {url}: {e}"); return None
+    except httpx.HTTPStatusError as e: logger.error(f"[Web Scrape Helper] HTTP error {e.response.status_code} fetching {url}: {e}")
+    except httpx.TimeoutException: logger.error(f"[Web Scrape Helper] Timeout error fetching {url}")
+    except httpx.TooManyRedirects: logger.error(f"[Web Scrape Helper] Too many redirects fetching {url}")
+    except httpx.RequestError as e: logger.error(f"[Web Scrape Helper] Request error fetching {url}: {e}")
+    except Exception as e: logger.error(f"[Web Scrape Helper] Unexpected error fetching {url}: {e}", exc_info=True)
+    return None
+
+# Fallback 1: Direct BS4 Scraping (Renamed original function)
+async def get_website_content_via_bs4(url: str) -> Optional[str]:
+    """Fetches and extracts text content using BeautifulSoup (Fallback 1)."""
+    global _bs4_fallback_web_enabled
+    if not _bs4_fallback_web_enabled:
+        logger.warning("[BS4 Fallback] Called but disabled.") # Should not happen unless manually disabled
+        return None
+    if not url: logger.error("[BS4 Fallback] No URL provided"); return None
+    logger.info(f"[BS4 Fallback] Fetching website content for: {url}")
+    html_content = await fetch_url_content_for_scrape(url) # Use the helper
+    if not html_content:
+        logger.warning(f"[BS4 Fallback] fetch_url_content_for_scrape failed for {url}")
+        return None
     try:
         def parse_html(content):
             soup = BeautifulSoup(content, DEFAULT_PARSER)
+            # Keep the existing cleaning logic
             for element in soup(["script", "style", "header", "footer", "nav", "aside", "form", "button", "input", "iframe", "img", "svg", "link", "meta", "noscript", "figure"]): element.extract()
             main_content = soup.find('main') or soup.find('article') or soup.find(id='content') or soup.find(class_='content') or soup.find(id='main-content') or soup.find(class_='main-content') or soup.find(role='main')
             target_element = main_content if main_content else soup.body
-            if not target_element: logger.warning(f"[Primary Web] Could not find body/main for parsing {url}"); return None
+            if not target_element: logger.warning(f"[BS4 Fallback] Could not find body/main for parsing {url}"); return None
             lines = [line.strip() for line in target_element.get_text(separator='\n', strip=True).splitlines() if line.strip()]
             text = " ".join(lines)
-            if not text: logger.warning(f"[Primary Web] Extracted text empty after clean for {url}"); return None
+            if not text: logger.warning(f"[BS4 Fallback] Extracted text empty after clean for {url}"); return None
             return text
+
         text_content = await asyncio.to_thread(parse_html, html_content)
-        if text_content: logger.info(f"[Primary Web] Success scrape for {url} (final len: {len(text_content)})"); return text_content
-        else: return None
-    except Exception as e: logger.error(f"[Primary Web] Error scraping/parsing {url}: {e}", exc_info=True); return None
+        if text_content: logger.info(f"[BS4 Fallback] Success scrape for {url} (final len: {len(text_content)})"); return text_content
+        else:
+             logger.warning(f"[BS4 Fallback] parse_html returned None for {url}")
+             return None
+    except Exception as e: logger.error(f"[BS4 Fallback] Error scraping/parsing {url}: {e}", exc_info=True); return None
 
+# Fallback 2: urltotext.com API (Unchanged function)
 async def get_website_content_via_api(url: str, api_key: str) -> Optional[str]:
-    if not url: logger.error("[Fallback Web API] No URL"); return None
-    if not api_key: logger.error("[Fallback Web API] urltotext.com API key missing."); return None
-    logger.info(f"[Fallback Web API] Attempting fetch for: {url} using urltotext.com API")
+    """Fetches website content using urltotext.com API (Fallback 2)."""
+    global _urltotext_fallback_web_enabled
+    if not _urltotext_fallback_web_enabled:
+        logger.warning("[urltotext API Fallback] Called but disabled (no API key).")
+        return None
+    if not url: logger.error("[urltotext API Fallback] No URL"); return None
+    if not api_key: logger.error("[urltotext API Fallback] urltotext.com API key missing."); return None # Redundant check but safe
+    logger.info(f"[urltotext API Fallback] Attempting fetch for: {url} using urltotext.com API")
     api_endpoint = "https://urltotext.com/api/v1/urltotext/"
     payload = { "url": url, "output_format": "text", "extract_main_content": True, "render_javascript": True, "residential_proxy": False }
     headers = { "Authorization": f"Token {api_key}", "Content-Type": "application/json" }
     try:
         async with httpx.AsyncClient(timeout=45.0) as client:
-            logger.debug(f"[Fallback Web API] Sending request to urltotext.com API for {url}")
+            logger.debug(f"[urltotext API Fallback] Sending request to urltotext.com API for {url}")
             response = await client.post(api_endpoint, headers=headers, json=payload)
-            logger.debug(f"[Fallback Web API] Received status {response.status_code} from urltotext.com API for {url}")
+            logger.debug(f"[urltotext API Fallback] Received status {response.status_code} from urltotext.com API for {url}")
             if response.status_code == 200:
                 try:
                     data = response.json()
                     content = data.get("data", {}).get("content"); credits = data.get("credits_used", "N/A"); warning = data.get("data", {}).get("warning")
-                    if warning: logger.warning(f"[Fallback Web API] urltotext.com API Warning for {url}: {warning}")
-                    if content: logger.info(f"[Fallback Web API] Success via urltotext.com API for {url}. Len: {len(content)}. Credits: {credits}"); return content.strip()
-                    else: logger.warning(f"[Fallback Web API] urltotext.com API success but content empty for {url}. Resp: {data}"); return None
-                except json.JSONDecodeError: logger.error(f"[Fallback Web API] Failed JSON decode urltotext.com for {url}. Resp:{response.text[:500]}"); return None
-                except Exception as e: logger.error(f"[Fallback Web API] Error processing urltotext.com success response for {url}: {e}", exc_info=True); return None
-            elif response.status_code in [400, 401, 402, 403, 422, 500]: logger.error(f"[Fallback Web API] Error {response.status_code} from urltotext.com API for {url}. Resp:{response.text[:200]}"); return None
-            else: logger.error(f"[Fallback Web API] Unexpected status {response.status_code} from urltotext.com API for {url}. Resp:{response.text[:200]}"); return None
-    except httpx.TimeoutException: logger.error(f"[Fallback Web API] Timeout connecting to urltotext.com API for {url}"); return None
-    except httpx.RequestError as e: logger.error(f"[Fallback Web API] Request error connecting to urltotext.com API for {url}: {e}"); return None
-    except Exception as e: logger.error(f"[Fallback Web API] Unexpected error during urltotext.com API call for {url}: {e}", exc_info=True); return None
-
-# --- Summarization Functions ---
+                    if warning: logger.warning(f"[urltotext API Fallback] urltotext.com API Warning for {url}: {warning}")
+                    if content: logger.info(f"[urltotext API Fallback] Success via urltotext.com API for {url}. Len: {len(content)}. Credits: {credits}"); return content.strip()
+                    else: logger.warning(f"[urltotext API Fallback] urltotext.com API success but content empty for {url}. Resp: {data}"); return None
+                except json.JSONDecodeError: logger.error(f"[urltotext API Fallback] Failed JSON decode urltotext.com for {url}. Resp:{response.text[:500]}"); return None
+                except Exception as e: logger.error(f"[urltotext API Fallback] Error processing urltotext.com success response for {url}: {e}", exc_info=True); return None
+            elif response.status_code in [400, 401, 402, 403, 422, 500]: logger.error(f"[urltotext API Fallback] Error {response.status_code} from urltotext.com API for {url}. Resp:{response.text[:200]}"); return None
+            else: logger.error(f"[urltotext API Fallback] Unexpected status {response.status_code} from urltotext.com API for {url}. Resp:{response.text[:200]}"); return None
+    except httpx.TimeoutException: logger.error(f"[urltotext API Fallback] Timeout connecting to urltotext.com API for {url}"); return None
+    except httpx.RequestError as e: logger.error(f"[urltotext API Fallback] Request error connecting to urltotext.com API for {url}: {e}"); return None
+    except Exception as e: logger.error(f"[urltotext API Fallback] Unexpected error during urltotext.com API call for {url}: {e}", exc_info=True); return None
+
+# --- Summarization Functions (Unchanged) ---
 
 async def _call_gemini(text: str, summary_type: str) -> Tuple[Optional[str], Optional[str]]:
     """Internal function to call Gemini API. Returns (summary, error_message)."""
@@ -336,7 +415,7 @@ async def _call_gemini(text: str, summary_type: str) -> Tuple[Optional[str], Opt
         return None, "Error: Primary AI service (Gemini) not configured/available."
     logger.info(f"[Gemini Primary] Generating {summary_type} summary using {GEMINI_MODEL}. Input length: {len(text)}")
 
-    # Define prompts
+    # Define prompts (Keep existing prompts)
     if summary_type == "paragraph":
         prompt = ("You are an AI model designed to provide concise summaries using British English spellings. Your output MUST be:\n"
                   "• Clear and simple language suitable for someone unfamiliar with the topic.\n"
@@ -412,9 +491,6 @@ async def _call_gemini(text: str, summary_type: str) -> Tuple[Optional[str], Opt
 
         if summary:
             logger.info(f"[Gemini Primary] Success generating summary. Output len: {len(summary)}");
-            # Escape Markdown for Telegram only if necessary (Removed escaping as it might conflict with plain heading)
-            # escaped_summary = summary.strip().replace('_', r'\_').replace('*', r'\*').replace('[', r'\[').replace('`', r'\`')
-            # Let's see if the AI respects the instruction without further escaping. If not, we might need selective escaping.
             return summary.strip(), None
         else:
             finish_reason = response.candidates[0].finish_reason if response.candidates else 'N/A'
@@ -433,7 +509,7 @@ async def _call_openrouter(text: str, summary_type: str) -> Tuple[Optional[str],
         return None, "Error: Fallback AI service (OpenRouter) not configured/available."
     logger.info(f"[OpenRouter Fallback] Generating {summary_type} summary using {OPENROUTER_MODEL}. Input length: {len(text)}")
 
-    # Define prompts
+    # Define prompts (Keep existing prompts)
     if summary_type == "paragraph":
         prompt = ("You are an AI model designed to provide concise summaries using British English spellings. Your output MUST be:\n"
                   "• Clear and simple language suitable for someone unfamiliar with the topic.\n"
@@ -493,8 +569,6 @@ async def _call_openrouter(text: str, summary_type: str) -> Tuple[Optional[str],
                             summary = message.get("content")
                             if summary:
                                 logger.info(f"[OpenRouter Fallback] Success via OpenRouter. Output len: {len(summary)}")
-                                # Escape Markdown for Telegram only if necessary (Removed escaping)
-                                # escaped_summary = summary.strip().replace('_', r'\_').replace('*', r'\*').replace('[', r'\[').replace('`', r'\`')
                                 return summary.strip(), None
                             else:
                                 logger.warning(f"[OpenRouter Fallback] OpenRouter success but content empty. Resp: {data}")
@@ -585,14 +659,21 @@ async def generate_summary(text: str, summary_type: str) -> str:
     return "Sorry, an unknown error occurred during summary generation after trying all available models."
 
 
-# (process_summary_task, handlers, setup, lifespan, routes, etc. remain the same)
+# --- Main Task Processing (Modified Web Fetching Logic) ---
 async def process_summary_task( user_id: int, chat_id: int, message_id_to_edit: Optional[int], url: str, summary_type: str, bot_token: str ) -> None:
     task_id = f"{user_id}-{message_id_to_edit or 'new'}"; logger.info(f"[Task {task_id}] Starting processing for URL: {url}")
     background_request: Optional[BaseRequest] = None; bot: Optional[Bot] = None
     try: background_request = HTTPXRequest( connect_timeout=15.0, read_timeout=60.0, write_timeout=60.0, pool_timeout=60.0 ); bot = Bot(token=bot_token, request=background_request)
     except Exception as e: logger.critical(f"[Task {task_id}] Failed to create background bot: {e}", exc_info=True); return
-    content = None; user_feedback_message = None; success = False; status_message_id = message_id_to_edit; message_to_delete_later_id : Optional[int] = None
+
+    content: Optional[str] = None
+    user_feedback_message: Optional[str] = None
+    success = False
+    status_message_id = message_id_to_edit
+    message_to_delete_later_id : Optional[int] = None
+
     try:
+        # Send initial "Processing..." message (or edit existing)
         processing_message_text = f"Got it! Generating '{summary_type}' summary for:\n`{url}`\n\nThis might take a moment..."
         if status_message_id:
             try: await retry_bot_operation( bot.edit_message_text, chat_id=chat_id, message_id=status_message_id, text=processing_message_text, parse_mode=ParseMode.MARKDOWN, reply_markup=None ); logger.debug(f"[Task {task_id}] Edited message {status_message_id} to 'Processing'")
@@ -603,67 +684,132 @@ async def process_summary_task( user_id: int, chat_id: int, message_id_to_edit:
                  if status_message: message_to_delete_later_id = status_message.message_id; logger.debug(f"[Task {task_id}] Sent new status message {message_to_delete_later_id}")
                  else: raise RuntimeError("Failed to send status message after retries.")
              except Exception as e: logger.error(f"[Task {task_id}] CRITICAL: Failed to send new status message: {e}"); raise
+
         try:
             await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
-            is_youtube = is_youtube_url(url); logger.debug(f"[Task {task_id}] URL type: {'YouTube' if is_youtube else 'Website'}")
+            is_youtube = is_youtube_url(url)
+            logger.debug(f"[Task {task_id}] URL type: {'YouTube' if is_youtube else 'Website'}")
+
+            # --- YouTube Processing (Unchanged) ---
             if is_youtube:
                 video_id = extract_youtube_id(url)
-                if video_id: content = await get_youtube_transcript(video_id, url)
-                else: user_feedback_message = "Sorry, I couldn't understand that YouTube URL format."
-                if not content and not user_feedback_message: user_feedback_message = "Sorry, I couldn't get the transcript for that YouTube video using any available method (unavailable/private/no captions?)."
+                if video_id:
+                    content = await get_youtube_transcript(video_id, url) # Tries lib -> Supadata -> Apify
+                else:
+                    user_feedback_message = "Sorry, I couldn't understand that YouTube URL format."
+
+                if not content and not user_feedback_message:
+                    user_feedback_message = "Sorry, I couldn't get the transcript for that YouTube video using any available method (unavailable/private/no captions?)."
+
+            # --- Website Processing (NEW Logic) ---
             else:
-                content = await get_website_content(url)
+                # Method 1: Crawl4AI (Primary)
+                if _crawl4ai_primary_web_enabled:
+                    logger.info(f"[Task {task_id}] Trying primary web method: Crawl4AI")
+                    await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
+                    content = await get_website_content_via_crawl4ai(url)
+                    if content:
+                        logger.info(f"[Task {task_id}] Success via Crawl4AI for {url} (len: {len(content)})")
+                    else:
+                        logger.warning(f"[Task {task_id}] Crawl4AI failed or returned empty for {url}.")
+                else:
+                    logger.warning(f"[Task {task_id}] Crawl4AI is disabled. Skipping.")
+
+                # Method 2: BeautifulSoup (Fallback 1)
+                if not content and _bs4_fallback_web_enabled:
+                    logger.warning(f"[Task {task_id}] Trying fallback web method 1: BeautifulSoup")
+                    await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
+                    content = await get_website_content_via_bs4(url) # Use the renamed function
+                    if content:
+                        logger.info(f"[Task {task_id}] Success via BS4 scrape for {url} (len: {len(content)})")
+                    else:
+                        logger.warning(f"[Task {task_id}] BS4 scrape failed or returned empty for {url}.")
+
+                # Method 3: urltotext.com API (Fallback 2)
+                if not content and _urltotext_fallback_web_enabled:
+                    logger.warning(f"[Task {task_id}] Trying fallback web method 2: urltotext.com API")
+                    await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
+                    content = await get_website_content_via_api(url, URLTOTEXT_API_KEY) # API key checked inside function
+                    if content:
+                         logger.info(f"[Task {task_id}] Success via urltotext.com API for {url} (len: {len(content)})")
+                    else:
+                         logger.warning(f"[Task {task_id}] urltotext.com API failed or returned empty for {url}.")
+
+                # Final check if any website method succeeded
                 if not content:
-                    logger.warning(f"[Task {task_id}] Primary web scrape failed for {url}. Trying fallback API.")
-                    global URLTOTEXT_API_KEY
-                    if URLTOTEXT_API_KEY:
-                         await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
-                         content = await get_website_content_via_api(url, URLTOTEXT_API_KEY)
-                         if not content: user_feedback_message = "Sorry, I couldn't fetch content from that website using either method (blocked/inaccessible/empty?)."
-                    else: user_feedback_message = "Sorry, I couldn't fetch content from that website (blocked/inaccessible/empty?). The fallback method is not configured."
+                    methods_tried = []
+                    if _crawl4ai_primary_web_enabled: methods_tried.append("Crawl4AI")
+                    if _bs4_fallback_web_enabled: methods_tried.append("BS4")
+                    if _urltotext_fallback_web_enabled: methods_tried.append("API")
+                    tried_str = ", ".join(methods_tried) if methods_tried else "configured methods"
+                    user_feedback_message = f"Sorry, I couldn't fetch content from that website using any available method ({tried_str}). It might be blocked, inaccessible, or empty."
+
+            # --- Summarization ---
             if content:
                 logger.info(f"[Task {task_id}] Content fetched (len:{len(content)}). Generating summary.")
                 await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
-                final_summary = await generate_summary(content, summary_type) # This now calls Gemini first, then OpenRouter
-                if final_summary.startswith("Error:") or final_summary.startswith("Sorry,"): user_feedback_message = final_summary; logger.warning(f"[Task {task_id}] Summary generation failed: {final_summary}")
+                # NOTE: `content` might be Markdown (from Crawl4AI) or plain text (from others).
+                # The LLM prompts should handle this reasonably well.
+                final_summary = await generate_summary(content, summary_type) # Calls Gemini -> OpenRouter
+
+                if final_summary.startswith("Error:") or final_summary.startswith("Sorry,"):
+                    user_feedback_message = final_summary # Pass AI error message to user
+                    logger.warning(f"[Task {task_id}] Summary generation failed: {final_summary}")
                 else:
                     max_length = 4096; summary_parts = [final_summary[i:i+max_length] for i in range(0, len(final_summary), max_length)]
-                    # Sending potentially unescaped summary directly - assuming AI follows instructions
-                    # ParseMode.MARKDOWN might interpret unintended things, test needed. Consider ParseMode=None if issues arise.
+                    # Send summary without explicit Markdown parsing, assuming LLM followed instructions
+                    # for plain headings and standard bullet points. Using parse_mode=None.
                     await retry_bot_operation( bot.send_message, chat_id=chat_id, text=summary_parts[0], parse_mode=None, link_preview_options={'is_disabled': True} )
-                    for part in summary_parts[1:]: await asyncio.sleep(0.5); await retry_bot_operation( bot.send_message, chat_id=chat_id, text=part, parse_mode=None, link_preview_options={'is_disabled': True} )
-                    success = True; logger.info(f"[Task {task_id}] Successfully sent summary ({len(summary_parts)} parts)."); user_feedback_message = None
-            elif user_feedback_message: logger.warning(f"[Task {task_id}] Sending failure feedback: {user_feedback_message}"); await retry_bot_operation( bot.send_message, chat_id=chat_id, text=user_feedback_message, link_preview_options={'is_disabled': True} )
+                    for part in summary_parts[1:]:
+                        await asyncio.sleep(0.5)
+                        await retry_bot_operation( bot.send_message, chat_id=chat_id, text=part, parse_mode=None, link_preview_options={'is_disabled': True} )
+                    success = True
+                    logger.info(f"[Task {task_id}] Successfully sent summary ({len(summary_parts)} parts).")
+                    user_feedback_message = None # Clear any previous fetch error if summary succeeded
+
+            # --- Send Feedback if Fetching Failed ---
+            elif user_feedback_message:
+                 logger.warning(f"[Task {task_id}] Sending failure feedback: {user_feedback_message}")
+                 await retry_bot_operation( bot.send_message, chat_id=chat_id, text=user_feedback_message, link_preview_options={'is_disabled': True} )
+
         except Exception as e:
-            logger.error(f"[Task {task_id}] Unexpected error during processing: {e}", exc_info=True); user_feedback_message = "Oops! Something went really wrong. Please try again later."
+            logger.error(f"[Task {task_id}] Unexpected error during processing: {e}", exc_info=True)
+            user_feedback_message = "Oops! Something went really wrong while processing your request. Please try again later."
             try: await retry_bot_operation( bot.send_message, chat_id=chat_id, text=user_feedback_message )
             except Exception: logger.error(f"[Task {task_id}] Failed to send unexpected error feedback.")
+
     except Exception as outer_e:
         logger.critical(f"[Task {task_id}] Critical outer error: {outer_e}", exc_info=True)
         try:
-             if bot: await retry_bot_operation( bot.send_message, chat_id=chat_id, text="❌ Critical internal error occurred." )
+             if bot: await retry_bot_operation( bot.send_message, chat_id=chat_id, text="❌ Critical internal error occurred processing your request." )
         except Exception: logger.exception(f"[Task {task_id}] Failed even to send critical error message.")
+
     finally:
+        # Delete the "Processing..." or button message
         delete_target_id = message_to_delete_later_id if message_to_delete_later_id else status_message_id
         if delete_target_id and bot:
             try: await retry_bot_operation(bot.delete_message, chat_id=chat_id, message_id=delete_target_id); logger.debug(f"[Task {task_id}] Deleted status/button message {delete_target_id}")
             except Exception as del_e: logger.warning(f"[Task {task_id}] Failed to delete status/button message {delete_target_id}: {del_e}")
+
+        # Close the background bot's HTTP client
         if background_request and hasattr(background_request, '_client') and background_request._client:
              try: await background_request._client.aclose(); logger.debug(f"[Task {task_id}] Background bot's HTTPX client closed.")
              except Exception as e: logger.warning(f"[Task {task_id}] Error closing background bot's client: {e}")
         logger.info(f"[Task {task_id}] Task completed. Success: {success}")
 
+
+# --- Bot Handlers (Unchanged) ---
 async def start(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     user = update.effective_user; mention = user.mention_html()
     if not user or not update.message: return
     logger.info(f"User {user.id} used /start.")
-    await update.message.reply_html( f"👋 Hello {mention}! I can summarise YouTube links or website URLs.\n\nJust send me a link anytime!" ) # Changed 'summarize'
+    await update.message.reply_html( f"👋 Hello {mention}! I can summarise YouTube links or website URLs.\n\nJust send me a link anytime!" )
 
 async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     user = update.effective_user
     if not user or not update.message: return
     logger.info(f"User {user.id} used /help.")
-    help_text = ( "🔍 How to use this bot:\n\n" "1. Send me any YouTube video link or website URL.\n" "2. I'll ask you how you want it summarised (paragraph or points).\n" # Changed 'summarized'
+    help_text = ( "🔍 How to use this bot:\n\n" "1. Send me any YouTube video link or website URL.\n" "2. I'll ask you how you want it summarised (paragraph or points).\n"
                   "3. Click the button for your choice.\n" "4. Wait for the summary!\n\n" "I'll try multiple methods to get content if the first one fails (especially for YouTube transcripts).\n\n" "Commands:\n" "`/start` - Display welcome message\n" "`/help` - Show this help message" )
     await update.message.reply_text(help_text, parse_mode=ParseMode.MARKDOWN)
 
@@ -671,67 +817,111 @@ async def handle_potential_url(update: Update, context: ContextTypes.DEFAULT_TYP
     if not update.message or not update.message.text: return
     url = update.message.text.strip(); user = update.effective_user
     if not user: return
-    if not (url.startswith('http://') or url.startswith('https://')) or '.' not in url[8:]: logger.debug(f"Ignoring non-URL from {user.id}: {url}"); return
+    # Basic URL validation
+    if not (url.startswith('http://') or url.startswith('https://')) or '.' not in url[8:]:
+        logger.debug(f"Ignoring non-URL message from {user.id}")
+        # Optional: Reply if it looks like they tried to send something else?
+        # await update.message.reply_text("Please send a valid website URL (starting with http:// or https://) or a YouTube link.")
+        return
     logger.info(f"User {user.id} sent potential URL: {url}")
-    context.user_data['url_to_summarize'] = url; context.user_data['original_message_id'] = update.message.message_id
+    context.user_data['url_to_summarize'] = url
+    context.user_data['original_message_id'] = update.message.message_id # Store original message ID if needed later
     keyboard = [[ InlineKeyboardButton("Paragraph Summary", callback_data="paragraph"), InlineKeyboardButton("Points Summary", callback_data="points") ]]
     reply_markup = InlineKeyboardMarkup(keyboard)
-    await update.message.reply_text( f"Okay, I see this link:\n{url}\n\nHow would you like it summarised?", reply_markup=reply_markup, disable_web_page_preview=True ) # Changed 'summarized'
+    await update.message.reply_text( f"Okay, I see this link:\n{url}\n\nHow would you like it summarised?", reply_markup=reply_markup, disable_web_page_preview=True )
 
 async def handle_summary_type_callback(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     query = update.callback_query
     if not query or not query.message or not query.from_user: logger.warning("Callback query missing data."); return
     user = query.from_user; summary_type = query.data; query_id = query.id
     try: await query.answer(); logger.debug(f"Ack callback {query_id} from {user.id}")
-    except Exception as e: logger.error(f"Error answering callback {query_id}: {e}", exc_info=True)
-    url = context.user_data.get('url_to_summarize'); message_id_to_edit = query.message.message_id
+    except Exception as e: logger.error(f"Error answering callback {query_id}: {e}", exc_info=True) # Log but continue
+
+    url = context.user_data.get('url_to_summarize')
+    message_id_to_edit = query.message.message_id # The message with the buttons
     logger.info(f"User {user.id} chose '{summary_type}' for msg {message_id_to_edit}. URL in context: {'Yes' if url else 'No'}")
+
     if not url:
-        logger.warning(f"No URL in context for user {user.id} (cb {query_id}).")
+        logger.warning(f"No URL in context for user {user.id} (cb {query_id}). Might be an old button.")
         try: await query.edit_message_text(text="Sorry, I couldn't find the URL associated with this request. Please send the link again.")
         except Exception as e:
-            logger.error(f"Failed edit 'URL not found' msg: {e}")
-            try: await context.bot.send_message(chat_id=user.id, text="Sorry, context lost. Send link again.")
-            except Exception: pass
+            logger.error(f"Failed to edit 'URL not found' message: {e}")
+            # Attempt to send a new message as a fallback
+            try: await context.bot.send_message(chat_id=user.id, text="Sorry, the context for your previous request seems to have expired. Please send the link again.")
+            except Exception as send_e: logger.error(f"Failed even to send new message about lost context: {send_e}")
         return
 
-    context.user_data.pop('url_to_summarize', None); context.user_data.pop('original_message_id', None); logger.debug(f"Cleared URL context for user {user.id}")
+    # Clear context *after* checking it exists
+    context.user_data.pop('url_to_summarize', None)
+    context.user_data.pop('original_message_id', None) # Clear original ID too
+    logger.debug(f"Cleared URL context for user {user.id}")
 
-    global TELEGRAM_TOKEN, GEMINI_API_KEY, OPENROUTER_API_KEY, _gemini_primary_enabled, _openrouter_fallback_enabled
+    # Check critical configurations before scheduling task
+    global TELEGRAM_TOKEN, _gemini_primary_enabled, _openrouter_fallback_enabled
     if not TELEGRAM_TOKEN:
-        logger.critical("TG TOKEN missing!")
-        try: await query.edit_message_text(text="❌ Bot config error.")
-        except Exception: pass
+        logger.critical("TELEGRAM_TOKEN missing! Cannot schedule task.")
+        try: await query.edit_message_text(text="❌ Bot configuration error (Token). Task cannot be started.")
+        except Exception: pass # Ignore if edit fails
         return
     if not _gemini_primary_enabled and not _openrouter_fallback_enabled:
-        logger.critical("Neither Gemini nor OpenRouter API keys are configured/valid!")
-        try: await query.edit_message_text(text="❌ AI config error: No models available.")
-        except Exception: pass
+        logger.critical("Neither Gemini nor OpenRouter API keys are configured/valid! Cannot schedule task.")
+        try: await query.edit_message_text(text="❌ AI configuration error: No summarization models available. Task cannot be started.")
+        except Exception: pass # Ignore if edit fails
         return
     elif not _gemini_primary_enabled:
-         logger.warning("Primary AI (Gemini) is unavailable, will rely on fallback.")
-         # No need to inform user unless fallback also fails later
+         logger.warning("Primary AI (Gemini) is unavailable, will rely on fallback for summarization.")
+         # User will be notified by generate_summary if fallback also fails
     elif not _openrouter_fallback_enabled:
-         logger.warning("Fallback AI (OpenRouter) is unavailable.")
-         # No need to inform user unless primary fails later
+         logger.warning("Fallback AI (OpenRouter) is unavailable for summarization.")
+         # User will be notified by generate_summary if primary fails
 
+    # Schedule the background task
     logger.info(f"Scheduling task for user {user.id}, chat {query.message.chat_id}, msg {message_id_to_edit}")
-    asyncio.create_task( process_summary_task( user_id=user.id, chat_id=query.message.chat_id, message_id_to_edit=message_id_to_edit, url=url, summary_type=summary_type, bot_token=TELEGRAM_TOKEN ), name=f"SummaryTask-{user.id}-{message_id_to_edit}" )
+    asyncio.create_task(
+        process_summary_task(
+            user_id=user.id,
+            chat_id=query.message.chat_id,
+            message_id_to_edit=message_id_to_edit, # Pass the button message ID to edit/delete
+            url=url,
+            summary_type=summary_type,
+            bot_token=TELEGRAM_TOKEN
+        ),
+        name=f"SummaryTask-{user.id}-{message_id_to_edit}"
+    )
+    # Don't edit the message here; the task will handle it immediately.
 
 async def error_handler(update: object, context: ContextTypes.DEFAULT_TYPE) -> None:
-    ignore_errors = (AttributeError, )
-    if isinstance(context.error, ignore_errors) and "object has no attribute" in str(context.error): logger.debug(f"Ignoring known/handled error in error_handler: {context.error}"); return
+    # Ignore specific, known errors if necessary
+    ignore_errors = (AttributeError, ) # Example
+    if isinstance(context.error, ignore_errors) and "object has no attribute" in str(context.error):
+        logger.debug(f"Ignoring known/handled error in error_handler: {context.error}")
+        return
+
     logger.error("Exception while handling an update:", exc_info=context.error)
+    # Optionally, try to inform the user if it's a direct message context
+    # if isinstance(update, Update) and update.effective_chat:
+    #     try: await context.bot.send_message(chat_id=update.effective_chat.id, text="An internal error occurred.")
+    #     except Exception: logger.error("Failed to send error message to user.")
 
+# --- Application Setup (Unchanged) ---
 async def setup_bot_config() -> Application:
     logger.info("Configuring Telegram Application..."); global TELEGRAM_TOKEN
     if not TELEGRAM_TOKEN: raise ValueError("TELEGRAM_TOKEN missing.")
+    # Configure HTTPX client for PTB
     custom_request = HTTPXRequest( connect_timeout=10.0, read_timeout=30.0, write_timeout=30.0, pool_timeout=60.0 )
-    application = Application.builder().token(TELEGRAM_TOKEN).request(custom_request).build()
-    application.add_handler(CommandHandler("start", start)); application.add_handler(CommandHandler("help", help_command))
-    application.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND, handle_potential_url)); application.add_handler(CallbackQueryHandler(handle_summary_type_callback))
-    application.add_error_handler(error_handler); logger.info("Telegram application handlers configured."); return application
-
+    application = ( Application.builder() .token(TELEGRAM_TOKEN) .request(custom_request) .build() )
+    # Add handlers
+    application.add_handler(CommandHandler("start", start))
+    application.add_handler(CommandHandler("help", help_command))
+    # Message handler for potential URLs (non-command text)
+    application.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND, handle_potential_url))
+    # Callback handler for summary type buttons
+    application.add_handler(CallbackQueryHandler(handle_summary_type_callback))
+    # Error handler
+    application.add_error_handler(error_handler)
+    logger.info("Telegram application handlers configured."); return application
+
+# --- ASGI Lifespan & Webhook (Unchanged) ---
 @contextlib.asynccontextmanager
 async def lifespan(app: Starlette):
     global ptb_app, WEBHOOK_SECRET, TELEGRAM_TOKEN
@@ -739,75 +929,156 @@ async def lifespan(app: Starlette):
     if not TELEGRAM_TOKEN: logger.critical("TG TOKEN missing."); raise RuntimeError("Telegram token missing.")
     try:
         ptb_app = await setup_bot_config(); await ptb_app.initialize(); bot_info = await ptb_app.bot.get_me(); logger.info(f"Bot initialized: @{bot_info.username} (ID: {bot_info.id})")
+        # Webhook setup logic
         current_webhook_info = await ptb_app.bot.get_webhook_info()
         if current_webhook_info and current_webhook_info.url:
             logger.info(f"Found existing webhook: {current_webhook_info.url}. Deleting...")
             try:
                 if await ptb_app.bot.delete_webhook(drop_pending_updates=True): logger.info("Webhook deleted.")
                 else: logger.warning("Failed delete webhook (API returned False).")
-            except Exception as e: logger.warning(f"Could not delete webhook: {e}"); await asyncio.sleep(1)
-        space_host = os.environ.get("SPACE_HOST"); webhook_path = "/webhook"; full_webhook_url = None
+            except Exception as e: logger.warning(f"Could not delete webhook: {e}"); await asyncio.sleep(1) # Brief pause before setting new one
+
+        space_host = os.environ.get("SPACE_HOST") # Provided by Hugging Face Spaces
+        webhook_path = "/webhook" # Matches the Starlette route below
+        full_webhook_url = None
         if space_host:
-            protocol = "https"; host = space_host.split('://')[-1]; full_webhook_url = f"{protocol}://{host.rstrip('/')}{webhook_path}"
-            if full_webhook_url:
-                logger.info(f"Setting webhook: {full_webhook_url}"); set_webhook_args = { "url": full_webhook_url, "allowed_updates": Update.ALL_TYPES, "drop_pending_updates": True }
-                if WEBHOOK_SECRET: set_webhook_args["secret_token"] = WEBHOOK_SECRET; logger.info("Using webhook secret.")
-                await asyncio.sleep(1.0)
-                try:
-                    await ptb_app.bot.set_webhook(**set_webhook_args); webhook_info = await ptb_app.bot.get_webhook_info()
-                    if webhook_info.url == full_webhook_url: logger.info(f"Webhook set: URL='{webhook_info.url}', Secret={bool(WEBHOOK_SECRET)}")
-                    else: logger.error(f"Webhook URL mismatch! Expected '{full_webhook_url}', Got '{webhook_info.url}'"); raise RuntimeError("Webhook URL mismatch.")
-                    await ptb_app.start(); logger.info("PTB Application started (webhook mode).")
-                except Exception as e: logger.error(f"FATAL: Failed set webhook: {e}", exc_info=True); raise RuntimeError(f"Failed set webhook: {e}") from e
-            else: logger.critical("Could not construct webhook URL."); raise RuntimeError("Webhook URL undetermined.")
-        else: logger.critical("SPACE_HOST missing."); raise RuntimeError("SPACE_HOST env var missing.")
-        logger.info("ASGI Lifespan: Startup complete."); yield
+            protocol = "https"
+            host = space_host.split('://')[-1] # Remove potential protocol prefix from env var
+            full_webhook_url = f"{protocol}://{host.rstrip('/')}{webhook_path}"
+
+        if full_webhook_url:
+            logger.info(f"Setting webhook: {full_webhook_url}")
+            set_webhook_args: Dict[str, Any] = {
+                "url": full_webhook_url,
+                "allowed_updates": Update.ALL_TYPES,
+                "drop_pending_updates": True
+            }
+            if WEBHOOK_SECRET:
+                set_webhook_args["secret_token"] = WEBHOOK_SECRET
+                logger.info("Using webhook secret token.")
+
+            await asyncio.sleep(1.0) # Short delay before setting webhook
+
+            try:
+                await ptb_app.bot.set_webhook(**set_webhook_args)
+                webhook_info = await ptb_app.bot.get_webhook_info() # Verify
+                if webhook_info.url == full_webhook_url:
+                    logger.info(f"Webhook set successfully: URL='{webhook_info.url}', Secret Configured={bool(WEBHOOK_SECRET)}")
+                else:
+                    logger.error(f"Webhook URL mismatch after setting! Expected '{full_webhook_url}', Got '{webhook_info.url}'. Check SPACE_HOST env var.")
+                    raise RuntimeError("Webhook URL mismatch after setting.")
+
+                await ptb_app.start() # Start listening for updates via webhook
+                logger.info("PTB Application started (webhook mode).")
+
+            except Exception as e:
+                logger.critical(f"FATAL: Failed to set webhook: {e}", exc_info=True)
+                raise RuntimeError(f"Failed to set webhook: {e}") from e
+        else:
+            logger.critical("Could not construct webhook URL. SPACE_HOST environment variable might be missing or invalid.")
+            raise RuntimeError("Webhook URL could not be determined.")
+
+        logger.info("ASGI Lifespan: Startup complete."); yield # Application runs here
+
     except Exception as startup_err:
         logger.critical(f"Application startup failed: {startup_err}", exc_info=True)
+        # Ensure cleanup even if startup fails midway
         if ptb_app:
              if ptb_app.running: await ptb_app.stop()
              await ptb_app.shutdown()
-        raise
-    finally:
+        raise # Reraise the exception to stop the ASGI server
+
+    finally: # Shutdown phase
         logger.info("ASGI Lifespan: Shutdown initiated...")
         if ptb_app:
-            if ptb_app.running: logger.info("Stopping PTB..."); await ptb_app.stop()
-            logger.info("Shutting down PTB..."); await ptb_app.shutdown(); logger.info("PTB Application shut down.")
-        else: logger.info("PTB application not initialized or failed.")
+            if ptb_app.running:
+                logger.info("Stopping PTB application...")
+                await ptb_app.stop()
+            logger.info("Shutting down PTB application...")
+            await ptb_app.shutdown()
+            logger.info("PTB Application shut down.")
+        else:
+            logger.info("PTB application was not initialized or failed during startup.")
         logger.info("ASGI Lifespan: Shutdown complete.")
 
 async def health_check(request: Request) -> PlainTextResponse:
-    global OPENROUTER_MODEL, GEMINI_MODEL, APIFY_ACTOR_ID, _apify_token_exists, _gemini_primary_enabled, _openrouter_fallback_enabled
+    global OPENROUTER_MODEL, GEMINI_MODEL, APIFY_ACTOR_ID, _apify_token_exists, _gemini_primary_enabled, _openrouter_fallback_enabled, _crawl4ai_primary_web_enabled, _bs4_fallback_web_enabled, _urltotext_fallback_web_enabled
     bot_status = "Not Initialized"
     if ptb_app and ptb_app.bot:
         try:
-             if ptb_app.running: bot_info = await ptb_app.bot.get_me(); bot_status = f"Running (@{bot_info.username})"
+             if ptb_app.running:
+                 bot_info = await ptb_app.bot.get_me()
+                 bot_status = f"Running (@{bot_info.username})"
              else: bot_status = "Initialized/Not running"
         except Exception as e: bot_status = f"Error checking status: {e}"
-    return PlainTextResponse( f"TG Bot Summariser - Status: {bot_status}\n" # Changed 'Summarizer'
-                              f"Primary Model: {GEMINI_MODEL if _gemini_primary_enabled else 'N/A (Disabled)'}\n"
-                              f"Fallback Model: {OPENROUTER_MODEL if _openrouter_fallback_enabled else 'N/A (Disabled)'}\n"
-                              f"Apify Actor: {APIFY_ACTOR_ID if _apify_token_exists else 'N/A (No Token)'}" )
+
+    web_status = f"Web Primary: {'Crawl4AI' if _crawl4ai_primary_web_enabled else 'DISABLED'}\n" \
+                 f"Web Fallback 1: {'BS4' if _bs4_fallback_web_enabled else 'DISABLED'}\n" \
+                 f"Web Fallback 2: {'API' if _urltotext_fallback_web_enabled else 'DISABLED'}"
+    summary_status = f"Summarizer Primary: {GEMINI_MODEL if _gemini_primary_enabled else 'DISABLED'}\n" \
+                     f"Summarizer Fallback: {OPENROUTER_MODEL if _openrouter_fallback_enabled else 'DISABLED'}"
+    yt_status = f"YT Fallback 1: {'Supadata' if SUPADATA_API_KEY else 'DISABLED'}\n" \
+                f"YT Fallback 2: {APIFY_ACTOR_ID if _apify_token_exists else 'DISABLED'}"
+
+
+    return PlainTextResponse( f"TG Bot Summariser - Status: {bot_status}\n\n"
+                              f"{web_status}\n\n"
+                              f"{summary_status}\n\n"
+                              f"{yt_status}" )
 
 async def telegram_webhook(request: Request) -> Response:
     global WEBHOOK_SECRET
-    if not ptb_app: logger.error("Webhook recv but PTB not initialized."); return PlainTextResponse('Bot not initialized', status_code=503)
-    if not ptb_app.running: logger.warning("Webhook recv but PTB not running."); return PlainTextResponse('Bot not running', status_code=503)
+    if not ptb_app:
+        logger.error("Webhook received but PTB application is not initialized.")
+        return PlainTextResponse('Bot not initialized', status_code=503) # Service Unavailable
+    if not ptb_app.running:
+        logger.warning("Webhook received but PTB application is not running.")
+        return PlainTextResponse('Bot not running', status_code=503) # Service Unavailable
+
     try:
+        # Validate secret token if configured
         if WEBHOOK_SECRET:
              token_header = request.headers.get("X-Telegram-Bot-Api-Secret-Token")
-             if token_header != WEBHOOK_SECRET: logger.warning(f"Webhook invalid secret. Header: '{token_header}'"); return Response(content="Invalid secret token", status_code=403)
-        update_data = await request.json(); update = Update.de_json(data=update_data, bot=ptb_app.bot); logger.debug(f"Processing update_id: {update.update_id} via webhook")
-        await ptb_app.process_update(update); return Response(status_code=200) # OK
-    except json.JSONDecodeError: logger.error("Webhook invalid JSON."); return PlainTextResponse('Bad Request: Invalid JSON', status_code=400)
-    except Exception as e: logger.error(f"Error processing webhook update: {e}", exc_info=True); return Response(status_code=200) # OK despite error
-
-app = Starlette( debug=False, lifespan=lifespan, routes=[ Route("/", endpoint=health_check, methods=["GET"]), Route("/webhook", endpoint=telegram_webhook, methods=["POST"]), ] )
-logger.info("Starlette ASGI application created with native routes.")
+             if token_header != WEBHOOK_SECRET:
+                 logger.warning(f"Webhook received with invalid secret token. Header: '{token_header}'")
+                 return Response(content="Invalid secret token", status_code=403) # Forbidden
+
+        # Process the update
+        update_data = await request.json()
+        update = Update.de_json(data=update_data, bot=ptb_app.bot)
+        logger.debug(f"Processing update_id: {update.update_id} via webhook")
+        await ptb_app.process_update(update)
+        return Response(status_code=200) # OK - Tell Telegram we received it
+
+    except json.JSONDecodeError:
+        logger.error("Webhook received invalid JSON payload.")
+        return PlainTextResponse('Bad Request: Invalid JSON', status_code=400)
+    except Exception as e:
+        logger.error(f"Error processing webhook update: {e}", exc_info=True)
+        # Still return OK to Telegram to prevent retries for potentially poison-pill updates,
+        # but log the error for debugging.
+        return Response(status_code=200)
+
+# --- ASGI App Definition (Unchanged) ---
+app = Starlette(
+    debug=False, # Keep debug False in production
+    lifespan=lifespan,
+    routes=[
+        Route("/", endpoint=health_check, methods=["GET"]),
+        Route("/webhook", endpoint=telegram_webhook, methods=["POST"]),
+    ]
+)
+logger.info("Starlette ASGI application created with health check and webhook routes.")
 
+# --- Direct Run (for local testing, unchanged) ---
 if __name__ == '__main__':
     import uvicorn
-    logger.warning("Running in development mode using Uvicorn directly")
+    logger.warning("Running in development mode using Uvicorn directly (not for production)")
     log_level = os.environ.get("LOGGING_LEVEL", "info").lower()
+    # Use PORT from env var (like HF Spaces provides) or default to 8080 for local dev
     local_port = int(os.environ.get('PORT', 8080))
+    # Disable webhook setup for local Uvicorn run if needed (manual polling instead)
+    # You might need to comment out the webhook setting logic in lifespan for local testing
+    # or run with ngrok/similar and set SPACE_HOST manually.
+    # For simplicity, assuming webhook setup will just log errors if SPACE_HOST isn't set locally.
     uvicorn.run("__main__:app", host='0.0.0.0', port=local_port, log_level=log_level, reload=True)