Update main.py

main.py

@@ -1,4 +1,4 @@
-# main.py (Corrected SyntaxError at line 424 - Now with Gemini 2.0 as primary)
+# main.py (Corrected SyntaxError at line 424 - Now with Gemini 2.0 as primary AND new scraping fallbacks)
 import os
 import re
 import logging
@@ -7,6 +7,7 @@ import json
 import html
 import contextlib
 import traceback
+import urllib.parse # Added for URL encoding
 from typing import Optional, Dict, Any, Tuple
 
 # --- Frameworks ---
@@ -81,11 +82,12 @@ def get_secret(secret_name):
 
 TELEGRAM_TOKEN = get_secret('TELEGRAM_TOKEN')
 OPENROUTER_API_KEY = get_secret('OPENROUTER_API_KEY') # Now Fallback
-URLTOTEXT_API_KEY = get_secret('URLTOTEXT_API_KEY')
-SUPADATA_API_KEY = get_secret('SUPADATA_API_KEY')
-APIFY_API_TOKEN = get_secret('APIFY_API_TOKEN')
+URLTOTEXT_API_KEY = get_secret('URLTOTEXT_API_KEY') # Scrape Fallback 1
+SUPADATA_API_KEY = get_secret('SUPADATA_API_KEY') # YT Fallback 1
+APIFY_API_TOKEN = get_secret('APIFY_API_TOKEN')   # YT Fallback 2
+RAPIDAPI_KEY = get_secret('RAPIDAPI_KEY')         # Scrape Fallbacks 2 & 3 (NEW)
 WEBHOOK_SECRET = get_secret('WEBHOOK_SECRET')
-GEMINI_API_KEY = get_secret('GEMINI_API_KEY') # Now Primary
+GEMINI_API_KEY = get_secret('GEMINI_API_KEY')     # Now Primary Summarizer
 
 # Models (User can still configure via env vars)
 OPENROUTER_MODEL = os.environ.get("OPENROUTER_MODEL", "deepseek/deepseek-chat-v3-0324:free") # Fallback Model
@@ -95,6 +97,7 @@ GEMINI_MODEL = os.environ.get("GEMINI_MODEL", "gemini-2.0-flash-001") # Primary
 if not TELEGRAM_TOKEN: logger.critical("❌ FATAL: TELEGRAM_TOKEN not found."); raise RuntimeError("Exiting: Telegram token missing.")
 if not GEMINI_API_KEY: logger.error("❌ ERROR: GEMINI_API_KEY not found. Primary summarization (Gemini) will fail.")
 if not OPENROUTER_API_KEY: logger.warning("⚠️ WARNING: OPENROUTER_API_KEY not found. Fallback summarization will fail.")
+if not RAPIDAPI_KEY: logger.warning("⚠️ WARNING: RAPIDAPI_KEY not found. RapidAPI scraping fallbacks will be unavailable.") # New check
 
 _gemini_primary_enabled = _gemini_available and bool(GEMINI_API_KEY)
 if not _gemini_available: logger.warning("⚠️ WARNING: google-generativeai library missing. Gemini disabled.")
@@ -104,9 +107,9 @@ _openrouter_fallback_enabled = bool(OPENROUTER_API_KEY)
 if not _openrouter_fallback_enabled: logger.warning("⚠️ WARNING: OPENROUTER_API_KEY not found. Fallback disabled.")
 
 
-if not URLTOTEXT_API_KEY: pass
-if not SUPADATA_API_KEY: pass
-if not APIFY_API_TOKEN: pass
+if not URLTOTEXT_API_KEY: logger.warning("Optional secret 'URLTOTEXT_API_KEY' not found. First web scraping fallback unavailable.") # Adjusted warning
+if not SUPADATA_API_KEY: logger.warning("Optional secret 'SUPADATA_API_KEY' not found. First YT transcript fallback unavailable.") # Adjusted warning
+if not APIFY_API_TOKEN: logger.warning("Optional secret 'APIFY_API_TOKEN' not found. Second YT transcript fallback unavailable.") # Adjusted warning
 if not WEBHOOK_SECRET: logger.info("Optional secret 'WEBHOOK_SECRET' not found. Webhook security disabled.")
 
 logger.info("Secret loading and configuration check finished.")
@@ -114,6 +117,8 @@ logger.info(f"Using Gemini Model (Primary): {GEMINI_MODEL if _gemini_primary_ena
 logger.info(f"Using OpenRouter Model (Fallback): {OPENROUTER_MODEL if _openrouter_fallback_enabled else 'DISABLED'}")
 logger.info(f"Using Apify Actor (via REST): {APIFY_ACTOR_ID}")
 _apify_token_exists = bool(APIFY_API_TOKEN)
+_urltotext_key_exists = bool(URLTOTEXT_API_KEY)
+_rapidapi_key_exists = bool(RAPIDAPI_KEY)
 
 
 if _gemini_primary_enabled:
@@ -143,26 +148,8 @@ def extract_youtube_id(url):
 
 
 # --- Content Fetching Functions ---
-# (fetch_url_content_for_scrape, get_transcript_via_supadata, get_transcript_via_apify, get_youtube_transcript, get_website_content, get_website_content_via_api remain the same as previous version)
-async def fetch_url_content_for_scrape(url: str, timeout: int = 25) -> Optional[str]:
-    headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Connection': 'keep-alive', 'DNT': '1', 'Upgrade-Insecure-Requests': '1' }
-    try:
-        async with httpx.AsyncClient(follow_redirects=True, timeout=timeout, headers=headers) as client:
-            logger.debug(f"[Web Scrape] Sending request to {url}")
-            response = await client.get(url)
-            logger.debug(f"[Web Scrape] Received response {response.status_code} from {url}")
-            response.raise_for_status()
-            content_type = response.headers.get('content-type', '').lower()
-            if 'html' not in content_type: logger.warning(f"[Web Scrape] Non-HTML content type from {url}: {content_type}"); return None
-            try: return response.text
-            except Exception as e: logger.error(f"[Web Scrape] Error decoding response for {url}: {e}"); return None
-    except httpx.HTTPStatusError as e: logger.error(f"[Web Scrape] HTTP error {e.response.status_code} fetching {url}: {e}")
-    except httpx.TimeoutException: logger.error(f"[Web Scrape] Timeout error fetching {url}")
-    except httpx.TooManyRedirects: logger.error(f"[Web Scrape] Too many redirects fetching {url}")
-    except httpx.RequestError as e: logger.error(f"[Web Scrape] Request error fetching {url}: {e}")
-    except Exception as e: logger.error(f"[Web Scrape] Unexpected error fetching {url}: {e}", exc_info=True)
-    return None
 
+# --- YouTube Transcript Fetching (Unchanged) ---
 async def get_transcript_via_supadata(video_id: str, api_key: str) -> Optional[str]:
     if not video_id: logger.error("[Supadata] No video_id provided"); return None
     if not api_key: logger.error("[Supadata] API key missing."); return None
@@ -250,6 +237,7 @@ async def get_youtube_transcript(video_id: str, video_url: str) -> Optional[str]
     if not video_id: logger.error("get_youtube_transcript: No video_id"); return None
     logger.info(f"Fetching transcript for video ID: {video_id} (URL: {video_url})")
     transcript_text = None
+    # Method 1: youtube-transcript-api (Primary)
     logger.info("[Primary YT] Attempting youtube-transcript-api...")
     try:
         transcript_list = await asyncio.to_thread( YouTubeTranscriptApi.get_transcript, video_id, languages=['en', 'en-GB', 'en-US'] )
@@ -261,6 +249,7 @@ async def get_youtube_transcript(video_id: str, video_url: str) -> Optional[str]
         if "No transcript found" in str(e): logger.warning(f"[Primary YT] No transcript found.")
         elif "disabled" in str(e): logger.warning(f"[Primary YT] Transcripts disabled.")
         transcript_text = None
+    # Method 2: Supadata (Fallback 1)
     if transcript_text is None:
         logger.info("[Fallback YT 1] Trying Supadata API...")
         if SUPADATA_API_KEY:
@@ -268,6 +257,7 @@ async def get_youtube_transcript(video_id: str, video_url: str) -> Optional[str]
             if transcript_text: logger.info(f"[Fallback YT 1] Success via Supadata for {video_id}"); return transcript_text
             else: logger.warning(f"[Fallback YT 1] Supadata failed or no content for {video_id}.")
         else: logger.warning("[Fallback YT 1] Supadata API key unavailable. Skipping.")
+    # Method 3: Apify (Fallback 2)
     if transcript_text is None:
         logger.info("[Fallback YT 2] Trying Apify REST API (SyncItems)...")
         if APIFY_API_TOKEN:
@@ -275,58 +265,196 @@ async def get_youtube_transcript(video_id: str, video_url: str) -> Optional[str]
             if transcript_text: logger.info(f"[Fallback YT 2] Success via Apify SyncItems REST for {video_url}"); return transcript_text
             else: logger.warning(f"[Fallback YT 2] Apify SyncItems REST failed or no content for {video_url}.")
         else: logger.warning("[Fallback YT 2] Apify API token unavailable. Skipping.")
+    # Final Result
     if transcript_text is None: logger.error(f"All methods failed for YT transcript: {video_id}"); return None
     return transcript_text
 
+# --- Website Content Fetching (MODIFIED SECTION) ---
+
+async def fetch_url_content_for_scrape(url: str, timeout: int = 25) -> Optional[str]:
+    """Directly fetches URL content using httpx. (Primary Web Method - Fetching part)"""
+    headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Connection': 'keep-alive', 'DNT': '1', 'Upgrade-Insecure-Requests': '1' }
+    try:
+        async with httpx.AsyncClient(follow_redirects=True, timeout=timeout, headers=headers) as client:
+            logger.debug(f"[Web Scrape Direct] Sending request to {url}")
+            response = await client.get(url)
+            logger.debug(f"[Web Scrape Direct] Received response {response.status_code} from {url}")
+            response.raise_for_status()
+            content_type = response.headers.get('content-type', '').lower()
+            if 'html' not in content_type: logger.warning(f"[Web Scrape Direct] Non-HTML content type from {url}: {content_type}"); return None
+            try: return response.text
+            except Exception as e: logger.error(f"[Web Scrape Direct] Error decoding response for {url}: {e}"); return None
+    except httpx.HTTPStatusError as e: logger.error(f"[Web Scrape Direct] HTTP error {e.response.status_code} fetching {url}: {e}")
+    except httpx.TimeoutException: logger.error(f"[Web Scrape Direct] Timeout error fetching {url}")
+    except httpx.TooManyRedirects: logger.error(f"[Web Scrape Direct] Too many redirects fetching {url}")
+    except httpx.RequestError as e: logger.error(f"[Web Scrape Direct] Request error fetching {url}: {e}")
+    except Exception as e: logger.error(f"[Web Scrape Direct] Unexpected error fetching {url}: {e}", exc_info=True)
+    return None
+
 async def get_website_content(url: str) -> Optional[str]:
+    """Primary method: Fetches HTML directly and parses with BeautifulSoup."""
     if not url: logger.error("get_website_content: No URL"); return None
-    logger.info(f"[Primary Web] Fetching website content for: {url}")
+    logger.info(f"[Web Scrape Primary] Fetching website content for: {url}")
     html_content = await fetch_url_content_for_scrape(url)
     if not html_content: return None
     try:
         def parse_html(content):
             soup = BeautifulSoup(content, DEFAULT_PARSER)
-            for element in soup(["script", "style", "header", "footer", "nav", "aside", "form", "button", "input", "iframe", "img", "svg", "link", "meta", "noscript", "figure"]): element.extract()
+            # Remove common non-content tags
+            for element in soup(["script", "style", "header", "footer", "nav", "aside", "form", "button", "input", "iframe", "img", "svg", "link", "meta", "noscript", "figure"]):
+                element.extract()
+            # Try to find main content areas
             main_content = soup.find('main') or soup.find('article') or soup.find(id='content') or soup.find(class_='content') or soup.find(id='main-content') or soup.find(class_='main-content') or soup.find(role='main')
             target_element = main_content if main_content else soup.body
-            if not target_element: logger.warning(f"[Primary Web] Could not find body/main for parsing {url}"); return None
+            if not target_element: logger.warning(f"[Web Scrape Primary] Could not find body/main for parsing {url}"); return None
+            # Get text, clean up whitespace, join lines
             lines = [line.strip() for line in target_element.get_text(separator='\n', strip=True).splitlines() if line.strip()]
             text = " ".join(lines)
-            if not text: logger.warning(f"[Primary Web] Extracted text empty after clean for {url}"); return None
+            if not text: logger.warning(f"[Web Scrape Primary] Extracted text empty after clean for {url}"); return None
             return text
+        # Run parsing in a separate thread to avoid blocking
         text_content = await asyncio.to_thread(parse_html, html_content)
-        if text_content: logger.info(f"[Primary Web] Success scrape for {url} (final len: {len(text_content)})"); return text_content
+        if text_content: logger.info(f"[Web Scrape Primary] Success scrape for {url} (final len: {len(text_content)})"); return text_content
         else: return None
-    except Exception as e: logger.error(f"[Primary Web] Error scraping/parsing {url}: {e}", exc_info=True); return None
+    except Exception as e: logger.error(f"[Web Scrape Primary] Error scraping/parsing {url}: {e}", exc_info=True); return None
 
 async def get_website_content_via_api(url: str, api_key: str) -> Optional[str]:
-    if not url: logger.error("[Fallback Web API] No URL"); return None
-    if not api_key: logger.error("[Fallback Web API] urltotext.com API key missing."); return None
-    logger.info(f"[Fallback Web API] Attempting fetch for: {url} using urltotext.com API")
+    """Fallback 1: Fetches website content using urltotext.com API."""
+    if not url: logger.error("[Web Scrape Fallback 1] No URL"); return None
+    if not api_key: logger.error("[Web Scrape Fallback 1] urltotext.com API key missing."); return None
+    logger.info(f"[Web Scrape Fallback 1] Attempting fetch for: {url} using urltotext.com API")
     api_endpoint = "https://urltotext.com/api/v1/urltotext/"
     payload = { "url": url, "output_format": "text", "extract_main_content": True, "render_javascript": True, "residential_proxy": False }
     headers = { "Authorization": f"Token {api_key}", "Content-Type": "application/json" }
     try:
         async with httpx.AsyncClient(timeout=45.0) as client:
-            logger.debug(f"[Fallback Web API] Sending request to urltotext.com API for {url}")
+            logger.debug(f"[Web Scrape Fallback 1] Sending request to urltotext.com API for {url}")
             response = await client.post(api_endpoint, headers=headers, json=payload)
-            logger.debug(f"[Fallback Web API] Received status {response.status_code} from urltotext.com API for {url}")
+            logger.debug(f"[Web Scrape Fallback 1] Received status {response.status_code} from urltotext.com API for {url}")
             if response.status_code == 200:
                 try:
                     data = response.json()
                     content = data.get("data", {}).get("content"); credits = data.get("credits_used", "N/A"); warning = data.get("data", {}).get("warning")
-                    if warning: logger.warning(f"[Fallback Web API] urltotext.com API Warning for {url}: {warning}")
-                    if content: logger.info(f"[Fallback Web API] Success via urltotext.com API for {url}. Len: {len(content)}. Credits: {credits}"); return content.strip()
-                    else: logger.warning(f"[Fallback Web API] urltotext.com API success but content empty for {url}. Resp: {data}"); return None
-                except json.JSONDecodeError: logger.error(f"[Fallback Web API] Failed JSON decode urltotext.com for {url}. Resp:{response.text[:500]}"); return None
-                except Exception as e: logger.error(f"[Fallback Web API] Error processing urltotext.com success response for {url}: {e}", exc_info=True); return None
-            elif response.status_code in [400, 401, 402, 403, 422, 500]: logger.error(f"[Fallback Web API] Error {response.status_code} from urltotext.com API for {url}. Resp:{response.text[:200]}"); return None
-            else: logger.error(f"[Fallback Web API] Unexpected status {response.status_code} from urltotext.com API for {url}. Resp:{response.text[:200]}"); return None
-    except httpx.TimeoutException: logger.error(f"[Fallback Web API] Timeout connecting to urltotext.com API for {url}"); return None
-    except httpx.RequestError as e: logger.error(f"[Fallback Web API] Request error connecting to urltotext.com API for {url}: {e}"); return None
-    except Exception as e: logger.error(f"[Fallback Web API] Unexpected error during urltotext.com API call for {url}: {e}", exc_info=True); return None
-
-# --- Summarization Functions ---
+                    if warning: logger.warning(f"[Web Scrape Fallback 1] urltotext.com API Warning for {url}: {warning}")
+                    if content and isinstance(content, str): logger.info(f"[Web Scrape Fallback 1] Success via urltotext.com API for {url}. Len: {len(content)}. Credits: {credits}"); return content.strip()
+                    else: logger.warning(f"[Web Scrape Fallback 1] urltotext.com API success but content empty for {url}. Resp: {data}"); return None
+                except json.JSONDecodeError: logger.error(f"[Web Scrape Fallback 1] Failed JSON decode urltotext.com for {url}. Resp:{response.text[:500]}"); return None
+                except Exception as e: logger.error(f"[Web Scrape Fallback 1] Error processing urltotext.com success response for {url}: {e}", exc_info=True); return None
+            elif response.status_code in [400, 401, 402, 403, 422, 500]: logger.error(f"[Web Scrape Fallback 1] Error {response.status_code} from urltotext.com API for {url}. Resp:{response.text[:200]}"); return None
+            else: logger.error(f"[Web Scrape Fallback 1] Unexpected status {response.status_code} from urltotext.com API for {url}. Resp:{response.text[:200]}"); return None
+    except httpx.TimeoutException: logger.error(f"[Web Scrape Fallback 1] Timeout connecting to urltotext.com API for {url}"); return None
+    except httpx.RequestError as e: logger.error(f"[Web Scrape Fallback 1] Request error connecting to urltotext.com API for {url}: {e}"); return None
+    except Exception as e: logger.error(f"[Web Scrape Fallback 1] Unexpected error during urltotext.com API call for {url}: {e}", exc_info=True); return None
+
+# --- NEW Fallback Functions ---
+
+async def get_website_content_via_scrapers_proxy(url: str, api_key: str) -> Optional[str]:
+    """Fallback 2 (NEW): Fetches website content using Scraper's Proxy Parser via RapidAPI."""
+    if not url: logger.error("[Web Scrape Fallback 2] No URL provided"); return None
+    if not api_key: logger.error("[Web Scrape Fallback 2] RapidAPI key missing."); return None
+    logger.info(f"[Web Scrape Fallback 2] Attempting fetch for: {url} using Scraper's Proxy Parser API")
+
+    api_host = "scrapers-proxy2.p.rapidapi.com"
+    encoded_url = urllib.parse.quote(url, safe='') # URL Encode the target URL
+    api_endpoint = f"https://{api_host}/parser?url={encoded_url}&auto_detect=true"
+    headers = {
+        "x-rapidapi-host": api_host,
+        "x-rapidapi-key": api_key
+    }
+    try:
+        async with httpx.AsyncClient(timeout=40.0) as client: # Increased timeout slightly
+            logger.debug(f"[Web Scrape Fallback 2] Sending GET request to {api_host} for {url}")
+            response = await client.get(api_endpoint, headers=headers)
+            logger.debug(f"[Web Scrape Fallback 2] Received status {response.status_code} from {api_host} for {url}")
+
+            if response.status_code == 200:
+                try:
+                    data = response.json()
+                    # Try to extract content, potentially combining title and content
+                    content = data.get("content")
+                    title = data.get("title")
+                    extracted_text = ""
+                    if title and isinstance(title, str): extracted_text += title.strip() + ". "
+                    if content and isinstance(content, str): extracted_text += content.strip()
+
+                    if extracted_text:
+                        logger.info(f"[Web Scrape Fallback 2] Success via Scraper's Proxy Parser API for {url}. Len: {len(extracted_text)}")
+                        return extracted_text
+                    else:
+                        logger.warning(f"[Web Scrape Fallback 2] Scraper's Proxy API success but content/title empty/invalid for {url}. Response keys: {list(data.keys())}")
+                        return None
+                except json.JSONDecodeError:
+                    logger.error(f"[Web Scrape Fallback 2] Failed JSON decode Scraper's Proxy API for {url}. Status:{response.status_code}. Resp:{response.text[:500]}")
+                    return None
+                except Exception as e:
+                    logger.error(f"[Web Scrape Fallback 2] Error processing Scraper's Proxy API success response for {url}: {e}", exc_info=True)
+                    return None
+            # Handle RapidAPI specific errors if known, otherwise general errors
+            elif response.status_code == 401: logger.error(f"[Web Scrape Fallback 2] Auth error (401) with {api_host}. Check RapidAPI key."); return None
+            elif response.status_code == 403: logger.error(f"[Web Scrape Fallback 2] Forbidden (403) from {api_host}. Check API subscription/limits."); return None
+            elif response.status_code == 429: logger.warning(f"[Web Scrape Fallback 2] Rate Limit (429) from {api_host}."); return None
+            elif response.status_code >= 500: logger.error(f"[Web Scrape Fallback 2] Server error ({response.status_code}) from {api_host}. Resp:{response.text[:200]}"); return None # API itself failed
+            else: logger.error(f"[Web Scrape Fallback 2] Unexpected status {response.status_code} from {api_host} API for {url}. Resp:{response.text[:200]}"); return None
+    except httpx.TimeoutException: logger.error(f"[Web Scrape Fallback 2] Timeout connecting to {api_host} API for {url}"); return None
+    except httpx.RequestError as e: logger.error(f"[Web Scrape Fallback 2] Request error connecting to {api_host} API for {url}: {e}"); return None
+    except Exception as e: logger.error(f"[Web Scrape Fallback 2] Unexpected error during {api_host} API call for {url}: {e}", exc_info=True); return None
+
+async def get_website_content_via_ai_web_scraper(url: str, api_key: str) -> Optional[str]:
+    """Fallback 3 (NEW): Fetches website content using AI Web Scraper via RapidAPI."""
+    if not url: logger.error("[Web Scrape Fallback 3] No URL provided"); return None
+    if not api_key: logger.error("[Web Scrape Fallback 3] RapidAPI key missing."); return None
+    logger.info(f"[Web Scrape Fallback 3] Attempting fetch for: {url} using AI Web Scraper API")
+
+    api_host = "ai-web-scraper.p.rapidapi.com"
+    api_endpoint = f"https://{api_host}/extract_content/v1"
+    headers = {
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'x-rapidapi-host': api_host,
+        'x-rapidapi-key': api_key
+    }
+    # Data needs to be form-encoded, httpx handles this with `data=` param
+    payload = {'url': url}
+
+    try:
+        async with httpx.AsyncClient(timeout=45.0) as client: # Slightly longer timeout for potential AI processing
+            logger.debug(f"[Web Scrape Fallback 3] Sending POST request to {api_host} for {url}")
+            response = await client.post(api_endpoint, headers=headers, data=payload)
+            logger.debug(f"[Web Scrape Fallback 3] Received status {response.status_code} from {api_host} for {url}")
+
+            if response.status_code == 200:
+                try:
+                    data = response.json()
+                    # Infer response structure - Try common keys for content
+                    content = None
+                    if isinstance(data, dict):
+                        content = data.get("content") or data.get("text") or data.get("extracted_text") or data.get("result")
+                        # If it's a simple string response directly
+                    elif isinstance(data, str):
+                        content = data
+
+                    if content and isinstance(content, str):
+                        logger.info(f"[Web Scrape Fallback 3] Success via AI Web Scraper API for {url}. Len: {len(content)}")
+                        return content.strip()
+                    else:
+                        logger.warning(f"[Web Scrape Fallback 3] AI Web Scraper API success but content empty/invalid format for {url}. Response type: {type(data)}, Keys: {list(data.keys()) if isinstance(data, dict) else 'N/A'}")
+                        return None
+                except json.JSONDecodeError:
+                    logger.error(f"[Web Scrape Fallback 3] Failed JSON decode AI Web Scraper API for {url}. Status:{response.status_code}. Resp:{response.text[:500]}")
+                    return None
+                except Exception as e:
+                    logger.error(f"[Web Scrape Fallback 3] Error processing AI Web Scraper API success response for {url}: {e}", exc_info=True)
+                    return None
+            # Handle RapidAPI specific errors if known, otherwise general errors
+            elif response.status_code == 401: logger.error(f"[Web Scrape Fallback 3] Auth error (401) with {api_host}. Check RapidAPI key."); return None
+            elif response.status_code == 403: logger.error(f"[Web Scrape Fallback 3] Forbidden (403) from {api_host}. Check API subscription/limits."); return None
+            elif response.status_code == 429: logger.warning(f"[Web Scrape Fallback 3] Rate Limit (429) from {api_host}."); return None
+            elif response.status_code >= 500: logger.error(f"[Web Scrape Fallback 3] Server error ({response.status_code}) from {api_host}. Resp:{response.text[:200]}"); return None # API itself failed
+            else: logger.error(f"[Web Scrape Fallback 3] Unexpected status {response.status_code} from {api_host} API for {url}. Resp:{response.text[:200]}"); return None
+    except httpx.TimeoutException: logger.error(f"[Web Scrape Fallback 3] Timeout connecting to {api_host} API for {url}"); return None
+    except httpx.RequestError as e: logger.error(f"[Web Scrape Fallback 3] Request error connecting to {api_host} API for {url}: {e}"); return None
+    except Exception as e: logger.error(f"[Web Scrape Fallback 3] Unexpected error during {api_host} API call for {url}: {e}", exc_info=True); return None
+
+# --- Summarization Functions (Unchanged) ---
 
 async def _call_gemini(text: str, summary_type: str) -> Tuple[Optional[str], Optional[str]]:
     """Internal function to call Gemini API. Returns (summary, error_message)."""
@@ -412,9 +540,6 @@ async def _call_gemini(text: str, summary_type: str) -> Tuple[Optional[str], Opt
 
         if summary:
             logger.info(f"[Gemini Primary] Success generating summary. Output len: {len(summary)}");
-            # Escape Markdown for Telegram only if necessary (Removed escaping as it might conflict with plain heading)
-            # escaped_summary = summary.strip().replace('_', r'\_').replace('*', r'\*').replace('[', r'\[').replace('`', r'\`')
-            # Let's see if the AI respects the instruction without further escaping. If not, we might need selective escaping.
             return summary.strip(), None
         else:
             finish_reason = response.candidates[0].finish_reason if response.candidates else 'N/A'
@@ -493,8 +618,6 @@ async def _call_openrouter(text: str, summary_type: str) -> Tuple[Optional[str],
                             summary = message.get("content")
                             if summary:
                                 logger.info(f"[OpenRouter Fallback] Success via OpenRouter. Output len: {len(summary)}")
-                                # Escape Markdown for Telegram only if necessary (Removed escaping)
-                                # escaped_summary = summary.strip().replace('_', r'\_').replace('*', r'\*').replace('[', r'\[').replace('`', r'\`')
                                 return summary.strip(), None
                             else:
                                 logger.warning(f"[OpenRouter Fallback] OpenRouter success but content empty. Resp: {data}")
@@ -585,14 +708,23 @@ async def generate_summary(text: str, summary_type: str) -> str:
     return "Sorry, an unknown error occurred during summary generation after trying all available models."
 
 
-# (process_summary_task, handlers, setup, lifespan, routes, etc. remain the same)
+# --- Main Processing Logic (MODIFIED) ---
+
 async def process_summary_task( user_id: int, chat_id: int, message_id_to_edit: Optional[int], url: str, summary_type: str, bot_token: str ) -> None:
+    """Handles the entire process: fetching content (with fallbacks) and summarizing."""
     task_id = f"{user_id}-{message_id_to_edit or 'new'}"; logger.info(f"[Task {task_id}] Starting processing for URL: {url}")
     background_request: Optional[BaseRequest] = None; bot: Optional[Bot] = None
     try: background_request = HTTPXRequest( connect_timeout=15.0, read_timeout=60.0, write_timeout=60.0, pool_timeout=60.0 ); bot = Bot(token=bot_token, request=background_request)
     except Exception as e: logger.critical(f"[Task {task_id}] Failed to create background bot: {e}", exc_info=True); return
-    content = None; user_feedback_message = None; success = False; status_message_id = message_id_to_edit; message_to_delete_later_id : Optional[int] = None
+
+    content: Optional[str] = None
+    user_feedback_message: Optional[str] = None
+    success: bool = False
+    status_message_id: Optional[int] = message_id_to_edit
+    message_to_delete_later_id : Optional[int] = None
+
     try:
+        # --- 1. Initial User Feedback ---
         processing_message_text = f"Got it! Generating '{summary_type}' summary for:\n`{url}`\n\nThis might take a moment..."
         if status_message_id:
             try: await retry_bot_operation( bot.edit_message_text, chat_id=chat_id, message_id=status_message_id, text=processing_message_text, parse_mode=ParseMode.MARKDOWN, reply_markup=None ); logger.debug(f"[Task {task_id}] Edited message {status_message_id} to 'Processing'")
@@ -602,62 +734,118 @@ async def process_summary_task( user_id: int, chat_id: int, message_id_to_edit:
                  status_message = await retry_bot_operation( bot.send_message, chat_id=chat_id, text=processing_message_text, parse_mode=ParseMode.MARKDOWN )
                  if status_message: message_to_delete_later_id = status_message.message_id; logger.debug(f"[Task {task_id}] Sent new status message {message_to_delete_later_id}")
                  else: raise RuntimeError("Failed to send status message after retries.")
-             except Exception as e: logger.error(f"[Task {task_id}] CRITICAL: Failed to send new status message: {e}"); raise
+             except Exception as e: logger.error(f"[Task {task_id}] CRITICAL: Failed to send new status message: {e}"); raise # Don't proceed if we can't communicate
+
         try:
+            # --- 2. Content Fetching ---
             await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
             is_youtube = is_youtube_url(url); logger.debug(f"[Task {task_id}] URL type: {'YouTube' if is_youtube else 'Website'}")
+
             if is_youtube:
+                # --- YouTube Transcript Logic (Unchanged) ---
                 video_id = extract_youtube_id(url)
-                if video_id: content = await get_youtube_transcript(video_id, url)
+                if video_id: content = await get_youtube_transcript(video_id, url) # Tries lib -> Supadata -> Apify
                 else: user_feedback_message = "Sorry, I couldn't understand that YouTube URL format."
                 if not content and not user_feedback_message: user_feedback_message = "Sorry, I couldn't get the transcript for that YouTube video using any available method (unavailable/private/no captions?)."
             else:
+                # --- Website Scraping Logic (with New Fallbacks) ---
+                global URLTOTEXT_API_KEY, RAPIDAPI_KEY, _urltotext_key_exists, _rapidapi_key_exists
+
+                # Method 1: Primary Scrape (Direct Fetch + BS4)
                 content = await get_website_content(url)
+
+                # Method 2: Fallback 1 (urltotext.com)
                 if not content:
-                    logger.warning(f"[Task {task_id}] Primary web scrape failed for {url}. Trying fallback API.")
-                    global URLTOTEXT_API_KEY
-                    if URLTOTEXT_API_KEY:
+                    logger.warning(f"[Task {task_id}] Primary web scrape failed for {url}. Trying Fallback 1 (urltotext.com).")
+                    if _urltotext_key_exists:
                          await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
                          content = await get_website_content_via_api(url, URLTOTEXT_API_KEY)
-                         if not content: user_feedback_message = "Sorry, I couldn't fetch content from that website using either method (blocked/inaccessible/empty?)."
-                    else: user_feedback_message = "Sorry, I couldn't fetch content from that website (blocked/inaccessible/empty?). The fallback method is not configured."
+                         if not content: logger.warning(f"[Task {task_id}] Fallback 1 (urltotext.com) also failed.")
+                    else: logger.warning("[Task {task_id}] Fallback 1 (urltotext.com) API key unavailable. Skipping.")
+
+                # Method 3: Fallback 2 (Scraper's Proxy via RapidAPI - NEW)
+                if not content:
+                    logger.warning(f"[Task {task_id}] Fallbacks 1 failed. Trying Fallback 2 (Scraper's Proxy).")
+                    if _rapidapi_key_exists:
+                        await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
+                        content = await get_website_content_via_scrapers_proxy(url, RAPIDAPI_KEY)
+                        if not content: logger.warning(f"[Task {task_id}] Fallback 2 (Scraper's Proxy) also failed.")
+                    else: logger.warning("[Task {task_id}] Fallback 2 (Scraper's Proxy) RapidAPI key unavailable. Skipping.")
+
+                # Method 4: Fallback 3 (AI Web Scraper via RapidAPI - NEW)
+                if not content:
+                    logger.warning(f"[Task {task_id}] Fallbacks 2 failed. Trying Fallback 3 (AI Web Scraper).")
+                    if _rapidapi_key_exists:
+                        await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
+                        content = await get_website_content_via_ai_web_scraper(url, RAPIDAPI_KEY)
+                        if not content: logger.warning(f"[Task {task_id}] Fallback 3 (AI Web Scraper) also failed.")
+                    else: logger.warning("[Task {task_id}] Fallback 3 (AI Web Scraper) RapidAPI key unavailable. Skipping.")
+
+                # Final check for website content
+                if not content and not user_feedback_message:
+                    user_feedback_message = "Sorry, I couldn't fetch content from that website using any available method (blocked/inaccessible/empty?)."
+
+            # --- 3. Summarization ---
             if content:
                 logger.info(f"[Task {task_id}] Content fetched (len:{len(content)}). Generating summary.")
                 await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
-                final_summary = await generate_summary(content, summary_type) # This now calls Gemini first, then OpenRouter
-                if final_summary.startswith("Error:") or final_summary.startswith("Sorry,"): user_feedback_message = final_summary; logger.warning(f"[Task {task_id}] Summary generation failed: {final_summary}")
+                final_summary = await generate_summary(content, summary_type) # Calls Gemini -> OpenRouter
+
+                if final_summary.startswith("Error:") or final_summary.startswith("Sorry,"):
+                    user_feedback_message = final_summary # Use the error message from summarizer
+                    logger.warning(f"[Task {task_id}] Summary generation failed: {final_summary}")
                 else:
-                    max_length = 4096; summary_parts = [final_summary[i:i+max_length] for i in range(0, len(final_summary), max_length)]
-                    # Sending potentially unescaped summary directly - assuming AI follows instructions
-                    # ParseMode.MARKDOWN might interpret unintended things, test needed. Consider ParseMode=None if issues arise.
+                    # Split long messages if needed
+                    max_length = 4096
+                    summary_parts = [final_summary[i:i+max_length] for i in range(0, len(final_summary), max_length)]
+                    # Send summary parts (using ParseMode=None as requested by AI prompt instructions)
                     await retry_bot_operation( bot.send_message, chat_id=chat_id, text=summary_parts[0], parse_mode=None, link_preview_options={'is_disabled': True} )
-                    for part in summary_parts[1:]: await asyncio.sleep(0.5); await retry_bot_operation( bot.send_message, chat_id=chat_id, text=part, parse_mode=None, link_preview_options={'is_disabled': True} )
-                    success = True; logger.info(f"[Task {task_id}] Successfully sent summary ({len(summary_parts)} parts)."); user_feedback_message = None
-            elif user_feedback_message: logger.warning(f"[Task {task_id}] Sending failure feedback: {user_feedback_message}"); await retry_bot_operation( bot.send_message, chat_id=chat_id, text=user_feedback_message, link_preview_options={'is_disabled': True} )
+                    for part in summary_parts[1:]:
+                        await asyncio.sleep(0.5) # Small delay between parts
+                        await retry_bot_operation( bot.send_message, chat_id=chat_id, text=part, parse_mode=None, link_preview_options={'is_disabled': True} )
+                    success = True
+                    logger.info(f"[Task {task_id}] Successfully sent summary ({len(summary_parts)} parts).")
+                    user_feedback_message = None # Clear any previous error message if summarization succeeded
+
+            # --- 4. Handle Final Failure Feedback ---
+            if user_feedback_message: # If any step failed and set a message
+                logger.warning(f"[Task {task_id}] Sending failure feedback: {user_feedback_message}")
+                await retry_bot_operation( bot.send_message, chat_id=chat_id, text=user_feedback_message, link_preview_options={'is_disabled': True} )
+
         except Exception as e:
-            logger.error(f"[Task {task_id}] Unexpected error during processing: {e}", exc_info=True); user_feedback_message = "Oops! Something went really wrong. Please try again later."
+            # Catch unexpected errors during the inner try block (fetching/summarizing)
+            logger.error(f"[Task {task_id}] Unexpected error during processing: {e}", exc_info=True)
+            user_feedback_message = "Oops! Something went really wrong during processing. Please try again later."
             try: await retry_bot_operation( bot.send_message, chat_id=chat_id, text=user_feedback_message )
             except Exception: logger.error(f"[Task {task_id}] Failed to send unexpected error feedback.")
+
     except Exception as outer_e:
+        # Catch errors in the outer setup (bot creation, status message sending)
         logger.critical(f"[Task {task_id}] Critical outer error: {outer_e}", exc_info=True)
         try:
-             if bot: await retry_bot_operation( bot.send_message, chat_id=chat_id, text="❌ Critical internal error occurred." )
+             if bot: await retry_bot_operation( bot.send_message, chat_id=chat_id, text="❌ Critical internal error occurred. Could not start processing." )
         except Exception: logger.exception(f"[Task {task_id}] Failed even to send critical error message.")
     finally:
+        # --- 5. Cleanup ---
+        # Delete the "Processing..." or original button message
         delete_target_id = message_to_delete_later_id if message_to_delete_later_id else status_message_id
         if delete_target_id and bot:
             try: await retry_bot_operation(bot.delete_message, chat_id=chat_id, message_id=delete_target_id); logger.debug(f"[Task {task_id}] Deleted status/button message {delete_target_id}")
             except Exception as del_e: logger.warning(f"[Task {task_id}] Failed to delete status/button message {delete_target_id}: {del_e}")
+        # Close the background bot's HTTP client
         if background_request and hasattr(background_request, '_client') and background_request._client:
              try: await background_request._client.aclose(); logger.debug(f"[Task {task_id}] Background bot's HTTPX client closed.")
              except Exception as e: logger.warning(f"[Task {task_id}] Error closing background bot's client: {e}")
         logger.info(f"[Task {task_id}] Task completed. Success: {success}")
 
+
+# --- Telegram Handlers (Unchanged) ---
+
 async def start(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     user = update.effective_user; mention = user.mention_html()
     if not user or not update.message: return
     logger.info(f"User {user.id} used /start.")
-    await update.message.reply_html( f"👋 Hello {mention}! I can summarise YouTube links or website URLs.\n\nJust send me a link anytime!" ) # Changed 'summarize'
+    await update.message.reply_html( f"👋 Hello {mention}! I can summarise YouTube links or website URLs.\n\nJust send me a link anytime!" )
 
 async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     user = update.effective_user
@@ -671,41 +859,53 @@ async def handle_potential_url(update: Update, context: ContextTypes.DEFAULT_TYP
     if not update.message or not update.message.text: return
     url = update.message.text.strip(); user = update.effective_user
     if not user: return
-    if not (url.startswith('http://') or url.startswith('https://')) or '.' not in url[8:]: logger.debug(f"Ignoring non-URL from {user.id}: {url}"); return
+    # Basic URL check
+    if not (url.startswith('http://') or url.startswith('https://')) or '.' not in url[8:]:
+        logger.debug(f"Ignoring non-URL message from {user.id}")
+        # Optionally, send a message if you want to guide the user
+        # await update.message.reply_text("Please send a valid URL starting with http:// or https://")
+        return
+
     logger.info(f"User {user.id} sent potential URL: {url}")
     context.user_data['url_to_summarize'] = url; context.user_data['original_message_id'] = update.message.message_id
     keyboard = [[ InlineKeyboardButton("Paragraph Summary", callback_data="paragraph"), InlineKeyboardButton("Points Summary", callback_data="points") ]]
     reply_markup = InlineKeyboardMarkup(keyboard)
-    await update.message.reply_text( f"Okay, I see this link:\n{url}\n\nHow would you like it summarised?", reply_markup=reply_markup, disable_web_page_preview=True ) # Changed 'summarized'
+    await update.message.reply_text( f"Okay, I see this link:\n`{url}`\n\nHow would you like it summarised?", reply_markup=reply_markup, disable_web_page_preview=True, parse_mode=ParseMode.MARKDOWN ) # Changed 'summarized', added Markdown
 
 async def handle_summary_type_callback(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     query = update.callback_query
     if not query or not query.message or not query.from_user: logger.warning("Callback query missing data."); return
     user = query.from_user; summary_type = query.data; query_id = query.id
     try: await query.answer(); logger.debug(f"Ack callback {query_id} from {user.id}")
-    except Exception as e: logger.error(f"Error answering callback {query_id}: {e}", exc_info=True)
+    except Exception as e: logger.error(f"Error answering callback {query_id}: {e}", exc_info=True) # Log ack errors
+
     url = context.user_data.get('url_to_summarize'); message_id_to_edit = query.message.message_id
     logger.info(f"User {user.id} chose '{summary_type}' for msg {message_id_to_edit}. URL in context: {'Yes' if url else 'No'}")
+
     if not url:
-        logger.warning(f"No URL in context for user {user.id} (cb {query_id}).")
-        try: await query.edit_message_text(text="Sorry, I couldn't find the URL associated with this request. Please send the link again.")
+        logger.warning(f"No URL in context for user {user.id} (cb {query_id}). Might be an old button.")
+        try:
+            await query.edit_message_text(text="Sorry, I couldn't find the URL associated with this request (it might be too old). Please send the link again.")
+        except BadRequest as e:
+            if "message is not modified" in str(e).lower(): pass # Ignore if already edited
+            else: logger.error(f"Failed edit 'URL not found' msg: {e}")
         except Exception as e:
             logger.error(f"Failed edit 'URL not found' msg: {e}")
-            try: await context.bot.send_message(chat_id=user.id, text="Sorry, context lost. Send link again.")
-            except Exception: pass
+            # Don't send a new message here, as it might confuse the user if they didn't interact recently
         return
 
+    # Clear context *after* checking it's valid for this interaction
     context.user_data.pop('url_to_summarize', None); context.user_data.pop('original_message_id', None); logger.debug(f"Cleared URL context for user {user.id}")
 
-    global TELEGRAM_TOKEN, GEMINI_API_KEY, OPENROUTER_API_KEY, _gemini_primary_enabled, _openrouter_fallback_enabled
+    global TELEGRAM_TOKEN, _gemini_primary_enabled, _openrouter_fallback_enabled
     if not TELEGRAM_TOKEN:
-        logger.critical("TG TOKEN missing!")
-        try: await query.edit_message_text(text="❌ Bot config error.")
+        logger.critical("TG TOKEN missing in callback!")
+        try: await query.edit_message_text(text="❌ Bot configuration error. Cannot proceed.")
         except Exception: pass
         return
     if not _gemini_primary_enabled and not _openrouter_fallback_enabled:
-        logger.critical("Neither Gemini nor OpenRouter API keys are configured/valid!")
-        try: await query.edit_message_text(text="❌ AI config error: No models available.")
+        logger.critical("Neither Gemini nor OpenRouter API keys are configured/valid in callback!")
+        try: await query.edit_message_text(text="❌ AI configuration error: No summarization models available.")
         except Exception: pass
         return
     elif not _gemini_primary_enabled:
@@ -715,22 +915,62 @@ async def handle_summary_type_callback(update: Update, context: ContextTypes.DEF
          logger.warning("Fallback AI (OpenRouter) is unavailable.")
          # No need to inform user unless primary fails later
 
-    logger.info(f"Scheduling task for user {user.id}, chat {query.message.chat_id}, msg {message_id_to_edit}")
-    asyncio.create_task( process_summary_task( user_id=user.id, chat_id=query.message.chat_id, message_id_to_edit=message_id_to_edit, url=url, summary_type=summary_type, bot_token=TELEGRAM_TOKEN ), name=f"SummaryTask-{user.id}-{message_id_to_edit}" )
+    logger.info(f"Scheduling background task for user {user.id}, chat {query.message.chat_id}, msg {message_id_to_edit}")
+    # Schedule the background task
+    asyncio.create_task(
+        process_summary_task(
+            user_id=user.id,
+            chat_id=query.message.chat_id,
+            message_id_to_edit=message_id_to_edit,
+            url=url,
+            summary_type=summary_type,
+            bot_token=TELEGRAM_TOKEN
+        ),
+        name=f"SummaryTask-{user.id}-{message_id_to_edit}"
+    )
 
 async def error_handler(update: object, context: ContextTypes.DEFAULT_TYPE) -> None:
-    ignore_errors = (AttributeError, )
-    if isinstance(context.error, ignore_errors) and "object has no attribute" in str(context.error): logger.debug(f"Ignoring known/handled error in error_handler: {context.error}"); return
+    """Log Errors caused by Updates."""
+    # Ignore specific, known errors that are often harmless or handled elsewhere
+    ignore_errors = (AttributeError, BadRequest, TimedOut, NetworkError, RetryAfter)
+    if isinstance(context.error, ignore_errors):
+         # More specific logging for potentially ignorable errors
+         if isinstance(context.error, BadRequest) and any(err in str(context.error).lower() for err in ["message is not modified", "query is too old", "message to edit not found"]):
+             logger.debug(f"Ignoring known BadRequest in error_handler: {context.error}")
+             return
+         elif isinstance(context.error, AttributeError) and "object has no attribute" in str(context.error):
+             logger.debug(f"Ignoring handled AttributeError in error_handler: {context.error}")
+             return
+         else:
+             # Log other potentially recoverable network/timeout errors as warnings
+             logger.warning(f"Handled networking/API error in error_handler: {context.error}")
+             return
+
+    # Log all other exceptions as errors
     logger.error("Exception while handling an update:", exc_info=context.error)
+    # Optionally, notify the user about unexpected errors, but be careful not to spam
+    # if update and isinstance(update, Update) and update.effective_chat:
+    #    try: await context.bot.send_message(chat_id=update.effective_chat.id, text="An unexpected error occurred.")
+    #    except Exception: logger.error("Failed to send error message to user.")
+
+
+# --- Application Setup & Web Framework (Unchanged) ---
 
 async def setup_bot_config() -> Application:
     logger.info("Configuring Telegram Application..."); global TELEGRAM_TOKEN
     if not TELEGRAM_TOKEN: raise ValueError("TELEGRAM_TOKEN missing.")
     custom_request = HTTPXRequest( connect_timeout=10.0, read_timeout=30.0, write_timeout=30.0, pool_timeout=60.0 )
     application = Application.builder().token(TELEGRAM_TOKEN).request(custom_request).build()
-    application.add_handler(CommandHandler("start", start)); application.add_handler(CommandHandler("help", help_command))
-    application.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND, handle_potential_url)); application.add_handler(CallbackQueryHandler(handle_summary_type_callback))
-    application.add_error_handler(error_handler); logger.info("Telegram application handlers configured."); return application
+    # Command Handlers
+    application.add_handler(CommandHandler("start", start))
+    application.add_handler(CommandHandler("help", help_command))
+    # Message Handler (URLs)
+    application.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND & filters.Entity("url") | filters.Entity("text_link"), handle_potential_url))
+    # Callback Query Handler (Buttons)
+    application.add_handler(CallbackQueryHandler(handle_summary_type_callback))
+    # Error Handler
+    application.add_error_handler(error_handler)
+    logger.info("Telegram application handlers configured."); return application
 
 @contextlib.asynccontextmanager
 async def lifespan(app: Starlette):
@@ -745,69 +985,149 @@ async def lifespan(app: Starlette):
             try:
                 if await ptb_app.bot.delete_webhook(drop_pending_updates=True): logger.info("Webhook deleted.")
                 else: logger.warning("Failed delete webhook (API returned False).")
-            except Exception as e: logger.warning(f"Could not delete webhook: {e}"); await asyncio.sleep(1)
-        space_host = os.environ.get("SPACE_HOST"); webhook_path = "/webhook"; full_webhook_url = None
+            except Exception as e: logger.warning(f"Could not delete webhook: {e}"); await asyncio.sleep(1) # Short delay before setting new one
+
+        # Setup Webhook using SPACE_HOST env var from Hugging Face
+        space_host = os.environ.get("SPACE_HOST")
+        webhook_path = "/webhook" # Must match the route below
+        full_webhook_url = None
         if space_host:
-            protocol = "https"; host = space_host.split('://')[-1]; full_webhook_url = f"{protocol}://{host.rstrip('/')}{webhook_path}"
+            protocol = "https" # HF Spaces use HTTPS
+            host = space_host.split('://')[-1] # Get the host part
+            full_webhook_url = f"{protocol}://{host.rstrip('/')}{webhook_path}"
+
             if full_webhook_url:
-                logger.info(f"Setting webhook: {full_webhook_url}"); set_webhook_args = { "url": full_webhook_url, "allowed_updates": Update.ALL_TYPES, "drop_pending_updates": True }
-                if WEBHOOK_SECRET: set_webhook_args["secret_token"] = WEBHOOK_SECRET; logger.info("Using webhook secret.")
-                await asyncio.sleep(1.0)
+                logger.info(f"Setting webhook to: {full_webhook_url}")
+                set_webhook_args = {
+                    "url": full_webhook_url,
+                    "allowed_updates": Update.ALL_TYPES, # Or specify types like [Update.MESSAGE, Update.CALLBACK_QUERY]
+                    "drop_pending_updates": True
+                }
+                if WEBHOOK_SECRET:
+                    set_webhook_args["secret_token"] = WEBHOOK_SECRET
+                    logger.info("Webhook secret token is configured.")
+
+                await asyncio.sleep(1.0) # Give Telegram servers a moment after potential delete
                 try:
-                    await ptb_app.bot.set_webhook(**set_webhook_args); webhook_info = await ptb_app.bot.get_webhook_info()
-                    if webhook_info.url == full_webhook_url: logger.info(f"Webhook set: URL='{webhook_info.url}', Secret={bool(WEBHOOK_SECRET)}")
-                    else: logger.error(f"Webhook URL mismatch! Expected '{full_webhook_url}', Got '{webhook_info.url}'"); raise RuntimeError("Webhook URL mismatch.")
-                    await ptb_app.start(); logger.info("PTB Application started (webhook mode).")
-                except Exception as e: logger.error(f"FATAL: Failed set webhook: {e}", exc_info=True); raise RuntimeError(f"Failed set webhook: {e}") from e
-            else: logger.critical("Could not construct webhook URL."); raise RuntimeError("Webhook URL undetermined.")
-        else: logger.critical("SPACE_HOST missing."); raise RuntimeError("SPACE_HOST env var missing.")
-        logger.info("ASGI Lifespan: Startup complete."); yield
+                    await ptb_app.bot.set_webhook(**set_webhook_args)
+                    webhook_info = await ptb_app.bot.get_webhook_info() # Verify
+                    if webhook_info.url == full_webhook_url:
+                        logger.info(f"Webhook successfully set: URL='{webhook_info.url}', Secret={bool(WEBHOOK_SECRET)}")
+                    else:
+                        logger.error(f"Webhook URL mismatch after setting! Expected '{full_webhook_url}', Got '{webhook_info.url}'. Check SPACE_HOST and path.")
+                        raise RuntimeError("Webhook URL mismatch after setting.")
+
+                    await ptb_app.start() # Start the PTB application polling (for webhook updates)
+                    logger.info("PTB Application started in webhook mode.")
+
+                except Exception as e:
+                    logger.error(f"FATAL: Failed to set webhook: {e}", exc_info=True)
+                    raise RuntimeError(f"Failed to set webhook: {e}") from e
+            else:
+                logger.critical("Could not construct webhook URL from SPACE_HOST.")
+                raise RuntimeError("Webhook URL could not be determined.")
+        else:
+            logger.critical("SPACE_HOST environment variable not found. Cannot set webhook automatically.")
+            raise RuntimeError("SPACE_HOST environment variable is missing.")
+
+        logger.info("ASGI Lifespan: Startup complete."); yield # Application runs here
+
     except Exception as startup_err:
         logger.critical(f"Application startup failed: {startup_err}", exc_info=True)
         if ptb_app:
              if ptb_app.running: await ptb_app.stop()
              await ptb_app.shutdown()
-        raise
+        raise # Propagate error to stop Starlette
     finally:
+        # --- Shutdown ---
         logger.info("ASGI Lifespan: Shutdown initiated...")
         if ptb_app:
-            if ptb_app.running: logger.info("Stopping PTB..."); await ptb_app.stop()
-            logger.info("Shutting down PTB..."); await ptb_app.shutdown(); logger.info("PTB Application shut down.")
-        else: logger.info("PTB application not initialized or failed.")
+            if ptb_app.running: logger.info("Stopping PTB Application..."); await ptb_app.stop()
+            logger.info("Shutting down PTB Application..."); await ptb_app.shutdown(); logger.info("PTB Application shut down.")
+            # Attempt to clean up webhook on shutdown (optional, might fail if app is stopping forcefully)
+            try:
+                logger.info("Attempting to delete webhook on shutdown...")
+                await ptb_app.bot.delete_webhook(drop_pending_updates=True)
+                logger.info("Webhook deleted on shutdown.")
+            except Exception as e:
+                logger.warning(f"Could not delete webhook during shutdown: {e}")
+        else: logger.info("PTB application was not fully initialized or failed during startup.")
         logger.info("ASGI Lifespan: Shutdown complete.")
 
 async def health_check(request: Request) -> PlainTextResponse:
+    """Simple health check endpoint."""
     global OPENROUTER_MODEL, GEMINI_MODEL, APIFY_ACTOR_ID, _apify_token_exists, _gemini_primary_enabled, _openrouter_fallback_enabled
     bot_status = "Not Initialized"
     if ptb_app and ptb_app.bot:
         try:
-             if ptb_app.running: bot_info = await ptb_app.bot.get_me(); bot_status = f"Running (@{bot_info.username})"
-             else: bot_status = "Initialized/Not running"
-        except Exception as e: bot_status = f"Error checking status: {e}"
-    return PlainTextResponse( f"TG Bot Summariser - Status: {bot_status}\n" # Changed 'Summarizer'
-                              f"Primary Model: {GEMINI_MODEL if _gemini_primary_enabled else 'N/A (Disabled)'}\n"
-                              f"Fallback Model: {OPENROUTER_MODEL if _openrouter_fallback_enabled else 'N/A (Disabled)'}\n"
-                              f"Apify Actor: {APIFY_ACTOR_ID if _apify_token_exists else 'N/A (No Token)'}" )
+             if ptb_app.running:
+                 bot_info = await ptb_app.bot.get_me()
+                 bot_status = f"Running (@{bot_info.username})"
+             else:
+                 bot_status = "Initialized but Not running"
+        except Exception as e:
+             bot_status = f"Error checking status: {e}"
+
+    return PlainTextResponse(
+        f"TG Bot Summariser - Status: {bot_status}\n" # Changed 'Summarizer'
+        f"Primary Model (Gemini): {GEMINI_MODEL if _gemini_primary_enabled else 'N/A (Disabled)'}\n"
+        f"Fallback Model (OpenRouter): {OPENROUTER_MODEL if _openrouter_fallback_enabled else 'N/A (Disabled)'}\n"
+        f"YT Fallback (Apify Actor): {APIFY_ACTOR_ID if _apify_token_exists else 'N/A (No Token)'}\n"
+        f"Web Fallback 1 (urltotext): {'Enabled' if _urltotext_key_exists else 'Disabled'}\n"
+        f"Web Fallbacks 2/3 (RapidAPI): {'Enabled' if _rapidapi_key_exists else 'Disabled'}"
+    )
 
 async def telegram_webhook(request: Request) -> Response:
+    """Handles incoming updates from Telegram."""
     global WEBHOOK_SECRET
-    if not ptb_app: logger.error("Webhook recv but PTB not initialized."); return PlainTextResponse('Bot not initialized', status_code=503)
-    if not ptb_app.running: logger.warning("Webhook recv but PTB not running."); return PlainTextResponse('Bot not running', status_code=503)
+    if not ptb_app: logger.error("Webhook received but PTB application not initialized."); return PlainTextResponse('Bot not initialized', status_code=503)
+    if not ptb_app.running: logger.warning("Webhook received but PTB application not running."); return PlainTextResponse('Bot not running, cannot process update', status_code=503)
+
+    # --- Security Check ---
+    if WEBHOOK_SECRET:
+         token_header = request.headers.get("X-Telegram-Bot-Api-Secret-Token")
+         if token_header != WEBHOOK_SECRET:
+             logger.warning(f"Webhook received with invalid secret token. Header: '{token_header}'")
+             return Response(content="Invalid secret token", status_code=403) # Forbidden
+
+    # --- Process Update ---
     try:
-        if WEBHOOK_SECRET:
-             token_header = request.headers.get("X-Telegram-Bot-Api-Secret-Token")
-             if token_header != WEBHOOK_SECRET: logger.warning(f"Webhook invalid secret. Header: '{token_header}'"); return Response(content="Invalid secret token", status_code=403)
-        update_data = await request.json(); update = Update.de_json(data=update_data, bot=ptb_app.bot); logger.debug(f"Processing update_id: {update.update_id} via webhook")
-        await ptb_app.process_update(update); return Response(status_code=200) # OK
-    except json.JSONDecodeError: logger.error("Webhook invalid JSON."); return PlainTextResponse('Bad Request: Invalid JSON', status_code=400)
-    except Exception as e: logger.error(f"Error processing webhook update: {e}", exc_info=True); return Response(status_code=200) # OK despite error
-
-app = Starlette( debug=False, lifespan=lifespan, routes=[ Route("/", endpoint=health_check, methods=["GET"]), Route("/webhook", endpoint=telegram_webhook, methods=["POST"]), ] )
+        update_data = await request.json()
+        update = Update.de_json(data=update_data, bot=ptb_app.bot)
+        logger.debug(f"Processing update_id: {update.update_id} via webhook")
+        await ptb_app.process_update(update)
+        return Response(status_code=200) # OK - Tell Telegram we received it
+    except json.JSONDecodeError:
+        logger.error("Webhook received invalid JSON.")
+        return PlainTextResponse('Bad Request: Invalid JSON', status_code=400)
+    except Exception as e:
+        logger.error(f"Error processing webhook update: {e}", exc_info=True)
+        # Still return 200 OK to Telegram, otherwise it will keep retrying the same failed update.
+        # The error is logged for debugging.
+        return Response(status_code=200)
+
+# --- Starlette App Definition ---
+app = Starlette(
+    debug=False, # Set to True for more verbose errors during development ONLY
+    lifespan=lifespan,
+    routes=[
+        Route("/", endpoint=health_check, methods=["GET"]),
+        Route("/webhook", endpoint=telegram_webhook, methods=["POST"]), # Matches webhook_path
+    ]
+)
 logger.info("Starlette ASGI application created with native routes.")
 
+# --- Development Server (if run directly) ---
 if __name__ == '__main__':
     import uvicorn
-    logger.warning("Running in development mode using Uvicorn directly")
+    logger.warning("Running in development mode using Uvicorn directly - NOT for production!")
+    # Use environment variables for config if available, otherwise defaults
     log_level = os.environ.get("LOGGING_LEVEL", "info").lower()
-    local_port = int(os.environ.get('PORT', 8080))
-    uvicorn.run("__main__:app", host='0.0.0.0', port=local_port, log_level=log_level, reload=True)
+    local_port = int(os.environ.get('PORT', 8080)) # Use PORT env var common in PaaS
+    uvicorn.run(
+        "__main__:app",
+        host='0.0.0.0', # Listen on all available network interfaces
+        port=local_port,
+        log_level=log_level,
+        reload=True # Enable auto-reload for development
+    )