Update main.py

main.py

@@ -111,6 +111,7 @@ GEMINI_FLASH_MODEL = os.environ.get("GEMINI_FLASH_MODEL", "gemini-2.0-flash-001"
 OPENROUTER_DEEPSEEK_MODEL = os.environ.get("OPENROUTER_DEEPSEEK_MODEL", "deepseek/deepseek-chat-v3-0324:free") # Specific DeepSeek model
 
 APIFY_ACTOR_ID = os.environ.get("APIFY_ACTOR_ID", "1s7eXiaukVuOr4Ueg") # YT Default
+APIFY_STRUCTURED_YT_ACTOR_ID = "gpjTCWkGZS1lHc9pR" # YT Fallback 1 (New Structured Extractor)
 APIFY_CRAWLER_ACTOR_ID = "aYG0l9s7dbB7j3gbS" # Scrape Fallback 4
 APIFY_TEXT_SCRAPER_ACTOR_ID = "2gbQiRSpJIIag2FdR" # Scrape Fallback 5
 
@@ -147,9 +148,12 @@ logger.info(f"Summarizer 1 (Groq): {GROQ_LLAMA4_MODEL if _groq_enabled else 'DIS
 logger.info(f"Summarizer 2 (Gemini Pro Exp): {GEMINI_PRO_EXP_MODEL if _gemini_api_enabled else 'DISABLED'}")
 logger.info(f"Summarizer 3 (Gemini Flash): {GEMINI_FLASH_MODEL if _gemini_api_enabled else 'DISABLED'}")
 logger.info(f"Summarizer 4 (OpenRouter): {OPENROUTER_DEEPSEEK_MODEL if _openrouter_fallback_enabled else 'DISABLED'}")
-logger.info(f"Using Apify Actor (YT Default): {APIFY_ACTOR_ID}")
+# --- Updated Logging ---
+logger.info(f"Using Apify Actor (YT Fallback 1 - Structured): {APIFY_STRUCTURED_YT_ACTOR_ID}")
+logger.info(f"Using Apify Actor (YT Fallback 3 - Default): {APIFY_ACTOR_ID}") # Now fallback 3
 logger.info(f"Using Apify Actor (Web Scrape Fallback 4): {APIFY_CRAWLER_ACTOR_ID}")
 logger.info(f"Using Apify Actor (Web Scrape Fallback 5): {APIFY_TEXT_SCRAPER_ACTOR_ID}")
+# --- End Updated Logging ---
 
 # Flags for scraper key existence
 _apify_token_exists = bool(APIFY_API_TOKEN)
@@ -275,35 +279,107 @@ async def get_transcript_via_apify(video_url: str, api_token: str) -> Optional[s
         logger.error(f"[Apify YT] Unexpected error during Apify YT call for {video_url}: {e}", exc_info=True)
         return None
 
+async def get_transcript_via_apify_structured_extractor(video_url: str, api_token: str) -> Optional[str]:
+    """Fallback YT 1: Fetches YouTube transcript using the Structured Extractor Apify Actor."""
+    global APIFY_STRUCTURED_YT_ACTOR_ID # Use the new ID
+    if not video_url: logger.error("[Apify Structured YT] No video_url provided"); return None
+    if not api_token: logger.error("[Apify Structured YT] API token missing."); return None
+    logger.info(f"[YT Fallback 1] Attempting fetch for URL: {video_url} (Actor: {APIFY_STRUCTURED_YT_ACTOR_ID})")
+
+    # Use the generic helper function.
+    # We assume the standard input format used by the helper for non-specific actors
+    # (like the text scraper: {"urls": [url]}) is sufficient, or that the helper's
+    # existing logic correctly identifies and parses the output from this actor.
+    # The helper already has logic to parse various text/content/captions fields,
+    # including list structures for captions as described in the actor's docs.
+    return await _run_apify_actor_for_web_content(
+        url=video_url,
+        api_token=api_token,
+        actor_id=APIFY_STRUCTURED_YT_ACTOR_ID,
+        actor_name="Apify Structured YT" # Specific name for logging
+    )
+
 async def get_youtube_transcript(video_id: str, video_url: str) -> Optional[str]:
-    global SUPADATA_API_KEY, APIFY_API_TOKEN, _apify_token_exists
+    """Fetches YouTube transcript using a primary library and multiple fallback methods."""
+    global SUPADATA_API_KEY, APIFY_API_TOKEN, _apify_token_exists # Keep existing globals
     if not video_id: logger.error("get_youtube_transcript: No video_id"); return None
+
     logger.info(f"Fetching transcript for video ID: {video_id} (URL: {video_url})")
-    transcript_text = None
+    transcript_text: Optional[str] = None
+
+    # --- Primary Method: youtube-transcript-api ---
     logger.info("[Primary YT] Attempting youtube-transcript-api...")
     try:
-        transcript_list = await asyncio.to_thread( YouTubeTranscriptApi.get_transcript, video_id, languages=['en', 'en-GB', 'en-US'] )
-        if transcript_list: transcript_text = " ".join([item['text'] for item in transcript_list if 'text' in item])
-        if transcript_text: logger.info(f"[Primary YT] Success via lib for {video_id} (len: {len(transcript_text)})"); return transcript_text
-        else: logger.warning(f"[Primary YT] Transcript list/text empty for {video_id}"); transcript_text = None
-    except NoTranscriptFound: logger.warning(f"[Primary YT] No transcript found via lib for {video_id}.")
-    except TranscriptsDisabled: logger.warning(f"[Primary YT] Transcripts disabled via lib for {video_id}.")
-    except Exception as e: logger.warning(f"[Primary YT] Error via lib for {video_id}: {e}"); transcript_text = None
+        # Prefer English variants first
+        transcript_list = await asyncio.to_thread(
+            YouTubeTranscriptApi.get_transcript, video_id, languages=['en', 'en-GB', 'en-US']
+        )
+        if transcript_list:
+            transcript_text = " ".join([item['text'] for item in transcript_list if 'text' in item])
+        if transcript_text:
+            logger.info(f"[Primary YT] Success via lib for {video_id} (len: {len(transcript_text)})")
+            return transcript_text.strip() # Return immediately on success
+        else:
+            logger.warning(f"[Primary YT] Transcript list returned but text content was empty for {video_id}")
+            transcript_text = None # Ensure it's None to trigger fallbacks
+    except NoTranscriptFound:
+        logger.warning(f"[Primary YT] No transcript found via lib for {video_id}.")
+        transcript_text = None
+    except TranscriptsDisabled:
+        logger.warning(f"[Primary YT] Transcripts disabled via lib for {video_id}.")
+        transcript_text = None
+    except Exception as e:
+        # Log more specific errors if possible, e.g., timeouts, network issues
+        logger.warning(f"[Primary YT] Error via lib for {video_id}: {e}")
+        transcript_text = None
+
+    # --- Fallback 1: Apify Structured Transcript Extractor (NEW) ---
     if transcript_text is None:
-        logger.info("[Fallback YT 1] Trying Supadata API...")
+        logger.info("[Fallback YT 1] Trying Apify Structured Transcript Extractor...")
+        if _apify_token_exists:
+            transcript_text = await get_transcript_via_apify_structured_extractor(video_url, APIFY_API_TOKEN)
+            if transcript_text:
+                logger.info(f"[Fallback YT 1] Success via Apify Structured Extractor for {video_url}")
+                return transcript_text # Return on success
+            else:
+                logger.warning(f"[Fallback YT 1] Apify Structured Extractor failed or no content for {video_url}.")
+        else:
+            logger.warning("[Fallback YT 1] Apify API token unavailable. Skipping Apify Structured Extractor.")
+
+    # --- Fallback 2: Supadata API ---
+    if transcript_text is None:
+        logger.info("[Fallback YT 2] Trying Supadata API...") # <<<< UPDATED NUMBER
         if SUPADATA_API_KEY:
             transcript_text = await get_transcript_via_supadata(video_id, SUPADATA_API_KEY)
-            if transcript_text: logger.info(f"[Fallback YT 1] Success via Supadata for {video_id}"); return transcript_text
-            else: logger.warning(f"[Fallback YT 1] Supadata failed or no content for {video_id}.")
-        else: logger.warning("[Fallback YT 1] Supadata API key unavailable. Skipping.")
+            if transcript_text:
+                logger.info(f"[Fallback YT 2] Success via Supadata for {video_id}") # <<<< UPDATED NUMBER
+                return transcript_text # Return on success
+            else:
+                logger.warning(f"[Fallback YT 2] Supadata failed or no content for {video_id}.") # <<<< UPDATED NUMBER
+        else:
+            logger.warning("[Fallback YT 2] Supadata API key unavailable. Skipping.") # <<<< UPDATED NUMBER
+
+    # --- Fallback 3: Apify Default YT Actor ---
     if transcript_text is None:
-        logger.info("[Fallback YT 2] Trying Apify REST API (Default YT Actor)...")
+        logger.info("[Fallback YT 3] Trying Apify REST API (Default YT Actor)...") # <<<< UPDATED NUMBER
         if _apify_token_exists:
+            # This function already uses the generic helper _run_apify_actor_for_web_content
             transcript_text = await get_transcript_via_apify(video_url, APIFY_API_TOKEN)
-            if transcript_text: logger.info(f"[Fallback YT 2] Success via Apify Default YT Actor for {video_url}"); return transcript_text
-            else: logger.warning(f"[Fallback YT 2] Apify Default YT Actor failed or no content for {video_url}.")
-        else: logger.warning("[Fallback YT 2] Apify API token unavailable. Skipping.")
-    if transcript_text is None: logger.error(f"All methods failed for YT transcript: {video_id}"); return None
+            if transcript_text:
+                logger.info(f"[Fallback YT 3] Success via Apify Default YT Actor for {video_url}") # <<<< UPDATED NUMBER
+                return transcript_text # Return on success
+            else:
+                logger.warning(f"[Fallback YT 3] Apify Default YT Actor failed or no content for {video_url}.") # <<<< UPDATED NUMBER
+        else:
+            logger.warning("[Fallback YT 3] Apify API token unavailable. Skipping Default YT Actor.") # <<<< UPDATED NUMBER
+
+    # --- Final Outcome ---
+    if transcript_text is None:
+        logger.error(f"All methods failed for YT transcript: {video_id}")
+        return None # Explicitly return None if all failed
+
+    # This line should theoretically not be reached if logic above is correct,
+    # but return transcript_text just in case.
     return transcript_text
 
 async def fetch_url_content_for_scrape(url: str, timeout: int = 25) -> Optional[str]:
@@ -1274,10 +1350,12 @@ async def lifespan(app: Starlette):
 
 async def health_check(request: Request) -> PlainTextResponse:
     """Simple health check endpoint."""
-    global GROQ_LLAMA4_MODEL, GEMINI_PRO_EXP_MODEL, GEMINI_FLASH_MODEL, OPENROUTER_DEEPSEEK_MODEL, APIFY_ACTOR_ID
+    global GROQ_LLAMA4_MODEL, GEMINI_PRO_EXP_MODEL, GEMINI_FLASH_MODEL, OPENROUTER_DEEPSEEK_MODEL
+    global APIFY_ACTOR_ID, APIFY_STRUCTURED_YT_ACTOR_ID # Add new ID here
     global _groq_enabled, _gemini_api_enabled, _openrouter_fallback_enabled
     global _apify_token_exists, _urltotext_key_exists, _rapidapi_key_exists, SUPADATA_API_KEY
 
+    # ... (rest of the bot status checking logic remains the same) ...
     bot_status = "Not Initialized"; bot_username = "N/A"
     if ptb_app:
         try:
@@ -1292,7 +1370,8 @@ async def health_check(request: Request) -> PlainTextResponse:
              bot_status = f"Error checking status: {type(e).__name__}"; logger.warning(f"Health check: Error getting bot info: {e}")
     else: bot_status = "Not Initialized"; bot_username = "N/A"
 
-    # <<< Update response string with specific model names >>>
+
+    # <<< Update response string with specific model names AND YT Fallback order >>>
     return PlainTextResponse(
         f"TG Bot Summariser - Status: {bot_status} ({bot_username})\n"
         f"---\n"
@@ -1303,12 +1382,16 @@ async def health_check(request: Request) -> PlainTextResponse:
         f"4. OpenRouter API: {OPENROUTER_DEEPSEEK_MODEL if _openrouter_fallback_enabled else 'DISABLED'}\n"
         f"---\n"
         f"Content Fetching Status:\n"
-        f"YT Fallback 1 (Supadata): {'Enabled' if SUPADATA_API_KEY else 'Disabled'}\n"
-        f"YT Fallback 2 (Apify Actor): {APIFY_ACTOR_ID if _apify_token_exists else 'DISABLED'}\n"
+        # --- Updated YT Fallback List ---
+        f"YT Primary (Lib): Enabled\n"
+        f"YT Fallback 1 (Apify Structured): {APIFY_STRUCTURED_YT_ACTOR_ID if _apify_token_exists else 'DISABLED'}\n"
+        f"YT Fallback 2 (Supadata): {'Enabled' if SUPADATA_API_KEY else 'Disabled'}\n"
+        f"YT Fallback 3 (Apify Default): {APIFY_ACTOR_ID if _apify_token_exists else 'DISABLED'}\n"
+        # --- End Updated List ---
         f"Web Scrape 1 (Direct+BS4): Enabled\n"
         f"Web Scrape 2 (urltotext): {'Enabled' if _urltotext_key_exists else 'Disabled'}\n"
         f"Web Scrape 3/4 (RapidAPI): {'Enabled' if _rapidapi_key_exists else 'Disabled'}\n"
-        f"Web Scrape 5/6 (Apify Actors): {'Enabled' if _apify_token_exists else 'Disabled'}"
+        f"Web Scrape 5/6 (Apify Actors): {'Enabled' if _apify_token_exists else 'Disabled'}" # Web scrape order already updated previously
     )
 
 async def telegram_webhook(request: Request) -> Response: