Update main.py

main.py

@@ -205,9 +205,11 @@ def extract_youtube_id(url):
 # --- Content Fetching Functions ---
 
 async def get_transcript_via_supadata(video_id: str, api_key: str) -> Optional[str]:
+    """Fallback YT 3: Fetches YouTube transcript using Supadata API.""" # <<< UPDATED DOCSTRING
     if not video_id: logger.error("[Supadata] No video_id provided"); return None
     if not api_key: logger.error("[Supadata] API key missing."); return None
-    logger.info(f"[YT Fallback 1] Attempting fetch for video ID: {video_id} via Supadata")
+    # <<< UPDATED LOG MESSAGE NUMBER >>>
+    logger.info(f"[YT Fallback 3] Attempting fetch for video ID: {video_id} via Supadata")
     api_endpoint = "https://api.supadata.ai/v1/youtube/transcript"
     params = {"videoId": video_id, "format": "text"}; headers = {"X-API-Key": api_key}
     try:
@@ -220,7 +222,10 @@ async def get_transcript_via_supadata(video_id: str, api_key: str) -> Optional[s
                     content = None
                     if data: content = data if isinstance(data, str) else data.get("transcript") or data.get("text") or data.get("data")
                     if not content and response.text: content = response.text
-                    if content and isinstance(content, str): logger.info(f"[Supadata] Success for {video_id}. Length: {len(content)}"); return content.strip()
+                    if content and isinstance(content, str):
+                        # <<< UPDATED LOG MESSAGE NUMBER >>>
+                        logger.info(f"[Supadata] Success (Fallback 3) for {video_id}. Length: {len(content)}");
+                        return content.strip()
                     else: logger.warning(f"[Supadata] Success but content empty/invalid for {video_id}. Response: {response.text[:200]}"); return None
                 except json.JSONDecodeError: logger.warning(f"[Supadata] Received 200 but failed JSON decode for {video_id}. Using raw text if available. Response: {response.text[:200]}"); return response.text.strip() if response.text else None
                 except Exception as e: logger.error(f"[Supadata] Error processing success response for {video_id}: {e}", exc_info=True); return None
@@ -235,69 +240,27 @@ async def get_transcript_via_supadata(video_id: str, api_key: str) -> Optional[s
     except Exception as e: logger.error(f"[Supadata] Unexpected error for {video_id}: {e}", exc_info=True); return None
 
 async def get_transcript_via_apify(video_url: str, api_token: str) -> Optional[str]:
-    """Fallback YT 2: Fetches YouTube transcript using default Apify Actor."""
+    """Fallback YT 1: Fetches YouTube transcript using default Apify Actor via generic function.""" # <<< UPDATED DOCSTRING & NUMBER
     global APIFY_ACTOR_ID
-    if not video_url: logger.error("[Apify YT] No video_url provided"); return None
-    if not api_token: logger.error("[Apify YT] API token missing."); return None
-    logger.info(f"[YT Fallback 2] Attempting fetch for URL: {video_url} (Actor: {APIFY_ACTOR_ID})")
-    sync_items_endpoint = f"https://api.apify.com/v2/acts/{APIFY_ACTOR_ID}/run-sync-get-dataset-items"
-    params = {"token": api_token}
-    payload = { "urls": [video_url], "outputFormat": "singleStringText", "maxRetries": 5, "channelHandleBoolean": False, "channelNameBoolean": False, "datePublishedBoolean": False, "relativeDateTextBoolean": False, }
-    headers = {"Content-Type": "application/json"}
-    try: # OUTER TRY Block
-        async with httpx.AsyncClient(timeout=120.0) as client:
-            logger.debug(f"[Apify YT] POST Request to {sync_items_endpoint} for {video_url}")
-            response = await client.post(sync_items_endpoint, headers=headers, params=params, json=payload)
-            logger.debug(f"[Apify YT] Received status code {response.status_code} for {video_url}")
-
-            if response.status_code in [200, 201]:
-                if response.status_code == 201:
-                    logger.info(f"[Apify YT] Received status 201 (Created) from run-sync endpoint, processing results anyway.")
-                try: # INNER TRY for JSON processing
-                    results = response.json(); content = None
-                    # ... (existing logic for processing successful response) ...
-                    if isinstance(results, list) and len(results) > 0:
-                        # ... (extract content) ...
-                        if content and isinstance(content, str):
-                            logger.info(f"[Apify YT] Success via REST (Status {response.status_code}) for {video_url}. Length: {len(content)}")
-                            return content.strip()
-                        else:
-                            logger.warning(f"[Apify YT] Dataset item parsed (Status {response.status_code}) but transcript content empty/invalid format for {video_url}. Item keys: {list(results[0].keys() if results and isinstance(results[0],dict) else [])}")
-                            return None
-                    else:
-                        logger.warning(f"[Apify YT] Actor call successful (Status {response.status_code}) but dataset was empty for {video_url}. Response: {results}")
-                        return None
-                except json.JSONDecodeError: # INNER EXCEPT
-                    logger.error(f"[Apify YT] Failed JSON decode. Status:{response.status_code}. Resp:{response.text[:200]}");
-                    return None
-                except Exception as e: # INNER EXCEPT
-                    logger.error(f"[Apify YT] Error processing success response (Status {response.status_code}) for {video_url}: {e}", exc_info=True);
-                    return None
-            # ELIF chain for specific HTTP error codes from the API call
-            elif response.status_code == 400: logger.error(f"[Apify YT] Bad Request (400) for {video_url}. Check payload. Resp:{response.text[:200]}"); return None
-            elif response.status_code == 401: logger.error("[Apify YT] Auth error (401). Check token."); return None
-            elif response.status_code == 404: logger.error(f"[Apify YT] Endpoint/Actor Not Found (404). Actor: {APIFY_ACTOR_ID} Resp:{response.text[:200]}"); return None
-            else: logger.error(f"[Apify YT] Unexpected status {response.status_code} for {video_url}. Resp:{response.text[:200]}"); return None
-        # --- ADDED EXCEPTIONS FOR OUTER TRY ---
-    except httpx.TimeoutException as e:
-        logger.error(f"[Apify YT] Timeout during API interaction for {video_url}: {e}")
-        return None
-    except httpx.HTTPStatusError as e: # In case raise_for_status is added later or for specific HTTP errors not caught above
-        logger.error(f"[Apify YT] HTTP Status Error during API interaction for {video_url}: {e}")
-        return None
-    except httpx.RequestError as e: # Catches connection errors, DNS errors etc.
-        logger.error(f"[Apify YT] Request error during API interaction for {video_url}: {e}")
-        return None
-    except Exception as e: # Catch-all for any other unexpected error during the httpx call setup/execution
-        logger.error(f"[Apify YT] Unexpected error during Apify YT call for {video_url}: {e}", exc_info=True)
-        return None
+    # The specific run_input logic is now handled within _run_apify_actor_for_web_content
+    # when it detects the actor_id matches APIFY_ACTOR_ID
+    # <<< UPDATED LOG MESSAGE >>>
+    logger.debug(f"[get_transcript_via_apify - Fallback 1] Calling generic runner for URL: {video_url}")
+    return await _run_apify_actor_for_web_content(
+        url=video_url,  # Pass video_url as the 'url' parameter
+        api_token=api_token,
+        actor_id=APIFY_ACTOR_ID,
+        # <<< UPDATED ACTOR NAME IN LOGS >>>
+        actor_name="Apify YT Default (Fallback 1)"
+    )
 
 async def get_transcript_via_apify_structured_extractor(video_url: str, api_token: str) -> Optional[str]:
-    """Fallback YT 1: Fetches YouTube transcript using the Structured Extractor Apify Actor."""
+    """Fallback YT 2: Fetches YouTube transcript using the Structured Extractor Apify Actor.""" # <<< UPDATED DOCSTRING & NUMBER
     global APIFY_STRUCTURED_YT_ACTOR_ID # Use the new ID
     if not video_url: logger.error("[Apify Structured YT] No video_url provided"); return None
     if not api_token: logger.error("[Apify Structured YT] API token missing."); return None
-    logger.info(f"[YT Fallback 1] Attempting fetch for URL: {video_url} (Actor: {APIFY_STRUCTURED_YT_ACTOR_ID})")
+    # <<< UPDATED LOG MESSAGE NUMBER >>>
+    logger.info(f"[YT Fallback 2] Attempting fetch for URL: {video_url} (Actor: {APIFY_STRUCTURED_YT_ACTOR_ID})")
 
     # Use the generic helper function.
     # We assume the standard input format used by the helper for non-specific actors
@@ -309,90 +272,71 @@ async def get_transcript_via_apify_structured_extractor(video_url: str, api_toke
         url=video_url,
         api_token=api_token,
         actor_id=APIFY_STRUCTURED_YT_ACTOR_ID,
-        actor_name="Apify Structured YT" # Specific name for logging
+        # <<< UPDATED ACTOR NAME IN LOGS >>>
+        actor_name="Apify Structured YT (Fallback 2)" # Specific name for logging
     )
 
 async def get_youtube_transcript(video_id: str, video_url: str) -> Optional[str]:
-    """Fetches YouTube transcript using a primary library and multiple fallback methods."""
-    global SUPADATA_API_KEY, APIFY_API_TOKEN, _apify_token_exists # Keep existing globals
+    """
+    Fetches YouTube transcript using multiple fallback methods in the specified order:
+    1. Apify Default Actor (1s7eXiaukVuOr4Ueg)
+    2. Apify Structured Actor (gpjTCWkGZS1lHc9pR)
+    3. Supadata API
+    """
+    global SUPADATA_API_KEY, APIFY_API_TOKEN, _apify_token_exists
     if not video_id: logger.error("get_youtube_transcript: No video_id"); return None
 
-    logger.info(f"Fetching transcript for video ID: {video_id} (URL: {video_url})")
+    logger.info(f"Fetching transcript for video ID: {video_id} (URL: {video_url}) - NEW Fallback Order")
     transcript_text: Optional[str] = None
 
-    # --- Primary Method: youtube-transcript-api ---
-    logger.info("[Primary YT] Attempting youtube-transcript-api...")
-    try:
-        # Prefer English variants first
-        transcript_list = await asyncio.to_thread(
-            YouTubeTranscriptApi.get_transcript, video_id, languages=['en', 'en-GB', 'en-US']
-        )
-        if transcript_list:
-            transcript_text = " ".join([item['text'] for item in transcript_list if 'text' in item])
-        if transcript_text:
-            logger.info(f"[Primary YT] Success via lib for {video_id} (len: {len(transcript_text)})")
-            return transcript_text.strip() # Return immediately on success
-        else:
-            logger.warning(f"[Primary YT] Transcript list returned but text content was empty for {video_id}")
-            transcript_text = None # Ensure it's None to trigger fallbacks
-    except NoTranscriptFound:
-        logger.warning(f"[Primary YT] No transcript found via lib for {video_id}.")
-        transcript_text = None
-    except TranscriptsDisabled:
-        logger.warning(f"[Primary YT] Transcripts disabled via lib for {video_id}.")
-        transcript_text = None
-    except Exception as e:
-        # Log more specific errors if possible, e.g., timeouts, network issues
-        logger.warning(f"[Primary YT] Error via lib for {video_id}: {e}")
-        transcript_text = None
+    # --- Primary Method: REMOVED (youtube-transcript-api) ---
+    # logger.info("[Primary YT] Attempting youtube-transcript-api...") # Removed
 
-    # --- Fallback 1: Apify Structured Transcript Extractor (NEW) ---
+    # --- Fallback 1: Apify Default YT Actor (1s7eXiaukVuOr4Ueg) ---
     if transcript_text is None:
-        logger.info("[Fallback YT 1] Trying Apify Structured Transcript Extractor...")
+        logger.info("[Fallback YT 1] Trying Apify REST API (Default YT Actor)...") # <<<< NEW Fallback 1
         if _apify_token_exists:
-            transcript_text = await get_transcript_via_apify_structured_extractor(video_url, APIFY_API_TOKEN)
+            transcript_text = await get_transcript_via_apify(video_url, APIFY_API_TOKEN)
             if transcript_text:
-                logger.info(f"[Fallback YT 1] Success via Apify Structured Extractor for {video_url}")
+                logger.info(f"[Fallback YT 1] Success via Apify Default YT Actor for {video_url}") # <<<< UPDATED NUMBER
                 return transcript_text # Return on success
             else:
-                logger.warning(f"[Fallback YT 1] Apify Structured Extractor failed or no content for {video_url}.")
+                logger.warning(f"[Fallback YT 1] Apify Default YT Actor failed or no content for {video_url}.") # <<<< UPDATED NUMBER
         else:
-            logger.warning("[Fallback YT 1] Apify API token unavailable. Skipping Apify Structured Extractor.")
+            logger.warning("[Fallback YT 1] Apify API token unavailable. Skipping Default YT Actor.") # <<<< UPDATED NUMBER
 
-    # --- Fallback 2: Supadata API ---
+    # --- Fallback 2: Apify Structured Transcript Extractor (gpjTCWkGZS1lHc9pR) ---
     if transcript_text is None:
-        logger.info("[Fallback YT 2] Trying Supadata API...") # <<<< UPDATED NUMBER
-        if SUPADATA_API_KEY:
-            transcript_text = await get_transcript_via_supadata(video_id, SUPADATA_API_KEY)
+        logger.info("[Fallback YT 2] Trying Apify Structured Transcript Extractor...") # <<<< NEW Fallback 2
+        if _apify_token_exists:
+            transcript_text = await get_transcript_via_apify_structured_extractor(video_url, APIFY_API_TOKEN)
             if transcript_text:
-                logger.info(f"[Fallback YT 2] Success via Supadata for {video_id}") # <<<< UPDATED NUMBER
+                logger.info(f"[Fallback YT 2] Success via Apify Structured Extractor for {video_url}") # <<<< UPDATED NUMBER
                 return transcript_text # Return on success
             else:
-                logger.warning(f"[Fallback YT 2] Supadata failed or no content for {video_id}.") # <<<< UPDATED NUMBER
+                logger.warning(f"[Fallback YT 2] Apify Structured Extractor failed or no content for {video_url}.") # <<<< UPDATED NUMBER
         else:
-            logger.warning("[Fallback YT 2] Supadata API key unavailable. Skipping.") # <<<< UPDATED NUMBER
+            logger.warning("[Fallback YT 2] Apify API token unavailable. Skipping Apify Structured Extractor.") # <<<< UPDATED NUMBER
 
-    # --- Fallback 3: Apify Default YT Actor ---
+    # --- Fallback 3: Supadata API ---
     if transcript_text is None:
-        logger.info("[Fallback YT 3] Trying Apify REST API (Default YT Actor)...") # <<<< UPDATED NUMBER
-        if _apify_token_exists:
-            # This function already uses the generic helper _run_apify_actor_for_web_content
-            transcript_text = await get_transcript_via_apify(video_url, APIFY_API_TOKEN)
+        logger.info("[Fallback YT 3] Trying Supadata API...") # <<<< NEW Fallback 3
+        if SUPADATA_API_KEY:
+            transcript_text = await get_transcript_via_supadata(video_id, SUPADATA_API_KEY)
             if transcript_text:
-                logger.info(f"[Fallback YT 3] Success via Apify Default YT Actor for {video_url}") # <<<< UPDATED NUMBER
+                logger.info(f"[Fallback YT 3] Success via Supadata for {video_id}") # <<<< UPDATED NUMBER
                 return transcript_text # Return on success
             else:
-                logger.warning(f"[Fallback YT 3] Apify Default YT Actor failed or no content for {video_url}.") # <<<< UPDATED NUMBER
+                logger.warning(f"[Fallback YT 3] Supadata failed or no content for {video_id}.") # <<<< UPDATED NUMBER
         else:
-            logger.warning("[Fallback YT 3] Apify API token unavailable. Skipping Default YT Actor.") # <<<< UPDATED NUMBER
+            logger.warning("[Fallback YT 3] Supadata API key unavailable. Skipping.") # <<<< UPDATED NUMBER
 
     # --- Final Outcome ---
     if transcript_text is None:
-        logger.error(f"All methods failed for YT transcript: {video_id}")
+        logger.error(f"All fallback methods failed for YT transcript: {video_id}")
         return None # Explicitly return None if all failed
 
-    # This line should theoretically not be reached if logic above is correct,
-    # but return transcript_text just in case.
+    # This line should only be reached if a fallback succeeded but wasn't returned early (shouldn't happen).
     return transcript_text
 
 async def fetch_url_content_for_scrape(url: str, timeout: int = 25) -> Optional[str]:
@@ -1505,7 +1449,7 @@ async def health_check(request: Request) -> PlainTextResponse:
     else: bot_status = "Not Initialized"; bot_username = "N/A"
 
 
-    # <<< Update response string with the new model order >>>
+    # <<< Update response string with the NEW YT fallback order >>>
     return PlainTextResponse(
         f"TG Bot Summariser - Status: {bot_status} ({bot_username})\n"
         f"---\n"
@@ -1517,12 +1461,12 @@ async def health_check(request: Request) -> PlainTextResponse:
         f"5. Groq API: {GROQ_LLAMA4_MODEL if _groq_enabled else 'DISABLED'} (Last Fallback)\n"
         f"---\n"
         f"Content Fetching Status:\n"
-        # --- YT Fallback List (Order already correct) ---
-        f"YT Primary (Lib): Enabled\n"
-        f"YT Fallback 1 (Apify Structured): {APIFY_STRUCTURED_YT_ACTOR_ID if _apify_token_exists else 'DISABLED'}\n"
-        f"YT Fallback 2 (Supadata): {'Enabled' if SUPADATA_API_KEY else 'Disabled'}\n"
-        f"YT Fallback 3 (Apify Default): {APIFY_ACTOR_ID if _apify_token_exists else 'DISABLED'}\n"
-        # --- Web Scrape Fallback List (Order already correct) ---
+        # --- YT Fallback List (NEW ORDER) ---
+        f"YT Primary (Lib): REMOVED\n"
+        f"YT Fallback 1 (Apify Default): {APIFY_ACTOR_ID if _apify_token_exists else 'DISABLED'}\n"
+        f"YT Fallback 2 (Apify Structured): {APIFY_STRUCTURED_YT_ACTOR_ID if _apify_token_exists else 'DISABLED'}\n"
+        f"YT Fallback 3 (Supadata): {'Enabled' if SUPADATA_API_KEY else 'Disabled'}\n"
+        # --- Web Scrape Fallback List (Order Unchanged) ---
         f"Web Scrape 1 (Direct+BS4): Enabled\n"
         f"Web Scrape 2 (urltotext): {'Enabled' if _urltotext_key_exists else 'Disabled'}\n"
         f"Web Scrape 3/4 (RapidAPI): {'Enabled' if _rapidapi_key_exists else 'Disabled'}\n"