Update main.py

main.py

@@ -1,4 +1,4 @@
-# main.py (Implementing Async Apify Flow)
+# main.py (Changing Default Apify Actor ID)
 import os
 import re
 import logging
@@ -71,7 +71,8 @@ APIFY_API_TOKEN = get_secret('APIFY_API_TOKEN')
 WEBHOOK_SECRET = get_secret('WEBHOOK_SECRET')
 
 OPENROUTER_MODEL = os.environ.get("OPENROUTER_MODEL", "deepseek/deepseek-chat-v3-0324:free")
-APIFY_ACTOR_ID = os.environ.get("APIFY_ACTOR_ID", "karamelo/youtube-transcripts")
+# *** FIX: Change default actor back to pocesar/youtube-scraper ***
+APIFY_ACTOR_ID = os.environ.get("APIFY_ACTOR_ID", "pocesar/youtube-scraper")
 
 if not TELEGRAM_TOKEN: logger.critical("❌ FATAL: TELEGRAM_TOKEN not found."); raise RuntimeError("Exiting: Telegram token missing.")
 if not OPENROUTER_API_KEY: logger.error("❌ ERROR: OPENROUTER_API_KEY not found. Summarization will fail.")
@@ -83,7 +84,7 @@ if not WEBHOOK_SECRET: logger.info("Optional secret 'WEBHOOK_SECRET' not found.
 
 logger.info("Secret loading and configuration check finished.")
 logger.info(f"Using OpenRouter Model: {OPENROUTER_MODEL}")
-logger.info(f"Using Apify Actor (via REST): {APIFY_ACTOR_ID}")
+logger.info(f"Using Apify Actor (via REST): {APIFY_ACTOR_ID}") # Will show pocesar now if default
 
 _apify_token_exists = bool(APIFY_API_TOKEN)
 
@@ -109,6 +110,7 @@ def extract_youtube_id(url):
     else: logger.warning(f"Could not extract YT ID from {url}"); return None
 
 # --- Content Fetching Functions ---
+# (fetch_url_content_for_scrape, get_transcript_via_supadata remain the same)
 async def fetch_url_content_for_scrape(url: str, timeout: int = 25) -> Optional[str]:
     headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Connection': 'keep-alive', 'DNT': '1', 'Upgrade-Insecure-Requests': '1' }
     try:
@@ -160,74 +162,69 @@ async def get_transcript_via_supadata(video_id: str, api_key: str) -> Optional[s
 
 async def get_transcript_via_apify(video_url: str, api_token: str) -> Optional[str]:
     """Fetches YouTube transcript using Apify REST API (async start + poll + dataset fetch)."""
-    global APIFY_ACTOR_ID
+    global APIFY_ACTOR_ID # Uses the globally defined actor ID
     if not video_url: logger.error("[Apify Async] No video_url provided"); return None
     if not api_token: logger.error("[Apify Async] API token missing."); return None
     logger.info(f"[Apify Async] Attempting fetch for URL: {video_url} (Actor: {APIFY_ACTOR_ID})")
 
     start_run_endpoint = f"https://api.apify.com/v2/acts/{APIFY_ACTOR_ID}/runs"
     params_base = {"token": api_token}
+    # Adjust payload based on the specific actor (pocesar vs karamelo) if necessary
+    # This payload seems generic enough for pocesar/youtube-scraper too, might need adjustment
     payload = {
-        "urls": [video_url], "outputFormat": "singleStringText", "maxRetries": 5,
-        "channelHandleBoolean": False, "channelNameBoolean": False,
-        "datePublishedBoolean": False, "relativeDateTextBoolean": False,
+        "startUrls": [{"url": video_url}], # pocesar often uses startUrls
+        # "urls": [video_url], # karamelo used urls
+        "proxyConfiguration": {"useApifyProxy": True}, # Common setting
+        "subtitles": True, # Explicitly request if pocesar supports it like this
+        "maxResultStreams": 0, "maxResults": 1, # Limit results
+        # Check pocesar/youtube-scraper docs for exact options
     }
     headers = {"Content-Type": "application/json"}
 
     run_id: Optional[str] = None
     dataset_id: Optional[str] = None
-    max_wait_seconds = 120 # Total time to wait for the run to finish
-    poll_interval = 5 # Seconds between status checks
+    max_wait_seconds = 120
+    poll_interval = 5
 
     try:
-        async with httpx.AsyncClient(timeout=30.0) as client: # Shorter timeout for individual API calls
+        async with httpx.AsyncClient(timeout=30.0) as client:
             # 1. Start the run
-            logger.debug(f"[Apify Async] Starting actor run for {video_url}")
+            logger.debug(f"[Apify Async] Starting actor run for {video_url} using actor {APIFY_ACTOR_ID}")
             response_start = await client.post(start_run_endpoint, headers=headers, params=params_base, json=payload)
             logger.debug(f"[Apify Async] Start run status: {response_start.status_code}")
 
-            if response_start.status_code == 201: # Expected status for starting a run
+            if response_start.status_code == 201:
                 try:
                     run_data = response_start.json().get("data", {})
                     run_id = run_data.get("id")
                     dataset_id = run_data.get("defaultDatasetId")
-                    if not run_id or not dataset_id:
-                        logger.error(f"[Apify Async] Started run but missing runId or datasetId. Data: {run_data}")
-                        return None
+                    if not run_id or not dataset_id: logger.error(f"[Apify Async] Started run but missing runId or datasetId. Data: {run_data}"); return None
                     logger.info(f"[Apify Async] Run started. Run ID: {run_id}, Dataset ID: {dataset_id}")
-                except Exception as e:
-                    logger.error(f"[Apify Async] Error parsing start run response: {e}. Response: {response_start.text[:200]}", exc_info=True)
-                    return None
+                except Exception as e: logger.error(f"[Apify Async] Error parsing start run response: {e}. Response: {response_start.text[:200]}", exc_info=True); return None
             else:
-                logger.error(f"[Apify Async] Failed to start run. Status: {response_start.status_code}. Response: {response_start.text[:200]}")
+                # Log specific error if available from Apify response
+                error_info = ""
+                try: error_info = response_start.json().get("error", {}).get("message", "")
+                except Exception: pass
+                logger.error(f"[Apify Async] Failed to start run. Status: {response_start.status_code}. Error: {error_info} Resp: {response_start.text[:200]}")
                 return None
 
             # 2. Poll for run completion
             run_status_endpoint = f"https://api.apify.com/v2/actor-runs/{run_id}"
-            elapsed_time = 0
-            final_status = None
+            elapsed_time = 0; final_status = None
             while elapsed_time < max_wait_seconds:
-                await asyncio.sleep(poll_interval)
-                elapsed_time += poll_interval
+                await asyncio.sleep(poll_interval); elapsed_time += poll_interval
                 logger.debug(f"[Apify Async] Polling status for run {run_id} ({elapsed_time}s elapsed)")
                 try:
                     response_status = await client.get(run_status_endpoint, params=params_base)
                     if response_status.status_code == 200:
-                        status_data = response_status.json().get("data", {})
-                        final_status = status_data.get("status")
+                        status_data = response_status.json().get("data", {}); final_status = status_data.get("status")
                         logger.debug(f"[Apify Async] Run status: {final_status}")
-                        if final_status in ["SUCCEEDED", "FAILED", "ABORTED", "TIMED-OUT"]:
-                            break # Exit polling loop on terminal status
-                    else:
-                        logger.warning(f"[Apify Async] Non-200 status ({response_status.status_code}) polling run {run_id}.")
-                        await asyncio.sleep(poll_interval * 2) # Wait longer on error
-                except Exception as poll_err:
-                    logger.error(f"[Apify Async] Error polling run status {run_id}: {poll_err}")
-                    await asyncio.sleep(poll_interval * 2) # Wait longer on error
-
-            if final_status != "SUCCEEDED":
-                logger.warning(f"[Apify Async] Run {run_id} did not succeed. Final status: {final_status}")
-                return None
+                        if final_status in ["SUCCEEDED", "FAILED", "ABORTED", "TIMED-OUT"]: break
+                    else: logger.warning(f"[Apify Async] Non-200 status ({response_status.status_code}) polling run {run_id}."); await asyncio.sleep(poll_interval * 2)
+                except Exception as poll_err: logger.error(f"[Apify Async] Error polling run status {run_id}: {poll_err}"); await asyncio.sleep(poll_interval * 2)
+
+            if final_status != "SUCCEEDED": logger.warning(f"[Apify Async] Run {run_id} did not succeed. Final status: {final_status}"); return None
 
             # 3. Fetch dataset items
             logger.info(f"[Apify Async] Run {run_id} succeeded. Fetching items from dataset {dataset_id}")
@@ -235,14 +232,27 @@ async def get_transcript_via_apify(video_url: str, api_token: str) -> Optional[s
             params_dataset = {"token": api_token, "format": "json", "limit": 5}
             response_dataset = await client.get(dataset_endpoint, params=params_dataset)
             logger.debug(f"[Apify Async] Dataset fetch status: {response_dataset.status_code}")
-            response_dataset.raise_for_status() # Raise exception for non-2xx status
+            response_dataset.raise_for_status()
 
             results = response_dataset.json()
             if isinstance(results, list) and len(results) > 0:
                 item = results[0]
-                content = item.get("text") or item.get("transcript") or item.get("captions_concatenated")
-                if not content and item.get("captions") and isinstance(item["captions"], list):
-                    content = " ".join(cap.get("text", "") for cap in item["captions"] if cap.get("text"))
+                # --- Adapt parsing for pocesar/youtube-scraper ---
+                # This actor often puts the transcript directly under a 'subtitles' key,
+                # possibly as a single string or sometimes structured.
+                content = item.get("subtitles") # Check 'subtitles' first
+                if not content: content = item.get("text") # Fallback check
+                if not content: content = item.get("transcript") # Another fallback
+
+                # If subtitles is a list of dicts (like karamelo's output sometimes)
+                if isinstance(content, list) and len(content) > 0 and isinstance(content[0], dict) and 'lines' in content[0]:
+                     logger.info("[Apify Async] Processing structured subtitles format.")
+                     content = " ".join(line.get("text", "") for line in content[0].get('lines', []) if line.get("text"))
+                elif isinstance(content, list): # Handle simple list of strings if found
+                     logger.info("[Apify Async] Processing list of strings format.")
+                     content = " ".join(content)
+
+                # Final check if we have a non-empty string
                 if content and isinstance(content, str):
                     logger.info(f"[Apify Async] Success via ASYNC REST for {video_url}. Length: {len(content)}")
                     return content.strip()
@@ -254,8 +264,8 @@ async def get_transcript_via_apify(video_url: str, api_token: str) -> Optional[s
     except httpx.RequestError as e: logger.error(f"[Apify Async] Request error during API interaction for {video_url}: {e}"); return None
     except Exception as e: logger.error(f"[Apify Async] Unexpected error during Apify Async REST call for {video_url}: {e}", exc_info=True); return None
 
+# (get_youtube_transcript, get_website_content, get_website_content_via_api, generate_summary remain the same)
 async def get_youtube_transcript(video_id: str, video_url: str) -> Optional[str]:
-    # This function remains the same, calling the updated helper functions
     global SUPADATA_API_KEY, APIFY_API_TOKEN
     if not video_id: logger.error("get_youtube_transcript: No video_id"); return None
     logger.info(f"Fetching transcript for video ID: {video_id} (URL: {video_url})")
@@ -279,17 +289,16 @@ async def get_youtube_transcript(video_id: str, video_url: str) -> Optional[str]
             else: logger.warning(f"[Fallback YT 1] Supadata failed or no content for {video_id}.")
         else: logger.warning("[Fallback YT 1] Supadata API key unavailable. Skipping.")
     if transcript_text is None:
-        logger.info("[Fallback YT 2] Trying Apify REST API (Async)...") # Updated log message
+        logger.info("[Fallback YT 2] Trying Apify REST API (Async)...")
         if APIFY_API_TOKEN:
-            transcript_text = await get_transcript_via_apify(video_url, APIFY_API_TOKEN) # Calls new async version
+            transcript_text = await get_transcript_via_apify(video_url, APIFY_API_TOKEN)
             if transcript_text: logger.info(f"[Fallback YT 2] Success via Apify ASYNC REST for {video_url}"); return transcript_text
             else: logger.warning(f"[Fallback YT 2] Apify ASYNC REST failed or no content for {video_url}.")
         else: logger.warning("[Fallback YT 2] Apify API token unavailable. Skipping.")
     if transcript_text is None: logger.error(f"All methods failed for YT transcript: {video_id}"); return None
-    return transcript_text # Will be None if all failed
+    return transcript_text
 
 async def get_website_content(url: str) -> Optional[str]:
-    # This function remains the same
     if not url: logger.error("get_website_content: No URL"); return None
     logger.info(f"[Primary Web] Fetching website content for: {url}")
     html_content = await fetch_url_content_for_scrape(url)
@@ -311,7 +320,6 @@ async def get_website_content(url: str) -> Optional[str]:
     except Exception as e: logger.error(f"[Primary Web] Error scraping/parsing {url}: {e}", exc_info=True); return None
 
 async def get_website_content_via_api(url: str, api_key: str) -> Optional[str]:
-    # This function remains the same
     if not url: logger.error("[Fallback Web API] No URL"); return None
     if not api_key: logger.error("[Fallback Web API] urltotext.com API key missing."); return None
     logger.info(f"[Fallback Web API] Attempting fetch for: {url} using urltotext.com API")
@@ -338,8 +346,6 @@ async def get_website_content_via_api(url: str, api_key: str) -> Optional[str]:
     except httpx.RequestError as e: logger.error(f"[Fallback Web API] Request error connecting to urltotext.com API for {url}: {e}"); return None
     except Exception as e: logger.error(f"[Fallback Web API] Unexpected error during urltotext.com API call for {url}: {e}", exc_info=True); return None
 
-# --- Summarization Function ---
-# (generate_summary remains the same)
 async def generate_summary(text: str, summary_type: str) -> str:
     global OPENROUTER_API_KEY, OPENROUTER_MODEL
     logger.info(f"Generating {summary_type} summary using {OPENROUTER_MODEL}. Input length: {len(text)}")
@@ -514,9 +520,7 @@ async def error_handler(update: object, context: ContextTypes.DEFAULT_TYPE) -> N
     if isinstance(context.error, ignore_errors) and "object has no attribute" in str(context.error): logger.debug(f"Ignoring known/handled error in error_handler: {context.error}"); return
     logger.error("Exception while handling an update:", exc_info=context.error)
 
-
 # --- Bot Setup ---
-# (setup_bot_config remains the same)
 async def setup_bot_config() -> Application:
     logger.info("Configuring Telegram Application..."); global TELEGRAM_TOKEN
     if not TELEGRAM_TOKEN: raise ValueError("TELEGRAM_TOKEN missing.")
@@ -572,6 +576,7 @@ async def lifespan(app: Starlette):
         else: logger.info("PTB application not initialized or failed.")
         logger.info("ASGI Lifespan: Shutdown complete.")
 
+
 # --- Starlette Route Handlers ---
 # (health_check and telegram_webhook remain the same)
 async def health_check(request: Request) -> PlainTextResponse:
@@ -596,6 +601,7 @@ async def telegram_webhook(request: Request) -> Response:
     except json.JSONDecodeError: logger.error("Webhook invalid JSON."); return PlainTextResponse('Bad Request: Invalid JSON', status_code=400)
     except Exception as e: logger.error(f"Error processing webhook update: {e}", exc_info=True); return Response(status_code=200) # OK despite error
 
+
 # --- Create Starlette ASGI Application ---
 # (app definition remains the same)
 app = Starlette( debug=False, lifespan=lifespan, routes=[ Route("/", endpoint=health_check, methods=["GET"]), Route("/webhook", endpoint=telegram_webhook, methods=["POST"]), ] )