Update main.py

main.py

@@ -1,4 +1,4 @@
-# main.py (Correcting SyntaxError at line 192)
+# main.py (Correcting Supadata URL and Apify Endpoint/Logic)
 import os
 import re
 import logging
@@ -71,7 +71,7 @@ APIFY_API_TOKEN = get_secret('APIFY_API_TOKEN')
 WEBHOOK_SECRET = get_secret('WEBHOOK_SECRET')
 
 OPENROUTER_MODEL = os.environ.get("OPENROUTER_MODEL", "deepseek/deepseek-chat-v3-0324:free")
-APIFY_ACTOR_ID = os.environ.get("APIFY_ACTOR_ID", "karamelo~youtube-transcripts")
+APIFY_ACTOR_ID = os.environ.get("APIFY_ACTOR_ID", "karamelo~youtube-transcripts") # Keep karamelo as per docs
 
 if not TELEGRAM_TOKEN: logger.critical("❌ FATAL: TELEGRAM_TOKEN not found."); raise RuntimeError("Exiting: Telegram token missing.")
 if not OPENROUTER_API_KEY: logger.error("❌ ERROR: OPENROUTER_API_KEY not found. Summarization will fail.")
@@ -132,7 +132,8 @@ async def get_transcript_via_supadata(video_id: str, api_key: str) -> Optional[s
     if not video_id: logger.error("[Supadata] No video_id provided"); return None
     if not api_key: logger.error("[Supadata] API key missing."); return None
     logger.info(f"[Supadata] Attempting fetch for video ID: {video_id}")
-    api_endpoint = "https://api.supadata.net/v1/youtube/transcript"
+    # *** FIX: Use correct base URL ***
+    api_endpoint = "https://api.supadata.ai/v1/youtube/transcript"
     params = {"videoId": video_id, "format": "text"}; headers = {"X-API-Key": api_key}
     try:
         async with httpx.AsyncClient(timeout=30.0) as client:
@@ -144,7 +145,7 @@ async def get_transcript_via_supadata(video_id: str, api_key: str) -> Optional[s
                     except json.JSONDecodeError: data = None
                     content = None
                     if data: content = data if isinstance(data, str) else data.get("transcript") or data.get("text") or data.get("data")
-                    if not content and response.text: content = response.text
+                    if not content and response.text: content = response.text # Check plain text response
                     if content and isinstance(content, str): logger.info(f"[Supadata] Success for {video_id}. Length: {len(content)}"); return content.strip()
                     else: logger.warning(f"[Supadata] Success but content empty/invalid for {video_id}. Response: {response.text[:200]}"); return None
                 except Exception as e: logger.error(f"[Supadata] Error processing success response for {video_id}: {e}", exc_info=True); return None
@@ -153,94 +154,73 @@ async def get_transcript_via_supadata(video_id: str, api_key: str) -> Optional[s
             else: logger.error(f"[Supadata] Unexpected status {response.status_code} for {video_id}. Resp: {response.text[:200]}"); return None
     except httpx.TimeoutException: logger.error(f"[Supadata] Timeout connecting for {video_id}"); return None
     except httpx.RequestError as e:
+        # SSL errors usually fall under RequestError
         if "CERTIFICATE_VERIFY_FAILED" in str(e): logger.error(f"[Supadata] SSL Cert Verify Failed for {video_id}: {e}")
         else: logger.error(f"[Supadata] Request error for {video_id}: {e}")
         return None
     except Exception as e: logger.error(f"[Supadata] Unexpected error for {video_id}: {e}", exc_info=True); return None
 
-
 async def get_transcript_via_apify(video_url: str, api_token: str) -> Optional[str]:
-    """Fetches YouTube transcript using Apify REST API (async start + poll + dataset fetch)."""
-    global APIFY_ACTOR_ID
-    if not video_url: logger.error("[Apify Async] No video_url provided"); return None
-    if not api_token: logger.error("[Apify Async] API token missing."); return None
-    logger.info(f"[Apify Async] Attempting fetch for URL: {video_url} (Actor: {APIFY_ACTOR_ID})")
-
-    start_run_endpoint = f"https://api.apify.com/v2/acts/{APIFY_ACTOR_ID}/runs"
-    params_base = {} # Token goes in header now
-    payload = { "urls": [video_url], "outputFormat": "singleStringText", "maxRetries": 5, "channelHandleBoolean": False, "channelNameBoolean": False, "datePublishedBoolean": False, "relativeDateTextBoolean": False, }
-    headers = { "Content-Type": "application/json", "Authorization": f"Bearer {api_token}" }
-
-    run_id: Optional[str] = None; dataset_id: Optional[str] = None
-    max_wait_seconds = 120; poll_interval = 5
+    """Fetches YouTube transcript using Apify REST API (run-sync-get-dataset-items endpoint)."""
+    global APIFY_ACTOR_ID # Should be karamelo~youtube-transcripts
+    if not video_url: logger.error("[Apify SyncItems] No video_url provided"); return None
+    if not api_token: logger.error("[Apify SyncItems] API token missing."); return None
+    logger.info(f"[Apify SyncItems] Attempting fetch for URL: {video_url} (Actor: {APIFY_ACTOR_ID})")
+
+    # *** FIX: Use the run-sync-get-dataset-items endpoint ***
+    sync_items_endpoint = f"https://api.apify.com/v2/acts/{APIFY_ACTOR_ID}/run-sync-get-dataset-items"
+    params = {"token": api_token} # Token in param as per OpenAPI spec for this endpoint
+    payload = {
+        "urls": [video_url],
+        "outputFormat": "singleStringText",
+        "maxRetries": 5,
+        "channelHandleBoolean": False, "channelNameBoolean": False,
+        "datePublishedBoolean": False, "relativeDateTextBoolean": False,
+    }
+    headers = {"Content-Type": "application/json"} # No Auth header needed if token in params
 
     try:
-        async with httpx.AsyncClient(timeout=30.0, headers=headers) as client:
-            log_headers = {k: v for k, v in headers.items() if k.lower() != 'authorization'}
-            logger.debug(f"[Apify Async] POST Request Details:\nURL: {start_run_endpoint}\nHeaders: {log_headers}\nPayload: {json.dumps(payload)}")
-            response_start = await client.post(start_run_endpoint, json=payload)
-            logger.debug(f"[Apify Async] Start run status: {response_start.status_code}")
-
-            if response_start.status_code == 201:
-                try:
-                    run_data = response_start.json().get("data", {})
-                    run_id = run_data.get("id"); dataset_id = run_data.get("defaultDatasetId")
-                    if not run_id or not dataset_id: logger.error(f"[Apify Async] Started run but missing runId or datasetId. Data: {run_data}"); return None
-                    logger.info(f"[Apify Async] Run started. Run ID: {run_id}, Dataset ID: {dataset_id}")
-                except Exception as e: logger.error(f"[Apify Async] Error parsing start run response: {e}. Response: {response_start.text[:200]}", exc_info=True); return None
-            else:
-                # *** FIX: Correct Indentation for try/except block ***
-                error_info = ""
-                try:
-                    # Attempt to get the error message from the JSON response
-                    error_info = response_start.json().get("error", {}).get("message", "")
-                except Exception:
-                    # If parsing fails or structure is unexpected, just pass
-                    pass
-                logger.error(f"[Apify Async] Failed to start run. Status: {response_start.status_code}. Error: '{error_info}' Resp: {response_start.text[:200]}")
-                return None
-
-            run_status_endpoint = f"https://api.apify.com/v2/actor-runs/{run_id}"; elapsed_time = 0; final_status = None
-            while elapsed_time < max_wait_seconds:
-                await asyncio.sleep(poll_interval); elapsed_time += poll_interval
-                logger.debug(f"[Apify Async] Polling status for run {run_id} ({elapsed_time}s elapsed)")
+        # Use a longer timeout for this synchronous endpoint
+        async with httpx.AsyncClient(timeout=120.0) as client:
+            log_headers = {k: v for k, v in headers.items()} # Log headers
+            logger.debug(f"[Apify SyncItems] POST Request Details:\nURL: {sync_items_endpoint}\nParams: {params}\nHeaders: {log_headers}\nPayload: {json.dumps(payload)}")
+            # *** FIX: POST to the sync items endpoint ***
+            response = await client.post(sync_items_endpoint, headers=headers, params=params, json=payload)
+            logger.debug(f"[Apify SyncItems] Received status code {response.status_code} for {video_url}")
+
+            # *** FIX: Expect 200 OK for this endpoint ***
+            if response.status_code == 200:
                 try:
-                    response_status = await client.get(run_status_endpoint)
-                    if response_status.status_code == 200:
-                        status_data = response_status.json().get("data", {}); final_status = status_data.get("status")
-                        logger.debug(f"[Apify Async] Run status: {final_status}")
-                        if final_status in ["SUCCEEDED", "FAILED", "ABORTED", "TIMED-OUT"]: break
-                    else: logger.warning(f"[Apify Async] Non-200 status ({response_status.status_code}) polling run {run_id}."); await asyncio.sleep(poll_interval * 2)
-                except Exception as poll_err: logger.error(f"[Apify Async] Error polling run status {run_id}: {poll_err}"); await asyncio.sleep(poll_interval * 2)
-
-            if final_status != "SUCCEEDED": logger.warning(f"[Apify Async] Run {run_id} did not succeed. Final status: {final_status}"); return None
-
-            logger.info(f"[Apify Async] Run {run_id} succeeded. Fetching items from dataset {dataset_id}")
-            dataset_endpoint = f"https://api.apify.com/v2/datasets/{dataset_id}/items"
-            params_dataset = {"format": "json", "limit": 5}
-            response_dataset = await client.get(dataset_endpoint, params=params_dataset)
-            logger.debug(f"[Apify Async] Dataset fetch status: {response_dataset.status_code}")
-            response_dataset.raise_for_status()
-
-            results = response_dataset.json()
-            if isinstance(results, list) and len(results) > 0:
-                item = results[0]
-                content = None
-                if "captions" in item and isinstance(item["captions"], str): logger.info("[Apify Async] Found 'captions' key with string content."); content = item["captions"]
-                elif "text" in item and isinstance(item["text"], str): logger.info("[Apify Async] Found 'text' key with string content."); content = item["text"]
-                elif "transcript" in item and isinstance(item["transcript"], str): logger.info("[Apify Async] Found 'transcript' key with string content."); content = item["transcript"]
-                elif "captions" in item and isinstance(item["captions"], list):
-                     logger.warning("[Apify Async] Received list format for 'captions' unexpectedly. Processing...")
-                     if len(item["captions"]) > 0 and isinstance(item["captions"][0], dict) and 'text' in item["captions"][0]: content = " ".join(line.get("text", "") for line in item["captions"] if line.get("text"))
-                     elif len(item["captions"]) > 0 and isinstance(item["captions"][0], str): content = " ".join(item["captions"])
-                if content and isinstance(content, str): logger.info(f"[Apify Async] Success via ASYNC REST for {video_url}. Length: {len(content)}"); return content.strip()
-                else: logger.warning(f"[Apify Async] Dataset item parsed but transcript content empty/invalid format for {video_url}. Item keys: {list(item.keys())}"); return None
-            else: logger.warning(f"[Apify Async] Dataset {dataset_id} was empty for {video_url}. Response: {results}"); return None
-
-    except httpx.TimeoutException as e: logger.error(f"[Apify Async] Timeout during API interaction for {video_url}: {e}"); return None
-    except httpx.HTTPStatusError as e: logger.error(f"[Apify Async] HTTP Status Error during API interaction for {video_url}: {e}"); return None
-    except httpx.RequestError as e: logger.error(f"[Apify Async] Request error during API interaction for {video_url}: {e}"); return None
-    except Exception as e: logger.error(f"[Apify Async] Unexpected error during Apify Async REST call for {video_url}: {e}", exc_info=True); return None
+                    # Response body *is* the dataset items array
+                    results = response.json()
+                    if isinstance(results, list) and len(results) > 0:
+                        item = results[0]
+                        # Parsing logic (same as before)
+                        content = None
+                        if "captions" in item and isinstance(item["captions"], str): content = item["captions"]
+                        elif "text" in item and isinstance(item["text"], str): content = item["text"]
+                        elif "transcript" in item and isinstance(item["transcript"], str): content = item["transcript"]
+                        elif "captions" in item and isinstance(item["captions"], list):
+                             logger.warning("[Apify SyncItems] Received list format for 'captions' unexpectedly. Processing...")
+                             if len(item["captions"]) > 0 and isinstance(item["captions"][0], dict) and 'text' in item["captions"][0]: content = " ".join(line.get("text", "") for line in item["captions"] if line.get("text"))
+                             elif len(item["captions"]) > 0 and isinstance(item["captions"][0], str): content = " ".join(item["captions"])
+
+                        if content and isinstance(content, str): logger.info(f"[Apify SyncItems] Success via REST for {video_url}. Length: {len(content)}"); return content.strip()
+                        else: logger.warning(f"[Apify SyncItems] Dataset item parsed but transcript content empty/invalid format for {video_url}. Item keys: {list(item.keys())}"); return None
+                    else: logger.warning(f"[Apify SyncItems] Actor success but dataset was empty for {video_url}. Response: {results}"); return None
+                except json.JSONDecodeError: logger.error(f"[Apify SyncItems] Failed JSON decode. Status:{response.status_code}. Resp:{response.text[:200]}"); return None
+                except Exception as e: logger.error(f"[Apify SyncItems] Error processing success response for {video_url}: {e}", exc_info=True); return None
+            elif response.status_code == 400: logger.error(f"[Apify SyncItems] Bad Request (400) for {video_url}. Check payload. Resp:{response.text[:200]}"); return None
+            elif response.status_code == 401: logger.error("[Apify SyncItems] Auth error (401). Check token."); return None
+            elif response.status_code == 404: # This was the error before, log details if it happens again
+                 error_info = ""; try: error_info = response.json().get("error", {}).get("message", "") except Exception: pass
+                 logger.error(f"[Apify SyncItems] Endpoint/Actor Not Found (404). Error: '{error_info}' Resp:{response.text[:200]}"); return None
+            else: logger.error(f"[Apify SyncItems] Unexpected status {response.status_code} for {video_url}. Resp:{response.text[:200]}"); return None
+
+    except httpx.TimeoutException as e: logger.error(f"[Apify SyncItems] Timeout during API interaction for {video_url}: {e}"); return None
+    except httpx.HTTPStatusError as e: logger.error(f"[Apify SyncItems] HTTP Status Error during API interaction for {video_url}: {e}"); return None
+    except httpx.RequestError as e: logger.error(f"[Apify SyncItems] Request error during API interaction for {video_url}: {e}"); return None
+    except Exception as e: logger.error(f"[Apify SyncItems] Unexpected error during Apify SyncItems REST call for {video_url}: {e}", exc_info=True); return None
 
 
 # (get_youtube_transcript, get_website_content, get_website_content_via_api, generate_summary remain the same)
@@ -268,11 +248,11 @@ async def get_youtube_transcript(video_id: str, video_url: str) -> Optional[str]
             else: logger.warning(f"[Fallback YT 1] Supadata failed or no content for {video_id}.")
         else: logger.warning("[Fallback YT 1] Supadata API key unavailable. Skipping.")
     if transcript_text is None:
-        logger.info("[Fallback YT 2] Trying Apify REST API (Async)...")
+        logger.info("[Fallback YT 2] Trying Apify REST API (SyncItems)...") # Updated log
         if APIFY_API_TOKEN:
-            transcript_text = await get_transcript_via_apify(video_url, APIFY_API_TOKEN)
-            if transcript_text: logger.info(f"[Fallback YT 2] Success via Apify ASYNC REST for {video_url}"); return transcript_text
-            else: logger.warning(f"[Fallback YT 2] Apify ASYNC REST failed or no content for {video_url}.")
+            transcript_text = await get_transcript_via_apify(video_url, APIFY_API_TOKEN) # Calls updated func
+            if transcript_text: logger.info(f"[Fallback YT 2] Success via Apify SyncItems REST for {video_url}"); return transcript_text
+            else: logger.warning(f"[Fallback YT 2] Apify SyncItems REST failed or no content for {video_url}.")
         else: logger.warning("[Fallback YT 2] Apify API token unavailable. Skipping.")
     if transcript_text is None: logger.error(f"All methods failed for YT transcript: {video_id}"); return None
     return transcript_text
@@ -325,6 +305,7 @@ async def get_website_content_via_api(url: str, api_key: str) -> Optional[str]:
     except httpx.RequestError as e: logger.error(f"[Fallback Web API] Request error connecting to urltotext.com API for {url}: {e}"); return None
     except Exception as e: logger.error(f"[Fallback Web API] Unexpected error during urltotext.com API call for {url}: {e}", exc_info=True); return None
 
+
 async def generate_summary(text: str, summary_type: str) -> str:
     global OPENROUTER_API_KEY, OPENROUTER_MODEL
     logger.info(f"Generating {summary_type} summary using {OPENROUTER_MODEL}. Input length: {len(text)}")