Update main.py

main.py

@@ -558,36 +558,60 @@ async def _run_apify_actor_for_web_content(url: str, api_token: str, actor_id: s
     logger.info(f"[{actor_name}] Attempting fetch for URL: {url} (Actor: {actor_id})")
     sync_items_endpoint = f"https://api.apify.com/v2/acts/{actor_id}/run-sync-get-dataset-items"
     params = {"token": api_token}
-
-    # --- Define base input, adjust for specific actors ---
-    run_input = { "startUrls": [{"url": url}], "maxCrawlPages": 1, "crawlerType": "playwright:firefox" }
-    is_yt_actor = actor_id == APIFY_ACTOR_ID
-    log_prefix = "[Apify YT]" if is_yt_actor else f"[{actor_name}]"
-
-    if is_yt_actor:
-        # Use input specific to the default YT actor
-        # REMOVED outputFormat: "singleStringText" as it seems unreliable based on observed output
-        run_input = { "urls": [url], # Pass the URL correctly
-                      # "outputFormat": "singleStringText", # <<< REMOVED THIS LINE
-                      "maxRetries": 5, # Keep retries
-                      # Keep other flags as they might affect which data is returned overall
-                      "channelHandleBoolean": False,
-                      "channelNameBoolean": False,
-                      "datePublishedBoolean": False,
-                      "relativeDateTextBoolean": False }
-        logger.debug(f"{log_prefix} Using YouTube-specific input (default array output expected)")
+    log_prefix = f"[{actor_name}]" # Use actor_name for logging prefix
+
+    # --- Define input based on actor ID ---
+    run_input: Dict[str, Any] = {} # Initialize empty dict
+
+    if actor_id == APIFY_ACTOR_ID:
+        # Input specific to the default YT actor (1s7eXiaukVuOr4Ueg)
+        run_input = {
+            "urls": [url],
+            "maxRetries": 5,
+            "channelHandleBoolean": False,
+            "channelNameBoolean": False,
+            "datePublishedBoolean": False,
+            "relativeDateTextBoolean": False
+        }
+        logger.debug(f"{log_prefix} Using input format for Default YT Actor ({APIFY_ACTOR_ID})")
+    elif actor_id == APIFY_STRUCTURED_YT_ACTOR_ID: # <<< --- ADDED THIS BLOCK --- >>>
+        # Input specific to the Structured YT Actor (gpjTCWkGZS1lHc9pR)
+        # Based on the error message "Field input.urls is required"
+        run_input = {
+            "urls": [url]
+            # Add other specific parameters for this actor if needed/known
+        }
+        logger.debug(f"{log_prefix} Using input format for Structured YT Actor ({APIFY_STRUCTURED_YT_ACTOR_ID})")
     elif actor_id == APIFY_TEXT_SCRAPER_ACTOR_ID:
-        run_input = { "urls": [url] }
-        logger.debug(f"{log_prefix} Using simplified input for Text Scraper: {run_input}")
-    # Add other actor-specific input adjustments here if necessary
+        # Input specific to Text Scraper Free (2gbQiRSpJIIag2FdR)
+        run_input = {
+            "urls": [url]
+        }
+        logger.debug(f"{log_prefix} Using input format for Text Scraper ({APIFY_TEXT_SCRAPER_ACTOR_ID})")
+    elif actor_id == APIFY_CRAWLER_ACTOR_ID:
+         # Input specific to Website Content Crawler (aYG0l9s7dbB7j3gbS)
+         run_input = {
+            "startUrls": [{"url": url}],
+            "maxCrawlPages": 1,
+            "crawlerType": "playwright:firefox" # Or adjust as needed
+            # Add other parameters specific to the crawler if necessary
+         }
+         logger.debug(f"{log_prefix} Using input format for Website Content Crawler ({APIFY_CRAWLER_ACTOR_ID})")
+    else:
+        # Fallback default input if actor ID doesn't match known ones
+        # Using the simple {"urls": [url]} format seems safest for generic text/content extractors
+        run_input = {"urls": [url]}
+        logger.warning(f"{log_prefix} Unknown Actor ID '{actor_id}'. Using default input format: {run_input}")
+
 
     headers = {"Content-Type": "application/json"}
     try:
         async with httpx.AsyncClient(timeout=120.0) as client:
-            logger.debug(f"{log_prefix} POST Request to {sync_items_endpoint} for {url}")
+            logger.debug(f"{log_prefix} POST Request to {sync_items_endpoint} for {url} with input: {run_input}") # Log the input being sent
             response = await client.post(sync_items_endpoint, headers=headers, params=params, json=run_input)
             logger.debug(f"{log_prefix} Received status code {response.status_code} for {url}")
 
+            # --- Start of response processing (Remains the same as before) ---
             if response.status_code in [200, 201]:
                 if response.status_code == 201:
                     logger.info(f"{log_prefix} Received status 201 (Created) from run-sync endpoint, processing results anyway.")
@@ -662,8 +686,8 @@ async def _run_apify_actor_for_web_content(url: str, api_token: str, actor_id: s
                              logger.warning(f"{log_prefix} Dataset item parsed (Status {response.status_code}) but final content empty/short/invalid format after checking all known keys for {url}. Item keys: {item_keys_str}. Length: {content_len}")
                              return None # Return None if no valid content found
                     else:
-                        # Handle empty dataset list '[]'
-                        logger.warning(f"{log_prefix} Actor call successful (Status {response.status_code}) but dataset was empty for {url}. Response: {results}")
+                        # Handle empty dataset list '[]' or non-list response
+                        logger.warning(f"{log_prefix} Actor call successful (Status {response.status_code}) but dataset was empty or not a list for {url}. Response type: {type(results)}. Response sample: {str(results)[:200]}")
                         return None
                 # --- End of success processing logic ---
                 except json.JSONDecodeError:
@@ -685,9 +709,6 @@ async def _run_apify_actor_for_web_content(url: str, api_token: str, actor_id: s
     except httpx.RequestError as e: logger.error(f"{log_prefix} Request error during API interaction for {url}: {e}"); return None
     except Exception as e: logger.error(f"{log_prefix} Unexpected error during {actor_name} call for {url}: {e}", exc_info=True); return None
 
-# --- Ensure YT Transcript function uses the generic one ---
-# You can simplify the get_transcript_via_apify function now
-
 async def get_transcript_via_apify(video_url: str, api_token: str) -> Optional[str]:
     """Fallback YT 2: Fetches YouTube transcript using default Apify Actor via generic function."""
     global APIFY_ACTOR_ID