Update main.py

main.py

@@ -491,47 +491,64 @@ async def _run_apify_actor_for_web_content(url: str, api_token: str, actor_id: s
             response = await client.post(sync_items_endpoint, headers=headers, params=params, json=run_input)
             logger.debug(f"[{actor_name}] Received status code {response.status_code} for {url}")
 
-            # <<< MODIFIED check for success: Accept 200 OR 201 >>>
             if response.status_code in [200, 201]:
-                # Log if it was 201, as it's slightly unexpected for sync but apparently happens
                 if response.status_code == 201:
                     logger.info(f"[{actor_name}] Received status 201 (Created) from run-sync endpoint, processing results anyway.")
 
                 try:
                     results = response.json(); content = None
-                    # --- The rest of the success processing logic stays the same ---
+                    # Check if results is a list and not empty
                     if isinstance(results, list) and len(results) > 0:
                         item = results[0]
-                    content = None # Reset content
-
-                    # <<< MODIFIED PARSING LOGIC >>>
-                    if "text" in item and isinstance(item["text"], str):
-                        content = item["text"]
-                    elif "content" in item and isinstance(item["content"], str):
-                        content = item["content"]
-                    elif "markdown" in item and isinstance(item["markdown"], str):
-                        content = item["markdown"]
-                    # --- ADD THIS CHECK ---
-                    elif "captions" in item and isinstance(item["captions"], str):
-                        logger.info("[Apify YT] Found transcript text in 'captions' field.")
-                        content = item["captions"]
-                    # --- END ADDITION ---
-                    elif "html" in item and isinstance(item["html"], str):
-                         logger.warning(f"[Apify YT] No direct text/markdown/captions found, attempting to parse 'html' from result.")
-                         soup = BeautifulSoup(item["html"], DEFAULT_PARSER)
-                         content = " ".join(line.strip() for line in soup.get_text(separator='\n', strip=True).splitlines() if line.strip())
-
-                    # Check content validity AFTER parsing
-                    if content and isinstance(content, str) and len(content) > 30:
-                        logger.info(f"[Apify YT] Success via REST (Status {response.status_code}) for {video_url}. Length: {len(content)}")
-                        return content.strip()
-                    else:
-                         content_len = len(content) if content and isinstance(content, str) else 0
-                         # Adjust warning message if needed to show 'captions' was checked
-                         logger.warning(f"[Apify YT] Dataset item parsed (Status {response.status_code}) but final transcript content empty/short/invalid format for {video_url}. Item keys: {list(item.keys())}. Length: {content_len}")
-                         return None
+                        content = None # Reset content
+
+                        # Parsing Logic - try extracting text from various possible keys
+                        if "text" in item and isinstance(item["text"], str):
+                            content = item["text"]
+                        elif "content" in item and isinstance(item["content"], str):
+                            content = item["content"]
+                        elif "markdown" in item and isinstance(item["markdown"], str):
+                            content = item["markdown"]
+                        elif "captions" in item and isinstance(item["captions"], str):
+                            # Specifically check if this is the YT actor by name or ID if needed,
+                            # otherwise this assumes 'captions' might contain text for other actors too.
+                            is_yt_actor = actor_id == APIFY_ACTOR_ID # Check if it's the specific YT actor
+                            log_prefix = "[Apify YT]" if is_yt_actor else f"[{actor_name}]"
+                            logger.info(f"{log_prefix} Found text content in 'captions' field.")
+                            content = item["captions"]
+                        elif "html" in item and isinstance(item["html"], str):
+                             is_yt_actor = actor_id == APIFY_ACTOR_ID
+                             log_prefix = "[Apify YT]" if is_yt_actor else f"[{actor_name}]"
+                             logger.warning(f"{log_prefix} No direct text/markdown/captions found, attempting to parse 'html' from result.")
+                             # Run potentially blocking BS4 parsing in a separate thread
+                             def parse_html_sync(html_str):
+                                 try:
+                                     soup = BeautifulSoup(html_str, DEFAULT_PARSER)
+                                     return " ".join(line.strip() for line in soup.get_text(separator='\n', strip=True).splitlines() if line.strip())
+                                 except Exception as e:
+                                     logger.error(f"{log_prefix} Error parsing HTML in thread: {e}")
+                                     return None
+                             content = await asyncio.to_thread(parse_html_sync, item["html"])
+
+                        # Check content validity AFTER attempting all parsing methods
+                        if content and isinstance(content, str) and len(content) > 30:
+                            is_yt_actor = actor_id == APIFY_ACTOR_ID
+                            log_prefix = "[Apify YT]" if is_yt_actor else f"[{actor_name}]"
+                            # Use 'url' which is the function parameter
+                            logger.info(f"{log_prefix} Success via REST (Status {response.status_code}) for {url}. Length: {len(content)}")
+                            return content.strip()
+                        else:
+                             # Content is invalid or parsing failed
+                             content_len = len(content) if content and isinstance(content, str) else 0
+                             is_yt_actor = actor_id == APIFY_ACTOR_ID
+                             log_prefix = "[Apify YT]" if is_yt_actor else f"[{actor_name}]"
+                             item_keys_str = list(item.keys()) if isinstance(item, dict) else "N/A"
+                             # Use 'url' which is the function parameter
+                             logger.warning(f"{log_prefix} Dataset item parsed (Status {response.status_code}) but final content empty/short/invalid format for {url}. Item keys: {item_keys_str}. Length: {content_len}")
+                             return None
+                    # <<< CORRECTED INDENTATION FOR THIS ELSE >>>
                     else:
-                        # Handle empty dataset list '[]' which might be returned with 200 or 201
+                        # Handle empty dataset list '[]'
                         logger.warning(f"[{actor_name}] Actor call successful (Status {response.status_code}) but dataset was empty for {url}. Response: {results}")
                         return None
                 # --- End of success processing logic ---