Update main.py

main.py

@@ -857,32 +857,63 @@ async def process_summary_task( user_id: int, chat_id: int, message_id_to_edit:
                 if video_id: content = await get_youtube_transcript(video_id, url)
                 else: user_feedback_message = "Sorry, I couldn't understand that YouTube URL format."
                 if not content and not user_feedback_message: user_feedback_message = "Sorry, I couldn't get the transcript for that YouTube video using any available method (unavailable/private/no captions?)."
-            else:
+                        else:
+                # Website URL processing with NEW fallback order: 1 -> 5 -> 6 -> 3 -> 4 -> 2
                 global URLTOTEXT_API_KEY, RAPIDAPI_KEY, APIFY_API_TOKEN, _urltotext_key_exists, _rapidapi_key_exists, _apify_token_exists
 
-                logger.info(f"[Task {task_id}] Trying Web Scrape Method 1 (Direct Fetch + BS4)..."); content = await get_website_content(url)
+                # --- Method 1: Direct Fetch + BS4 (Primary) ---
+                logger.info(f"[Task {task_id}] Trying Web Scrape Method 1 (Direct Fetch + BS4)...");
+                content = await get_website_content(url)
+
+                # --- Method 5: Apify Crawler (Fallback 1) ---
+                if not content:
+                    logger.warning(f"[Task {task_id}] Method 1 failed. Trying Method 5 (Apify Crawler)...")
+                    if _apify_token_exists:
+                        await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
+                        content = await get_website_content_via_apify_crawler(url, APIFY_API_TOKEN)
+                    else:
+                        logger.warning("[Task {task_id}] Method 5 (Apify Crawler) APIFY_API_TOKEN unavailable. Skipping.")
+
+                # --- Method 6: Apify Text Scraper (Fallback 2) ---
                 if not content:
-                    logger.warning(f"[Task {task_id}] Method 1 failed. Trying Method 2 (urltotext.com)...")
-                    if _urltotext_key_exists: await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing'); content = await get_website_content_via_api(url, URLTOTEXT_API_KEY)
-                    else: logger.warning("[Task {task_id}] Method 2 (urltotext.com) API key unavailable. Skipping.")
+                    logger.warning(f"[Task {task_id}] Method 5 failed. Trying Method 6 (Apify Text Scraper)...")
+                    if _apify_token_exists:
+                        await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
+                        content = await get_website_content_via_apify_text_scraper(url, APIFY_API_TOKEN)
+                    else:
+                        logger.warning("[Task {task_id}] Method 6 (Apify Text Scraper) APIFY_API_TOKEN unavailable. Skipping.")
+
+                # --- Method 3: RapidAPI - Scraper's Proxy (Fallback 3) ---
                 if not content:
-                    logger.warning(f"[Task {task_id}] Method 2 failed. Trying Method 3 (Scraper's Proxy)...")
-                    if _rapidapi_key_exists: await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing'); content = await get_website_content_via_scrapers_proxy(url, RAPIDAPI_KEY)
-                    else: logger.warning("[Task {task_id}] Method 3 (Scraper's Proxy) RapidAPI key unavailable. Skipping.")
+                    logger.warning(f"[Task {task_id}] Method 6 failed. Trying Method 3 (Scraper's Proxy)...")
+                    if _rapidapi_key_exists:
+                        await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
+                        content = await get_website_content_via_scrapers_proxy(url, RAPIDAPI_KEY)
+                    else:
+                        logger.warning("[Task {task_id}] Method 3 (Scraper's Proxy) RapidAPI key unavailable. Skipping.")
+
+                # --- Method 4: RapidAPI - AI Web Scraper (Fallback 4) ---
                 if not content:
                     logger.warning(f"[Task {task_id}] Method 3 failed. Trying Method 4 (AI Web Scraper)...")
-                    if _rapidapi_key_exists: await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing'); content = await get_website_content_via_ai_web_scraper(url, RAPIDAPI_KEY)
-                    else: logger.warning("[Task {task_id}] Method 4 (AI Web Scraper) RapidAPI key unavailable. Skipping.")
-                if not content:
-                    logger.warning(f"[Task {task_id}] Method 4 failed. Trying Method 5 (Apify Crawler)...")
-                    if _apify_token_exists: await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing'); content = await get_website_content_via_apify_crawler(url, APIFY_API_TOKEN)
-                    else: logger.warning("[Task {task_id}] Method 5 (Apify Crawler) APIFY_API_TOKEN unavailable. Skipping.")
+                    if _rapidapi_key_exists:
+                        await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
+                        content = await get_website_content_via_ai_web_scraper(url, RAPIDAPI_KEY)
+                    else:
+                        logger.warning("[Task {task_id}] Method 4 (AI Web Scraper) RapidAPI key unavailable. Skipping.")
+
+                # --- Method 2: urltotext.com API (Fallback 5) ---
                 if not content:
-                    logger.warning(f"[Task {task_id}] Method 5 failed. Trying Method 6 (Apify Text Scraper)...")
-                    if _apify_token_exists: await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing'); content = await get_website_content_via_apify_text_scraper(url, APIFY_API_TOKEN)
-                    else: logger.warning("[Task {task_id}] Method 6 (Apify Text Scraper) APIFY_API_TOKEN unavailable. Skipping.")
+                    logger.warning(f"[Task {task_id}] Method 4 failed. Trying Method 2 (urltotext.com)...")
+                    if _urltotext_key_exists:
+                        await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
+                        content = await get_website_content_via_api(url, URLTOTEXT_API_KEY)
+                    else:
+                        logger.warning("[Task {task_id}] Method 2 (urltotext.com) API key unavailable. Skipping.")
 
-                if not content and not user_feedback_message: user_feedback_message = "Sorry, I couldn't fetch readable content from that website using multiple methods (blocked/dynamic content/empty?)."
+                # --- Final check after all attempts ---
+                if not content and not user_feedback_message:
+                    logger.error(f"[Task {task_id}] All website fetching methods (1, 5, 6, 3, 4, 2) failed for {url}")
+                    user_feedback_message = "Sorry, I couldn't fetch readable content from that website using multiple methods (blocked/dynamic content/empty?)."
 
             # --- 3. Summarization ---
             if content: