Update main.py

main.py

@@ -1,4 +1,4 @@
-# main.py (Correcting SyntaxError at line 238 & Updating Gemini Model)
+# main.py (Updating Gemini Model and Safety Settings)
 import os
 import re
 import logging
@@ -62,7 +62,6 @@ logging.getLogger("urllib3").setLevel(logging.INFO)
 logging.getLogger('gunicorn.error').setLevel(logging.INFO)
 logging.getLogger('uvicorn').setLevel(logging.INFO)
 logging.getLogger('starlette').setLevel(logging.INFO)
-# Reduce Gemini log noise if needed
 if _gemini_available: logging.getLogger("google.ai.generativelanguage").setLevel(logging.WARNING)
 logger = logging.getLogger(__name__)
 logger.info(f"Logging configured. Using BS4 parser: {DEFAULT_PARSER}")
@@ -88,16 +87,17 @@ APIFY_API_TOKEN = get_secret('APIFY_API_TOKEN')
 WEBHOOK_SECRET = get_secret('WEBHOOK_SECRET')
 GEMINI_API_KEY = get_secret('GEMINI_API_KEY')
 
-# --- Model Configuration ---
 OPENROUTER_MODEL = os.environ.get("OPENROUTER_MODEL", "deepseek/deepseek-chat-v3-0324:free")
 APIFY_ACTOR_ID = os.environ.get("APIFY_ACTOR_ID", "karamelo~youtube-transcripts")
-# *** FIX: Update default Gemini model ***
+# *** FIX: Update default Gemini model to gemini-2.0-flash ***
 GEMINI_MODEL = os.environ.get("GEMINI_MODEL", "gemini-2.0-flash")
 
-# --- Key Checks ---
 if not TELEGRAM_TOKEN: logger.critical("❌ FATAL: TELEGRAM_TOKEN not found."); raise RuntimeError("Exiting: Telegram token missing.")
 if not OPENROUTER_API_KEY: logger.error("❌ ERROR: OPENROUTER_API_KEY not found. Primary summarization will fail.")
-if _gemini_available and not GEMINI_API_KEY: logger.warning("⚠️ WARNING: GEMINI_API_KEY not found. Fallback summarization via Gemini disabled.")
+_gemini_fallback_enabled = _gemini_available and bool(GEMINI_API_KEY)
+if _gemini_fallback_enabled and not GEMINI_API_KEY: logger.warning("⚠️ WARNING: GEMINI_API_KEY found in env but value seems empty. Fallback disabled.")
+elif not _gemini_fallback_enabled and _gemini_available : logger.warning("⚠️ WARNING: GEMINI_API_KEY not found. Fallback disabled.")
+
 
 if not URLTOTEXT_API_KEY: pass
 if not SUPADATA_API_KEY: pass
@@ -106,21 +106,15 @@ if not WEBHOOK_SECRET: logger.info("Optional secret 'WEBHOOK_SECRET' not found.
 
 logger.info("Secret loading and configuration check finished.")
 logger.info(f"Using OpenRouter Model (Primary): {OPENROUTER_MODEL}")
-_gemini_fallback_enabled = _gemini_available and bool(GEMINI_API_KEY) # Define after checking key
 if _gemini_fallback_enabled: logger.info(f"Using Gemini Model (Fallback): {GEMINI_MODEL}")
-else: logger.info("Gemini Fallback: Disabled (library or API key missing)")
+else: logger.info("Gemini Fallback: Disabled")
 logger.info(f"Using Apify Actor (via REST): {APIFY_ACTOR_ID}")
 _apify_token_exists = bool(APIFY_API_TOKEN)
 
 
-# --- Configure Gemini Client ---
 if _gemini_fallback_enabled:
-    try:
-        genai.configure(api_key=GEMINI_API_KEY)
-        logger.info("Google GenAI client configured successfully.")
-    except Exception as e:
-        logger.error(f"Failed to configure Google GenAI client: {e}")
-        _gemini_fallback_enabled = False
+    try: genai.configure(api_key=GEMINI_API_KEY); logger.info("Google GenAI client configured successfully.")
+    except Exception as e: logger.error(f"Failed to configure Google GenAI client: {e}"); _gemini_fallback_enabled = False
 
 # --- Retry Decorator ---
 @retry( stop=stop_after_attempt(4), wait=wait_exponential(multiplier=1, min=2, max=15), retry=retry_if_exception_type((NetworkError, RetryAfter, TimedOut, BadRequest)), before_sleep=before_sleep_log(logger, logging.WARNING), reraise=True )
@@ -145,6 +139,7 @@ def extract_youtube_id(url):
 
 
 # --- Content Fetching Functions ---
+# (fetch_url_content_for_scrape, get_transcript_via_supadata, get_transcript_via_apify, get_youtube_transcript, get_website_content, get_website_content_via_api remain the same as previous version)
 async def fetch_url_content_for_scrape(url: str, timeout: int = 25) -> Optional[str]:
     headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Connection': 'keep-alive', 'DNT': '1', 'Upgrade-Insecure-Requests': '1' }
     try:
@@ -235,8 +230,7 @@ async def get_transcript_via_apify(video_url: str, api_token: str) -> Optional[s
             elif response.status_code == 401: logger.error("[Apify SyncItems] Auth error (401). Check token."); return None
             elif response.status_code == 404:
                  error_info = "";
-                 try: # Correctly indented try/except
-                      error_info = response.json().get("error", {}).get("message", "")
+                 try: error_info = response.json().get("error", {}).get("message", "")
                  except Exception: pass
                  logger.error(f"[Apify SyncItems] Endpoint/Actor Not Found (404). Error: '{error_info}' Resp:{response.text[:200]}");
                  return None
@@ -247,7 +241,6 @@ async def get_transcript_via_apify(video_url: str, api_token: str) -> Optional[s
     except httpx.RequestError as e: logger.error(f"[Apify SyncItems] Request error during API interaction for {video_url}: {e}"); return None
     except Exception as e: logger.error(f"[Apify SyncItems] Unexpected error during Apify SyncItems REST call for {video_url}: {e}", exc_info=True); return None
 
-
 async def get_youtube_transcript(video_id: str, video_url: str) -> Optional[str]:
     global SUPADATA_API_KEY, APIFY_API_TOKEN
     if not video_id: logger.error("get_youtube_transcript: No video_id"); return None
@@ -341,23 +334,54 @@ async def generate_summary_gemini(text: str, summary_type: str) -> str:
     MAX_INPUT_LENGTH = 1000000
     if len(text) > MAX_INPUT_LENGTH: logger.warning(f"[Gemini Fallback] Input length ({len(text)}) exceeds limit ({MAX_INPUT_LENGTH}). Truncating."); text = text[:MAX_INPUT_LENGTH] + "... (Content truncated)"
     full_prompt = f"{prompt}\n\n{text}"
-    safety_settings = { HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE, HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE, HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE, HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE, }
+
+    # *** FIX: Set all safety settings to BLOCK_NONE ***
+    safety_settings = {
+        HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,
+        HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,
+        HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,
+        HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,
+        # Note: CIVIC_INTEGRITY might not be available/needed for all models,
+        # but setting it won't hurt if the HarmCategory exists.
+        # If it causes an error later, remove this line.
+        getattr(HarmCategory, 'HARM_CATEGORY_CIVIC_INTEGRITY', None): HarmBlockThreshold.BLOCK_NONE
+    }
+    # Filter out None keys in case CIVIC_INTEGRITY doesn't exist
+    safety_settings = {k: v for k, v in safety_settings.items() if k is not None}
+    logger.debug(f"[Gemini Fallback] Using safety settings: {safety_settings}")
+
 
     try:
         logger.debug(f"[Gemini Fallback] Initializing model {GEMINI_MODEL}")
         model = genai.GenerativeModel(GEMINI_MODEL)
         logger.info(f"[Gemini Fallback] Sending request to Gemini ({GEMINI_MODEL})...")
-        response = await model.generate_content_async( full_prompt, safety_settings=safety_settings )
+        request_options = {"timeout": 120}
+        response = await model.generate_content_async( full_prompt, safety_settings=safety_settings, request_options=request_options )
         logger.info("[Gemini Fallback] Received response from Gemini.")
 
-        if response.prompt_feedback.block_reason: logger.error(f"[Gemini Fallback] Request blocked. Reason: {response.prompt_feedback.block_reason}"); return f"Sorry, fallback AI blocked summary due to safety filters ({response.prompt_feedback.block_reason})."
-        summary = response.text
+        # Checking block reason is still useful for debugging, though unlikely now
+        if response.prompt_feedback.block_reason: logger.warning(f"[Gemini Fallback] Request blocked unexpectedly. Reason: {response.prompt_feedback.block_reason}"); # Changed to warning
+        # Check candidates for safety blocks as well
+        for cand in response.candidates:
+            if cand.finish_reason == 'SAFETY':
+                logger.warning(f"[Gemini Fallback] Candidate blocked due to SAFETY. Ratings: {cand.safety_ratings}")
+
+
+        # Attempt to get text even if blocked (might be None)
+        try:
+            summary = response.text
+        except ValueError as e:
+            # Handle cases where accessing response.text raises ValueError (e.g., blocked content)
+            logger.warning(f"[Gemini Fallback] Error accessing response text (likely blocked content): {e}")
+            summary = None # Ensure summary is None
+
         if summary: logger.info(f"[Gemini Fallback] Success generating summary. Output len: {len(summary)}"); return summary.strip().replace('_', r'\_').replace('*', r'\*').replace('[', r'\[').replace('`', r'\`')
-        else: logger.warning(f"[Gemini Fallback] Gemini returned empty summary. Finish reason: {response.candidates[0].finish_reason if response.candidates else 'N/A'}"); return "Sorry, the fallback AI model returned an empty summary."
+        else: logger.warning(f"[Gemini Fallback] Gemini returned empty summary or content was blocked. Finish reason: {response.candidates[0].finish_reason if response.candidates else 'N/A'}"); return "Sorry, the fallback AI model did not provide a summary (possibly due to content filters)."
     except Exception as e: logger.error(f"[Gemini Fallback] Unexpected error during Gemini API call: {e}", exc_info=True); return "Sorry, an unexpected error occurred while using the fallback AI service."
 
+
 async def generate_summary(text: str, summary_type: str) -> str:
-    """Generates summary using OpenRouter (Primary) with Gemini fallback on ReadTimeout."""
+    """Generates summary using OpenRouter (Primary) with Gemini fallback on 10s ReadTimeout."""
     global OPENROUTER_API_KEY, OPENROUTER_MODEL, _gemini_fallback_enabled
     logger.info(f"[Primary Summary] Generating {summary_type} summary using {OPENROUTER_MODEL}. Input length: {len(text)}")
     if not OPENROUTER_API_KEY: logger.error("[Primary Summary] OpenRouter key missing."); return "Error: AI model configuration key missing."
@@ -368,7 +392,8 @@ async def generate_summary(text: str, summary_type: str) -> str:
     if len(text) > MAX_INPUT_LENGTH: logger.warning(f"[Primary Summary] Input length ({len(text)}) exceeds limit ({MAX_INPUT_LENGTH}). Truncating."); text = text[:MAX_INPUT_LENGTH] + "... (Content truncated)"
     full_prompt = f"{prompt}\n\n{text}"
     headers = { "Authorization": f"Bearer {OPENROUTER_API_KEY}", "Content-Type": "application/json" }; payload = { "model": OPENROUTER_MODEL, "messages": [{"role": "user", "content": full_prompt}] }; openrouter_api_endpoint = "https://openrouter.ai/api/v1/chat/completions"
-    api_timeouts = httpx.Timeout(15.0, read=180.0, write=15.0, pool=60.0); response = None
+    api_timeouts = httpx.Timeout(15.0, read=10.0, write=15.0, pool=60.0) # 10 second read timeout
+    response = None
 
     try:
         async with httpx.AsyncClient(timeout=api_timeouts) as client:
@@ -396,12 +421,7 @@ async def generate_summary(text: str, summary_type: str) -> str:
                 elif response.status_code == 429: logger.warning("[Primary Summary] OpenRouter Rate Limit Exceeded (429)."); return "Sorry, primary AI model is busy. Try again."
                 elif response.status_code == 500: logger.error(f"[Primary Summary] OpenRouter Internal Server Error (500). Resp:{response.text[:500]}"); return "Sorry, primary AI service internal error."
                 else:
-                    # *** FIX: Correct Indentation for error info extraction ***
-                    error_info = ""
-                    try:
-                        error_info = response.json().get("error", {}).get("message", "")
-                    except Exception:
-                        pass
+                    error_info = ""; try: error_info = response.json().get("error", {}).get("message", "") except Exception: pass
                     logger.error(f"[Primary Summary] Unexpected status {response.status_code} from OpenRouter. Error: '{error_info}' Resp:{response.text[:500]}");
                     return f"Sorry, primary AI service returned unexpected status ({response.status_code})."
 
@@ -457,7 +477,7 @@ async def process_summary_task( user_id: int, chat_id: int, message_id_to_edit:
             if content:
                 logger.info(f"[Task {task_id}] Content fetched (len:{len(content)}). Generating summary.")
                 await retry_bot_operation(bot.send_chat_action, chat_id=chat_id, action='typing')
-                final_summary = await generate_summary(content, summary_type)
+                final_summary = await generate_summary(content, summary_type) # This now handles the fallback internally
                 if final_summary.startswith("Error:") or final_summary.startswith("Sorry,"): user_feedback_message = final_summary; logger.warning(f"[Task {task_id}] Summary generation failed: {final_summary}")
                 else:
                     max_length = 4096; summary_parts = [final_summary[i:i+max_length] for i in range(0, len(final_summary), max_length)]