Spaces:
Running
Running
| # main.py (Revised: Starlette Lifespan + ptb_app._running fix) | |
| import os | |
| import re | |
| import logging | |
| import asyncio | |
| import json | |
| import html # For unescaping HTML entities | |
| import contextlib # For async context manager (lifespan) | |
| # --- Frameworks --- | |
| from flask import Flask, request, Response # Core web routes | |
| from starlette.applications import Starlette # ASGI App & Lifespan | |
| from starlette.routing import Mount # Mount Flask within Starlette | |
| from starlette.middleware.wsgi import WSGIMiddleware # Wrap Flask for Starlette | |
| # --- Telegram Bot --- | |
| from telegram import Update, InlineKeyboardButton, InlineKeyboardMarkup | |
| from telegram.ext import ( | |
| Application, | |
| CommandHandler, | |
| MessageHandler, | |
| filters, | |
| ContextTypes, | |
| CallbackQueryHandler, | |
| ) | |
| from telegram.constants import ParseMode | |
| from telegram.error import NetworkError, RetryAfter # Import RetryAfter | |
| # --- Other Libraries --- | |
| from youtube_transcript_api import YouTubeTranscriptApi | |
| import requests | |
| from bs4 import BeautifulSoup | |
| _apify_token_exists = bool(os.environ.get('APIFY_API_TOKEN')) | |
| if _apify_token_exists: | |
| from apify_client import ApifyClient | |
| else: | |
| ApifyClient = None | |
| # --- Logging Setup --- | |
| logging.basicConfig( | |
| format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', | |
| level=logging.DEBUG | |
| ) | |
| logging.getLogger("httpx").setLevel(logging.WARNING) | |
| if ApifyClient: logging.getLogger("apify_client").setLevel(logging.WARNING) | |
| logging.getLogger("telegram.ext").setLevel(logging.INFO) | |
| logging.getLogger('telegram.bot').setLevel(logging.INFO) | |
| logging.getLogger("urllib3").setLevel(logging.INFO) | |
| logging.getLogger('gunicorn.error').setLevel(logging.INFO) | |
| logging.getLogger('uvicorn').setLevel(logging.INFO) | |
| logging.getLogger('starlette').setLevel(logging.INFO) | |
| logger = logging.getLogger(__name__) | |
| logger.info("Logging configured.") | |
| # --- Global variable for PTB app (initialized during lifespan) --- | |
| ptb_app: Application | None = None | |
| # --- Environment Variable Loading --- | |
| logger.info("Attempting to load secrets...") | |
| def get_secret(secret_name): | |
| logger.debug(f"Attempting to read secret: {secret_name}") | |
| value = os.environ.get(secret_name) | |
| if value: logger.info(f"Secret '{secret_name}': Found (Value length: {len(value)})") | |
| else: logger.warning(f"Secret '{secret_name}': Not Found") | |
| return value | |
| TELEGRAM_TOKEN = get_secret('TELEGRAM_TOKEN') | |
| OPENROUTER_API_KEY = get_secret('OPENROUTER_API_KEY') | |
| URLTOTEXT_API_KEY = get_secret('URLTOTEXT_API_KEY') | |
| SUPADATA_API_KEY = get_secret('SUPADATA_API_KEY') | |
| APIFY_API_TOKEN = get_secret('APIFY_API_TOKEN') | |
| logger.info("Secret loading attempt finished.") | |
| # --- Bot Logic Functions --- | |
| # Helper Functions | |
| def is_youtube_url(url): | |
| """Checks if the URL is a valid YouTube video or shorts URL.""" | |
| youtube_regex = r'(https?://)?(www\.)?(youtube\.com/(watch\?v=|shorts/)|youtu\.be/)([\w-]{11})' | |
| match = re.search(youtube_regex, url) | |
| logger.debug(f"is_youtube_url check for '{url}': {'Match found' if match else 'No match'}") | |
| return bool(match) | |
| def extract_youtube_id(url): | |
| """Extracts the YouTube video ID from a URL.""" | |
| youtube_id_regex = r'(?:youtube\.com/(?:watch\?v=|shorts/)|youtu\.be/)([\w-]{11})(?:\?|&|\s|$)' | |
| match = re.search(youtube_id_regex, url) | |
| if match: | |
| video_id = match.group(1) | |
| logger.debug(f"Extracted YouTube ID '{video_id}' from URL: {url}") | |
| return video_id | |
| else: | |
| logger.warning(f"Could not extract YouTube ID from URL: {url}") | |
| return None | |
| # Supadata Transcript Fetching | |
| async def get_transcript_via_supadata(video_id: str, api_key: str): | |
| """Fetches YouTube transcript via Supadata API.""" | |
| if not video_id: logger.error("[Supadata] get_transcript_via_supadata called with no video_id"); return None | |
| if not api_key: logger.error("[Supadata] API key is missing."); return None | |
| logger.info(f"[Supadata] Attempting fetch for video ID: {video_id}") | |
| api_endpoint = f"https://api.supadata.net/v1/youtube/transcript" | |
| params = {"videoId": video_id, "format": "text"} | |
| headers = {"X-API-Key": api_key} | |
| try: | |
| logger.warning("[Supadata] Making request with verify=False (Attempting to bypass SSL verification)") | |
| response = await asyncio.to_thread(requests.get, api_endpoint, headers=headers, params=params, timeout=30, verify=False) | |
| logger.debug(f"[Supadata] Received status code {response.status_code} for {video_id}") | |
| if response.status_code == 200: | |
| try: | |
| data = response.json() | |
| content = data if isinstance(data, str) else data.get("transcript") or data.get("text") or data.get("data") | |
| if content and isinstance(content, str): | |
| logger.info(f"[Supadata] Successfully fetched transcript for {video_id}. Length: {len(content)}") | |
| return content.strip() | |
| else: | |
| logger.warning(f"[Supadata] API success but content empty/invalid for {video_id}. Response: {data}") | |
| return None | |
| except json.JSONDecodeError: | |
| if response.text: | |
| logger.info(f"[Supadata] Successfully fetched transcript (plain text) for {video_id}. Length: {len(response.text)}") | |
| return response.text.strip() | |
| else: | |
| logger.error(f"[Supadata] Failed to decode JSON response (and no text body) for {video_id}. Response: {response.text[:200]}...") | |
| return None | |
| except Exception as e: | |
| logger.error(f"[Supadata] Error processing successful response for {video_id}: {e}", exc_info=True) | |
| return None | |
| elif response.status_code in [401, 403]: | |
| logger.error(f"[Supadata] Authentication error ({response.status_code}). Check API key.") | |
| return None | |
| elif response.status_code == 404: | |
| logger.warning(f"[Supadata] Transcript not found ({response.status_code}) for {video_id}.") | |
| return None | |
| else: | |
| logger.error(f"[Supadata] Unexpected status code {response.status_code} for {video_id}. Response: {response.text[:200]}...") | |
| return None | |
| except requests.exceptions.Timeout: | |
| logger.error(f"[Supadata] Timeout error connecting to API for {video_id}") | |
| return None | |
| except requests.exceptions.RequestException as e: | |
| logger.error(f"[Supadata] Request error connecting to API for {video_id}: {e}") | |
| if isinstance(e, requests.exceptions.SSLError): | |
| logger.error(f"[Supadata] SSL Error occurred despite using verify=False. Details: {e}") | |
| return None | |
| except Exception as e: | |
| logger.error(f"[Supadata] Unexpected error during API call for {video_id}: {e}", exc_info=True) | |
| return None | |
| # Apify Transcript Fetching (with fixed fallback parsing) | |
| async def get_transcript_via_apify(video_url: str, api_token: str): | |
| """Fetches YouTube transcript via Apify API.""" | |
| if not video_url: logger.error("[Apify] get_transcript_via_apify called with no video_url"); return None | |
| if not api_token: logger.error("[Apify] API token is missing."); return None | |
| if not ApifyClient: logger.error("[Apify] ApifyClient not available/imported."); return None | |
| logger.info(f"[Apify] Attempting fetch for URL: {video_url}") | |
| actor_id = "karamelo~youtube-transcripts" | |
| api_endpoint = f"https://api.apify.com/v2/acts/{actor_id}/run-sync-get-dataset-items" | |
| params = {"token": api_token} | |
| payload = json.dumps({ | |
| "urls": [video_url], | |
| "outputFormat": "singleStringText", # Still request this primarily | |
| "maxRetries": 3, | |
| "channelHandleBoolean": False, | |
| "channelNameBoolean": False, | |
| "datePublishedBoolean": False, | |
| "relativeDateTextBoolean": False, | |
| }) | |
| headers = {"Content-Type": "application/json"} | |
| try: | |
| logger.debug(f"[Apify] Sending request to run actor {actor_id} synchronously for {video_url}") | |
| response = await asyncio.to_thread(requests.post, api_endpoint, headers=headers, params=params, data=payload, timeout=90) | |
| logger.debug(f"[Apify] Received status code {response.status_code} for {video_url}") | |
| if response.status_code in [200, 201]: | |
| try: | |
| results = response.json() | |
| if isinstance(results, list) and len(results) > 0: | |
| item = results[0] | |
| content = item.get("text") or item.get("transcript") or item.get("captions_concatenated") | |
| if not content and item.get("captions"): | |
| captions_data = item["captions"] | |
| if isinstance(captions_data, str): | |
| logger.info("[Apify] Processing 'captions' string format as fallback.") | |
| content = captions_data.strip() | |
| if len(content) < 50 and "error" in content.lower(): | |
| logger.warning(f"[Apify] 'captions' string looks like an error: {content}") | |
| content = None | |
| elif isinstance(captions_data, list): | |
| logger.info("[Apify] Processing 'captions' list format as fallback.") | |
| texts = [cap.get("text", "") for cap in captions_data if isinstance(cap, dict) and cap.get("text")] | |
| content = " ".join(texts).strip() | |
| else: | |
| logger.warning(f"[Apify] 'captions' field found but is neither string nor list: {type(captions_data)}") | |
| content = None | |
| if content: | |
| try: | |
| content = html.unescape(content) # Use imported html module | |
| except Exception as unescape_err: | |
| logger.warning(f"[Apify] Error during html unescaping: {unescape_err}") | |
| if content and isinstance(content, str): | |
| logger.info(f"[Apify] Successfully fetched transcript via run-sync for {video_url} (Status: {response.status_code}). Length: {len(content)}") | |
| return content | |
| else: | |
| if item.get("text") or item.get("transcript") or item.get("captions_concatenated"): logger.warning(f"[Apify] Actor success ({response.status_code}) but primary fields empty for {video_url}.") | |
| elif not item.get("captions"): logger.warning(f"[Apify] Actor success ({response.status_code}) but no relevant fields found for {video_url}. Item: {item}") | |
| else: logger.warning(f"[Apify] Actor success ({response.status_code}), 'captions' found but fallback parsing failed for {video_url}.") | |
| return None | |
| else: | |
| logger.warning(f"[Apify] Actor success ({response.status_code}) but dataset result list empty for {video_url}. Response: {results}") | |
| return None | |
| except json.JSONDecodeError: | |
| logger.error(f"[Apify] Failed JSON decode for {video_url}. Status: {response.status_code}. Resp: {response.text[:200]}...") | |
| return None | |
| except Exception as e: | |
| logger.error(f"[Apify] Error processing successful response ({response.status_code}) for {video_url}: {e}", exc_info=True) | |
| return None | |
| elif response.status_code == 400: logger.error(f"[Apify] Bad Request (400) for {video_url}. Resp: {response.text[:200]}..."); return None | |
| elif response.status_code == 401: logger.error("[Apify] Auth error (401). Check token."); return None | |
| else: logger.error(f"[Apify] Unexpected status {response.status_code} for {video_url}. Resp: {response.text[:200]}..."); return None | |
| except requests.exceptions.Timeout: logger.error(f"[Apify] Timeout error running actor for {video_url}"); return None | |
| except requests.exceptions.RequestException as e: logger.error(f"[Apify] Request error running actor for {video_url}: {e}"); return None | |
| except Exception as e: logger.error(f"[Apify] Unexpected error during Apify call for {video_url}: {e}", exc_info=True); return None | |
| # Combined YouTube Transcript Function | |
| async def get_youtube_transcript(video_id: str, video_url: str, supadata_key: str | None, apify_token: str | None): | |
| """Fetches YouTube transcript using library, then Supadata, then Apify.""" | |
| if not video_id: logger.error("get_youtube_transcript called with no video_id"); return None | |
| logger.info(f"Fetching transcript for video ID: {video_id} (URL: {video_url})") | |
| transcript_text = None | |
| try: # Primary: Library | |
| logger.info("[Primary YT] Attempting youtube-transcript-api...") | |
| transcript_list = await asyncio.to_thread(YouTubeTranscriptApi.get_transcript, video_id, languages=['en', 'en-GB', 'en-US']) | |
| if transcript_list: | |
| transcript_text = " ".join([item['text'] for item in transcript_list if 'text' in item]) | |
| transcript_text = re.sub(r'\s+', ' ', transcript_text).strip() | |
| if transcript_text: logger.info(f"[Primary YT] Success via library. Length: {len(transcript_text)}"); return transcript_text | |
| else: logger.warning("[Primary YT] Joined text empty after cleaning."); transcript_text = None | |
| else: logger.warning("[Primary YT] Transcript list empty."); transcript_text = None | |
| except Exception as e: | |
| logger.warning(f"[Primary YT] Error via library: {type(e).__name__} - {e}") | |
| if "YouTube is blocking requests" in str(e) or "HTTP Error 429" in str(e): logger.warning("[Primary YT] IP likely blocked.") | |
| elif "No transcript found" in str(e): logger.warning("[Primary YT] No transcript in languages.") | |
| elif "TranscriptsDisabled" in str(e) or "disabled" in str(e): logger.warning("[Primary YT] Transcripts disabled.") | |
| transcript_text = None | |
| if transcript_text is None: # Fallback 1: Supadata | |
| logger.info("[Fallback YT 1] Trying Supadata API...") | |
| if supadata_key: | |
| transcript_text = await get_transcript_via_supadata(video_id, supadata_key) | |
| if transcript_text: logger.info(f"[Fallback YT 1] Success via Supadata. Length: {len(transcript_text)}"); return transcript_text | |
| else: logger.warning("[Fallback YT 1] Supadata failed or no content.") | |
| else: logger.warning("[Fallback YT 1] Supadata key not available.") | |
| if transcript_text is None: # Fallback 2: Apify | |
| logger.info("[Fallback YT 2] Trying Apify API...") | |
| if apify_token: | |
| transcript_text = await get_transcript_via_apify(video_url, apify_token) | |
| if transcript_text: logger.info(f"[Fallback YT 2] Success via Apify. Length: {len(transcript_text)}"); return transcript_text | |
| else: logger.warning("[Fallback YT 2] Apify failed or no content.") | |
| else: logger.warning("[Fallback YT 2] Apify token not available.") | |
| if transcript_text is None: logger.error(f"All methods failed for video ID: {video_id}") | |
| return transcript_text | |
| # Website Content via Requests/BS4 | |
| async def get_website_content_via_requests(url): | |
| """Attempts to scrape website content using requests/BeautifulSoup.""" | |
| if not url: logger.error("[Web Scraper - Requests/BS4] no URL"); return None | |
| logger.info(f"[Web Scraper - Requests/BS4] Fetching: {url}") | |
| try: | |
| headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8','Accept-Language': 'en-US,en;q=0.9','Connection': 'keep-alive','DNT': '1','Upgrade-Insecure-Requests': '1'} | |
| response = await asyncio.to_thread(requests.get, url, headers=headers, timeout=25, allow_redirects=True) | |
| response.raise_for_status() | |
| logger.debug(f"[Web Scraper - Requests/BS4] Status {response.status_code} for {url}") | |
| content_type = response.headers.get('content-type', '').lower() | |
| if 'html' not in content_type: | |
| logger.warning(f"[Web Scraper - Requests/BS4] Non-HTML: {content_type}. Plain text?") | |
| if 'text/plain' in content_type and response.text: return response.text.strip() | |
| return None | |
| soup = BeautifulSoup(response.text, 'html.parser') | |
| for element in soup(["script", "style", "header", "footer", "nav", "aside", "form", "button", "input", "textarea", "select", "option", "label", "iframe", "img", "svg", "link", "meta", "noscript", "figure", "figcaption", "video", "audio"]): element.extract() | |
| main_content = soup.find('main') or soup.find('article') or soup.find(id='content') or soup.find(class_='content') or soup.find(id='main-content') or soup.find(class_='main-content') or soup.find(role='main') | |
| target_element = main_content if main_content else soup.body | |
| if not target_element: logger.warning(f"[Web Scraper - Requests/BS4] No body/main content for {url}"); return None | |
| lines = [line.strip() for line in target_element.get_text(separator='\n', strip=True).splitlines() if line.strip()] | |
| text = "\n".join(lines) | |
| MIN_TEXT_LENGTH = 50 | |
| if not text or len(text) < MIN_TEXT_LENGTH: logger.warning(f"[Web Scraper - Requests/BS4] Text too short (<{MIN_TEXT_LENGTH} chars) for {url} (Len: {len(text)})") | |
| logger.info(f"[Web Scraper - Requests/BS4] Success scraping {url} (Len: {len(text)})") | |
| return text | |
| except requests.exceptions.Timeout: logger.error(f"[Web Scraper - Requests/BS4] Timeout: {url}"); return None | |
| except requests.exceptions.TooManyRedirects: logger.error(f"[Web Scraper - Requests/BS4] Redirects: {url}"); return None | |
| except requests.exceptions.RequestException as e: logger.error(f"[Web Scraper - Requests/BS4] Request error {url}: {e}"); return None | |
| except Exception as e: logger.error(f"[Web Scraper - Requests/BS4] Parsing error {url}: {e}", exc_info=True); return None | |
| # Website Content via URLToText API | |
| async def get_website_content_via_urltotext_api(url: str, api_key: str): | |
| """Fetches website content using the URLToText API.""" | |
| if not url: logger.error("[Web Scraper - URLToText API] no URL"); return None | |
| if not api_key: logger.error("[Web Scraper - URLToText API] API key missing."); return None | |
| logger.info(f"[Web Scraper - URLToText API] Attempting fetch: {url}") | |
| api_endpoint = "https://urltotext.com/api/v1/urltotext/" | |
| payload = json.dumps({"url": url, "output_format": "text", "extract_main_content": True, "render_javascript": True, "residential_proxy": False}) | |
| headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} | |
| try: | |
| response = await asyncio.to_thread(requests.post, api_endpoint, headers=headers, data=payload, timeout=45) | |
| logger.debug(f"[Web Scraper - URLToText API] Status {response.status_code} for {url}") | |
| if response.status_code == 200: | |
| try: | |
| data = response.json() | |
| content = data.get("data", {}).get("content"); credits = data.get("credits_used", "N/A"); warning = data.get("data", {}).get("warning") | |
| if warning: logger.warning(f"[Web Scraper - URLToText API] Warning for {url}: {warning}") | |
| if content: logger.info(f"[Web Scraper - URLToText API] Success via API. Length: {len(content)}. Credits: {credits}"); return content.strip() | |
| else: logger.warning(f"[Web Scraper - URLToText API] API success (200) but content empty. Resp: {data}"); return None | |
| except json.JSONDecodeError: logger.error(f"[Web Scraper - URLToText API] Failed JSON decode. Status: {response.status_code}. Resp: {response.text[:500]}..."); return None | |
| except Exception as e: logger.error(f"[Web Scraper - URLToText API] Error processing API response: {e}", exc_info=True); return None | |
| elif response.status_code == 400: logger.error(f"[Web Scraper - URLToText API] Bad Request (400). Resp: {response.text[:200]}...") | |
| elif response.status_code == 401: logger.error(f"[Web Scraper - URLToText API] Unauthorized (401). Check Key. Resp: {response.text[:200]}...") | |
| elif response.status_code == 402: logger.error(f"[Web Scraper - URLToText API] Payment Required (402). Check credits. Resp: {response.text[:200]}...") | |
| elif response.status_code == 422: logger.warning(f"[Web Scraper - URLToText API] Unprocessable URL (422). Resp: {response.text[:200]}...") | |
| elif response.status_code >= 500: logger.error(f"[Web Scraper - URLToText API] Server Error ({response.status_code}). Resp: {response.text[:200]}...") | |
| else: logger.error(f"[Web Scraper - URLToText API] Unexpected status {response.status_code}. Resp: {response.text[:200]}...") | |
| return None | |
| except requests.exceptions.Timeout: logger.error(f"[Web Scraper - URLToText API] Timeout: {url}"); return None | |
| except requests.exceptions.RequestException as e: logger.error(f"[Web Scraper - URLToText API] Request error: {e}"); return None | |
| except Exception as e: logger.error(f"[Web Scraper - URLToText API] Unexpected error: {e}", exc_info=True); return None | |
| # DeepSeek Summary Function | |
| async def generate_summary(text: str, summary_type: str, api_key: str) -> str: | |
| """Generates summary using DeepSeek via OpenRouter API.""" | |
| logger.info(f"Generating '{summary_type}' summary. Input length: {len(text)}") | |
| if not api_key: logger.error("OpenRouter API key missing."); return "Error: AI config key missing." | |
| if not text: logger.warning("generate_summary called with empty text."); return "Error: No content to summarize." | |
| openrouter_api_endpoint = "https://openrouter.ai/api/v1/chat/completions"; model_name = "deepseek/deepseek-chat:free" | |
| if summary_type == "paragraph": prompt = "Please provide a concise, well-written paragraph summarizing the key information and main points of the following text. Focus on capturing the essence of the content accurately." | |
| elif summary_type == "points": prompt = "Please summarize the following text into clear, distinct bullet points. Each point should highlight a key piece of information, finding, or main topic discussed. Aim for clarity and conciseness." | |
| else: logger.error(f"Invalid summary_type '{summary_type}'."); return f"Error: Invalid summary type ('{summary_type}')." | |
| MAX_INPUT_LENGTH = 500000 | |
| if len(text) > MAX_INPUT_LENGTH: logger.warning(f"Input text ({len(text)}) > limit ({MAX_INPUT_LENGTH}). Truncating."); text = text[:MAX_INPUT_LENGTH] + "... (Truncated)" | |
| full_prompt = f"{prompt}\n\n--- Start of Text ---\n\n{text}\n\n--- End of Text ---" | |
| space_host = os.environ.get("SPACE_HOST", "huggingface.co/spaces/YOUR_SPACE_NAME") | |
| referer_url = f"https://{space_host}" if not space_host.startswith("http") else space_host | |
| headers = {"Authorization": f"Bearer {api_key}","Content-Type": "application/json","HTTP-Referer": referer_url,"X-Title": "Telegram URL Summarizer Bot"} | |
| payload = json.dumps({"model": model_name, "messages": [{"role": "user", "content": full_prompt}]}) | |
| try: | |
| logger.debug(f"Sending request to OpenRouter (Model: {model_name})...") | |
| response = await asyncio.to_thread(requests.post, openrouter_api_endpoint, headers=headers, data=payload, timeout=90) | |
| logger.debug(f"Received status {response.status_code} from OpenRouter.") | |
| if response.status_code == 200: | |
| try: | |
| data = response.json() | |
| if data.get("choices") and isinstance(data["choices"], list) and len(data["choices"]) > 0: | |
| message = data["choices"][0].get("message") | |
| if message and message.get("content"): | |
| summary = message["content"].strip() | |
| if summary: logger.info(f"Success generating summary. Len: {len(summary)}"); return summary | |
| else: logger.warning(f"OpenRouter success but empty content. Resp: {data}"); return "Sorry, AI model returned empty summary." | |
| else: logger.warning(f"OpenRouter success but missing content field. Resp: {data}"); return "Sorry, could not parse AI response (content)." | |
| elif data.get("error"): logger.error(f"OpenRouter API Error: {data['error']}"); return f"Sorry, AI service error: {data['error'].get('message', 'Unknown')}" | |
| else: logger.error(f"Unexpected OpenRouter choices structure. Resp: {data}"); return "Sorry, could not parse AI response (choices)." | |
| except json.JSONDecodeError: logger.error(f"Failed JSON decode from OpenRouter. Status: {response.status_code}. Resp: {response.text[:500]}..."); return "Sorry, failed to understand AI response format." | |
| except Exception as e: logger.error(f"Error processing OpenRouter success resp: {e}", exc_info=True); return "Sorry, error processing AI response." | |
| elif response.status_code == 401: logger.error("OpenRouter key invalid (401)."); return "Error: AI model config key invalid." | |
| elif response.status_code == 402: logger.error("OpenRouter Payment Required (402)."); return "Sorry, issue with AI service limits/payment." | |
| elif response.status_code == 429: logger.warning("OpenRouter Rate Limit (429)."); return "Sorry, AI model busy. Try again." | |
| elif response.status_code >= 500: logger.error(f"OpenRouter Internal Error ({response.status_code}). Resp: {response.text[:500]}..."); return "Sorry, AI model service error. Try again later." | |
| else: | |
| logger.error(f"Unexpected status {response.status_code} from OpenRouter. Resp: {response.text[:500]}...") | |
| try: error_data = response.json(); error_msg = error_data.get("error", {}).get("message", response.text[:100]); return f"Sorry, AI service error ({response.status_code}): {error_msg}" | |
| except: return f"Sorry, AI service returned status {response.status_code}." | |
| except requests.exceptions.Timeout: logger.error("Timeout connecting to OpenRouter."); return "Sorry, request to AI model timed out." | |
| except requests.exceptions.RequestException as e: logger.error(f"Request error connecting to OpenRouter: {e}"); return "Sorry, error connecting to AI model service." | |
| except Exception as e: logger.error(f"Unexpected error in generate_summary: {e}", exc_info=True); return "Sorry, unexpected error generating summary." | |
| # --- Telegram Bot Handlers --- | |
| async def start(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: | |
| user = update.effective_user; | |
| if not user: return | |
| logger.info(f"User {user.id} ({user.username or 'NoUsername'}) /start.") | |
| mention = user.mention_html() if user.username else user.first_name | |
| await update.message.reply_html(f"👋 Hello {mention}! Send a URL to summarize.") | |
| async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: | |
| user = update.effective_user; logger.info(f"User {user.id if user else '?'} /help.") | |
| help_text = ("**How:**\n1. Send URL.\n2. Choose Paragraph/Points.\n3. Get summary!\n\n" | |
| "**Notes:**\n- YT transcripts can fail.\n- Complex sites hard to scrape.\n- AI errors possible.") | |
| await update.message.reply_text(help_text, parse_mode=ParseMode.MARKDOWN) | |
| async def handle_potential_url(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: | |
| if not update.message or not update.message.text: return | |
| message_text = update.message.text.strip(); user = update.effective_user; | |
| if not user: return | |
| url_pattern = r'https?://[^\s/$.?#].[^\s]*'; match = re.search(url_pattern, message_text) | |
| if match: | |
| url = match.group(0); logger.info(f"User {user.id} sent URL: {url}") | |
| context.user_data['url_to_summarize'] = url; logger.debug(f"Stored URL '{url}' for user {user.id}") | |
| keyboard = [[InlineKeyboardButton("Paragraph", callback_data="paragraph"), InlineKeyboardButton("Points", callback_data="points")]] | |
| reply_markup = InlineKeyboardMarkup(keyboard) | |
| await update.message.reply_text(f"Link:\n{url}\n\nChoose summary type:", reply_markup=reply_markup, link_preview_options={'is_disabled': True}) | |
| else: logger.debug(f"Ignoring non-URL from {user.id}: {message_text[:100]}") | |
| async def handle_summary_type_callback(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: | |
| query = update.callback_query; user = query.from_user | |
| try: await query.answer(); logger.debug(f"Answered query {query.id}") | |
| except Exception as e: logger.error(f"Failed answer query {query.id}: {e}") | |
| summary_type = query.data; url = context.user_data.get('url_to_summarize') | |
| logger.info(f"User {user.id} chose '{summary_type}'. URL: '{url}'.") | |
| if not url: | |
| logger.warning(f"User {user.id} pressed button, NO URL in context."); | |
| try: await query.edit_message_text(text="Context lost. Send link again.") | |
| except Exception as edit_err: logger.error(f"Failed edit on lost context: {edit_err}") | |
| return | |
| context.user_data.pop('url_to_summarize', None); logger.debug(f"Cleared URL {url} for user {user.id}") | |
| current_openrouter_key = os.environ.get('OPENROUTER_API_KEY'); current_urltotext_key = os.environ.get('URLTOTEXT_API_KEY') | |
| current_supadata_key = os.environ.get('SUPADATA_API_KEY'); current_apify_token = os.environ.get('APIFY_API_TOKEN') | |
| logger.debug(f"Keys: OR={'Y' if current_openrouter_key else 'N'}, UTT={'Y' if current_urltotext_key else 'N'}, SD={'Y' if current_supadata_key else 'N'}, AP={'Y' if current_apify_token else 'N'}") | |
| if not current_openrouter_key: | |
| logger.error("OpenRouter key missing."); | |
| try: await query.edit_message_text(text="⚠️ AI service config error (key missing).") | |
| except Exception as edit_err: logger.error(f"Failed edit on missing OR key: {edit_err}") | |
| return | |
| processing_message_text = f"Working on '{summary_type}' summary for:\n{url}\n..."; message_to_delete_later_id = None | |
| try: await query.edit_message_text(text=processing_message_text); logger.debug(f"Edited message query {query.id}") | |
| except Exception as e: | |
| logger.warning(f"Could not edit message {query.message.message_id if query.message else 'N/A'}: {e}. Sending new."); | |
| try: status_message = await context.bot.send_message(chat_id=user.id, text=processing_message_text); message_to_delete_later_id = status_message.message_id; logger.debug(f"Sent new status message {message_to_delete_later_id}") | |
| except Exception as send_err: logger.error(f"Failed sending new status message: {send_err}") | |
| content = None; user_feedback_message = None; success = False | |
| try: | |
| logger.debug(f"Sending 'typing' action for chat {user.id}"); await context.bot.send_chat_action(chat_id=user.id, action='typing') | |
| is_yt = is_youtube_url(url); logger.debug(f"URL is YouTube: {is_yt}") | |
| if is_yt: | |
| video_id = extract_youtube_id(url) | |
| if video_id: | |
| logger.info(f"Fetching YT transcript: {video_id}"); content = await get_youtube_transcript(video_id, url, current_supadata_key, current_apify_token) | |
| if not content: user_feedback_message = "Sorry, couldn't get YT transcript (unavailable/private/no captions?)." | |
| logger.info(f"YT transcript fetch done. Found: {bool(content)}") | |
| else: logger.warning(f"Failed YT ID extraction: {url}"); user_feedback_message = "Sorry, couldn't parse YT video ID." | |
| else: | |
| logger.info(f"Scraping website (Requests/BS4): {url}"); content = await get_website_content_via_requests(url) | |
| if content: logger.info("Website scrape (Requests/BS4) OK."); user_feedback_message = None | |
| else: | |
| logger.warning(f"Website scrape failed for {url}. Trying URLToText API."); | |
| if current_urltotext_key: | |
| await context.bot.send_chat_action(chat_id=user.id, action='typing'); content = await get_website_content_via_urltotext_api(url, current_urltotext_key) | |
| if content: logger.info("URLToText API scrape OK."); user_feedback_message = None | |
| else: logger.warning(f"URLToText scrape failed for {url}."); user_feedback_message = "Sorry, couldn't fetch web content (both methods)." | |
| else: logger.warning("URLToText key not configured."); user_feedback_message = "Sorry, couldn't fetch web content (fallback not configured)." | |
| if content: | |
| logger.info("Content found, generating summary."); await context.bot.send_chat_action(chat_id=user.id, action='typing') | |
| summary = await generate_summary(content, summary_type, current_openrouter_key) | |
| if summary.startswith("Error:") or summary.startswith("Sorry,"): user_feedback_message = summary; logger.warning(f"Summary generation failed: {summary}") | |
| else: logger.info("Summary generated OK. Sending."); await context.bot.send_message(chat_id=user.id, text=summary, parse_mode=ParseMode.MARKDOWN, link_preview_options={'is_disabled': True}); success = True; user_feedback_message = None | |
| elif not user_feedback_message: user_feedback_message = "Sorry, couldn't retrieve content from link." | |
| if user_feedback_message and not success: logger.warning(f"Sending failure feedback: {user_feedback_message}"); await context.bot.send_message(chat_id=user.id, text=user_feedback_message) | |
| except Exception as e: | |
| logger.error(f"Unexpected error in callback processing: {e}", exc_info=True); | |
| try: await context.bot.send_message(chat_id=user.id, text="Oops! Internal error processing request.") | |
| except Exception as final_err: logger.error(f"Failed sending final error message: {final_err}") | |
| finally: | |
| logger.debug("Cleaning up status message..."); | |
| try: | |
| if message_to_delete_later_id: await context.bot.delete_message(chat_id=user.id, message_id=message_to_delete_later_id); logger.debug("Deleted separate status msg.") | |
| elif query.message: await query.delete_message(); logger.debug(f"Deleted original message query {query.id}.") | |
| except Exception as del_e: logger.warning(f"Could not delete status/button message: {del_e}") | |
| async def error_handler(update: object, context: ContextTypes.DEFAULT_TYPE) -> None: | |
| """Log Errors caused by Updates.""" | |
| logger.error(f"Exception while handling an update: {context.error}", exc_info=context.error) | |
| # --- Bot Setup Function (Configure Only) --- | |
| async def setup_bot_config() -> Application: | |
| """Configures the PTB Application but does NOT initialize or start it.""" | |
| logger.info("Configuring Telegram Application...") | |
| if not TELEGRAM_TOKEN: | |
| logger.critical("CRITICAL: TELEGRAM_TOKEN environment variable not found.") | |
| raise ValueError("TELEGRAM_TOKEN environment variable not set.") | |
| application = Application.builder().token(TELEGRAM_TOKEN).build() | |
| application.add_handler(CommandHandler("start", start)) | |
| application.add_handler(CommandHandler("help", help_command)) | |
| application.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND, handle_potential_url)) | |
| application.add_handler(CallbackQueryHandler(handle_summary_type_callback)) | |
| application.add_error_handler(error_handler) | |
| logger.info("Telegram handlers configured.") | |
| return application | |
| # --- ASGI Lifespan Context Manager --- | |
| async def lifespan(app: Starlette): # app argument is the Starlette instance | |
| """Handles PTB startup and shutdown during ASGI lifespan.""" | |
| global ptb_app | |
| logger.info("ASGI Lifespan: Startup commencing...") | |
| loop = asyncio.get_running_loop() | |
| try: | |
| ptb_app = await setup_bot_config() | |
| logger.info("PTB App configured. Initializing...") | |
| await ptb_app.initialize() | |
| logger.info("PTB App initialized. Starting background tasks...") | |
| await ptb_app.start() | |
| logger.info(f"PTB App started. Bot details: {ptb_app.bot.username}") | |
| # Set webhook after start | |
| WEBHOOK_URL_BASE = os.environ.get("SPACE_HOST") | |
| if WEBHOOK_URL_BASE: | |
| if not WEBHOOK_URL_BASE.startswith("https://"): WEBHOOK_URL_BASE = f"https://{WEBHOOK_URL_BASE}" | |
| webhook_path = "/webhook" | |
| full_webhook_url = f"{WEBHOOK_URL_BASE.rstrip('/')}{webhook_path}" | |
| logger.info(f"Attempting to set webhook to: {full_webhook_url}") | |
| try: | |
| # Add a small delay before setting webhook, especially with multiple workers | |
| # This might help avoid the initial rate limit error, although one worker succeeding is enough. | |
| await asyncio.sleep(1.5) # Wait 1.5 seconds | |
| await ptb_app.bot.set_webhook(url=full_webhook_url, allowed_updates=Update.ALL_TYPES) | |
| webhook_info = await ptb_app.bot.get_webhook_info() | |
| logger.info(f"Webhook set successfully! Info: {webhook_info}") | |
| except RetryAfter as e: | |
| logger.warning(f"Webhook set failed due to rate limit (RetryAfter: {e.retry_after}s). Another worker likely succeeded.") | |
| except Exception as e: | |
| logger.error(f"Failed to set webhook: {e}", exc_info=True) | |
| else: logger.warning("SPACE_HOST env variable not found. Skipping webhook setup.") | |
| logger.info("ASGI Lifespan: Startup complete. Application ready.") | |
| yield # Application runs here | |
| except Exception as startup_err: | |
| logger.critical(f"CRITICAL ERROR during ASGI startup: {startup_err}", exc_info=True) | |
| raise | |
| finally: | |
| # --- Shutdown --- | |
| logger.info("ASGI Lifespan: Shutdown commencing...") | |
| # Use the correct attribute to check if running before stopping/shutting down | |
| if ptb_app and ptb_app._running: # <--- Use _running here too | |
| try: | |
| logger.info("Stopping PTB App...") | |
| await ptb_app.stop() | |
| logger.info("Shutting down PTB App...") | |
| await ptb_app.shutdown() | |
| logger.info("PTB App shut down successfully.") | |
| except Exception as shutdown_err: | |
| logger.error(f"Error during PTB shutdown: {shutdown_err}", exc_info=True) | |
| elif ptb_app: | |
| logger.warning("PTB App instance exists but was not running at shutdown.") | |
| else: | |
| logger.warning("No PTB App instance found at shutdown.") | |
| logger.info("ASGI Lifespan: Shutdown complete.") | |
| # --- Flask App Setup (for Routes) --- | |
| flask_core_app = Flask(__name__) | |
| logger.info("Core Flask app instance created (for routing via Starlette).") | |
| # --- Define Flask Routes on flask_core_app --- | |
| def index(): | |
| """Basic health check endpoint.""" | |
| logger.debug("Health check '/' accessed.") | |
| bot_status = "UNKNOWN" | |
| if ptb_app: | |
| # --- CORRECTED CHECK --- | |
| bot_status = "Running" if ptb_app._running else "Initialized/Stopped/Starting/Error" | |
| # --- END CORRECTION --- | |
| else: | |
| bot_status = "Not Initialized" | |
| return f"Telegram Bot Webhook Listener ({bot_status}) running via Starlette." | |
| async def webhook() -> Response: | |
| """Webhook endpoint for Telegram updates.""" | |
| if not ptb_app: # Check if instance exists first | |
| logger.error("Webhook triggered, but PTB Application instance (ptb_app) is None.") | |
| return Response('Bot service not configured.', status=503) | |
| # --- CORRECTED CHECK --- | |
| # Use the internal _running attribute as suggested by the AttributeError | |
| if not ptb_app._running: | |
| # --- END CORRECTION --- | |
| status = "Not Running" # If instance exists but not running | |
| logger.error(f"Webhook triggered, but PTB Application is {status}.") | |
| return Response('Bot service not ready.', status=503) | |
| logger.debug("Webhook request received (POST)...") | |
| if request.is_json: | |
| try: | |
| update_data = request.get_json() | |
| update = Update.de_json(update_data, ptb_app.bot) | |
| logger.debug(f"Processing update ID: {update.update_id} via webhook") | |
| await ptb_app.process_update(update) # Queue/process the update | |
| logger.debug(f"Finished processing update ID: {update.update_id}") | |
| return Response('ok', status=200) | |
| except json.JSONDecodeError: logger.error("Failed JSON decode from Telegram."); return Response('Bad Request: Invalid JSON', status=400) | |
| except Exception as e: logger.error(f"Error processing update in webhook handler: {e}", exc_info=True); return Response('Internal Server Error processing update.', status=500) | |
| else: logger.warning("Received non-JSON request to webhook."); return Response('Bad Request: Expected JSON', status=400) | |
| # --- Create Starlette App with Lifespan & Mount Flask --- | |
| app = Starlette( | |
| lifespan=lifespan, | |
| routes=[ | |
| Mount("/", app=WSGIMiddleware(flask_core_app)) | |
| ] | |
| ) | |
| logger.info("Starlette application created with lifespan and Flask app mounted at '/'.") | |
| # --- Main Execution Block (for local testing ONLY) --- | |
| if __name__ == '__main__': | |
| logger.warning("Running Flask development server directly (LOCAL TESTING ONLY).") | |
| logger.warning("NOTE: This mode does NOT initialize PTB via ASGI lifespan.") | |
| logger.warning("Use 'uvicorn main:app --reload --port 8080' for proper local ASGI testing.") | |
| if not TELEGRAM_TOKEN: logger.critical("Aborting local Flask start: TELEGRAM_TOKEN missing.") | |
| else: | |
| local_port = int(os.environ.get('PORT', 8080)) | |
| logger.info(f"Flask dev server starting on http://0.0.0.0:{local_port}") | |
| flask_core_app.run(host='0.0.0.0', port=local_port, debug=True) |