Upload 5 files

app.py

@@ -0,0 +1,247 @@
+import streamlit as st
+import time
+import asyncio
+import nest_asyncio
+import traceback
+from typing import List, Dict, Any
+import re  # for extracting citation IDs
+
+# --- Configuration and Service Initialization ---
+try:
+    print("App: Loading config...")
+    import config
+    print("App: Loading utils...")
+    from utils import clean_source_text
+    print("App: Loading services...")
+    from services.retriever import init_retriever, get_retriever_status
+    from services.openai_service import init_openai_client, get_openai_status
+    print("App: Loading RAG processor... ")
+    from rag_processor import execute_validate_generate_pipeline, PIPELINE_VALIDATE_GENERATE_GPT4O
+    print("App: Imports successful.")
+except ImportError as e:
+    st.error(f"Fatal Error: Module import failed. {e}", icon="🚨")
+    traceback.print_exc()
+    st.stop()
+except Exception as e:
+    st.error(f"Fatal Error during initial setup: {e}", icon="🚨")
+    traceback.print_exc()
+    st.stop()
+
+nest_asyncio.apply()
+
+# --- Initialize Required Services ---
+print("App: Initializing services...")
+try:
+    retriever_ready_init, retriever_msg_init = init_retriever()
+    openai_ready_init, openai_msg_init = init_openai_client()
+    print("App: Service initialization calls complete.")
+except Exception as init_err:
+    st.error(f"Error during service initialization: {init_err}", icon="🔥")
+    traceback.print_exc()
+
+# --- Streamlit Page Configuration and Styling ---
+st.set_page_config(page_title="Divrey Yoel AI Chat (GPT-4o Gen)", layout="wide")
+st.markdown("""<style> /* ... Keep existing styles ... */ </style>""", unsafe_allow_html=True)
+st.markdown("<h1 class='rtl-text'>דברות קודש - חיפוש ועיון</h1>", unsafe_allow_html=True)
+st.markdown("<p class='rtl-text'>מבוסס על ספרי דברי יואל מסאטמאר זצוק'ל - אחזור מידע חכם (RAG)</p>", unsafe_allow_html=True)
+st.markdown("<p class='rtl-text' style='font-size: 0.9em; color: #555;'>תהליך: אחזור → אימות (GPT-4o) → יצירה (GPT-4o)</p>", unsafe_allow_html=True)
+
+# --- UI Helper Functions ---
+def display_sidebar() -> Dict[str, Any]:
+    st.sidebar.markdown("<h3 class='rtl-text'>מצב המערכת</h3>", unsafe_allow_html=True)
+    retriever_ready, _ = get_retriever_status()
+    openai_ready, _ = get_openai_status()
+    st.sidebar.markdown(
+        f"<p class='rtl-text'><strong>מאחזר (Pinecone):</strong> {'✅' if retriever_ready else '❌'}</p>",
+        unsafe_allow_html=True
+    )
+    if not retriever_ready:
+        st.sidebar.error("מאחזר אינו זמין.", icon="🛑")
+        st.stop()
+    st.sidebar.markdown(
+        f"<p class='rtl-text'><strong>OpenAI ({config.OPENAI_VALIDATION_MODEL} / {config.OPENAI_GENERATION_MODEL}):</strong> {'✅' if openai_ready else '❌'}</p>",
+        unsafe_allow_html=True
+    )
+    if not openai_ready:
+        st.sidebar.error("OpenAI אינו זמין.", icon="⚠️")
+    st.sidebar.markdown("<hr>", unsafe_allow_html=True)
+
+    n_retrieve = st.sidebar.slider("מספר פסקאות לאחזור", 1, 300, config.DEFAULT_N_RETRIEVE)
+    max_validate = min(n_retrieve, 100)
+    n_validate = st.sidebar.slider(
+        "פסקאות לאימות (GPT-4o)",
+        1,
+        max_validate,
+        min(config.DEFAULT_N_VALIDATE, max_validate),
+        disabled=not openai_ready
+    )
+    st.sidebar.info("התשובות מבוססות רק על המקורות שאומתו.", icon="ℹ️")
+
+    # Prompt editors
+    with st.sidebar.expander("Edit RAG Prompts", expanded=False):
+        gen_prompt = st.text_area(
+            "System prompt (generation)",
+            value=config.OPENAI_SYSTEM_PROMPT,
+            height=200
+        )
+        val_prompt = st.text_area(
+            "Validation prompt (GPT-4o)",
+            value=config.VALIDATION_PROMPT_TEMPLATE,
+            height=200
+        )
+    config.OPENAI_SYSTEM_PROMPT = gen_prompt
+    config.VALIDATION_PROMPT_TEMPLATE = val_prompt
+
+    return {"n_retrieve": n_retrieve, "n_validate": n_validate, "services_ready": (retriever_ready and openai_ready)}
+
+
+def display_chat_message(message: Dict[str, Any]):
+    role = message.get("role", "assistant")
+    with st.chat_message(role):
+        st.markdown(message.get('content', ''), unsafe_allow_html=True)
+        if role == "assistant" and message.get("final_docs"):
+            docs = message["final_docs"]
+            # Expander for all passed docs
+            exp_title = f"<span class='rtl-text'>הצג {len(docs)} קטעי מקור שנשלחו למחולל</span>"
+            with st.expander(exp_title, expanded=False):
+                st.markdown("<div dir='rtl' class='expander-content'>", unsafe_allow_html=True)
+                for i, doc in enumerate(docs, start=1):
+                    source = doc.get('source_name', '') or 'מקור לא ידוע'
+                    text = clean_source_text(doc.get('hebrew_text', ''))
+                    st.markdown(
+                        f"<div class='source-info rtl-text'><strong>מקור {i}:</strong> {source}</div>",
+                        unsafe_allow_html=True
+                    )
+                    st.markdown(f"<div class='hebrew-text'>{text}</div>", unsafe_allow_html=True)
+                st.markdown("</div>", unsafe_allow_html=True)
+
+
+def display_status_updates(status_log: List[str]):
+    if status_log:
+        with st.expander("<span class='rtl-text'>הצג פרטי עיבוד</span>", expanded=False):
+            for u in status_log:
+                st.markdown(
+                    f"<code class='status-update rtl-text'>- {u}</code>",
+                    unsafe_allow_html=True
+                )
+
+# --- Main Application Logic ---
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+rag_params = display_sidebar()
+
+# Render history
+for msg in st.session_state.messages:
+    display_chat_message(msg)
+
+if prompt := st.chat_input("שאל שאלה בענייני חסידות...", disabled=not rag_params["services_ready"]):
+    st.session_state.messages.append({"role": "user", "content": prompt})
+    display_chat_message(st.session_state.messages[-1])
+
+    with st.chat_message("assistant"):
+        msg_placeholder = st.empty()
+        status_container = st.status("מעבד בקשה...", expanded=True)
+        chunks: List[str] = []
+        try:
+            def status_cb(m): status_container.update(label=f"<span class='rtl-text'>{m}</span>")
+            def stream_cb(c):
+                chunks.append(c)
+                msg_placeholder.markdown(
+                    f"<div dir='rtl' class='rtl-text'>{''.join(chunks)}▌</div>",
+                    unsafe_allow_html=True
+                )
+
+            loop = asyncio.get_event_loop()
+            final_rag = loop.run_until_complete(
+                execute_validate_generate_pipeline(
+                    history=st.session_state.messages,
+                    params=rag_params,
+                    status_callback=status_cb,
+                    stream_callback=stream_cb
+                )
+            )
+
+            if isinstance(final_rag, dict):
+                raw = final_rag.get("final_response", "")
+                err = final_rag.get("error")
+                log = final_rag.get("status_log", [])
+                docs = final_rag.get("generator_input_documents", [])
+                pipeline = final_rag.get("pipeline_used", PIPELINE_VALIDATE_GENERATE_GPT4O)
+
+                # wrap result
+                final = raw
+                if not (err and final.strip().startswith("<div")) and not final.strip().startswith((
+                    '<div', '<p', '<ul', '<ol', '<strong'
+                )):
+                    final = f"<div dir='rtl' class='rtl-text'>{final or 'לא התקבלה תשובה מהמחולל.'}</div>"
+                msg_placeholder.markdown(final, unsafe_allow_html=True)
+
+                # Identify cited IDs
+                cited_ids = set(re.findall(r'\(מקור\s*([0-9]+)\)', raw))
+                if cited_ids:
+                    enumerated_docs = list(enumerate(docs, start=1))
+                    docs_to_show = [(idx, doc) for idx, doc in enumerated_docs if str(idx) in cited_ids]
+                else:
+                    docs_to_show = list(enumerate(docs, start=1))
+
+                if docs_to_show:
+                    label = f"<span class='rtl-text'>הצג {len(docs_to_show)} קטעי מקור שהוזכרו בתשובה</span>"
+                    with st.expander(label, expanded=False):
+                        st.markdown("<div dir='rtl' class='expander-content'>", unsafe_allow_html=True)
+                        for idx, doc in docs_to_show:
+                            source = doc.get('source_name', '') or 'מקור לא ידוע'
+                            text = clean_source_text(doc.get('hebrew_text', ''))
+                            st.markdown(
+                                f"<div class='source-info rtl-text'><strong>מקור {idx}:</strong> {source}</div>",
+                                unsafe_allow_html=True
+                            )
+                            st.markdown(f"<div class='hebrew-text'>{text}</div>", unsafe_allow_html=True)
+                        st.markdown("</div>", unsafe_allow_html=True)
+
+                # store message
+                assistant_data = {
+                    "role": "assistant",
+                    "content": final,
+                    "final_docs": docs,
+                    "pipeline_used": pipeline,
+                    "status_log": log,
+                    "error": err
+                }
+                st.session_state.messages.append(assistant_data)
+                display_status_updates(log)
+                if err:
+                    status_container.update(label="שגיאה בעיבוד!", state="error", expanded=False)
+                else:
+                    status_container.update(label="העיבוד הושלם!", state="complete", expanded=False)
+            else:
+                msg_placeholder.markdown(
+                    "<div dir='rtl' class='rtl-text'><strong>שגיאה בלתי צפויה בתקשורת.</strong></div>",
+                    unsafe_allow_html=True
+                )
+                st.session_state.messages.append({
+                    "role": "assistant",
+                    "content": "שגיאה בלתי צפויה בתקשורת.",
+                    "final_docs": [],
+                    "pipeline_used": "Error",
+                    "status_log": ["Unexpected result"],
+                    "error": "Unexpected"
+                })
+                status_container.update(label="שגיאה בלתי צפויה!", state="error", expanded=False)
+
+        except Exception as e:
+            traceback.print_exc()
+            err_html = (
+                f"<div dir='rtl' class='rtl-text'><strong>שגיאה קריטית!</strong><br>נסה לרענן."  
+                f"<details><summary>פרטים</summary><pre>{traceback.format_exc()}</pre></details></div>"
+            )
+            msg_placeholder.error(err_html, icon="🔥")
+            st.session_state.messages.append({
+                "role": "assistant",
+                "content": err_html,
+                "final_docs": [],
+                "pipeline_used": "Critical Error",
+                "status_log": [f"Critical: {type(e).__name__}"],
+                "error": str(e)
+            })
+            status_container.update(label=str(e), state="error", expanded=False)

code (9).txt

@@ -0,0 +1,7 @@
+# requirements.txt
+streamlit
+pinecone-client # Correct package name
+openai
+langsmith
+nest_asyncio
+python-dotenv # Optional, but harmless

config.py

@@ -0,0 +1,87 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# --- LangSmith Configuration ---
+LANGSMITH_ENDPOINT = os.environ.get("LANGSMITH_ENDPOINT", "https://api.smith.langchain.com")
+LANGSMITH_TRACING = os.environ.get("LANGSMITH_TRACING", "true")
+LANGSMITH_API_KEY = os.environ.get("LANGSMITH_API_KEY")
+LANGSMITH_PROJECT = os.environ.get("LANGSMITH_PROJECT", "DivreyYoel-RAG-GPT4-Gen")
+
+# --- API Keys (Required) ---
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+PINECONE_API_KEY = os.environ.get("PINECONE_API_KEY")
+
+# --- Model Configuration ---
+EMBEDDING_MODEL = os.environ.get("OPENAI_EMBEDDING_MODEL", "text-embedding-3-large")
+OPENAI_VALIDATION_MODEL = os.environ.get("OPENAI_VALIDATION_MODEL", "gpt-4o")
+OPENAI_GENERATION_MODEL = os.environ.get("OPENAI_GENERATION_MODEL", "o3")
+
+# --- Pinecone Configuration ---
+PINECONE_INDEX_NAME = os.environ.get("PINECONE_INDEX_NAME", "chassidus-index")
+
+# --- Default RAG Pipeline Parameters ---
+DEFAULT_N_RETRIEVE = 300
+DEFAULT_N_VALIDATE = 100
+
+# --- System Prompts ---
+OPENAI_SYSTEM_PROMPT = """You are an expert assistant specializing in Chassidic texts, particularly the works of the Satmar Rebbe, Rabbi Yoel Teitelbaum (Divrei Yoel).
+Your task is to answer the user's question based *exclusively* on the provided source text snippets (paragraphs from relevant books). Do not use any prior knowledge or external information.
+
+**Source Text Format:**
+The relevant source texts will be provided below under the heading "Source Texts:". Each source is numbered and includes an ID.
+
+**Response Requirements:**
+1.  **Language:** Respond **exclusively in Hebrew**.
+2.  **Basis:** Base your answer *strictly* on the information contained within the provided "Source Texts:". Do not infer, add external knowledge, or answer if the context does not contain relevant information.
+3.  **Attribution (Optional but Recommended):** When possible, mention the source number (e.g., "כפי שמופיע במקור 3") where the information comes from. Do not invent information. Use quotes sparingly and only when essential, quoting the Hebrew text directly.
+4.  **Completeness:** Synthesize information from *multiple* relevant sources if they contribute to the answer.
+5.  **Handling Lack of Information:** If the provided sources do not contain information relevant to the question, state clearly in Hebrew that the provided texts do not contain the answer (e.g., "על פי המקורות שסופקו, אין מידע לענות על שאלה זו."). Do not attempt to answer based on outside knowledge.
+6.  **Clarity and Conciseness:** Provide a clear, well-structured, and concise answer in Hebrew. Focus on directly answering the user's question.
+7.  **Tone:** Maintain a formal and respectful tone appropriate for discussing religious texts.
+8.  **No Greetings/Closings:** Do not include introductory greetings (e.g., "שלום") or concluding remarks (e.g., "בברכה", "מקווה שעזרתי"). Focus solely on the answer.
+"""
+
+VALIDATION_PROMPT_TEMPLATE = """
+User Question (Hebrew):
+\"{user_question}\"
+
+Text Paragraph (Paragraph {paragraph_index}):
+Hebrew:
+---
+{hebrew_text}
+---
+English:
+---
+{english_text}
+---
+
+Instruction:
+Analyze the Text Paragraph. Determine if it contains information that *directly* answers or significantly contributes to answering the User Question.
+Respond ONLY with valid JSON: {{\"contains_relevant_info\": boolean, \"justification\": \"Brief Hebrew explanation\"}}.
+Output only the JSON object.
+"""
+
+# --- Helper Functions ---
+def check_env_vars():
+    missing_keys = []
+    if not LANGSMITH_API_KEY: missing_keys.append("LANGSMITH_API_KEY")
+    if not OPENAI_API_KEY: missing_keys.append("OPENAI_API_KEY")
+    if not PINECONE_API_KEY: missing_keys.append("PINECONE_API_KEY")
+    return missing_keys
+
+def configure_langsmith():
+    os.environ["LANGSMITH_ENDPOINT"] = LANGSMITH_ENDPOINT
+    os.environ["LANGSMITH_TRACING"] = LANGSMITH_TRACING
+    if LANGSMITH_API_KEY: os.environ["LANGSMITH_API_KEY"] = LANGSMITH_API_KEY
+    if LANGSMITH_PROJECT: os.environ["LANGSMITH_PROJECT"] = LANGSMITH_PROJECT
+    print(f"LangSmith configured: Endpoint={LANGSMITH_ENDPOINT}, Tracing={LANGSMITH_TRACING}, Project={LANGSMITH_PROJECT or 'Default'}")
+
+missing = check_env_vars()
+if missing:
+    print(f"Warning: Missing essential API keys: {', '.join(missing)}")
+else:
+    print("All essential API keys found.")
+
+configure_langsmith()

rag_processor.py

@@ -0,0 +1,219 @@
+import time
+import asyncio
+import traceback
+from typing import List, Dict, Any, Optional, Callable, Tuple
+from langsmith import traceable
+
+try:
+    import config
+    from services import retriever, openai_service
+except ImportError:
+    print("Error: Failed to import config or services in rag_processor.py")
+    raise SystemExit("Failed imports in rag_processor.py")
+
+PIPELINE_VALIDATE_GENERATE_GPT4O = "GPT-4o Validator + GPT-4o Synthesizer"
+StatusCallback = Callable[[str], None]
+
+# --- Step Functions ---
+
+@traceable(name="rag-step-retrieve")
+async def run_retrieval_step(query: str, n_retrieve: int, update_status: StatusCallback) -> List[Dict]:
+    update_status(f"1. מאחזר עד {n_retrieve} פסקאות מ-Pinecone...")
+    start_time = time.time()
+    retrieved_docs = retriever.retrieve_documents(query_text=query, n_results=n_retrieve)
+    retrieval_time = time.time() - start_time
+    status_msg = f"אוחזרו {len(retrieved_docs)} פסקאות ב-{retrieval_time:.2f} שניות."
+    update_status(f"1. {status_msg}")
+    if not retrieved_docs:
+        update_status("1. לא אותרו מסמכים.")
+    return retrieved_docs
+
+@traceable(name="rag-step-gpt4o-filter")
+async def run_gpt4o_validation_filter_step(
+    docs_to_process: List[Dict], query: str, n_validate: int, update_status: StatusCallback
+) -> List[Dict]:
+    if not docs_to_process:
+        update_status("2. [GPT-4o] דילוג על אימות - אין פסקאות.")
+        return []
+    validation_count = min(len(docs_to_process), n_validate)
+    update_status(f"2. [GPT-4o] מתחיל אימות מקבילי ({validation_count} / {len(docs_to_process)} פסקאות)...")
+    validation_start_time = time.time()
+    tasks = [openai_service.validate_relevance_openai(doc, query, i)
+             for i, doc in enumerate(docs_to_process[:validation_count])]
+    validation_results = await asyncio.gather(*tasks, return_exceptions=True)
+    passed_docs = []
+    passed_count = failed_validation_count = error_count = 0
+    update_status("3. [GPT-4o] סינון פסקאות לפי תוצאות אימות...")
+    for i, res in enumerate(validation_results):
+        original_doc = docs_to_process[i]
+        if isinstance(res, Exception):
+            print(f"GPT-4o Validation Exception doc {i}: {res}")
+            error_count += 1
+        elif isinstance(res, dict) and 'validation' in res:
+            if res['validation'].get('contains_relevant_info'):
+                original_doc['validation_result'] = res['validation']
+                passed_docs.append(original_doc)
+                passed_count += 1
+            else:
+                failed_validation_count += 1
+        else:
+            print(f"GPT-4o Validation Unexpected result doc {i}: {type(res)}")
+            error_count += 1
+    validation_time = time.time() - validation_start_time
+    status_msg_val = (f"אימות GPT-4o הושלם ({passed_count} עברו, "
+                      f"{failed_validation_count} נדחו, {error_count} שגיאות) "
+                      f"ב-{validation_time:.2f} שניות.")
+    update_status(f"2. {status_msg_val}")
+    status_msg_filter = f"נאספו {len(passed_docs)} פסקאות רלוונטיות לאחר אימות GPT-4o."
+    update_status(f"3. {status_msg_filter}")
+    return passed_docs
+
+@traceable(name="rag-step-openai-generate")
+async def run_openai_generation_step(
+    history: List[Dict], context_documents: List[Dict],
+    update_status: StatusCallback, stream_callback: Callable[[str], None]
+) -> Tuple[str, Optional[str]]:
+    generator_name = "OpenAI"
+    if not context_documents:
+        update_status(f"4. [{generator_name}] דילוג על יצירה - אין פסקאות להקשר.")
+        return "לא סופקו פסקאות רלוונטיות ליצירת התשובה.", None
+    update_status(f"4. [{generator_name}] מחולל תשובה סופית מ-{len(context_documents)} קטעי הקשר...")
+    start_gen_time = time.time()
+    try:
+        full_response = []
+        error_msg = None
+        generator = openai_service.generate_openai_stream(
+            messages=history, context_documents=context_documents
+        )
+        async for chunk in generator:
+            if isinstance(chunk, str) and chunk.strip().startswith("--- Error:"):
+                if not error_msg:
+                    error_msg = chunk.strip()
+                print(f"OpenAI stream yielded error: {chunk.strip()}")
+                break
+            if isinstance(chunk, str):
+                full_response.append(chunk)
+                stream_callback(chunk)
+        final_response_text = "".join(full_response)
+        gen_time = time.time() - start_gen_time
+        if error_msg:
+            update_status(f"4. שגיאה ביצירת התשובה ({generator_name}) ב-{gen_time:.2f} שניות.")
+            return final_response_text, error_msg
+        update_status(f"4. יצירת התשובה ({generator_name}) הושלמה ב-{gen_time:.2f} שניות.")
+        return final_response_text, None
+    except Exception as gen_err:
+        gen_time = time.time() - start_gen_time
+        error_msg_critical = (f"--- Error: Critical failure during {generator_name} generation "
+                              f"({type(gen_err).__name__}): {gen_err} ---")
+        update_status(f"4. שגיאה קריטית ביצירת התשובה ({generator_name}) ב-{gen_time:.2f} שניות.")
+        traceback.print_exc()
+        return "", error_msg_critical
+
+@traceable(name="rag-execute-validate-generate-gpt4o-pipeline")
+async def execute_validate_generate_pipeline(
+    history: List[Dict], params: Dict[str, Any],
+    status_callback: StatusCallback, stream_callback: Callable[[str], None]
+) -> Dict[str, Any]:
+    result: Dict[str, Any] = {
+        "final_response": "",
+        "validated_documents_full": [],
+        "generator_input_documents": [],
+        "status_log": [],
+        "error": None,
+        "pipeline_used": PIPELINE_VALIDATE_GENERATE_GPT4O
+    }
+    status_log_internal: List[str] = []
+
+    def update_status_and_log(message: str):
+        print(f"Status Update: {message}")
+        status_log_internal.append(message)
+        status_callback(message)
+
+    current_query_text = ""
+    if history and isinstance(history, list):
+        for msg_ in reversed(history):
+            if isinstance(msg_, dict) and msg_.get("role") == "user":
+                current_query_text = str(msg_.get("content") or "")
+                break
+    if not current_query_text:
+        result["error"] = "לא זוהתה שאלה."
+        result["final_response"] = f"<div class='rtl-text'>{result['error']}</div>"
+        result["status_log"] = status_log_internal
+        return result
+
+    try:
+        # 1. Retrieval
+        retrieved_docs = await run_retrieval_step(
+            current_query_text, params['n_retrieve'], update_status_and_log
+        )
+        if not retrieved_docs:
+            result["error"] = "לא אותרו מקורות."
+            result["final_response"] = f"<div class='rtl-text'>{result['error']}</div>"
+            result["status_log"] = status_log_internal
+            return result
+
+        # 2. Validation
+        validated_docs_full = await run_gpt4o_validation_filter_step(
+            retrieved_docs, current_query_text, params['n_validate'], update_status_and_log
+        )
+        result["validated_documents_full"] = validated_docs_full
+        if not validated_docs_full:
+            result["error"] = "לא נמצאו פסקאות רלוונטיות."
+            result["final_response"] = f"<div class='rtl-text'>{result['error']}</div>"
+            update_status_and_log(f"4. {result['error']} לא ניתן להמשיך.")
+            return result
+
+        # --- Simplify Docs for Generation ---
+        simplified_docs_for_generation: List[Dict[str, Any]] = []
+        print(f"Processor: Simplifying {len(validated_docs_full)} docs...")
+        for doc in validated_docs_full:
+            if isinstance(doc, dict):
+                hebrew_text = doc.get('hebrew_text', '')
+                validation = doc.get('validation_result')
+                if hebrew_text:
+                    simplified_doc: Dict[str, Any] = {
+                        'hebrew_text': hebrew_text,
+                        'original_id': doc.get('original_id', 'unknown')
+                    }
+                    if doc.get('source_name'):
+                        simplified_doc['source_name'] = doc.get('source_name')
+                    if validation is not None:
+                        simplified_doc['validation_result'] = validation  # include judgment
+                    simplified_docs_for_generation.append(simplified_doc)
+            else:
+                print(f"Warn: Skipping non-dict item: {doc}")
+        result["generator_input_documents"] = simplified_docs_for_generation
+        print(f"Processor: Created {len(simplified_docs_for_generation)} simplified docs with validation results.")
+
+        # 3. Generation
+        final_response_text, generation_error = await run_openai_generation_step(
+            history=history,
+            context_documents=simplified_docs_for_generation,
+            update_status=update_status_and_log,
+            stream_callback=stream_callback
+        )
+        result["final_response"] = final_response_text
+        result["error"] = generation_error
+
+        if generation_error and not result["final_response"].strip().startswith(("<div", "לא סופקו")):
+            result["final_response"] = (
+                f"<div class='rtl-text'><strong>שגיאה ביצירת התשובה.</strong><br>"
+                f"פרטים: {generation_error}<br>---<br>{result['final_response']}</div>"
+            )
+        elif result["final_response"] == "לא סופקו פסקאות רלוונטיות ליצירת התשובה.":
+            result["final_response"] = f"<div class='rtl-text'>{result['final_response']}</div>"
+
+    except Exception as e:
+        error_type = type(e).__name__
+        error_msg = f"שגיאה קריטית RAG ({error_type}): {e}"
+        print(f"Critical RAG Error: {error_msg}")
+        traceback.print_exc()
+        result["error"] = error_msg
+        result["final_response"] = (
+            f"<div class='rtl-text'><strong>שגיאה קריטית! ({error_type})</strong><br>נסה שוב."
+            f"<details><summary>פרטים</summary><pre>{traceback.format_exc()}</pre></details></div>"
+        )
+        update_status_and_log(f"שגיאה קריטית: {error_type}")
+
+    result["status_log"] = status_log_internal
+    return result

utils.py

@@ -0,0 +1,93 @@
+# utils.py (Updated for OpenAI context formatting)
+import re
+import os
+import time
+import traceback
+import openai
+from typing import Optional, List, Dict
+
+try:
+    import config
+except ImportError:
+    print("Error: config.py not found. Cannot proceed.")
+    raise SystemExit("config.py not found")
+
+# ... (keep openai_client init, clean_source_text, get_embedding) ...
+openai_client = None
+if config.OPENAI_API_KEY:
+    try:
+        openai_client = openai.OpenAI(api_key=config.OPENAI_API_KEY)
+        print("Utils: OpenAI client initialized for embeddings.")
+    except Exception as e:
+        print(f"Utils: Error initializing OpenAI client for embeddings: {e}")
+else:
+    print("Utils: Warning - OPENAI_API_KEY not found. Embeddings will fail.")
+
+def clean_source_text(text: Optional[str]) -> str:
+    if not text: return ""
+    text = text.replace('\x00', '').replace('\ufffd', '')
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text
+
+def get_embedding(text: str, model: str = config.EMBEDDING_MODEL, max_retries: int = 3) -> Optional[List[float]]:
+    global openai_client
+    if not openai_client:
+        print("Error: OpenAI client not initialized (utils.py). Cannot get embedding.")
+        return None
+    if not text or not isinstance(text, str):
+        print("Error: Invalid input text for embedding.")
+        return None
+    cleaned_text = text.replace("\n", " ").strip()
+    if not cleaned_text:
+         print("Warning: Text is empty after cleaning, cannot get embedding.")
+         return None
+    attempt = 0
+    while attempt < max_retries:
+        try:
+            response = openai_client.embeddings.create(input=[cleaned_text], model=model)
+            return response.data[0].embedding
+        except openai.RateLimitError as e:
+            wait_time = (2 ** attempt); print(f"Rate limit embedding. Retrying in {wait_time}s..."); time.sleep(wait_time)
+            attempt += 1
+        except openai.APIConnectionError as e:
+             print(f"Connection error embedding. Retrying..."); time.sleep(2)
+             attempt += 1
+        except Exception as e:
+            print(f"Error generating embedding (Attempt {attempt + 1}/{max_retries}): {type(e).__name__}")
+            attempt += 1
+    print(f"Failed embedding after {max_retries} attempts.")
+    return None
+
+# --- REMOVED format_context_for_anthropic ---
+
+# --- NEW Function to format context for OpenAI ---
+def format_context_for_openai(documents: List[Dict]) -> str:
+    """Formats documents for the OpenAI prompt context section using numbered list."""
+    if not documents:
+        return "No source texts provided."
+    formatted_docs = []
+    language_key = 'hebrew_text'
+    id_key = 'original_id'
+    source_key = 'source_name' # Optional: Include source name if available
+
+    for index, doc in enumerate(documents):
+        if not isinstance(doc, dict):
+            print(f"Warning: Skipping non-dict item in documents list: {doc}")
+            continue
+
+        text = clean_source_text(doc.get(language_key, ''))
+        doc_id = doc.get(id_key, f'unknown_{index+1}')
+        source_name = doc.get(source_key, '') # Get source name
+
+        if text:
+            # Start with 1-based indexing for readability
+            header = f"Source {index + 1} (ID: {doc_id}"
+            if source_name:
+                header += f", SourceName: {source_name}"
+            header += ")"
+            formatted_docs.append(f"{header}:\n{text}\n---") # Add separator
+
+    if not formatted_docs:
+         return "No valid source texts could be formatted."
+
+    return "\n".join(formatted_docs)