The full chain is now a single LCEL chain

app.py

@@ -18,6 +18,7 @@ import pstuts_rag.datastore
 class ApplicationParameters:
     filename = "data/test.json"
     embedding_model = "text-embedding-3-small"
+    n_context_docs = 2
 
 
 class ApplicationState:
@@ -25,7 +26,6 @@ class ApplicationState:
     docs: List[Document] = []
     qdrantclient: QdrantClient = None
     vectorstore: QdrantVectorStore = None
-    n_context_docs = 2
     retriever = None
 
 
@@ -55,15 +55,19 @@ async def on_chat_start():
 
     _ = state.vectorstore.add_documents(documents=state.docs)
     state.retriever = state.vectorstore.as_retriever(
-        search_kwargs={"k": state.n_context_docs}
+        search_kwargs={"k": params.n_context_docs}
     )
 
+    await cl.Message(content=f"Populated vector database.").send()
+
 
 @cl.on_message
 async def main(message: cl.Message):
     # Send a response back to the user
 
-    await cl.Message(content=f"Hello! You said: {message.content}").send()
+    v = await state.retriever.ainvoke(message.content)
+
+    await cl.Message(content=f"Hello! {len(v)}").send()
 
 
 if __name__ == "__main__":

notebooks/transcript_rag.ipynb

@@ -32,6 +32,22 @@
     "%autoreload 2\n"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dataclasses import dataclass\n",
+    "@dataclass\n",
+    "class ApplicationParameters:\n",
+    "    filename = \"data/test.json\"\n",
+    "    embedding_model = \"text-embedding-3-small\"\n",
+    "    n_context_docs = 2\n",
+    "\n",
+    "params = ApplicationParameters()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 4,
@@ -62,7 +78,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -74,7 +90,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -102,7 +118,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 45,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -110,19 +126,17 @@
     "from qdrant_client import QdrantClient\n",
     "from qdrant_client.http.models import Distance, VectorParams\n",
     "\n",
-    "client = QdrantClient(\":memory:\")\n",
+    "qdrantclient = QdrantClient(\":memory:\")\n",
     "\n",
-    "collection_name = f\"{filename}_qdrant\"\n",
-    "\n",
-    "client.create_collection(\n",
-    "    collection_name=collection_name,\n",
-    "    vectors_config=VectorParams(size=1536, distance=Distance.COSINE),\n",
+    "vectorstore = pstuts_rag.datastore.initialize_vectorstore(\n",
+    "    client=qdrantclient,\n",
+    "    collection_name=f\"{params.filename}_qdrant\",\n",
+    "    embeddings=embeddings,\n",
     ")\n",
     "\n",
-    "vector_store = QdrantVectorStore(\n",
-    "    client=client,\n",
-    "    collection_name=collection_name,\n",
-    "    embedding=embeddings,\n",
+    "_ = vectorstore.add_documents(documents=docs_chunks_semantic)\n",
+    "retriever =vectorstore.as_retriever(\n",
+    "    search_kwargs={\"k\": params.n_context_docs}\n",
     ")"
    ]
   },
@@ -131,18 +145,14 @@
    "execution_count": 46,
    "metadata": {},
    "outputs": [],
-   "source": [
-    "_ = vector_store.add_documents(documents=docs_chunks_semantic)"
-   ]
+   "source": []
   },
   {
    "cell_type": "code",
-   "execution_count": 47,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [],
    "source": [
-    "retriever = vector_store.as_retriever(search_kwargs={\"k\":2})\n",
-    "\n",
     "def retrieve(state):\n",
     "    retrieved_docs = retriever.invoke(state[\"question\"])\n",
     "    return {\"context\":retrieved_docs}\n"
@@ -150,9 +160,66 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [],
+   "source": [
+    "from pprint import pp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(\"Layers are the building blocks of any image in Photoshop CC. So, it's \"\n",
+      " \"important to understand, what layers are and why to use them - which we'll \"\n",
+      " \"cover in this video. If you're following along, open this layered image from \"\n",
+      " 'the downloadable practice files for this tutorial. You might think of layers '\n",
+      " 'like separate flat pints of glass, stacked one on top of the other. Each '\n",
+      " 'layer contains separate pieces of content. To get a sense of how layers are '\n",
+      " \"constructed, let's take a look at this Layers panel. I've closed my other \"\n",
+      " 'panels, so that we can focus on the Layers panel. But you can skip that. By '\n",
+      " \"the way: If your Layers panel isn't showing, go up to the Window menu and \"\n",
+      " 'choose Layers from there. The Layers panel is where you go to select and '\n",
+      " 'work with layers. In this image there are 4 layers, each with separate '\n",
+      " 'content. If you click the Eye icon to the left of a layer, you can toggle '\n",
+      " \"the visibility of that layer off and on. So, I'm going to turn off the \"\n",
+      " 'visibility of the tailor layer. And keep your eye on the image, so you can '\n",
+      " \"see what's on that layer.\")\n",
+      "(\"Now let's take a look at just one layer, the tailor layer. A quick way to \"\n",
+      " 'turn off all the layers except the tailor layer, is to hold down the Option '\n",
+      " 'key on the Mac, or the ALT key on the PC, and click on the Eye icon to the '\n",
+      " 'left of the tailor layer. In the Document window, you can see that this '\n",
+      " 'layer contains just the one small photo surrounded by a gray and white '\n",
+      " 'checkerboard pattern. That pattern represents transparent pixels, which '\n",
+      " 'allow us to see down through the corresponding part of this layer to the '\n",
+      " \"content of the layers below. So, let's turn that content back on by going \"\n",
+      " 'back to the Layers panel, again holding the Option key on the Mac or the ALT '\n",
+      " 'key on the PC and clicking on the Eye icon to the left of the tailor layer. '\n",
+      " 'And all the other layers and their Eye icons come back into view. So again: '\n",
+      " 'You might think of layers like a stack of pints of glass, each with its own '\n",
+      " 'artwork and in some cases transparent areas that let you see down through to '\n",
+      " 'the layers below. The biggest benefit of having items on separate layers '\n",
+      " \"like this, is that you'll be able to edit pieces of an image independently \"\n",
+      " 'without affecting the rest of the image.')\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "[None, None]"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "a = retrieve({\"question\":\"What is a layer?\"})\n",
     "[ pp(d.page_content) for d in a[\"context\"] ]"
@@ -169,13 +236,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 49,
+   "execution_count": 85,
    "metadata": {},
    "outputs": [],
    "source": [
     "from langchain.prompts import ChatPromptTemplate\n",
     "\n",
-    "SYSTEM_PROMPT = \"\"\"\\\n",
+    "prompt_template = ChatPromptTemplate.from_messages([\n",
+    "    (\"system\", \"\"\"\\\n",
     "You are a helpful an expert on Photoshop and your goal is to help users\n",
     "gain knowledge from a database of training videos. \n",
     "You answer questions based on provided context. \n",
@@ -191,12 +259,9 @@
     "\"What is ...\"\n",
     "\n",
     "When appropriate, provide your answers in a step-by-step form.\n",
-    "ALWAYS list the URL and the title of the reference video.\n",
     "NEVER invent the explanation. ALWAYS use ONLY the context information.\n",
-    "\n",
-    "\"\"\"\n",
-    "\n",
-    "RAG_PROMPT=\"\"\"\\\n",
+    "\"\"\"),\n",
+    "    (\"user\",\"\"\"\\\n",
     "\n",
     "### Question\n",
     "{question}\n",
@@ -206,14 +271,22 @@
     "### Context\n",
     "{context}\n",
     "\n",
-    "\n",
-    "\"\"\"\n",
-    "\n",
-    "rag_prompt = ChatPromptTemplate(\n",
-    "    [(\"system\",SYSTEM_PROMPT), \n",
-    "     (\"human\",RAG_PROMPT)\n",
-    "     ]\n",
-    "    )"
+    "\"\"\")])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 86,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def compile_references(context):\n",
+    "    references = [ \n",
+    "                    {k: doc.metadata[k] for k in (\"title\",\"source\",\"start\",\"stop\")} \n",
+    "                    for doc in context\n",
+    "    ] \n",
+    "    print(type(references))\n",
+    "    return json.dumps(references,indent=2)\n"
    ]
   },
   {
@@ -227,7 +300,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 50,
+   "execution_count": 87,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -238,26 +311,80 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 51,
+   "execution_count": 96,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<class 'list'>\n"
+     ]
+    }
+   ],
    "source": [
-    "def generate(state):\n",
-    "  docs_content = \"\\n\\n\".join(doc.page_content for doc in state[\"context\"])\n",
+    "from operator import itemgetter\n",
+    "from langchain.schema.output_parser import StrOutputParser\n",
+    "from langchain_core.runnables import RunnableLambda\n",
+    "\n",
+    "form_context = {\n",
+    "    \"context\": itemgetter(\"question\") | retriever, \n",
+    "    \"question\": itemgetter(\"question\") \n",
+    "    } | RunnablePassthrough()\n",
+    "\n",
+    "answer_chain = prompt_template | llm | StrOutputParser()\n",
     "\n",
-    "  references = [ \n",
-    "                {k: doc.metadata[k] for k in (\"title\",\"source\",\"start\",\"stop\")} \n",
-    "                for doc in state[\"context\"] \n",
-    "  ] \n",
+    "get_videos =   form_context | \\\n",
+    "    {\"input\":RunnablePassthrough(),\"answer\": answer_chain} |\\\n",
+    "    RunnableLambda( lambda d: \n",
+    "        {**d[\"input\"], \"answer\": d[\"answer\"] + \n",
+    "         \"\\nReferences:\\n\" +\n",
+    "         compile_references(d[\"input\"][\"context\"]) \n",
+    "        } )\n",
+    "    \n",
     "\n",
     "\n",
-    "  messages = rag_prompt.format_messages(question=state[\"question\"], \n",
-    "                                        context=docs_content)\n",
-    "  response = llm.invoke(messages)\n",
-    "  retval = {\"response\":f\"{response.content}\\n\\n**References**:\\n{json.dumps(references,indent=2)}\",\n",
-    "            \"context\":state[\"context\"]}\n",
-    "  \n",
-    "  return retval\n"
+    "val = get_videos.invoke({\"question\":\"What are layers\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 94,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "dict_keys(['context', 'question', 'answer'])"
+      ]
+     },
+     "execution_count": 94,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "val.keys()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "AIMessage(content='Layers are the building blocks of any image in Photoshop CC. 🖼️ They can be thought of as separate flat pints of glass, stacked one on top of the other. Each layer contains separate pieces of content, and some layers may have transparent areas that let you see through to the layers below. The Layers panel is where you select and work with layers, and you can toggle their visibility by clicking the Eye icon. The main benefit of layers is that they allow you to edit parts of an image independently without affecting the rest. \\n\\n📺 Watch the full explanation in the video titled \"Understand layers\" here: [https://images-tv.adobe.com/avp/vr/b758b4c4-2a74-41f4-8e67-e2f2eab83c6a/f810fc5b-2b04-4e23-8fa4-5c532e7de6f8/e268fe4d-e5c7-415c-9f5c-d34d024b14d8_20170727011753.1280x720at2400_h264.mp4](https://images-tv.adobe.com/avp/vr/b758b4c4-2a74-41f4-8e67-e2f2eab83c6a/f810fc5b-2b04-4e23-8fa4-5c532e7de6f8/e268fe4d-e5c7-415c-9f5c-d34d024b14d8_20170727011753.1280x720at2400_h264.mp4)', additional_kwargs={'refusal': None}, response_metadata={'token_usage': {'completion_tokens': 349, 'prompt_tokens': 1630, 'total_tokens': 1979, 'completion_tokens_details': {'accepted_prediction_tokens': 0, 'audio_tokens': 0, 'reasoning_tokens': 0, 'rejected_prediction_tokens': 0}, 'prompt_tokens_details': {'audio_tokens': 0, 'cached_tokens': 1536}}, 'model_name': 'gpt-4.1-nano-2025-04-14', 'system_fingerprint': 'fp_8fd43718b3', 'finish_reason': 'stop', 'logprobs': None}, id='run--3f91ffdc-d962-40ca-9e06-71040894c9a9-0', usage_metadata={'input_tokens': 1630, 'output_tokens': 349, 'total_tokens': 1979, 'input_token_details': {'audio': 0, 'cache_read': 1536}, 'output_token_details': {'audio': 0, 'reasoning': 0}})"
+      ]
+     },
+     "execution_count": 42,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "value"
    ]
   },
   {