update major frontend and backend features

README.md

@@ -4,7 +4,7 @@ emoji: 🐨
 colorFrom: green
 colorTo: indigo
 sdk: gradio
-sdk_version: 4.36.1
+sdk_version: 4.37.2
 app_file: app.py
 pinned: false
 hf_oauth: true

app.py

@@ -1,15 +1,15 @@
+import gradio as gr
 import time
 import yaml
-import logging
-import gradio as gr
 from langchain.prompts.chat import ChatPromptTemplate
-from huggingface_hub import hf_hub_download, whoami
+from huggingface_hub import hf_hub_download
 from spinoza_project.source.backend.llm_utils import get_llm, get_vectorstore
 from spinoza_project.source.backend.document_store import pickle_to_document_store
 from spinoza_project.source.backend.get_prompts import get_qa_prompts
 from spinoza_project.source.frontend.utils import (
     make_html_source,
     make_html_presse_source,
+    parse_output_llm_with_sources,
     init_env,
 )
 from spinoza_project.source.backend.prompt_utils import (
@@ -17,6 +17,13 @@ from spinoza_project.source.backend.prompt_utils import (
     SpecialTokens,
 )
 
+from assets.utils_javascript import (
+    accordion_trigger,
+    accordion_trigger_end,
+    accordion_trigger_spinoza,
+    accordion_trigger_spinoza_end,
+)
+
 init_env()
 
 with open("./spinoza_project/config.yaml") as f:
@@ -55,18 +62,12 @@ for source, prompt in prompts.items():
     chat_qa_prompt, chat_reformulation_prompt = get_qa_prompts(config, prompt)
     chat_qa_prompts[source] = chat_qa_prompt
     chat_reformulation_prompts[source] = chat_reformulation_prompt
-    # chat_summarize_memory_prompts[source] = chat_summarize_memory_prompt
 
 
 with open("./assets/style.css", "r") as f:
     css = f.read()
 
 
-def update_tabs(outil, visible_tabs):
-    visible_tabs = outil
-    return visible_tabs
-
-
 special_tokens = SpecialTokens(config)
 
 synthesis_template = """You are a factual journalist that summarize the secialized awnsers from thechnical sources.
@@ -117,80 +118,6 @@ def zip_longest_fill(*args, fillvalue=None):
         yield tuple(values)
 
 
-def build_data_dict(config):
-    data_dict = {}
-    for tab in config["tabs"]:
-        data_dict[tab] = {
-            "tab": {
-                "init_value": tab,
-                "component": None,
-                "elem_id": "tab",
-            },
-            "description": {
-                "init_value": config["tabs"][tab],
-                "component": None,
-                "elem_id": "desc",
-            },
-            "question": {
-                "init_value": None,
-                "component": None,
-                "elem_id": "question",
-            },
-            "answer": {
-                "init_value": None,
-                "component": None,
-                "elem_id": "answer",
-            },
-            "sources": {
-                "init_value": None,
-                "component": None,
-                "elem_id": "src",
-            },
-        }
-    return data_dict
-
-
-def init_gradio(data, config=config):
-    for t in data:
-        data[t]["tab"]["component"] = gr.Tab(
-            data[t]["tab"]["init_value"], elem_id="tab"
-        )
-        with data[t]["tab"]["component"]:
-            for fields in data[t]:
-                if fields == "question":
-                    data[t][fields]["component"] = gr.Textbox(
-                        elem_id=data[t][fields]["elem_id"],
-                        show_label=False,
-                        interactive=True,
-                        placeholder="",
-                    )
-                # elif fields == "answer":
-                #     data[t][fields]["component"] = gr.Textbox(
-                #         elem_id=data[t][fields]["elem_id"],
-                #         show_label=True,
-                #         interactive=True,
-                #         placeholder="",
-                #         show_copy_button=True
-                #     )
-                elif fields != "tab":
-                    data[t][fields]["component"] = gr.Markdown(
-                        data[t][fields]["init_value"],
-                        elem_id=data[t][fields]["elem_id"],
-                    )
-                    # data[t][fields]["component"] = gr.Textbox(
-                    #     value=data[t][fields]["init_value"],
-                    #     elem_id=data[t][fields]["elem_id"],
-                    #     show_label=True,
-                    #     interactive=False,
-                    #     show_copy_button=True,
-                    # )
-    return data
-
-
-def add_warning():
-    return "*Les éléments cochés ont commencé à être généré dans les onglets spécifiques, la synthèse ne sera disponible qu'après la mise à disposition de ces derniers.*"
-
-
 def format_question(question):
     return f"{question}"  # ###
 
@@ -202,8 +129,8 @@ def parse_question(question):
     return x
 
 
-def reformulate(outils, question, tab, config=config):
-    if tab in outils:
+def reformulate(question, tab, config=config):
+    if tab in list(config["tabs"].keys()):
         return llm.stream(
             chat_reformulation_prompts[config["source_mapping"][tab]],
             {"question": parse_question(question)},
@@ -212,15 +139,15 @@ def reformulate(outils, question, tab, config=config):
         return iter([None] * 5)
 
 
-def reformulate_single_question(outils, question, tab, config=config):
-    for elt in reformulate(outils, question, tab, config=config):
+def reformulate_single_question(question, tab, config=config):
+    for elt in reformulate(question, tab, config=config):
         time.sleep(0.02)
         yield elt
 
 
-def reformulate_questions(outils, question, config=config):
+def reformulate_questions(question, config=config):
     for elt in zip_longest_fill(
-        *[reformulate(outils, question, tab, config=config) for tab in config["tabs"]]
+        *[reformulate(question, tab, config=config) for tab in config["tabs"]]
     ):
         time.sleep(0.02)
         yield elt
@@ -230,8 +157,8 @@ def add_question(question):
     return question
 
 
-def answer(question, source, outils, tab, config=config):
-    if tab in outils:
+def answer(question, source, tab, config=config):
+    if tab in list(config["tabs"].keys()):
         if len(source) < 10:
             return iter(["Aucune source trouvée, veuillez reformuler votre question"])
         else:
@@ -247,49 +174,35 @@ def answer(question, source, outils, tab, config=config):
         return iter([None] * 5)
 
 
-def answer_single_question(outils, source, question, tab, config=config):
-    for elt in answer(question, source, outils, tab, config=config):
+def answer_single_question(source, question, tab, config=config):
+    for elt in answer(question, source, tab, config=config):
         time.sleep(0.02)
         yield elt
 
 
-def answer_questions(outils, *questions_sources, config=config):
-
+def answer_questions(*questions_sources, config=config):
     questions = [elt for elt in questions_sources[: len(questions_sources) // 2]]
     sources = [elt for elt in questions_sources[len(questions_sources) // 2 :]]
 
     for elt in zip_longest_fill(
         *[
-            answer(question, source, outils, tab, config=config)
+            answer(question, source, tab, config=config)
             for question, source, tab in zip(questions, sources, config["tabs"])
         ]
     ):
         time.sleep(0.02)
-        yield elt
-
-
-def get_source_link(metadata):
-    return metadata["file_url"] + f"#page={metadata['content_page_number'] + 1}"
-
-
-def get_button(i, tag):
-    return f"""<button id="btn_{tag}_{i}" type="button" style="margin: 0; display: inline; align="right">[{i}]</button>"""
-
-
-def get_html_sources(buttons, cards):
-    return f"""
-<p style="margin: 0; display: inline;"><strong><br>Sources utilisées : </strong></p>
-{buttons}
-{cards}
-"""
+        yield [
+            [(question, parse_output_llm_with_sources(ans))]
+            for question, ans in zip(questions, elt)
+        ]
 
 
-def get_sources(
-    outils, question, tab, qdrants=qdrants, bdd_presse=bdd_presse, config=config
-):
+def get_sources(questions, qdrants=qdrants, bdd_presse=bdd_presse, config=config):
     k = config["num_document_retrieved"]
     min_similarity = config["min_similarity"]
-    if tab in outils:
+    formated = []
+    text = []
+    for i, (question, tab) in enumerate(zip(questions, list(config["tabs"].keys()))):
         sources = (
             (
                 bdd_presse.similarity_search_with_relevance_scores(
@@ -307,66 +220,49 @@ def get_sources(
             )
         )
 
-        sources = [(doc, score) for doc, score in sources if score >= min_similarity]
-
-        buttons_ids = list(range(len(sources)))
-        buttons = " ".join(
-            [get_button(i, tab) for i, source in zip(buttons_ids, sources)]
-        )
-        formated = (
-            "\n\n".join(
-                [
-                    make_html_presse_source(source[0], i, tab, source[1], config)
-                    for i, source in zip(buttons_ids, sources)
-                ]
-            )
+        sources = [
+            (doc, score) for doc, score in sources
+        ]  # if score >= min_similarity]
+        formated.extend(
+            [
+                make_html_presse_source(source[0], j, source[1], config)
+                for j, source in zip(range(k * i + 1, k * (i + 1) + 1), sources)
+            ]
             if tab == "Presse"
-            else "\n\n".join(
-                [
-                    make_html_source(source[0], i, tab, source[1], config)
-                    for i, source in zip(buttons_ids, sources)
-                ]
-            )
+            else [
+                make_html_source(source[0], j, source[1], config)
+                for j, source in zip(range(k * i + 1, k * (i + 1) + 1), sources)
+            ]
         )
-        formated = get_html_sources(buttons, formated) if sources else ""
-        text = "\n\n".join(
+        text.extend(
             [
-                f"Doc {str(i)} with source type {elt[0].metadata.get('file_source_type')}:\n"
-                + elt[0].page_content
-                for i, elt in enumerate(sources)
+                "\n\n".join(
+                    [
+                        f"Doc {str(j)} with source type {source[0].metadata.get('file_source_type')}:\n"
+                        + source[0].page_content
+                        for j, source in zip(range(k * i + 1, k * (i + 1) + 1), sources)
+                    ]
+                )
             ]
         )
-        return str(formated), str(text)  # formated_sources, text_sources
-    else:
-        return "", ""
+
+    formated = "".join(formated)
+
+    return formated, text
 
 
 def retrieve_sources(
-    outils, *questions, qdrants=qdrants, bdd_presse=bdd_presse, config=config
+    *questions, qdrants=qdrants, bdd_presse=bdd_presse, config=config
 ):
-    results = [
-        get_sources(outils, question, tab, qdrants, bdd_presse, config)
-        for question, tab in zip(questions, config["tabs"])
-    ]
-    formated_sources = [source[0] for source in results]
-    text_sources = [source[1] for source in results]
-    return tuple(formated_sources + text_sources)
-
+    formated_sources, text_sources = get_sources(questions, qdrants, bdd_presse, config)
 
-def get_experts(outils, *answers, config=config):
-    return "\n\n".join(
-        [
-            f"{tab}\n{answers[i]}"
-            for i, tab in enumerate(config["tabs"])
-            if (tab in outils)
-        ]
-    )
+    return (formated_sources, *text_sources)
 
 
-def get_synthesis(outils, question, *answers, config=config):
+def get_synthesis(question, *answers, config=config):
     answer = []
     for i, tab in enumerate(config["tabs"]):
-        if (tab in outils) & (len(str(answers[i])) >= 100):
+        if (len(str(answers[i])) >= 100):
             answer.append(
                 f"{tab}\n{answers[i]}".replace("<p>", "").replace("</p>\n", "")
             )
@@ -382,67 +278,7 @@ def get_synthesis(outils, question, *answers, config=config):
             },
         ):
             time.sleep(0.01)
-            yield elt
-
-
-def get_listener():
-    return """
-    function my_func_body() {
-        const body = document.querySelector("body");
-        body.addEventListener("click", e => {
-            console.log(e)
-            const sourceId = "btn_" + e.target.id.split("_")[1] + "_" + e.target.id.split("_")[2] + "_source"
-            console.log(sourceId)
-                if (document.getElementById(sourceId).style.display === "none") {
-                document.getElementById(sourceId).style.display = "";
-            } else {
-                document.getElementById(sourceId).style.display = "none";
-            }
-        }
-    )}
-    """
-
-
-def get_source_template(buttons, divs_source):
-    return """
-    <div class="source">
-        <p style="margin: 0; display: inline;"><strong><br>Sources utilisées :</strong></p>
-        {buttons}
-        {divs_source}
-        </div>
-    </div>
-    """
-
-
-def activate_questions(outils, *textboxes, config=config):
-    activated_textboxes = []
-    for i, tab in enumerate(config["tabs"]):
-        if tab in outils:
-            activated_textboxes.append(
-                gr.Textbox(
-                    show_label=False,
-                    interactive=True,
-                    placeholder="Sélectionnez cet outil et posez une question sur l'onglet de synthèse",
-                )
-            )
-
-        else:
-            activated_textboxes.append(
-                gr.Textbox(
-                    show_label=False,
-                    interactive=False,
-                    placeholder="Sélectionnez cet outil et posez une question sur l'onglet de synthèse",
-                )
-            )
-    return activated_textboxes
-
-
-def empty():
-    return ""
-
-
-def empty_none():
-    return None
+            yield [(question, elt)]
 
 
 theme = gr.themes.Base(
@@ -451,151 +287,219 @@ theme = gr.themes.Base(
     font=[gr.themes.GoogleFont("Poppins"), "ui-sans-serif", "system-ui", "sans-serif"],
 )
 
+with open("./assets/style.css", "r") as f:
+    css = f.read()
 
-init_prompt = """
-Hello, I am Spinoza Q&A, a conversational assistant designed to help journalists by providing secialized answers from technical sources. I will answer your questions based **on the official definition of each ESRS as well as guidelines**.
-
-⚠️ Limitations
-*Please note that this chatbot is in an early stage phase, it is not perfect and may sometimes give irrelevant answers. If you are not satisfied with the answer, please ask a more specific question or report your feedback to help us improve the system.*
-
-What do you want to learn ?
-"""
+with open("./assets/source_information.md", "r") as f:
+    source_information = f.read()
 
-logo_rsf = config["logo_rsf"]
-logo_ap = config["logo_ap"]
+def start_agents():
+    gr.Info(message="The agents and Spinoza are loading...", duration=3)
 
-data = build_data_dict(config)
 
+def end_agents():
+    gr.Info(
+        message="The agents and Spinoza have finished answering your question",
+        duration=3,
+    )
 
-def update_visible(oauth_token: gr.OAuthToken | None):
-    if oauth_token is None:
-        return {
-            bloc_1: gr.update(visible=True),
-            bloc_2: gr.update(visible=False),
-            bloc_3: gr.update(visible=False),
-        }
 
-    org_names = [org["name"] for org in whoami(oauth_token.token)["orgs"]]
+def next_call():
+    print("Next call")
 
-    if "SpinozaProject" in org_names:  # logged in group
-        return {
-            bloc_1: gr.update(visible=False),
-            bloc_2: gr.update(visible=True),
-            bloc_3: gr.update(visible=False),
-        }
+init_prompt = """
+Hello, I am Spinoza, a conversational assistant designed to help you in your journalistic journey. I will answer your questions based **on the provided sources**.
 
-    else:  # logged but not in group
-        return {
-            bloc_1: gr.update(visible=False),
-            bloc_2: gr.update(visible=False),
-            bloc_3: gr.update(visible=True),
-        }
+⚠️ Limitations
+*Please note that this chatbot is in an early stage phase, it is not perfect and may sometimes give irrelevant answers. If you are not satisfied with the answer, please ask a more specific question or report your feedback to help us improve the system.*
 
+What do you want to learn ?
+"""
 
 with gr.Blocks(
-    title=f"🔍{config['demo_name']}",
+    title=f"🔍 Spinoza",
     css=css,
-    js=get_listener(),
     theme=theme,
 ) as demo:
-    with gr.Column(visible=True):
-        gr.HTML(
-            f"""<div class="row_logo">
-                        <img src={logo_rsf} alt="logo RSF" style="float:left; width:120px; height:70px">
-                        <img src={logo_ap} alt="logo AP" style="width:120px; height:70px">
-                </div>"""
-        )
+    chatbots = {}
+    question = gr.State("")
+    docs_textbox = gr.State([""])
+    agent_questions = {elt: gr.State("") for elt in config["tabs"]}
+    component_sources = {elt: gr.State("") for elt in config["tabs"]}
+    text_sources = {elt: gr.State("") for elt in config["tabs"]}
+    tab_states = {elt: gr.State(elt) for elt in config["tabs"]}
+    chatbot_states = [
+        gr.State(name)
+        for name in ["science", "presse", "politique", "legal", "spinoza"]
+    ]
 
-        text_sources = {elt: gr.State("") for elt in config["tabs"]}
-        tab_states = {elt: gr.State(elt) for elt in config["tabs"]}
+    with gr.Tab("Q&A", elem_id="main-component"):
+        with gr.Row(elem_id="chatbot-row"):
+            with gr.Column(scale=2, elem_id="center-panel"):
+                with gr.Group(elem_id="chatbot-group"):
+                    with gr.Accordion(
+                        "Science agent",
+                        open=False,
+                        elem_id="accordion-science",
+                        elem_classes="accordion",
+                    ):
+                        chatbots[list(config["tabs"].keys())[0]] = gr.Chatbot(
+                            show_copy_button=True,
+                            show_share_button=False,
+                            show_label=False,
+                            elem_id="chatbot-science",
+                            layout="panel",
+                            avatar_images=(
+                                "./assets/logos/help.png",
+                                None,
+                            ),
+                        )
+
+                    with gr.Accordion(
+                        "Law agent",
+                        open=False,
+                        elem_id="accordion-legal",
+                        elem_classes="accordion",
+                    ):
+                        chatbots[list(config["tabs"].keys())[1]] = gr.Chatbot(
+                            show_copy_button=True,
+                            show_share_button=False,
+                            show_label=False,
+                            elem_id="chatbot-legal",
+                            layout="panel",
+                            avatar_images=(
+                                "./assets/logos/help.png",
+                                None,
+                            ),
+                        )
+
+                    with gr.Accordion(
+                        "Politics agent",
+                        open=False,
+                        elem_id="accordion-politique",
+                        elem_classes="accordion",
+                    ):
+                        chatbots[list(config["tabs"].keys())[2]] = gr.Chatbot(
+                            show_copy_button=True,
+                            show_share_button=False,
+                            show_label=False,
+                            elem_id="chatbot-politique",
+                            layout="panel",
+                            avatar_images=(
+                                "./assets/logos/help.png",
+                                None,  # "https://i.ibb.co/cN0czLp/celsius-logo.png",
+                            ),
+                        )
+
+                    with gr.Accordion(
+                        "ADEME agent",
+                        open=False,
+                        elem_id="accordion-ademe",
+                        elem_classes="accordion",
+                    ):
+                        chatbots[list(config["tabs"].keys())[3]] = gr.Chatbot(
+                            show_copy_button=True,
+                            show_share_button=False,
+                            show_label=False,
+                            elem_id="chatbot-ademe",
+                            layout="panel",
+                            avatar_images=(
+                                "./assets/logos/help.png",
+                                None,  # "https://i.ibb.co/cN0czLp/celsius-logo.png",
+                            ),
+                        )
+
+                    with gr.Accordion(
+                        "Press agent",
+                        open=False,
+                        elem_id="accordion-presse",
+                        elem_classes="accordion",
+                    ):
+                        chatbots[list(config["tabs"].keys())[4]] = gr.Chatbot(
+                            show_copy_button=True,
+                            show_share_button=False,
+                            show_label=False,
+                            elem_id="chatbot-presse",
+                            layout="panel",
+                            avatar_images=(
+                                "./assets/logos/help.png",
+                                None,  # "https://i.ibb.co/cN0czLp/celsius-logo.png",
+                            ),
+                        )
+
+                    with gr.Accordion(
+                        "Spinoza",
+                        open=True,
+                        elem_id="accordion-spinoza",
+                        elem_classes="accordion",
+                    ):
+                        chatbots["spinoza"] = gr.Chatbot(
+                            value=[(None, init_prompt)],
+                            show_copy_button=True,
+                            show_share_button=False,
+                            show_label=False,
+                            elem_id="chatbot-spinoza",
+                            layout="panel",
+                            avatar_images=(
+                                "./assets/logos/help.png",
+                                "./assets/logos/spinoza.png",
+                            ),
+                        )
+
+                with gr.Row(elem_id="input-message"):
+                    ask = gr.Textbox(
+                        placeholder="Ask me anything here!",
+                        show_label=False,
+                        scale=7,
+                        lines=1,
+                        interactive=True,
+                        elem_id="input-textbox",
+                    )
+
+            with gr.Column(scale=1, variant="panel", elem_id="right-panel"):
+                with gr.TabItem("Sources", elem_id="tab-sources", id=0):
+                    sources_textbox = gr.HTML(
+                        show_label=False, elem_id="sources-textbox"
+                    )
+
+    with gr.Tab("Source information", elem_id="source-component"):
         with gr.Row():
-            with gr.Column(scale=3):
-                outils = gr.CheckboxGroup(
-                    choices=list(config["tabs"].keys()),
-                    value=list(config["tabs"].keys()),
-                    type="value",
-                    label="Choisir les bases de données à interroger",
-                )
             with gr.Column(scale=1):
-                submit_btn = gr.Button(
-                    "Relancer la Synthèse", variant="primary", elem_id="synthese_btn"
-                )
+                gr.Markdown(source_information)
 
-        # Synthesis tab
-        synthesis_tab = gr.Tab("Synthesis", elem_id="tab")
-        with synthesis_tab:
-            question = gr.Textbox(
-                show_label=True,
-                label="Posez une question à Spinoza",
-                placeholder="Quelle est votre question ?",
-            )
-            md_question = gr.Markdown(None, visible=False)
-            warning = gr.Markdown(None, elem_id="warn")
-            synthesis = gr.Markdown(None, elem_id="synthesis")
-
-        data = init_gradio(data)
-        (
-            question.submit(add_question, [question], [md_question])
-            .then(add_warning, [], [warning])
-            .then(empty, [], [synthesis])
-            .then(
-                reformulate_questions,
-                [outils, md_question],
-                [data[tab]["question"]["component"] for tab in config["tabs"]],
-            )
-            .then(
-                retrieve_sources,
-                [outils]
-                + [data[tab]["question"]["component"] for tab in config["tabs"]],
-                [data[tab]["sources"]["component"] for tab in config["tabs"]]
-                + [text_sources[tab] for tab in config["tabs"]],
-            )
-            .then(
-                answer_questions,
-                [outils]
-                + [data[tab]["question"]["component"] for tab in config["tabs"]]
-                + [text_sources[tab] for tab in config["tabs"]],
-                [data[tab]["answer"]["component"] for tab in config["tabs"]],
-            )
-            .then(
-                get_synthesis,
-                [outils, md_question]
-                + [data[tab]["answer"]["component"] for tab in config["tabs"]],
-                [synthesis],
-            )
-        )
-
-        for tab in config["tabs"]:
-            (
-                data[tab]["question"]["component"]
-                .submit(empty, [], [data[tab]["sources"]["component"]])
-                .then(empty, [], [text_sources[tab]])
-                .then(empty, [], [data[tab]["answer"]["component"]])
-                .then(
-                    get_sources,
-                    [outils, data[tab]["question"]["component"], tab_states[tab]],
-                    [data[tab]["sources"]["component"], text_sources[tab]],
-                )
-                .then(
-                    answer_single_question,
-                    [
-                        outils,
-                        text_sources[tab],
-                        data[tab]["question"]["component"],
-                        tab_states[tab],
-                    ],
-                    [data[tab]["answer"]["component"]],
-                )
-            )
+    with gr.Tab("Contact", elem_id="contact-component"):
+        with gr.Row():
+            with gr.Column(scale=1):
+                gr.Markdown("For any issue contact **spinoza.[email protected]**.")
+
+    ask.submit(start_agents, inputs=[], outputs=[], js=accordion_trigger()).then(
+        fn=reformulate_questions,
+        inputs=[ask],
+        outputs=[agent_questions[tab] for tab in config["tabs"]],
+    ).then(
+        fn=retrieve_sources,
+        inputs=[agent_questions[tab] for tab in config["tabs"]],
+        outputs=[sources_textbox] + [text_sources[tab] for tab in config["tabs"]],
+    ).then(
+        fn=answer_questions,
+        inputs=[agent_questions[tab] for tab in config["tabs"]]
+        + [text_sources[tab] for tab in config["tabs"]],
+        outputs=[chatbots[tab] for tab in config["tabs"]],
+    ).then(
+        fn=next_call, inputs=[], outputs=[], js=accordion_trigger_end()
+    ).then(
+        fn=next_call, inputs=[], outputs=[], js=accordion_trigger_spinoza()
+    ).then(
+        fn=get_synthesis,
+        inputs=[agent_questions[list(config["tabs"].keys())[1]]]
+        + [chatbots[tab] for tab in config["tabs"]],
+        outputs=[chatbots["spinoza"]],
+    ).then(
+        fn=next_call, inputs=[], outputs=[], js=accordion_trigger_spinoza_end()
+    ).then(
+        fn=end_agents, inputs=[], outputs=[]
+    )
 
-        (
-            submit_btn.click(empty, [], [synthesis]).then(
-                get_synthesis,
-                [outils, md_question]
-                + [data[tab]["answer"]["component"] for tab in config["tabs"]],
-                [synthesis],
-            )
-        )
 
 if __name__ == "__main__":
     demo.queue().launch(share=True, debug=True)

assets/source_information.md

@@ -0,0 +1,14 @@
+Here is a quick introduction to the sources of data accessed by the different agents.
+    
+1. **Science:** this tool is composed if IPCC and IPBES reports.  
+
+2. **Legal:** this tool is based on french law, it gathers 21 of the "codes" that were modified by "la loi climat" of 2021.  
+
+3. **Politic:** this tool is questioning the national french low carbon policy : SNBC.  
+
+4. **ADEME:** this tool is dedicated to ADEME data and we have selected various categories of reports :  
+
+    - Guides made available to the general public 
+    - Reports on experiences with new technologies 
+    - Studies and research on local impacts institutional documents (analyses requested by France & activity reports) 
+    - Sectoral transition plans for the industrial sectors with the highest emissions (glass, paper, cement, steel, aluminium, chemicals, sugar)

assets/style.css

@@ -1,6 +1,6 @@
-:root {
+/* :root {
     --user-image: url('https://ih1.redbubble.net/image.4776899543.6215/st,small,507x507-pad,600x600,f8f8f8.jpg');
-  }
+} */
 
 .warning-box {
     background-color: #fff3cd;
@@ -11,21 +11,32 @@
     color: #856404;
     display: inline-block;
     margin-bottom: 15px;
-  }
+}
+
 
 .tip-box {
     background-color: #f0f9ff;
     border: 1px solid #80d4fa;
     border-radius: 4px;
-    margin-top:20px;
+    margin-top: 20px;
     padding: 15px 20px;
     font-size: 14px;
-    color: #006064;
     display: inline-block;
     margin-bottom: 15px;
     width: auto;
+    color: black !important;
+}
+
+body.dark .warning-box * {
+    color: black !important;
+}
+
+
+body.dark .tip-box * {
+    color: black !important;
 }
 
+
 .tip-box-title {
     font-weight: bold;
     font-size: 14px;
@@ -37,14 +48,16 @@
     margin-right: 5px;
 }
 
-.gr-box {border-color: #d6c37c}
+.gr-box {
+    border-color: #d6c37c
+}
 
-#hidden-message{
-    display:none;
+#hidden-message {
+    display: none;
 }
 
-.message{
-    font-size:14px !important;
+.message {
+    font-size: 14px !important;
 }
 
 
@@ -60,7 +73,7 @@ a {
     overflow: hidden;
     display: flex;
     flex-direction: column;
-    margin:20px;
+    margin: 20px;
 }
 
 .card-content {
@@ -71,8 +84,9 @@ a {
     font-size: 14px !important;
     font-weight: bold;
     margin-bottom: 10px;
-    margin-top:0px !important;
-    color:#577b9b!important;;
+    margin-top: 0px !important;
+    color: #dc2626 !important;
+    ;
 }
 
 .card-content p {
@@ -80,16 +94,6 @@ a {
     margin-bottom: 0;
 }
 
-.card-content-column-1 {
-  float: left;
-  width: 20%;
-}
-
-.card-content-column-2 {
-  float: left;
-  width: 80%;
-}
-
 .card-footer {
     background-color: #f4f4f4;
     font-size: 10px;
@@ -109,55 +113,51 @@ a {
     display: inline-flex;
     align-items: center;
     margin-left: auto;
-    text-decoration: none!important;
+    text-decoration: none !important;
     font-size: 14px;
 }
 
-.message.user{
-    background-color:#7494b0 !important;
-    border:none;
-    color:white!important;
+
+
+.message.user {
+    /* background-color:#7494b0 !important; */
+    border: none;
+    /* color:white!important; */
 }
 
-.message.bot{
-    background-color:#f2f2f7 !important;
-    border:none;
+.message.bot {
+    /* background-color:#f2f2f7 !important; */
+    border: none;
 }
 
-.gallery-item > div:hover{
+/* .gallery-item > div:hover{
     background-color:#7494b0 !important;
     color:white!important;
 }
-
 .gallery-item:hover{
     border:#7494b0 !important;
 }
-
 .gallery-item > div{
     background-color:white !important;
     color:#577b9b!important;
 }
-
 .label{
     color:#577b9b!important;
-}
+} */
 
-label.selected{
-    background:none !important;
-}
-
-.paginate{
+/* .paginate{
     color:#577b9b!important;
-}
+} */
 
 
-label > span{
-    background-color:white !important;
-    color:#577b9b!important;
-}
+
+/* span[data-testid="block-info"]{
+    background:none !important;
+    color:#577b9b;
+  } */
 
 /* Pseudo-element for the circularly cropped picture */
-.message.bot::before {
+/* .message.bot::before {
     content: '';
     position: absolute;
     top: -10px;
@@ -169,4 +169,269 @@ label > span{
     background-position: center;
     border-radius: 50%;
     z-index: 10;
-  }
+  }
+   */
+
+label.selected {
+    background: none !important;
+}
+
+#submit-button {
+    padding: 0px !important;
+}
+
+
+@media screen and (min-width: 1024px) {
+    div#sources-textbox {
+        height: calc(100vh - 190px) !important;
+        overflow-y: auto !important;
+        scrollbar-width: none;
+        -ms-overflow-style: none;
+    }
+
+    div#sources-textbox::-webkit-scrollbar {
+        width: 0;
+        height: 0;
+    }
+
+    div.svelte-iyf88w {
+        scrollbar-width: none;
+    }
+
+    div.svelte-iyf88w::-webkit-scrollbar {
+        width: 0;
+        height: 0;
+    }
+
+    div#chatbot-row {
+        height: calc(100vh - 90px) !important;
+    }
+
+    .max-height {
+        height: calc(100vh - 90px) !important;
+        overflow-y: auto;
+    }
+
+    div.svelte-iyf88w {
+        height: calc(100vh - 160px) !important;
+        overflow-y: auto;
+    }
+
+    #accordion-spinoza {
+        height: calc(100vh - 160px) !important;
+    }
+
+    .form {
+        position: relative;
+        top: 10px;
+    }
+
+    #accordion-spinoza>open>span:nth-child(1) {
+        color: #000000;
+        font-size: large;
+        font-weight: bold;
+    }
+
+    #accordion-spinoza>button:nth-child(2)>span:nth-child(1) {
+        color: #000000;
+        font-size: large;
+        font-weight: bold;
+    }
+
+    #accordion-science>button:nth-child(2)>span:nth-child(1) {
+        color: #9ca1a5e7;
+        font-weight: bold;
+    }
+
+    #accordion-presse>button:nth-child(2)>span:nth-child(1) {
+        color: #9ca1a5e7;
+        font-weight: bold;
+    }
+
+    #accordion-legal>button:nth-child(2)>span:nth-child(1) {
+        color: #9ca1a5e7;
+        font-weight: bold;
+    }
+
+    #accordion-politique>button:nth-child(2)>span:nth-child(1) {
+        color: #9ca1a5e7;
+        font-weight: bold;
+    }
+
+    #accordion-ademe>button:nth-child(2)>span:nth-child(1) {
+        color: #9ca1a5e7;
+        font-weight: bold;
+    }
+
+}
+
+footer {
+    visibility: hidden;
+    display: none !important;
+}
+
+/* @media screen and (max-width: 767px) {
+    /* Your mobile-specific styles go here */
+
+div#chatbot {
+    height: 500px !important;
+}
+
+#submit-button {
+    padding: 0px !important;
+    min-width: 80px;
+}
+
+/* This will hide all list items */
+div.tab-nav button {
+    display: none !important;
+}
+
+/* This will show only the first list item */
+div.tab-nav button:first-child {
+    display: block !important;
+}
+
+/* This will show only the first list item */
+div.tab-nav button:nth-child(2) {
+    display: block !important;
+}
+
+/* This will show only the first list item */
+div.tab-nav button:nth-child(3) {
+    display: block !important;
+}
+
+#right-panel button {
+    display: block !important;
+}
+
+/* ... add other mobile-specific styles ... */
+
+*/ body.dark .card {
+    background-color: #374151;
+}
+
+body.dark .card-content h2 {
+    color: #f4dbd3 !important;
+}
+
+body.dark .card-footer {
+    background-color: #404652;
+}
+
+body.dark .card-footer span {
+    color: white !important;
+}
+
+
+.doc-ref {
+    color: #dc2626 !important;
+    margin-right: 1px;
+}
+
+.tabitem {
+    border: none !important;
+}
+
+.other-tabs>div {
+    padding-left: 40px;
+    padding-right: 40px;
+    padding-top: 10px;
+}
+
+.gallery-item>div {
+    white-space: normal !important;
+    /* Allow the text to wrap */
+    word-break: break-word !important;
+    /* Break words to prevent overflow */
+    overflow-wrap: break-word !important;
+    /* Break long words if necessary */
+}
+
+span.chatbot>p>img {
+    margin-top: 40px !important;
+    max-height: none !important;
+    max-width: 80% !important;
+    border-radius: 0px !important;
+}
+
+
+.chatbot-caption {
+    font-size: 11px;
+    font-style: italic;
+    color: #508094;
+}
+
+.ai-generated {
+    font-size: 11px !important;
+    font-style: italic;
+    color: #73b8d4 !important;
+}
+
+.card-image>.card-content {
+    background-color: #f1f7fa !important;
+}
+
+
+
+.tab-nav>button.selected {
+    color: #4b8ec3;
+    font-weight: bold;
+    border: none;
+}
+
+.tab-nav {
+    border: none !important;
+}
+
+#input-textbox>label>textarea {
+    border-radius: 40px;
+    padding-left: 30px;
+    resize: none;
+}
+
+#input-message>div {
+    border: none;
+}
+
+#dropdown-samples {
+    /*! border:none !important; */
+    /*! border-width:0px !important; */
+    background: none !important;
+
+}
+
+#dropdown-samples>.container>.wrap {
+    background-color: white;
+}
+
+/* HTML: <div class="loader"></div> */
+/* .loader {
+    display: inline-flex;
+    gap: 5px;
+    animation: l2-0 1s infinite;
+  }
+  .loader:before,
+  .loader:after {
+    content: "";
+    width: 25px;
+    aspect-ratio: 1;
+    box-shadow: 0 0 0 3px inset #fff;
+    animation: l2-1 1s infinite;
+  }
+  .loader:after {
+    --s: -1;
+  }
+  @keyframes l2-0 {
+    0%,
+    50% {transform:rotate(0deg)}
+    80%,
+    100% {transform:rotate(180deg)}
+  }
+  @keyframes l2-1 {
+    0%   {transform:translate(0)}
+    50%,
+    80%  {transform:translate(calc(var(--s,1)*2.5px))}
+    100% {transform:translate(0)}
+  } */

assets/utils_javascript.py

@@ -0,0 +1,108 @@
+def accordion_trigger():
+    return """
+    function accordion_trigger() {
+        input_textbox = document.getElementById("input-textbox")
+        input_textbox.addEventListener('keyup', function (e) {
+            if (e.key === 'Enter' || e.keyCode === 13) {
+                var accordion_science = document.getElementById("accordion-science")
+                var accordion_presse = document.getElementById("accordion-presse")
+                var accordion_politique = document.getElementById("accordion-politique")
+                var accordion_legal = document.getElementById("accordion-legal")
+                var accordion_ademe= document.getElementById("accordion-ademe")
+                accordion_science.children[1].children[0].textContent = "Science agent - loading";
+                accordion_science.children[1].children[1].classList.add('loader');
+                accordion_presse.children[1].children[0].textContent = "Press agent - loading";
+                accordion_presse.children[1].children[0].classList.add('loader');
+                accordion_politique.children[1].children[0].textContent = "Politics agent - loading";
+                accordion_politique.children[1].children[0].classList.add('loader');
+                accordion_legal.children[1].children[0].textContent = "Law agent - loading";
+                accordion_legal.children[1].children[0].classList.add('loader');
+                accordion_ademe.children[1].children[0].textContent = "ADEME agent - loading";
+                accordion_ademe.children[1].children[0].classList.add('loader');
+            }
+        });
+    }
+    """
+
+
+def accordion_trigger_end():
+    return """
+    function accordion_trigger_end() {
+        var accordion_science = document.getElementById("accordion-science")
+        var accordion_presse = document.getElementById("accordion-presse")
+        var accordion_politique = document.getElementById("accordion-politique")
+        var accordion_legal = document.getElementById("accordion-legal")
+        var accordion_ademe = document.getElementById("accordion-ademe")
+        accordion_science.children[1].children[0].textContent = "Science agent - ready";
+        accordion_science.children[1].children[1].classList.remove('loader');
+        accordion_presse.children[1].children[0].textContent = "Press agent - ready";
+        accordion_presse.children[1].children[0].classList.remove('loader');
+        accordion_politique.children[1].children[0].textContent = "Politics agent - ready";
+        accordion_politique.children[1].children[0].classList.remove('loader');
+        accordion_legal.children[1].children[0].textContent = "Law agent - ready";
+        accordion_legal.children[1].children[0].classList.remove('loader');
+        accordion_ademe.children[1].children[0].textContent = "ADEME agent - ready";
+        accordion_ademe.children[1].children[0].classList.remove('loader');
+
+    }
+    """
+
+
+def accordion_trigger_spinoza():
+    return """
+    function accordion_trigger_spinoza() {
+        var accordion_spinoza = document.getElementById("accordion-spinoza")
+        accordion_spinoza.children[1].children[0].textContent = "Spinoza - generating";
+        accordion_spinoza.children[1].children[0].classList.add('loader');
+    }
+    """
+
+
+def accordion_trigger_spinoza_end():
+    return """
+    function accordion_trigger_spinoza_end() {
+        var accordion_spinoza = document.getElementById("accordion-spinoza")
+        accordion_spinoza.children[1].children[0].textContent = "Spinoza - ready";
+        accordion_spinoza.children[1].children[0].classList.remove('loader');
+    }
+    """
+
+
+def accordion_trigger_end_science():
+    return """
+    function accordion_trigger_end() {
+        var accordion_science = document.getElementById("accordion-science")
+        accordion_science.children[1].children[0].textContent = "Science agent - ready";
+        console.log("DONE - end science")
+    }
+    """
+
+
+def accordion_trigger_end_presse():
+    return """
+    function accordion_trigger_end() {
+        var accordion_presse = document.getElementById("accordion-presse")
+        accordion_presse.children[1].children[0].textContent = "Presse agent - ready";
+        console.log("DONE - end presse")
+    }
+    """
+
+
+def accordion_trigger_end_politique():
+    return """
+    function accordion_trigger_end() {
+        var accordion_politique = document.getElementById("accordion-politique")
+        accordion_politique.children[1].children[0].textContent = "Politique agent - ready";
+        console.log("DONE - end politique")
+    }
+    """
+
+
+def accordion_trigger_end_legal():
+    return """
+    function accordion_trigger_end() {
+        var accordion_legal = document.getElementById("accordion-legal")
+        accordion_legal.children[1].children[0].textContent = "Legal agent - ready";
+        console.log("DONE - end legal")
+    }
+    """

pyproject.toml

@@ -9,7 +9,7 @@ package-mode = true
 [tool.poetry.dependencies]
 python = "^3.10"
 langchain = "^0.2.5"
-gradio = {extras = ["oauth"], version = "^4.36.1"}
+gradio = "4.37.2"
 sentence-transformers = "2.2.2"
 msal = "^1.28.1"
 langchain-openai = "^0.1.8"

spinoza_project/source/frontend/utils.py

@@ -1,5 +1,6 @@
 from queue import SimpleQueue
 from dotenv import load_dotenv
+import re
 from langchain.callbacks.base import BaseCallbackHandler
 
 job_done = object()  # signals the processing is done
@@ -57,51 +58,66 @@ def get_source_link(metadata):
     return metadata["file_url"] + f"#page={metadata['content_page_number'] + 1}"
 
 
-def make_html_presse_source(source, i, tag, score, config):
+def make_html_presse_source(source, i, score, config):
     meta = source.metadata
     return f"""
-<div class="card" id="btn_{tag}_{i}_source" style="display:none;">
+<div class="card" id="doc{i}">
     <div class="card-content">
-        <div class="card-content-column-1">
-            <p><strong>Titre :</strong> {meta['file_title']}</p>
-            <p><strong>Auteur/s :</strong> {meta['file_source_type']}</p>
-            <p><strong>Date :</strong> {meta['file_date_publishing']}</p>
-        </div>
-        <div class="card-content-column-2">
-            <p><strong>Paragraphe id :</strong> {source.page_content}</p>
-        </div>
+        <h2>Doc {i} - {meta['file_title']}</h2>
+        <p>{source.page_content}</p>
     </div>
     <div class="card-footer">
-        <span>[{i}]</span>
+        <span>{meta['file_source_type']}</span>
         <span>Relevance Score : {round(100*score,1)}%</span>
+        <a href="https://fr.wikipedia.org/wiki/Baruch_Spinoza" target="_blank">
+            <span role="img" aria-label="Open PDF">🔗</span>
+        </a>
     </div>
 </div>
 """
 
 
-def make_html_source(source, i, tag, score, config):
+def make_html_source(source, i, score, config):
     meta = source.metadata
     return f"""
-<div class="card" id="btn_{tag}_{i}_source" style="display:none;">
+<div class="card" id="doc{i}">
     <div class="card-content">
-        <div class="card-content-column-1">
-            <p><strong>Titre :</strong> {meta['file_title']}</p>
-            <p><strong>Auteur/s :</strong> {meta['file_source_type']}</p>
-            <p><strong>Date :</strong> {meta['file_date_publishing']}</p>
-        </div>
-        <div class="card-content-column-2">
-            <p><strong>Paragraphe id :</strong> {source.page_content.replace(config["passage_preprompt"], "")}</p>
-        </div>
+        <h2>Doc {i} - {meta['file_title']} - Page {meta['content_page_number'] + 1}</h2>
+        <p>{source.page_content.replace(config["passage_preprompt"], "")}</p>
     </div>
     <div class="card-footer">
-        <span>[{i}]</span>
-        <span><a href="{get_source_link(meta)}" target="_blank">Lien source</a></span>
-        <span>Page {meta['content_page_number'] + 1}</span>
+        <span>{meta['file_source_type']}</span>
         <span>Relevance Score : {round(100*score,1)}%</span>
+        <a href="{get_source_link(meta)}" target="_blank">
+            <span role="img" aria-label="Open PDF">🔗</span>
+        </a>
     </div>
 </div>
 """
 
+def parse_output_llm_with_sources(output):
+    content_parts = re.split(
+        r"[\[(]?(Doc\s?\d+(?:,\s?Doc\s?\d+)*|doc\s?\d+(?:,\s?doc\s?\d+)*|Doc\s\d+)[\])?]",
+        output,
+    )
+    parts = []
+    for part in content_parts:
+        if part.lower().startswith("doc"):
+            subparts = part.split(",")
+            subparts = [
+                subpart.lower().replace("doc", "").strip() for subpart in subparts
+            ]
+            subparts = [
+                f"""<a href="#doc{subpart}" class="a-doc-ref" target="_self"><span class='doc-ref'><sup>{subpart}</sup></span></a>"""
+                for subpart in subparts
+            ]
+            parts.append("".join(subparts))
+        else:
+            parts.append(part)
+    content_parts = "".join(parts)
+
+    return content_parts
+
 
 def clear_text_box(textbox):
     return ""