Upload 8 files

Upload the main files.

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+chromedriver.exe filter=lfs diff=lfs merge=lfs -text

Edu-Researcher.py

@@ -0,0 +1,126 @@
+import streamlit as st
+import requests
+import logging
+import os
+from duckduckgo_search import DDGS  # Using DuckDuckGo search library
+from langchain.embeddings import HuggingFaceEmbeddings
+from langgraph.graph import START, END, StateGraph
+from typing import Dict, Any
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Initialize session state for messages
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+# Refresh messages if switching to Edu-Researcher
+if st.session_state.get("active_function") != "Edu-Researcher":
+    st.session_state.messages = []
+    st.session_state.active_function = "Edu-Researcher"
+
+# Sidebar configuration
+with st.sidebar:
+    st.header("Researcher Configuration")
+    st.markdown("[Get HuggingFace Token](https://huggingface.co/settings/tokens)")
+    st.info("Using DuckDuckGo search for web results")
+    system_message = st.text_area(
+        "System Message",
+        value="You are an assistant for research. Use the retrieved web snippets to answer the query concisely.",
+        height=100
+    )
+    max_tokens = st.slider("Max Tokens", 10, 4000, 300)
+    temperature = st.slider("Temperature", 0.1, 4.0, 0.3)
+    top_p = st.slider("Top-p", 0.1, 1.0, 0.6)
+
+# Set up embedding model (for context retrieval)
+embedding_model = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
+
+st.title("Edu-Researcher")
+st.caption("Powered by DuckDuckGo Search, LangGraph, and Hugging Face Inference API")
+
+# Input research query
+query_input = st.text_input("Enter your research query:")
+
+# Define our research state as a dictionary
+# The state will hold: query, sources, snippets, and response
+ResearchState = Dict[str, Any]
+
+def search_web_node(state: ResearchState) -> ResearchState:
+    # Use DuckDuckGo to search for the query (max 3 results) using DDGS
+    sources = []
+    snippets = []
+    with DDGS() as ddgs:
+        results = ddgs.text(state["query"], max_results=3)
+        for res in results:
+            sources.append(res.get("href", ""))
+            snippet = res.get("body") or res.get("title", "")
+            snippets.append(snippet)
+    state["sources"] = sources
+    state["snippets"] = snippets
+    return state
+
+def generate_answer_node(state: ResearchState) -> ResearchState:
+    # Combine retrieved snippets into a context string
+    context = "\n\n".join(state.get("snippets", []))
+    full_prompt = (
+        f"{system_message}\n\n"
+        f"Context: {context}\n\n"
+        f"Query: {state['query']}\n\n"
+        f"Please provide a succinct and complete answer within {max_tokens} tokens."
+    )
+    # Query the Hugging Face API using the selected model endpoint
+    model_endpoint = "https://api-inference.huggingface.co/models/Qwen/Qwen2.5-72B-Instruct"
+    headers = {"Authorization": f"Bearer {st.secrets['HF_TOKEN']}"}
+    logger.info(f"Sending request to {model_endpoint} with prompt: {full_prompt}")
+    response = requests.post(model_endpoint, headers=headers, json={
+        "inputs": full_prompt,
+        "parameters": {
+            "max_new_tokens": max_tokens,
+            "temperature": temperature,
+            "top_p": top_p,
+            "return_full_text": False
+        }
+    })
+    logger.info(f"Received response: {response.status_code}, {response.text}")
+    try:
+        output = response.json()
+    except requests.exceptions.JSONDecodeError:
+        logger.error(f"Failed to decode JSON response: {response.text}")
+        output = None
+    if output and isinstance(output, list) and len(output) > 0 and "generated_text" in output[0]:
+        state["response"] = output[0]["generated_text"].strip()
+    else:
+        state["response"] = "No response generated - please try again."
+    return state
+
+def render_message(content):
+    # Render as LaTeX if enclosed by $$, else as markdown.
+    if content.strip().startswith("$$") and content.strip().endswith("$$"):
+        st.latex(content.strip()[2:-2])
+    else:
+        st.markdown(content)
+
+# Build the state graph using langgraph
+builder = StateGraph(dict)
+builder.add_node("search_web", search_web_node)
+builder.add_node("generate_answer", generate_answer_node)
+builder.add_edge(START, "search_web")
+builder.add_edge("search_web", "generate_answer")
+builder.add_edge("generate_answer", END)
+graph = builder.compile()
+
+if query_input:
+    # Initialize state with the query
+    initial_state = {"query": query_input}
+    with st.spinner("Searching the web and generating response..."):
+        result_state = graph.invoke(initial_state)
+    if result_state:
+        render_message(result_state["response"])
+        st.subheader("Sources:")
+        for src in result_state.get("sources", []):
+            st.write(src)
+    else:
+        st.error("No response generated.")
+    st.session_state.messages.append({"role": "researcher", "content": result_state.get("response", "No answer.")})

Edu-Scraper.py

@@ -0,0 +1,153 @@
+import streamlit as st
+import requests
+import logging
+import os
+from langchain_community.document_loaders import SeleniumURLLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import InMemoryVectorStore
+from langchain.embeddings import HuggingFaceEmbeddings
+
+# Disable proxy for Hugging Face endpoints
+os.environ["NO_PROXY"] = "huggingface.co"
+
+# PATCH: Force SeleniumURLLoader to use local chromedriver.exe using Service
+import langchain_community.document_loaders.url_selenium as url_selenium
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.chrome.service import Service
+
+def patched_get_driver(self):
+    chrome_options = Options()
+    chrome_options.add_argument("--headless")
+    # Use a simple path: assume "chromedriver.exe" is in the current working directory
+    service = Service("chromedriver.exe")
+    return webdriver.Chrome(service=service, options=chrome_options)
+
+url_selenium.SeleniumURLLoader._get_driver = patched_get_driver
+# END PATCH
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Initialize session state for chat history and vector store
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+if "vector_store" not in st.session_state:
+    st.session_state.vector_store = None
+
+# Set up embedding model (used for indexing)
+embedding_model = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
+
+# Sidebar configuration (similar to pdf_rag.py)
+with st.sidebar:
+    st.header("Model Configuration")
+    st.markdown("[Get HuggingFace Token](https://huggingface.co/settings/tokens)")
+    # Updated model options list with additional option for Qwen2.5-72B-Instruct
+    model_options = [
+        "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B",
+        "Qwen/Qwen2.5-72B-Instruct"
+    ]
+    selected_model = st.selectbox("Select Model", model_options, index=0)
+    system_message = st.text_area(
+        "System Message",
+        value="You are an assistant for question-answering tasks created by ruslanmv.com. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. Use three sentences maximum for each question and keep the answer concise.",
+        height=100
+    )
+    max_tokens = st.slider("Max Tokens", 10, 4000, 300)
+    temperature = st.slider("Temperature", 0.1, 4.0, 0.3)
+    top_p = st.slider("Top-p", 0.1, 1.0, 0.6)
+
+# After sidebar configuration and session_state initialization
+if st.session_state.get("active_function") != "Edu-Scraper":
+    st.session_state.messages = []
+    st.session_state.active_function = "Edu-Scraper"
+
+# Main interface
+st.title("Edu Scraper with RAG")
+st.caption("Powered by Hugging Face Inference API - Configure parameters in the sidebar.")
+
+# URL input section (instead of PDF upload)
+url = st.text_input("Enter URL for scraping:")
+
+if url:
+    try:
+        with st.spinner("Loading page..."):
+            documents = SeleniumURLLoader(urls=[url]).load()
+        with st.spinner("Splitting content..."):
+            splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200, add_start_index=True)
+            chunks = splitter.split_documents(documents)
+        # Create and store vector store
+        vector_store = InMemoryVectorStore.from_documents(chunks, embedding_model)
+        st.session_state.vector_store = vector_store
+        st.success("Page processed and indexed successfully!")
+    except Exception as e:
+        st.error(f"Error processing URL: {str(e)}")
+
+def render_message(content):
+    # If content is enclosed by $$, render as LaTeX; else as markdown.
+    if content.strip().startswith("$$") and content.strip().endswith("$$"):
+        st.latex(content.strip()[2:-2])
+    else:
+        st.markdown(content)
+
+# Display chat history
+for message in st.session_state.messages:
+    with st.chat_message(message["role"]):
+        render_message(message["content"])
+
+# Function to query Hugging Face API
+def query(payload, api_url):
+    headers = {"Authorization": f"Bearer {st.secrets['HF_TOKEN']}"}
+    logger.info(f"Sending request to {api_url} with payload: {payload}")
+    response = requests.post(api_url, headers=headers, json=payload)
+    logger.info(f"Received response: {response.status_code}, {response.text}")
+    try:
+        return response.json()
+    except requests.exceptions.JSONDecodeError:
+        logger.error(f"Failed to decode JSON response: {response.text}")
+        return None
+
+# Handle user chat input
+if prompt := st.chat_input("Type your message..."):
+    st.session_state.messages.append({"role": "user", "content": prompt})
+    with st.chat_message("user"):
+        render_message(prompt)
+    try:
+        with st.spinner("Generating response..."):
+            if not st.session_state.vector_store:
+                st.error("Please enter a URL and process the page first.")
+                st.stop()
+            vs = st.session_state.vector_store
+            related_docs = vs.similarity_search(prompt, k=3)
+            context = "\n\n".join([doc.page_content for doc in related_docs])
+            full_prompt = (
+                f"{system_message}\n\n"
+                f"Context: {context}\n\n"
+                f"{prompt}\n\n"
+                f"Please provide a succinct, direct, and complete answer within {max_tokens} tokens, without extra reasoning steps. Use three sentences maximum for each question and keep the answer concise."
+            )
+            payload = {
+                "inputs": full_prompt,
+                "parameters": {
+                    "max_new_tokens": max_tokens,
+                    "temperature": temperature,
+                    "top_p": top_p,
+                    "return_full_text": False
+                }
+            }
+            api_url = f"https://api-inference.huggingface.co/models/{selected_model}"
+            output = query(payload, api_url)
+            if output and isinstance(output, list) and len(output) > 0 and "generated_text" in output[0]:
+                assistant_response = output[0]["generated_text"].strip()
+                with st.chat_message("assistant"):
+                    render_message(assistant_response)
+                st.session_state.messages.append({
+                    "role": "assistant",
+                    "content": assistant_response
+                })
+            else:
+                st.error("No response generated - please try again")
+    except Exception as e:
+        logger.error(f"Error: {str(e)}", exc_info=True)
+        st.error(f"An error occurred: {str(e)}")

Hellopage.py

@@ -0,0 +1,40 @@
+import streamlit as st
+import os
+
+st.set_page_config(
+    page_title="Welcome to Edu_AI",
+    page_icon="👋",
+)
+
+# Sidebar navigation
+st.sidebar.title("Navigation")
+# Updated navigation options to include AI Researcher
+page = st.sidebar.selectbox("Go to", ["Welcome", "PDF Chatbot with RAG", "Edu Scraper with RAG", "Edu Researcher"])
+
+if page == "Welcome":
+    st.write("# Welcome to Edu_AI 👋")
+    st.sidebar.success("Select a feature from the sidebar to get started.")
+
+    st.markdown(
+        """
+        Edu_AI is a streamlit web application that is powered by artificial intelligence
+        functions for both research and educational purposes. 
+        **👈 Select a demo from the sidebar** to see some examples
+        of what Edu_AI can do!
+        
+### Made by the talented batch 10 UIT Section-A students group 7
+        """
+    )
+    st.image("webp.png", caption="Welcome to Edu_AI")
+elif page == "PDF Chatbot with RAG":
+    # Run pdf_rag.py
+    with open(os.path.join(os.path.dirname(__file__), 'pdf_rag.py')) as f:
+        exec(f.read())
+elif page == "Edu Scraper with RAG":
+    # Run Edu-Scraper.py
+    with open(os.path.join(os.path.dirname(__file__), 'Edu-Scraper.py')) as f:
+        exec(f.read())
+elif page == "Edu Researcher":
+    # Run Edu-Researcher.py
+    with open(os.path.join(os.path.dirname(__file__), 'Edu-Researcher.py')) as f:
+        exec(f.read())

README.md

@@ -1,3 +1,21 @@
----
-license: mit
----
+# Chat with PDF
+
+# Pre-requisites
+Install Ollama on your local machine from the [official website](https://ollama.com/). And then pull the Deepseek model:
+
+```bash
+ollama pull deepseek-r1:14b
+```
+
+Install the dependencies using pip:
+
+```bash
+pip install -r requirements.txt
+```
+
+# Run
+Run the Streamlit app:
+
+```bash
+streamlit run pdf_rag.py
+```

chromedriver.exe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbef5f36638e7f728ed79dbf72043c5651759633fd5ac355fa44cd7aa853a113
+size 18541056

pdf_rag.py

@@ -0,0 +1,169 @@
+import streamlit as st
+import requests
+import logging
+import os
+from langchain_community.document_loaders import PDFPlumberLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import InMemoryVectorStore
+from langchain.embeddings import HuggingFaceEmbeddings
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Initialize session state for chat history and vector store
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+if "vector_store" not in st.session_state:
+    st.session_state.vector_store = None
+
+# Refresh messages if switching to PDF Chat (pdf_rag)
+if st.session_state.get("active_function") != "pdf_rag":
+    st.session_state.messages = []
+    st.session_state.active_function = "pdf_rag"
+
+# Set up PDF directory and embedding model
+pdfs_directory = "chat-with-pdf\pdfs"
+os.makedirs(pdfs_directory, exist_ok=True)
+embedding_model = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
+
+# Sidebar configuration
+with st.sidebar:
+    st.header("Model Configuration")
+    st.markdown("[Get HuggingFace Token](https://huggingface.co/settings/tokens)")
+
+    # Dropdown to select model
+    model_options = [
+        "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B",
+        "Qwen/Qwen2.5-72B-Instruct"
+    ]
+    selected_model = st.selectbox("Select Model", model_options, index=0)
+
+    system_message = st.text_area(
+        "System Message",
+        value="You are an assistant for question-answering tasks created by ruslanmv.com. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know.Use three sentences maximum for each question and keep the answer concise.",
+        height=100
+    )
+    max_tokens = st.slider("Max Tokens", 10, 4000, 300)
+    temperature = st.slider("Temperature", 0.1, 4.0, 0.3)
+    top_p = st.slider("Top-p", 0.1, 1.0, 0.6)
+
+# Main interface
+st.title(u"\U0001F4D1 PDF Chatbot with RAG")  # fixed emoji encoding
+st.caption("Powered by Hugging Face Inference API - You can configure the temperature,tokens and top-p values in the sidebar.")
+
+# PDF upload section
+uploaded_file = st.file_uploader(
+    "Upload a PDF for context",
+    type="pdf",
+    accept_multiple_files=False
+)
+
+if uploaded_file:
+    try:
+        # Save uploaded PDF
+        pdf_path = os.path.join(pdfs_directory, uploaded_file.name)
+        with open(pdf_path, "wb") as f:
+            f.write(uploaded_file.getbuffer())
+        
+        # Load and process PDF
+        loader = PDFPlumberLoader(pdf_path)
+        documents = loader.load()
+        
+        # Split text into chunks
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=1000,
+            chunk_overlap=200
+        )
+        chunks = text_splitter.split_documents(documents)
+        
+        # Create and store vector store
+        vector_store = InMemoryVectorStore.from_documents(chunks, embedding_model)
+        st.session_state.vector_store = vector_store
+        st.success("PDF processed and indexed successfully!")
+    except PermissionError:
+        st.error(f"Permission denied: Unable to save the file to {pdf_path}. Please check the directory permissions.")
+    except Exception as e:
+        st.error(f"Error processing PDF: {str(e)}")
+
+# Display chat history
+for message in st.session_state.messages:
+    with st.chat_message(message["role"]):
+        st.markdown(message["content"])
+
+# Function to query Hugging Face API
+def query(payload, api_url):
+    headers = {"Authorization": f"Bearer {st.secrets['HF_TOKEN']}"}
+    logger.info(f"Sending request to {api_url} with payload: {payload}")
+    response = requests.post(api_url, headers=headers, json=payload)
+    logger.info(f"Received response: {response.status_code}, {response.text}")
+    try:
+        return response.json()
+    except requests.exceptions.JSONDecodeError:
+        logger.error(f"Failed to decode JSON response: {response.text}")
+        return None
+
+# Handle user input
+if prompt := st.chat_input("Type your message..."):
+    st.session_state.messages.append({"role": "user", "content": prompt})
+    
+    with st.chat_message("user"):
+        st.markdown(prompt)
+
+    try:
+        with st.spinner("Generating response..."):
+            # Check if vector store is available
+            if not st.session_state.vector_store:
+                st.error("Please upload a PDF first to provide context.")
+                st.stop()
+            
+            # Retrieve relevant documents
+            vector_store = st.session_state.vector_store
+            related_docs = vector_store.similarity_search(prompt, k=3)
+            
+            # Build context
+            context = "\n\n".join([doc.page_content for doc in related_docs])
+            
+            # Prepare full prompt with an instruction for brevity and completeness
+            full_prompt = (
+                f"{system_message}\n\n"
+                f"Context: {context}\n\n"
+                f"{prompt}\n\n"
+                f"Please provide a succinct, direct, and complete answer within {max_tokens} tokens, without extra reasoning steps. Use three sentences maximum for each question and keep the answer concise."
+            )
+            
+            # Prepare API payload
+            payload = {
+                "inputs": full_prompt,
+                "parameters": {
+                    "max_new_tokens": max_tokens,
+                    "temperature": temperature,
+                    "top_p": top_p,
+                    "return_full_text": False
+                }
+            }
+
+            # Query API
+            api_url = f"https://api-inference.huggingface.co/models/{selected_model}"
+            output = query(payload, api_url)
+
+            # Handle response
+            if output and isinstance(output, list) and len(output) > 0:
+                if 'generated_text' in output[0]:
+                    assistant_response = output[0]['generated_text'].strip()
+                    
+                    with st.chat_message("assistant"):
+                        st.markdown(assistant_response)
+                    
+                    st.session_state.messages.append({
+                        "role": "assistant",
+                        "content": assistant_response
+                    })
+                else:
+                    st.error("Unexpected response format from the model")
+            else:
+                st.error("No response generated - please try again")
+                
+    except Exception as e:
+        logger.error(f"Error: {str(e)}", exc_info=True)
+        st.error(f"An error occurred: {str(e)}")

requirements.txt

@@ -0,0 +1,11 @@
+streamlit
+langchain_core
+langchain_community
+langchain_ollama
+duckduckgo_search
+langgraph
+pdfplumber
+selenium
+unstructured
+chromedriver
+sentence-transformers