Add scraper and update RAG

app.py

@@ -7,40 +7,39 @@ from llama_index.llms.openai import OpenAI
 from llama_index.vector_stores.pinecone import PineconeVectorStore
 from llama_index.core import VectorStoreIndex, StorageContext
 from dotenv import load_dotenv
-from pinecone import Pinecone  # Correct import
+from pinecone import Pinecone
+from llama_index.embeddings.openai import OpenAIEmbedding
 
 load_dotenv()
 
 app = Flask(__name__)
 app.debug = True
 
-# Configure logging
 logging.basicConfig(level=logging.DEBUG)
 
-# Set up OpenAI API key
 openai.api_key = os.getenv('OPENAI_API_KEY')
 
-# Initialize Pinecone
 pc = Pinecone(
     api_key=os.getenv('PINECONE_API_KEY')
 )
 
-# Name of your existing Pinecone index
 PINECONE_INDEX_NAME = os.getenv('PINECONE_INDEX')
 
-# Initialize Pinecone index
 pinecone_index = pc.Index(PINECONE_INDEX_NAME)
 
+# After initializing the Pinecone index
+stats = pinecone_index.describe_index_stats()
+app.logger.debug(f"Pinecone index stats: {stats}")
+
 # Set up LlamaIndex global settings
 Settings.llm = OpenAI(
-    model=os.getenv('OPENAI_MODEL', 'gpt-3.5-turbo'),  # Default to 'gpt-3.5-turbo' if not specified
+    model=os.getenv('OPENAI_MODEL', 'gpt-3.5-turbo'),
     temperature=0
 )
 
-# Set up Pinecone Vector Store
 vector_store = PineconeVectorStore(
     pinecone_index=pinecone_index,
-    namespace=None  # Specify a namespace if used during ingestion
+    namespace="ai"
 )
 
 # Create Storage Context with the Vector Store
@@ -56,7 +55,7 @@ index = VectorStoreIndex.from_vector_store(
 def predict():
     try:
         data = request.json
-        app.logger.debug(f"Received data: {data}")
+        app.logger.debug(f"Received data: {data}") # data => {'query': 'What is LangChain?'}
 
         if not data:
             app.logger.error("No data provided in the request.")
@@ -68,16 +67,60 @@ def predict():
             app.logger.error("No query provided in the request.")
             return jsonify({'error': 'No query provided.'}), 400
 
-        # Perform the query using LlamaIndex
-        response = index.as_query_engine().query(user_query)
-        # app.logger.debug(f"Generated response: {response}")
+        # Log Pinecone query details
+        app.logger.debug(f"Querying Pinecone index: {PINECONE_INDEX_NAME}")
+        app.logger.debug(f"Query: {user_query}")
 
-        return jsonify({'response': str(response)})
+        # Perform the query using LlamaIndex
+        query_engine = index.as_query_engine(similarity_top_k=5)
+        app.logger.debug(f"Query engine: {query_engine}")
+        
+        response = query_engine.query(user_query)
+        app.logger.debug(f"Raw response object: {response}")
+        app.logger.debug(f"Response type: {type(response)}")
+        
+        if hasattr(response, 'source_nodes'):
+            app.logger.debug(f"Number of source nodes: {len(response.source_nodes)}")
+            for i, node in enumerate(response.source_nodes):
+                app.logger.debug(f"Source node {i}: {node.node.text[:100]}...")  # Log first 100 chars of each source node
+        else:
+            app.logger.warning("No source nodes found in the response")
+
+        if hasattr(response, 'response'):
+            response_text = response.response
+        else:
+            response_text = str(response)
+        
+        app.logger.debug(f"Response text: {response_text}")
+
+        return jsonify({'response': response_text})
 
     except Exception as e:
-        app.logger.error(f"Error processing request: {e}")
+        app.logger.error(f"Error processing request: {e}", exc_info=True)
         return jsonify({"error": "An error occurred while processing the request"}), 500
 
+@app.route('/empty-datastore', methods=['DELETE'])
+def empty_datastore():
+    try:
+        # Attempt to delete all vectors in the default namespace
+        delete_response = pinecone_index.delete(delete_all=True, namespace="")
+        app.logger.debug(f"Delete response: {delete_response}")
+        
+        # Verify the index is empty
+        stats = pinecone_index.describe_index_stats()
+        app.logger.debug(f"Index stats after deletion: {stats}")
+        
+        if stats['total_vector_count'] == 0:
+            app.logger.info("Datastore emptied successfully.")
+            return jsonify({'message': 'Datastore emptied successfully'}), 200
+        else:
+            app.logger.warning("Datastore not fully emptied.")
+            return jsonify({'message': 'Datastore not fully emptied'}), 500
+    
+    except Exception as e:
+        app.logger.error(f"Error emptying datastore: {e}")
+        return jsonify({'error': f'An error occurred while emptying the datastore: {str(e)}'}), 500
+
 if __name__ == '__main__':
     from os import environ
     app.run(host='0.0.0.0', port=int(environ.get('PORT', 7860)))

scraper.py

@@ -0,0 +1,81 @@
+import requests
+from bs4 import BeautifulSoup
+import os
+from urllib.parse import urljoin, urlparse, urldefrag
+import time
+import logging
+
+# Set up logging for error handling
+logging.basicConfig(filename='scrape_errors.log', level=logging.ERROR)
+
+def scrape_docs(base_url, save_dir, delay=1):
+    if not os.path.exists(save_dir):
+        os.makedirs(save_dir)
+
+    visited = set()
+
+    def normalize_url(url):
+        # Remove fragments and query parameters, and normalize slashes
+        url, _ = urldefrag(url)  # Remove the fragment
+        parsed_url = urlparse(url)
+        normalized_url = parsed_url._replace(query="").geturl().rstrip('/')
+        return normalized_url
+
+    def scrape_page(url):
+        normalized_url = normalize_url(url)
+
+        if normalized_url in visited:
+            return
+        visited.add(normalized_url)
+
+        try:
+            response = requests.get(normalized_url)
+            if response.status_code != 200:
+                logging.error(f"Failed to retrieve {normalized_url}, status code: {response.status_code}")
+                return
+
+            soup = BeautifulSoup(response.text, 'html.parser')
+
+            # Save the page content
+            parsed_url = urlparse(normalized_url)
+            relative_path = parsed_url.path.lstrip('/')
+            file_path = os.path.join(save_dir, relative_path)
+
+            # Ensure the directory exists
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
+            # Determine the file name: append 'index.html' if it's a directory
+            if parsed_url.path.endswith('/') or not os.path.basename(file_path):
+                file_path = os.path.join(file_path, 'index.html')
+            elif not file_path.endswith('.html'):
+                file_path += '.html'
+
+            with open(file_path, 'w', encoding='utf-8') as f:
+                f.write(response.text)
+
+            print(f"Scraped: {normalized_url}")
+
+            # Find all links on the page
+            for link in soup.find_all('a', href=True):
+                href = link['href']
+                full_url = urljoin(normalized_url, href)
+
+                # Only follow links within the base URL
+                if full_url.startswith(base_url):
+                    scrape_page(full_url)
+
+            # Respect server rate limits
+            time.sleep(delay)
+
+        except Exception as e:
+            logging.error(f"Failed to scrape {normalized_url}: {e}")
+            print(f"Failed to scrape {normalized_url}, see log for details.")
+
+    scrape_page(base_url)
+
+if __name__ == "__main__":
+    base_url = "https://docs.llamaindex.ai/en/stable/api_reference/"
+    save_dir = "llamaindex_docs"
+    
+    scrape_docs(base_url, save_dir, delay=1)
+    print("Scraping completed.")