Upload genaisession.ipynb

genaisession.ipynb

@@ -0,0 +1,267 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "source": [
+        "pip install faiss-cpu numpy pypdf sentence-transformers\n"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "LqTTG2cy0L1A",
+        "outputId": "c8be3a59-e763-47a7-f1de-4a010dae06f4"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Requirement already satisfied: faiss-cpu in /usr/local/lib/python3.11/dist-packages (1.10.0)\n",
+            "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (1.26.4)\n",
+            "Requirement already satisfied: pypdf in /usr/local/lib/python3.11/dist-packages (5.3.0)\n",
+            "Requirement already satisfied: sentence-transformers in /usr/local/lib/python3.11/dist-packages (3.4.1)\n",
+            "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from faiss-cpu) (24.2)\n",
+            "Requirement already satisfied: transformers<5.0.0,>=4.41.0 in /usr/local/lib/python3.11/dist-packages (from sentence-transformers) (4.48.3)\n",
+            "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (from sentence-transformers) (4.67.1)\n",
+            "Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.11/dist-packages (from sentence-transformers) (2.5.1+cu124)\n",
+            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.11/dist-packages (from sentence-transformers) (1.6.1)\n",
+            "Requirement already satisfied: scipy in /usr/local/lib/python3.11/dist-packages (from sentence-transformers) (1.13.1)\n",
+            "Requirement already satisfied: huggingface-hub>=0.20.0 in /usr/local/lib/python3.11/dist-packages (from sentence-transformers) (0.28.1)\n",
+            "Requirement already satisfied: Pillow in /usr/local/lib/python3.11/dist-packages (from sentence-transformers) (11.1.0)\n",
+            "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (3.17.0)\n",
+            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (2024.10.0)\n",
+            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (6.0.2)\n",
+            "Requirement already satisfied: requests in /usr/local/lib/python3.11/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (2.32.3)\n",
+            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub>=0.20.0->sentence-transformers) (4.12.2)\n",
+            "Requirement already satisfied: networkx in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (3.4.2)\n",
+            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (3.1.5)\n",
+            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (12.4.127)\n",
+            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (12.4.127)\n",
+            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (12.4.127)\n",
+            "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (9.1.0.70)\n",
+            "Requirement already satisfied: nvidia-cublas-cu12==12.4.5.8 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (12.4.5.8)\n",
+            "Requirement already satisfied: nvidia-cufft-cu12==11.2.1.3 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (11.2.1.3)\n",
+            "Requirement already satisfied: nvidia-curand-cu12==10.3.5.147 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (10.3.5.147)\n",
+            "Requirement already satisfied: nvidia-cusolver-cu12==11.6.1.9 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (11.6.1.9)\n",
+            "Requirement already satisfied: nvidia-cusparse-cu12==12.3.1.170 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (12.3.1.170)\n",
+            "Requirement already satisfied: nvidia-nccl-cu12==2.21.5 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (2.21.5)\n",
+            "Requirement already satisfied: nvidia-nvtx-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (12.4.127)\n",
+            "Requirement already satisfied: nvidia-nvjitlink-cu12==12.4.127 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (12.4.127)\n",
+            "Requirement already satisfied: triton==3.1.0 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (3.1.0)\n",
+            "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.11/dist-packages (from torch>=1.11.0->sentence-transformers) (1.13.1)\n",
+            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from sympy==1.13.1->torch>=1.11.0->sentence-transformers) (1.3.0)\n",
+            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/dist-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (2024.11.6)\n",
+            "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.11/dist-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (0.21.0)\n",
+            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.11/dist-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (0.5.2)\n",
+            "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn->sentence-transformers) (1.4.2)\n",
+            "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn->sentence-transformers) (3.5.0)\n",
+            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->torch>=1.11.0->sentence-transformers) (3.0.2)\n",
+            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests->huggingface-hub>=0.20.0->sentence-transformers) (3.4.1)\n",
+            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests->huggingface-hub>=0.20.0->sentence-transformers) (3.10)\n",
+            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests->huggingface-hub>=0.20.0->sentence-transformers) (2.3.0)\n",
+            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests->huggingface-hub>=0.20.0->sentence-transformers) (2025.1.31)\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "V2T0CkLD0Cnh",
+        "outputId": "176443e5-f99f-4d65-c6c5-e1ca43699006"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Extracting text from PDF...\n",
+            "Extracted text (first 500 chars): Machine Learning For Absolute\n",
+            "Beginners\n",
+            " \n",
+            " \n",
+            " \n",
+            " \n",
+            "Oliver Theobald\n",
+            " \n",
+            " \n",
+            " \n",
+            " \n",
+            " \n",
+            "Second Edition\n",
+            "Copyright © 2017 by Oliver Theobald\n",
+            "All rights reserved. No part of this publication may be reproduced,\n",
+            "distributed, or transmitted in any form or by any means, including\n",
+            "photocopying, recording, or other electronic or mechanical\n",
+            "methods, without the prior written permission of the publisher,\n",
+            "except in the case of brief quotations embodied in critical reviews\n",
+            "and certain other non-commercial uses permitted b\n",
+            "Chunking text...\n",
+            "Total chunks created: 53\n",
+            "Generating embeddings...\n",
+            "Embedding 1/53 generated, Shape: (1, 384)\n",
+            "Embedding 2/53 generated, Shape: (1, 384)\n",
+            "Embedding 3/53 generated, Shape: (1, 384)\n",
+            "Embedding 4/53 generated, Shape: (1, 384)\n",
+            "Embedding 5/53 generated, Shape: (1, 384)\n",
+            "Embedding 6/53 generated, Shape: (1, 384)\n",
+            "Embedding 7/53 generated, Shape: (1, 384)\n",
+            "Embedding 8/53 generated, Shape: (1, 384)\n",
+            "Embedding 9/53 generated, Shape: (1, 384)\n",
+            "Embedding 10/53 generated, Shape: (1, 384)\n",
+            "Embedding 11/53 generated, Shape: (1, 384)\n",
+            "Embedding 12/53 generated, Shape: (1, 384)\n",
+            "Embedding 13/53 generated, Shape: (1, 384)\n",
+            "Embedding 14/53 generated, Shape: (1, 384)\n",
+            "Embedding 15/53 generated, Shape: (1, 384)\n",
+            "Embedding 16/53 generated, Shape: (1, 384)\n",
+            "Embedding 17/53 generated, Shape: (1, 384)\n",
+            "Embedding 18/53 generated, Shape: (1, 384)\n",
+            "Embedding 19/53 generated, Shape: (1, 384)\n",
+            "Embedding 20/53 generated, Shape: (1, 384)\n",
+            "Embedding 21/53 generated, Shape: (1, 384)\n",
+            "Embedding 22/53 generated, Shape: (1, 384)\n",
+            "Embedding 23/53 generated, Shape: (1, 384)\n",
+            "Embedding 24/53 generated, Shape: (1, 384)\n",
+            "Embedding 25/53 generated, Shape: (1, 384)\n",
+            "Embedding 26/53 generated, Shape: (1, 384)\n",
+            "Embedding 27/53 generated, Shape: (1, 384)\n",
+            "Embedding 28/53 generated, Shape: (1, 384)\n",
+            "Embedding 29/53 generated, Shape: (1, 384)\n",
+            "Embedding 30/53 generated, Shape: (1, 384)\n",
+            "Embedding 31/53 generated, Shape: (1, 384)\n",
+            "Embedding 32/53 generated, Shape: (1, 384)\n",
+            "Embedding 33/53 generated, Shape: (1, 384)\n",
+            "Embedding 34/53 generated, Shape: (1, 384)\n",
+            "Embedding 35/53 generated, Shape: (1, 384)\n",
+            "Embedding 36/53 generated, Shape: (1, 384)\n",
+            "Embedding 37/53 generated, Shape: (1, 384)\n",
+            "Embedding 38/53 generated, Shape: (1, 384)\n",
+            "Embedding 39/53 generated, Shape: (1, 384)\n",
+            "Embedding 40/53 generated, Shape: (1, 384)\n",
+            "Embedding 41/53 generated, Shape: (1, 384)\n",
+            "Embedding 42/53 generated, Shape: (1, 384)\n",
+            "Embedding 43/53 generated, Shape: (1, 384)\n",
+            "Embedding 44/53 generated, Shape: (1, 384)\n",
+            "Embedding 45/53 generated, Shape: (1, 384)\n",
+            "Embedding 46/53 generated, Shape: (1, 384)\n",
+            "Embedding 47/53 generated, Shape: (1, 384)\n",
+            "Embedding 48/53 generated, Shape: (1, 384)\n",
+            "Embedding 49/53 generated, Shape: (1, 384)\n",
+            "Embedding 50/53 generated, Shape: (1, 384)\n",
+            "Embedding 51/53 generated, Shape: (1, 384)\n",
+            "Embedding 52/53 generated, Shape: (1, 384)\n",
+            "Embedding 53/53 generated, Shape: (1, 384)\n",
+            "Storing in FAISS...\n",
+            "FAISS database saved as 'vector_database.faiss'\n"
+          ]
+        }
+      ],
+      "source": [
+        "import os\n",
+        "import faiss\n",
+        "import numpy as np\n",
+        "import pypdf  # Using pypdf for text extraction\n",
+        "from sentence_transformers import SentenceTransformer\n",
+        "\n",
+        "# Load an open-source embedding model from Hugging Face\n",
+        "model = SentenceTransformer(\"sentence-transformers/all-MiniLM-L6-v2\")\n",
+        "\n",
+        "# Load text from PDF using pypdf\n",
+        "def load_pdf(pdf_path):\n",
+        "    text = \"\"\n",
+        "    with open(pdf_path, \"rb\") as file:\n",
+        "        reader = pypdf.PdfReader(file)\n",
+        "        for page in reader.pages:\n",
+        "            text += page.extract_text() + \"\\n\" if page.extract_text() else \"\"  # Handle empty pages\n",
+        "    return text.strip() if text.strip() else None  # Ensure non-empty text\n",
+        "\n",
+        "# Split text into chunks\n",
+        "def chunk_text(text, chunk_size=500):\n",
+        "    words = text.split()\n",
+        "    chunks = [\" \".join(words[i:i+chunk_size]) for i in range(0, len(words), chunk_size)]\n",
+        "    return [c for c in chunks if c.strip()]  # Remove empty chunks\n",
+        "\n",
+        "# Generate embeddings using Hugging Face model\n",
+        "def get_embedding(text):\n",
+        "    return model.encode(text, convert_to_numpy=True).reshape(1, -1)  # Ensure 2D shape\n",
+        "\n",
+        "# Store embeddings in FAISS\n",
+        "def store_in_faiss(embeddings):\n",
+        "    if len(embeddings) == 0:\n",
+        "        raise ValueError(\"No embeddings found! Check your text extraction and chunking.\")\n",
+        "\n",
+        "    embeddings = np.vstack(embeddings)  # Stack into 2D array\n",
+        "    dim = embeddings.shape[1]\n",
+        "    index = faiss.IndexFlatL2(dim)\n",
+        "    index.add(embeddings)\n",
+        "    faiss.write_index(index, \"vector_database.faiss\")\n",
+        "\n",
+        "def main():\n",
+        "    pdf_path = \"/content/[Oliver_Theobald]_Machine_Learning_for_Absolute_Be.pdf\"\n",
+        "\n",
+        "    print(\"Extracting text from PDF...\")\n",
+        "    text = load_pdf(pdf_path)\n",
+        "    if text is None:\n",
+        "        raise ValueError(\"No text extracted from PDF. Check if it's a scanned document!\")\n",
+        "\n",
+        "    print(\"Extracted text (first 500 chars):\", text[:500])\n",
+        "\n",
+        "    print(\"Chunking text...\")\n",
+        "    chunks = chunk_text(text)\n",
+        "    print(f\"Total chunks created: {len(chunks)}\")\n",
+        "    if not chunks:\n",
+        "        raise ValueError(\"No valid text chunks found!\")\n",
+        "\n",
+        "    print(\"Generating embeddings...\")\n",
+        "    embeddings = []\n",
+        "    for i, chunk in enumerate(chunks):\n",
+        "        emb = get_embedding(chunk)\n",
+        "        print(f\"Embedding {i+1}/{len(chunks)} generated, Shape: {emb.shape}\")\n",
+        "        embeddings.append(emb)\n",
+        "\n",
+        "    if not embeddings:\n",
+        "        raise ValueError(\"No embeddings were generated! Check the text chunks.\")\n",
+        "\n",
+        "    embeddings = np.vstack(embeddings)\n",
+        "\n",
+        "    print(\"Storing in FAISS...\")\n",
+        "    store_in_faiss(embeddings)\n",
+        "\n",
+        "    print(\"FAISS database saved as 'vector_database.faiss'\")\n",
+        "\n",
+        "if __name__ == \"__main__\":\n",
+        "    main()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [],
+      "metadata": {
+        "id": "4FyvMg221DIg"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}