Initial upload

.gitignore

@@ -0,0 +1,93 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual Environment
+venv/
+env/
+ENV/
+.env
+.venv
+env.bak/
+venv.bak/
+
+# IDE specific files
+.idea/
+.vscode/
+*.swp
+*.swo
+.project
+.pydevproject
+.settings/
+
+# Project specific
+# data/
+# *.pkl
+# temp_query_image.jpg
+# embeddings.pkl
+*.h5
+models/
+temp/
+logs/
+
+# OS specific
+.DS_Store
+Thumbs.db
+*.db
+*.sqlite3
+
+# Jupyter Notebook
+.ipynb_checkpoints
+*.ipynb
+
+# Distribution / packaging
+.Python
+*.manifest
+*.spec
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Logs
+*.log
+local_settings.py
+db.sqlite3
+
+# Environment variables
+.env
+.env.local
+.env.*.local
+
+# Docker
+Dockerfile
+docker-compose.yml
+.docker/

README.md

@@ -0,0 +1,109 @@
+## Image Similarity Search Engine
+A deep learning-based image similarity search engine that uses EfficientNetB0 for feature extraction and FAISS for fast similarity search. The application provides a web interface built with Streamlit for easy interaction.
+
+Features
+- Deep Feature Extraction: Uses EfficientNetB0 (pre-trained on ImageNet) to extract meaningful features from images
+- Fast Similarity Search: Implements FAISS for efficient nearest-neighbor search
+- Interactive Web Interface: User-friendly interface built with Streamlit
+- Real-time Processing: Shows progress and time estimates during feature extraction
+- Scalable Architecture: Designed to handle large image datasets efficiently
+
+## Installation
+## Prerequisites
+
+Python 3.8 or higher
+pip package manager
+
+## Setup
+
+1. Clone the repository:
+```
+git clone https://github.com/yourusername/image-similarity-search.git
+cd image-similarity-search
+```
+2. Create and activate a virtual environment:
+```
+python -m venv venv
+source venv/bin/activate  # On Windows use: venv\Scripts\activate
+```
+3. Install required packages:
+```
+pip install -r requirements.txt
+```
+
+## Project Structure
+```
+image-similarity-search/
+├── data/
+│   ├── images/                     # Directory for train dataset images
+│   ├── sample-test-images/         # Directory for test dataset images
+│   └── embeddings.pkl              # Pre-computed image embeddings
+├── src/
+│   ├── feature_extractor.py    # EfficientNetB0 feature extraction
+│   ├── preprocessing.py        # Image preprocessing and embedding computation
+│   ├── similarity_search.py    # FAISS-based similarity search
+│   └── main.py                 # Streamlit web interface
+├── requirements.txt
+├── README.md
+└── .gitignore
+```
+## Usage
+
+1. **Prepare Your Dataset:**
+Get training image dataset from drive:
+```
+https://drive.google.com/file/d/1U2PljA7NE57jcSSzPs21ZurdIPXdYZtN/view?usp=drive_link
+```
+Place your image dataset in the data/images directory
+Supported formats: JPG, JPEG, PNG
+
+2. **Generate Embeddings:**
+```
+python -m src.preprocessing
+```
+
+**This will**:
+- Process all images in the dataset
+- Show progress and time estimates
+- Save embeddings to data/embeddings.pkl
+
+3. **Run the Web Interface:**
+```
+streamlit run src/main.py
+```
+
+4. Using the Interface:
+
+- Upload a query image using the file uploader
+- Click "Search Similar Images"
+- View the most similar images from your dataset
+
+
+
+## Technical Details
+**Feature Extraction**
+- Uses EfficientNetB0 without top layers
+- Input image size: 224x224 pixels
+- Output feature dimension: 1280
+
+**Similarity Search**
+- Uses FAISS IndexFlatL2 for L2 distance-based search
+- Returns top-k most similar images (default k=5)
+
+**Web Interface**
+- Responsive design with Streamlit
+- Displays original and similar images with similarity scores
+- Progress tracking during processing
+
+**Dependencies**
+- TensorFlow 2.x
+- FAISS-cpu (or FAISS-gpu for GPU support)
+- Streamlit
+- Pillow
+- NumPy
+- tqdm
+
+**Performance**
+- Feature extraction: ~1 second per image on CPU
+- Similarity search: Near real-time for datasets up to 100k images
+- Memory usage depends on dataset size (approximately 5KB per image embedding)

data/embeddings.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdc230f965cc668bcb0bb7d024c3ef0cde267dca763df634b93b58f512aedc73
+size 229887733

requirements.txt

@@ -0,0 +1,7 @@
+tensorflow
+numpy
+opencv-python
+scikit-learn
+streamlit
+Pillow
+faiss-cpu

src/__init__.py

@@ -0,0 +1,29 @@
+from dotenv import load_dotenv
+import os
+from huggingface_hub import HfApi, HfFolder, create_repo, upload_folder
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Load the access token from the environment variable
+access_token = os.getenv("HF_ACCESS_TOKEN")
+
+if access_token is None:
+    raise ValueError("HF_ACCESS_TOKEN environment variable is not set")
+
+# Save the token to the Hugging Face folder
+HfFolder.save_token(access_token)
+
+# Authenticate
+HfFolder.get_token()
+
+# Create a new repository
+create_repo("image-search-engine-fashion", private=False)
+
+# Upload the folder
+upload_folder(
+    folder_path=r"C:\Users\rudra\Documents\GitHub\image-search-engine",
+    repo_id="rudra0410/image-search-engine-fashion",
+    repo_type="model",
+    commit_message="Initial upload"
+)

src/feature_extractor.py

@@ -0,0 +1,22 @@
+import tensorflow as tf
+from tensorflow.keras.applications import EfficientNetB0
+from tensorflow.keras.preprocessing import image
+from tensorflow.keras.applications.efficientnet import preprocess_input
+import numpy as np
+
+class FeatureExtractor:
+    def __init__(self):
+        # Load pretrained EfficientNetB0 model without top layers
+        base_model = EfficientNetB0(weights='imagenet', include_top=False, pooling='avg')
+        self.model = tf.keras.Model(inputs=base_model.input, outputs=base_model.output)
+
+    def extract_features(self, img_path):
+        # Load and preprocess the image
+        img = image.load_img(img_path, target_size=(224, 224))
+        img_array = image.img_to_array(img)
+        expanded_img = np.expand_dims(img_array, axis=0)
+        preprocessed_img = preprocess_input(expanded_img)
+
+        # Extract features
+        features = self.model.predict(preprocessed_img)
+        return features.flatten()

src/main.py

@@ -0,0 +1,50 @@
+import streamlit as st
+from PIL import Image
+from feature_extractor import FeatureExtractor
+from similarity_search import SimilaritySearchEngine
+
+def main():
+    st.title('Image Similarity Search')
+
+    # Upload query image
+    uploaded_file = st.file_uploader("Choose an image...", type=["jpg", "png", "jpeg"])
+
+    if uploaded_file is not None:
+        # Load the uploaded image
+        query_img = Image.open(uploaded_file)
+
+        # Resize and display the query image
+        query_img_resized = query_img.resize((263, 385))
+        st.image(query_img_resized, caption='Uploaded Image', use_container_width=False)
+
+        # Feature extraction and similarity search
+        if st.button("Search Similar Images"):
+            with st.spinner("Analyzing query image..."):
+                try:
+                    # Initialize feature extractor and search engine
+                    extractor = FeatureExtractor()
+                    search_engine = SimilaritySearchEngine()
+
+                    # Save the uploaded image temporarily
+                    query_img_path = 'temp_query_image.jpg'
+                    query_img.save(query_img_path)
+
+                    # Extract features from the query image
+                    query_embedding = extractor.extract_features(query_img_path)
+
+                    # Perform similarity search
+                    similar_images, distances = search_engine.search_similar_images(query_embedding)
+
+                    # Display similar images
+                    st.subheader('Similar Images')
+                    cols = st.columns(len(similar_images))
+                    for i, (img_path, dist) in enumerate(zip(similar_images, distances)):
+                        with cols[i]:
+                            similar_img = Image.open(img_path).resize((375, 550))
+                            st.image(similar_img, caption=f'Distance: {dist:.2f}', use_container_width=True)
+
+                except Exception as e:
+                    st.error(f"Error during similarity search: {e}")
+
+if __name__ == '__main__':
+    main()

src/preprocessing.py

@@ -0,0 +1,62 @@
+import os
+import pickle
+from .feature_extractor import FeatureExtractor
+import time
+from tqdm import tqdm
+
+def precompute_embeddings(image_dir='data/images', output_path='data/embeddings.pkl'):
+    # Initialize the feature extractor
+    extractor = FeatureExtractor()
+
+    embeddings = []
+    image_paths = []
+
+    # Get total number of valid images
+    valid_images = [f for f in os.listdir(image_dir) 
+                   if f.lower().endswith(('.png', '.jpg', '.jpeg'))]
+    total_images = len(valid_images)
+    
+    print(f"\nFound {total_images} images to process")
+    
+    # Estimate time (assuming ~1 second per image for EfficientNetB0)
+    estimated_time = total_images * 1  # 1 second per image
+    print(f"Estimated time: {estimated_time//60} minutes and {estimated_time%60} seconds\n")
+
+    # Use tqdm for progress bar
+    start_time = time.time()
+    for idx, filename in enumerate(tqdm(valid_images, desc="Processing images")):
+        if filename.endswith(('.png', '.jpg', '.jpeg')):
+            img_path = os.path.join(image_dir, filename)
+            try:
+                # Show current image being processed
+                print(f"\rProcessing image {idx+1}/{total_images}: {filename}", end="")
+                
+                embedding = extractor.extract_features(img_path)
+                embeddings.append(embedding)
+                image_paths.append(img_path)
+                
+                # Calculate and show remaining time
+                elapsed_time = time.time() - start_time
+                avg_time_per_image = elapsed_time / (idx + 1)
+                remaining_images = total_images - (idx + 1)
+                estimated_remaining_time = remaining_images * avg_time_per_image
+                
+                print(f" | Remaining time: {estimated_remaining_time//60:.0f}m {estimated_remaining_time%60:.0f}s")
+                
+            except Exception as e:
+                print(f"\nError processing {filename}: {e}")
+
+    # Save embeddings and paths
+    with open(output_path, 'wb') as f:
+        pickle.dump({'embeddings': embeddings, 'image_paths': image_paths}, f)
+
+    total_time = time.time() - start_time
+    print(f"\nProcessing complete!")
+    print(f"Total time taken: {total_time//60:.0f} minutes and {total_time%60:.0f} seconds")
+    print(f"Successfully processed {len(embeddings)}/{total_images} images")
+    print(f"Embeddings saved to {output_path}")
+    
+    return embeddings, image_paths
+
+if __name__ == "__main__":
+    precompute_embeddings()

src/similarity_search.py

@@ -0,0 +1,22 @@
+import faiss
+import numpy as np
+import pickle
+import os
+
+class SimilaritySearchEngine:
+    def __init__(self, embeddings_path='data/embeddings.pkl'):
+        # Load precomputed embeddings
+        with open(embeddings_path, 'rb') as f:
+            data = pickle.load(f)
+            self.embeddings = data['embeddings']
+            self.image_paths = data['image_paths']
+
+        # Create FAISS index
+        dimension = len(self.embeddings[0])
+        self.index = faiss.IndexFlatL2(dimension)
+        self.index.add(np.array(self.embeddings))
+
+    def search_similar_images(self, query_embedding, top_k=5):
+        # Perform similarity search
+        distances, indices = self.index.search(np.array([query_embedding]), top_k)
+        return [self.image_paths[idx] for idx in indices[0]], distances[0]