""" Utility functions for the Universal Image Classifier """ import torch import torchvision.transforms as transforms from torch.utils.data import Dataset, DataLoader from PIL import Image import os import json import matplotlib.pyplot as plt import seaborn as sns from sklearn.metrics import confusion_matrix, classification_report import numpy as np from typing import List, Dict, Tuple, Optional from pathlib import Path import pandas as pd def load_dataset_from_folder( data_dir: str, class_names: Optional[List[str]] = None, valid_extensions: Tuple[str, ...] = ('.jpg', '.jpeg', '.png', '.bmp', '.tiff') ) -> Tuple[List[str], List[int], List[str]]: """ Load dataset from folder structure: data_dir/ ├── class1/ │ ├── img1.jpg │ └── img2.jpg └── class2/ ├── img3.jpg └── img4.jpg Returns: image_paths: List of image file paths labels: List of corresponding class indices class_names: List of class names """ data_path = Path(data_dir) if not data_path.exists(): raise ValueError(f"Data directory {data_dir} does not exist") # Get class directories class_dirs = [d for d in data_path.iterdir() if d.is_dir()] class_dirs.sort() if class_names is None: class_names = [d.name for d in class_dirs] image_paths = [] labels = [] for class_idx, class_dir in enumerate(class_dirs): class_name = class_dir.name if class_name not in class_names: continue class_label = class_names.index(class_name) # Get all image files in this class directory for img_file in class_dir.iterdir(): if img_file.suffix.lower() in valid_extensions: image_paths.append(str(img_file)) labels.append(class_label) print(f"Loaded {len(image_paths)} images from {len(class_names)} classes") for i, class_name in enumerate(class_names): count = labels.count(i) print(f" {class_name}: {count} images") return image_paths, labels, class_names def load_dataset_from_csv( csv_file: str, image_col: str = 'image_path', label_col: str = 'label', class_names: Optional[List[str]] = None ) -> Tuple[List[str], List[int], List[str]]: """ Load dataset from CSV file with columns for image paths and labels Args: csv_file: Path to CSV file image_col: Column name for image paths label_col: Column name for labels class_names: Optional list of class names Returns: image_paths: List of image file paths labels: List of corresponding class indices class_names: List of class names """ df = pd.read_csv(csv_file) image_paths = df[image_col].tolist() # Handle both string and numeric labels if df[label_col].dtype == 'object': # String labels unique_labels = sorted(df[label_col].unique()) if class_names is None: class_names = unique_labels label_to_idx = {label: idx for idx, label in enumerate(class_names)} labels = [label_to_idx[label] for label in df[label_col]] else: # Numeric labels labels = df[label_col].tolist() if class_names is None: class_names = [f"class_{i}" for i in range(max(labels) + 1)] print(f"Loaded {len(image_paths)} images from CSV") return image_paths, labels, class_names def visualize_dataset( image_paths: List[str], labels: List[int], class_names: List[str], num_samples: int = 16, figsize: Tuple[int, int] = (12, 8) ): """Visualize random samples from the dataset""" import random # Sample random images indices = random.sample(range(len(image_paths)), min(num_samples, len(image_paths))) rows = int(np.sqrt(num_samples)) cols = int(np.ceil(num_samples / rows)) fig, axes = plt.subplots(rows, cols, figsize=figsize) axes = axes.flatten() if num_samples > 1 else [axes] for i, idx in enumerate(indices): if i >= len(axes): break img = Image.open(image_paths[idx]).convert('RGB') axes[i].imshow(img) axes[i].set_title(f"{class_names[labels[idx]]}") axes[i].axis('off') # Hide unused subplots for i in range(len(indices), len(axes)): axes[i].axis('off') plt.tight_layout() plt.show() def plot_class_distribution(labels: List[int], class_names: List[str]): """Plot the distribution of classes in the dataset""" class_counts = [labels.count(i) for i in range(len(class_names))] plt.figure(figsize=(10, 6)) bars = plt.bar(class_names, class_counts) plt.title('Class Distribution') plt.xlabel('Classes') plt.ylabel('Number of Images') plt.xticks(rotation=45) # Add count labels on bars for bar, count in zip(bars, class_counts): plt.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.1, str(count), ha='center', va='bottom') plt.tight_layout() plt.show() def evaluate_model( model: torch.nn.Module, test_loader: DataLoader, class_names: List[str], device: str = "cpu" ) -> Dict: """Evaluate model performance and generate metrics""" model.eval() all_predictions = [] all_labels = [] with torch.no_grad(): for images, labels in test_loader: images, labels = images.to(device), labels.to(device) outputs = model(images) _, predicted = torch.max(outputs, 1) all_predictions.extend(predicted.cpu().numpy()) all_labels.extend(labels.cpu().numpy()) # Calculate metrics accuracy = np.mean(np.array(all_predictions) == np.array(all_labels)) # Classification report report = classification_report( all_labels, all_predictions, target_names=class_names, output_dict=True ) return { 'accuracy': accuracy, 'predictions': all_predictions, 'true_labels': all_labels, 'classification_report': report } def plot_confusion_matrix( true_labels: List[int], predictions: List[int], class_names: List[str], figsize: Tuple[int, int] = (8, 6) ): """Plot confusion matrix""" cm = confusion_matrix(true_labels, predictions) plt.figure(figsize=figsize) sns.heatmap( cm, annot=True, fmt='d', cmap='Blues', xticklabels=class_names, yticklabels=class_names ) plt.title('Confusion Matrix') plt.xlabel('Predicted') plt.ylabel('Actual') plt.tight_layout() plt.show() def save_predictions( predictions: List[int], image_paths: List[str], class_names: List[str], output_file: str ): """Save predictions to CSV file""" df = pd.DataFrame({ 'image_path': image_paths, 'predicted_class_id': predictions, 'predicted_class_name': [class_names[pred] for pred in predictions] }) df.to_csv(output_file, index=False) print(f"Predictions saved to {output_file}") def create_model_card( model_name: str, model_config: Dict, training_config: Dict, performance_metrics: Dict, class_names: List[str], dataset_info: str, output_file: str = "MODEL_CARD.md" ): """Create a model card for Hugging Face""" card_content = f"""--- license: mit tags: - image-classification - pytorch - computer-vision datasets: - custom metrics: - accuracy library_name: pytorch --- # Universal Image Classifier - {model_name} ## Model Description This is a {model_name} model trained using the Universal Image Classifier framework. It's designed for flexible image classification tasks and can be easily adapted to different datasets. ## Model Architecture - **Model Type**: {model_name} - **Input Size**: {model_config.get('input_height', 64)}x{model_config.get('input_width', 64)} - **Number of Classes**: {model_config.get('num_classes', len(class_names))} - **Hidden Dimensions**: {model_config.get('hidden_dim', 128)} - **Number of Layers**: {model_config.get('num_layers', 4)} - **Dropout Rate**: {model_config.get('dropout_rate', 0.1)} - **Batch Normalization**: {model_config.get('use_batch_norm', True)} ## Training Configuration - **Batch Size**: {training_config.get('batch_size', 32)} - **Learning Rate**: {training_config.get('learning_rate', 0.001)} - **Weight Decay**: {training_config.get('weight_decay', 1e-4)} - **Early Stopping Patience**: {training_config.get('early_stopping_patience', 10)} ## Performance - **Accuracy**: {performance_metrics.get('accuracy', 'N/A'):.4f} - **Classes**: {', '.join(class_names)} ## Dataset {dataset_info} ## Usage ```python from inference import load_classifier # Load the model classifier = load_classifier( model_path="path/to/model.pth", config_path="path/to/config.json", class_names={class_names} ) # Make predictions result = classifier.predict("path/to/image.jpg") print(f"Predicted: {{result['predicted_class']}} ({{result['confidence']:.3f}})") ``` ## Training This model was trained using the Universal Image Classifier framework. To train your own model: ```python from train import UniversalTrainer from config import ModelConfig, TrainingConfig, DataConfig # Configure and train trainer = UniversalTrainer(model_config, training_config, data_config) trainer.train(train_dataset, val_dataset, model_name="{model_name}") ``` ## Framework Built with the Universal Image Classifier framework - a flexible PyTorch-based solution for image classification tasks. ## License MIT License """ with open(output_file, 'w') as f: f.write(card_content) print(f"Model card saved to {output_file}") def export_to_huggingface( model_path: str, config_path: str, class_names: List[str], model_name: str, output_dir: str = "huggingface_model" ): """Export model for Hugging Face Hub""" output_path = Path(output_dir) output_path.mkdir(exist_ok=True) # Copy model files import shutil shutil.copy(model_path, output_path / "pytorch_model.pth") shutil.copy(config_path, output_path / "config.json") # Create class names file with open(output_path / "class_names.json", 'w') as f: json.dump(class_names, f, indent=2) # Create inference script inference_script = ''' from inference import UniversalImageClassifier import json def load_model(): """Load the model for inference""" with open("class_names.json", "r") as f: class_names = json.load(f) classifier = UniversalImageClassifier( model_path="pytorch_model.pth", config_path="config.json", class_names=class_names ) return classifier def predict(image_path): """Predict image class""" classifier = load_model() return classifier.predict(image_path) ''' with open(output_path / "inference_hf.py", 'w') as f: f.write(inference_script) print(f"Model exported to {output_path} for Hugging Face Hub")