Upload Vision-GPT model (FP32 and FP16 versions)

.gitattributes

@@ -1,35 +1,4 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
+# Git LFS tracking
 *.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,189 @@
+---
+language:
+- en
+license: mit
+tags:
+- vision
+- image-to-text
+- image-captioning
+- gpt2
+- vision-transformer
+- flickr8k
+pipeline_tag: image-to-text
+datasets:
+- jxie/flickr8k
+---
+
+# Vision-GPT: Image Captioning Model
+
+A multimodal model combining Vision Transformer (ViT-B/16) and GPT-2 for image captioning, trained on Flickr8K dataset.
+
+## Model Description
+
+This model generates natural language captions for images by:
+1. Encoding images using a pre-trained ViT-B/16 vision encoder
+2. Projecting visual features into GPT-2's embedding space
+3. Generating captions autoregressively with GPT-2
+
+## Training Details
+
+- **Dataset**: Flickr8K (all splits: train, validation, test)
+- **Vision Encoder**: ViT-B/16 (frozen)
+- **Language Model**: GPT-2 (frozen backbone, trainable projection)
+- **Training**: Only the vision-to-text projection layer is trained
+
+## Model Versions
+
+### 📦 FP32 (Full Precision)
+- **Size**: ~0.83 GB
+- **Precision**: 32-bit floating point
+- **Use case**: Maximum accuracy, research
+
+### 📦 FP16 (Half Precision)
+- **Size**: ~0.83 GB
+- **Precision**: 16-bit floating point
+- **Use case**: Faster inference, reduced memory (~50% smaller)
+
+## Usage
+
+### Installation
+
+```bash
+pip install torch torchvision transformers pillow huggingface_hub
+```
+
+### Loading the Model (FP32)
+
+```python
+import torch
+from transformers import GPT2Tokenizer
+from PIL import Image
+from torchvision import transforms
+
+# Load checkpoint
+checkpoint = torch.load("model_fp32/model_checkpoint.pth", map_location="cpu")
+
+# Load tokenizer
+tokenizer = GPT2Tokenizer.from_pretrained("model_fp32/tokenizer")
+
+# Load your model architecture (you need to define this)
+# model = YourVisionGPTModel(config)
+# model.load_state_dict(checkpoint['model_state_dict'])
+# model.eval()
+
+print("Model loaded successfully!")
+```
+
+### Loading the Model (FP16)
+
+```python
+# Load FP16 checkpoint
+checkpoint = torch.load("model_fp16/model_checkpoint.pth", map_location="cpu")
+
+# Load model and convert to FP16
+# model = YourVisionGPTModel(config)
+# model.load_state_dict(checkpoint['model_state_dict'])
+# model.half()  # Convert to FP16
+# model.eval()
+
+# For inference with FP16, also convert input images to FP16
+```
+
+### Image Preprocessing
+
+```python
+image_transform = transforms.Compose([
+    transforms.Resize((224, 224)),
+    transforms.Lambda(lambda x: x.convert('RGB')),
+    transforms.ToTensor(),
+    transforms.Normalize(
+        mean=[0.485, 0.456, 0.406],
+        std=[0.229, 0.224, 0.225]
+    ),
+])
+
+# Load and preprocess image
+image = Image.open("your_image.jpg")
+image_tensor = image_transform(image).unsqueeze(0)  # Add batch dimension
+```
+
+### Generate Caption
+
+```python
+# Generate caption
+with torch.no_grad():
+    # Forward pass
+    generated_ids = model.generate(
+        image_tensor,
+        max_length=50,
+        num_beams=5,
+        temperature=0.7
+    )
+    
+    # Decode caption
+    caption = tokenizer.decode(generated_ids[0], skip_special_tokens=True)
+    print(f"Generated caption: {caption}")
+```
+
+## Model Architecture
+
+```
+┌─────────────────┐
+│  Input Image    │
+│   (224x224)     │
+└────────┬────────┘
+         │
+         ▼
+┌─────────────────┐
+│   ViT-B/16      │
+│   (frozen)      │
+└────────┬────────┘
+         │
+         ▼
+┌─────────────────┐
+│  Projection     │
+│  (trainable)    │
+└────────┬────────┘
+         │
+         ▼
+┌─────────────────┐
+│    GPT-2        │
+│   (frozen)      │
+└────────┬────────┘
+         │
+         ▼
+┌─────────────────┐
+│  Caption Output │
+└─────────────────┘
+```
+
+## Limitations
+
+- Trained only on Flickr8K (limited domain)
+- English captions only
+- Input images must be 224x224
+- May generate generic captions for out-of-domain images
+
+## Citation
+
+If you use this model, please cite:
+
+```bibtex
+@misc{vision-gpt-flickr8k,
+  author = {gurumurthy3},
+  title = {Vision-GPT: Image Captioning with ViT and GPT-2},
+  year = {2025},
+  publisher = {Hugging Face},
+  howpublished = {\url{https://huggingface.co/gurumurthy3/vision-gpt-flickr8k}}
+}
+```
+
+## License
+
+MIT License
+
+## Acknowledgments
+
+- Vision Transformer (ViT): Dosovitskiy et al.
+- GPT-2: OpenAI
+- Flickr8K Dataset: Hodosh et al.

model_fp16/config.json

@@ -0,0 +1,24 @@
+{
+  "vocab_size": 50258,
+  "emb_dim": 768,
+  "context_length": 256,
+  "n_heads": 12,
+  "n_layers": 12,
+  "drop_rate": 0.1,
+  "qkv_bias": true,
+  "cross_attention_pos": [
+    3,
+    6,
+    9
+  ],
+  "vision_enabled": true,
+  "vision_encoder_type": "torchvision_vit_b_16",
+  "vision_pretrained": true,
+  "vision_freeze": true,
+  "perceiver_num_latents": 64,
+  "perceiver_depth": 2,
+  "perceiver_heads": 8,
+  "perceiver_dim_head": 64,
+  "weight_decay": 0.01,
+  "learning_rate": 0.0001
+}

model_fp16/model_checkpoint.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d0a5fe0249503f0deecd8c319a1133c010cadef4294d4e301970eb7498de357
+size 891677724

model_fp16/tokenizer/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<|pad|>": 50257
+}

model_fp16/tokenizer/special_tokens_map.json

@@ -0,0 +1,12 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": "<|endoftext|>"
+}

model_fp16/tokenizer/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50257": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

model_fp32/config.json

@@ -0,0 +1,24 @@
+{
+  "vocab_size": 50258,
+  "emb_dim": 768,
+  "context_length": 256,
+  "n_heads": 12,
+  "n_layers": 12,
+  "drop_rate": 0.1,
+  "qkv_bias": true,
+  "cross_attention_pos": [
+    3,
+    6,
+    9
+  ],
+  "vision_enabled": true,
+  "vision_encoder_type": "torchvision_vit_b_16",
+  "vision_pretrained": true,
+  "vision_freeze": true,
+  "perceiver_num_latents": 64,
+  "perceiver_depth": 2,
+  "perceiver_heads": 8,
+  "perceiver_dim_head": 64,
+  "weight_decay": 0.01,
+  "learning_rate": 0.0001
+}

model_fp32/model_checkpoint.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d0a5fe0249503f0deecd8c319a1133c010cadef4294d4e301970eb7498de357
+size 891677724

model_fp32/tokenizer/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<|pad|>": 50257
+}

model_fp32/tokenizer/special_tokens_map.json

@@ -0,0 +1,12 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": "<|endoftext|>"
+}

model_fp32/tokenizer/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50257": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}