Upload HfMoondream

image_crops.py

@@ -1,10 +1,18 @@
 import math
 import numpy as np
 import torch
-import pyvips
 
 from typing import TypedDict
 
+try:
+    import pyvips
+
+    HAS_VIPS = True
+except:
+    from PIL import Image
+
+    HAS_VIPS = False
+
 
 def select_tiling(
     height: int, width: int, crop_size: int, max_crops: int
@@ -113,18 +121,33 @@ def overlap_crop_image(
         tiling[1] * crop_window_size + total_margin_pixels,
     )
 
-    # Convert to vips for resizing
-    vips_image = pyvips.Image.new_from_array(image)
-    scale_x = target_size[1] / image.shape[1]
-    scale_y = target_size[0] / image.shape[0]
-    resized = vips_image.resize(scale_x, vscale=scale_y)
-    image = resized.numpy()
-
-    # Create global crop
-    scale_x = base_size[1] / vips_image.width
-    scale_y = base_size[0] / vips_image.height
-    global_vips = vips_image.resize(scale_x, vscale=scale_y)
-    crops[0] = global_vips.numpy()
+    if HAS_VIPS:
+        # Convert to vips for resizing
+        vips_image = pyvips.Image.new_from_array(image)
+        scale_x = target_size[1] / image.shape[1]
+        scale_y = target_size[0] / image.shape[0]
+        resized = vips_image.resize(scale_x, vscale=scale_y)
+        image = resized.numpy()
+
+        # Create global crop
+        scale_x = base_size[1] / vips_image.width
+        scale_y = base_size[0] / vips_image.height
+        global_vips = vips_image.resize(scale_x, vscale=scale_y)
+        crops[0] = global_vips.numpy()
+    else:
+        # Fallback to PIL
+        pil_img = Image.fromarray(image)
+        resized = pil_img.resize(
+            (int(target_size[1]), int(target_size[0])),
+            resample=Image.Resampling.LANCZOS,
+        )
+        image = np.asarray(resized)
+
+        # Create global crop
+        global_pil = pil_img.resize(
+            (int(base_size[1]), int(base_size[0])), resample=Image.Resampling.LANCZOS
+        )
+        crops[0] = np.asarray(global_pil)
 
     for i in range(tiling[0]):
         for j in range(tiling[1]):

moondream.py

@@ -182,6 +182,7 @@ class MoondreamModel(nn.Module):
 
     def _run_vision_encoder(self, image: Image.Image) -> torch.Tensor:
         all_crops, tiling = prepare_crops(image, self.config.vision, device=self.device)
+
         torch._dynamo.mark_dynamic(all_crops, 0)
 
         outputs = self._vis_enc(all_crops)
@@ -249,6 +250,7 @@ class MoondreamModel(nn.Module):
         with torch.inference_mode():
             prompt_emb = text_encoder(prompt_tokens, self.text)
             torch._dynamo.mark_dynamic(prompt_emb, 1)
+
             mask = self.attn_mask[:, :, pos : pos + prompt_emb.size(1), :]
             pos_ids = torch.arange(pos, pos + prompt_emb.size(1), dtype=torch.long)
             hidden = self._prefill(prompt_emb, mask, pos_ids)

text.py

@@ -35,18 +35,6 @@ def attn(
     k = k.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2)
     v = v.view(bsz, q_len, n_kv_heads, head_dim).transpose(1, 2)
 
-    # q = qkv_out[..., :q_dim].view(bsz, q_len, n_heads, head_dim).transpose(1, 2)
-    # k = (
-    #     qkv_out[..., q_dim : q_dim + kv_dim]
-    #     .view(bsz, q_len, n_kv_heads, head_dim)
-    #     .transpose(1, 2)
-    # )
-    # v = (
-    #     qkv_out[..., q_dim + kv_dim :]
-    #     .view(bsz, q_len, n_kv_heads, head_dim)
-    #     .transpose(1, 2)
-    # )
-
     q = apply_rotary_emb(q, freqs_cis, position_ids, n_heads)
     k = apply_rotary_emb(k, freqs_cis, position_ids, n_kv_heads)