Change methode request gpu

app.py

@@ -20,16 +20,24 @@ subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'SimpleITK'])
 # Imports Hugging Face
 from huggingface_hub import hf_hub_download, login
 import spaces
+import numpy as np
+from PIL import Image
+import torch
+import torch.nn.functional as F
+from torchvision import transforms
+from translator import translate_text
 
 # Imports locaux
 from modeling.BaseModel import BaseModel
 from modeling import build_model
-from utilities.distributed import init_distributed
 from utilities.arguments import load_opt_from_config_files
 from utilities.constants import BIOMED_CLASSES
-from inference_utils.inference import interactive_infer_image
+from modeling.language.loss import vl_similarity
 from inference_utils.output_processing import check_mask_stats
-from inference_utils.processing_utils import read_rgb, get_instances
+
+t = []
+t.append(transforms.Resize((1024, 1024), interpolation=Image.BICUBIC))
+transform = transforms.Compose(t)
 
 def init_huggingface():
     """Initialize Hugging Face connection and download the model."""
@@ -45,151 +53,124 @@ def init_huggingface():
     )
     return pretrained_path
 
-def apply_distributed(opt):
-    """Applique les paramètres distribués pour le mode multi-processus."""
-    print(f"Configuration distribuée appliquée : {opt}")
-
+@torch.no_grad()
+@spaces.GPU
 def init_distributed(opt):
-    """Initialize distributed mode without premature CUDA initialization."""
     opt['CUDA'] = opt.get('CUDA', True) and torch.cuda.is_available()
     if 'OMPI_COMM_WORLD_SIZE' not in os.environ:
-        # Application started without MPI
+        # application was started without MPI
+        # default to single node with single process
         opt['env_info'] = 'no MPI'
         opt['world_size'] = 1
         opt['local_size'] = 1
         opt['rank'] = 0
-        opt['local_rank'] = 0  # Ensure this is set to 0
+        opt['local_rank'] = 0
         opt['master_address'] = '127.0.0.1'
         opt['master_port'] = '8673'
     else:
-        # Application started with MPI
+        # application was started with MPI
+        # get MPI parameters
         opt['world_size'] = int(os.environ['OMPI_COMM_WORLD_SIZE'])
         opt['local_size'] = int(os.environ['OMPI_COMM_WORLD_LOCAL_SIZE'])
         opt['rank'] = int(os.environ['OMPI_COMM_WORLD_RANK'])
         opt['local_rank'] = int(os.environ['OMPI_COMM_WORLD_LOCAL_RANK'])
 
+    # set up device
     if not opt['CUDA']:
-        assert opt['world_size'] == 1, 'Multi-GPU training without CUDA is not supported since we use NCCL as communication backend'
+        assert opt['world_size'] == 1, 'multi-GPU training without CUDA is not supported since we use NCCL as communication backend'
         opt['device'] = torch.device("cpu")
     else:
-        opt['device'] = torch.device("cuda", opt['local_rank'])  # Ensure local_rank is integer
+        torch.cuda.set_device(opt['local_rank'])
+        opt['device'] = torch.device("cuda", opt['local_rank'])
 
-    apply_distributed(opt)
     return opt
 
 def setup_model():
     """Initialize the model on CPU without CUDA initialization."""
     opt = load_opt_from_config_files(["configs/biomedparse_inference.yaml"])
     opt = init_distributed(opt)
-    opt['device'] = 'cpu'
-    
     pretrained_path = init_huggingface()
-    model = BaseModel(opt, build_model(opt))
-    state_dict = torch.load(pretrained_path, map_location='cpu', weights_only=True)
-    model.load_state_dict(state_dict, strict=False)
-    
-    # Initialize train_class_names
-    model.train_class_names = BIOMED_CLASSES + ["background"]
-    
-    return model.eval()
-
-import numpy as np
-from PIL import Image
+    model = BaseModel(opt, build_model(opt)).from_pretrained(pretrained_path).eval()
+    return model
 
-def preprocess_image(image):
-    """Preprocess image for SEEM model input."""
-    if isinstance(image, Image.Image):
-        # Convert PIL Image to numpy array
-        image = np.array(image)
-    
-    # Ensure image is float32 and normalized
-    image = image.astype(np.float32) / 255.0
+@torch.no_grad()
+@spaces.GPU 
+def predict_image(model, image, prompts):
+    model = model.cuda()
+    print("====================== Model moved to GPU via predict_image ======================")
+    prompts = [translate_text(p, "fr", "en") for p in prompts]
+    # Initialiser les embeddings textuels avant l'évaluation
+    lang_encoder = model.model.sem_seg_head.predictor.lang_encoder
+    lang_encoder.get_text_embeddings(BIOMED_CLASSES + ["background"], is_eval=True)   
+    model.model.sem_seg_head.predictor.lang_encoder.get_text_embeddings(BIOMED_CLASSES + ["background"], is_eval=True)  
+    # Préparer l'image
+    image_resize = transform(image)
+    width = image.size[0]
+    height = image.size[1]
+    image_resize = np.asarray(image_resize)
+    image = torch.from_numpy(image_resize.copy()).permute(2,0,1).cuda()
+
+    # Préparer les données d'entrée
+    data = {"image": image, 'text': prompts, "height": height, "width": width}
     
-    # Ensure correct dimensions (B, C, H, W)
-    if len(image.shape) == 3:
-        image = np.transpose(image, (2, 0, 1))  # HWC -> CHW
-        image = np.expand_dims(image, axis=0)  # Add batch dimension
+    # Configurer les tâches
+    model.model.task_switch['spatial'] = False
+    model.model.task_switch['visual'] = False
+    model.model.task_switch['grounding'] = True
+    model.model.task_switch['audio'] = False
+
+    # Évaluation
+    batch_inputs = [data]
+    results, image_size, extra = model.model.evaluate_demo(batch_inputs)
+
+    # Traitement des prédictions
+    pred_masks = results['pred_masks'][0]
+    v_emb = results['pred_captions'][0]
+    t_emb = extra['grounding_class']
+
+    # Normalisation
+    t_emb = t_emb / (t_emb.norm(dim=-1, keepdim=True) + 1e-7)
+    v_emb = v_emb / (v_emb.norm(dim=-1, keepdim=True) + 1e-7)
+
+    # Calcul de similarité
+    temperature = lang_encoder.logit_scale
+    out_prob = vl_similarity(v_emb, t_emb, temperature=temperature)
     
-    return image
+    # Sélection des masques
+    matched_id = out_prob.max(0)[1]
+    pred_masks_pos = pred_masks[matched_id,:,:]
 
-@spaces.GPU 
-def predict_image(model, image, prompts):
-    """Process image prediction with proper formatting."""
-    try:
-        # Convert PIL Image to numpy array if needed
-        if isinstance(image, Image.Image):
-            image = np.array(image)
-            
-        # Ensure image is in float32 and normalized
-        image = image.astype(np.float32) / 255.0
-        
-        # Transpose from HWC to CHW format
-        if len(image.shape) == 3:
-            image = np.transpose(image, (2, 0, 1))
-            
-        # Add batch dimension if needed
-        if len(image.shape) == 3:
-            image = np.expand_dims(image, axis=0)
-            
-        # Convert to tensor
-        image_tensor = torch.from_numpy(image)
-        
-        # Move to GPU if available
-        if torch.cuda.is_available():
-            device = torch.device("cuda", 0)
-            model = model.to(device)
-            image_tensor = image_tensor.to(device)
-        else:
-            device = torch.device("cpu")
-            
-        # Create batched input
-        batched_inputs = [{
-            "image": image_tensor,
-            "prompt": prompts,
-            "height": image_tensor.shape[-2],
-            "width": image_tensor.shape[-1]
-        }]
-        
-        with torch.no_grad():
-            pred_masks = model(batched_inputs)
-            
-        # Move back to CPU if needed
-        if device.type == "cuda":
-            model = model.to("cpu")
-            pred_masks = [mask.cpu() for mask in pred_masks]
-                
-        return pred_masks
-            
-    except Exception as e:
-        print(f"Error processing image: {str(e)}")
-        raise
+    # Redimensionnement à la taille d'origine
+    pred_mask_prob = F.interpolate(pred_masks_pos[None,], (data['height'], data['width']), 
+                                   mode='bilinear')[0,:,:data['height'],:data['width']].sigmoid().cpu().detach().numpy()
+    return pred_mask_prob
 
-def process_image(image, text, model):
+def process_image(image, prompts, model):
     """Process image with proper error handling."""
     try:
-        prompts = [p.strip() for p in text.split(',') if p.strip()]
+        if isinstance(image, str):
+            image = Image.open(image)
+        else:
+            image = Image.fromarray(image)
+
+        prompts = [p.strip() for p in prompts.split(',')]
+
         if not prompts:
             raise ValueError("No valid prompts provided")
             
         pred_masks = predict_image(model, image, prompts)
-        
-        # Create visualization
-        fig = plt.figure(figsize=(5 * (len(pred_masks) + 1), 5))
-        
-        # Show original image
+        fig = plt.figure(figsize=(10, 5))
         plt.subplot(1, len(pred_masks) + 1, 1)
-        plt.imshow(preprocess_image(image))
-        plt.title("Original")
+        plt.imshow(image)
+        plt.title('Image originale')
         plt.axis('off')
-        
-        # Show predictions
+
         for i, mask in enumerate(pred_masks):
-            plt.subplot(1, len(pred_masks) + 1, i+2)
-            plt.imshow(preprocess_image(image))
-            plt.imshow(mask.cpu().numpy(), alpha=0.5, cmap='Reds')
+            plt.subplot(1, len(pred_masks) + 1, i + 2)
+            plt.imshow(image)
+            plt.imshow(mask, alpha=0.5, cmap='Reds')
             plt.title(prompts[i])
             plt.axis('off')
-        
         return fig
         
     except Exception as e:
@@ -199,6 +180,7 @@ def process_image(image, text, model):
 def setup_gradio_interface(model):
     """Configure l'interface Gradio."""
     return gr.Interface(
+        theme=gr.Theme.from_hub("JohnSmith9982/small_and_pretty"),
         fn=lambda img, txt: process_image(img, txt, model),
         inputs=[
             gr.Image(type="numpy", label="Image médicale"),
@@ -209,8 +191,12 @@ def setup_gradio_interface(model):
             )
         ],
         outputs=gr.Plot(),
-        title="Core IA - Traitement d'image medicale",
-        description="Chargez une image médicale et spécifiez les éléments à segmenter",
+        title=" 🇬🇦 Core IA - Traitement d'image medicale",
+        description="""Chargez une image médicale de type (IRM , Echographie,  ) et spécifiez les éléments à segmenter : Les cas d’utilisation incluent des tâches variées d’analyse d’images médicales. En imagerie CT, le modèle peut détecter et segmenter des organes et pathologies dans des régions telles que l’abdomen, le côlon, le foie, les poumons ou le bassin. Avec l’IRM, il est capable de traiter des structures abdominales, cérébrales, cardiaques et prostatiques selon les différentes séquences (FLAIR, T1-Gd, T2). En radiographie, il peut identifier des anomalies pulmonaires ou des infections liées à la COVID-19.
+
+Le modèle couvre également d’autres modalités comme la dermoscopie, l’endoscopie, le fond d’œil, et la pathologie, avec des applications pour la détection de lésions, de polypes ou de cellules néoplasiques dans divers tissus et organes. En échographie, il identifie des anomalies mammaires, cardiaques ou fœtales. Enfin, en TOCT, il peut analyser des structures rétiniennes.
+
+Bien que performant dans ces contextes, il est important de considérer les spécificités des ensembles de données externes et de procéder à un ajustement pour des résultats précis.""",
         examples=[
             ["examples/144DME_as_F.jpeg", "Dans cette image donne moi l'œdème"],
             ["examples/T0011.jpg", "disque optique, cupule optique"],
@@ -224,7 +210,7 @@ def main():
     """Entry point avoiding CUDA initialization in main process."""
     try:
         init_huggingface()
-        model = setup_model()  # Load on CPU
+        model = setup_model() 
         interface = setup_gradio_interface(model)
         interface.launch(debug=True)
     except Exception as e:

translator.py

@@ -0,0 +1,38 @@
+import os
+import requests
+
+def translate_text(text, source_lang, target_lang):
+    url = "https://google-translator9.p.rapidapi.com/v2"
+    payload = {
+        "q": text,
+        "source": source_lang,
+        "target": target_lang,
+        "format": "text"
+    }
+    headers = {
+        "x-rapidapi-key": os.getenv("RAPIDAPI_KEY"),
+        "x-rapidapi-host": "google-translator9.p.rapidapi.com",
+        "Content-Type": "application/json"
+    }
+
+    try:
+        response = requests.post(url, json=payload, headers=headers)
+        response.raise_for_status()
+        print('=====================', response.json())
+        translations = response.json().get('data', {}).get('translations', [])
+        if translations:
+            translated_text = translations[0].get('translatedText', '')
+            return translated_text
+        else:
+            return text
+    except requests.exceptions.RequestException as e:
+        print(f"An error occurred: {e}")
+        return text
+
+if __name__ == "__main__":
+    text_to_translate = "Dans cette image donne moi l'œdème"
+    source_language = "fr"
+    target_language = "en"
+    translation = translate_text(text_to_translate, source_language, target_language)
+    if translation:
+        print(translation)

utilities/distributed.py

@@ -30,7 +30,6 @@ def apply_distributed(opt):
                                              init_method=init_method_url,
                                              world_size=world_size,
                                              rank=rank)
-
 def init_distributed(opt):
     opt['CUDA'] = opt.get('CUDA', True) and torch.cuda.is_available()
     if 'OMPI_COMM_WORLD_SIZE' not in os.environ: