Update app.py (#3)

- Update app.py (85ff0541fcb00e1d44afff004d05c4e103e21599)

app.py

@@ -1,14 +1,12 @@
-import gradio as gr
 import spaces
+import gradio as gr
 import torch
 from diffusers import AutoencoderKL, TCDScheduler
 from diffusers.models.model_loading_utils import load_state_dict
 from gradio_imageslider import ImageSlider
 from huggingface_hub import hf_hub_download
-
 from controlnet_union import ControlNetModel_Union
 from pipeline_fill_sd_xl import StableDiffusionXLFillPipeline
-
 from PIL import Image, ImageDraw
 import numpy as np
 
@@ -23,7 +21,6 @@ config_file = hf_hub_download(
     "xinsir/controlnet-union-sdxl-1.0",
     filename="config_promax.json",
 )
-
 config = ControlNetModel_Union.load_config(config_file)
 controlnet_model = ControlNetModel_Union.from_config(config)
 model_file = hf_hub_download(
@@ -35,11 +32,9 @@ model, _, _, _, _ = ControlNetModel_Union._load_pretrained_model(
     controlnet_model, state_dict, model_file, "xinsir/controlnet-union-sdxl-1.0"
 )
 model.to(device="cuda", dtype=torch.float16)
-
 vae = AutoencoderKL.from_pretrained(
     "madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16
 ).to("cuda")
-
 pipe = StableDiffusionXLFillPipeline.from_pretrained(
     "SG161222/RealVisXL_V5.0_Lightning",
     torch_dtype=torch.float16,
@@ -47,34 +42,21 @@ pipe = StableDiffusionXLFillPipeline.from_pretrained(
     controlnet=model,
     variant="fp16",
 )
-
 pipe = StableDiffusionXLFillPipeline.from_pretrained(
     "GraydientPlatformAPI/lustify-lightning",
     torch_dtype=torch.float16,
     vae=vae,
     controlnet=model,
 )
-
 pipe.scheduler = TCDScheduler.from_config(pipe.scheduler.config)
-
 pipe.to("cuda")
 
-# inpaint_model = hf_hub_download(
-#     "andro-flock/LUSTIFY-SDXL-NSFW-checkpoint-v2-0-INPAINTING",
-#     "lustifySDXLNSFW_v20-inpainting.safetensors",
-# )
-# pipe_inpaint = StableDiffusionXLFillPipeline.from_single_file(
-#     "https://huggingface.co/andro-flock/LUSTIFY-SDXL-NSFW-checkpoint-v2-0-INPAINTING/raw/main/lustifySDXLNSFW_v20-inpainting.safetensors",
-#     torch_dtype=torch.float16,
-#     vae=vae,
-#     controlnet=model,
-#     use_safetensors=True
-# )
-# pipe_inpaint.to("cuda")
-
 @spaces.GPU(duration=12)
 def fill_image(prompt, image, model_selection, paste_back):
-    print(f"Received image: {image}")  # Debugging statement
+    print(f"Received image: {image}")
+    if image is None:
+        yield None, None
+        return
 
     (
         prompt_embeds,
@@ -82,10 +64,8 @@ def fill_image(prompt, image, model_selection, paste_back):
         pooled_prompt_embeds,
         negative_pooled_prompt_embeds,
     ) = pipe.encode_prompt(prompt, "cuda", True)
-
     source = image["background"]
     mask = image["layers"][0]
-
     alpha_channel = mask.split()[3]
     binary_mask = alpha_channel.point(lambda p: p > 0 and 255)
     cnet_image = source.copy()
@@ -102,21 +82,17 @@ def fill_image(prompt, image, model_selection, paste_back):
 
     print(f"{model_selection=}")
     print(f"{paste_back=}")
-
     if paste_back:
         image = image.convert("RGBA")
         cnet_image.paste(image, (0, 0), binary_mask)
     else:
         cnet_image = image
-
     yield source, cnet_image
 
-
 def clear_result():
     return gr.update(value=None)
-    
+
 def can_expand(source_width, source_height, target_width, target_height, alignment):
-    """Checks if the image can be expanded based on the alignment."""
     if alignment in ("Left", "Right") and source_width >= target_width:
         return False
     if alignment in ("Top", "Bottom") and source_height >= target_height:
@@ -125,16 +101,11 @@ def can_expand(source_width, source_height, target_width, target_height, alignme
 
 def prepare_image_and_mask(image, width, height, overlap_percentage, resize_option, custom_resize_percentage, alignment, overlap_left, overlap_right, overlap_top, overlap_bottom):
     target_size = (width, height)
-
-    # Calculate the scaling factor to fit the image within the target size
     scale_factor = min(target_size[0] / image.width, target_size[1] / image.height)
     new_width = int(image.width * scale_factor)
     new_height = int(image.height * scale_factor)
-    
-    # Resize the source image to fit within target size
-    source = image.resize((new_width, new_height), Image.LANCZOS)
 
-    # Apply resize option using percentages
+    source = image.resize((new_width, new_height), Image.LANCZOS)
     if resize_option == "Full":
         resize_percentage = 100
     elif resize_option == "80%":
@@ -148,27 +119,19 @@ def prepare_image_and_mask(image, width, height, overlap_percentage, resize_opti
     else:  # Custom
         resize_percentage = custom_resize_percentage
 
-    # Calculate new dimensions based on percentage
     resize_factor = resize_percentage / 100
     new_width = int(source.width * resize_factor)
     new_height = int(source.height * resize_factor)
-
-    # Ensure minimum size of 64 pixels
     new_width = max(new_width, 64)
     new_height = max(new_height, 64)
 
-    # Resize the image
     source = source.resize((new_width, new_height), Image.LANCZOS)
 
-    # Calculate the overlap in pixels based on the percentage
     overlap_x = int(new_width * (overlap_percentage / 100))
     overlap_y = int(new_height * (overlap_percentage / 100))
-
-    # Ensure minimum overlap of 1 pixel
     overlap_x = max(overlap_x, 1)
     overlap_y = max(overlap_y, 1)
 
-    # Calculate margins based on alignment
     if alignment == "Middle":
         margin_x = (target_size[0] - new_width) // 2
         margin_y = (target_size[1] - new_height) // 2
@@ -185,26 +148,21 @@ def prepare_image_and_mask(image, width, height, overlap_percentage, resize_opti
         margin_x = (target_size[0] - new_width) // 2
         margin_y = target_size[1] - new_height
 
-    # Adjust margins to eliminate gaps
     margin_x = max(0, min(margin_x, target_size[0] - new_width))
     margin_y = max(0, min(margin_y, target_size[1] - new_height))
 
-    # Create a new background image and paste the resized source image
     background = Image.new('RGB', target_size, (255, 255, 255))
     background.paste(source, (margin_x, margin_y))
 
-    # Create the mask
     mask = Image.new('L', target_size, 255)
     mask_draw = ImageDraw.Draw(mask)
 
-    # Calculate overlap areas
     white_gaps_patch = 2
-
     left_overlap = margin_x + overlap_x if overlap_left else margin_x + white_gaps_patch
     right_overlap = margin_x + new_width - overlap_x if overlap_right else margin_x + new_width - white_gaps_patch
     top_overlap = margin_y + overlap_y if overlap_top else margin_y + white_gaps_patch
     bottom_overlap = margin_y + new_height - overlap_y if overlap_bottom else margin_y + new_height - white_gaps_patch
-    
+
     if alignment == "Left":
         left_overlap = margin_x + overlap_x if overlap_left else margin_x
     elif alignment == "Right":
@@ -214,34 +172,21 @@ def prepare_image_and_mask(image, width, height, overlap_percentage, resize_opti
     elif alignment == "Bottom":
         bottom_overlap = margin_y + new_height - overlap_y if overlap_bottom else margin_y + new_height
 
-
-    # Draw the mask
     mask_draw.rectangle([
         (left_overlap, top_overlap),
         (right_overlap, bottom_overlap)
     ], fill=0)
-
     return background, mask
 
 def preview_image_and_mask(image, width, height, overlap_percentage, resize_option, custom_resize_percentage, alignment, overlap_left, overlap_right, overlap_top, overlap_bottom):
     background, mask = prepare_image_and_mask(image, width, height, overlap_percentage, resize_option, custom_resize_percentage, alignment, overlap_left, overlap_right, overlap_top, overlap_bottom)
-    
-    # Create a preview image showing the mask
     preview = background.copy().convert('RGBA')
-    
-    # Create a semi-transparent red overlay
-    red_overlay = Image.new('RGBA', background.size, (255, 0, 0, 64))  # Reduced alpha to 64 (25% opacity)
-    
-    # Convert black pixels in the mask to semi-transparent red
+    red_overlay = Image.new('RGBA', background.size, (255, 0, 0, 64))
     red_mask = Image.new('RGBA', background.size, (0, 0, 0, 0))
     red_mask.paste(red_overlay, (0, 0), mask)
-    
-    # Overlay the red mask on the background
     preview = Image.alpha_composite(preview, red_mask)
-    
     return preview
 
-
 @spaces.GPU(duration=12)
 def inpaint(prompt, image, inpaint_model, paste_back):
     global pipe
@@ -252,39 +197,27 @@ def inpaint(prompt, image, inpaint_model, paste_back):
             vae=vae,
             controlnet=model,
         ).to("cuda")
-    # if pipe.config.model_name == "Lustify Inpaint":
-        
-    
     mask = Image.fromarray(image["mask"]).convert("L")
     image = Image.fromarray(image["image"])
-    
     result = pipe(prompt=prompt, image=image, mask_image=mask).images[0]
-    # result = pipe_inpaint(prompt=prompt, image=image, mask_image=mask).images[0]
-
     if paste_back:
         result.paste(image, (0, 0), Image.fromarray(255 - np.array(mask)))
-    
     return result
 
 @spaces.GPU(duration=12)
 def outpaint(image, width, height, overlap_percentage, num_inference_steps, resize_option, custom_resize_percentage, prompt_input, alignment, overlap_left, overlap_right, overlap_top, overlap_bottom):
     background, mask = prepare_image_and_mask(image, width, height, overlap_percentage, resize_option, custom_resize_percentage, alignment, overlap_left, overlap_right, overlap_top, overlap_bottom)
-    
     if not can_expand(background.width, background.height, width, height, alignment):
         alignment = "Middle"
-
     cnet_image = background.copy()
     cnet_image.paste(0, (0, 0), mask)
-
     final_prompt = f"{prompt_input} , high quality, 4k"
-
     (
         prompt_embeds,
         negative_prompt_embeds,
         pooled_prompt_embeds,
         negative_pooled_prompt_embeds,
     ) = pipe.encode_prompt(final_prompt, "cuda", True)
-
     for image in pipe(
         prompt_embeds=prompt_embeds,
         negative_prompt_embeds=negative_prompt_embeds,
@@ -294,31 +227,24 @@ def outpaint(image, width, height, overlap_percentage, num_inference_steps, resi
         num_inference_steps=num_inference_steps
     ):
         yield cnet_image, image
-
     image = image.convert("RGBA")
     cnet_image.paste(image, (0, 0), mask)
-
     yield background, cnet_image
 
 @spaces.GPU(duration=12)
 def infer(image, width, height, overlap_percentage, num_inference_steps, resize_option, custom_resize_percentage, prompt_input, alignment, overlap_left, overlap_right, overlap_top, overlap_bottom):
     background, mask = prepare_image_and_mask(image, width, height, overlap_percentage, resize_option, custom_resize_percentage, alignment, overlap_left, overlap_right, overlap_top, overlap_bottom)
-    
     if not can_expand(background.width, background.height, width, height, alignment):
         alignment = "Middle"
-
     cnet_image = background.copy()
     cnet_image.paste(0, (0, 0), mask)
-
     final_prompt = f"{prompt_input} , high quality, 4k"
-
     (
         prompt_embeds,
         negative_prompt_embeds,
         pooled_prompt_embeds,
         negative_pooled_prompt_embeds,
     ) = pipe.encode_prompt(final_prompt, "cuda", True)
-
     for image in pipe(
         prompt_embeds=prompt_embeds,
         negative_prompt_embeds=negative_prompt_embeds,
@@ -328,17 +254,14 @@ def infer(image, width, height, overlap_percentage, num_inference_steps, resize_
         num_inference_steps=num_inference_steps
     ):
         yield cnet_image, image
-
     image = image.convert("RGBA")
     cnet_image.paste(image, (0, 0), mask)
-
     yield background, cnet_image
-def clear_result():
-    """Clears the result ImageSlider."""
-    return gr.update(value=None)
+
+def use_output_as_input(output_image):
+    return gr.update(value=output_image[1])
 
 def preload_presets(target_ratio, ui_width, ui_height):
-    """Updates the width and height sliders based on the selected aspect ratio."""
     if target_ratio == "9:16":
         changed_width = 720
         changed_height = 1280
@@ -357,6 +280,8 @@ def preload_presets(target_ratio, ui_width, ui_height):
         return changed_width, changed_height, gr.update()
     elif target_ratio == "Custom":
         return ui_width, ui_height, gr.update(open=True)
+    else:
+        return ui_width, ui_height, gr.update()
 
 def select_the_right_preset(user_width, user_height):
     if user_width == 720 and user_height == 1280:
@@ -374,7 +299,6 @@ def toggle_custom_resize_slider(resize_option):
     return gr.update(visible=(resize_option == "Custom"))
 
 def update_history(new_image, history):
-    """Updates the history gallery with the new image."""
     if history is None:
         history = []
     history.insert(0, new_image)
@@ -399,14 +323,13 @@ title = """<h1 align="center">Diffusers Image Outpaint</h1>
     <p style="display: flex;gap: 6px;">
          <a href="https://huggingface.co/spaces/fffiloni/diffusers-image-outpout?duplicate=true">
             <img src="https://huggingface.co/datasets/huggingface/badges/resolve/main/duplicate-this-space-md.svg" alt="Duplicate this Space">
-        </a> to skip the queue and enjoy faster inference on the GPU of your choice 
+        </a> to skip the queue and enjoy faster inference on the GPU of your choice
     </p>
 </div>
 """
 
 with gr.Blocks(css=css, fill_height=True) as demo:
     gr.Markdown("# Diffusers Inpaint and Outpaint")
-    
     with gr.Tabs():
         with gr.TabItem("Inpaint"):
             with gr.Column():
@@ -423,31 +346,21 @@ with gr.Blocks(css=css, fill_height=True) as demo:
                             value="RealVisXL V5.0 Lightning",
                             label="Model",
                         )
-            
                         with gr.Row():
                             run_button = gr.Button("Generate")
                             paste_back = gr.Checkbox(True, label="Paste back original")
-            
                 with gr.Row(equal_height=False):
                     input_image = gr.ImageMask(
                         type="pil", label="Input Image", layers=True
-                        # type="pil", label="Input Image", crop_size=(1024, 1024), layers=False
                     )
-            
                     result = ImageSlider(
                         interactive=False,
                         label="Generated Image",
                     )
-            
                 use_as_input_button = gr.Button("Use as Input Image", visible=False)
-            
-                def use_output_as_input(output_image):
-                    return gr.update(value=output_image[1])
-            
                 use_as_input_button.click(
                     fn=use_output_as_input, inputs=[result], outputs=[input_image]
                 )
-            
                 run_button.click(
                     fn=clear_result,
                     inputs=None,
@@ -459,13 +372,12 @@ with gr.Blocks(css=css, fill_height=True) as demo:
                 ).then(
                     fn=fill_image,
                     inputs=[prompt, input_image, model_selection, paste_back],
-                    outputs=result,
+                    outputs=[result],
                 ).then(
                     fn=lambda: gr.update(visible=True),
                     inputs=None,
                     outputs=use_as_input_button,
                 )
-            
                 prompt.submit(
                     fn=clear_result,
                     inputs=None,
@@ -477,29 +389,25 @@ with gr.Blocks(css=css, fill_height=True) as demo:
                 ).then(
                     fn=fill_image,
                     inputs=[prompt, input_image, model_selection, paste_back],
-                    outputs=result,
+                    outputs=[result],
                 ).then(
                     fn=lambda: gr.update(visible=True),
                     inputs=None,
                     outputs=use_as_input_button,
                 )
-
         with gr.TabItem("Outpaint"):
             with gr.Column():
-
                 with gr.Row():
                     with gr.Column():
-                        outpaint_input_image = gr.Image(
+                        input_image_outpaint = gr.Image(
                             type="pil",
                             label="Input Image"
                         )
-        
                         with gr.Row():
                             with gr.Column(scale=2):
                                 prompt_input = gr.Textbox(label="Prompt (Optional)")
                             with gr.Column(scale=1):
                                 runout_button = gr.Button("Generate")
-        
                         with gr.Row():
                             target_ratio = gr.Radio(
                                 label="Expected Ratio",
@@ -507,13 +415,11 @@ with gr.Blocks(css=css, fill_height=True) as demo:
                                 value="1:1",
                                 scale=2
                             )
-                            
                             alignment_dropdown = gr.Dropdown(
                                 choices=["Middle", "Left", "Right", "Top", "Bottom"],
                                 value="Middle",
                                 label="Alignment"
                             )
-        
                         with gr.Accordion(label="Advanced settings", open=False) as settings_panel:
                             with gr.Column():
                                 with gr.Row():
@@ -522,16 +428,15 @@ with gr.Blocks(css=css, fill_height=True) as demo:
                                         minimum=720,
                                         maximum=1536,
                                         step=8,
-                                        value=1280,  # Set a default value
+                                        value=1280,
                                     )
                                     height_slider = gr.Slider(
                                         label="Target Height",
                                         minimum=720,
                                         maximum=1536,
                                         step=8,
-                                        value=1280,  # Set a default value
+                                        value=1280,
                                     )
-                                
                                 num_inference_steps = gr.Slider(label="Steps", minimum=4, maximum=12, step=1, value=8)
                                 with gr.Group():
                                     overlap_percentage = gr.Slider(
@@ -561,11 +466,8 @@ with gr.Blocks(css=css, fill_height=True) as demo:
                                         value=50,
                                         visible=False
                                     )
-                                
                                 with gr.Column():
                                     preview_button = gr.Button("Preview alignment and mask")
-                                    
-                                    
                         gr.Examples(
                             examples=[
                                 ["./examples/example_1.webp", 1280, 720, "Middle"],
@@ -573,136 +475,65 @@ with gr.Blocks(css=css, fill_height=True) as demo:
                                 ["./examples/example_3.jpg", 1024, 1024, "Top"],
                                 ["./examples/example_3.jpg", 1024, 1024, "Bottom"],
                             ],
-                            inputs=[outpaint_input_image, width_slider, height_slider, alignment_dropdown],
+                            inputs=[input_image_outpaint, width_slider, height_slider, alignment_dropdown],
                         )
-        
-                        
-        
                     with gr.Column():
-                        result = ImageSlider(
+                        result_outpaint = ImageSlider(
                             interactive=False,
                             label="Generated Image",
                         )
-                        use_as_input_button = gr.Button("Use as Input Image", visible=False)
-        
+                        use_as_input_button_outpaint = gr.Button("Use as Input Image", visible=False)
                         history_gallery = gr.Gallery(label="History", columns=6, object_fit="contain", interactive=False)
                         preview_image = gr.Image(label="Preview")
-
-        
-
-    def use_output_as_input(output_image):
-        """Sets the generated output as the new input image."""
-        return gr.update(value=output_image[1])
-
-    use_as_input_button.click(
+    use_as_input_button_outpaint.click(
         fn=use_output_as_input,
-        inputs=[result],
-        outputs=[outpaint_input_image]
-    )
-    
-    # Set up event handlers
-    run_button.click(
-        fn=fill_image,
-        inputs=[prompt, input_image, model_selection, paste_back],
-        outputs=result,
-    )
-
-    target_ratio.change(
-        fn=preload_presets,
-        inputs=[target_ratio, width_slider, height_slider],
-        outputs=[width_slider, height_slider, settings_panel],
-        queue=False
-    )
-
-    width_slider.change(
-        fn=select_the_right_preset,
-        inputs=[width_slider, height_slider],
-        outputs=[target_ratio],
-        queue=False
-    )
-
-    height_slider.change(
-        fn=select_the_right_preset,
-        inputs=[width_slider, height_slider],
-        outputs=[target_ratio],
-        queue=False
+        inputs=[result_outpaint],
+        outputs=[input_image_outpaint]
     )
-
-    resize_option.change(
-        fn=toggle_custom_resize_slider,
-        inputs=[resize_option],
-        outputs=[custom_resize_percentage],
-        queue=False
-    )
-    
-    runout_button.click(  # Clear the result
+    runout_button.click(
         fn=clear_result,
         inputs=None,
-        outputs=result,
-    ).then(  # Generate the new image
+        outputs=result_outpaint,
+    ).then(
         fn=infer,
-        inputs=[outpaint_input_image, width_slider, height_slider, overlap_percentage, num_inference_steps,
+        inputs=[input_image_outpaint, width_slider, height_slider, overlap_percentage, num_inference_steps,
                 resize_option, custom_resize_percentage, prompt_input, alignment_dropdown,
                 overlap_left, overlap_right, overlap_top, overlap_bottom],
-        outputs=result,
-    ).then(  # Update the history gallery
+        outputs=[result_outpaint],
+    ).then(
         fn=lambda x, history: update_history(x[1], history),
-        inputs=[result, history_gallery],
+        inputs=[result_outpaint, history_gallery],
         outputs=history_gallery,
-    ).then(  # Show the "Use as Input Image" button
+    ).then(
         fn=lambda: gr.update(visible=True),
         inputs=None,
-        outputs=use_as_input_button,
+        outputs=[use_as_input_button_outpaint],
     )
-
-    prompt_input.submit(  # Clear the result
+    prompt_input.submit(
         fn=clear_result,
         inputs=None,
-        outputs=result,
-    ).then(  # Generate the new image
+        outputs=result_outpaint,
+    ).then(
         fn=infer,
-        inputs=[outpaint_input_image, width_slider, height_slider, overlap_percentage, num_inference_steps,
+        inputs=[input_image_outpaint, width_slider, height_slider, overlap_percentage, num_inference_steps,
                 resize_option, custom_resize_percentage, prompt_input, alignment_dropdown,
                 overlap_left, overlap_right, overlap_top, overlap_bottom],
-        outputs=result,
-    ).then(  # Update the history gallery
+        outputs=[result_outpaint],
+    ).then(
         fn=lambda x, history: update_history(x[1], history),
-        inputs=[result, history_gallery],
+        inputs=[result_outpaint, history_gallery],
         outputs=history_gallery,
-    ).then(  # Show the "Use as Input Image" button
+    ).then(
         fn=lambda: gr.update(visible=True),
         inputs=None,
-        outputs=use_as_input_button,
-    )
-
-    preview_button.click(
-        fn=preview_image_and_mask,
-        inputs=[outpaint_input_image, width_slider, height_slider, overlap_percentage, resize_option, custom_resize_percentage, alignment_dropdown,
-                overlap_left, overlap_right, overlap_top, overlap_bottom],
-        outputs=preview_image,
-        queue=False
-    )
-
-    runout_button.click(
-        fn=infer,
-        inputs=[outpaint_input_image, width_slider, height_slider, overlap_percentage, num_inference_steps,
-                resize_option, custom_resize_percentage, prompt_input, alignment_dropdown,
-                overlap_left, overlap_right, overlap_top, overlap_bottom],
-        outputs=result,
+        outputs=[use_as_input_button_outpaint],
     )
-
     preview_button.click(
         fn=preview_image_and_mask,
-        inputs=[outpaint_input_image, width_slider, height_slider, overlap_percentage, resize_option, custom_resize_percentage, alignment_dropdown,
+        inputs=[input_image_outpaint, width_slider, height_slider, overlap_percentage, resize_option, custom_resize_percentage, alignment_dropdown,
                 overlap_left, overlap_right, overlap_top, overlap_bottom],
-        outputs=preview_image,
+        outputs=[preview_image],
         queue=False
     )
 
-    resize_option.change(
-        fn=lambda x: gr.update(visible=(x == "Custom")),
-        inputs=[resize_option],
-        outputs=[custom_resize_percentage]
-    )
-
 demo.launch(show_error=True)