Force push - update all files

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/a.png filter=lfs diff=lfs merge=lfs -text
+examples/b.png filter=lfs diff=lfs merge=lfs -text
+templates/fist-back.png filter=lfs diff=lfs merge=lfs -text
+templates/opened-palm.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,174 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc

README.md

@@ -1,14 +1,14 @@
 ---
-title: Handcraft
-emoji: 📚
-colorFrom: gray
-colorTo: indigo
+title: HandCraft
+emoji: 🖐️
+colorFrom: pink
+colorTo: pink
 sdk: gradio
 sdk_version: 5.21.0
 app_file: app.py
 pinned: false
 license: cc
-short_description: Sourcecode of HandCraft (WACV 2025)
+short_description: Official sourcecode for HandCraft (WACV 2025)
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

component/__init__.py

@@ -0,0 +1,3 @@
+from .bbox import bbox
+from .skeleton import skeleton
+from .control import control

component/bbox.py

@@ -0,0 +1,48 @@
+from ultralytics import YOLO
+import cv2
+import numpy as np
+
+
+hand_dict = {0: 'Standard', 1: 'Non-Standard'}
+
+def bbox(image, include_standard, mask_expand_ratio):
+    model = YOLO('model/yolo.pt')
+
+    solution = model.predict(image)
+
+    # detect bbox
+    result = []
+    for det in solution[0]:
+        x1, y1, x2, y2, conf, cls = list(det.boxes.data[0])
+        bbox = [int(x1), int(y1), int(x2), int(y2), int(cls), float(conf), hand_dict[int(cls)]]
+        result.append(bbox)
+
+        print('BBox Result:', bbox, image.shape)
+
+    # draw bbox
+    for bbox in result:
+        COLOR = (0, 255, 0) if bbox[4] == 0 else (255, 0, 0)
+        cv2.rectangle(image, (bbox[0], bbox[1]), (bbox[2], bbox[3]), COLOR, 2)
+        cv2.putText(image, str(bbox[6]), (bbox[0], bbox[1]), cv2.FONT_HERSHEY_SIMPLEX, 1, COLOR, 2, cv2.LINE_AA)
+
+    # get mask
+    image_mask = np.zeros(image.shape, dtype=np.uint8)
+    for bbox in result:
+        # only non-standard
+        if not include_standard and bbox[4] == 0:
+            continue
+
+        # expand
+        bbox[0] = max(0, bbox[0] - int((bbox[2] - bbox[0]) * (mask_expand_ratio - 1) * 0.5))
+        bbox[1] = max(0, bbox[1] - int((bbox[3] - bbox[1]) * (mask_expand_ratio - 1) * 0.5))
+        bbox[2] = min(image.shape[1], bbox[2] + int((bbox[2] - bbox[0]) * (mask_expand_ratio - 1) * 0.5))
+        bbox[3] = min(image.shape[0], bbox[3] + int((bbox[3] - bbox[1]) * (mask_expand_ratio - 1) * 0.5))
+
+        image_mask[bbox[1]:bbox[3], bbox[0]:bbox[2]] = 255
+
+    # get bbox info
+    bbox_info = f'Number of hands: {len(result)}\n'
+    for i, hand in enumerate(result):
+        bbox_info += f'Hand {i}: {hand[6]} with confidence {hand[5]:.4f}.\n'
+
+    return bbox_info, image, image_mask

component/control.py

@@ -0,0 +1,123 @@
+import cv2
+import numpy as np
+import json
+from datetime import datetime
+
+
+def read_template(name, type):
+    path_template = f'templates/{name}.png'
+    path_keypoint = f'templates/{name}.txt'
+
+    template = cv2.cvtColor(cv2.imread(path_template), cv2.COLOR_BGR2RGB)
+    template_keypoint = np.loadtxt(path_keypoint, dtype=np.int32, delimiter=',')
+
+    if type == 'b':
+        template = cv2.flip(template, 1)
+        template_keypoint[:, 0] = template.shape[1] - template_keypoint[:, 0]
+
+    return template, template_keypoint
+
+
+def affine(image, image_keypoint, template, template_keypoint, expand_ratio):
+    # pre process
+    template_white = np.full(template.shape, 255, dtype=np.uint8)
+
+    # resize
+    scale = np.linalg.norm(image_keypoint[0] - image_keypoint[2]) / np.linalg.norm(
+        template_keypoint[0] - template_keypoint[2])
+    scale *= expand_ratio
+    M = np.array([
+        [scale, 0, 0],
+        [0, scale, 0],
+    ], dtype=np.float32)
+    size = (int(template.shape[1] * scale), int(template.shape[0] * scale))
+    template_white = cv2.warpAffine(template_white, M, size)
+    template = cv2.warpAffine(template, M, size)
+    template_keypoint = template_keypoint * scale
+
+    # move
+    distance = image_keypoint[0] - template_keypoint[0]
+    M = np.array([
+        [1, 0, distance[0]],
+        [0, 1, distance[1]],
+    ], dtype=np.float32)
+    template_white = cv2.warpAffine(template_white, M, image.shape[1::-1])
+    template = cv2.warpAffine(template, M, image.shape[1::-1])
+
+    # rotate
+    v1 = template_keypoint[2] - template_keypoint[0]
+    v2 = image_keypoint[2] - image_keypoint[0]
+
+    angle = np.rad2deg(np.arccos(np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))))
+    if np.cross(v1, v2) > 0:
+        angle = -angle
+
+    M = cv2.getRotationMatrix2D(image_keypoint[0], angle, 1)
+    template_white = cv2.warpAffine(template_white, M, image.shape[1::-1])
+    template = cv2.warpAffine(template, M, image.shape[1::-1])
+
+    return template_white, template
+
+
+def control(image, image_bbox_mask, res_skeleton, expand_ratio, template_name, image_skeleton, force):
+    # left hand
+    template_white_l, template_l = np.zeros_like(image), np.zeros_like(image)
+    image_keypoint = np.array(json.loads(res_skeleton))[0]
+    if np.max(image_keypoint) < 1:
+        image_keypoint[:, 0], image_keypoint[:, 1] = image_keypoint[:, 1] * image.shape[1], image_keypoint[:, 0] * image.shape[0]
+        # check hand be detected by yolo
+        be_detected = False
+        for keypoint in image_keypoint:
+            if np.sum(image_bbox_mask[int(keypoint[1]), int(keypoint[0])]) > 1:
+                be_detected = True
+                break
+        if be_detected or force:
+
+            if np.cross(image_keypoint[2] - image_keypoint[0], image_keypoint[3] - image_keypoint[1]) > 0:
+                template, template_keypoint = read_template(template_name, 'a')
+            else:
+                template, template_keypoint = read_template(template_name, 'b')
+
+            template_white_l, template_l = affine(image, image_keypoint, template, template_keypoint, expand_ratio)
+
+    # right hand
+    template_white_r, template_r = np.zeros_like(image), np.zeros_like(image)
+
+    image_keypoint = np.array(json.loads(res_skeleton))[1]
+    if np.max(image_keypoint) < 1:
+        image_keypoint[:, 0], image_keypoint[:, 1] = image_keypoint[:, 1] * image.shape[1], image_keypoint[:, 0] * image.shape[0]
+        # check hand be detected by yolo
+        be_detected = False
+        for keypoint in image_keypoint:
+            if np.sum(image_bbox_mask[int(keypoint[1]), int(keypoint[0])]) > 1:
+                be_detected = True
+                break
+        if be_detected or force:
+
+            if np.cross(image_keypoint[2] - image_keypoint[0], image_keypoint[3] - image_keypoint[1]) > 0:
+                template, template_keypoint = read_template(template_name, 'a')
+            else:
+                template, template_keypoint = read_template(template_name, 'b')
+
+            template_white_r, template_r = affine(image, image_keypoint, template, template_keypoint, expand_ratio)
+
+
+    # control image
+    template = np.clip(template_l.astype(np.int32) + template_r.astype(np.int32), 0, 255).astype(np.uint8)
+    # combine image
+    image_combine = np.clip(image.astype(np.int32) + template, 0, 255).astype(np.uint8)
+    # control mask
+    template_white = np.clip(template_white_l.astype(np.int32) + template_white_r.astype(np.int32), 0, 255).astype(np.uint8)
+    # union mask
+    image_union_mask = np.clip(image_bbox_mask.astype(np.int32) + template_white.astype(np.int32), 0, 255).astype(np.uint8)
+
+    # visualization
+    image_visualization = np.clip(
+        template.astype(np.int32) * - 1 +
+        image_skeleton.astype(np.int32) * 0.6 +
+        image_bbox_mask.astype(np.int32) * 0.3 +
+        template_white.astype(np.int32) * 0.3
+        , 0, 255).astype(np.uint8)
+
+    return image_visualization, template, template_white, image_union_mask
+

component/skeleton.py

@@ -0,0 +1,26 @@
+import mediapipe as mp
+import numpy as np
+import json
+from datetime import datetime
+
+def skeleton(image):
+    mp_pose = mp.solutions.pose.Pose()
+    mp_drawing = mp.solutions.drawing_utils
+
+    solution = mp_pose.process(image)
+
+    # default value
+    result = np.zeros((33, 2))
+
+    # detect and draw skeleton
+    if solution.pose_landmarks:
+        mp_drawing.draw_landmarks(image, solution.pose_landmarks, mp.solutions.pose.POSE_CONNECTIONS)
+        result = np.array([
+            (landmark.y, landmark.x)
+            for landmark in solution.pose_landmarks.landmark
+        ])
+
+    # serialization
+    skeleton_info = json.dumps(np.array([result[15:22:2], result[16:23:2]]).tolist())
+
+    return skeleton_info, image

main.py

@@ -0,0 +1,207 @@
+import gradio as gr
+import cv2
+from component import bbox
+from component import skeleton
+from component import control
+
+# Load example image
+image_example = cv2.cvtColor(cv2.imread("examples/a.png"), cv2.COLOR_BGR2RGB)
+uni_height = 800
+
+# Create the interface with a cute theme
+with gr.Blocks(theme=gr.themes.Soft()) as interface:
+    # Title with kawaii emojis
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown(f"""
+            <div style='text-align: center; 
+                        padding: 25px; 
+                        border-radius: 12px;
+                        background: #f3f4f6;
+                        box-shadow: 0 10px 25px rgba(0, 0, 0, 0.08),
+                                    0 2px 4px rgba(0, 0, 0, 0.05);
+                        margin: 20px 0;
+                        border: 1px solid rgba(0,0,0,0.1);
+                        width: 100%;
+                        position: relative;
+                        transition: transform 0.2s ease, box-shadow 0.2s ease;'>
+                <h1 style='margin: 0; 
+                        color: #BE830E;
+                        font-family: "Comic Sans MS", cursive;
+                        text-shadow: 2px 2px 4px rgba(190, 131, 14, 0.1);
+                        letter-spacing: 1.2px;
+                        position: relative;
+                        z-index: 2;'>
+                    🖐️ HandCraft 🖐️
+                </h1>
+                <h3 style='margin: 10px 0 0;
+                        color: #374151;
+                        font-family: "Comic Sans MS", cursive;
+                        font-weight: 500;
+                        position: relative;
+                        z-index: 2;
+                        line-height: 1.4;
+                        text-shadow: 0 1px 2px rgba(0,0,0,0.05);'>
+                    🐾✨ Anatomically Correct Restoration of Malformed Hands in Diffusion Generated Images ✨🐾
+                </h3>
+            </div>
+            """)
+    
+    # Shared input image at the top
+    input_image = gr.Image(
+        type="numpy", 
+        label="📸 Upload Your Image with Hands", 
+        height=uni_height,
+        value=image_example
+    )
+    
+    # Button to trigger the cascade
+    generate_btn = gr.Button("✨🪄 Generate Control Mask 🪄✨", variant="primary", size="lg")
+    
+    # State variables to store intermediate results
+    bbox_mask_state = gr.State()
+    keypoints_state = gr.State()
+    skeleton_state = gr.State()
+    
+    # Results section with tabs for each step
+    with gr.Tabs():
+        with gr.TabItem("🐾 Step 1: Malformed Hand Detection"):
+            with gr.Row():
+                with gr.Column(scale=1):
+                    output_bbox_result = gr.Textbox(label="🏷️ Number of Hands & Classification with Confidence")
+                    include_standard = gr.Checkbox(
+                        label="🤲 Include Standard Hands", 
+                        value=False
+                    )
+                    expand_ratio = gr.Slider(
+                        minimum=0.5, 
+                        maximum=2, 
+                        step=0.01, 
+                        value=1, 
+                        label="📏 Bounding Box Expand Ratio"
+                    )
+                with gr.Column(scale=2):
+                    with gr.Row():
+                        with gr.Column(scale=1):
+                            output_bbox_vis = gr.Image(type="numpy", label="📦 Bounding Box", height=uni_height)
+                        with gr.Column(scale=1):
+                            output_bbox_mask = gr.Image(type="numpy", label="🎭 Bounding Box Mask", height=uni_height)
+        
+        with gr.TabItem("💃 Step 2: Body Pose Estimation"):
+            with gr.Row():
+                with gr.Column(scale=1):
+                    output_keypoints = gr.Textbox(label="📊 Key Points String")
+                with gr.Column(scale=2):
+                    output_skeleton = gr.Image(type="numpy", label="💪 Body Skeleton", height=uni_height)
+        
+        with gr.TabItem("🎨 Step 3: Control Image Generation"):
+            with gr.Row():
+                with gr.Column(scale=1):
+                    hand_template = gr.Radio(
+                        ["opened-palm", "fist-back"], 
+                        label="👋 Hand Template", 
+                        value="opened-palm"
+                    )
+                    control_expand = gr.Slider(
+                        minimum=0.5, 
+                        maximum=2, 
+                        step=0.01, 
+                        value=1,
+                        label="🔍 Control Image Expand Ratio"
+                    )
+                    include_undetected = gr.Checkbox(
+                        label="🔎 Include Undetected Hand", 
+                        value=False
+                    )
+                with gr.Column(scale=2):
+                    with gr.Row():
+                        with gr.Column():
+                            output_viz = gr.Image(type="numpy", label="👁️ Visualization Image", height=300)
+                        with gr.Column():
+                            output_control = gr.Image(type="numpy", label="🎮 Control Image", height=300)
+                    with gr.Row():
+                        with gr.Column():
+                            output_control_mask = gr.Image(type="numpy", label="🎭 Control Mask", height=300)
+                        with gr.Column():
+                            output_union_mask = gr.Image(type="numpy", label="🔄 Union Mask", height=300)
+                            
+        with gr.TabItem("🎉 Output Control Image"):
+            gr.Markdown("""
+            ### ✨🌈 Control Image 🌈✨
+            Your hand-fixed image is ready! (ﾉ◕ヮ◕)ﾉ*:･ﾟ✧
+            """)
+            with gr.Row():
+                with gr.Column():
+                    output_final_control = gr.Image(type="numpy", label="👐 Fixed Hand Image", interactive=False, height=uni_height)
+                with gr.Column():
+                    gr.Markdown("""
+                    ### 🌟✨ How to Use Your Control Image ✨🌟
+                    
+                    1. 🪄 Take this Control Image to your favorite Stable Diffusion model
+                    2. 🎀 Apply that to the ControlNet
+                    3. 🍬 Sprinkle some parameters until it looks just right!
+                    
+                    """)
+    
+    # Citation information with cute emojis
+    with gr.Accordion("📚✨ Citation Information ✨📚", open=False):
+        gr.Markdown("""
+        If you find this tool helpful for your research, please cite our paper: 📝
+        
+        ```bibtex
+        @InProceedings{2025_wacv_handcraft,
+            author    = {Qin, Zhenyue and Zhang, Yiqun and Liu, Yang and Campbell, Dylan},
+            title     = {HandCraft: Anatomically Correct Restoration of Malformed Hands in Diffusion Generated Images},
+            booktitle = {Proceedings of the Winter Conference on Applications of Computer Vision (WACV)},
+            month     = {February},
+            year      = {2025},
+            pages     = {3925-3933}
+        }
+        ```
+        
+        Thank you for using HandCraft! ✨👐✨
+        """)
+    
+    # Define the step functions with improved data flow
+    def run_step1(image, include_std, expand_r):
+        # Step 1: Run hand detection
+        bbox_result, bbox_vis, bbox_mask = bbox(image, include_std, expand_r)
+        return bbox_result, bbox_vis, bbox_mask, bbox_mask
+    
+    def run_step2(image):
+        # Step 2: Run pose estimation
+        keypoints, skeleton_img = skeleton(image)
+        return keypoints, skeleton_img, keypoints, skeleton_img
+    
+    def run_step3(image, bbox_mask, keypoints, control_exp, hand_tmpl, skeleton_img, include_undetect):
+        # Step 3: Generate control images
+        viz, control_img, control_mask, union_mask = control(
+            image, bbox_mask, keypoints, control_exp, hand_tmpl, skeleton_img, include_undetect
+        )
+        return viz, control_img, control_mask, union_mask, control_img
+    
+    # Connect the Generate button to trigger all steps in sequence
+    generate_btn.click(
+        fn=run_step1,
+        inputs=[input_image, include_standard, expand_ratio],
+        outputs=[output_bbox_result, output_bbox_vis, output_bbox_mask, bbox_mask_state]
+    ).then(
+        fn=run_step2,
+        inputs=[input_image],
+        outputs=[output_keypoints, output_skeleton, keypoints_state, skeleton_state]
+    ).then(
+        fn=run_step3,
+        inputs=[
+            input_image, 
+            bbox_mask_state, 
+            keypoints_state,
+            control_expand,
+            hand_template,
+            skeleton_state,
+            include_undetected
+        ],
+        outputs=[output_viz, output_control, output_control_mask, output_union_mask, output_final_control]
+    )
+
+# Launch the interface
+interface.launch(server_name="0.0.0.0", server_port=7860, share=True)

model/yolo.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fa7a64d136d9b9356a72ed2238fa4744833cb1dc6078fc395e39c2d664372e2
+size 136682110

requirements.txt

@@ -0,0 +1,10 @@
+torch==2.0.1
+gradio==3.48.0
+mediapipe==0.10.5
+numpy==1.26.1
+opencv_contrib_python==4.8.0.76
+opencv_python==4.8.0.76
+Pillow==10.1.0
+ultralytics==8.0.180
+accelerate==0.23.0
+transformers==4.33.2

templates/fist-back.txt

@@ -0,0 +1,5 @@
+153, 84
+87, 268
+175, 342
+270, 300
+256, 267

templates/opened-palm.txt

@@ -0,0 +1,5 @@
+414, 127
+460, 365
+362, 477
+286, 393
+646, 960