Upload app.py

app.py

@@ -32,6 +32,15 @@ def create_repo(repo_id: str, repo_type: str, private: bool, hf_token: str):
     if repo_type == "space": api.create_repo(repo_id=repo_id, repo_type=repo_type, private=private, exist_ok=True, space_sdk="gradio", token=hf_token)
     else: api.create_repo(repo_id=repo_id, repo_type=repo_type, private=private, exist_ok=True, token=hf_token)
 
+def remove_dup_pathstr(path: str) -> str:
+    try:
+        if not path or "/" not in path: return path
+        first = path.split("/")[0]
+        path = re.sub(f"^{first}/{first}", first, path, count=1)
+        return path
+    except Exception:
+        return path
+
 def extract_src_reponame(source_repo: str):
     try:
         if is_valid_reponame(source_repo): target = ""
@@ -90,12 +99,17 @@ def duplicate(source_repo, dst_repo, repo_type, private, overwrite, auto_dir, re
             create_repo(dst_repo, repo_type, private, hf_token)
             for path in api.list_repo_files(repo_id=source_repo, repo_type=repo_type, token=hf_token):
                 if target and target not in path: continue
+                path_in_repo = f"{subfolder}/{path}" if subfolder else path
+                if is_same_file_hf(source_repo, path, repo_type, dst_repo, path_in_repo, repo_type, hf_token):
+                    print(f"{dst_repo}/{path_in_repo} is already exists. Skipping.")
+                    progress(0, desc=f"{dst_repo}/{path_in_repo} is already exists. Skipping.")
+                    continue
                 file = hf_hub_download(repo_id=source_repo, filename=path, repo_type=repo_type, local_dir=temp_dir, token=hf_token)
                 if not Path(file).exists(): continue
                 if Path(file).is_dir(): # unused for now
-                    api.upload_folder(repo_id=dst_repo, folder_path=file, path_in_repo=f"{subfolder}/{path}" if subfolder else path, repo_type=repo_type, token=hf_token)
+                    api.upload_folder(repo_id=dst_repo, folder_path=file, path_in_repo=path_in_repo, repo_type=repo_type, token=hf_token)
                 elif Path(file).is_file():
-                    api.upload_file(repo_id=dst_repo, path_or_fileobj=file, path_in_repo=f"{subfolder}/{path}" if subfolder else path, repo_type=repo_type, token=hf_token)
+                    api.upload_file(repo_id=dst_repo, path_or_fileobj=file, path_in_repo=path_in_repo, repo_type=repo_type, token=hf_token)
                 if Path(file).exists(): Path(file).unlink()
             if repo_type == "dataset": repo_url = f"https://huggingface.co/datasets/{dst_repo}"
             elif repo_type == "space": repo_url = f"https://huggingface.co/spaces/{dst_repo}"
@@ -141,7 +155,7 @@ def is_same_file_hf(src_repo: str, src_path: str, src_type: str, dst_repo: str,
     if src_info[0].size == dst_info[0].size and src_info[0].lfs.sha256 == dst_info[0].lfs.sha256: return True
     return False
 
-def duplicate_m2o(source_repos_str, dst_repo, repo_type, private, overwrite, oauth_token: gr.OAuthToken | None, progress=gr.Progress(track_tqdm=True)):
+def duplicate_m2o(source_repos_str, dst_repo, repo_type, private, overwrite, avoid_dup_dir, oauth_token: gr.OAuthToken | None, progress=gr.Progress(track_tqdm=True)):
     hf_token = oauth_token.token
     api = HfApi(token=hf_token)
     try:
@@ -160,8 +174,8 @@ def duplicate_m2o(source_repos_str, dst_repo, repo_type, private, overwrite, oau
         source_repos = parse_repos(source_repos_str)
         for source_repo in source_repos:
             source_repo, target = extract_src_reponame(source_repo)
-
             subfolder = subfolder_prefix + "/" + source_repo if subfolder_prefix else source_repo
+            if avoid_dup_dir: subfolder = remove_dup_pathstr(subfolder)
 
             temp_dir = tempfile.mkdtemp()
             create_repo(dst_repo, repo_type, private, hf_token)
@@ -170,6 +184,7 @@ def duplicate_m2o(source_repos_str, dst_repo, repo_type, private, overwrite, oau
                 path_in_repo = f"{subfolder}/{path}" if subfolder else path
                 if is_same_file_hf(source_repo, path, repo_type, dst_repo, path_in_repo, repo_type, hf_token):
                     print(f"{dst_repo}/{path_in_repo} is already exists. Skipping.")
+                    progress(0, desc=f"{dst_repo}/{path_in_repo} is already exists. Skipping.")
                     continue
                 file = hf_hub_download(repo_id=source_repo, filename=path, repo_type=repo_type, local_dir=temp_dir, token=hf_token)
                 if not Path(file).exists(): continue
@@ -270,6 +285,7 @@ with gr.Blocks(css=css) as demo:
                     with gr.Row():
                         submit_button = gr.Button("Submit", variant="primary")
                         clear_button = gr.Button("Clear", variant="secondary")
+                        stop_button = gr.Button("Stop", variant="stop")
                 with gr.Column():
                     output_md = gr.Markdown(label="output")
                     output_image = gr.Image(show_label=False)
@@ -289,9 +305,11 @@ with gr.Blocks(css=css) as demo:
                         with gr.Row():
                             m2o_is_private = gr.Checkbox(label="Make new repo private?", value=True)
                             m2o_is_overwrite = gr.Checkbox(label="Overwrite existing repo?", value=True)
+                            m2o_is_nodupdir = gr.Checkbox(label="Avoid creating duplicated directory?", value=True)
                     with gr.Row():
                         m2o_submit_button = gr.Button("Submit", variant="primary")
                         m2o_clear_button = gr.Button("Clear", variant="secondary")
+                        m2o_stop_button = gr.Button("Stop", variant="stop")
                 with gr.Column():
                     m2o_output_md = gr.Markdown(label="output")
                     m2o_output_image = gr.Image(show_label=False)
@@ -318,20 +336,25 @@ with gr.Blocks(css=css) as demo:
                     with gr.Row():
                         m2m_submit_button = gr.Button("Submit", variant="primary")
                         m2m_clear_button = gr.Button("Clear", variant="secondary")
+                        m2m_stop_button = gr.Button("Stop", variant="stop")
                 with gr.Column():
                     m2m_output_md = gr.Markdown(label="output")
                     m2m_output_image = gr.Image(show_label=False)
     demo.load(fn=swap_visibilty, outputs=main_ui)
-    submit_button.click(duplicate, [search, dst_repo, repo_type, is_private, is_overwrite, is_subdir, is_remtag], [output_md, output_image])
+    submit_event = submit_button.click(duplicate, [search, dst_repo, repo_type, is_private, is_overwrite, is_subdir, is_remtag], [output_md, output_image])
+    stop_button.click(fn=None, inputs=None, outputs=None, cancels=[submit_event])
     clear_button.click(lambda: ("", HF_REPO, "model", True, True, True, True), None, [search, dst_repo, repo_type, is_private, is_overwrite, is_subdir, is_remtag], queue=False)
     m2o_search.submit(add_repo_text, [m2o_search, m2o_source_repos], [m2o_source_repos], queue=False)
-    m2o_submit_button.click(duplicate_m2o, [m2o_source_repos, m2o_dst_repo, m2o_repo_type, m2o_is_private, m2o_is_overwrite], [m2o_output_md, m2o_output_image])
+    m2o_submit_event = m2o_submit_button.click(duplicate_m2o, [m2o_source_repos, m2o_dst_repo, m2o_repo_type, m2o_is_private, m2o_is_overwrite, m2o_is_nodupdir], [m2o_output_md, m2o_output_image])
+    m2o_stop_button.click(fn=None, inputs=None, outputs=None, cancels=[m2o_submit_event])
     m2o_clear_button.click(lambda: ("", HF_REPO, "model", True, True, ""), None,
                            [m2o_search, m2o_dst_repo, m2o_repo_type, m2o_is_private, m2o_is_overwrite, m2o_source_repos], queue=False)
     m2m_search.submit(add_repo_text, [m2m_search, m2m_source_repos], [m2m_source_repos], queue=False)
-    m2m_submit_button.click(duplicate_m2m, [m2m_source_repos, m2m_user, m2m_repo_type, m2m_is_private, m2m_is_overwrite, m2m_is_remtag, m2m_prefix, m2m_suffix],
+    m2m_submit_event = m2m_submit_button.click(duplicate_m2m, [m2m_source_repos, m2m_user, m2m_repo_type, m2m_is_private, m2m_is_overwrite, m2m_is_remtag, m2m_prefix, m2m_suffix],
                             [m2m_output_md, m2m_output_image])
+    m2m_stop_button.click(fn=None, inputs=None, outputs=None, cancels=[m2m_submit_event])
     m2m_clear_button.click(lambda: ("", HF_USER, "model", True, False, True, "", HF_REPO_PREFIX, HF_REPO_SUFFIX), None,
                            [m2m_search, m2m_user, m2m_repo_type, m2m_is_private, m2m_is_overwrite, m2m_is_remtag, m2m_source_repos, m2m_prefix, m2m_suffix], queue=False)
+    
 
 demo.queue().launch()