first commit

.gitignore

@@ -0,0 +1,186 @@
+.DS_Store
+
+tmp/
+
+
+### Generated by gibo (https://github.com/simonwhitaker/gibo)
+### https://raw.github.com/github/gitignore/4488915eec0b3a45b5c63ead28f286819c0917de/Global/VisualStudioCode.gitignore
+
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+
+### https://raw.github.com/github/gitignore/4488915eec0b3a45b5c63ead28f286819c0917de/Python.gitignore
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+# lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "python.formatting.provider": "black",
+    "editor.codeActionsOnSave": {
+        "source.organizeImports": true
+    },
+    "editor.formatOnSave": true,
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 ddPn08
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README-ja.md

@@ -0,0 +1,54 @@
+<h1 align="center">RVC-WebUI</h1>
+<div align="center">
+<p>
+
+[`liujing04/Retrieval-based-Voice-Conversion-WebUI`](https://github.com/liujing04/Retrieval-based-Voice-Conversion-WebUI) の再構築プロジェクト
+
+</p>
+</div>
+
+---
+
+<div align="center">
+<p>
+
+[日本語](README-ja.md) | [English](README.md)
+
+</p>
+</div>
+
+<br >
+
+# 起動
+
+## Windows
+`webui-user.bat` をダブルクリックして、webuiを起動します。
+
+## Linux or Mac
+`webui.sh` を実行して、webuiを起動します。
+
+<br >
+
+```
+テスト環境: Windows 10, Python 3.10.9, torch 2.0.0+cu118
+```
+
+<br >
+
+# トラブルシューティング
+
+## `error: Microsoft Visual C++ 14.0 or greater is required.`
+
+Microsoft C++ Build Tools がインストールされている必要があります。
+
+### Step 1: インストーラーをダウンロード
+[Download](https://visualstudio.microsoft.com/ja/thank-you-downloading-visual-studio/?sku=BuildTools&rel=16)
+
+### Step 2: `C++ Build Tools` をインストール
+インストーラーを実行し、`Workloads` タブで `C++ Build Tools` を選択します。
+
+<br >
+
+# クレジット
+- [`liujing04/Retrieval-based-Voice-Conversion-WebUI`](https://github.com/liujing04/Retrieval-based-Voice-Conversion-WebUI)
+- [`teftef6220/Voice_Separation_and_Selection`](https://github.com/teftef6220/Voice_Separation_and_Selection)

README.md

@@ -1,10 +1,54 @@
+<h1 align="center">RVC-WebUI</h1>
+<div align="center">
+<p>
+
+[`liujing04/Retrieval-based-Voice-Conversion-WebUI`](https://github.com/liujing04/Retrieval-based-Voice-Conversion-WebUI) reconstruction project
+
+</p>
+</div>
+
 ---
-title: Rvc Webui
-emoji: 😻
-colorFrom: purple
-colorTo: pink
-sdk: docker
-pinned: false
----
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+<div align="center">
+<p>
+
+[日本語](README-ja.md) | [English](README.md)
+
+</p>
+</div>
+
+<br >
+
+# Launch
+
+## Windows
+Double click `webui-user.bat` to start the webui.
+
+## Linux or Mac
+Run `webui.sh` to start the webui.
+
+<br >
+
+```
+Tested environment: Windows 10, Python 3.10.9, torch 2.0.0+cu118
+```
+
+<br >
+
+# Troubleshooting
+
+## `error: Microsoft Visual C++ 14.0 or greater is required.`
+
+Microsoft C++ Build Tools must be installed.
+
+### Step 1: Download the installer
+[Download](https://visualstudio.microsoft.com/ja/thank-you-downloading-visual-studio/?sku=BuildTools&rel=16)
+
+### Step 2: Install `C++ Build Tools`
+Run the installer and select `C++ Build Tools` in the `Workloads` tab.
+
+<br >
+
+# Credits
+- [`liujing04/Retrieval-based-Voice-Conversion-WebUI`](https://github.com/liujing04/Retrieval-based-Voice-Conversion-WebUI)
+- [`teftef6220/Voice_Separation_and_Selection`](https://github.com/teftef6220/Voice_Separation_and_Selection)

bin/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

configs/32k-768.json

@@ -0,0 +1,47 @@
+{
+  "train": {
+    "log_interval": 200,
+    "seed": 1234,
+    "epochs": 20000,
+    "learning_rate": 1e-4,
+    "betas": [0.8, 0.99],
+    "eps": 1e-9,
+    "batch_size": 4,
+    "fp16_run": true,
+    "lr_decay": 0.999875,
+    "segment_size": 12800,
+    "init_lr_ratio": 1,
+    "warmup_epochs": 0,
+    "c_mel": 45,
+    "c_kl": 1.0
+  },
+  "data": {
+    "max_wav_value": 32768.0,
+    "sampling_rate": 32000,
+    "filter_length": 1024,
+    "hop_length": 320,
+    "win_length": 1024,
+    "n_mel_channels": 80,
+    "mel_fmin": 0.0,
+    "mel_fmax": null
+  },
+  "model": {
+    "inter_channels": 192,
+    "hidden_channels": 192,
+    "filter_channels": 768,
+    "n_heads": 2,
+    "n_layers": 6,
+    "kernel_size": 3,
+    "p_dropout": 0,
+    "resblock": "1",
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+    "upsample_rates": [10,4,2,2,2],
+    "upsample_initial_channel": 512,
+    "upsample_kernel_sizes": [16,16,4,4,4],
+    "use_spectral_norm": false,
+    "gin_channels": 256,
+    "emb_channels": 768,
+    "spk_embed_dim": 109
+  }
+}

configs/32k.json

@@ -0,0 +1,47 @@
+{
+  "train": {
+    "log_interval": 200,
+    "seed": 1234,
+    "epochs": 20000,
+    "learning_rate": 1e-4,
+    "betas": [0.8, 0.99],
+    "eps": 1e-9,
+    "batch_size": 4,
+    "fp16_run": true,
+    "lr_decay": 0.999875,
+    "segment_size": 12800,
+    "init_lr_ratio": 1,
+    "warmup_epochs": 0,
+    "c_mel": 45,
+    "c_kl": 1.0
+  },
+  "data": {
+    "max_wav_value": 32768.0,
+    "sampling_rate": 32000,
+    "filter_length": 1024,
+    "hop_length": 320,
+    "win_length": 1024,
+    "n_mel_channels": 80,
+    "mel_fmin": 0.0,
+    "mel_fmax": null
+  },
+  "model": {
+    "inter_channels": 192,
+    "hidden_channels": 192,
+    "filter_channels": 768,
+    "n_heads": 2,
+    "n_layers": 6,
+    "kernel_size": 3,
+    "p_dropout": 0,
+    "resblock": "1",
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+    "upsample_rates": [10,4,2,2,2],
+    "upsample_initial_channel": 512,
+    "upsample_kernel_sizes": [16,16,4,4,4],
+    "use_spectral_norm": false,
+    "gin_channels": 256,
+    "emb_channels": 256,
+    "spk_embed_dim": 109
+  }
+}

configs/40k-768.json

@@ -0,0 +1,47 @@
+{
+  "train": {
+    "log_interval": 200,
+    "seed": 1234,
+    "epochs": 20000,
+    "learning_rate": 1e-4,
+    "betas": [0.8, 0.99],
+    "eps": 1e-9,
+    "batch_size": 4,
+    "fp16_run": true,
+    "lr_decay": 0.999875,
+    "segment_size": 12800,
+    "init_lr_ratio": 1,
+    "warmup_epochs": 0,
+    "c_mel": 45,
+    "c_kl": 1.0
+  },
+  "data": {
+    "max_wav_value": 32768.0,
+    "sampling_rate": 40000,
+    "filter_length": 2048,
+    "hop_length": 400,
+    "win_length": 2048,
+    "n_mel_channels": 125,
+    "mel_fmin": 0.0,
+    "mel_fmax": null
+  },
+  "model": {
+    "inter_channels": 192,
+    "hidden_channels": 192,
+    "filter_channels": 768,
+    "n_heads": 2,
+    "n_layers": 6,
+    "kernel_size": 3,
+    "p_dropout": 0,
+    "resblock": "1",
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+    "upsample_rates": [10,10,2,2],
+    "upsample_initial_channel": 512,
+    "upsample_kernel_sizes": [16,16,4,4],
+    "use_spectral_norm": false,
+    "gin_channels": 256,
+    "emb_channels": 768,
+    "spk_embed_dim": 109
+  }
+}

configs/40k.json

@@ -0,0 +1,47 @@
+{
+  "train": {
+    "log_interval": 200,
+    "seed": 1234,
+    "epochs": 20000,
+    "learning_rate": 1e-4,
+    "betas": [0.8, 0.99],
+    "eps": 1e-9,
+    "batch_size": 4,
+    "fp16_run": true,
+    "lr_decay": 0.999875,
+    "segment_size": 12800,
+    "init_lr_ratio": 1,
+    "warmup_epochs": 0,
+    "c_mel": 45,
+    "c_kl": 1.0
+  },
+  "data": {
+    "max_wav_value": 32768.0,
+    "sampling_rate": 40000,
+    "filter_length": 2048,
+    "hop_length": 400,
+    "win_length": 2048,
+    "n_mel_channels": 125,
+    "mel_fmin": 0.0,
+    "mel_fmax": null
+  },
+  "model": {
+    "inter_channels": 192,
+    "hidden_channels": 192,
+    "filter_channels": 768,
+    "n_heads": 2,
+    "n_layers": 6,
+    "kernel_size": 3,
+    "p_dropout": 0,
+    "resblock": "1",
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+    "upsample_rates": [10,10,2,2],
+    "upsample_initial_channel": 512,
+    "upsample_kernel_sizes": [16,16,4,4],
+    "use_spectral_norm": false,
+    "gin_channels": 256,
+    "emb_channels": 256,
+    "spk_embed_dim": 109
+  }
+}

configs/48k-768.json

@@ -0,0 +1,47 @@
+{
+  "train": {
+    "log_interval": 200,
+    "seed": 1234,
+    "epochs": 20000,
+    "learning_rate": 1e-4,
+    "betas": [0.8, 0.99],
+    "eps": 1e-9,
+    "batch_size": 4,
+    "fp16_run": true,
+    "lr_decay": 0.999875,
+    "segment_size": 11520,
+    "init_lr_ratio": 1,
+    "warmup_epochs": 0,
+    "c_mel": 45,
+    "c_kl": 1.0
+  },
+  "data": {
+    "max_wav_value": 32768.0,
+    "sampling_rate": 48000,
+    "filter_length": 2048,
+    "hop_length": 480,
+    "win_length": 2048,
+    "n_mel_channels": 128,
+    "mel_fmin": 0.0,
+    "mel_fmax": null
+  },
+  "model": {
+    "inter_channels": 192,
+    "hidden_channels": 192,
+    "filter_channels": 768,
+    "n_heads": 2,
+    "n_layers": 6,
+    "kernel_size": 3,
+    "p_dropout": 0,
+    "resblock": "1",
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+    "upsample_rates": [10,6,2,2,2],
+    "upsample_initial_channel": 512,
+    "upsample_kernel_sizes": [16,16,4,4,4],
+    "use_spectral_norm": false,
+    "gin_channels": 256,
+    "emb_channels": 768,
+    "spk_embed_dim": 109
+  }
+}

configs/48k.json

@@ -0,0 +1,47 @@
+{
+  "train": {
+    "log_interval": 200,
+    "seed": 1234,
+    "epochs": 20000,
+    "learning_rate": 1e-4,
+    "betas": [0.8, 0.99],
+    "eps": 1e-9,
+    "batch_size": 4,
+    "fp16_run": true,
+    "lr_decay": 0.999875,
+    "segment_size": 11520,
+    "init_lr_ratio": 1,
+    "warmup_epochs": 0,
+    "c_mel": 45,
+    "c_kl": 1.0
+  },
+  "data": {
+    "max_wav_value": 32768.0,
+    "sampling_rate": 48000,
+    "filter_length": 2048,
+    "hop_length": 480,
+    "win_length": 2048,
+    "n_mel_channels": 128,
+    "mel_fmin": 0.0,
+    "mel_fmax": null
+  },
+  "model": {
+    "inter_channels": 192,
+    "hidden_channels": 192,
+    "filter_channels": 768,
+    "n_heads": 2,
+    "n_layers": 6,
+    "kernel_size": 3,
+    "p_dropout": 0,
+    "resblock": "1",
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+    "upsample_rates": [10,6,2,2,2],
+    "upsample_initial_channel": 512,
+    "upsample_kernel_sizes": [16,16,4,4,4],
+    "use_spectral_norm": false,
+    "gin_channels": 256,
+    "emb_channels": 256,
+    "spk_embed_dim": 109
+  }
+}

dev.py

@@ -0,0 +1,3 @@
+import modules.ui as ui
+
+demo = ui.create_ui()

launch.py

@@ -0,0 +1,139 @@
+import importlib.util
+import os
+import shlex
+import subprocess
+import sys
+
+commandline_args = os.environ.get("COMMANDLINE_ARGS", "")
+sys.argv += shlex.split(commandline_args)
+
+python = sys.executable
+git = os.environ.get("GIT", "git")
+index_url = os.environ.get("INDEX_URL", "")
+stored_commit_hash = None
+skip_install = False
+
+
+def run(command, desc=None, errdesc=None, custom_env=None):
+    if desc is not None:
+        print(desc)
+
+    result = subprocess.run(
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        shell=True,
+        env=os.environ if custom_env is None else custom_env,
+    )
+
+    if result.returncode != 0:
+        message = f"""{errdesc or 'Error running command'}.
+Command: {command}
+Error code: {result.returncode}
+stdout: {result.stdout.decode(encoding="utf8", errors="ignore") if len(result.stdout)>0 else '<empty>'}
+stderr: {result.stderr.decode(encoding="utf8", errors="ignore") if len(result.stderr)>0 else '<empty>'}
+"""
+        raise RuntimeError(message)
+
+    return result.stdout.decode(encoding="utf8", errors="ignore")
+
+
+def check_run(command):
+    result = subprocess.run(
+        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
+    )
+    return result.returncode == 0
+
+
+def is_installed(package):
+    try:
+        spec = importlib.util.find_spec(package)
+    except ModuleNotFoundError:
+        return False
+
+    return spec is not None
+
+
+def commit_hash():
+    global stored_commit_hash
+
+    if stored_commit_hash is not None:
+        return stored_commit_hash
+
+    try:
+        stored_commit_hash = run(f"{git} rev-parse HEAD").strip()
+    except Exception:
+        stored_commit_hash = "<none>"
+
+    return stored_commit_hash
+
+
+def run_pip(args, desc=None):
+    if skip_install:
+        return
+
+    index_url_line = f" --index-url {index_url}" if index_url != "" else ""
+    return run(
+        f'"{python}" -m pip {args} --prefer-binary{index_url_line}',
+        desc=f"Installing {desc}",
+        errdesc=f"Couldn't install {desc}",
+    )
+
+
+def run_python(code, desc=None, errdesc=None):
+    return run(f'"{python}" -c "{code}"', desc, errdesc)
+
+
+def extract_arg(args, name):
+    return [x for x in args if x != name], name in args
+
+
+def prepare_environment():
+    commit = commit_hash()
+
+    print(f"Python {sys.version}")
+    print(f"Commit hash: {commit}")
+
+    torch_command = os.environ.get(
+        "TORCH_COMMAND",
+        "pip install torch torchaudio --extra-index-url https://download.pytorch.org/whl/cu118",
+    )
+
+    sys.argv, skip_install = extract_arg(sys.argv, "--skip-install")
+    if skip_install:
+        return
+
+    sys.argv, reinstall_torch = extract_arg(sys.argv, "--reinstall-torch")
+    ngrok = "--ngrok" in sys.argv
+
+    if reinstall_torch or not is_installed("torch") or not is_installed("torchaudio"):
+        run(
+            f'"{python}" -m {torch_command}',
+            "Installing torch and torchaudio",
+            "Couldn't install torch",
+        )
+
+    if not is_installed("pyngrok") and ngrok:
+        run_pip("install pyngrok", "ngrok")
+
+    run(
+        f'"{python}" -m pip install -r requirements.txt',
+        desc=f"Installing requirements",
+        errdesc=f"Couldn't install requirements",
+    )
+
+
+def start():
+    os.environ["PATH"] = (
+        os.path.join(os.path.dirname(__file__), "bin")
+        + os.pathsep
+        + os.environ.get("PATH", "")
+    )
+    subprocess.run(
+        [python, "webui.py", *sys.argv[1:]],
+    )
+
+
+if __name__ == "__main__":
+    prepare_environment()
+    start()

lib/rvc/attentions.py

@@ -0,0 +1,415 @@
+import math
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from . import commons
+from .modules import LayerNorm
+
+
+class Encoder(nn.Module):
+    def __init__(
+        self,
+        hidden_channels,
+        filter_channels,
+        n_heads,
+        n_layers,
+        kernel_size=1,
+        p_dropout=0.0,
+        window_size=10,
+        **kwargs
+    ):
+        super().__init__()
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.window_size = window_size
+
+        self.drop = nn.Dropout(p_dropout)
+        self.attn_layers = nn.ModuleList()
+        self.norm_layers_1 = nn.ModuleList()
+        self.ffn_layers = nn.ModuleList()
+        self.norm_layers_2 = nn.ModuleList()
+        for i in range(self.n_layers):
+            self.attn_layers.append(
+                MultiHeadAttention(
+                    hidden_channels,
+                    hidden_channels,
+                    n_heads,
+                    p_dropout=p_dropout,
+                    window_size=window_size,
+                )
+            )
+            self.norm_layers_1.append(LayerNorm(hidden_channels))
+            self.ffn_layers.append(
+                FFN(
+                    hidden_channels,
+                    hidden_channels,
+                    filter_channels,
+                    kernel_size,
+                    p_dropout=p_dropout,
+                )
+            )
+            self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+    def forward(self, x, x_mask):
+        attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+        x = x * x_mask
+        for i in range(self.n_layers):
+            y = self.attn_layers[i](x, x, attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_1[i](x + y)
+
+            y = self.ffn_layers[i](x, x_mask)
+            y = self.drop(y)
+            x = self.norm_layers_2[i](x + y)
+        x = x * x_mask
+        return x
+
+
+class Decoder(nn.Module):
+    def __init__(
+        self,
+        hidden_channels,
+        filter_channels,
+        n_heads,
+        n_layers,
+        kernel_size=1,
+        p_dropout=0.0,
+        proximal_bias=False,
+        proximal_init=True,
+        **kwargs
+    ):
+        super().__init__()
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.proximal_bias = proximal_bias
+        self.proximal_init = proximal_init
+
+        self.drop = nn.Dropout(p_dropout)
+        self.self_attn_layers = nn.ModuleList()
+        self.norm_layers_0 = nn.ModuleList()
+        self.encdec_attn_layers = nn.ModuleList()
+        self.norm_layers_1 = nn.ModuleList()
+        self.ffn_layers = nn.ModuleList()
+        self.norm_layers_2 = nn.ModuleList()
+        for i in range(self.n_layers):
+            self.self_attn_layers.append(
+                MultiHeadAttention(
+                    hidden_channels,
+                    hidden_channels,
+                    n_heads,
+                    p_dropout=p_dropout,
+                    proximal_bias=proximal_bias,
+                    proximal_init=proximal_init,
+                )
+            )
+            self.norm_layers_0.append(LayerNorm(hidden_channels))
+            self.encdec_attn_layers.append(
+                MultiHeadAttention(
+                    hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout
+                )
+            )
+            self.norm_layers_1.append(LayerNorm(hidden_channels))
+            self.ffn_layers.append(
+                FFN(
+                    hidden_channels,
+                    hidden_channels,
+                    filter_channels,
+                    kernel_size,
+                    p_dropout=p_dropout,
+                    causal=True,
+                )
+            )
+            self.norm_layers_2.append(LayerNorm(hidden_channels))
+
+    def forward(self, x, x_mask, h, h_mask):
+        """
+        x: decoder input
+        h: encoder output
+        """
+        self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(
+            device=x.device, dtype=x.dtype
+        )
+        encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
+        x = x * x_mask
+        for i in range(self.n_layers):
+            y = self.self_attn_layers[i](x, x, self_attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_0[i](x + y)
+
+            y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
+            y = self.drop(y)
+            x = self.norm_layers_1[i](x + y)
+
+            y = self.ffn_layers[i](x, x_mask)
+            y = self.drop(y)
+            x = self.norm_layers_2[i](x + y)
+        x = x * x_mask
+        return x
+
+
+class MultiHeadAttention(nn.Module):
+    def __init__(
+        self,
+        channels,
+        out_channels,
+        n_heads,
+        p_dropout=0.0,
+        window_size=None,
+        heads_share=True,
+        block_length=None,
+        proximal_bias=False,
+        proximal_init=False,
+    ):
+        super().__init__()
+        assert channels % n_heads == 0
+
+        self.channels = channels
+        self.out_channels = out_channels
+        self.n_heads = n_heads
+        self.p_dropout = p_dropout
+        self.window_size = window_size
+        self.heads_share = heads_share
+        self.block_length = block_length
+        self.proximal_bias = proximal_bias
+        self.proximal_init = proximal_init
+        self.attn = None
+
+        self.k_channels = channels // n_heads
+        self.conv_q = nn.Conv1d(channels, channels, 1)
+        self.conv_k = nn.Conv1d(channels, channels, 1)
+        self.conv_v = nn.Conv1d(channels, channels, 1)
+        self.conv_o = nn.Conv1d(channels, out_channels, 1)
+        self.drop = nn.Dropout(p_dropout)
+
+        if window_size is not None:
+            n_heads_rel = 1 if heads_share else n_heads
+            rel_stddev = self.k_channels**-0.5
+            self.emb_rel_k = nn.Parameter(
+                torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
+                * rel_stddev
+            )
+            self.emb_rel_v = nn.Parameter(
+                torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
+                * rel_stddev
+            )
+
+        nn.init.xavier_uniform_(self.conv_q.weight)
+        nn.init.xavier_uniform_(self.conv_k.weight)
+        nn.init.xavier_uniform_(self.conv_v.weight)
+        if proximal_init:
+            with torch.no_grad():
+                self.conv_k.weight.copy_(self.conv_q.weight)
+                self.conv_k.bias.copy_(self.conv_q.bias)
+
+    def forward(self, x, c, attn_mask=None):
+        q = self.conv_q(x)
+        k = self.conv_k(c)
+        v = self.conv_v(c)
+
+        x, self.attn = self.attention(q, k, v, mask=attn_mask)
+
+        x = self.conv_o(x)
+        return x
+
+    def attention(self, query, key, value, mask=None):
+        # reshape [b, d, t] -> [b, n_h, t, d_k]
+        b, d, t_s, t_t = (*key.size(), query.size(2))
+        query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
+        key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+        value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
+
+        scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
+        if self.window_size is not None:
+            assert (
+                t_s == t_t
+            ), "Relative attention is only available for self-attention."
+            key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
+            rel_logits = self._matmul_with_relative_keys(
+                query / math.sqrt(self.k_channels), key_relative_embeddings
+            )
+            scores_local = self._relative_position_to_absolute_position(rel_logits)
+            scores = scores + scores_local
+        if self.proximal_bias:
+            assert t_s == t_t, "Proximal bias is only available for self-attention."
+            scores = scores + self._attention_bias_proximal(t_s).to(
+                device=scores.device, dtype=scores.dtype
+            )
+        if mask is not None:
+            scores = scores.masked_fill(mask == 0, -1e4)
+            if self.block_length is not None:
+                assert (
+                    t_s == t_t
+                ), "Local attention is only available for self-attention."
+                block_mask = (
+                    torch.ones_like(scores)
+                    .triu(-self.block_length)
+                    .tril(self.block_length)
+                )
+                scores = scores.masked_fill(block_mask == 0, -1e4)
+        p_attn = F.softmax(scores, dim=-1)  # [b, n_h, t_t, t_s]
+        p_attn = self.drop(p_attn)
+        output = torch.matmul(p_attn, value)
+        if self.window_size is not None:
+            relative_weights = self._absolute_position_to_relative_position(p_attn)
+            value_relative_embeddings = self._get_relative_embeddings(
+                self.emb_rel_v, t_s
+            )
+            output = output + self._matmul_with_relative_values(
+                relative_weights, value_relative_embeddings
+            )
+        output = (
+            output.transpose(2, 3).contiguous().view(b, d, t_t)
+        )  # [b, n_h, t_t, d_k] -> [b, d, t_t]
+        return output, p_attn
+
+    def _matmul_with_relative_values(self, x, y):
+        """
+        x: [b, h, l, m]
+        y: [h or 1, m, d]
+        ret: [b, h, l, d]
+        """
+        ret = torch.matmul(x, y.unsqueeze(0))
+        return ret
+
+    def _matmul_with_relative_keys(self, x, y):
+        """
+        x: [b, h, l, d]
+        y: [h or 1, m, d]
+        ret: [b, h, l, m]
+        """
+        ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
+        return ret
+
+    def _get_relative_embeddings(self, relative_embeddings, length):
+        max_relative_position = 2 * self.window_size + 1
+        # Pad first before slice to avoid using cond ops.
+        pad_length = max(length - (self.window_size + 1), 0)
+        slice_start_position = max((self.window_size + 1) - length, 0)
+        slice_end_position = slice_start_position + 2 * length - 1
+        if pad_length > 0:
+            padded_relative_embeddings = F.pad(
+                relative_embeddings,
+                commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]),
+            )
+        else:
+            padded_relative_embeddings = relative_embeddings
+        used_relative_embeddings = padded_relative_embeddings[
+            :, slice_start_position:slice_end_position
+        ]
+        return used_relative_embeddings
+
+    def _relative_position_to_absolute_position(self, x):
+        """
+        x: [b, h, l, 2*l-1]
+        ret: [b, h, l, l]
+        """
+        batch, heads, length, _ = x.size()
+        # Concat columns of pad to shift from relative to absolute indexing.
+        x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]]))
+
+        # Concat extra elements so to add up to shape (len+1, 2*len-1).
+        x_flat = x.view([batch, heads, length * 2 * length])
+        x_flat = F.pad(
+            x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [0, length - 1]])
+        )
+
+        # Reshape and slice out the padded elements.
+        x_final = x_flat.view([batch, heads, length + 1, 2 * length - 1])[
+            :, :, :length, length - 1 :
+        ]
+        return x_final
+
+    def _absolute_position_to_relative_position(self, x):
+        """
+        x: [b, h, l, l]
+        ret: [b, h, l, 2*l-1]
+        """
+        batch, heads, length, _ = x.size()
+        # padd along column
+        x = F.pad(
+            x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]])
+        )
+        x_flat = x.view([batch, heads, length**2 + length * (length - 1)])
+        # add 0's in the beginning that will skew the elements after reshape
+        x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
+        x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:]
+        return x_final
+
+    def _attention_bias_proximal(self, length):
+        """Bias for self-attention to encourage attention to close positions.
+        Args:
+          length: an integer scalar.
+        Returns:
+          a Tensor with shape [1, 1, length, length]
+        """
+        r = torch.arange(length, dtype=torch.float32)
+        diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
+        return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
+
+
+class FFN(nn.Module):
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+        filter_channels,
+        kernel_size,
+        p_dropout=0.0,
+        activation=None,
+        causal=False,
+    ):
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.filter_channels = filter_channels
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.activation = activation
+        self.causal = causal
+
+        if causal:
+            self.padding = self._causal_padding
+        else:
+            self.padding = self._same_padding
+
+        self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
+        self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
+        self.drop = nn.Dropout(p_dropout)
+
+    def forward(self, x, x_mask):
+        x = self.conv_1(self.padding(x * x_mask))
+        if self.activation == "gelu":
+            x = x * torch.sigmoid(1.702 * x)
+        else:
+            x = torch.relu(x)
+        x = self.drop(x)
+        x = self.conv_2(self.padding(x * x_mask))
+        return x * x_mask
+
+    def _causal_padding(self, x):
+        if self.kernel_size == 1:
+            return x
+        pad_l = self.kernel_size - 1
+        pad_r = 0
+        padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+        x = F.pad(x, commons.convert_pad_shape(padding))
+        return x
+
+    def _same_padding(self, x):
+        if self.kernel_size == 1:
+            return x
+        pad_l = (self.kernel_size - 1) // 2
+        pad_r = self.kernel_size // 2
+        padding = [[0, 0], [0, 0], [pad_l, pad_r]]
+        x = F.pad(x, commons.convert_pad_shape(padding))
+        return x

lib/rvc/checkpoints.py

@@ -0,0 +1,149 @@
+import os
+from collections import OrderedDict
+from typing import *
+
+import torch
+
+
+def write_config(state_dict: Dict[str, Any], cfg: Dict[str, Any]):
+    state_dict["config"] = []
+    for key, x in cfg.items():
+        state_dict["config"].append(x)
+    state_dict["params"] = cfg
+
+
+def create_trained_model(
+    weights: Dict[str, Any],
+    version: Literal["v1", "v2"],
+    sr: str,
+    f0: bool,
+    emb_name: str,
+    emb_ch: int,
+    emb_output_layer: int,
+    epoch: int,
+    speaker_info: Optional[dict[str, int]]
+):
+    state_dict = OrderedDict()
+    state_dict["weight"] = {}
+    for key in weights.keys():
+        if "enc_q" in key:
+            continue
+        state_dict["weight"][key] = weights[key].half()
+    if sr == "40k":
+        write_config(
+            state_dict,
+            {
+                "spec_channels": 1025,
+                "segment_size": 32,
+                "inter_channels": 192,
+                "hidden_channels": 192,
+                "filter_channels": 768,
+                "n_heads": 2,
+                "n_layers": 6,
+                "kernel_size": 3,
+                "p_dropout": 0,
+                "resblock": "1",
+                "resblock_kernel_sizes": [3, 7, 11],
+                "resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]],
+                "upsample_rates": [10, 10, 2, 2],
+                "upsample_initial_channel": 512,
+                "upsample_kernel_sizes": [16, 16, 4, 4],
+                "spk_embed_dim": 109 if speaker_info is None else len(speaker_info),
+                "gin_channels": 256,
+                "emb_channels": emb_ch,
+                "sr": 40000,
+            },
+        )
+    elif sr == "48k":
+        write_config(
+            state_dict,
+            {
+                "spec_channels": 1025,
+                "segment_size": 32,
+                "inter_channels": 192,
+                "hidden_channels": 192,
+                "filter_channels": 768,
+                "n_heads": 2,
+                "n_layers": 6,
+                "kernel_size": 3,
+                "p_dropout": 0,
+                "resblock": "1",
+                "resblock_kernel_sizes": [3, 7, 11],
+                "resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]],
+                "upsample_rates": [10, 6, 2, 2, 2],
+                "upsample_initial_channel": 512,
+                "upsample_kernel_sizes": [16, 16, 4, 4, 4],
+                "spk_embed_dim": 109 if speaker_info is None else len(speaker_info),
+                "gin_channels": 256,
+                "emb_channels": emb_ch,
+                "sr": 48000,
+            },
+        )
+    elif sr == "32k":
+        write_config(
+            state_dict,
+            {
+                "spec_channels": 513,
+                "segment_size": 32,
+                "inter_channels": 192,
+                "hidden_channels": 192,
+                "filter_channels": 768,
+                "n_heads": 2,
+                "n_layers": 6,
+                "kernel_size": 3,
+                "p_dropout": 0,
+                "resblock": "1",
+                "resblock_kernel_sizes": [3, 7, 11],
+                "resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]],
+                "upsample_rates": [10, 4, 2, 2, 2],
+                "upsample_initial_channel": 512,
+                "upsample_kernel_sizes": [16, 16, 4, 4, 4],
+                "spk_embed_dim": 109 if speaker_info is None else len(speaker_info),
+                "gin_channels": 256,
+                "emb_channels": emb_ch,
+                "sr": 32000,
+            },
+        )
+    state_dict["version"] = version
+    state_dict["info"] = f"{epoch}epoch"
+    state_dict["sr"] = sr
+    state_dict["f0"] = 1 if f0 else 0
+    state_dict["embedder_name"] = emb_name
+    state_dict["embedder_output_layer"] = emb_output_layer
+    if not speaker_info is None:
+        state_dict["speaker_info"] = {str(v): str(k) for k, v in speaker_info.items()}
+    return state_dict
+
+
+def save(
+    model,
+    version: Literal["v1", "v2"],
+    sr: str,
+    f0: bool,
+    emb_name: str,
+    emb_ch: int,
+    emb_output_layer: int,
+    filepath: str,
+    epoch: int,
+    speaker_info: Optional[dict[str, int]]
+):
+    if hasattr(model, "module"):
+        state_dict = model.module.state_dict()
+    else:
+        state_dict = model.state_dict()
+
+    print(f"save: emb_name: {emb_name} {emb_ch}")
+
+    state_dict = create_trained_model(
+        state_dict,
+        version,
+        sr,
+        f0,
+        emb_name,
+        emb_ch,
+        emb_output_layer,
+        epoch,
+        speaker_info
+    )
+    os.makedirs(os.path.dirname(filepath), exist_ok=True)
+    torch.save(state_dict, filepath)

lib/rvc/commons.py

@@ -0,0 +1,163 @@
+import math
+
+import torch
+from torch.nn import functional as F
+
+
+def init_weights(m, mean=0.0, std=0.01):
+    classname = m.__class__.__name__
+    if classname.find("Conv") != -1:
+        m.weight.data.normal_(mean, std)
+
+
+def get_padding(kernel_size, dilation=1):
+    return int((kernel_size * dilation - dilation) / 2)
+
+
+def convert_pad_shape(pad_shape):
+    l = pad_shape[::-1]
+    pad_shape = [item for sublist in l for item in sublist]
+    return pad_shape
+
+
+def kl_divergence(m_p, logs_p, m_q, logs_q):
+    """KL(P||Q)"""
+    kl = (logs_q - logs_p) - 0.5
+    kl += (
+        0.5 * (torch.exp(2.0 * logs_p) + ((m_p - m_q) ** 2)) * torch.exp(-2.0 * logs_q)
+    )
+    return kl
+
+
+def rand_gumbel(shape):
+    """Sample from the Gumbel distribution, protect from overflows."""
+    uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
+    return -torch.log(-torch.log(uniform_samples))
+
+
+def rand_gumbel_like(x):
+    g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
+    return g
+
+
+def slice_segments(x, ids_str, segment_size=4):
+    ret = torch.zeros_like(x[:, :, :segment_size])
+    for i in range(x.size(0)):
+        idx_str = ids_str[i]
+        idx_end = idx_str + segment_size
+        ret[i] = x[i, :, idx_str:idx_end]
+    return ret
+
+
+def slice_segments2(x, ids_str, segment_size=4):
+    ret = torch.zeros_like(x[:, :segment_size])
+    for i in range(x.size(0)):
+        idx_str = ids_str[i]
+        idx_end = idx_str + segment_size
+        ret[i] = x[i, idx_str:idx_end]
+    return ret
+
+
+def rand_slice_segments(x, x_lengths=None, segment_size=4):
+    b, d, t = x.size()
+    if x_lengths is None:
+        x_lengths = t
+    ids_str_max = x_lengths - segment_size + 1
+    ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
+    ret = slice_segments(x, ids_str, segment_size)
+    return ret, ids_str
+
+
+def get_timing_signal_1d(length, channels, min_timescale=1.0, max_timescale=1.0e4):
+    position = torch.arange(length, dtype=torch.float)
+    num_timescales = channels // 2
+    log_timescale_increment = math.log(float(max_timescale) / float(min_timescale)) / (
+        num_timescales - 1
+    )
+    inv_timescales = min_timescale * torch.exp(
+        torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment
+    )
+    scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
+    signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
+    signal = F.pad(signal, [0, 0, 0, channels % 2])
+    signal = signal.view(1, channels, length)
+    return signal
+
+
+def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
+    b, channels, length = x.size()
+    signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
+    return x + signal.to(dtype=x.dtype, device=x.device)
+
+
+def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
+    b, channels, length = x.size()
+    signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
+    return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
+
+
+def subsequent_mask(length):
+    mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
+    return mask
+
+
+@torch.jit.script
+def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
+    n_channels_int = n_channels[0]
+    in_act = input_a + input_b
+    t_act = torch.tanh(in_act[:, :n_channels_int, :])
+    s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
+    acts = t_act * s_act
+    return acts
+
+
+def convert_pad_shape(pad_shape):
+    l = pad_shape[::-1]
+    pad_shape = [item for sublist in l for item in sublist]
+    return pad_shape
+
+
+def shift_1d(x):
+    x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
+    return x
+
+
+def sequence_mask(length, max_length=None):
+    if max_length is None:
+        max_length = length.max()
+    x = torch.arange(max_length, dtype=length.dtype, device=length.device)
+    return x.unsqueeze(0) < length.unsqueeze(1)
+
+
+def generate_path(duration, mask):
+    """
+    duration: [b, 1, t_x]
+    mask: [b, 1, t_y, t_x]
+    """
+    b, _, t_y, t_x = mask.shape
+    cum_duration = torch.cumsum(duration, -1)
+
+    cum_duration_flat = cum_duration.view(b * t_x)
+    path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
+    path = path.view(b, t_x, t_y)
+    path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
+    path = path.unsqueeze(1).transpose(2, 3) * mask
+    return path
+
+
+def clip_grad_value_(parameters, clip_value, norm_type=2):
+    if isinstance(parameters, torch.Tensor):
+        parameters = [parameters]
+    parameters = list(filter(lambda p: p.grad is not None, parameters))
+    norm_type = float(norm_type)
+    if clip_value is not None:
+        clip_value = float(clip_value)
+
+    total_norm = 0
+    for p in parameters:
+        param_norm = p.grad.data.norm(norm_type)
+        total_norm += param_norm.item() ** norm_type
+        if clip_value is not None:
+            p.grad.data.clamp_(min=-clip_value, max=clip_value)
+    total_norm = total_norm ** (1.0 / norm_type)
+    return total_norm

lib/rvc/config.py

@@ -0,0 +1,71 @@
+from typing import *
+
+from pydantic import BaseModel
+
+
+class TrainConfigTrain(BaseModel):
+    log_interval: int
+    seed: int
+    epochs: int
+    learning_rate: float
+    betas: List[float]
+    eps: float
+    batch_size: int
+    fp16_run: bool
+    lr_decay: float
+    segment_size: int
+    init_lr_ratio: int
+    warmup_epochs: int
+    c_mel: int
+    c_kl: float
+
+
+class TrainConfigData(BaseModel):
+    max_wav_value: float
+    sampling_rate: int
+    filter_length: int
+    hop_length: int
+    win_length: int
+    n_mel_channels: int
+    mel_fmin: float
+    mel_fmax: Any
+
+
+class TrainConfigModel(BaseModel):
+    inter_channels: int
+    hidden_channels: int
+    filter_channels: int
+    n_heads: int
+    n_layers: int
+    kernel_size: int
+    p_dropout: int
+    resblock: str
+    resblock_kernel_sizes: List[int]
+    resblock_dilation_sizes: List[List[int]]
+    upsample_rates: List[int]
+    upsample_initial_channel: int
+    upsample_kernel_sizes: List[int]
+    use_spectral_norm: bool
+    gin_channels: int
+    emb_channels: int
+    spk_embed_dim: int
+
+
+class TrainConfig(BaseModel):
+    version: Literal["v1", "v2"] = "v2"
+    train: TrainConfigTrain
+    data: TrainConfigData
+    model: TrainConfigModel
+
+
+class DatasetMetaItem(BaseModel):
+    gt_wav: str
+    co256: str
+    f0: Optional[str]
+    f0nsf: Optional[str]
+    speaker_id: int
+
+
+class DatasetMetadata(BaseModel):
+    files: Dict[str, DatasetMetaItem]
+    # mute: DatasetMetaItem

lib/rvc/data_utils.py

@@ -0,0 +1,515 @@
+import os
+import traceback
+
+import numpy as np
+import torch
+import torch.utils.data
+
+from .config import DatasetMetadata, DatasetMetaItem, TrainConfigData
+from .mel_processing import spectrogram_torch
+from .utils import load_wav_to_torch
+
+
+class TextAudioLoader(torch.utils.data.Dataset):
+    """
+    1) loads audio, text pairs
+    2) normalizes text and converts them to sequences of integers
+    3) computes spectrograms from audio files.
+    """
+
+    def __init__(self, dataset_meta: DatasetMetadata, data: TrainConfigData):
+        self.dataset_meta = dataset_meta
+        self.max_wav_value = data.max_wav_value
+        self.sampling_rate = data.sampling_rate
+        self.filter_length = data.filter_length
+        self.hop_length = data.hop_length
+        self.win_length = data.win_length
+        self.sampling_rate = data.sampling_rate
+        self.min_text_len = getattr(data, "min_text_len", 1)
+        self.max_text_len = getattr(data, "max_text_len", 5000)
+        self._filter()
+
+    def _filter(self):
+        """
+        Filter text & store spec lengths
+        """
+        # Store spectrogram lengths for Bucketing
+        # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2)
+        # spec_length = wav_length // hop_length
+        lengths = []
+        for key, data in self.dataset_meta.files.items():
+            if (
+                self.min_text_len <= len(data.co256)
+                and len(data.co256) <= self.max_text_len
+            ):
+                lengths.append(os.path.getsize(data.gt_wav) // (2 * self.hop_length))
+            else:
+                del self.dataset_meta.files[key]
+        self.lengths = lengths
+
+    def get_sid(self, sid):
+        sid = torch.LongTensor([int(sid)])
+        return sid
+
+    def get_audio_text_pair(self, data: DatasetMetaItem):
+        # separate filename and text
+        file = data.gt_wav
+        phone = data.co256
+        dv = data.speaker_id
+
+        phone = self.get_labels(phone)
+        spec, wav = self.get_audio(file)
+        dv = self.get_sid(dv)
+
+        len_phone = phone.size()[0]
+        len_spec = spec.size()[-1]
+        if len_phone != len_spec:
+            len_min = min(len_phone, len_spec)
+            len_wav = len_min * self.hop_length
+            spec = spec[:, :len_min]
+            wav = wav[:, :len_wav]
+            phone = phone[:len_min, :]
+        return (spec, wav, phone, dv)
+
+    def get_labels(self, phone):
+        phone = np.load(phone)
+        phone = np.repeat(phone, 2, axis=0)
+        n_num = min(phone.shape[0], 900)  # DistributedBucketSampler
+        phone = phone[:n_num, :]
+        phone = torch.FloatTensor(phone)
+        return phone
+
+    def get_audio(self, filename):
+        audio, sampling_rate = load_wav_to_torch(filename)
+        if sampling_rate != self.sampling_rate:
+            raise ValueError(
+                "{} SR doesn't match target {} SR".format(
+                    sampling_rate, self.sampling_rate
+                )
+            )
+        # audio_norm = audio / self.max_wav_value
+        audio_norm = audio.unsqueeze(0)
+        spec_filename = filename.replace(".wav", ".spec.pt")
+        if os.path.exists(spec_filename):
+            try:
+                spec = torch.load(spec_filename)
+            except:
+                print(spec_filename, traceback.format_exc())
+                spec = spectrogram_torch(
+                    audio_norm,
+                    self.filter_length,
+                    self.sampling_rate,
+                    self.hop_length,
+                    self.win_length,
+                    center=False,
+                )
+                spec = torch.squeeze(spec, 0)
+                torch.save(spec, spec_filename, _use_new_zipfile_serialization=False)
+        else:
+            spec = spectrogram_torch(
+                audio_norm,
+                self.filter_length,
+                self.sampling_rate,
+                self.hop_length,
+                self.win_length,
+                center=False,
+            )
+            spec = torch.squeeze(spec, 0)
+            torch.save(spec, spec_filename, _use_new_zipfile_serialization=False)
+        return spec, audio_norm
+
+    def __getitem__(self, index):
+        _, data = list(self.dataset_meta.files.items())[index]
+        return self.get_audio_text_pair(data)
+
+    def __len__(self):
+        return len(self.dataset_meta.files)
+
+
+class TextAudioLoaderMultiNSFsid(torch.utils.data.Dataset):
+    """
+    1) loads audio, text pairs
+    2) normalizes text and converts them to sequences of integers
+    3) computes spectrograms from audio files.
+    """
+
+    def __init__(self, dataset_meta: DatasetMetadata, data: TrainConfigData):
+        self.dataset_meta = dataset_meta
+        self.max_wav_value = data.max_wav_value
+        self.sampling_rate = data.sampling_rate
+        self.filter_length = data.filter_length
+        self.hop_length = data.hop_length
+        self.win_length = data.win_length
+        self.sampling_rate = data.sampling_rate
+        self.min_text_len = getattr(data, "min_text_len", 1)
+        self.max_text_len = getattr(data, "max_text_len", 5000)
+        self._filter()
+
+    def _filter(self):
+        """
+        Filter text & store spec lengths
+        """
+        # Store spectrogram lengths for Bucketing
+        # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2)
+        # spec_length = wav_length // hop_length
+        lengths = []
+        for key, data in self.dataset_meta.files.items():
+            if (
+                self.min_text_len <= len(data.co256)
+                and len(data.co256) <= self.max_text_len
+            ):
+                lengths.append(os.path.getsize(data.gt_wav) // (2 * self.hop_length))
+            else:
+                del self.dataset_meta.files[key]
+        self.lengths = lengths
+
+    def get_sid(self, sid):
+        sid = torch.LongTensor([int(sid)])
+        return sid
+
+    def get_audio_text_pair(self, data: DatasetMetaItem):
+        # separate filename and text
+        file = data.gt_wav
+        phone = data.co256
+        pitch = data.f0
+        pitchf = data.f0nsf
+        dv = data.speaker_id
+
+        phone, pitch, pitchf = self.get_labels(phone, pitch, pitchf)
+        spec, wav = self.get_audio(file)
+        dv = self.get_sid(dv)
+
+        len_phone = phone.size()[0]
+        len_spec = spec.size()[-1]
+        # print(123,phone.shape,pitch.shape,spec.shape)
+        if len_phone != len_spec:
+            len_min = min(len_phone, len_spec)
+            # amor
+            len_wav = len_min * self.hop_length
+
+            spec = spec[:, :len_min]
+            wav = wav[:, :len_wav]
+
+            phone = phone[:len_min, :]
+            pitch = pitch[:len_min]
+            pitchf = pitchf[:len_min]
+
+        return (spec, wav, phone, pitch, pitchf, dv)
+
+    def get_labels(self, phone, pitch, pitchf):
+        phone = np.load(phone)
+        phone = np.repeat(phone, 2, axis=0)
+        pitch = np.load(pitch)
+        pitchf = np.load(pitchf)
+        n_num = min(phone.shape[0], 900)  # DistributedBucketSampler
+        # print(234,phone.shape,pitch.shape)
+        phone = phone[:n_num, :]
+        pitch = pitch[:n_num]
+        pitchf = pitchf[:n_num]
+        phone = torch.FloatTensor(phone)
+        pitch = torch.LongTensor(pitch)
+        pitchf = torch.FloatTensor(pitchf)
+        return phone, pitch, pitchf
+
+    def get_audio(self, filename):
+        audio, sampling_rate = load_wav_to_torch(filename)
+        if sampling_rate != self.sampling_rate:
+            raise ValueError(
+                "{} SR doesn't match target {} SR".format(
+                    sampling_rate, self.sampling_rate
+                )
+            )
+        # audio_norm = audio / self.max_wav_value
+        audio_norm = audio.unsqueeze(0)
+        spec_filename = filename.replace(".wav", ".spec.pt")
+        if os.path.exists(spec_filename):
+            try:
+                spec = torch.load(spec_filename)
+            except:
+                print(spec_filename, traceback.format_exc())
+                spec = spectrogram_torch(
+                    audio_norm,
+                    self.filter_length,
+                    self.sampling_rate,
+                    self.hop_length,
+                    self.win_length,
+                    center=False,
+                )
+                spec = torch.squeeze(spec, 0)
+                torch.save(spec, spec_filename, _use_new_zipfile_serialization=False)
+        else:
+            spec = spectrogram_torch(
+                audio_norm,
+                self.filter_length,
+                self.sampling_rate,
+                self.hop_length,
+                self.win_length,
+                center=False,
+            )
+            spec = torch.squeeze(spec, 0)
+            torch.save(spec, spec_filename, _use_new_zipfile_serialization=False)
+        return spec, audio_norm
+
+    def __getitem__(self, index):
+        _, data = list(self.dataset_meta.files.items())[index]
+        return self.get_audio_text_pair(data)
+
+    def __len__(self):
+        return len(self.dataset_meta.files)
+
+
+class TextAudioCollateMultiNSFsid:
+    """Zero-pads model inputs and targets"""
+
+    def __init__(self, return_ids=False):
+        self.return_ids = return_ids
+
+    def __call__(self, batch):
+        """Collate's training batch from normalized text and aduio
+        PARAMS
+        ------
+        batch: [text_normalized, spec_normalized, wav_normalized]
+        """
+        # Right zero-pad all one-hot text sequences to max input length
+        _, ids_sorted_decreasing = torch.sort(
+            torch.LongTensor([x[0].size(1) for x in batch]), dim=0, descending=True
+        )
+
+        max_spec_len = max([x[0].size(1) for x in batch])
+        max_wave_len = max([x[1].size(1) for x in batch])
+        spec_lengths = torch.LongTensor(len(batch))
+        wave_lengths = torch.LongTensor(len(batch))
+        spec_padded = torch.FloatTensor(len(batch), batch[0][0].size(0), max_spec_len)
+        wave_padded = torch.FloatTensor(len(batch), 1, max_wave_len)
+        spec_padded.zero_()
+        wave_padded.zero_()
+
+        max_phone_len = max([x[2].size(0) for x in batch])
+        phone_lengths = torch.LongTensor(len(batch))
+        phone_padded = torch.FloatTensor(
+            len(batch), max_phone_len, batch[0][2].shape[1]
+        )  # (spec, wav, phone, pitch)
+        pitch_padded = torch.LongTensor(len(batch), max_phone_len)
+        pitchf_padded = torch.FloatTensor(len(batch), max_phone_len)
+        phone_padded.zero_()
+        pitch_padded.zero_()
+        pitchf_padded.zero_()
+        # dv = torch.FloatTensor(len(batch), 256)#gin=256
+        sid = torch.LongTensor(len(batch))
+
+        for i in range(len(ids_sorted_decreasing)):
+            row = batch[ids_sorted_decreasing[i]]
+
+            spec = row[0]
+            spec_padded[i, :, : spec.size(1)] = spec
+            spec_lengths[i] = spec.size(1)
+
+            wave = row[1]
+            wave_padded[i, :, : wave.size(1)] = wave
+            wave_lengths[i] = wave.size(1)
+
+            phone = row[2]
+            phone_padded[i, : phone.size(0), :] = phone
+            phone_lengths[i] = phone.size(0)
+
+            pitch = row[3]
+            pitch_padded[i, : pitch.size(0)] = pitch
+            pitchf = row[4]
+            pitchf_padded[i, : pitchf.size(0)] = pitchf
+
+            # dv[i] = row[5]
+            sid[i] = row[5]
+
+        return (
+            phone_padded,
+            phone_lengths,
+            pitch_padded,
+            pitchf_padded,
+            spec_padded,
+            spec_lengths,
+            wave_padded,
+            wave_lengths,
+            # dv
+            sid,
+        )
+
+
+class TextAudioCollate:
+    """Zero-pads model inputs and targets"""
+
+    def __init__(self, return_ids=False):
+        self.return_ids = return_ids
+
+    def __call__(self, batch):
+        """Collate's training batch from normalized text and aduio
+        PARAMS
+        ------
+        batch: [text_normalized, spec_normalized, wav_normalized]
+        """
+        # Right zero-pad all one-hot text sequences to max input length
+        _, ids_sorted_decreasing = torch.sort(
+            torch.LongTensor([x[0].size(1) for x in batch]), dim=0, descending=True
+        )
+
+        max_spec_len = max([x[0].size(1) for x in batch])
+        max_wave_len = max([x[1].size(1) for x in batch])
+        spec_lengths = torch.LongTensor(len(batch))
+        wave_lengths = torch.LongTensor(len(batch))
+        spec_padded = torch.FloatTensor(len(batch), batch[0][0].size(0), max_spec_len)
+        wave_padded = torch.FloatTensor(len(batch), 1, max_wave_len)
+        spec_padded.zero_()
+        wave_padded.zero_()
+
+        max_phone_len = max([x[2].size(0) for x in batch])
+        phone_lengths = torch.LongTensor(len(batch))
+        phone_padded = torch.FloatTensor(
+            len(batch), max_phone_len, batch[0][2].shape[1]
+        )
+        phone_padded.zero_()
+        sid = torch.LongTensor(len(batch))
+
+        for i in range(len(ids_sorted_decreasing)):
+            row = batch[ids_sorted_decreasing[i]]
+
+            spec = row[0]
+            spec_padded[i, :, : spec.size(1)] = spec
+            spec_lengths[i] = spec.size(1)
+
+            wave = row[1]
+            wave_padded[i, :, : wave.size(1)] = wave
+            wave_lengths[i] = wave.size(1)
+
+            phone = row[2]
+            phone_padded[i, : phone.size(0), :] = phone
+            phone_lengths[i] = phone.size(0)
+
+            sid[i] = row[3]
+
+        return (
+            phone_padded,
+            phone_lengths,
+            spec_padded,
+            spec_lengths,
+            wave_padded,
+            wave_lengths,
+            sid,
+        )
+
+
+class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
+    """
+    Maintain similar input lengths in a batch.
+    Length groups are specified by boundaries.
+    Ex) boundaries = [b1, b2, b3] -> any batch is included either {x | b1 < length(x) <=b2} or {x | b2 < length(x) <= b3}.
+
+    It removes samples which are not included in the boundaries.
+    Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded.
+    """
+
+    def __init__(
+        self,
+        dataset,
+        batch_size,
+        boundaries,
+        num_replicas=None,
+        rank=None,
+        shuffle=True,
+    ):
+        super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle)
+        self.lengths = dataset.lengths
+        self.batch_size = batch_size
+        self.boundaries = boundaries
+
+        self.buckets, self.num_samples_per_bucket = self._create_buckets()
+        self.total_size = sum(self.num_samples_per_bucket)
+        self.num_samples = self.total_size // self.num_replicas
+
+    def _create_buckets(self):
+        buckets = [[] for _ in range(len(self.boundaries) - 1)]
+        for i in range(len(self.lengths)):
+            length = self.lengths[i]
+            idx_bucket = self._bisect(length)
+            if idx_bucket != -1:
+                buckets[idx_bucket].append(i)
+
+        for i in range(len(buckets) - 1, -1, -1):  #
+            if len(buckets[i]) == 0:
+                buckets.pop(i)
+                self.boundaries.pop(i + 1)
+
+        num_samples_per_bucket = []
+        for i in range(len(buckets)):
+            len_bucket = len(buckets[i])
+            total_batch_size = self.num_replicas * self.batch_size
+            rem = (
+                total_batch_size - (len_bucket % total_batch_size)
+            ) % total_batch_size
+            num_samples_per_bucket.append(len_bucket + rem)
+        return buckets, num_samples_per_bucket
+
+    def __iter__(self):
+        # deterministically shuffle based on epoch
+        g = torch.Generator()
+        g.manual_seed(self.epoch)
+
+        indices = []
+        if self.shuffle:
+            for bucket in self.buckets:
+                indices.append(torch.randperm(len(bucket), generator=g).tolist())
+        else:
+            for bucket in self.buckets:
+                indices.append(list(range(len(bucket))))
+
+        batches = []
+        for i in range(len(self.buckets)):
+            bucket = self.buckets[i]
+            len_bucket = len(bucket)
+            ids_bucket = indices[i]
+            num_samples_bucket = self.num_samples_per_bucket[i]
+
+            # add extra samples to make it evenly divisible
+            rem = num_samples_bucket - len_bucket
+            ids_bucket = (
+                ids_bucket
+                + ids_bucket * (rem // len_bucket)
+                + ids_bucket[: (rem % len_bucket)]
+            )
+
+            # subsample
+            ids_bucket = ids_bucket[self.rank :: self.num_replicas]
+
+            # batching
+            for j in range(len(ids_bucket) // self.batch_size):
+                batch = [
+                    bucket[idx]
+                    for idx in ids_bucket[
+                        j * self.batch_size : (j + 1) * self.batch_size
+                    ]
+                ]
+                batches.append(batch)
+
+        if self.shuffle:
+            batch_ids = torch.randperm(len(batches), generator=g).tolist()
+            batches = [batches[i] for i in batch_ids]
+        self.batches = batches
+
+        assert len(self.batches) * self.batch_size == self.num_samples
+        return iter(self.batches)
+
+    def _bisect(self, x, lo=0, hi=None):
+        if hi is None:
+            hi = len(self.boundaries) - 1
+
+        if hi > lo:
+            mid = (hi + lo) // 2
+            if self.boundaries[mid] < x and x <= self.boundaries[mid + 1]:
+                return mid
+            elif x <= self.boundaries[mid]:
+                return self._bisect(x, lo, mid)
+            else:
+                return self._bisect(x, mid + 1, hi)
+        else:
+            return -1
+
+    def __len__(self):
+        return self.num_samples // self.batch_size

lib/rvc/losses.py

@@ -0,0 +1,58 @@
+import torch
+
+
+def feature_loss(fmap_r, fmap_g):
+    loss = 0
+    for dr, dg in zip(fmap_r, fmap_g):
+        for rl, gl in zip(dr, dg):
+            rl = rl.float().detach()
+            gl = gl.float()
+            loss += torch.mean(torch.abs(rl - gl))
+
+    return loss * 2
+
+
+def discriminator_loss(disc_real_outputs, disc_generated_outputs):
+    loss = 0
+    r_losses = []
+    g_losses = []
+    for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
+        dr = dr.float()
+        dg = dg.float()
+        r_loss = torch.mean((1 - dr) ** 2)
+        g_loss = torch.mean(dg**2)
+        loss += r_loss + g_loss
+        r_losses.append(r_loss.item())
+        g_losses.append(g_loss.item())
+
+    return loss, r_losses, g_losses
+
+
+def generator_loss(disc_outputs):
+    loss = 0
+    gen_losses = []
+    for dg in disc_outputs:
+        dg = dg.float()
+        l = torch.mean((1 - dg) ** 2)
+        gen_losses.append(l)
+        loss += l
+
+    return loss, gen_losses
+
+
+def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
+    """
+    z_p, logs_q: [b, h, t_t]
+    m_p, logs_p: [b, h, t_t]
+    """
+    z_p = z_p.float()
+    logs_q = logs_q.float()
+    m_p = m_p.float()
+    logs_p = logs_p.float()
+    z_mask = z_mask.float()
+
+    kl = logs_p - logs_q - 0.5
+    kl += 0.5 * ((z_p - m_p) ** 2) * torch.exp(-2.0 * logs_p)
+    kl = torch.sum(kl * z_mask)
+    l = kl / torch.sum(z_mask)
+    return l

lib/rvc/mel_processing.py

@@ -0,0 +1,113 @@
+import torch
+import torch.utils.data
+from librosa.filters import mel as librosa_mel_fn
+
+MAX_WAV_VALUE = 32768.0
+
+
+def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
+    """
+    PARAMS
+    ------
+    C: compression factor
+    """
+    return torch.log(torch.clamp(x, min=clip_val) * C)
+
+
+def dynamic_range_decompression_torch(x, C=1):
+    """
+    PARAMS
+    ------
+    C: compression factor used to compress
+    """
+    return torch.exp(x) / C
+
+
+def spectral_normalize_torch(magnitudes):
+    return dynamic_range_compression_torch(magnitudes)
+
+
+def spectral_de_normalize_torch(magnitudes):
+    return dynamic_range_decompression_torch(magnitudes)
+
+
+mel_basis = {}
+hann_window = {}
+
+
+def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False):
+    if torch.min(y) < -1.07:
+        print("min value is ", torch.min(y))
+    if torch.max(y) > 1.07:
+        print("max value is ", torch.max(y))
+
+    global hann_window
+    dtype_device = str(y.dtype) + "_" + str(y.device)
+    wnsize_dtype_device = str(win_size) + "_" + dtype_device
+    if wnsize_dtype_device not in hann_window:
+        hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(
+            dtype=y.dtype, device=y.device
+        )
+
+    y = torch.nn.functional.pad(
+        y.unsqueeze(1),
+        (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)),
+        mode="reflect",
+    )
+    y = y.squeeze(1)
+
+    # mps does not support torch.stft.
+    if y.device.type == "mps":
+        i = y.cpu()
+        win = hann_window[wnsize_dtype_device].cpu()
+    else:
+        i = y
+        win = hann_window[wnsize_dtype_device]
+    spec = torch.stft(
+        i,
+        n_fft,
+        hop_length=hop_size,
+        win_length=win_size,
+        window=win,
+        center=center,
+        pad_mode="reflect",
+        normalized=False,
+        onesided=True,
+        return_complex=False,
+    ).to(device=y.device)
+
+    spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
+    return spec
+
+
+def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax):
+    global mel_basis
+    dtype_device = str(spec.dtype) + "_" + str(spec.device)
+    fmax_dtype_device = str(fmax) + "_" + dtype_device
+    if fmax_dtype_device not in mel_basis:
+        mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
+        mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(
+            dtype=spec.dtype, device=spec.device
+        )
+    melspec = torch.matmul(mel_basis[fmax_dtype_device], spec)
+    melspec = spectral_normalize_torch(melspec)
+    return melspec
+
+
+def mel_spectrogram_torch(
+    y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False
+):
+    """Convert waveform into Mel-frequency Log-amplitude spectrogram.
+
+    Args:
+        y       :: (B, T)           - Waveforms
+    Returns:
+        melspec :: (B, Freq, Frame) - Mel-frequency Log-amplitude spectrogram
+    """
+    # Linear-frequency Linear-amplitude spectrogram :: (B, T) -> (B, Freq, Frame)
+    spec = spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center)
+
+    # Mel-frequency Log-amplitude spectrogram :: (B, Freq, Frame) -> (B, Freq=num_mels, Frame)
+    melspec = spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax)
+
+    return melspec

lib/rvc/models.py

@@ -0,0 +1,853 @@
+import math
+
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import Conv1d, Conv2d, ConvTranspose1d
+from torch.nn import functional as F
+from torch.nn.utils import remove_weight_norm, spectral_norm, weight_norm
+
+from . import attentions, commons, modules
+from .commons import get_padding, init_weights
+
+
+class TextEncoder(nn.Module):
+    def __init__(
+        self,
+        out_channels: int,
+        hidden_channels: int,
+        filter_channels: int,
+        emb_channels: int,
+        n_heads: int,
+        n_layers: int,
+        kernel_size: int,
+        p_dropout: int,
+        f0: bool = True,
+    ):
+        super().__init__()
+        self.out_channels = out_channels
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.emb_channels = emb_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.emb_phone = nn.Linear(emb_channels, hidden_channels)
+        self.lrelu = nn.LeakyReLU(0.1, inplace=True)
+        if f0 == True:
+            self.emb_pitch = nn.Embedding(256, hidden_channels)  # pitch 256
+        self.encoder = attentions.Encoder(
+            hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout
+        )
+        self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
+
+    def forward(self, phone, pitch, lengths):
+        if pitch == None:
+            x = self.emb_phone(phone)
+        else:
+            x = self.emb_phone(phone) + self.emb_pitch(pitch)
+        x = x * math.sqrt(self.hidden_channels)  # [b, t, h]
+        x = self.lrelu(x)
+        x = torch.transpose(x, 1, -1)  # [b, h, t]
+        x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to(
+            x.dtype
+        )
+        x = self.encoder(x * x_mask, x_mask)
+        stats = self.proj(x) * x_mask
+
+        m, logs = torch.split(stats, self.out_channels, dim=1)
+        return m, logs, x_mask
+
+
+class ResidualCouplingBlock(nn.Module):
+    def __init__(
+        self,
+        channels,
+        hidden_channels,
+        kernel_size,
+        dilation_rate,
+        n_layers,
+        n_flows=4,
+        gin_channels=0,
+    ):
+        super().__init__()
+        self.channels = channels
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.n_flows = n_flows
+        self.gin_channels = gin_channels
+
+        self.flows = nn.ModuleList()
+        for i in range(n_flows):
+            self.flows.append(
+                modules.ResidualCouplingLayer(
+                    channels,
+                    hidden_channels,
+                    kernel_size,
+                    dilation_rate,
+                    n_layers,
+                    gin_channels=gin_channels,
+                    mean_only=True,
+                )
+            )
+            self.flows.append(modules.Flip())
+
+    def forward(self, x, x_mask, g=None, reverse=False):
+        if not reverse:
+            for flow in self.flows:
+                x, _ = flow(x, x_mask, g=g, reverse=reverse)
+        else:
+            for flow in reversed(self.flows):
+                x = flow(x, x_mask, g=g, reverse=reverse)
+        return x
+
+    def remove_weight_norm(self):
+        for i in range(self.n_flows):
+            self.flows[i * 2].remove_weight_norm()
+
+
+class PosteriorEncoder(nn.Module):
+    def __init__(
+        self,
+        in_channels,
+        out_channels,
+        hidden_channels,
+        kernel_size,
+        dilation_rate,
+        n_layers,
+        gin_channels=0,
+    ):
+        super().__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.gin_channels = gin_channels
+
+        self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
+        self.enc = modules.WN(
+            hidden_channels,
+            kernel_size,
+            dilation_rate,
+            n_layers,
+            gin_channels=gin_channels,
+        )
+        self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
+
+    def forward(self, x, x_lengths, g=None):
+        x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(
+            x.dtype
+        )
+        x = self.pre(x) * x_mask
+        x = self.enc(x, x_mask, g=g)
+        stats = self.proj(x) * x_mask
+        m, logs = torch.split(stats, self.out_channels, dim=1)
+        z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
+        return z, m, logs, x_mask
+
+    def remove_weight_norm(self):
+        self.enc.remove_weight_norm()
+
+
+class Generator(torch.nn.Module):
+    def __init__(
+        self,
+        initial_channel,
+        resblock,
+        resblock_kernel_sizes,
+        resblock_dilation_sizes,
+        upsample_rates,
+        upsample_initial_channel,
+        upsample_kernel_sizes,
+        gin_channels=0,
+    ):
+        super(Generator, self).__init__()
+        self.num_kernels = len(resblock_kernel_sizes)
+        self.num_upsamples = len(upsample_rates)
+        self.conv_pre = Conv1d(
+            initial_channel, upsample_initial_channel, 7, 1, padding=3
+        )
+        resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2
+
+        self.ups = nn.ModuleList()
+        for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
+            self.ups.append(
+                weight_norm(
+                    ConvTranspose1d(
+                        upsample_initial_channel // (2**i),
+                        upsample_initial_channel // (2 ** (i + 1)),
+                        k,
+                        u,
+                        padding=(k - u) // 2,
+                    )
+                )
+            )
+
+        self.resblocks = nn.ModuleList()
+        for i in range(len(self.ups)):
+            ch = upsample_initial_channel // (2 ** (i + 1))
+            for j, (k, d) in enumerate(
+                zip(resblock_kernel_sizes, resblock_dilation_sizes)
+            ):
+                self.resblocks.append(resblock(ch, k, d))
+
+        self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
+        self.ups.apply(init_weights)
+
+        if gin_channels != 0:
+            self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
+
+    def forward(self, x, g=None):
+        x = self.conv_pre(x)
+        if g is not None:
+            x = x + self.cond(g)
+
+        for i in range(self.num_upsamples):
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            x = self.ups[i](x)
+            xs = None
+            for j in range(self.num_kernels):
+                if xs is None:
+                    xs = self.resblocks[i * self.num_kernels + j](x)
+                else:
+                    xs += self.resblocks[i * self.num_kernels + j](x)
+            x = xs / self.num_kernels
+        x = F.leaky_relu(x)
+        x = self.conv_post(x)
+        x = torch.tanh(x)
+
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.ups:
+            remove_weight_norm(l)
+        for l in self.resblocks:
+            l.remove_weight_norm()
+
+
+class SineGen(torch.nn.Module):
+    """Definition of sine generator
+    SineGen(samp_rate, harmonic_num = 0,
+            sine_amp = 0.1, noise_std = 0.003,
+            voiced_threshold = 0,
+            flag_for_pulse=False)
+    samp_rate: sampling rate in Hz
+    harmonic_num: number of harmonic overtones (default 0)
+    sine_amp: amplitude of sine-wavefrom (default 0.1)
+    noise_std: std of Gaussian noise (default 0.003)
+    voiced_thoreshold: F0 threshold for U/V classification (default 0)
+    flag_for_pulse: this SinGen is used inside PulseGen (default False)
+    Note: when flag_for_pulse is True, the first time step of a voiced
+        segment is always sin(np.pi) or cos(0)
+    """
+
+    def __init__(
+        self,
+        samp_rate,
+        harmonic_num=0,
+        sine_amp=0.1,
+        noise_std=0.003,
+        voiced_threshold=0,
+        flag_for_pulse=False,
+    ):
+        super(SineGen, self).__init__()
+        self.sine_amp = sine_amp
+        self.noise_std = noise_std
+        self.harmonic_num = harmonic_num
+        self.dim = self.harmonic_num + 1
+        self.sampling_rate = samp_rate
+        self.voiced_threshold = voiced_threshold
+
+    def _f02uv(self, f0):
+        # generate uv signal
+        uv = torch.ones_like(f0)
+        uv = uv * (f0 > self.voiced_threshold)
+        return uv
+
+    def forward(self, f0, upp):
+        """sine_tensor, uv = forward(f0)
+        input F0: tensor(batchsize=1, length, dim=1)
+                  f0 for unvoiced steps should be 0
+        output sine_tensor: tensor(batchsize=1, length, dim)
+        output uv: tensor(batchsize=1, length, 1)
+        """
+        with torch.no_grad():
+            f0 = f0[:, None].transpose(1, 2)
+            f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim, device=f0.device)
+            # fundamental component
+            f0_buf[:, :, 0] = f0[:, :, 0]
+            for idx in np.arange(self.harmonic_num):
+                f0_buf[:, :, idx + 1] = f0_buf[:, :, 0] * (
+                    idx + 2
+                )  # idx + 2: the (idx+1)-th overtone, (idx+2)-th harmonic
+            rad_values = (f0_buf / self.sampling_rate) % 1  ###%1意味着n_har的乘积无法后处理优化
+            rand_ini = torch.rand(
+                f0_buf.shape[0], f0_buf.shape[2], device=f0_buf.device
+            )
+            rand_ini[:, 0] = 0
+            rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini
+            tmp_over_one = torch.cumsum(rad_values, 1)  # % 1  #####%1意味着后面的cumsum无法再优化
+            tmp_over_one *= upp
+            tmp_over_one = F.interpolate(
+                tmp_over_one.transpose(2, 1),
+                scale_factor=upp,
+                mode="linear",
+                align_corners=True,
+            ).transpose(2, 1)
+            rad_values = F.interpolate(
+                rad_values.transpose(2, 1), scale_factor=upp, mode="nearest"
+            ).transpose(
+                2, 1
+            )  #######
+            tmp_over_one %= 1
+            tmp_over_one_idx = (tmp_over_one[:, 1:, :] - tmp_over_one[:, :-1, :]) < 0
+            cumsum_shift = torch.zeros_like(rad_values)
+            cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0
+            sine_waves = torch.sin(
+                torch.cumsum(rad_values + cumsum_shift, dim=1) * 2 * np.pi
+            )
+            sine_waves = sine_waves * self.sine_amp
+            uv = self._f02uv(f0)
+            uv = F.interpolate(
+                uv.transpose(2, 1), scale_factor=upp, mode="nearest"
+            ).transpose(2, 1)
+            noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3
+            noise = noise_amp * torch.randn_like(sine_waves)
+            sine_waves = sine_waves * uv + noise
+        return sine_waves, uv, noise
+
+
+class SourceModuleHnNSF(torch.nn.Module):
+    """SourceModule for hn-nsf
+    SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1,
+                 add_noise_std=0.003, voiced_threshod=0)
+    sampling_rate: sampling_rate in Hz
+    harmonic_num: number of harmonic above F0 (default: 0)
+    sine_amp: amplitude of sine source signal (default: 0.1)
+    add_noise_std: std of additive Gaussian noise (default: 0.003)
+        note that amplitude of noise in unvoiced is decided
+        by sine_amp
+    voiced_threshold: threhold to set U/V given F0 (default: 0)
+    Sine_source, noise_source = SourceModuleHnNSF(F0_sampled)
+    F0_sampled (batchsize, length, 1)
+    Sine_source (batchsize, length, 1)
+    noise_source (batchsize, length 1)
+    uv (batchsize, length, 1)
+    """
+
+    def __init__(
+        self,
+        sampling_rate,
+        harmonic_num=0,
+        sine_amp=0.1,
+        add_noise_std=0.003,
+        voiced_threshod=0,
+        is_half=True,
+    ):
+        super(SourceModuleHnNSF, self).__init__()
+
+        self.sine_amp = sine_amp
+        self.noise_std = add_noise_std
+        self.is_half = is_half
+        # to produce sine waveforms
+        self.l_sin_gen = SineGen(
+            sampling_rate, harmonic_num, sine_amp, add_noise_std, voiced_threshod
+        )
+
+        # to merge source harmonics into a single excitation
+        self.l_linear = torch.nn.Linear(harmonic_num + 1, 1)
+        self.l_tanh = torch.nn.Tanh()
+
+    def forward(self, x, upp=None):
+        sine_wavs, uv, _ = self.l_sin_gen(x, upp)
+        if self.is_half == True:
+            sine_wavs = sine_wavs.half()
+        sine_merge = self.l_tanh(self.l_linear(sine_wavs))
+        return sine_merge, None, None  # noise, uv
+
+
+class GeneratorNSF(torch.nn.Module):
+    def __init__(
+        self,
+        initial_channel,
+        resblock,
+        resblock_kernel_sizes,
+        resblock_dilation_sizes,
+        upsample_rates,
+        upsample_initial_channel,
+        upsample_kernel_sizes,
+        gin_channels,
+        sr,
+        is_half=False,
+    ):
+        super(GeneratorNSF, self).__init__()
+        self.num_kernels = len(resblock_kernel_sizes)
+        self.num_upsamples = len(upsample_rates)
+
+        self.f0_upsamp = torch.nn.Upsample(scale_factor=np.prod(upsample_rates))
+        self.m_source = SourceModuleHnNSF(
+            sampling_rate=sr, harmonic_num=0, is_half=is_half
+        )
+        self.noise_convs = nn.ModuleList()
+        self.conv_pre = Conv1d(
+            initial_channel, upsample_initial_channel, 7, 1, padding=3
+        )
+        resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2
+
+        self.ups = nn.ModuleList()
+        for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
+            c_cur = upsample_initial_channel // (2 ** (i + 1))
+            self.ups.append(
+                weight_norm(
+                    ConvTranspose1d(
+                        upsample_initial_channel // (2**i),
+                        upsample_initial_channel // (2 ** (i + 1)),
+                        k,
+                        u,
+                        padding=(k - u) // 2,
+                    )
+                )
+            )
+            if i + 1 < len(upsample_rates):
+                stride_f0 = np.prod(upsample_rates[i + 1 :])
+                self.noise_convs.append(
+                    Conv1d(
+                        1,
+                        c_cur,
+                        kernel_size=stride_f0 * 2,
+                        stride=stride_f0,
+                        padding=stride_f0 // 2,
+                    )
+                )
+            else:
+                self.noise_convs.append(Conv1d(1, c_cur, kernel_size=1))
+
+        self.resblocks = nn.ModuleList()
+        for i in range(len(self.ups)):
+            ch = upsample_initial_channel // (2 ** (i + 1))
+            for j, (k, d) in enumerate(
+                zip(resblock_kernel_sizes, resblock_dilation_sizes)
+            ):
+                self.resblocks.append(resblock(ch, k, d))
+
+        self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
+        self.ups.apply(init_weights)
+
+        if gin_channels != 0:
+            self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
+
+        self.upp = np.prod(upsample_rates)
+
+    def forward(self, x, f0, g=None):
+        har_source, noi_source, uv = self.m_source(f0, self.upp)
+        har_source = har_source.transpose(1, 2)
+        x = self.conv_pre(x)
+        if g is not None:
+            x = x + self.cond(g)
+
+        for i in range(self.num_upsamples):
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            x = self.ups[i](x)
+            x_source = self.noise_convs[i](har_source)
+            x = x + x_source
+            xs = None
+            for j in range(self.num_kernels):
+                if xs is None:
+                    xs = self.resblocks[i * self.num_kernels + j](x)
+                else:
+                    xs += self.resblocks[i * self.num_kernels + j](x)
+            x = xs / self.num_kernels
+        x = F.leaky_relu(x)
+        x = self.conv_post(x)
+        x = torch.tanh(x)
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.ups:
+            remove_weight_norm(l)
+        for l in self.resblocks:
+            l.remove_weight_norm()
+
+
+sr2sr = {
+    "32k": 32000,
+    "40k": 40000,
+    "48k": 48000,
+}
+
+
+class SynthesizerTrnMs256NSFSid(nn.Module):
+    def __init__(
+        self,
+        spec_channels,
+        segment_size,
+        inter_channels,
+        hidden_channels,
+        filter_channels,
+        n_heads,
+        n_layers,
+        kernel_size,
+        p_dropout,
+        resblock,
+        resblock_kernel_sizes,
+        resblock_dilation_sizes,
+        upsample_rates,
+        upsample_initial_channel,
+        upsample_kernel_sizes,
+        spk_embed_dim,
+        gin_channels,
+        emb_channels,
+        sr,
+        **kwargs
+    ):
+        super().__init__()
+        if type(sr) == type("strr"):
+            sr = sr2sr[sr]
+        self.spec_channels = spec_channels
+        self.inter_channels = inter_channels
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.resblock = resblock
+        self.resblock_kernel_sizes = resblock_kernel_sizes
+        self.resblock_dilation_sizes = resblock_dilation_sizes
+        self.upsample_rates = upsample_rates
+        self.upsample_initial_channel = upsample_initial_channel
+        self.upsample_kernel_sizes = upsample_kernel_sizes
+        self.segment_size = segment_size
+        self.gin_channels = gin_channels
+        self.emb_channels = emb_channels
+        self.sr = sr
+        # self.hop_length = hop_length#
+        self.spk_embed_dim = spk_embed_dim
+        self.enc_p = TextEncoder(
+            inter_channels,
+            hidden_channels,
+            filter_channels,
+            emb_channels,
+            n_heads,
+            n_layers,
+            kernel_size,
+            p_dropout,
+        )
+        self.dec = GeneratorNSF(
+            inter_channels,
+            resblock,
+            resblock_kernel_sizes,
+            resblock_dilation_sizes,
+            upsample_rates,
+            upsample_initial_channel,
+            upsample_kernel_sizes,
+            gin_channels=gin_channels,
+            sr=sr,
+            is_half=kwargs["is_half"],
+        )
+        self.enc_q = PosteriorEncoder(
+            spec_channels,
+            inter_channels,
+            hidden_channels,
+            5,
+            1,
+            16,
+            gin_channels=gin_channels,
+        )
+        self.flow = ResidualCouplingBlock(
+            inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels
+        )
+        self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels)
+        print(
+            "gin_channels:",
+            gin_channels,
+            "self.spk_embed_dim:",
+            self.spk_embed_dim,
+            "emb_channels:",
+            emb_channels,
+        )
+
+    def remove_weight_norm(self):
+        self.dec.remove_weight_norm()
+        self.flow.remove_weight_norm()
+        self.enc_q.remove_weight_norm()
+
+    def forward(
+        self, phone, phone_lengths, pitch, pitchf, y, y_lengths, ds
+    ):  # 这里ds是id，[bs,1]
+        # print(1,pitch.shape)#[bs,t]
+        g = self.emb_g(ds).unsqueeze(-1)  # [b, 256, 1]##1是t，广播的
+        m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)
+        z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
+        z_p = self.flow(z, y_mask, g=g)
+        z_slice, ids_slice = commons.rand_slice_segments(
+            z, y_lengths, self.segment_size
+        )
+        # print(-1,pitchf.shape,ids_slice,self.segment_size,self.hop_length,self.segment_size//self.hop_length)
+        pitchf = commons.slice_segments2(pitchf, ids_slice, self.segment_size)
+        # print(-2,pitchf.shape,z_slice.shape)
+        o = self.dec(z_slice, pitchf, g=g)
+        return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q)
+
+    def infer(self, phone, phone_lengths, pitch, nsff0, sid, max_len=None):
+        g = self.emb_g(sid).unsqueeze(-1)
+        m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)
+        z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask
+        z = self.flow(z_p, x_mask, g=g, reverse=True)
+        o = self.dec((z * x_mask)[:, :, :max_len], nsff0, g=g)
+        return o, x_mask, (z, z_p, m_p, logs_p)
+
+
+class SynthesizerTrnMs256NSFSidNono(nn.Module):
+    def __init__(
+        self,
+        spec_channels,
+        segment_size,
+        inter_channels,
+        hidden_channels,
+        filter_channels,
+        n_heads,
+        n_layers,
+        kernel_size,
+        p_dropout,
+        resblock,
+        resblock_kernel_sizes,
+        resblock_dilation_sizes,
+        upsample_rates,
+        upsample_initial_channel,
+        upsample_kernel_sizes,
+        spk_embed_dim,
+        gin_channels,
+        emb_channels,
+        sr=None,
+        **kwargs
+    ):
+        super().__init__()
+        self.spec_channels = spec_channels
+        self.inter_channels = inter_channels
+        self.hidden_channels = hidden_channels
+        self.filter_channels = filter_channels
+        self.n_heads = n_heads
+        self.n_layers = n_layers
+        self.kernel_size = kernel_size
+        self.p_dropout = p_dropout
+        self.resblock = resblock
+        self.resblock_kernel_sizes = resblock_kernel_sizes
+        self.resblock_dilation_sizes = resblock_dilation_sizes
+        self.upsample_rates = upsample_rates
+        self.upsample_initial_channel = upsample_initial_channel
+        self.upsample_kernel_sizes = upsample_kernel_sizes
+        self.segment_size = segment_size
+        self.gin_channels = gin_channels
+        self.emb_channels = emb_channels
+        self.sr = sr
+        # self.hop_length = hop_length#
+        self.spk_embed_dim = spk_embed_dim
+        self.enc_p = TextEncoder(
+            inter_channels,
+            hidden_channels,
+            filter_channels,
+            emb_channels,
+            n_heads,
+            n_layers,
+            kernel_size,
+            p_dropout,
+            f0=False,
+        )
+        self.dec = Generator(
+            inter_channels,
+            resblock,
+            resblock_kernel_sizes,
+            resblock_dilation_sizes,
+            upsample_rates,
+            upsample_initial_channel,
+            upsample_kernel_sizes,
+            gin_channels=gin_channels,
+        )
+        self.enc_q = PosteriorEncoder(
+            spec_channels,
+            inter_channels,
+            hidden_channels,
+            5,
+            1,
+            16,
+            gin_channels=gin_channels,
+        )
+        self.flow = ResidualCouplingBlock(
+            inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels
+        )
+        self.emb_g = nn.Embedding(self.spk_embed_dim, gin_channels)
+        print(
+            "gin_channels:",
+            gin_channels,
+            "self.spk_embed_dim:",
+            self.spk_embed_dim,
+            "emb_channels:",
+            emb_channels,
+        )
+
+    def remove_weight_norm(self):
+        self.dec.remove_weight_norm()
+        self.flow.remove_weight_norm()
+        self.enc_q.remove_weight_norm()
+
+    def forward(self, phone, phone_lengths, y, y_lengths, ds):  # 这里ds是id，[bs,1]
+        g = self.emb_g(ds).unsqueeze(-1)  # [b, 256, 1]##1是t，广播的
+        m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths)
+        z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
+        z_p = self.flow(z, y_mask, g=g)
+        z_slice, ids_slice = commons.rand_slice_segments(
+            z, y_lengths, self.segment_size
+        )
+        o = self.dec(z_slice, g=g)
+        return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q)
+
+    def infer(self, phone, phone_lengths, sid, max_len=None):
+        g = self.emb_g(sid).unsqueeze(-1)
+        m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths)
+        z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask
+        z = self.flow(z_p, x_mask, g=g, reverse=True)
+        o = self.dec((z * x_mask)[:, :, :max_len], g=g)
+        return o, x_mask, (z, z_p, m_p, logs_p)
+
+
+class DiscriminatorS(torch.nn.Module):
+    def __init__(self, use_spectral_norm=False):
+        super(DiscriminatorS, self).__init__()
+        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+        self.convs = nn.ModuleList(
+            [
+                norm_f(Conv1d(1, 16, 15, 1, padding=7)),
+                norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)),
+                norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)),
+                norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)),
+                norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)),
+                norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
+            ]
+        )
+        self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
+
+    def forward(self, x):
+        fmap = []
+
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x, fmap
+
+
+class DiscriminatorP(torch.nn.Module):
+    def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
+        super(DiscriminatorP, self).__init__()
+        self.period = period
+        self.use_spectral_norm = use_spectral_norm
+        norm_f = weight_norm if use_spectral_norm == False else spectral_norm
+        self.convs = nn.ModuleList(
+            [
+                norm_f(
+                    Conv2d(
+                        1,
+                        32,
+                        (kernel_size, 1),
+                        (stride, 1),
+                        padding=(get_padding(kernel_size, 1), 0),
+                    )
+                ),
+                norm_f(
+                    Conv2d(
+                        32,
+                        128,
+                        (kernel_size, 1),
+                        (stride, 1),
+                        padding=(get_padding(kernel_size, 1), 0),
+                    )
+                ),
+                norm_f(
+                    Conv2d(
+                        128,
+                        512,
+                        (kernel_size, 1),
+                        (stride, 1),
+                        padding=(get_padding(kernel_size, 1), 0),
+                    )
+                ),
+                norm_f(
+                    Conv2d(
+                        512,
+                        1024,
+                        (kernel_size, 1),
+                        (stride, 1),
+                        padding=(get_padding(kernel_size, 1), 0),
+                    )
+                ),
+                norm_f(
+                    Conv2d(
+                        1024,
+                        1024,
+                        (kernel_size, 1),
+                        1,
+                        padding=(get_padding(kernel_size, 1), 0),
+                    )
+                ),
+            ]
+        )
+        self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
+
+    def forward(self, x):
+        fmap = []
+
+        # 1d to 2d
+        b, c, t = x.shape
+        if t % self.period != 0:  # pad first
+            n_pad = self.period - (t % self.period)
+            x = F.pad(x, (0, n_pad), "reflect")
+            t = t + n_pad
+        x = x.view(b, c, t // self.period, self.period)
+
+        for l in self.convs:
+            x = l(x)
+            x = F.leaky_relu(x, modules.LRELU_SLOPE)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        x = torch.flatten(x, 1, -1)
+
+        return x, fmap
+
+
+class MultiPeriodDiscriminator(torch.nn.Module):
+    def __init__(self, use_spectral_norm=False, periods=[2, 3, 5, 7, 11, 17]):
+        super(MultiPeriodDiscriminator, self).__init__()
+
+        discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]
+        discs = discs + [
+            DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods
+        ]
+        self.discriminators = nn.ModuleList(discs)
+
+    def forward(self, y, y_hat):
+        y_d_rs = []  #
+        y_d_gs = []
+        fmap_rs = []
+        fmap_gs = []
+        for i, d in enumerate(self.discriminators):
+            y_d_r, fmap_r = d(y)
+            y_d_g, fmap_g = d(y_hat)
+            # for j in range(len(fmap_r)):
+            #     print(i,j,y.shape,y_hat.shape,fmap_r[j].shape,fmap_g[j].shape)
+            y_d_rs.append(y_d_r)
+            y_d_gs.append(y_d_g)
+            fmap_rs.append(fmap_r)
+            fmap_gs.append(fmap_g)
+
+        return y_d_rs, y_d_gs, fmap_rs, fmap_gs

lib/rvc/modules.py

@@ -0,0 +1,518 @@
+import math
+
+import torch
+from torch import nn
+from torch.nn import Conv1d
+from torch.nn import functional as F
+from torch.nn.utils import remove_weight_norm, weight_norm
+
+from . import commons
+from .commons import get_padding, init_weights
+from .transforms import piecewise_rational_quadratic_transform
+
+LRELU_SLOPE = 0.1
+
+
+class LayerNorm(nn.Module):
+    def __init__(self, channels, eps=1e-5):
+        super().__init__()
+        self.channels = channels
+        self.eps = eps
+
+        self.gamma = nn.Parameter(torch.ones(channels))
+        self.beta = nn.Parameter(torch.zeros(channels))
+
+    def forward(self, x):
+        x = x.transpose(1, -1)
+        x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
+        return x.transpose(1, -1)
+
+
+class ConvReluNorm(nn.Module):
+    def __init__(
+        self,
+        in_channels,
+        hidden_channels,
+        out_channels,
+        kernel_size,
+        n_layers,
+        p_dropout,
+    ):
+        super().__init__()
+        self.in_channels = in_channels
+        self.hidden_channels = hidden_channels
+        self.out_channels = out_channels
+        self.kernel_size = kernel_size
+        self.n_layers = n_layers
+        self.p_dropout = p_dropout
+        assert n_layers > 1, "Number of layers should be larger than 0."
+
+        self.conv_layers = nn.ModuleList()
+        self.norm_layers = nn.ModuleList()
+        self.conv_layers.append(
+            nn.Conv1d(
+                in_channels, hidden_channels, kernel_size, padding=kernel_size // 2
+            )
+        )
+        self.norm_layers.append(LayerNorm(hidden_channels))
+        self.relu_drop = nn.Sequential(nn.ReLU(), nn.Dropout(p_dropout))
+        for _ in range(n_layers - 1):
+            self.conv_layers.append(
+                nn.Conv1d(
+                    hidden_channels,
+                    hidden_channels,
+                    kernel_size,
+                    padding=kernel_size // 2,
+                )
+            )
+            self.norm_layers.append(LayerNorm(hidden_channels))
+        self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
+        self.proj.weight.data.zero_()
+        self.proj.bias.data.zero_()
+
+    def forward(self, x, x_mask):
+        x_org = x
+        for i in range(self.n_layers):
+            x = self.conv_layers[i](x * x_mask)
+            x = self.norm_layers[i](x)
+            x = self.relu_drop(x)
+        x = x_org + self.proj(x)
+        return x * x_mask
+
+
+class DDSConv(nn.Module):
+    """
+    Dialted and Depth-Separable Convolution
+    """
+
+    def __init__(self, channels, kernel_size, n_layers, p_dropout=0.0):
+        super().__init__()
+        self.channels = channels
+        self.kernel_size = kernel_size
+        self.n_layers = n_layers
+        self.p_dropout = p_dropout
+
+        self.drop = nn.Dropout(p_dropout)
+        self.convs_sep = nn.ModuleList()
+        self.convs_1x1 = nn.ModuleList()
+        self.norms_1 = nn.ModuleList()
+        self.norms_2 = nn.ModuleList()
+        for i in range(n_layers):
+            dilation = kernel_size**i
+            padding = (kernel_size * dilation - dilation) // 2
+            self.convs_sep.append(
+                nn.Conv1d(
+                    channels,
+                    channels,
+                    kernel_size,
+                    groups=channels,
+                    dilation=dilation,
+                    padding=padding,
+                )
+            )
+            self.convs_1x1.append(nn.Conv1d(channels, channels, 1))
+            self.norms_1.append(LayerNorm(channels))
+            self.norms_2.append(LayerNorm(channels))
+
+    def forward(self, x, x_mask, g=None):
+        if g is not None:
+            x = x + g
+        for i in range(self.n_layers):
+            y = self.convs_sep[i](x * x_mask)
+            y = self.norms_1[i](y)
+            y = F.gelu(y)
+            y = self.convs_1x1[i](y)
+            y = self.norms_2[i](y)
+            y = F.gelu(y)
+            y = self.drop(y)
+            x = x + y
+        return x * x_mask
+
+
+class WN(torch.nn.Module):
+    def __init__(
+        self,
+        hidden_channels,
+        kernel_size,
+        dilation_rate,
+        n_layers,
+        gin_channels=0,
+        p_dropout=0,
+    ):
+        super(WN, self).__init__()
+        assert kernel_size % 2 == 1
+        self.hidden_channels = hidden_channels
+        self.kernel_size = (kernel_size,)
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.gin_channels = gin_channels
+        self.p_dropout = p_dropout
+
+        self.in_layers = torch.nn.ModuleList()
+        self.res_skip_layers = torch.nn.ModuleList()
+        self.drop = nn.Dropout(p_dropout)
+
+        if gin_channels != 0:
+            cond_layer = torch.nn.Conv1d(
+                gin_channels, 2 * hidden_channels * n_layers, 1
+            )
+            self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name="weight")
+
+        for i in range(n_layers):
+            dilation = dilation_rate**i
+            padding = int((kernel_size * dilation - dilation) / 2)
+            in_layer = torch.nn.Conv1d(
+                hidden_channels,
+                2 * hidden_channels,
+                kernel_size,
+                dilation=dilation,
+                padding=padding,
+            )
+            in_layer = torch.nn.utils.weight_norm(in_layer, name="weight")
+            self.in_layers.append(in_layer)
+
+            # last one is not necessary
+            if i < n_layers - 1:
+                res_skip_channels = 2 * hidden_channels
+            else:
+                res_skip_channels = hidden_channels
+
+            res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1)
+            res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name="weight")
+            self.res_skip_layers.append(res_skip_layer)
+
+    def forward(self, x, x_mask, g=None, **kwargs):
+        output = torch.zeros_like(x)
+        n_channels_tensor = torch.IntTensor([self.hidden_channels])
+
+        if g is not None:
+            g = self.cond_layer(g)
+
+        for i in range(self.n_layers):
+            x_in = self.in_layers[i](x)
+            if g is not None:
+                cond_offset = i * 2 * self.hidden_channels
+                g_l = g[:, cond_offset : cond_offset + 2 * self.hidden_channels, :]
+            else:
+                g_l = torch.zeros_like(x_in)
+
+            acts = commons.fused_add_tanh_sigmoid_multiply(x_in, g_l, n_channels_tensor)
+            acts = self.drop(acts)
+
+            res_skip_acts = self.res_skip_layers[i](acts)
+            if i < self.n_layers - 1:
+                res_acts = res_skip_acts[:, : self.hidden_channels, :]
+                x = (x + res_acts) * x_mask
+                output = output + res_skip_acts[:, self.hidden_channels :, :]
+            else:
+                output = output + res_skip_acts
+        return output * x_mask
+
+    def remove_weight_norm(self):
+        if self.gin_channels != 0:
+            torch.nn.utils.remove_weight_norm(self.cond_layer)
+        for l in self.in_layers:
+            torch.nn.utils.remove_weight_norm(l)
+        for l in self.res_skip_layers:
+            torch.nn.utils.remove_weight_norm(l)
+
+
+class ResBlock1(torch.nn.Module):
+    def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
+        super(ResBlock1, self).__init__()
+        self.convs1 = nn.ModuleList(
+            [
+                weight_norm(
+                    Conv1d(
+                        channels,
+                        channels,
+                        kernel_size,
+                        1,
+                        dilation=dilation[0],
+                        padding=get_padding(kernel_size, dilation[0]),
+                    )
+                ),
+                weight_norm(
+                    Conv1d(
+                        channels,
+                        channels,
+                        kernel_size,
+                        1,
+                        dilation=dilation[1],
+                        padding=get_padding(kernel_size, dilation[1]),
+                    )
+                ),
+                weight_norm(
+                    Conv1d(
+                        channels,
+                        channels,
+                        kernel_size,
+                        1,
+                        dilation=dilation[2],
+                        padding=get_padding(kernel_size, dilation[2]),
+                    )
+                ),
+            ]
+        )
+        self.convs1.apply(init_weights)
+
+        self.convs2 = nn.ModuleList(
+            [
+                weight_norm(
+                    Conv1d(
+                        channels,
+                        channels,
+                        kernel_size,
+                        1,
+                        dilation=1,
+                        padding=get_padding(kernel_size, 1),
+                    )
+                ),
+                weight_norm(
+                    Conv1d(
+                        channels,
+                        channels,
+                        kernel_size,
+                        1,
+                        dilation=1,
+                        padding=get_padding(kernel_size, 1),
+                    )
+                ),
+                weight_norm(
+                    Conv1d(
+                        channels,
+                        channels,
+                        kernel_size,
+                        1,
+                        dilation=1,
+                        padding=get_padding(kernel_size, 1),
+                    )
+                ),
+            ]
+        )
+        self.convs2.apply(init_weights)
+
+    def forward(self, x, x_mask=None):
+        for c1, c2 in zip(self.convs1, self.convs2):
+            xt = F.leaky_relu(x, LRELU_SLOPE)
+            if x_mask is not None:
+                xt = xt * x_mask
+            xt = c1(xt)
+            xt = F.leaky_relu(xt, LRELU_SLOPE)
+            if x_mask is not None:
+                xt = xt * x_mask
+            xt = c2(xt)
+            x = xt + x
+        if x_mask is not None:
+            x = x * x_mask
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.convs1:
+            remove_weight_norm(l)
+        for l in self.convs2:
+            remove_weight_norm(l)
+
+
+class ResBlock2(torch.nn.Module):
+    def __init__(self, channels, kernel_size=3, dilation=(1, 3)):
+        super(ResBlock2, self).__init__()
+        self.convs = nn.ModuleList(
+            [
+                weight_norm(
+                    Conv1d(
+                        channels,
+                        channels,
+                        kernel_size,
+                        1,
+                        dilation=dilation[0],
+                        padding=get_padding(kernel_size, dilation[0]),
+                    )
+                ),
+                weight_norm(
+                    Conv1d(
+                        channels,
+                        channels,
+                        kernel_size,
+                        1,
+                        dilation=dilation[1],
+                        padding=get_padding(kernel_size, dilation[1]),
+                    )
+                ),
+            ]
+        )
+        self.convs.apply(init_weights)
+
+    def forward(self, x, x_mask=None):
+        for c in self.convs:
+            xt = F.leaky_relu(x, LRELU_SLOPE)
+            if x_mask is not None:
+                xt = xt * x_mask
+            xt = c(xt)
+            x = xt + x
+        if x_mask is not None:
+            x = x * x_mask
+        return x
+
+    def remove_weight_norm(self):
+        for l in self.convs:
+            remove_weight_norm(l)
+
+
+class Log(nn.Module):
+    def forward(self, x, x_mask, reverse=False, **kwargs):
+        if not reverse:
+            y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask
+            logdet = torch.sum(-y, [1, 2])
+            return y, logdet
+        else:
+            x = torch.exp(x) * x_mask
+            return x
+
+
+class Flip(nn.Module):
+    def forward(self, x, *args, reverse=False, **kwargs):
+        x = torch.flip(x, [1])
+        if not reverse:
+            logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)
+            return x, logdet
+        else:
+            return x
+
+
+class ElementwiseAffine(nn.Module):
+    def __init__(self, channels):
+        super().__init__()
+        self.channels = channels
+        self.m = nn.Parameter(torch.zeros(channels, 1))
+        self.logs = nn.Parameter(torch.zeros(channels, 1))
+
+    def forward(self, x, x_mask, reverse=False, **kwargs):
+        if not reverse:
+            y = self.m + torch.exp(self.logs) * x
+            y = y * x_mask
+            logdet = torch.sum(self.logs * x_mask, [1, 2])
+            return y, logdet
+        else:
+            x = (x - self.m) * torch.exp(-self.logs) * x_mask
+            return x
+
+
+class ResidualCouplingLayer(nn.Module):
+    def __init__(
+        self,
+        channels,
+        hidden_channels,
+        kernel_size,
+        dilation_rate,
+        n_layers,
+        p_dropout=0,
+        gin_channels=0,
+        mean_only=False,
+    ):
+        assert channels % 2 == 0, "channels should be divisible by 2"
+        super().__init__()
+        self.channels = channels
+        self.hidden_channels = hidden_channels
+        self.kernel_size = kernel_size
+        self.dilation_rate = dilation_rate
+        self.n_layers = n_layers
+        self.half_channels = channels // 2
+        self.mean_only = mean_only
+
+        self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
+        self.enc = WN(
+            hidden_channels,
+            kernel_size,
+            dilation_rate,
+            n_layers,
+            p_dropout=p_dropout,
+            gin_channels=gin_channels,
+        )
+        self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
+        self.post.weight.data.zero_()
+        self.post.bias.data.zero_()
+
+    def forward(self, x, x_mask, g=None, reverse=False):
+        x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
+        h = self.pre(x0) * x_mask
+        h = self.enc(h, x_mask, g=g)
+        stats = self.post(h) * x_mask
+        if not self.mean_only:
+            m, logs = torch.split(stats, [self.half_channels] * 2, 1)
+        else:
+            m = stats
+            logs = torch.zeros_like(m)
+
+        if not reverse:
+            x1 = m + x1 * torch.exp(logs) * x_mask
+            x = torch.cat([x0, x1], 1)
+            logdet = torch.sum(logs, [1, 2])
+            return x, logdet
+        else:
+            x1 = (x1 - m) * torch.exp(-logs) * x_mask
+            x = torch.cat([x0, x1], 1)
+            return x
+
+    def remove_weight_norm(self):
+        self.enc.remove_weight_norm()
+
+
+class ConvFlow(nn.Module):
+    def __init__(
+        self,
+        in_channels,
+        filter_channels,
+        kernel_size,
+        n_layers,
+        num_bins=10,
+        tail_bound=5.0,
+    ):
+        super().__init__()
+        self.in_channels = in_channels
+        self.filter_channels = filter_channels
+        self.kernel_size = kernel_size
+        self.n_layers = n_layers
+        self.num_bins = num_bins
+        self.tail_bound = tail_bound
+        self.half_channels = in_channels // 2
+
+        self.pre = nn.Conv1d(self.half_channels, filter_channels, 1)
+        self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.0)
+        self.proj = nn.Conv1d(
+            filter_channels, self.half_channels * (num_bins * 3 - 1), 1
+        )
+        self.proj.weight.data.zero_()
+        self.proj.bias.data.zero_()
+
+    def forward(self, x, x_mask, g=None, reverse=False):
+        x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
+        h = self.pre(x0)
+        h = self.convs(h, x_mask, g=g)
+        h = self.proj(h) * x_mask
+
+        b, c, t = x0.shape
+        h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2)  # [b, cx?, t] -> [b, c, t, ?]
+
+        unnormalized_widths = h[..., : self.num_bins] / math.sqrt(self.filter_channels)
+        unnormalized_heights = h[..., self.num_bins : 2 * self.num_bins] / math.sqrt(
+            self.filter_channels
+        )
+        unnormalized_derivatives = h[..., 2 * self.num_bins :]
+
+        x1, logabsdet = piecewise_rational_quadratic_transform(
+            x1,
+            unnormalized_widths,
+            unnormalized_heights,
+            unnormalized_derivatives,
+            inverse=reverse,
+            tails="linear",
+            tail_bound=self.tail_bound,
+        )
+
+        x = torch.cat([x0, x1], 1) * x_mask
+        logdet = torch.sum(logabsdet * x_mask, [1, 2])
+        if not reverse:
+            return x, logdet
+        else:
+            return x

lib/rvc/pipeline.py

@@ -0,0 +1,453 @@
+import os
+import traceback
+from typing import *
+
+import faiss
+import numpy as np
+import pyworld
+import scipy.signal as signal
+import torch
+import torch.nn.functional as F
+import torchcrepe
+from torch import Tensor
+# from faiss.swigfaiss_avx2 import IndexIVFFlat # cause crash on windows' faiss-cpu installed from pip
+from fairseq.models.hubert import HubertModel
+
+from .models import SynthesizerTrnMs256NSFSid
+
+
+class VocalConvertPipeline(object):
+    def __init__(self, tgt_sr: int, device: Union[str, torch.device], is_half: bool):
+        if isinstance(device, str):
+            device = torch.device(device)
+        if device.type == "cuda":
+            vram = torch.cuda.get_device_properties(device).total_memory / 1024**3
+        else:
+            vram = None
+
+        if vram is not None and vram <= 4:
+            self.x_pad = 1
+            self.x_query = 5
+            self.x_center = 30
+            self.x_max = 32
+        elif vram is not None and vram <= 5:
+            self.x_pad = 1
+            self.x_query = 6
+            self.x_center = 38
+            self.x_max = 41
+        else:
+            self.x_pad = 3
+            self.x_query = 10
+            self.x_center = 60
+            self.x_max = 65
+
+        self.sr = 16000  # hubert input sample rate
+        self.window = 160  # hubert input window
+        self.t_pad = self.sr * self.x_pad  # padding time for each utterance
+        self.t_pad_tgt = tgt_sr * self.x_pad
+        self.t_pad2 = self.t_pad * 2
+        self.t_query = self.sr * self.x_query  # query time before and after query point
+        self.t_center = self.sr * self.x_center  # query cut point position
+        self.t_max = self.sr * self.x_max  # max time for no query
+        self.device = device
+        self.is_half = is_half
+
+    def get_optimal_torch_device(self, index: int = 0) -> torch.device:
+        # Get cuda device
+        if torch.cuda.is_available():
+            return torch.device(f"cuda:{index % torch.cuda.device_count()}") # Very fast
+        elif torch.backends.mps.is_available():
+            return torch.device("mps")
+        # Insert an else here to grab "xla" devices if available. TO DO later. Requires the torch_xla.core.xla_model library
+        # Else wise return the "cpu" as a torch device, 
+        return torch.device("cpu")
+
+    def get_f0_crepe_computation(
+            self, 
+            x, 
+            f0_min,
+            f0_max,
+            p_len,
+            hop_length=64, # 512 before. Hop length changes the speed that the voice jumps to a different dramatic pitch. Lower hop lengths means more pitch accuracy but longer inference time.
+            model="full", # Either use crepe-tiny "tiny" or crepe "full". Default is full
+    ):
+        x = x.astype(np.float32) # fixes the F.conv2D exception. We needed to convert double to float.
+        x /= np.quantile(np.abs(x), 0.999)
+        torch_device = self.get_optimal_torch_device()
+        audio = torch.from_numpy(x).to(torch_device, copy=True)
+        audio = torch.unsqueeze(audio, dim=0)
+        if audio.ndim == 2 and audio.shape[0] > 1:
+            audio = torch.mean(audio, dim=0, keepdim=True).detach()
+        audio = audio.detach()
+        print("Initiating prediction with a crepe_hop_length of: " + str(hop_length))
+        pitch: Tensor = torchcrepe.predict(
+            audio,
+            self.sr,
+            hop_length,
+            f0_min,
+            f0_max,
+            model,
+            batch_size=hop_length * 2,
+            device=torch_device,
+            pad=True
+        )
+        p_len = p_len or x.shape[0] // hop_length
+        # Resize the pitch for final f0
+        source = np.array(pitch.squeeze(0).cpu().float().numpy())
+        source[source < 0.001] = np.nan
+        target = np.interp(
+            np.arange(0, len(source) * p_len, len(source)) / p_len,
+            np.arange(0, len(source)),
+            source
+        )
+        f0 = np.nan_to_num(target)
+        return f0 # Resized f0
+
+    def get_f0_official_crepe_computation(
+            self,
+            x,
+            f0_min,
+            f0_max,
+            model="full",
+    ):
+        # Pick a batch size that doesn't cause memory errors on your gpu
+        batch_size = 512
+        # Compute pitch using first gpu
+        audio = torch.tensor(np.copy(x))[None].float()
+        f0, pd = torchcrepe.predict(
+            audio,
+            self.sr,
+            self.window,
+            f0_min,
+            f0_max,
+            model,
+            batch_size=batch_size,
+            device=self.device,
+            return_periodicity=True,
+        )
+        pd = torchcrepe.filter.median(pd, 3)
+        f0 = torchcrepe.filter.mean(f0, 3)
+        f0[pd < 0.1] = 0
+        f0 = f0[0].cpu().numpy()
+        return f0
+
+    def get_f0(
+        self,
+        x: np.ndarray,
+        p_len: int,
+        f0_up_key: int,
+        f0_method: str,
+        inp_f0: np.ndarray = None,
+    ):
+        f0_min = 50
+        f0_max = 1100
+        f0_mel_min = 1127 * np.log(1 + f0_min / 700)
+        f0_mel_max = 1127 * np.log(1 + f0_max / 700)
+
+        if f0_method == "harvest":
+            f0, t = pyworld.harvest(
+                x.astype(np.double),
+                fs=self.sr,
+                f0_ceil=f0_max,
+                f0_floor=f0_min,
+                frame_period=10,
+            )
+            f0 = pyworld.stonemask(x.astype(np.double), f0, t, self.sr)
+            f0 = signal.medfilt(f0, 3)
+        elif f0_method == "dio":
+            f0, t = pyworld.dio(
+                x.astype(np.double),
+                fs=self.sr,
+                f0_ceil=f0_max,
+                f0_floor=f0_min,
+                frame_period=10,
+            )
+            f0 = pyworld.stonemask(x.astype(np.double), f0, t, self.sr)
+            f0 = signal.medfilt(f0, 3)
+        elif f0_method == "mangio-crepe":
+            f0 = self.get_f0_crepe_computation(x, f0_min, f0_max, p_len, 160, "full")
+        elif f0_method == "crepe":
+            f0 = self.get_f0_official_crepe_computation(x, f0_min, f0_max, "full")
+
+        f0 *= pow(2, f0_up_key / 12)
+        tf0 = self.sr // self.window  # f0 points per second
+        if inp_f0 is not None:
+            delta_t = np.round(
+                (inp_f0[:, 0].max() - inp_f0[:, 0].min()) * tf0 + 1
+            ).astype("int16")
+            replace_f0 = np.interp(
+                list(range(delta_t)), inp_f0[:, 0] * 100, inp_f0[:, 1]
+            )
+            shape = f0[self.x_pad * tf0 : self.x_pad * tf0 + len(replace_f0)].shape[0]
+            f0[self.x_pad * tf0 : self.x_pad * tf0 + len(replace_f0)] = replace_f0[
+                :shape
+            ]
+
+        f0bak = f0.copy()
+        f0_mel = 1127 * np.log(1 + f0 / 700)
+        f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (
+            f0_mel_max - f0_mel_min
+        ) + 1
+        f0_mel[f0_mel <= 1] = 1
+        f0_mel[f0_mel > 255] = 255
+        f0_coarse = np.rint(f0_mel).astype(np.int)
+        return f0_coarse, f0bak  # 1-0
+
+    def _convert(
+        self,
+        model: HubertModel,
+        embedding_output_layer: int,
+        net_g: SynthesizerTrnMs256NSFSid,
+        sid: int,
+        audio: np.ndarray,
+        pitch: np.ndarray,
+        pitchf: np.ndarray,
+        index: faiss.IndexIVFFlat,
+        big_npy: np.ndarray,
+        index_rate: float,
+    ):
+        feats = torch.from_numpy(audio)
+        if self.is_half:
+            feats = feats.half()
+        else:
+            feats = feats.float()
+        if feats.dim() == 2:  # double channels
+            feats = feats.mean(-1)
+        assert feats.dim() == 1, feats.dim()
+        feats = feats.view(1, -1)
+        padding_mask = torch.BoolTensor(feats.shape).to(self.device).fill_(False)
+
+        half_support = (
+            self.device.type == "cuda"
+            and torch.cuda.get_device_capability(self.device)[0] >= 5.3
+        )
+        is_feats_dim_768 = net_g.emb_channels == 768
+
+        if isinstance(model, tuple):
+            feats = model[0](
+                feats.squeeze(0).squeeze(0).to(self.device),
+                return_tensors="pt",
+                sampling_rate=16000,
+            )
+            if self.is_half:
+                feats = feats.input_values.to(self.device).half()
+            else:
+                feats = feats.input_values.to(self.device)
+            with torch.no_grad():
+                if is_feats_dim_768:
+                    feats = model[1](feats).last_hidden_state
+                else:
+                    feats = model[1](feats).extract_features
+        else:
+            inputs = {
+                "source": feats.half().to(self.device)
+                if half_support
+                else feats.to(self.device),
+                "padding_mask": padding_mask.to(self.device),
+                "output_layer": embedding_output_layer,
+            }
+
+            if not half_support:
+                model = model.float()
+                inputs["source"] = inputs["source"].float()
+
+            with torch.no_grad():
+                logits = model.extract_features(**inputs)
+                if is_feats_dim_768:
+                    feats = logits[0]
+                else:
+                    feats = model.final_proj(logits[0])
+
+        if (
+            isinstance(index, type(None)) == False
+            and isinstance(big_npy, type(None)) == False
+            and index_rate != 0
+        ):
+            npy = feats[0].cpu().numpy()
+            if self.is_half:
+                npy = npy.astype("float32")
+
+            score, ix = index.search(npy, k=8)
+            weight = np.square(1 / score)
+            weight /= weight.sum(axis=1, keepdims=True)
+            npy = np.sum(big_npy[ix] * np.expand_dims(weight, axis=2), axis=1)
+
+            if self.is_half:
+                npy = npy.astype("float16")
+            feats = (
+                torch.from_numpy(npy).unsqueeze(0).to(self.device) * index_rate
+                + (1 - index_rate) * feats
+            )
+
+        feats = F.interpolate(feats.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1)
+
+        p_len = audio.shape[0] // self.window
+        if feats.shape[1] < p_len:
+            p_len = feats.shape[1]
+            if pitch != None and pitchf != None:
+                pitch = pitch[:, :p_len]
+                pitchf = pitchf[:, :p_len]
+        p_len = torch.tensor([p_len], device=self.device).long()
+        with torch.no_grad():
+            if pitch != None and pitchf != None:
+                audio1 = (
+                    (net_g.infer(feats, p_len, pitch, pitchf, sid)[0][0, 0] * 32768)
+                    .data.cpu()
+                    .float()
+                    .numpy()
+                    .astype(np.int16)
+                )
+            else:
+                audio1 = (
+                    (net_g.infer(feats, p_len, sid)[0][0, 0] * 32768)
+                    .data.cpu()
+                    .float()
+                    .numpy()
+                    .astype(np.int16)
+                )
+        del feats, p_len, padding_mask
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        return audio1
+
+    def __call__(
+        self,
+        model: HubertModel,
+        embedding_output_layer: int,
+        net_g: SynthesizerTrnMs256NSFSid,
+        sid: int,
+        audio: np.ndarray,
+        transpose: int,
+        f0_method: str,
+        file_index: str,
+        index_rate: float,
+        if_f0: bool,
+        f0_file: str = None,
+    ):
+        if file_index != "" and os.path.exists(file_index) and index_rate != 0:
+            try:
+                index = faiss.read_index(file_index)
+                # big_npy = np.load(file_big_npy)
+                big_npy = index.reconstruct_n(0, index.ntotal)
+            except:
+                traceback.print_exc()
+                index = big_npy = None
+        else:
+            index = big_npy = None
+
+        bh, ah = signal.butter(N=5, Wn=48, btype="high", fs=16000)
+        audio = signal.filtfilt(bh, ah, audio)
+
+        audio_pad = np.pad(audio, (self.window // 2, self.window // 2), mode="reflect")
+        opt_ts = []
+        if audio_pad.shape[0] > self.t_max:
+            audio_sum = np.zeros_like(audio)
+            for i in range(self.window):
+                audio_sum += audio_pad[i : i - self.window]
+            for t in range(self.t_center, audio.shape[0], self.t_center):
+                opt_ts.append(
+                    t
+                    - self.t_query
+                    + np.where(
+                        np.abs(audio_sum[t - self.t_query : t + self.t_query])
+                        == np.abs(audio_sum[t - self.t_query : t + self.t_query]).min()
+                    )[0][0]
+                )
+
+        audio_pad = np.pad(audio, (self.t_pad, self.t_pad), mode="reflect")
+        p_len = audio_pad.shape[0] // self.window
+        inp_f0 = None
+        if hasattr(f0_file, "name"):
+            try:
+                with open(f0_file.name, "r") as f:
+                    lines = f.read().strip("\n").split("\n")
+                inp_f0 = []
+                for line in lines:
+                    inp_f0.append([float(i) for i in line.split(",")])
+                inp_f0 = np.array(inp_f0, dtype="float32")
+            except:
+                traceback.print_exc()
+        sid = torch.tensor(sid, device=self.device).unsqueeze(0).long()
+        pitch, pitchf = None, None
+        if if_f0 == 1:
+            pitch, pitchf = self.get_f0(audio_pad, p_len, transpose, f0_method, inp_f0)
+            pitch = pitch[:p_len]
+            pitchf = pitchf[:p_len]
+            if self.device.type == "mps":
+                pitchf = pitchf.astype(np.float32)
+            pitch = torch.tensor(pitch, device=self.device).unsqueeze(0).long()
+            pitchf = torch.tensor(pitchf, device=self.device).unsqueeze(0).float()
+
+        audio_opt = []
+
+        s = 0
+        t = None
+
+        for t in opt_ts:
+            t = t // self.window * self.window
+            if if_f0 == 1:
+                audio_opt.append(
+                    self._convert(
+                        model,
+                        embedding_output_layer,
+                        net_g,
+                        sid,
+                        audio_pad[s : t + self.t_pad2 + self.window],
+                        pitch[:, s // self.window : (t + self.t_pad2) // self.window],
+                        pitchf[:, s // self.window : (t + self.t_pad2) // self.window],
+                        index,
+                        big_npy,
+                        index_rate,
+                    )[self.t_pad_tgt : -self.t_pad_tgt]
+                )
+            else:
+                audio_opt.append(
+                    self._convert(
+                        model,
+                        embedding_output_layer,
+                        net_g,
+                        sid,
+                        audio_pad[s : t + self.t_pad2 + self.window],
+                        None,
+                        None,
+                        index,
+                        big_npy,
+                        index_rate,
+                    )[self.t_pad_tgt : -self.t_pad_tgt]
+                )
+            s = t
+        if if_f0 == 1:
+            audio_opt.append(
+                self._convert(
+                    model,
+                    embedding_output_layer,
+                    net_g,
+                    sid,
+                    audio_pad[t:],
+                    pitch[:, t // self.window :] if t is not None else pitch,
+                    pitchf[:, t // self.window :] if t is not None else pitchf,
+                    index,
+                    big_npy,
+                    index_rate,
+                )[self.t_pad_tgt : -self.t_pad_tgt]
+            )
+        else:
+            audio_opt.append(
+                self._convert(
+                    model,
+                    embedding_output_layer,
+                    net_g,
+                    sid,
+                    audio_pad[t:],
+                    None,
+                    None,
+                    index,
+                    big_npy,
+                    index_rate,
+                )[self.t_pad_tgt : -self.t_pad_tgt]
+            )
+        audio_opt = np.concatenate(audio_opt)
+        del pitch, pitchf, sid
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        return audio_opt

lib/rvc/preprocessing/extract_f0.py

@@ -0,0 +1,221 @@
+import os
+import traceback
+from concurrent.futures import ProcessPoolExecutor
+from typing import *
+import multiprocessing as mp
+
+import numpy as np
+import pyworld
+import torch
+import torchcrepe
+from torch import Tensor
+from tqdm import tqdm
+
+from lib.rvc.utils import load_audio
+
+def get_optimal_torch_device(index: int = 0) -> torch.device:
+    # Get cuda device
+    if torch.cuda.is_available():
+        return torch.device(f"cuda:{index % torch.cuda.device_count()}") # Very fast
+    elif torch.backends.mps.is_available():
+        return torch.device("mps")
+    # Insert an else here to grab "xla" devices if available. TO DO later. Requires the torch_xla.core.xla_model library
+    # Else wise return the "cpu" as a torch device, 
+    return torch.device("cpu")
+
+def get_f0_official_crepe_computation(
+        x,
+        sr,
+        f0_min,
+        f0_max,
+        model="full",
+):
+    batch_size = 512
+    torch_device = get_optimal_torch_device()
+    audio = torch.tensor(np.copy(x))[None].float()
+    f0, pd = torchcrepe.predict(
+        audio,
+        sr,
+        160,
+        f0_min,
+        f0_max,
+        model,
+        batch_size=batch_size,
+        device=torch_device,
+        return_periodicity=True,
+    )
+    pd = torchcrepe.filter.median(pd, 3)
+    f0 = torchcrepe.filter.mean(f0, 3)
+    f0[pd < 0.1] = 0
+    f0 = f0[0].cpu().numpy()
+    f0 = f0[1:] # Get rid of extra first frame
+    return f0
+
+def get_f0_crepe_computation(
+        x, 
+        sr,
+        f0_min,
+        f0_max,
+        hop_length=160, # 512 before. Hop length changes the speed that the voice jumps to a different dramatic pitch. Lower hop lengths means more pitch accuracy but longer inference time.
+        model="full", # Either use crepe-tiny "tiny" or crepe "full". Default is full
+):
+    x = x.astype(np.float32) # fixes the F.conv2D exception. We needed to convert double to float.
+    x /= np.quantile(np.abs(x), 0.999)
+    torch_device = get_optimal_torch_device()
+    audio = torch.from_numpy(x).to(torch_device, copy=True)
+    audio = torch.unsqueeze(audio, dim=0)
+    if audio.ndim == 2 and audio.shape[0] > 1:
+        audio = torch.mean(audio, dim=0, keepdim=True).detach()
+    audio = audio.detach()
+    print("Initiating prediction with a crepe_hop_length of: " + str(hop_length))
+    pitch: Tensor = torchcrepe.predict(
+        audio,
+        sr,
+        hop_length,
+        f0_min,
+        f0_max,
+        model,
+        batch_size=hop_length * 2,
+        device=torch_device,
+        pad=True
+    )
+    p_len = x.shape[0] // hop_length
+    # Resize the pitch for final f0
+    source = np.array(pitch.squeeze(0).cpu().float().numpy())
+    source[source < 0.001] = np.nan
+    target = np.interp(
+        np.arange(0, len(source) * p_len, len(source)) / p_len,
+        np.arange(0, len(source)),
+        source
+    )
+    f0 = np.nan_to_num(target)
+    f0 = f0[1:] # Get rid of extra first frame
+    return f0 # Resized f0
+
+
+def compute_f0(
+    path: str,
+    f0_method: str,
+    fs: int,
+    hop: int,
+    f0_max: float,
+    f0_min: float,
+):
+    x = load_audio(path, fs)
+    if f0_method == "harvest":
+        f0, t = pyworld.harvest(
+            x.astype(np.double),
+            fs=fs,
+            f0_ceil=f0_max,
+            f0_floor=f0_min,
+            frame_period=1000 * hop / fs,
+        )
+        f0 = pyworld.stonemask(x.astype(np.double), f0, t, fs)
+    elif f0_method == "dio":
+        f0, t = pyworld.dio(
+            x.astype(np.double),
+            fs=fs,
+            f0_ceil=f0_max,
+            f0_floor=f0_min,
+            frame_period=1000 * hop / fs,
+        )
+        f0 = pyworld.stonemask(x.astype(np.double), f0, t, fs)
+    elif f0_method == "mangio-crepe":
+        f0 = get_f0_crepe_computation(x, fs, f0_min, f0_max, 160, "full")
+    elif f0_method == "crepe":
+        f0 = get_f0_official_crepe_computation(x.astype(np.double), fs, f0_min, f0_max, "full")
+    return f0
+
+
+def coarse_f0(f0, f0_bin, f0_mel_min, f0_mel_max):
+    f0_mel = 1127 * np.log(1 + f0 / 700)
+    f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * (f0_bin - 2) / (
+        f0_mel_max - f0_mel_min
+    ) + 1
+
+    # use 0 or 1
+    f0_mel[f0_mel <= 1] = 1
+    f0_mel[f0_mel > f0_bin - 1] = f0_bin - 1
+    f0_coarse = np.rint(f0_mel).astype(np.int)
+    assert f0_coarse.max() <= 255 and f0_coarse.min() >= 1, (
+        f0_coarse.max(),
+        f0_coarse.min(),
+    )
+    return f0_coarse
+
+
+def processor(paths, f0_method, samplerate=16000, hop_size=160, process_id=0):
+    fs = samplerate
+    hop = hop_size
+
+    f0_bin = 256
+    f0_max = 1100.0
+    f0_min = 50.0
+    f0_mel_min = 1127 * np.log(1 + f0_min / 700)
+    f0_mel_max = 1127 * np.log(1 + f0_max / 700)
+    if len(paths) != 0:
+        for idx, (inp_path, opt_path1, opt_path2) in enumerate(
+            tqdm(paths, position=1 + process_id)
+        ):
+            try:
+                if (
+                    os.path.exists(opt_path1 + ".npy") == True
+                    and os.path.exists(opt_path2 + ".npy") == True
+                ):
+                    continue
+                featur_pit = compute_f0(inp_path, f0_method, fs, hop, f0_max, f0_min)
+                np.save(
+                    opt_path2,
+                    featur_pit,
+                    allow_pickle=False,
+                )  # nsf
+                coarse_pit = coarse_f0(featur_pit, f0_bin, f0_mel_min, f0_mel_max)
+                np.save(
+                    opt_path1,
+                    coarse_pit,
+                    allow_pickle=False,
+                )  # ori
+            except:
+                print(f"f0 failed {idx}: {inp_path} {traceback.format_exc()}")
+
+
+def run(training_dir: str, num_processes: int, f0_method: str):
+    paths = []
+    dataset_dir = os.path.join(training_dir, "1_16k_wavs")
+    opt_dir_f0 = os.path.join(training_dir, "2a_f0")
+    opt_dir_f0_nsf = os.path.join(training_dir, "2b_f0nsf")
+
+    if os.path.exists(opt_dir_f0) and os.path.exists(opt_dir_f0_nsf):
+        return
+
+    os.makedirs(opt_dir_f0, exist_ok=True)
+    os.makedirs(opt_dir_f0_nsf, exist_ok=True)
+
+    names = []
+
+    for pathname in sorted(list(os.listdir(dataset_dir))):
+        if os.path.isdir(os.path.join(dataset_dir, pathname)):
+            for f in sorted(list(os.listdir(os.path.join(dataset_dir, pathname)))):
+                if "spec" in f:
+                    continue
+                names.append(os.path.join(pathname, f))
+        else:
+            names.append(pathname)
+
+    for name in names:  # dataset_dir/{05d}/file.ext
+        filepath = os.path.join(dataset_dir, name)
+        if "spec" in filepath:
+            continue
+        opt_filepath_f0 = os.path.join(opt_dir_f0, name)
+        opt_filepath_f0_nsf = os.path.join(opt_dir_f0_nsf, name)
+        paths.append([filepath, opt_filepath_f0, opt_filepath_f0_nsf])
+
+    for dir in set([(os.path.dirname(p[1]), os.path.dirname(p[2])) for p in paths]):
+        os.makedirs(dir[0], exist_ok=True)
+        os.makedirs(dir[1], exist_ok=True)
+
+    with ProcessPoolExecutor(mp_context=mp.get_context("spawn")) as executer:
+        for i in range(num_processes):
+            executer.submit(processor, paths[i::num_processes], f0_method, process_id=i)
+
+    processor(paths, f0_method)

lib/rvc/preprocessing/extract_feature.py

@@ -0,0 +1,217 @@
+import multiprocessing as mp
+import os
+import traceback
+from concurrent.futures import ProcessPoolExecutor
+from typing import *
+
+import numpy as np
+import soundfile as sf
+import torch
+import torch.nn.functional as F
+from fairseq import checkpoint_utils
+from tqdm import tqdm
+
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+MODELS_DIR = os.path.join(ROOT_DIR, "models")
+EMBEDDINGS_LIST = {
+    "hubert-base-japanese": (
+        "rinna_hubert_base_jp.pt",
+        "hubert-base-japanese",
+        "local",
+    ),
+    "contentvec": ("checkpoint_best_legacy_500.pt", "contentvec", "local"),
+}
+
+def get_embedder(embedder_name):
+    if embedder_name in EMBEDDINGS_LIST:
+        return EMBEDDINGS_LIST[embedder_name]
+    return None
+
+
+def load_embedder(embedder_path: str, device):
+    try:
+        models, cfg, _ = checkpoint_utils.load_model_ensemble_and_task(
+            [embedder_path],
+            suffix="",
+        )
+        embedder_model = models[0]
+        embedder_model = embedder_model.to(device)
+        if device != "cpu":
+            embedder_model = embedder_model.half()
+        else:
+            embedder_model = embedder_model.float()
+        embedder_model.eval()
+    except Exception as e:
+        print(f"Error: {e} {embedder_path}")
+        traceback.print_exc()
+
+    return embedder_model, cfg
+
+
+# wave must be 16k, hop_size=320
+def readwave(wav_path, normalize=False):
+    wav, sr = sf.read(wav_path)
+    assert sr == 16000
+    feats = torch.from_numpy(wav).float()
+    if feats.dim() == 2:  # double channels
+        feats = feats.mean(-1)
+    assert feats.dim() == 1, feats.dim()
+    if normalize:
+        with torch.no_grad():
+            feats = F.layer_norm(feats, feats.shape)
+    feats = feats.view(1, -1)
+    return feats
+
+
+def processor(
+    todo: List[str],
+    device: torch.device,
+    embedder_path: str,
+    embedder_load_from: str,
+    embedding_channel: bool,
+    embedding_output_layer: int,
+    wav_dir: str,
+    out_dir: str,
+    process_id: int,
+):
+    half_support = (
+        device.type == "cuda" and torch.cuda.get_device_capability(device)[0] >= 5.3
+    )
+    is_feats_dim_768 = embedding_channel == 768
+
+    if embedder_load_from == "local" and not os.path.exists(embedder_path):
+        return f"Embedder not found: {embedder_path}"
+
+    model, cfg = load_embedder(embedder_path, device)
+
+    for file in tqdm(todo, position=1 + process_id):
+        try:
+            if file.endswith(".wav"):
+                wav_filepath = os.path.join(wav_dir, file)
+                out_filepath = os.path.join(out_dir, file.replace("wav", "npy"))
+
+                if os.path.exists(out_filepath):
+                    continue
+
+                os.makedirs(os.path.dirname(out_filepath), exist_ok=True)
+
+                is_normalize = False if cfg is None else cfg.task.normalize
+                feats = readwave(wav_filepath, normalize=is_normalize)
+                padding_mask = torch.BoolTensor(feats.shape).fill_(False)
+                if isinstance(model, tuple):
+                    feats = model[0](
+                        feats.squeeze(0).squeeze(0).to(device),
+                        return_tensors="pt",
+                        sampling_rate=16000,
+                    )
+                    if half_support:
+                        feats = feats.input_values.to(device).half()
+                    else:
+                        feats = feats.input_values.to(device).float()
+
+                    with torch.no_grad():
+                        if half_support:
+                            if is_feats_dim_768:
+                                feats = model[1](feats).last_hidden_state
+                            else:
+                                feats = model[1](feats).extract_features
+                        else:
+                            if is_feats_dim_768:
+                                feats = model[1].float()(feats).last_hidden_state
+                            else:
+                                feats = model[1].float()(feats).extract_features
+                else:
+                    inputs = {
+                        "source": feats.half().to(device)
+                        if half_support
+                        else feats.to(device),
+                        "padding_mask": padding_mask.to(device),
+                        "output_layer": embedding_output_layer,
+                    }
+
+                    # なんかまだこの時点でfloat16なので改めて変換
+                    if not half_support:
+                        model = model.float()
+                        inputs["source"] = inputs["source"].float()
+
+                    with torch.no_grad():
+                        logits = model.extract_features(**inputs)
+                        if is_feats_dim_768:
+                            feats = logits[0]
+                        else:
+                            feats = model.final_proj(logits[0])
+
+                feats = feats.squeeze(0).float().cpu().numpy()
+                if np.isnan(feats).sum() == 0:
+                    np.save(out_filepath, feats, allow_pickle=False)
+                else:
+                    print(f"{file} contains nan")
+        except Exception as e:
+            print(f"Error: {e} {file}")
+            traceback.print_exc()
+
+
+def run(
+    training_dir: str,
+    embedder_path: str,
+    embedder_load_from: str,
+    embedding_channel: int,
+    embedding_output_layer: int,
+    gpu_ids: List[int],
+    device: Optional[Union[torch.device, str]] = None,
+):
+    wav_dir = os.path.join(training_dir, "1_16k_wavs")
+    out_dir = os.path.join(training_dir, "3_feature256")
+
+    num_gpus = len(gpu_ids)
+
+    for gpu_id in gpu_ids:
+        if num_gpus < gpu_id + 1:
+            print(f"GPU {gpu_id} is not available")
+            return
+
+    if os.path.exists(out_dir):
+        return
+
+    os.makedirs(out_dir, exist_ok=True)
+
+    todo = [
+        os.path.join(dir, f)
+        for dir in sorted(list(os.listdir(wav_dir)))
+        if os.path.isdir(os.path.join(wav_dir, dir))
+        for f in sorted(list(os.listdir(os.path.join(wav_dir, dir))))
+    ]
+
+    if device is not None:
+        if type(device) == str:
+            device = torch.device(device)
+        if device.type == "mps":
+            device = torch.device(
+                "cpu"
+            )  # Mac(MPS) crashes when multiprocess, so change to CPU.
+        processor(
+            todo,
+            device,
+            embedder_path,
+            embedder_load_from,
+            embedding_channel,
+            embedding_output_layer,
+            wav_dir,
+            out_dir,
+            process_id=0,
+        )
+    else:
+        with ProcessPoolExecutor(mp_context=mp.get_context("spawn")) as executor:
+            for i, id in enumerate(gpu_ids):
+                executor.submit(
+                    processor,
+                    todo[i::num_gpus],
+                    torch.device(f"cuda:{id}"),
+                    embedder_path,
+                    embedder_load_from,
+                    embedding_channel,
+                    embedding_output_layer,
+                    wav_dir,
+                    out_dir,
+                    process_id=i,
+                )

lib/rvc/preprocessing/slicer.py

@@ -0,0 +1,179 @@
+import numpy as np
+
+
+# This function is obtained from librosa.
+def get_rms(
+    y,
+    frame_length=2048,
+    hop_length=512,
+    pad_mode="constant",
+):
+    padding = (int(frame_length // 2), int(frame_length // 2))
+    y = np.pad(y, padding, mode=pad_mode)
+
+    axis = -1
+    # put our new within-frame axis at the end for now
+    out_strides = y.strides + tuple([y.strides[axis]])
+    # Reduce the shape on the framing axis
+    x_shape_trimmed = list(y.shape)
+    x_shape_trimmed[axis] -= frame_length - 1
+    out_shape = tuple(x_shape_trimmed) + tuple([frame_length])
+    xw = np.lib.stride_tricks.as_strided(y, shape=out_shape, strides=out_strides)
+    if axis < 0:
+        target_axis = axis - 1
+    else:
+        target_axis = axis + 1
+    xw = np.moveaxis(xw, -1, target_axis)
+    # Downsample along the target axis
+    slices = [slice(None)] * xw.ndim
+    slices[axis] = slice(0, None, hop_length)
+    x = xw[tuple(slices)]
+
+    # Calculate power
+    power = np.mean(np.abs(x) ** 2, axis=-2, keepdims=True)
+
+    return np.sqrt(power)
+
+
+class Slicer:
+    def __init__(
+        self,
+        sr: int,
+        threshold: float = -40.0,
+        min_length: int = 5000,
+        min_interval: int = 300,
+        hop_size: int = 20,
+        max_sil_kept: int = 5000,
+    ):
+        if not min_length >= min_interval >= hop_size:
+            raise ValueError(
+                "The following condition must be satisfied: min_length >= min_interval >= hop_size"
+            )
+        if not max_sil_kept >= hop_size:
+            raise ValueError(
+                "The following condition must be satisfied: max_sil_kept >= hop_size"
+            )
+        min_interval = sr * min_interval / 1000
+        self.threshold = 10 ** (threshold / 20.0)
+        self.hop_size = round(sr * hop_size / 1000)
+        self.win_size = min(round(min_interval), 4 * self.hop_size)
+        self.min_length = round(sr * min_length / 1000 / self.hop_size)
+        self.min_interval = round(min_interval / self.hop_size)
+        self.max_sil_kept = round(sr * max_sil_kept / 1000 / self.hop_size)
+
+    def _apply_slice(self, waveform, begin, end):
+        if len(waveform.shape) > 1:
+            return waveform[
+                :, begin * self.hop_size : min(waveform.shape[1], end * self.hop_size)
+            ]
+        else:
+            return waveform[
+                begin * self.hop_size : min(waveform.shape[0], end * self.hop_size)
+            ]
+
+    # @timeit
+    def slice(self, waveform):
+        if len(waveform.shape) > 1:
+            samples = waveform.mean(axis=0)
+        else:
+            samples = waveform
+        if samples.shape[0] <= self.min_length:
+            return [waveform]
+        rms_list = get_rms(
+            y=samples, frame_length=self.win_size, hop_length=self.hop_size
+        ).squeeze(0)
+        sil_tags = []
+        silence_start = None
+        clip_start = 0
+        for i, rms in enumerate(rms_list):
+            # Keep looping while frame is silent.
+            if rms < self.threshold:
+                # Record start of silent frames.
+                if silence_start is None:
+                    silence_start = i
+                continue
+            # Keep looping while frame is not silent and silence start has not been recorded.
+            if silence_start is None:
+                continue
+            # Clear recorded silence start if interval is not enough or clip is too short
+            is_leading_silence = silence_start == 0 and i > self.max_sil_kept
+            need_slice_middle = (
+                i - silence_start >= self.min_interval
+                and i - clip_start >= self.min_length
+            )
+            if not is_leading_silence and not need_slice_middle:
+                silence_start = None
+                continue
+            # Need slicing. Record the range of silent frames to be removed.
+            if i - silence_start <= self.max_sil_kept:
+                pos = rms_list[silence_start : i + 1].argmin() + silence_start
+                if silence_start == 0:
+                    sil_tags.append((0, pos))
+                else:
+                    sil_tags.append((pos, pos))
+                clip_start = pos
+            elif i - silence_start <= self.max_sil_kept * 2:
+                pos = rms_list[
+                    i - self.max_sil_kept : silence_start + self.max_sil_kept + 1
+                ].argmin()
+                pos += i - self.max_sil_kept
+                pos_l = (
+                    rms_list[
+                        silence_start : silence_start + self.max_sil_kept + 1
+                    ].argmin()
+                    + silence_start
+                )
+                pos_r = (
+                    rms_list[i - self.max_sil_kept : i + 1].argmin()
+                    + i
+                    - self.max_sil_kept
+                )
+                if silence_start == 0:
+                    sil_tags.append((0, pos_r))
+                    clip_start = pos_r
+                else:
+                    sil_tags.append((min(pos_l, pos), max(pos_r, pos)))
+                    clip_start = max(pos_r, pos)
+            else:
+                pos_l = (
+                    rms_list[
+                        silence_start : silence_start + self.max_sil_kept + 1
+                    ].argmin()
+                    + silence_start
+                )
+                pos_r = (
+                    rms_list[i - self.max_sil_kept : i + 1].argmin()
+                    + i
+                    - self.max_sil_kept
+                )
+                if silence_start == 0:
+                    sil_tags.append((0, pos_r))
+                else:
+                    sil_tags.append((pos_l, pos_r))
+                clip_start = pos_r
+            silence_start = None
+        # Deal with trailing silence.
+        total_frames = rms_list.shape[0]
+        if (
+            silence_start is not None
+            and total_frames - silence_start >= self.min_interval
+        ):
+            silence_end = min(total_frames, silence_start + self.max_sil_kept)
+            pos = rms_list[silence_start : silence_end + 1].argmin() + silence_start
+            sil_tags.append((pos, total_frames + 1))
+        # Apply and return slices.
+        if len(sil_tags) == 0:
+            return [waveform]
+        else:
+            chunks = []
+            if sil_tags[0][0] > 0:
+                chunks.append(self._apply_slice(waveform, 0, sil_tags[0][0]))
+            for i in range(len(sil_tags) - 1):
+                chunks.append(
+                    self._apply_slice(waveform, sil_tags[i][1], sil_tags[i + 1][0])
+                )
+            if sil_tags[-1][1] < total_frames:
+                chunks.append(
+                    self._apply_slice(waveform, sil_tags[-1][1], total_frames)
+                )
+            return chunks

lib/rvc/preprocessing/split.py

@@ -0,0 +1,195 @@
+import operator
+import os
+from concurrent.futures import ProcessPoolExecutor
+from typing import *
+
+import librosa
+import numpy as np
+import scipy.signal as signal
+from scipy.io import wavfile
+from tqdm import tqdm
+
+from lib.rvc.utils import load_audio
+
+from .slicer import Slicer
+
+
+def norm_write(
+    tmp_audio: np.ndarray,
+    idx0: int,
+    idx1: int,
+    speaker_id: int,
+    outdir: str,
+    outdir_16k: str,
+    sampling_rate: int,
+    max: float,
+    alpha: float,
+    is_normalize: bool,
+):
+    if is_normalize:
+        tmp_audio = (tmp_audio / np.abs(tmp_audio).max() * (max * alpha)) + (
+            1 - alpha
+        ) * tmp_audio
+    else:
+        # clip level to max (cause sometimes when floating point decoding)
+        audio_min = np.min(tmp_audio)
+        if audio_min < -max:
+            tmp_audio = tmp_audio / -audio_min * max
+        audio_max = np.max(tmp_audio)
+        if audio_max > max:
+            tmp_audio = tmp_audio / audio_max * max
+
+    wavfile.write(
+        os.path.join(outdir, f"{speaker_id:05}", f"{idx0}_{idx1}.wav"),
+        sampling_rate,
+        tmp_audio.astype(np.float32),
+    )
+
+    tmp_audio = librosa.resample(
+        tmp_audio, orig_sr=sampling_rate, target_sr=16000, res_type="soxr_vhq"
+    )
+    wavfile.write(
+        os.path.join(outdir_16k, f"{speaker_id:05}", f"{idx0}_{idx1}.wav"),
+        16000,
+        tmp_audio.astype(np.float32),
+    )
+
+
+def write_mute(
+    mute_wave_filename: str,
+    speaker_id: int,
+    outdir: str,
+    outdir_16k: str,
+    sampling_rate: int,
+):
+    tmp_audio = load_audio(mute_wave_filename, sampling_rate)
+    wavfile.write(
+        os.path.join(outdir, f"{speaker_id:05}", "mute.wav"),
+        sampling_rate,
+        tmp_audio.astype(np.float32),
+    )
+    tmp_audio = librosa.resample(
+        tmp_audio, orig_sr=sampling_rate, target_sr=16000, res_type="soxr_vhq"
+    )
+    wavfile.write(
+        os.path.join(outdir_16k, f"{speaker_id:05}", "mute.wav"),
+        16000,
+        tmp_audio.astype(np.float32),
+    )
+
+
+def pipeline(
+    slicer: Slicer,
+    datasets: List[Tuple[str, int]],  # List[(path, speaker_id)]
+    outdir: str,
+    outdir_16k: str,
+    sampling_rate: int,
+    is_normalize: bool,
+    process_id: int = 0,
+):
+    per = 3.7
+    overlap = 0.3
+    tail = per + overlap
+    max = 0.95
+    alpha = 0.8
+
+    bh, ah = signal.butter(N=5, Wn=48, btype="high", fs=sampling_rate)
+
+    for index, (wave_filename, speaker_id) in tqdm(datasets, position=1 + process_id):
+        audio = load_audio(wave_filename, sampling_rate)
+        audio = signal.lfilter(bh, ah, audio)
+
+        idx1 = 0
+        for audio in slicer.slice(audio):
+            i = 0
+            while 1:
+                start = int(sampling_rate * (per - overlap) * i)
+                i += 1
+                if len(audio[start:]) > tail * sampling_rate:
+                    tmp_audio = audio[start : start + int(per * sampling_rate)]
+                    norm_write(
+                        tmp_audio,
+                        index,
+                        idx1,
+                        speaker_id,
+                        outdir,
+                        outdir_16k,
+                        sampling_rate,
+                        max,
+                        alpha,
+                        is_normalize,
+                    )
+                    idx1 += 1
+                else:
+                    tmp_audio = audio[start:]
+                    break
+            norm_write(
+                tmp_audio,
+                index,
+                idx1,
+                speaker_id,
+                outdir,
+                outdir_16k,
+                sampling_rate,
+                max,
+                alpha,
+                is_normalize,
+            )
+            idx1 += 1
+
+
+def preprocess_audio(
+    datasets: List[Tuple[str, int]],  # List[(path, speaker_id)]
+    sampling_rate: int,
+    num_processes: int,
+    training_dir: str,
+    is_normalize: bool,
+    mute_wav_path: str,
+):
+    waves_dir = os.path.join(training_dir, "0_gt_wavs")
+    waves16k_dir = os.path.join(training_dir, "1_16k_wavs")
+    if os.path.exists(waves_dir) and os.path.exists(waves16k_dir):
+        return
+
+    for speaker_id in set([spk for _, spk in datasets]):
+        os.makedirs(os.path.join(waves_dir, f"{speaker_id:05}"), exist_ok=True)
+        os.makedirs(os.path.join(waves16k_dir, f"{speaker_id:05}"), exist_ok=True)
+
+    all = [(i, x) for i, x in enumerate(sorted(datasets, key=operator.itemgetter(0)))]
+
+    # n of datasets per process
+    process_all_nums = [len(all) // num_processes] * num_processes
+    # add residual datasets
+    for i in range(len(all) % num_processes):
+        process_all_nums[i] += 1
+
+    assert len(all) == sum(process_all_nums), print(
+        f"len(all): {len(all)}, sum(process_all_nums): {sum(process_all_nums)}"
+    )
+
+    with ProcessPoolExecutor(max_workers=num_processes) as executor:
+        all_index = 0
+        for i in range(num_processes):
+            data = all[all_index : all_index + process_all_nums[i]]
+            slicer = Slicer(
+                sr=sampling_rate,
+                threshold=-42,
+                min_length=1500,
+                min_interval=400,
+                hop_size=15,
+                max_sil_kept=500,
+            )
+            executor.submit(
+                pipeline,
+                slicer,
+                data,
+                waves_dir,
+                waves16k_dir,
+                sampling_rate,
+                is_normalize,
+                process_id=i,
+            )
+            all_index += process_all_nums[i]
+
+    for speaker_id in set([spk for _, spk in datasets]):
+        write_mute(mute_wav_path, speaker_id, waves_dir, waves16k_dir, sampling_rate)

lib/rvc/train.py

@@ -0,0 +1,998 @@
+import glob
+import json
+import operator
+import os
+import shutil
+import time
+from random import shuffle
+from typing import *
+
+import faiss
+import numpy as np
+import torch
+import torch.distributed as dist
+import torch.multiprocessing as mp
+import torchaudio
+import tqdm
+from sklearn.cluster import MiniBatchKMeans
+from torch.cuda.amp import GradScaler, autocast
+from torch.nn import functional as F
+from torch.nn.parallel import DistributedDataParallel as DDP
+from torch.utils.data import DataLoader
+from torch.utils.tensorboard import SummaryWriter
+
+from . import commons, utils
+from .checkpoints import save
+from .config import DatasetMetadata, TrainConfig
+from .data_utils import (DistributedBucketSampler, TextAudioCollate,
+                         TextAudioCollateMultiNSFsid, TextAudioLoader,
+                         TextAudioLoaderMultiNSFsid)
+from .losses import discriminator_loss, feature_loss, generator_loss, kl_loss
+from .mel_processing import mel_spectrogram_torch, spec_to_mel_torch
+from .models import (MultiPeriodDiscriminator, SynthesizerTrnMs256NSFSid,
+                     SynthesizerTrnMs256NSFSidNono)
+from .preprocessing.extract_feature import (MODELS_DIR, get_embedder,
+                                            load_embedder)
+
+
+def is_audio_file(file: str):
+    if "." not in file:
+        return False
+    ext = os.path.splitext(file)[1]
+    return ext.lower() in [
+        ".wav",
+        ".flac",
+        ".ogg",
+        ".mp3",
+        ".m4a",
+        ".wma",
+        ".aiff",
+    ]
+
+
+def glob_dataset(
+    glob_str: str,
+    speaker_id: int,
+    multiple_speakers: bool = False,
+    recursive: bool = True,
+    training_dir: str = ".",
+):
+    globs = glob_str.split(",")
+    speaker_count = 0
+    datasets_speakers = []
+    speaker_to_id_mapping = {}
+    for glob_str in globs:
+        if os.path.isdir(glob_str):
+            if multiple_speakers:
+                # Multispeaker format:
+                # dataset_path/
+                # - speakername/
+                #     - {wav name here}.wav
+                #     - ...
+                # - next_speakername/
+                #     - {wav name here}.wav
+                #     - ...
+                # - ...
+                print("Multispeaker dataset enabled; Processing speakers.")
+                for dir in tqdm.tqdm(os.listdir(glob_str)):
+                    print("Speaker ID " + str(speaker_count) + ": " + dir)
+                    speaker_to_id_mapping[dir] = speaker_count
+                    speaker_path = glob_str + "/" + dir
+                    for audio in tqdm.tqdm(os.listdir(speaker_path)):
+                        if is_audio_file(glob_str + "/" + dir + "/" + audio):
+                            datasets_speakers.append((glob_str + "/" + dir + "/" + audio, speaker_count))
+                    speaker_count += 1
+                with open(os.path.join(training_dir, "speaker_info.json"), "w") as outfile:
+                    print("Dumped speaker info to {}".format(os.path.join(training_dir, "speaker_info.json")))
+                    json.dump(speaker_to_id_mapping, outfile)
+                continue # Skip the normal speaker extend
+
+            glob_str = os.path.join(glob_str, "**", "*")
+        print("Single speaker dataset enabled; Processing speaker as ID " + str(speaker_id) + ".")
+        datasets_speakers.extend(
+            [
+                (file, speaker_id)
+                for file in glob.iglob(glob_str, recursive=recursive)
+                if is_audio_file(file)
+            ]
+        )
+
+    return sorted(datasets_speakers)
+
+
+def create_dataset_meta(training_dir: str, f0: bool):
+    gt_wavs_dir = os.path.join(training_dir, "0_gt_wavs")
+    co256_dir = os.path.join(training_dir, "3_feature256")
+
+    def list_data(dir: str):
+        files = []
+        for subdir in os.listdir(dir):
+            speaker_dir = os.path.join(dir, subdir)
+            for name in os.listdir(speaker_dir):
+                files.append(os.path.join(subdir, name.split(".")[0]))
+        return files
+
+    names = set(list_data(gt_wavs_dir)) & set(list_data(co256_dir))
+
+    if f0:
+        f0_dir = os.path.join(training_dir, "2a_f0")
+        f0nsf_dir = os.path.join(training_dir, "2b_f0nsf")
+        names = names & set(list_data(f0_dir)) & set(list_data(f0nsf_dir))
+
+    meta = {
+        "files": {},
+    }
+
+    for name in names:
+        speaker_id = os.path.dirname(name).split("_")[0]
+        speaker_id = int(speaker_id) if speaker_id.isdecimal() else 0
+        if f0:
+            gt_wav_path = os.path.join(gt_wavs_dir, f"{name}.wav")
+            co256_path = os.path.join(co256_dir, f"{name}.npy")
+            f0_path = os.path.join(f0_dir, f"{name}.wav.npy")
+            f0nsf_path = os.path.join(f0nsf_dir, f"{name}.wav.npy")
+            meta["files"][name] = {
+                "gt_wav": gt_wav_path,
+                "co256": co256_path,
+                "f0": f0_path,
+                "f0nsf": f0nsf_path,
+                "speaker_id": speaker_id,
+            }
+        else:
+            gt_wav_path = os.path.join(gt_wavs_dir, f"{name}.wav")
+            co256_path = os.path.join(co256_dir, f"{name}.npy")
+            meta["files"][name] = {
+                "gt_wav": gt_wav_path,
+                "co256": co256_path,
+                "speaker_id": speaker_id,
+            }
+
+    with open(os.path.join(training_dir, "meta.json"), "w") as f:
+        json.dump(meta, f, indent=2)
+
+
+def change_speaker(net_g, speaker_info, embedder, embedding_output_layer, phone, phone_lengths, pitch, pitchf, spec_lengths):
+    """
+    random change formant
+    inspired by https://github.com/auspicious3000/contentvec/blob/d746688a32940f4bee410ed7c87ec9cf8ff04f74/contentvec/data/audio/audio_utils_1.py#L179
+    """
+    N = phone.shape[0]
+    device = phone.device
+    dtype = phone.dtype
+
+    f0_bin = 256
+    f0_max = 1100.0
+    f0_min = 50.0
+    f0_mel_min = 1127 * np.log(1 + f0_min / 700)
+    f0_mel_max = 1127 * np.log(1 + f0_max / 700)
+
+    pitch_median = torch.median(pitchf, 1).values
+    lo = 75. + 25. * (pitch_median >= 200).to(dtype=dtype)
+    hi = 250. + 150. * (pitch_median >= 200).to(dtype=dtype)
+    pitch_median = torch.clip(pitch_median, lo, hi).unsqueeze(1)
+
+    shift_pitch = torch.exp2((1. - 2. * torch.rand(N)) / 4).unsqueeze(1).to(device, dtype)   # ピッチを半オクターブの範囲でずらす
+
+    new_sid = np.random.choice(np.arange(len(speaker_info))[speaker_info > 0], size=N)
+    rel_pitch = pitchf / pitch_median
+    new_pitch_median = torch.from_numpy(speaker_info[new_sid]).to(device, dtype).unsqueeze(1) * shift_pitch
+    new_pitchf = new_pitch_median * rel_pitch
+    new_sid = torch.from_numpy(new_sid).to(device)
+
+    new_pitch = 1127. * torch.log(1. + new_pitchf / 700.)
+    new_pitch = (pitch - f0_mel_min) * (f0_bin - 2.) / (f0_mel_max - f0_mel_min) + 1.
+    new_pitch = torch.clip(new_pitch, 1, f0_bin - 1).to(dtype=torch.int)
+
+    aug_wave = net_g.infer(phone, phone_lengths, new_pitch, new_pitchf, new_sid)[0]
+    aug_wave_16k = torchaudio.functional.resample(aug_wave, net_g.sr, 16000, rolloff=0.99).squeeze(1)
+    padding_mask = torch.arange(aug_wave_16k.shape[1]).unsqueeze(0).to(device) > (spec_lengths.unsqueeze(1) * 160).to(device)
+
+    inputs = {
+        "source": aug_wave_16k.to(device, dtype),
+        "padding_mask": padding_mask.to(device),
+        "output_layer": embedding_output_layer
+    }
+    logits = embedder.extract_features(**inputs)
+    if phone.shape[-1] == 768:
+        feats = logits[0]
+    else:
+        feats = embedder.final_proj(logits[0])
+    feats = torch.repeat_interleave(feats, 2, 1)
+    new_phone = torch.zeros(phone.shape).to(device, dtype)
+    new_phone[:, :feats.shape[1]] = feats[:, :phone.shape[1]]
+    return new_phone.to(device), aug_wave
+
+
+def change_speaker_nono(net_g, embedder, embedding_output_layer, phone, phone_lengths, spec_lengths):
+    """
+    random change formant
+    inspired by https://github.com/auspicious3000/contentvec/blob/d746688a32940f4bee410ed7c87ec9cf8ff04f74/contentvec/data/audio/audio_utils_1.py#L179
+    """
+    N = phone.shape[0]
+    device = phone.device
+    dtype = phone.dtype
+
+    new_sid = np.random.randint(net_g.spk_embed_dim, size=N)
+    new_sid = torch.from_numpy(new_sid).to(device)
+
+    aug_wave = net_g.infer(phone, phone_lengths, new_sid)[0]
+    aug_wave_16k = torchaudio.functional.resample(aug_wave, net_g.sr, 16000, rolloff=0.99).squeeze(1)
+    padding_mask = torch.arange(aug_wave_16k.shape[1]).unsqueeze(0).to(device) > (spec_lengths.unsqueeze(1) * 160).to(device)
+
+    inputs = {
+        "source": aug_wave_16k.to(device, dtype),
+        "padding_mask": padding_mask.to(device),
+        "output_layer": embedding_output_layer
+    }
+
+    logits = embedder.extract_features(**inputs)
+    if phone.shape[-1] == 768:
+        feats = logits[0]
+    else:
+        feats = embedder.final_proj(logits[0])
+    feats = torch.repeat_interleave(feats, 2, 1)
+    new_phone = torch.zeros(phone.shape).to(device, dtype)
+    new_phone[:, :feats.shape[1]] = feats[:, :phone.shape[1]]
+    return new_phone.to(device), aug_wave
+
+
+def train_index(
+    training_dir: str,
+    model_name: str,
+    out_dir: str,
+    emb_ch: int,
+    num_cpu_process: int,
+    maximum_index_size: Optional[int],
+):
+    checkpoint_path = os.path.join(out_dir, model_name)
+    feature_256_dir = os.path.join(training_dir, "3_feature256")
+    index_dir = os.path.join(os.path.dirname(checkpoint_path), f"{model_name}_index")
+    os.makedirs(index_dir, exist_ok=True)
+    for speaker_id in tqdm.tqdm(
+        sorted([dir for dir in os.listdir(feature_256_dir) if dir.isdecimal()])
+    ):
+        feature_256_spk_dir = os.path.join(feature_256_dir, speaker_id)
+        speaker_id = int(speaker_id)
+        npys = []
+        for name in [
+            os.path.join(feature_256_spk_dir, file)
+            for file in os.listdir(feature_256_spk_dir)
+            if file.endswith(".npy")
+        ]:
+            phone = np.load(os.path.join(feature_256_spk_dir, name))
+            npys.append(phone)
+
+        # shuffle big_npy to prevent reproducing the sound source
+        big_npy = np.concatenate(npys, 0)
+        big_npy_idx = np.arange(big_npy.shape[0])
+        np.random.shuffle(big_npy_idx)
+        big_npy = big_npy[big_npy_idx]
+
+        if not maximum_index_size is None and big_npy.shape[0] > maximum_index_size:
+            kmeans = MiniBatchKMeans(
+                n_clusters=maximum_index_size,
+                batch_size=256 * num_cpu_process,
+                init="random",
+                compute_labels=False,
+            )
+            kmeans.fit(big_npy)
+            big_npy = kmeans.cluster_centers_
+
+        # recommend parameter in https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index
+        emb_ch = big_npy.shape[1]
+        emb_ch_half = emb_ch // 2
+        n_ivf = int(8 * np.sqrt(big_npy.shape[0]))
+        if big_npy.shape[0] >= 1_000_000:
+            index = faiss.index_factory(
+                emb_ch, f"IVF{n_ivf},PQ{emb_ch_half}x4fsr,RFlat"
+            )
+        else:
+            index = faiss.index_factory(emb_ch, f"IVF{n_ivf},Flat")
+
+        index.train(big_npy)
+        batch_size_add = 8192
+        for i in range(0, big_npy.shape[0], batch_size_add):
+            index.add(big_npy[i : i + batch_size_add])
+        np.save(
+            os.path.join(index_dir, f"{model_name}.{speaker_id}.big.npy"),
+            big_npy,
+        )
+        faiss.write_index(
+            index,
+            os.path.join(index_dir, f"{model_name}.{speaker_id}.index"),
+        )
+
+
+def train_model(
+    gpus: List[int],
+    config: TrainConfig,
+    training_dir: str,
+    model_name: str,
+    out_dir: str,
+    sample_rate: int,
+    f0: bool,
+    batch_size: int,
+    augment: bool,
+    augment_path: Optional[str],
+    speaker_info_path: Optional[str],
+    cache_batch: bool,
+    total_epoch: int,
+    save_every_epoch: int,
+    save_wav_with_checkpoint: bool,
+    pretrain_g: str,
+    pretrain_d: str,
+    embedder_name: str,
+    embedding_output_layer: int,
+    save_only_last: bool = False,
+    device: Optional[Union[str, torch.device]] = None,
+):
+    os.environ["MASTER_ADDR"] = "localhost"
+    os.environ["MASTER_PORT"] = str(utils.find_empty_port())
+
+    deterministic = torch.backends.cudnn.deterministic
+    benchmark = torch.backends.cudnn.benchmark
+    PREV_CUDA_VISIBLE_DEVICES = os.environ.get("CUDA_VISIBLE_DEVICES", None)
+
+    torch.backends.cudnn.deterministic = False
+    torch.backends.cudnn.benchmark = False
+
+    os.environ["CUDA_VISIBLE_DEVICES"] = ",".join([str(gpu) for gpu in gpus])
+
+    start = time.perf_counter()
+
+    # Mac(MPS)でやると、mp.spawnでなんかトラブルが出るので普通にtraining_runnerを呼び出す。
+    if device is not None:
+        training_runner(
+            0,  # rank
+            1,  # world size
+            config,
+            training_dir,
+            model_name,
+            out_dir,
+            sample_rate,
+            f0,
+            batch_size,
+            augment,
+            augment_path,
+            speaker_info_path,
+            cache_batch,
+            total_epoch,
+            save_every_epoch,
+            save_wav_with_checkpoint,
+            pretrain_g,
+            pretrain_d,
+            embedder_name,
+            embedding_output_layer,
+            save_only_last,
+            device,
+        )
+    else:
+        mp.spawn(
+            training_runner,
+            nprocs=len(gpus),
+            args=(
+                len(gpus),
+                config,
+                training_dir,
+                model_name,
+                out_dir,
+                sample_rate,
+                f0,
+                batch_size,
+                augment,
+                augment_path,
+                speaker_info_path,
+                cache_batch,
+                total_epoch,
+                save_every_epoch,
+                save_wav_with_checkpoint,
+                pretrain_g,
+                pretrain_d,
+                embedder_name,
+                embedding_output_layer,
+                save_only_last,
+                device,
+            ),
+        )
+
+    end = time.perf_counter()
+
+    print(f"Time: {end - start}")
+
+    if PREV_CUDA_VISIBLE_DEVICES is None:
+        del os.environ["CUDA_VISIBLE_DEVICES"]
+    else:
+        os.environ["CUDA_VISIBLE_DEVICES"] = PREV_CUDA_VISIBLE_DEVICES
+
+    torch.backends.cudnn.deterministic = deterministic
+    torch.backends.cudnn.benchmark = benchmark
+
+
+def training_runner(
+    rank: int,
+    world_size: List[int],
+    config: TrainConfig,
+    training_dir: str,
+    model_name: str,
+    out_dir: str,
+    sample_rate: int,
+    f0: bool,
+    batch_size: int,
+    augment: bool,
+    augment_path: Optional[str],
+    speaker_info_path: Optional[str],
+    cache_in_gpu: bool,
+    total_epoch: int,
+    save_every_epoch: int,
+    save_wav_with_checkpoint: bool,
+    pretrain_g: str,
+    pretrain_d: str,
+    embedder_name: str,
+    embedding_output_layer: int,
+    save_only_last: bool = False,
+    device: Optional[Union[str, torch.device]] = None,
+):
+    config.train.batch_size = batch_size
+    log_dir = os.path.join(training_dir, "logs")
+    state_dir = os.path.join(training_dir, "state")
+    training_files_path = os.path.join(training_dir, "meta.json")
+    training_meta = DatasetMetadata.parse_file(training_files_path)
+    embedder_out_channels = config.model.emb_channels
+
+    is_multi_process = world_size > 1
+
+    if device is not None:
+        if type(device) == str:
+            device = torch.device(device)
+
+    global_step = 0
+    is_main_process = rank == 0
+
+    if is_main_process:
+        os.makedirs(log_dir, exist_ok=True)
+        os.makedirs(state_dir, exist_ok=True)
+        writer = SummaryWriter(log_dir=log_dir)
+
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+
+    if not dist.is_initialized():
+        dist.init_process_group(
+            backend="gloo", init_method="env://", rank=rank, world_size=world_size
+        )
+
+    if is_multi_process:
+        torch.cuda.set_device(rank)
+
+    torch.manual_seed(config.train.seed)
+
+    if f0:
+        train_dataset = TextAudioLoaderMultiNSFsid(training_meta, config.data)
+    else:
+        train_dataset = TextAudioLoader(training_meta, config.data)
+
+    train_sampler = DistributedBucketSampler(
+        train_dataset,
+        config.train.batch_size * world_size,
+        [100, 200, 300, 400, 500, 600, 700, 800, 900],
+        num_replicas=world_size,
+        rank=rank,
+        shuffle=True,
+    )
+
+    if f0:
+        collate_fn = TextAudioCollateMultiNSFsid()
+    else:
+        collate_fn = TextAudioCollate()
+
+    train_loader = DataLoader(
+        train_dataset,
+        num_workers=4,
+        shuffle=False,
+        pin_memory=True,
+        collate_fn=collate_fn,
+        batch_sampler=train_sampler,
+        persistent_workers=True,
+        prefetch_factor=8,
+    )
+    speaker_info = None
+    if os.path.exists(os.path.join(training_dir, "speaker_info.json")):
+        with open(os.path.join(training_dir, "speaker_info.json"), "r") as f:
+            speaker_info = json.load(f)
+            config.model.spk_embed_dim = len(speaker_info)
+    if f0:
+        net_g = SynthesizerTrnMs256NSFSid(
+            config.data.filter_length // 2 + 1,
+            config.train.segment_size // config.data.hop_length,
+            **config.model.dict(),
+            is_half=False, # config.train.fp16_run,
+            sr=int(sample_rate[:-1] + "000"),
+        )
+    else:
+        net_g = SynthesizerTrnMs256NSFSidNono(
+            config.data.filter_length // 2 + 1,
+            config.train.segment_size // config.data.hop_length,
+            **config.model.dict(),
+            is_half=False, # config.train.fp16_run,
+            sr=int(sample_rate[:-1] + "000"),
+        )
+
+    if is_multi_process:
+        net_g = net_g.cuda(rank)
+    else:
+        net_g = net_g.to(device=device)
+
+    if config.version == "v1":
+        periods = [2, 3, 5, 7, 11, 17]
+    elif config.version == "v2":
+        periods = [2, 3, 5, 7, 11, 17, 23, 37]
+    net_d = MultiPeriodDiscriminator(config.model.use_spectral_norm, periods=periods)
+    if is_multi_process:
+        net_d = net_d.cuda(rank)
+    else:
+        net_d = net_d.to(device=device)
+
+    optim_g = torch.optim.AdamW(
+        net_g.parameters(),
+        config.train.learning_rate,
+        betas=config.train.betas,
+        eps=config.train.eps,
+    )
+    optim_d = torch.optim.AdamW(
+        net_d.parameters(),
+        config.train.learning_rate,
+        betas=config.train.betas,
+        eps=config.train.eps,
+    )
+
+    last_d_state = utils.latest_checkpoint_path(state_dir, "D_*.pth")
+    last_g_state = utils.latest_checkpoint_path(state_dir, "G_*.pth")
+
+    if last_d_state is None or last_g_state is None:
+        epoch = 1
+        global_step = 0
+        if os.path.exists(pretrain_g) and os.path.exists(pretrain_d):
+            net_g_state = torch.load(pretrain_g, map_location="cpu")["model"]
+            emb_spk_size = (config.model.spk_embed_dim, config.model.gin_channels)
+            emb_phone_size = (config.model.hidden_channels, config.model.emb_channels)
+            if emb_spk_size != net_g_state["emb_g.weight"].size():
+                original_weight = net_g_state["emb_g.weight"]
+                net_g_state["emb_g.weight"] = original_weight.mean(dim=0, keepdims=True) * torch.ones(emb_spk_size, device=original_weight.device, dtype=original_weight.dtype)
+            if emb_phone_size != net_g_state["enc_p.emb_phone.weight"].size():
+                # interpolate
+                orig_shape = net_g_state["enc_p.emb_phone.weight"].size()
+                if net_g_state["enc_p.emb_phone.weight"].dtype == torch.half:
+                    net_g_state["enc_p.emb_phone.weight"] = (
+                        F.interpolate(
+                            net_g_state["enc_p.emb_phone.weight"]
+                            .float()
+                            .unsqueeze(0)
+                            .unsqueeze(0),
+                            size=emb_phone_size,
+                            mode="bilinear",
+                        )
+                        .half()
+                        .squeeze(0)
+                        .squeeze(0)
+                    )
+                else:
+                    net_g_state["enc_p.emb_phone.weight"] = (
+                        F.interpolate(
+                            net_g_state["enc_p.emb_phone.weight"]
+                            .unsqueeze(0)
+                            .unsqueeze(0),
+                            size=emb_phone_size,
+                            mode="bilinear",
+                        )
+                        .squeeze(0)
+                        .squeeze(0)
+                    )
+                print(
+                    "interpolated pretrained state enc_p.emb_phone from",
+                    orig_shape,
+                    "to",
+                    emb_phone_size,
+                )
+            if is_multi_process:
+                net_g.module.load_state_dict(net_g_state)
+            else:
+                net_g.load_state_dict(net_g_state)
+
+            del net_g_state
+
+            if is_multi_process:
+                net_d.module.load_state_dict(
+                    torch.load(pretrain_d, map_location="cpu")["model"]
+                )
+            else:
+                net_d.load_state_dict(
+                    torch.load(pretrain_d, map_location="cpu")["model"]
+                )
+            if is_main_process:
+                print(f"loaded pretrained {pretrain_g} {pretrain_d}")
+
+    else:
+        _, _, _, epoch = utils.load_checkpoint(last_d_state, net_d, optim_d)
+        _, _, _, epoch = utils.load_checkpoint(last_g_state, net_g, optim_g)
+        if is_main_process:
+            print(f"loaded last state {last_d_state} {last_g_state}")
+
+        epoch += 1
+        global_step = (epoch - 1) * len(train_loader)
+
+    if augment:
+        # load embedder
+        embedder_filepath, _, embedder_load_from = get_embedder(embedder_name)
+
+        if embedder_load_from == "local":
+            embedder_filepath = os.path.join(
+                MODELS_DIR, "embeddings", embedder_filepath
+            )
+        embedder, _ = load_embedder(embedder_filepath, device)
+        if not config.train.fp16_run:
+            embedder = embedder.float()
+
+        if (augment_path is not None):
+            state_dict = torch.load(augment_path, map_location="cpu")
+            if state_dict["f0"] == 1:
+                augment_net_g = SynthesizerTrnMs256NSFSid(
+                    **state_dict["params"], is_half=config.train.fp16_run
+                )
+                augment_speaker_info = np.load(speaker_info_path)
+            else:
+                augment_net_g = SynthesizerTrnMs256NSFSidNono(
+                    **state_dict["params"], is_half=config.train.fp16_run
+                )
+
+            augment_net_g.load_state_dict(state_dict["weight"], strict=False)
+            augment_net_g.eval().to(device)
+
+        else:
+            augment_net_g = net_g
+            if f0:
+                medians = [[] for _ in range(augment_net_g.spk_embed_dim)]
+                for file in training_meta.files.values():
+                    f0f = np.load(file.f0nsf)
+                    if np.any(f0f > 0):
+                        medians[file.speaker_id].append(np.median(f0f[f0f > 0]))
+                augment_speaker_info = np.array([np.median(x) if len(x) else 0. for x in medians])
+                np.save(os.path.join(training_dir, "speaker_info.npy"), augment_speaker_info)
+
+    if is_multi_process:
+        net_g = DDP(net_g, device_ids=[rank])
+        net_d = DDP(net_d, device_ids=[rank])
+
+    scheduler_g = torch.optim.lr_scheduler.ExponentialLR(
+        optim_g, gamma=config.train.lr_decay, last_epoch=epoch - 2
+    )
+    scheduler_d = torch.optim.lr_scheduler.ExponentialLR(
+        optim_d, gamma=config.train.lr_decay, last_epoch=epoch - 2
+    )
+
+    scaler = GradScaler(enabled=config.train.fp16_run)
+
+    cache = []
+    progress_bar = tqdm.tqdm(range((total_epoch - epoch + 1) * len(train_loader)))
+    progress_bar.set_postfix(epoch=epoch)
+    step = -1 + len(train_loader) * (epoch - 1)
+    for epoch in range(epoch, total_epoch + 1):
+        train_loader.batch_sampler.set_epoch(epoch)
+
+        net_g.train()
+        net_d.train()
+
+        use_cache = len(cache) == len(train_loader)
+        data = cache if use_cache else enumerate(train_loader)
+
+        if is_main_process:
+            lr = optim_g.param_groups[0]["lr"]
+
+        if use_cache:
+            shuffle(cache)
+
+        for batch_idx, batch in data:
+            step += 1
+            progress_bar.update(1)
+            if f0:
+                (
+                    phone,
+                    phone_lengths,
+                    pitch,
+                    pitchf,
+                    spec,
+                    spec_lengths,
+                    wave,
+                    wave_lengths,
+                    sid,
+                ) = batch
+            else:
+                (
+                    phone,
+                    phone_lengths,
+                    spec,
+                    spec_lengths,
+                    wave,
+                    wave_lengths,
+                    sid,
+                ) = batch
+
+            if not use_cache:
+                phone, phone_lengths = (
+                    phone.to(device=device, non_blocking=True),
+                    phone_lengths.to(device=device, non_blocking=True),
+                )
+                if f0:
+                    pitch, pitchf = (
+                        pitch.to(device=device, non_blocking=True),
+                        pitchf.to(device=device, non_blocking=True),
+                    )
+                sid = sid.to(device=device, non_blocking=True)
+                spec, spec_lengths = (
+                    spec.to(device=device, non_blocking=True),
+                    spec_lengths.to(device=device, non_blocking=True),
+                )
+                wave, wave_lengths = (
+                    wave.to(device=device, non_blocking=True),
+                    wave_lengths.to(device=device, non_blocking=True),
+                )
+                if cache_in_gpu:
+                    if f0:
+                        cache.append(
+                            (
+                                batch_idx,
+                                (
+                                    phone,
+                                    phone_lengths,
+                                    pitch,
+                                    pitchf,
+                                    spec,
+                                    spec_lengths,
+                                    wave,
+                                    wave_lengths,
+                                    sid,
+                                ),
+                            )
+                        )
+                    else:
+                        cache.append(
+                            (
+                                batch_idx,
+                                (
+                                    phone,
+                                    phone_lengths,
+                                    spec,
+                                    spec_lengths,
+                                    wave,
+                                    wave_lengths,
+                                    sid,
+                                ),
+                            )
+                        )
+
+            with autocast(enabled=config.train.fp16_run):
+                if augment:
+                    with torch.no_grad():
+                        if type(augment_net_g) == SynthesizerTrnMs256NSFSid:
+                            new_phone, aug_wave = change_speaker(augment_net_g, augment_speaker_info, embedder, embedding_output_layer, phone, phone_lengths, pitch, pitchf, spec_lengths)
+                        else:
+                            new_phone, aug_wave = change_speaker_nono(augment_net_g, embedder, embedding_output_layer, phone, phone_lengths, spec_lengths)
+                        weight = np.power(.5, step / len(train_loader))  # 学習の初期はそのままのphone embeddingを使う
+                        phone = phone * weight + new_phone * (1. - weight)
+
+                if f0:
+                    (
+                        y_hat,
+                        ids_slice,
+                        x_mask,
+                        z_mask,
+                        (z, z_p, m_p, logs_p, m_q, logs_q),
+                    ) = net_g(
+                        phone, phone_lengths, pitch, pitchf, spec, spec_lengths, sid
+                    )
+                else:
+                    (
+                        y_hat,
+                        ids_slice,
+                        x_mask,
+                        z_mask,
+                        (z, z_p, m_p, logs_p, m_q, logs_q),
+                    ) = net_g(phone, phone_lengths, spec, spec_lengths, sid)
+                mel = spec_to_mel_torch(
+                    spec,
+                    config.data.filter_length,
+                    config.data.n_mel_channels,
+                    config.data.sampling_rate,
+                    config.data.mel_fmin,
+                    config.data.mel_fmax,
+                )
+                y_mel = commons.slice_segments(
+                    mel, ids_slice, config.train.segment_size // config.data.hop_length
+                )
+                with autocast(enabled=False):
+                    y_hat_mel = mel_spectrogram_torch(
+                        y_hat.float().squeeze(1),
+                        config.data.filter_length,
+                        config.data.n_mel_channels,
+                        config.data.sampling_rate,
+                        config.data.hop_length,
+                        config.data.win_length,
+                        config.data.mel_fmin,
+                        config.data.mel_fmax,
+                    )
+                if config.train.fp16_run == True and device != torch.device("mps"):
+                    y_hat_mel = y_hat_mel.half()
+                wave_slice = commons.slice_segments(
+                    wave, ids_slice * config.data.hop_length, config.train.segment_size
+                )  # slice
+
+                # Discriminator
+                y_d_hat_r, y_d_hat_g, _, _ = net_d(wave_slice, y_hat.detach())
+                with autocast(enabled=False):
+                    loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(
+                        y_d_hat_r, y_d_hat_g
+                    )
+            optim_d.zero_grad()
+            scaler.scale(loss_disc).backward()
+            scaler.unscale_(optim_d)
+            grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None)
+            scaler.step(optim_d)
+
+            with autocast(enabled=config.train.fp16_run):
+                # Generator
+                y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(wave_slice, y_hat)
+                with autocast(enabled=False):
+                    loss_mel = F.l1_loss(y_mel, y_hat_mel) * config.train.c_mel
+                    loss_kl = (
+                        kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * config.train.c_kl
+                    )
+                    loss_fm = feature_loss(fmap_r, fmap_g)
+                    loss_gen, losses_gen = generator_loss(y_d_hat_g)
+                    loss_gen_all = loss_gen + loss_fm + loss_mel + loss_kl
+            optim_g.zero_grad()
+            scaler.scale(loss_gen_all).backward()
+            scaler.unscale_(optim_g)
+            grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None)
+            scaler.step(optim_g)
+            scaler.update()
+
+            if is_main_process:
+                progress_bar.set_postfix(
+                    epoch=epoch,
+                    loss_g=float(loss_gen_all) if loss_gen_all is not None else 0.0,
+                    loss_d=float(loss_disc) if loss_disc is not None else 0.0,
+                    lr=float(lr) if lr is not None else 0.0,
+                    use_cache=use_cache,
+                )
+                if global_step % config.train.log_interval == 0:
+                    lr = optim_g.param_groups[0]["lr"]
+                    # Amor For Tensorboard display
+                    if loss_mel > 50:
+                        loss_mel = 50
+                    if loss_kl > 5:
+                        loss_kl = 5
+
+                    scalar_dict = {
+                        "loss/g/total": loss_gen_all,
+                        "loss/d/total": loss_disc,
+                        "learning_rate": lr,
+                        "grad_norm_d": grad_norm_d,
+                        "grad_norm_g": grad_norm_g,
+                    }
+                    scalar_dict.update(
+                        {
+                            "loss/g/fm": loss_fm,
+                            "loss/g/mel": loss_mel,
+                            "loss/g/kl": loss_kl,
+                        }
+                    )
+
+                    scalar_dict.update(
+                        {"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)}
+                    )
+                    scalar_dict.update(
+                        {
+                            "loss/d_r/{}".format(i): v
+                            for i, v in enumerate(losses_disc_r)
+                        }
+                    )
+                    scalar_dict.update(
+                        {
+                            "loss/d_g/{}".format(i): v
+                            for i, v in enumerate(losses_disc_g)
+                        }
+                    )
+                    image_dict = {
+                        "slice/mel_org": utils.plot_spectrogram_to_numpy(
+                            y_mel[0].data.cpu().numpy()
+                        ),
+                        "slice/mel_gen": utils.plot_spectrogram_to_numpy(
+                            y_hat_mel[0].data.cpu().numpy()
+                        ),
+                        "all/mel": utils.plot_spectrogram_to_numpy(
+                            mel[0].data.cpu().numpy()
+                        ),
+                    }
+                    utils.summarize(
+                        writer=writer,
+                        global_step=global_step,
+                        images=image_dict,
+                        scalars=scalar_dict,
+                    )
+            global_step += 1
+        if is_main_process and save_every_epoch != 0 and epoch % save_every_epoch == 0:
+            if save_only_last:
+                old_g_path = os.path.join(
+                    state_dir, f"G_{epoch - save_every_epoch}.pth"
+                )
+                old_d_path = os.path.join(
+                    state_dir, f"D_{epoch - save_every_epoch}.pth"
+                )
+                old_wav_path = os.path.join(
+                    state_dir, f"wav_sample_{epoch - save_every_epoch}"
+                )
+                if os.path.exists(old_g_path):
+                    os.remove(old_g_path)
+                if os.path.exists(old_d_path):
+                    os.remove(old_d_path)
+                if os.path.exists(old_wav_path):
+                    shutil.rmtree(old_wav_path)
+
+            if save_wav_with_checkpoint:
+                with autocast(enabled=config.train.fp16_run):
+                    with torch.no_grad():
+                        if f0:
+                            pred_wave = net_g.infer(phone, phone_lengths, pitch, pitchf, sid)[0]
+                        else:
+                            pred_wave = net_g.infer(phone, phone_lengths, sid)[0]
+                os.makedirs(os.path.join(state_dir, f"wav_sample_{epoch}"), exist_ok=True)
+                for i in range(pred_wave.shape[0]):
+                    torchaudio.save(filepath=os.path.join(state_dir, f"wav_sample_{epoch}", f"{i:02}_y_true.wav"), src=wave[i].detach().cpu().float(), sample_rate=int(sample_rate[:-1] + "000"))
+                    torchaudio.save(filepath=os.path.join(state_dir, f"wav_sample_{epoch}", f"{i:02}_y_pred.wav"), src=pred_wave[i].detach().cpu().float(), sample_rate=int(sample_rate[:-1] + "000"))
+                    if augment:
+                        torchaudio.save(filepath=os.path.join(state_dir, f"wav_sample_{epoch}", f"{i:02}_y_aug.wav"), src=aug_wave[i].detach().cpu().float(), sample_rate=int(sample_rate[:-1] + "000"))
+
+            utils.save_state(
+                net_g,
+                optim_g,
+                config.train.learning_rate,
+                epoch,
+                os.path.join(state_dir, f"G_{epoch}.pth"),
+            )
+            utils.save_state(
+                net_d,
+                optim_d,
+                config.train.learning_rate,
+                epoch,
+                os.path.join(state_dir, f"D_{epoch}.pth"),
+            )
+
+            save(
+                net_g,
+                config.version,
+                sample_rate,
+                f0,
+                embedder_name,
+                embedder_out_channels,
+                embedding_output_layer,
+                os.path.join(training_dir, "checkpoints", f"{model_name}-{epoch}.pth"),
+                epoch,
+                speaker_info
+            )
+
+        scheduler_g.step()
+        scheduler_d.step()
+
+    if is_main_process:
+        print("Training is done. The program is closed.")
+        save(
+            net_g,
+            config.version,
+            sample_rate,
+            f0,
+            embedder_name,
+            embedder_out_channels,
+            embedding_output_layer,
+            os.path.join(out_dir, f"{model_name}.pth"),
+            epoch,
+            speaker_info
+        )

lib/rvc/transforms.py

@@ -0,0 +1,207 @@
+import numpy as np
+import torch
+from torch.nn import functional as F
+
+DEFAULT_MIN_BIN_WIDTH = 1e-3
+DEFAULT_MIN_BIN_HEIGHT = 1e-3
+DEFAULT_MIN_DERIVATIVE = 1e-3
+
+
+def piecewise_rational_quadratic_transform(
+    inputs,
+    unnormalized_widths,
+    unnormalized_heights,
+    unnormalized_derivatives,
+    inverse=False,
+    tails=None,
+    tail_bound=1.0,
+    min_bin_width=DEFAULT_MIN_BIN_WIDTH,
+    min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
+    min_derivative=DEFAULT_MIN_DERIVATIVE,
+):
+    if tails is None:
+        spline_fn = rational_quadratic_spline
+        spline_kwargs = {}
+    else:
+        spline_fn = unconstrained_rational_quadratic_spline
+        spline_kwargs = {"tails": tails, "tail_bound": tail_bound}
+
+    outputs, logabsdet = spline_fn(
+        inputs=inputs,
+        unnormalized_widths=unnormalized_widths,
+        unnormalized_heights=unnormalized_heights,
+        unnormalized_derivatives=unnormalized_derivatives,
+        inverse=inverse,
+        min_bin_width=min_bin_width,
+        min_bin_height=min_bin_height,
+        min_derivative=min_derivative,
+        **spline_kwargs
+    )
+    return outputs, logabsdet
+
+
+def searchsorted(bin_locations, inputs, eps=1e-6):
+    bin_locations[..., -1] += eps
+    return torch.sum(inputs[..., None] >= bin_locations, dim=-1) - 1
+
+
+def unconstrained_rational_quadratic_spline(
+    inputs,
+    unnormalized_widths,
+    unnormalized_heights,
+    unnormalized_derivatives,
+    inverse=False,
+    tails="linear",
+    tail_bound=1.0,
+    min_bin_width=DEFAULT_MIN_BIN_WIDTH,
+    min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
+    min_derivative=DEFAULT_MIN_DERIVATIVE,
+):
+    inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound)
+    outside_interval_mask = ~inside_interval_mask
+
+    outputs = torch.zeros_like(inputs)
+    logabsdet = torch.zeros_like(inputs)
+
+    if tails == "linear":
+        unnormalized_derivatives = F.pad(unnormalized_derivatives, pad=(1, 1))
+        constant = np.log(np.exp(1 - min_derivative) - 1)
+        unnormalized_derivatives[..., 0] = constant
+        unnormalized_derivatives[..., -1] = constant
+
+        outputs[outside_interval_mask] = inputs[outside_interval_mask]
+        logabsdet[outside_interval_mask] = 0
+    else:
+        raise RuntimeError("{} tails are not implemented.".format(tails))
+
+    (
+        outputs[inside_interval_mask],
+        logabsdet[inside_interval_mask],
+    ) = rational_quadratic_spline(
+        inputs=inputs[inside_interval_mask],
+        unnormalized_widths=unnormalized_widths[inside_interval_mask, :],
+        unnormalized_heights=unnormalized_heights[inside_interval_mask, :],
+        unnormalized_derivatives=unnormalized_derivatives[inside_interval_mask, :],
+        inverse=inverse,
+        left=-tail_bound,
+        right=tail_bound,
+        bottom=-tail_bound,
+        top=tail_bound,
+        min_bin_width=min_bin_width,
+        min_bin_height=min_bin_height,
+        min_derivative=min_derivative,
+    )
+
+    return outputs, logabsdet
+
+
+def rational_quadratic_spline(
+    inputs,
+    unnormalized_widths,
+    unnormalized_heights,
+    unnormalized_derivatives,
+    inverse=False,
+    left=0.0,
+    right=1.0,
+    bottom=0.0,
+    top=1.0,
+    min_bin_width=DEFAULT_MIN_BIN_WIDTH,
+    min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
+    min_derivative=DEFAULT_MIN_DERIVATIVE,
+):
+    if torch.min(inputs) < left or torch.max(inputs) > right:
+        raise ValueError("Input to a transform is not within its domain")
+
+    num_bins = unnormalized_widths.shape[-1]
+
+    if min_bin_width * num_bins > 1.0:
+        raise ValueError("Minimal bin width too large for the number of bins")
+    if min_bin_height * num_bins > 1.0:
+        raise ValueError("Minimal bin height too large for the number of bins")
+
+    widths = F.softmax(unnormalized_widths, dim=-1)
+    widths = min_bin_width + (1 - min_bin_width * num_bins) * widths
+    cumwidths = torch.cumsum(widths, dim=-1)
+    cumwidths = F.pad(cumwidths, pad=(1, 0), mode="constant", value=0.0)
+    cumwidths = (right - left) * cumwidths + left
+    cumwidths[..., 0] = left
+    cumwidths[..., -1] = right
+    widths = cumwidths[..., 1:] - cumwidths[..., :-1]
+
+    derivatives = min_derivative + F.softplus(unnormalized_derivatives)
+
+    heights = F.softmax(unnormalized_heights, dim=-1)
+    heights = min_bin_height + (1 - min_bin_height * num_bins) * heights
+    cumheights = torch.cumsum(heights, dim=-1)
+    cumheights = F.pad(cumheights, pad=(1, 0), mode="constant", value=0.0)
+    cumheights = (top - bottom) * cumheights + bottom
+    cumheights[..., 0] = bottom
+    cumheights[..., -1] = top
+    heights = cumheights[..., 1:] - cumheights[..., :-1]
+
+    if inverse:
+        bin_idx = searchsorted(cumheights, inputs)[..., None]
+    else:
+        bin_idx = searchsorted(cumwidths, inputs)[..., None]
+
+    input_cumwidths = cumwidths.gather(-1, bin_idx)[..., 0]
+    input_bin_widths = widths.gather(-1, bin_idx)[..., 0]
+
+    input_cumheights = cumheights.gather(-1, bin_idx)[..., 0]
+    delta = heights / widths
+    input_delta = delta.gather(-1, bin_idx)[..., 0]
+
+    input_derivatives = derivatives.gather(-1, bin_idx)[..., 0]
+    input_derivatives_plus_one = derivatives[..., 1:].gather(-1, bin_idx)[..., 0]
+
+    input_heights = heights.gather(-1, bin_idx)[..., 0]
+
+    if inverse:
+        a = (inputs - input_cumheights) * (
+            input_derivatives + input_derivatives_plus_one - 2 * input_delta
+        ) + input_heights * (input_delta - input_derivatives)
+        b = input_heights * input_derivatives - (inputs - input_cumheights) * (
+            input_derivatives + input_derivatives_plus_one - 2 * input_delta
+        )
+        c = -input_delta * (inputs - input_cumheights)
+
+        discriminant = b.pow(2) - 4 * a * c
+        assert (discriminant >= 0).all()
+
+        root = (2 * c) / (-b - torch.sqrt(discriminant))
+        outputs = root * input_bin_widths + input_cumwidths
+
+        theta_one_minus_theta = root * (1 - root)
+        denominator = input_delta + (
+            (input_derivatives + input_derivatives_plus_one - 2 * input_delta)
+            * theta_one_minus_theta
+        )
+        derivative_numerator = input_delta.pow(2) * (
+            input_derivatives_plus_one * root.pow(2)
+            + 2 * input_delta * theta_one_minus_theta
+            + input_derivatives * (1 - root).pow(2)
+        )
+        logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator)
+
+        return outputs, -logabsdet
+    else:
+        theta = (inputs - input_cumwidths) / input_bin_widths
+        theta_one_minus_theta = theta * (1 - theta)
+
+        numerator = input_heights * (
+            input_delta * theta.pow(2) + input_derivatives * theta_one_minus_theta
+        )
+        denominator = input_delta + (
+            (input_derivatives + input_derivatives_plus_one - 2 * input_delta)
+            * theta_one_minus_theta
+        )
+        outputs = input_cumheights + numerator / denominator
+
+        derivative_numerator = input_delta.pow(2) * (
+            input_derivatives_plus_one * theta.pow(2)
+            + 2 * input_delta * theta_one_minus_theta
+            + input_derivatives * (1 - theta).pow(2)
+        )
+        logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator)
+
+        return outputs, logabsdet

lib/rvc/utils.py

@@ -0,0 +1,225 @@
+import glob
+import logging
+import os
+import shutil
+import socket
+import sys
+
+import ffmpeg
+import matplotlib
+import matplotlib.pylab as plt
+import numpy as np
+import torch
+from scipy.io.wavfile import read
+from torch.nn import functional as F
+
+from modules.shared import ROOT_DIR
+
+from .config import TrainConfig
+
+matplotlib.use("Agg")
+logging.getLogger("matplotlib").setLevel(logging.WARNING)
+
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+logger = logging
+
+
+def load_audio(file: str, sr):
+    try:
+        # https://github.com/openai/whisper/blob/main/whisper/audio.py#L26
+        # This launches a subprocess to decode audio while down-mixing and resampling as necessary.
+        # Requires the ffmpeg CLI and `ffmpeg-python` package to be installed.
+        file = (
+            file.strip(" ").strip('"').strip("\n").strip('"').strip(" ")
+        )  # Prevent small white copy path head and tail with spaces and " and return
+        out, _ = (
+            ffmpeg.input(file, threads=0)
+            .output("-", format="f32le", acodec="pcm_f32le", ac=1, ar=sr)
+            .run(cmd=["ffmpeg", "-nostdin"], capture_stdout=True, capture_stderr=True)
+        )
+    except Exception as e:
+        raise RuntimeError(f"Failed to load audio: {e}")
+
+    return np.frombuffer(out, np.float32).flatten()
+
+
+def find_empty_port():
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s.bind(("", 0))
+    s.listen(1)
+    port = s.getsockname()[1]
+    s.close()
+    return port
+
+
+def load_checkpoint(checkpoint_path, model, optimizer=None, load_opt=1):
+    assert os.path.isfile(checkpoint_path)
+    checkpoint_dict = torch.load(checkpoint_path, map_location="cpu")
+
+    saved_state_dict = checkpoint_dict["model"]
+    if hasattr(model, "module"):
+        state_dict = model.module.state_dict()
+    else:
+        state_dict = model.state_dict()
+    new_state_dict = {}
+    for k, v in state_dict.items():  # 模型需要的shape
+        try:
+            new_state_dict[k] = saved_state_dict[k]
+            if saved_state_dict[k].shape != state_dict[k].shape:
+                print(
+                    f"shape-{k}-mismatch|need-{state_dict[k].shape}|get-{saved_state_dict[k].shape}"
+                )
+                if saved_state_dict[k].dim() == 2:  # NOTE: check is this ok?
+                    # for embedded input 256 <==> 768
+                    # this achieves we can continue training from original's pretrained checkpoints when using embedder that 768-th dim output etc.
+                    if saved_state_dict[k].dtype == torch.half:
+                        new_state_dict[k] = (
+                            F.interpolate(
+                                saved_state_dict[k].float().unsqueeze(0).unsqueeze(0),
+                                size=state_dict[k].shape,
+                                mode="bilinear",
+                            )
+                            .half()
+                            .squeeze(0)
+                            .squeeze(0)
+                        )
+                    else:
+                        new_state_dict[k] = (
+                            F.interpolate(
+                                saved_state_dict[k].unsqueeze(0).unsqueeze(0),
+                                size=state_dict[k].shape,
+                                mode="bilinear",
+                            )
+                            .squeeze(0)
+                            .squeeze(0)
+                        )
+                    print(
+                        "interpolated new_state_dict",
+                        k,
+                        "from",
+                        saved_state_dict[k].shape,
+                        "to",
+                        new_state_dict[k].shape,
+                    )
+                else:
+                    raise KeyError
+        except Exception as e:
+            # print(traceback.format_exc())
+            print(f"{k} is not in the checkpoint")
+            print("error: %s" % e)
+            new_state_dict[k] = v  # 模型自带的随机值
+    if hasattr(model, "module"):
+        model.module.load_state_dict(new_state_dict, strict=False)
+    else:
+        model.load_state_dict(new_state_dict, strict=False)
+    print("Loaded model weights")
+
+    epoch = checkpoint_dict["epoch"]
+    learning_rate = checkpoint_dict["learning_rate"]
+    if optimizer is not None and load_opt == 1:
+        optimizer.load_state_dict(checkpoint_dict["optimizer"])
+    print("Loaded checkpoint '{}' (epoch {})".format(checkpoint_path, epoch))
+    return model, optimizer, learning_rate, epoch
+
+
+def save_state(model, optimizer, learning_rate, epoch, checkpoint_path):
+    print(
+        "Saving model and optimizer state at epoch {} to {}".format(
+            epoch, checkpoint_path
+        )
+    )
+    if hasattr(model, "module"):
+        state_dict = model.module.state_dict()
+    else:
+        state_dict = model.state_dict()
+    torch.save(
+        {
+            "model": state_dict,
+            "epoch": epoch,
+            "optimizer": optimizer.state_dict(),
+            "learning_rate": learning_rate,
+        },
+        checkpoint_path,
+    )
+
+
+def summarize(
+    writer,
+    global_step,
+    scalars={},
+    histograms={},
+    images={},
+    audios={},
+    audio_sampling_rate=22050,
+):
+    for k, v in scalars.items():
+        writer.add_scalar(k, v, global_step)
+    for k, v in histograms.items():
+        writer.add_histogram(k, v, global_step)
+    for k, v in images.items():
+        writer.add_image(k, v, global_step, dataformats="HWC")
+    for k, v in audios.items():
+        writer.add_audio(k, v, global_step, audio_sampling_rate)
+
+
+def latest_checkpoint_path(dir_path, regex="G_*.pth"):
+    filelist = glob.glob(os.path.join(dir_path, regex))
+    if len(filelist) == 0:
+        return None
+    filelist.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
+    filepath = filelist[-1]
+    return filepath
+
+
+def plot_spectrogram_to_numpy(spectrogram):
+    fig, ax = plt.subplots(figsize=(10, 2))
+    im = ax.imshow(spectrogram, aspect="auto", origin="lower", interpolation="none")
+    plt.colorbar(im, ax=ax)
+    plt.xlabel("Frames")
+    plt.ylabel("Channels")
+    plt.tight_layout()
+
+    fig.canvas.draw()
+    data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="")
+    data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
+    plt.close()
+    return data
+
+
+def plot_alignment_to_numpy(alignment, info=None):
+    fig, ax = plt.subplots(figsize=(6, 4))
+    im = ax.imshow(
+        alignment.transpose(), aspect="auto", origin="lower", interpolation="none"
+    )
+    fig.colorbar(im, ax=ax)
+    xlabel = "Decoder timestep"
+    if info is not None:
+        xlabel += "\n\n" + info
+    plt.xlabel(xlabel)
+    plt.ylabel("Encoder timestep")
+    plt.tight_layout()
+
+    fig.canvas.draw()
+    data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep="")
+    data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
+    plt.close()
+    return data
+
+
+def load_wav_to_torch(full_path):
+    sampling_rate, data = read(full_path)
+    return torch.FloatTensor(data.astype(np.float32)), sampling_rate
+
+
+def load_config(training_dir: str, sample_rate: int, emb_channels: int):
+    if emb_channels == 256:
+        config_path = os.path.join(ROOT_DIR, "configs", f"{sample_rate}.json")
+    else:
+        config_path = os.path.join(
+            ROOT_DIR, "configs", f"{sample_rate}-{emb_channels}.json"
+        )
+    config_save_path = os.path.join(training_dir, "config.json")
+
+    shutil.copyfile(config_path, config_save_path)
+
+    return TrainConfig.parse_file(config_save_path)

models/checkpoints/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

models/embeddings/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

models/pretrained/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

models/training/.gitignore

@@ -0,0 +1,6 @@
+*/**
+
+!mute/**/*
+!.gitignore
+
+mute/**/*.pt

models/training/models/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

models/training/mute/0_gt_wavs/mute32k.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9edcf85ec77e88bd01edf3d887bdc418d3596d573f7ad2694da546f41dae6baf
+size 192078

models/training/mute/0_gt_wavs/mute40k.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67a816e77b50cb9f016e49e5c01f07e080c4e3b82b7a8ac3e64bcb143f90f31b
+size 240078

models/training/mute/0_gt_wavs/mute48k.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f2bb4daaa106e351aebb001e5a25de985c0b472f22e8d60676bc924a79056ee
+size 288078

models/training/mute/1_16k_wavs/mute.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e233e86ba1be365e1133f157d56b61110086b89650ecfbdfc013c759e466250
+size 96078

models/training/mute/2a_f0/mute.wav.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b9acf9ab7facdb032e1d687fe35182670b0b94566c4b209ae48c239d19956a6
+size 1332

models/training/mute/2b_f0nsf/mute.wav.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30792849c8e72d67e6691754077f2888b101cb741e9c7f193c91dd9692870c87
+size 2536

models/training/mute/3_feature256/mute.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64d5abbac078e19a3f649c0d78a02cb33a71407ded3ddf2db78e6b803d0c0126
+size 152704

modules/cmd_opts.py

@@ -0,0 +1,22 @@
+import argparse
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--host", help="Host to connect to", type=str, default="127.0.0.1")
+parser.add_argument("--port", help="Port to connect to", type=int)
+parser.add_argument("--share", help="Enable gradio share", action="store_true")
+parser.add_argument(
+    "--models-dir", help="Path to models directory", type=str, default=None
+)
+parser.add_argument(
+    "--output-dir", help="Path to output directory", type=str, default=None
+)
+parser.add_argument(
+    "--precision",
+    help="Precision to use",
+    type=str,
+    default="fp16",
+    choices=["fp32", "fp16"],
+)
+
+opts, _ = parser.parse_known_args()

modules/core.py

@@ -0,0 +1,156 @@
+import hashlib
+import os
+import shutil
+import sys
+from concurrent.futures import ThreadPoolExecutor
+
+import requests
+
+from modules.models import MODELS_DIR
+from modules.shared import ROOT_DIR
+from modules.utils import download_file
+
+
+def get_hf_etag(url: str):
+    r = requests.head(url)
+
+    etag = r.headers["X-Linked-ETag"] if "X-Linked-ETag" in r.headers else ""
+
+    if etag.startswith('"') and etag.endswith('"'):
+        etag = etag[1:-1]
+
+    return etag
+
+
+def calc_sha256(filepath: str):
+    sha256 = hashlib.sha256()
+    with open(filepath, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            sha256.update(chunk)
+    return sha256.hexdigest()
+
+
+def download_models():
+    def hash_check(url: str, out: str):
+        if not os.path.exists(out):
+            return False
+        etag = get_hf_etag(url)
+        hash = calc_sha256(out)
+        return etag == hash
+
+    os.makedirs(os.path.join(MODELS_DIR, "pretrained", "v2"), exist_ok=True)
+
+    tasks = []
+    for template in [
+        "D{}k",
+        "G{}k",
+        "f0D{}k",
+        "f0G{}k",
+    ]:
+        basename = template.format("40")
+        url = f"https://huggingface.co/ddPn08/rvc-webui-models/resolve/main/pretrained/v2/{basename}.pth"
+        out = os.path.join(MODELS_DIR, "pretrained", "v2", f"{basename}.pth")
+
+        if hash_check(url, out):
+            continue
+
+        tasks.append((url, out))
+
+    for filename in [
+        "checkpoint_best_legacy_500.pt",
+    ]:
+        out = os.path.join(MODELS_DIR, "embeddings", filename)
+        url = f"https://huggingface.co/ddPn08/rvc-webui-models/resolve/main/embeddings/{filename}"
+
+        if hash_check(url, out):
+            continue
+
+        tasks.append(
+            (
+                f"https://huggingface.co/ddPn08/rvc-webui-models/resolve/main/embeddings/{filename}",
+                out,
+            )
+        )
+
+    # japanese-hubert-base (Fairseq)
+    # from official repo
+    # NOTE: change filename?
+    hubert_jp_url = f"https://huggingface.co/rinna/japanese-hubert-base/resolve/main/fairseq/model.pt"
+    out = os.path.join(MODELS_DIR, "embeddings", "rinna_hubert_base_jp.pt")
+    if not hash_check(hubert_jp_url, out):
+        tasks.append(
+            (
+                hubert_jp_url,
+                out,
+            )
+        )
+
+    if len(tasks) < 1:
+        return
+
+    with ThreadPoolExecutor() as pool:
+        pool.map(
+            download_file,
+            *zip(
+                *[(filename, out, i, True) for i, (filename, out) in enumerate(tasks)]
+            ),
+        )
+
+
+def install_ffmpeg():
+    if os.path.exists(os.path.join(ROOT_DIR, "bin", "ffmpeg.exe")):
+        return
+    tmpdir = os.path.join(ROOT_DIR, "tmp")
+    url = (
+        "https://www.gyan.dev/ffmpeg/builds/packages/ffmpeg-5.1.2-essentials_build.zip"
+    )
+    out = os.path.join(tmpdir, "ffmpeg.zip")
+    os.makedirs(os.path.dirname(out), exist_ok=True)
+    download_file(url, out)
+    shutil.unpack_archive(out, os.path.join(tmpdir, "ffmpeg"))
+    shutil.copyfile(
+        os.path.join(
+            tmpdir, "ffmpeg", "ffmpeg-5.1.2-essentials_build", "bin", "ffmpeg.exe"
+        ),
+        os.path.join(ROOT_DIR, "bin", "ffmpeg.exe"),
+    )
+    os.remove(os.path.join(tmpdir, "ffmpeg.zip"))
+    shutil.rmtree(os.path.join(tmpdir, "ffmpeg"))
+
+
+def update_modelnames():
+    for sr in ["32k", "40k", "48k"]:
+        files = [
+            f"f0G{sr}",
+            f"f0D{sr}",
+            f"G{sr}",
+            f"D{sr}",
+        ]
+        for file in files:
+            filepath = os.path.join(MODELS_DIR, "pretrained", f"{file}.pth")
+            if os.path.exists(filepath):
+                os.rename(
+                    filepath,
+                    os.path.join(MODELS_DIR, "pretrained", f"{file}256.pth"),
+                )
+
+    if not os.path.exists(os.path.join(MODELS_DIR, "embeddings")):
+        os.makedirs(os.path.join(MODELS_DIR, "embeddings"))
+
+    if os.path.exists(os.path.join(MODELS_DIR, "hubert_base.pt")):
+        os.rename(
+            os.path.join(MODELS_DIR, "hubert_base.pt"),
+            os.path.join(MODELS_DIR, "embeddings", "hubert_base.pt"),
+        )
+    if os.path.exists(os.path.join(MODELS_DIR, "checkpoint_best_legacy_500.pt")):
+        os.rename(
+            os.path.join(MODELS_DIR, "checkpoint_best_legacy_500.pt"),
+            os.path.join(MODELS_DIR, "embeddings", "checkpoint_best_legacy_500.pt"),
+        )
+
+
+def preload():
+    update_modelnames()
+    download_models()
+    if sys.platform == "win32":
+        install_ffmpeg()

modules/merge.py

@@ -0,0 +1,81 @@
+from collections import OrderedDict
+from typing import *
+
+import torch
+import tqdm
+
+
+def merge(
+    path_a: str,
+    path_b: str,
+    path_c: str,
+    alpha: float,
+    weights: Dict[str, float],
+    method: str,
+):
+    def extract(ckpt: Dict[str, Any]):
+        a = ckpt["model"]
+        opt = OrderedDict()
+        opt["weight"] = {}
+        for key in a.keys():
+            if "enc_q" in key:
+                continue
+            opt["weight"][key] = a[key]
+        return opt
+
+    def load_weight(path: str):
+        print(f"Loading {path}...")
+        state_dict = torch.load(path, map_location="cpu")
+        if "model" in state_dict:
+            weight = extract(state_dict)
+        else:
+            weight = state_dict["weight"]
+        return weight, state_dict
+
+    def get_alpha(key: str):
+        try:
+            filtered = sorted(
+                [x for x in weights.keys() if key.startswith(x)], key=len, reverse=True
+            )
+            if len(filtered) < 1:
+                return alpha
+            return weights[filtered[0]]
+        except:
+            return alpha
+
+    weight_a, state_dict = load_weight(path_a)
+    weight_b, _ = load_weight(path_b)
+    if path_c is not None:
+        weight_c, _ = load_weight(path_c)
+
+    if sorted(list(weight_a.keys())) != sorted(list(weight_b.keys())):
+        raise RuntimeError("Failed to merge models.")
+
+    merged = OrderedDict()
+    merged["weight"] = {}
+
+    def merge_weight(a, b, c, alpha):
+        if method == "weight_sum":
+            return (1 - alpha) * a + alpha * b
+        elif method == "add_diff":
+            return a + (b - c) * alpha
+
+    for key in tqdm.tqdm(weight_a.keys()):
+        a = get_alpha(key)
+        if path_c is not None:
+            merged["weight"][key] = merge_weight(
+                weight_a[key], weight_b[key], weight_c[key], a
+            )
+        else:
+            merged["weight"][key] = merge_weight(weight_a[key], weight_b[key], None, a)
+    merged["config"] = state_dict["config"]
+    merged["params"] = state_dict["params"] if "params" in state_dict else None
+    merged["version"] = state_dict.get("version", "v1")
+    merged["sr"] = state_dict["sr"]
+    merged["f0"] = state_dict["f0"]
+    merged["info"] = state_dict["info"]
+    merged["embedder_name"] = (
+        state_dict["embedder_name"] if "embedder_name" in state_dict else None
+    )
+    merged["embedder_output_layer"] = state_dict.get("embedder_output_layer", "12")
+    return merged

modules/models.py

@@ -0,0 +1,266 @@
+import os
+import re
+from typing import *
+
+import torch
+from fairseq import checkpoint_utils
+from fairseq.models.hubert.hubert import HubertModel
+from pydub import AudioSegment
+
+from lib.rvc.models import (SynthesizerTrnMs256NSFSid,
+                            SynthesizerTrnMs256NSFSidNono)
+from lib.rvc.pipeline import VocalConvertPipeline
+
+from .cmd_opts import opts
+from .shared import ROOT_DIR, device, is_half
+from .utils import load_audio
+
+AUDIO_OUT_DIR = opts.output_dir or os.path.join(ROOT_DIR, "outputs")
+
+
+EMBEDDINGS_LIST = {
+    "hubert-base-japanese": (
+        "rinna_hubert_base_jp.pt",
+        "hubert-base-japanese",
+        "local",
+    ),
+    "contentvec": ("checkpoint_best_legacy_500.pt", "contentvec", "local"),
+}
+
+
+def update_state_dict(state_dict):
+    if "params" in state_dict and state_dict["params"] is not None:
+        return
+    keys = [
+        "spec_channels",
+        "segment_size",
+        "inter_channels",
+        "hidden_channels",
+        "filter_channels",
+        "n_heads",
+        "n_layers",
+        "kernel_size",
+        "p_dropout",
+        "resblock",
+        "resblock_kernel_sizes",
+        "resblock_dilation_sizes",
+        "upsample_rates",
+        "upsample_initial_channel",
+        "upsample_kernel_sizes",
+        "spk_embed_dim",
+        "gin_channels",
+        "emb_channels",
+        "sr",
+    ]
+    state_dict["params"] = {}
+    n = 0
+    for i, key in enumerate(keys):
+        i = i - n
+        if len(state_dict["config"]) != 19 and key == "emb_channels":
+            # backward compat.
+            n += 1
+            continue
+        state_dict["params"][key] = state_dict["config"][i]
+
+    if not "emb_channels" in state_dict["params"]:
+        if state_dict.get("version", "v1") == "v1":
+            state_dict["params"]["emb_channels"] = 256  # for backward compat.
+            state_dict["embedder_output_layer"] = 9
+        else:
+            state_dict["params"]["emb_channels"] = 768  # for backward compat.
+            state_dict["embedder_output_layer"] = 12
+
+
+class VoiceConvertModel:
+    def __init__(self, model_name: str, state_dict: Dict[str, Any]) -> None:
+        update_state_dict(state_dict)
+        self.model_name = model_name
+        self.state_dict = state_dict
+        self.tgt_sr = state_dict["params"]["sr"]
+        f0 = state_dict.get("f0", 1)
+        state_dict["params"]["spk_embed_dim"] = state_dict["weight"][
+            "emb_g.weight"
+        ].shape[0]
+        if not "emb_channels" in state_dict["params"]:
+            state_dict["params"]["emb_channels"] = 256  # for backward compat.
+
+        if f0 == 1:
+            self.net_g = SynthesizerTrnMs256NSFSid(
+                **state_dict["params"], is_half=is_half
+            )
+        else:
+            self.net_g = SynthesizerTrnMs256NSFSidNono(**state_dict["params"])
+
+        del self.net_g.enc_q
+
+        self.net_g.load_state_dict(state_dict["weight"], strict=False)
+        self.net_g.eval().to(device)
+
+        if is_half:
+            self.net_g = self.net_g.half()
+        else:
+            self.net_g = self.net_g.float()
+
+        self.vc = VocalConvertPipeline(self.tgt_sr, device, is_half)
+        self.n_spk = state_dict["params"]["spk_embed_dim"]
+
+    def single(
+        self,
+        sid: int,
+        input_audio: str,
+        embedder_model_name: str,
+        embedding_output_layer: str,
+        f0_up_key: int,
+        f0_file: str,
+        f0_method: str,
+        auto_load_index: bool,
+        faiss_index_file: str,
+        index_rate: float,
+        output_dir: str = AUDIO_OUT_DIR,
+    ):
+        if not input_audio:
+            raise Exception("You need to set Source Audio")
+        f0_up_key = int(f0_up_key)
+        audio = load_audio(input_audio, 16000)
+
+        if embedder_model_name == "auto":
+            embedder_model_name = (
+                self.state_dict["embedder_name"]
+                if "embedder_name" in self.state_dict
+                else "hubert_base"
+            )
+            if embedder_model_name.endswith("768"):
+                embedder_model_name = embedder_model_name[:-3]
+
+        if embedder_model_name == "hubert_base":
+            embedder_model_name = "contentvec"
+
+        if not embedder_model_name in EMBEDDINGS_LIST.keys():
+            raise Exception(f"Not supported embedder: {embedder_model_name}")
+
+        if (
+            embedder_model == None
+            or loaded_embedder_model != EMBEDDINGS_LIST[embedder_model_name][1]
+        ):
+            print(f"load {embedder_model_name} embedder")
+            embedder_filename, embedder_name, load_from = get_embedder(
+                embedder_model_name
+            )
+            load_embedder(embedder_filename, embedder_name)
+
+        if embedding_output_layer == "auto":
+            embedding_output_layer = (
+                self.state_dict["embedding_output_layer"]
+                if "embedding_output_layer" in self.state_dict
+                else 12
+            )
+        else:
+            embedding_output_layer = int(embedding_output_layer)
+
+        f0 = self.state_dict.get("f0", 1)
+
+        if not faiss_index_file and auto_load_index:
+            faiss_index_file = self.get_index_path(sid)
+
+        audio_opt = self.vc(
+            embedder_model,
+            embedding_output_layer,
+            self.net_g,
+            sid,
+            audio,
+            f0_up_key,
+            f0_method,
+            faiss_index_file,
+            index_rate,
+            f0,
+            f0_file=f0_file,
+        )
+
+        audio = AudioSegment(
+            audio_opt,
+            frame_rate=self.tgt_sr,
+            sample_width=2,
+            channels=1,
+        )
+        os.makedirs(output_dir, exist_ok=True)
+        input_audio_splitext = os.path.splitext(os.path.basename(input_audio))[0]
+        model_splitext = os.path.splitext(self.model_name)[0]
+        index = 0
+        existing_files = os.listdir(output_dir)
+        for existing_file in existing_files:
+            result = re.match(r"\d+", existing_file)
+            if result:
+                prefix_num = int(result.group(0))
+                if index < prefix_num:
+                    index = prefix_num
+        audio.export(
+            os.path.join(
+                output_dir, f"{index+1}-{model_splitext}-{input_audio_splitext}.wav"
+            ),
+            format="wav",
+        )
+        return audio_opt
+
+    def get_index_path(self, speaker_id: int):
+        basename = os.path.splitext(self.model_name)[0]
+        speaker_index_path = os.path.join(
+            MODELS_DIR,
+            "checkpoints",
+            f"{basename}_index",
+            f"{basename}.{speaker_id}.index",
+        )
+        if os.path.exists(speaker_index_path):
+            return speaker_index_path
+        return os.path.join(MODELS_DIR, "checkpoints", f"{basename}.index")
+
+
+MODELS_DIR = opts.models_dir or os.path.join(ROOT_DIR, "models")
+vc_model: Optional[VoiceConvertModel] = None
+embedder_model: Optional[HubertModel] = None
+loaded_embedder_model = ""
+
+
+def get_models():
+    dir = os.path.join(ROOT_DIR, "models", "checkpoints")
+    os.makedirs(dir, exist_ok=True)
+    return [
+        file
+        for file in os.listdir(dir)
+        if any([x for x in [".ckpt", ".pth"] if file.endswith(x)])
+    ]
+
+
+def get_embedder(embedder_name):
+    if embedder_name in EMBEDDINGS_LIST:
+        return EMBEDDINGS_LIST[embedder_name]
+    return None
+
+
+def load_embedder(emb_file: str, emb_name: str):
+    global embedder_model, loaded_embedder_model
+    emb_file = os.path.join(MODELS_DIR, "embeddings", emb_file)
+    models, _, _ = checkpoint_utils.load_model_ensemble_and_task(
+        [emb_file],
+        suffix="",
+    )
+    embedder_model = models[0]
+    embedder_model = embedder_model.to(device)
+
+    if is_half:
+        embedder_model = embedder_model.half()
+    else:
+        embedder_model = embedder_model.float()
+    embedder_model.eval()
+
+    loaded_embedder_model = emb_name
+
+
+def get_vc_model(model_name: str):
+    model_path = os.path.join(MODELS_DIR, "checkpoints", model_name)
+    weight = torch.load(model_path, map_location="cpu")
+    return VoiceConvertModel(model_name, weight)
+
+
+def load_model(model_name: str):
+    global vc_model
+    vc_model = get_vc_model(model_name)

modules/separate.py

@@ -0,0 +1,82 @@
+import os
+from typing import *
+
+import tqdm
+from pydub import AudioSegment
+from pydub.silence import split_on_silence
+
+
+def separate_audio(
+    input: str,
+    output: str,
+    silence_thresh: int,
+    min_silence_len: int = 1000,
+    keep_silence: int = 100,
+    margin: int = 0,
+    padding: bool = False,
+    min: Optional[int] = None,
+    max: Optional[int] = None,
+):
+    if os.path.isfile(input):
+        input = [input]
+    elif os.path.isdir(input):
+        input = [os.path.join(input, f) for f in os.listdir(input)]
+    else:
+        raise ValueError("input must be a file or directory")
+
+    os.makedirs(output, exist_ok=True)
+
+    for file in input:
+        if os.path.splitext(file)[1] == ".mp3":
+            audio = AudioSegment.from_mp3(file)
+        elif os.path.splitext(file)[1] == ".wav":
+            audio = AudioSegment.from_wav(file)
+        elif os.path.splitext(file)[1] == ".flac":
+            audio = AudioSegment.from_file(file, "flac")
+        else:
+            raise ValueError(
+                "Invalid file format. Only MP3 and WAV files are supported."
+            )
+
+        chunks = split_on_silence(
+            audio,
+            min_silence_len=min_silence_len,
+            silence_thresh=silence_thresh,
+            keep_silence=keep_silence,
+        )
+
+        output_chunks: List[AudioSegment] = []
+
+        so_short = None
+
+        for chunk in tqdm.tqdm(chunks):
+            if so_short is not None:
+                chunk = so_short + chunk
+                so_short = None
+            if min is None or len(chunk) > min:
+                if max is not None and len(chunk) > max:
+                    sub_chunks = [
+                        chunk[i : i + max + margin]
+                        for i in range(0, len(chunk) - margin, max)
+                    ]
+
+                    if len(sub_chunks[-1]) < min:
+                        if padding and len(sub_chunks) > 2:
+                            output_chunks.extend(sub_chunks[0:-2])
+                            output_chunks.append(sub_chunks[-2] + sub_chunks[-1])
+                        else:
+                            output_chunks.extend(sub_chunks[0:-1])
+                    else:
+                        output_chunks.extend(sub_chunks)
+                else:
+                    output_chunks.append(chunk)
+            else:
+                if so_short is None:
+                    so_short = chunk
+                else:
+                    so_short += chunk
+        basename = os.path.splitext(os.path.basename(file))[0]
+
+        for i, chunk in enumerate(output_chunks):
+            filepath = os.path.join(output, f"{basename}_{i}.wav")
+            chunk.export(filepath, format="wav")

modules/server/model.py

@@ -0,0 +1,451 @@
+import os
+import re
+from typing import *
+
+import faiss
+import numpy as np
+import pyworld
+import scipy.signal as signal
+import torch
+import torch.nn.functional as F
+import torchaudio
+import torchcrepe
+from fairseq import checkpoint_utils
+from fairseq.models.hubert.hubert import HubertModel
+from pydub import AudioSegment
+from torch import Tensor
+
+from lib.rvc.models import (SynthesizerTrnMs256NSFSid,
+                            SynthesizerTrnMs256NSFSidNono)
+from lib.rvc.pipeline import VocalConvertPipeline
+from modules.cmd_opts import opts
+from modules.models import (EMBEDDINGS_LIST, MODELS_DIR, get_embedder,
+                            get_vc_model, update_state_dict)
+from modules.shared import ROOT_DIR, device, is_half
+
+MODELS_DIR = opts.models_dir or os.path.join(ROOT_DIR, "models")
+vc_model: Optional["VoiceServerModel"] = None
+embedder_model: Optional[HubertModel] = None
+loaded_embedder_model = ""
+
+
+class VoiceServerModel:
+    def __init__(self, rvc_model_file: str, faiss_index_file: str) -> None:
+        # setting vram
+        global device, is_half
+        if isinstance(device, str):
+            device = torch.device(device)
+        if device.type == "cuda":
+            vram = torch.cuda.get_device_properties(device).total_memory / 1024**3
+        else:
+            vram = None
+        if vram is not None and vram <= 4:
+            self.x_pad = 1
+            self.x_query = 5
+            self.x_center = 30
+            self.x_max = 32
+        elif vram is not None and vram <= 5:
+            self.x_pad = 1
+            self.x_query = 6
+            self.x_center = 38
+            self.x_max = 41
+        else:
+            self.x_pad = 3
+            self.x_query = 10
+            self.x_center = 60
+            self.x_max = 65
+
+        # load_model
+        state_dict = torch.load(rvc_model_file, map_location="cpu")
+        update_state_dict(state_dict)
+        self.state_dict = state_dict
+        self.tgt_sr = state_dict["params"]["sr"]
+        self.f0 = state_dict.get("f0", 1)
+        state_dict["params"]["spk_embed_dim"] = state_dict["weight"][
+            "emb_g.weight"
+        ].shape[0]
+        if not "emb_channels" in state_dict["params"]:
+            if state_dict.get("version", "v1") == "v1":
+                state_dict["params"]["emb_channels"] = 256  # for backward compat.
+                state_dict["embedder_output_layer"] = 9
+            else:
+                state_dict["params"]["emb_channels"] = 768  # for backward compat.
+                state_dict["embedder_output_layer"] = 12
+        if self.f0 == 1:
+            self.net_g = SynthesizerTrnMs256NSFSid(
+                **state_dict["params"], is_half=is_half
+            )
+        else:
+            self.net_g = SynthesizerTrnMs256NSFSidNono(**state_dict["params"])
+        del self.net_g.enc_q
+        self.net_g.load_state_dict(state_dict["weight"], strict=False)
+        self.net_g.eval().to(device)
+        if is_half:
+            self.net_g = self.net_g.half()
+        else:
+            self.net_g = self.net_g.float()
+
+        emb_name = state_dict.get("embedder_name", "contentvec")
+        if emb_name == "hubert_base":
+            emb_name = "contentvec"
+        emb_file = os.path.join(MODELS_DIR, "embeddings", EMBEDDINGS_LIST[emb_name][0])
+        models, _, _ = checkpoint_utils.load_model_ensemble_and_task(
+            [emb_file],
+            suffix="",
+        )
+        embedder_model = models[0]
+        embedder_model = embedder_model.to(device)
+
+        if is_half:
+            embedder_model = embedder_model.half()
+        else:
+            embedder_model = embedder_model.float()
+        embedder_model.eval()
+        self.embedder_model = embedder_model
+
+        self.embedder_output_layer = state_dict["embedder_output_layer"]
+
+        self.index = None
+        if faiss_index_file != "" and os.path.exists(faiss_index_file):
+            self.index = faiss.read_index(faiss_index_file)
+            self.big_npy = self.index.reconstruct_n(0, self.index.ntotal)
+
+        self.n_spk = state_dict["params"]["spk_embed_dim"]
+
+        self.sr = 16000  # hubert input sample rate
+        self.window = 160  # hubert input window
+        self.t_pad = self.sr * self.x_pad  # padding time for each utterance
+        self.t_pad_tgt = self.tgt_sr * self.x_pad
+        self.t_pad2 = self.t_pad * 2
+        self.t_query = self.sr * self.x_query  # query time before and after query point
+        self.t_center = self.sr * self.x_center  # query cut point position
+        self.t_max = self.sr * self.x_max  # max time for no query
+        self.device = device
+        self.is_half = is_half
+
+    def __call__(
+        self,
+        audio: np.ndarray,
+        sr: int,
+        sid: int,
+        transpose: int,
+        f0_method: str,
+        index_rate: float,
+    ):
+        # bh, ah = signal.butter(N=5, Wn=48, btype="high", fs=16000)
+        # audio = signal.filtfilt(bh, ah, audio)
+        if sr != self.sr:
+            audio = torchaudio.functional.resample(torch.from_numpy(audio), sr, self.sr, rolloff=0.99).detach().cpu().numpy()
+        audio_pad = np.pad(audio, (self.window // 2, self.window // 2), mode="reflect" if audio.shape[0] > self.window // 2 else "constant")
+
+        opt_ts = []
+        if audio_pad.shape[0] > self.t_max:
+            audio_sum = np.zeros_like(audio)
+            for i in range(self.window):
+                audio_sum += audio_pad[i : i - self.window]
+            for t in range(self.t_center, audio.shape[0], self.t_center):
+                opt_ts.append(
+                    t
+                    - self.t_query
+                    + np.where(
+                        np.abs(audio_sum[t - self.t_query : t + self.t_query])
+                        == np.abs(audio_sum[t - self.t_query : t + self.t_query]).min()
+                    )[0][0]
+                )
+        audio_pad = np.pad(audio, (self.t_pad, self.t_pad), mode="reflect" if audio.shape[0] > self.t_pad else "constant")
+        p_len = audio_pad.shape[0] // self.window
+
+        sid = torch.tensor(sid, device=self.device).unsqueeze(0).long()
+        pitch, pitchf = None, None
+        if self.f0 == 1:
+            pitch, pitchf = get_f0(audio_pad, self.sr, p_len, transpose, f0_method)
+            pitch = pitch[:p_len]
+            pitchf = pitchf[:p_len]
+            if self.device.type == "mps":
+                pitchf = pitchf.astype(np.float32)
+            pitch = torch.tensor(pitch, device=self.device).unsqueeze(0).long()
+            pitchf = torch.tensor(pitchf, device=self.device).unsqueeze(0).float()
+
+        audio_opt = []
+
+        s = 0
+        t = None
+
+        for t in opt_ts:
+            t = t // self.window * self.window
+            if self.f0 == 1:
+                audio_opt.append(
+                    self._convert(
+                        sid,
+                        audio_pad[s : t + self.t_pad2 + self.window],
+                        pitch[:, s // self.window : (t + self.t_pad2) // self.window],
+                        pitchf[:, s // self.window : (t + self.t_pad2) // self.window],
+                        index_rate,
+                    )[self.t_pad_tgt : -self.t_pad_tgt]
+                )
+            else:
+                audio_opt.append(
+                    self._convert(
+                        sid,
+                        audio_pad[s : t + self.t_pad2 + self.window],
+                        None,
+                        None,
+                        index_rate,
+                    )[self.t_pad_tgt : -self.t_pad_tgt]
+                )
+            s = t
+        if self.f0 == 1:
+            audio_opt.append(
+                self._convert(
+                    sid,
+                    audio_pad[t:],
+                    pitch[:, t // self.window :] if t is not None else pitch,
+                    pitchf[:, t // self.window :] if t is not None else pitchf,
+                    index_rate,
+                )[self.t_pad_tgt : -self.t_pad_tgt]
+            )
+        else:
+            audio_opt.append(
+                self._convert(
+                    sid,
+                    audio_pad[t:],
+                    None,
+                    None,
+                    index_rate,
+                )[self.t_pad_tgt : -self.t_pad_tgt]
+            )
+        audio_opt = np.concatenate(audio_opt)
+        del pitch, pitchf, sid
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        return audio_opt
+
+
+    def _convert(
+        self,
+        sid: int,
+        audio: np.ndarray,
+        pitch: Optional[np.ndarray],
+        pitchf: Optional[np.ndarray],
+        index_rate: float,
+    ):
+        feats = torch.from_numpy(audio)
+        if self.is_half:
+            feats = feats.half()
+        else:
+            feats = feats.float()
+        if feats.dim() == 2:  # double channels
+            feats = feats.mean(-1)
+        assert feats.dim() == 1, feats.dim()
+        feats = feats.view(1, -1)
+        padding_mask = torch.BoolTensor(feats.shape).to(self.device).fill_(False)
+
+        half_support = (
+            self.device.type == "cuda"
+            and torch.cuda.get_device_capability(self.device)[0] >= 5.3
+        )
+        is_feats_dim_768 = self.net_g.emb_channels == 768
+
+        if isinstance(self.embedder_model, tuple):
+            feats = self.embedder_model[0](
+                feats.squeeze(0).squeeze(0).to(self.device),
+                return_tensors="pt",
+                sampling_rate=16000,
+            )
+            if self.is_half:
+                feats = feats.input_values.to(self.device).half()
+            else:
+                feats = feats.input_values.to(self.device)
+            with torch.no_grad():
+                if is_feats_dim_768:
+                    feats = self.embedder_model[1](feats).last_hidden_state
+                else:
+                    feats = self.embedder_model[1](feats).extract_features
+        else:
+            inputs = {
+                "source": feats.half().to(self.device)
+                if half_support
+                else feats.to(self.device),
+                "padding_mask": padding_mask.to(self.device),
+                "output_layer": self.embedder_output_layer,
+            }
+
+            if not half_support:
+                self.embedder_model = self.embedder_model.float()
+                inputs["source"] = inputs["source"].float()
+
+            with torch.no_grad():
+                logits = self.embedder_model.extract_features(**inputs)
+                if is_feats_dim_768:
+                    feats = logits[0]
+                else:
+                    feats = self.embedder_model.final_proj(logits[0])
+
+        if (
+            isinstance(self.index, type(None)) == False
+            and isinstance(self.big_npy, type(None)) == False
+            and index_rate != 0
+        ):
+            npy = feats[0].cpu().numpy()
+            if self.is_half:
+                npy = npy.astype("float32")
+
+            _, ix = self.index.search(npy, k=1)
+            npy = self.big_npy[ix[:, 0]]
+
+            if self.is_half:
+                npy = npy.astype("float16")
+            feats = (
+                torch.from_numpy(npy).unsqueeze(0).to(self.device) * index_rate
+                + (1 - index_rate) * feats
+            )
+
+        feats = F.interpolate(feats.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1)
+
+        p_len = audio.shape[0] // self.window
+        if feats.shape[1] < p_len:
+            p_len = feats.shape[1]
+            if pitch != None and pitchf != None:
+                pitch = pitch[:, :p_len]
+                pitchf = pitchf[:, :p_len]
+        p_len = torch.tensor([p_len], device=self.device).long()
+        with torch.no_grad():
+            if pitch != None and pitchf != None:
+                audio1 = (
+                    (self.net_g.infer(feats, p_len, pitch, pitchf, sid)[0][0, 0] * 32768)
+                    .data.cpu()
+                    .float()
+                    .numpy()
+                    .astype(np.int16)
+                )
+            else:
+                audio1 = (
+                    (self.net_g.infer(feats, p_len, sid)[0][0, 0] * 32768)
+                    .data.cpu()
+                    .float()
+                    .numpy()
+                    .astype(np.int16)
+                )
+        del feats, p_len, padding_mask
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+        return audio1
+
+
+# F0 computation
+def get_f0_crepe_computation(
+        x,
+        sr,
+        f0_min,
+        f0_max,
+        p_len,
+        model="full", # Either use crepe-tiny "tiny" or crepe "full". Default is full
+):
+    hop_length = sr // 100
+    x = x.astype(np.float32) # fixes the F.conv2D exception. We needed to convert double to float.
+    x /= np.quantile(np.abs(x), 0.999)
+    torch_device = self.get_optimal_torch_device()
+    audio = torch.from_numpy(x).to(torch_device, copy=True)
+    audio = torch.unsqueeze(audio, dim=0)
+    if audio.ndim == 2 and audio.shape[0] > 1:
+        audio = torch.mean(audio, dim=0, keepdim=True).detach()
+    audio = audio.detach()
+    print("Initiating prediction with a crepe_hop_length of: " + str(hop_length))
+    pitch: Tensor = torchcrepe.predict(
+        audio,
+        sr,
+        sr // 100,
+        f0_min,
+        f0_max,
+        model,
+        batch_size=hop_length * 2,
+        device=torch_device,
+        pad=True
+    )
+    p_len = p_len or x.shape[0] // hop_length
+    # Resize the pitch for final f0
+    source = np.array(pitch.squeeze(0).cpu().float().numpy())
+    source[source < 0.001] = np.nan
+    target = np.interp(
+        np.arange(0, len(source) * p_len, len(source)) / p_len,
+        np.arange(0, len(source)),
+        source
+    )
+    f0 = np.nan_to_num(target)
+    return f0 # Resized f0
+
+def get_f0_official_crepe_computation(
+        x,
+        sr,
+        f0_min,
+        f0_max,
+        model="full",
+):
+    # Pick a batch size that doesn't cause memory errors on your gpu
+    batch_size = 512
+    # Compute pitch using first gpu
+    audio = torch.tensor(np.copy(x))[None].float()
+    f0, pd = torchcrepe.predict(
+        audio,
+        sr,
+        sr // 100,
+        f0_min,
+        f0_max,
+        model,
+        batch_size=batch_size,
+        device=device,
+        return_periodicity=True,
+    )
+    pd = torchcrepe.filter.median(pd, 3)
+    f0 = torchcrepe.filter.mean(f0, 3)
+    f0[pd < 0.1] = 0
+    f0 = f0[0].cpu().numpy()
+    return f0
+
+def get_f0(
+    x: np.ndarray,
+    sr: int,
+    p_len: int,
+    f0_up_key: int,
+    f0_method: str,
+):
+    f0_min = 50
+    f0_max = 1100
+    f0_mel_min = 1127 * np.log(1 + f0_min / 700)
+    f0_mel_max = 1127 * np.log(1 + f0_max / 700)
+
+    if f0_method == "harvest":
+        f0, t = pyworld.harvest(
+            x.astype(np.double),
+            fs=sr,
+            f0_ceil=f0_max,
+            f0_floor=f0_min,
+            frame_period=10,
+        )
+        f0 = pyworld.stonemask(x.astype(np.double), f0, t, sr)
+        f0 = signal.medfilt(f0, 3)
+    elif f0_method == "dio":
+        f0, t = pyworld.dio(
+            x.astype(np.double),
+            fs=sr,
+            f0_ceil=f0_max,
+            f0_floor=f0_min,
+            frame_period=10,
+        )
+        f0 = pyworld.stonemask(x.astype(np.double), f0, t, sr)
+        f0 = signal.medfilt(f0, 3)
+    elif f0_method == "mangio-crepe":
+        f0 = get_f0_crepe_computation(x, sr, f0_min, f0_max, p_len, "full")
+    elif f0_method == "crepe":
+        f0 = get_f0_official_crepe_computation(x, sr, f0_min, f0_max, "full")
+
+    f0 *= pow(2, f0_up_key / 12)
+    f0bak = f0.copy()
+    f0_mel = 1127 * np.log(1 + f0 / 700)
+    f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * 254 / (
+        f0_mel_max - f0_mel_min
+    ) + 1
+    f0_mel[f0_mel <= 1] = 1
+    f0_mel[f0_mel > 255] = 255
+    f0_coarse = np.rint(f0_mel).astype(np.int32)
+    return f0_coarse, f0bak  # 1-0

modules/shared.py

@@ -0,0 +1,44 @@
+import os
+import sys
+
+import torch
+
+from modules.cmd_opts import opts
+
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+MODELS_DIR = os.path.join(ROOT_DIR, "models")
+
+
+def has_mps():
+    if sys.platform != "darwin":
+        return False
+    else:
+        if not getattr(torch, "has_mps", False):
+            return False
+        try:
+            torch.zeros(1).to(torch.device("mps"))
+            return True
+        except Exception:
+            return False
+
+
+is_half = opts.precision == "fp16"
+half_support = (
+    torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 5.3
+)
+
+if not half_support:
+    print("WARNING: FP16 is not supported on this GPU")
+    is_half = False
+
+device = "cuda:0"
+
+if not torch.cuda.is_available():
+    if has_mps():
+        print("Using MPS")
+        device = "mps"
+    else:
+        print("Using CPU")
+        device = "cpu"
+
+device = torch.device(device)