first push of codes and models for g2p, t2u, tokenizer and detokenizer

.gitattributes

@@ -33,3 +33,17 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+semantic_detokenizer/ckpt/model.pt filter=lfs diff=lfs merge=lfs -text
+semantic_tokenizer/f40ms/ckpt/model.pt filter=lfs diff=lfs merge=lfs -text
+text2token/ckpt/40ms.checkpoint15.pt filter=lfs diff=lfs merge=lfs -text
+thirdparty/G2P/text/ja_userdic/userdict.csv filter=lfs diff=lfs merge=lfs -text
+figs/CADiT.jpg filter=lfs diff=lfs merge=lfs -text
+figs/F5-streaming.jpg filter=lfs diff=lfs merge=lfs -text
+figs/TTS.jpg filter=lfs diff=lfs merge=lfs -text
+figs/eval1.jpg filter=lfs diff=lfs merge=lfs -text
+figs/eval2.jpg filter=lfs diff=lfs merge=lfs -text
+figs/eval3.jpg filter=lfs diff=lfs merge=lfs -text
+figs/reconstruction.jpg filter=lfs diff=lfs merge=lfs -text
+figs/tokenizer.jpg filter=lfs diff=lfs merge=lfs -text
+004892.wav filter=lfs diff=lfs merge=lfs -text
+00004557-00000030.wav filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,138 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/

00004557-00000030.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a3b3747139278b3e4318d24e2e844cdbe534602d95fa5430786721d14480a9b
+size 249388

004892.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:255a96ce54c0b502b2aa785a4b1fdf0729d4c8975236fc4a1ba325c09637bf79
+size 297358

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,3 +1,45 @@
----
-license: apache-2.0
----
+# Discrete Speech Tokenization Toolkit
+## Release Notes:
+V1.0
+
+This release consists of the following models:
+1. A speech semantic tokenzier (25Hz, codebook size=4096) for Chinese and English
+2. A corresponding speech detokenizer for for Chinese and English
+3. A text2token model (T2U) converts text to speech tokens.
+
+these models achieved top-tier performance on TTS and speech reconstruction on the seed-tts-eval dataset:
+<p align="center"><img src="figs/eval1.jpg" width="1200"></p> 
+<p align="center"><img src="figs/eval2.jpg" width="1200"></p> 
+
+We also evaluated the ASR performance of our semantic tokenizer using a LLM as backbone. Our model achieve comparable performance to models that use continuous speech representation.
+<p align="center"><img src="figs/eval3.jpg"  width="1200"></p> 
+
+## Speech Semantic Tokenizer
+the speech semantic tokenizer is train with labelled data.
+<p align="center"><img src="figs/tokenizer.jpg" width="800"></p> 
+
+
+## Speech Detokenizer
+Our speech detokenizer is developed based on [F5-TTS](https://github.com/SWivid/F5-TTS) with two major updates added.
+1. we adopt DiT block with cross attention, which is similar to the detokenizer of [GLM-4-Voice](https://github.com/zai-org/GLM-4-Voice).
+<p align="center"><img src="figs/CADiT.jpg" height="600"></p>
+
+2. we intruduced a chunk-wise streaming inferece process that can generate speech of any length.
+<p align="center"><img src="figs/F5-streaming.jpg" width="1200"></p>
+
+## Text2Token(T2U)
+Text2Token is transformer machine translation model, which is trained on about 380k hours of speech-text pairs with [fairseq](https://github.com/facebookresearch/fairseq).
+
+## TTS pipeline
+As shown in tts_example.py, the 3 models could form a pipeline for TTS task.
+<p align="center"><img src="figs/TTS.jpg" width="1200"></p>
+
+## Non-parallel Speech Reconstruction Pipeline
+As shown in reconstruction_example.py, the tokenizer and detokenizer could form a pipeline for speech reconstruction task.
+<p align="center"><img src="figs/reconstruction.jpg" width="1200"></p>
+
+# Core Developers:
+[Daxin Tan]([email protected]), [Dehua Tao]([email protected]), [Yusen Sun]([email protected]) and [Xiao Chen]([email protected])
+
+## Contributors:
+[Hanlin Zhang]([email protected])

README_CN.md

@@ -0,0 +1,48 @@
+# Discrete Speech Tokenization Toolkit
+## Release Notes:
+V1.0
+
+本次发布的工具包包括一下几个模型:
+1. 一个语音tokenizer(25Hz, codebook size=4096)支持中英文
+2. 一个对应的detokenizer支持中英文
+3. 一个text2token模型(T2U)可将文本转换为语音token.
+
+这些模型在seed-tts-eval数据集的TTS和语音重建任务上上达到了一流的精度:
+<p align="center"><img src="figs/eval1.jpg" width="1200"></p> 
+<p align="center"><img src="figs/eval2.jpg" width="1200"></p> 
+
+我们基于LLM测试了本语音tokenizer的ASR精度，我们的tokenizer达到了与采用连续语音表征的模型相近的水平.
+<p align="center"><img src="figs/eval3.jpg"  width="1200"></p> 
+
+## Speech Semantic Tokenizer
+本tokenizer的采用了监督学习方法。训练中我们使用了从开源数据中采样的大约4000小时中英文语音文本对数据，语种比例为1:1.
+<p align="center"><img src="figs/tokenizer.jpg" width="800"></p> 
+
+
+## Speech Detokenizer
+本detokenizer是基于[F5-TTS](https://github.com/SWivid/F5-TTS)开发，但是对齐进行了两项改进：
+1. 采用了DiT with cross attention，类似于[GLM-4-Voice](https://github.com/zai-org/GLM-4-Voice)的detokenizer.
+<p align="center"><img src="figs/CADiT.jpg" height="600"></p>
+
+2. 开发了以chunk为单元的流式推理流程.
+<p align="center"><img src="figs/F5-streaming.jpg" width="1200"></p>
+
+本次发布的detokenizer使用了约6000小时的中英文数据包括Wenet4TTS（premium，standard），LibriTTS等等。
+
+## Text2Token(T2U)
+Text2Token是一个transformer机器翻译模型，用[fairseq](https://github.com/facebookresearch/fairseq)在约38万小时的语音文本对数据上训练得到。
+
+
+## TTS pipeline
+在tts_example.py中, 我们给出了一个实例，串联使用上述三个模型实现TTS的功能.
+<p align="center"><img src="figs/TTS.jpg" width="1200"></p>
+
+## Non-parallel Speech Reconstruction Pipeline
+在reconstruction_example.py中, 有另外一个实例，串联使用tokenizer和detokenizer实现语音重建的功能.
+<p align="center"><img src="figs/reconstruction.jpg" width="1200"></p>
+
+# Core Developers:
+[Daxin Tan]([email protected]), [Dehua Tao]([email protected]), [Yusen Sun]([email protected]) and [Xiao Chen]([email protected])
+
+## Contributors:
+[Hanlin Zhang]([email protected])

Third_Party_Open_Source_Software_Notice

@@ -0,0 +1,1289 @@
+OPEN SOURCE SOFTWARE NOTICE FOR LIST
+
+This document contains licenses and copyright notices for third-party software components used in the list project, which are legally required to be disclosed under their respective open source licenses.
+
+The below components are provided under separate licenses. The applicable license terms for each component are provided in the corresponding license text below. These licenses are granted by the original copyright holders.
+
+When using the open source software components contained in list, the applicable open source license(s) shall prevail over any other license terms that may be contained in this distribution, including but not limited to: Apache-2.0, BSD-3-Clause, MIT, NOASSERTION.
+
+WARRANTY DISCLAIMER
+
+THE OPEN SOURCE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+For full warranty details and limitations, refer to the individual license texts provided below.
+
+================================================================================
+
+[SoftWare] fairseq 0.12.2
+
+Copyright Notice(s):
+  • Copyright (c) Facebook, Inc. and its affiliates.
+
+License: MIT License
+
+Full License Text:
+Copyright (c) Facebook, Inc. and its affiliates.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+================================================================================
+
+[SoftWare] f5-tts 0.12.2
+
+Copyright Notice(s):
+  • Copyright (c) 2024 Yushen CHEN
+
+License: MIT License
+
+Full License Text:
+Copyright (c) 2024 Yushen CHEN
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+================================================================================
+
+[SoftWare] GPT-SoVITS
+
+Copyright Notice(s):
+  • Copyright (c) 2024 RVC-Boss
+
+License: MIT License
+
+Full License Text:
+Copyright (c) 2024 RVC-Boss
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+================================================================================
+
+[SoftWare] WeTextProcessing
+
+Copyright Notice(s):
+  • Copyright (c) 2022 Zhendong Peng ([email protected])
+
+License: Apache License V2.0
+
+Full License Text:
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+================================================================================
+
+[SoftWare] icefall 1.0
+
+Copyright Notice(s):
+  • Copyright (c) 2021-2024  Xiaomi Corp.
+
+License: Apache License V2.0
+
+Full License Text:
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+
+================================================================================
+
+[SoftWare] espnet 0.10.5
+
+Copyright Notice(s):
+  • Copyright 2017 Johns Hopkins University
+
+License: Apache License V2.0
+
+Full License Text:
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+================================================================================
+
+[SoftWare] vixen build-560
+
+License: freeware
+
+Full License Text:
+
+Vixen is unlicensed and free to use. This software is provided 'as-is', without any express or implied warranty. 
+In no event will the authors or any contributors be held liable for any damages or liabilities arising from the use of this software.
+
+Vixen uses many 3rd party libraries that are required to be present on the path for the full functionality. Some of those have licenses that can be found as follows.
+
+NLog http://raw.github.com/NLog/NLog/master/LICENSE.txt
+NLog.Config http://raw.github.com/NLog/NLog/master/LICENSE.txt
+NLog.Schema http://raw.github.com/NLog/NLog/master/LICENSE.txt
+DockPanelSuite http://www.opensource.org/licenses/mit-license.php
+DockPanelSuite.ThemeVS2015 http://www.opensource.org/licenses/mit-license.php
+Newtonsoft.Json https://raw.github.com/JamesNK/Newtonsoft.Json/master/LICENSE.md
+Nowin http://www.opensource.org/licenses/mit-license.php
+Owin https://github.com/owin-contrib/owin-hosting/blob/master/LICENSE.txt
+NShape https://nshape.codeplex.com/license
+ZedGraph https://www.gnu.org/licenses/lgpl-2.1.txt
+Accord.Net http://accord-framework.net/license.txt
+Accord.Video.FFMPEG http://accord-framework.net/gpl.txt
+
+Some portions of the Effect editor code are based on work that is licensed under the Apache 2.0 license.
+http://www.apache.org/licenses/LICENSE-2.0
+
+NumericTextBox is derived from NumericalTextBox control licensed under the Apache 2.0 License. http://www.apache.org/licenses/LICENSE-2.0 
+
+================================================================================
+
+[SoftWare] g2pK 3bb9d5a
+
+License: Apache License V2.0
+
+Full License Text:
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+================================================================================
+
+[SoftWare] Parakeet 0.4.0
+
+Copyright Notice(s):
+  • Copyright 2017 Johns Hopkins University
+
+License: Apache License V2.0
+
+Full License Text:
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+================================================================================
+
+[SoftWare] PaddleSpeech	0.2.0
+
+Copyright Notice(s):
+  • Copyright (c) 2021 PaddlePaddle Authors
+
+License: Apache License V2.0
+
+Full License Text: 
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+   
+================================================================================
+
+[SoftWare] espresso 3add524
+
+Copyright Notice(s):
+  • Copyright (c) Yiming Wang
+
+License: MIT License
+
+Full License Text: 
+MIT License
+
+Copyright for the original fairseq code are held by Facebook, Inc. and its 
+affiliates as part of project Espresso. All other copyright for project Espresso
+are held by Espresso authors.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+================================================================================

evaluation/README.md

@@ -0,0 +1,9 @@
+evaluation:
+
+for evaluation of speaker similarity, user should downloade the latest version of UniSpeech from github:
+
+git clone https://github.com/Sanyuan-Chen/UniSpeech.git
+
+and add the path of UniSpeech to PYTHONPATH:
+
+export PYTHONPATH=/path/to/UniSpeech/downstreams/speaker_verification:$PYTHONPATH

evaluation/eval_detok_en.py

@@ -0,0 +1,269 @@
+# Copyright (C) 2025. Huawei Technologies Co., Ltd. All Rights Reserved. (authors: Xiao Chen)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from transformers import WhisperProcessor, WhisperForConditionalGeneration
+import soundfile as sf
+import scipy
+import argparse
+from whisper_normalizer.english import EnglishTextNormalizer
+import os
+import string
+import lingvo.tasks.asr.tools.simple_wer_v2 as WER
+from tqdm import tqdm
+import logging
+import torch
+
+keyphrases = None
+english_normalizer = EnglishTextNormalizer()
+device = torch.device("cuda")
+en_asr_model_path = "./whisper-large-v3"
+
+
+wer_obj = WER.SimpleWER(
+    key_phrases=keyphrases,
+    html_handler=WER.HighlightAlignedHtmlHandler(WER.HighlightAlignedHtml),
+    preprocess_handler=WER.RemoveCommentTxtPreprocess,
+)
+
+
+def dummy_split_text(text):
+    return text
+
+
+def remove_punct(text):
+    puncts = set(string.punctuation)
+    output = ""
+    for char in text:
+        if char not in puncts:
+            output += char
+    output = output.replace("  ", " ")
+    return output
+
+
+def get_gt_ref_texts_and_wav_files(
+    args, gt_test_lst, gt_folder, punct_remover, text_spliter
+):
+    wav_file_list = []
+    reference = []
+    with open(gt_test_lst, "r") as fp:
+        for line in fp:
+            fields = line.strip().split("|")
+            wav_file = f"{gt_folder}/{fields[0]}.wav"
+
+            if not os.path.isfile(wav_file):
+                continue
+
+            wav_file_list.append(wav_file)
+            text = fields[-1].lower()
+            if args.norm_text:
+                truth_text = english_normalizer(text)  # " ".join(fields[-1])
+            elif args.remove_punct:
+                truth_text = punct_remover(text)
+            else:
+                truth_text = text
+            truth_text = text_spliter(truth_text)
+            reference.append([truth_text, fields[-1]])
+
+    assert len(reference) == len(wav_file_list)
+    return reference, wav_file_list
+
+
+def get_ref_texts_and_gen_files(
+    args, test_lst, test_folder, punct_remover, text_spliter
+):
+    reference = []
+    gen_file_list = []
+    with open(test_lst, "r") as fp:
+        for line in fp:
+            fields = line.strip().split("|")
+            filename = fields[2].split("/")[-1]
+            filename = filename.split(".")[0]
+            gen_file = f"{filename}_gen.wav"
+            gen_file_list.append(f"{test_folder}/{gen_file}")
+
+            text = fields[-1].lower()
+            if args.norm_text:
+                truth_text = english_normalizer(text)  # " ".join(fields[-1])
+            elif args.remove_punct:
+                truth_text = punct_remover(text)
+            else:
+                truth_text = text
+
+            truth_text = text_spliter(truth_text)
+            reference.append([truth_text, fields[-1]])
+
+    assert len(reference) == len(gen_file_list)
+    return reference, gen_file_list
+
+
+def get_hypo_texts(args, results_list, punct_remover, text_spliter):
+    hypothesis = []
+    for res in results_list:
+        text = res["text"].lower()
+        if args.norm_text:
+            hypo_text = english_normalizer(text)
+        elif args.remove_punct:
+            hypo_text = punct_remover(text)
+        else:
+            hypo_text = text
+        hypo_text = text_spliter(hypo_text)
+        hypothesis.append([hypo_text, res["text"]])
+
+    return hypothesis
+
+
+def calc_wer(reference, hypothesis, test_lst):
+    logging.info(f"calc WER:")
+    for idx in tqdm(range(len(hypothesis))):
+        hypo = hypothesis[idx][0].strip()
+        ref = reference[idx][0].strip()
+        wer_obj.AddHypRef(hypo, ref)
+
+    str_summary, str_details, str_keyphrases_info = wer_obj.GetSummaries()
+    logging.info(f"WER summary:")
+    logging.info(str_summary)
+    logging.info(str_details)
+    logging.info(str_keyphrases_info)
+
+    try:
+        fn_output = test_lst + "_diagnosis.html"
+        aligned_html = "<br>".join(wer_obj.aligned_htmls)
+        with open(fn_output, "wt") as fp:
+            fp.write("<body><html>")
+            fp.write("<div>%s</div>" % aligned_html)
+            fp.write("</body></html>")
+        fp.close()
+
+        text_output = test_lst + "_rawtext.lst"
+        with open(text_output, "w") as fp:
+            for ref, hypo in zip(reference, hypothesis):
+                fp.write(f"{ref[1]}|{hypo[1]}\n")
+        fp.close()
+        logging.info(f"Save {fn_output} and {text_output} for diagnosis")
+    except IOError:
+        logging.info("failed to write diagnosis html")
+
+
+def load_en_model():
+    processor = WhisperProcessor.from_pretrained(en_asr_model_path)
+    model = WhisperForConditionalGeneration.from_pretrained(en_asr_model_path).to(
+        device
+    )
+    return processor, model
+
+
+def process_wavs(wav_file_list, batch_size=300):
+    results = []
+    processor, model = load_en_model()
+    for wav_file_path in tqdm(wav_file_list):
+        wav, sr = sf.read(wav_file_path)
+        if sr != 16000:
+            wav = scipy.signal.resample(wav, int(len(wav) * 16000 / sr))
+        input_features = processor(
+            wav, sampling_rate=16000, return_tensors="pt"
+        ).input_features
+        input_features = input_features.to(device)
+        forced_decoder_ids = processor.get_decoder_prompt_ids(
+            language="english", task="transcribe"
+        )
+        predicted_ids = model.generate(
+            input_features, forced_decoder_ids=forced_decoder_ids
+        )
+        transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)[
+            0
+        ]
+        results.append({"text": transcription.strip()})
+
+    return results
+
+
+def main(args):
+    handler = logging.FileHandler(filename=args.log_file, mode="w")
+    logging.root.setLevel(logging.INFO)
+    logging.root.addHandler(handler)
+
+    test_path = (
+        args.test_path
+    )  # './40ms.AISHELL2.test_with_single_ref.base.chunk25.gen'
+    lst_path = args.test_lst  # "40ms.AISHELL2.test_with_single_ref.base.lst"
+    logging.info(
+        f"Evaluate {args.test_path} with Text Normalization: {args.norm_text} and Remove Punct: {args.remove_punct}"
+    )
+
+    if args.eval_gt:
+        logging.info(f"run ASR for GT: {lst_path}")
+        reference, wav_file_list = get_gt_ref_texts_and_wav_files(
+            args, lst_path, test_path, remove_punct, dummy_split_text
+        )
+        results = process_wavs(wav_file_list, batch_size=12)
+    else:
+        logging.info(f"run ASR for detok: {lst_path}")
+        reference, gen_file_list = get_ref_texts_and_gen_files(
+            args, lst_path, test_path, remove_punct, dummy_split_text
+        )
+        results = process_wavs(gen_file_list, batch_size=12)
+
+    hypothesis = get_hypo_texts(args, results, remove_punct, dummy_split_text)
+
+    assert len(hypothesis) == len(reference)
+    logging.info(f"Finish runing ASR for {lst_path}")
+    logging.info(f"hypothesis: {len(hypothesis)} vs reference: {len(reference)}")
+
+    calc_wer(reference, hypothesis, test_path)
+    logging.info(f"Finish evaluate {lst_path}, results are in {args.log_file}")
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--test-path",
+        required=True,
+        type=str,
+        help=f"folder of wav files",
+    )
+    parser.add_argument(
+        "--test-lst",
+        required=True,
+        type=str,
+        help=f"path to test file lst",
+    )
+    parser.add_argument(
+        "--log-file",
+        required=False,
+        type=str,
+        default=None,
+        help=f"path to test file lst",
+    )
+    parser.add_argument(
+        "--norm-text",
+        default=False,
+        action="store_true",
+        help=f"normalized GT and hypo texts",
+    )
+    parser.add_argument(
+        "--remove-punct",
+        default=False,
+        action="store_true",
+        help=f"remove punct from GT and hypo texts",
+    )
+    parser.add_argument(
+        "--eval-gt",
+        default=False,
+        action="store_true",
+        help=f"remove punct from GT and hypo texts",
+    )
+    args = parser.parse_args()
+
+    main(args)

evaluation/eval_detok_zh.py

@@ -0,0 +1,144 @@
+# Copyright (C) 2025. Huawei Technologies Co., Ltd. All Rights Reserved. (authors: Xiao Chen)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from funasr import AutoModel
+import argparse
+from zhon.hanzi import punctuation
+import zhconv
+import string
+from tqdm import tqdm
+from eval_detok_en import (
+    get_gt_ref_texts_and_wav_files,
+    get_ref_texts_and_gen_files,
+    get_hypo_texts,
+    calc_wer,
+)
+
+model_path = "./paraformer-zh"  # "./speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
+
+
+def split_text(text):
+    text = " ".join(text)
+    return text
+
+
+def dummy_split_text(text):
+    return text
+
+
+def remove_punct(text):
+    puncts = set(punctuation + string.punctuation)
+    output = ""
+    for char in text:
+        if char not in puncts:
+            output += char
+    output = output.replace("  ", " ")
+    return output
+
+
+def process_wavs(wav_file_list, batch_size=300):
+    model = AutoModel(
+        model=model_path,
+        disable_update=True,
+    )
+
+    results = []
+    for wav_file_path in tqdm(wav_file_list):
+        res = model.generate(
+            input=wav_file_path,
+            batch_size_s=batch_size,
+        )
+        transcription = zhconv.convert(res[0]["text"], "zh-cn")
+        results.append({"text": transcription})
+    return results
+
+
+def main(args):
+    handler = logging.FileHandler(filename=args.log_file, mode="w")
+    logging.root.setLevel(logging.INFO)
+    logging.root.addHandler(handler)
+
+    test_path = (
+        args.test_path
+    )  # './40ms.AISHELL2.test_with_single_ref.base.chunk25.gen'
+    lst_path = args.test_lst  # "40ms.AISHELL2.test_with_single_ref.base.lst"
+
+    if args.eval_gt:
+        logging.info(f"run ASR for GT: {lst_path}")
+        reference, wav_file_list = get_gt_ref_texts_and_wav_files(
+            args, lst_path, test_path, remove_punct, split_text
+        )
+        results = process_wavs(wav_file_list, batch_size=300)
+    else:
+        logging.info(f"run ASR for detok: {lst_path}")
+        reference, gen_file_list = get_ref_texts_and_gen_files(
+            args, lst_path, test_path, remove_punct, split_text
+        )
+        results = process_wavs(gen_file_list, batch_size=300)
+
+    hypothesis = get_hypo_texts(args, results, remove_punct, split_text)
+
+    assert len(hypothesis) == len(reference)
+    logging.info(f"Finish runing ASR for {lst_path}")
+    logging.info(f"hypothesis: {len(hypothesis)} vs reference: {len(reference)}")
+
+    calc_wer(reference, hypothesis, test_path)
+    logging.info(f"Finish evaluate {lst_path}, results are in {args.log_file}")
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--test-path",
+        required=True,
+        type=str,
+        help=f"folder of wav files",
+    )
+    parser.add_argument(
+        "--test-lst",
+        required=True,
+        type=str,
+        help=f"path to test file lst",
+    )
+    parser.add_argument(
+        "--log-file",
+        required=False,
+        type=str,
+        default=None,
+        help=f"path to test file lst",
+    )
+    parser.add_argument(
+        "--remove-punct",
+        default=False,
+        action="store_true",
+        help=f"remove punct from GT and hypo texts",
+    )
+    parser.add_argument(
+        "--norm-text",
+        default=False,
+        action="store_true",
+        help=f"normalized GT and hypo texts",
+    )
+    parser.add_argument(
+        "--eval-gt",
+        default=False,
+        action="store_true",
+        help=f"remove punct from GT and hypo texts",
+    )
+    args = parser.parse_args()
+    args.norm_text = False
+
+    main(args)

evaluation/eval_sim.py

@@ -0,0 +1,188 @@
+# Copyright (C) 2025. Huawei Technologies Co., Ltd. All Rights Reserved. (authors: Xiao Chen)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+from tqdm import tqdm
+import logging
+import os
+from verification import init_model, MODEL_LIST
+import soundfile as sf
+import torch
+import numpy as np
+import torch.nn.functional as F
+from torchaudio.transforms import Resample
+import torch.multiprocessing as mp
+
+console_format = logging.Formatter(
+    "[%(asctime)s][%(filename)s:%(levelname)s][%(process)d:%(threadName)s]%(message)s"
+)
+console_handler = logging.StreamHandler()
+console_handler.setFormatter(console_format)
+console_handler.setLevel(logging.INFO)
+if len(logging.root.handlers) > 0:
+    for handler in logging.root.handlers:
+        logging.root.removeHandler(handler)
+logging.root.addHandler(console_handler)
+logging.root.setLevel(logging.INFO)
+
+
+MODEL_NAME = "wavlm_large"
+S3PRL_PATH = os.environ.get("S3PRL_PATH")
+if S3PRL_PATH is not None:
+    import patch_unispeech
+    logging.info("Applying Patches for unispeech!!!")
+    patch_unispeech.patch_for_npu()
+
+
+def get_ref_and_gen_files(
+    test_lst, test_folder, task_queue
+):
+    with open(test_lst, "r") as fp:
+        for line in fp:
+            fields = line.strip().split("|")
+            gen_name = fields[2].split("/")[-1]
+            gen_name = gen_name.split(".")[0]
+            gen_file = f"{test_folder}/{gen_name}_gen.wav"
+            
+            ref_name = fields[0].split("/")[-1]
+            ref_name = ref_name.split(".")[0]
+            ref_file = f"{test_folder}/{ref_name}_ref.wav"
+
+            task_queue.put((ref_file, gen_file))
+
+    return
+
+
+def eval_speaker_similarity(model,  wav1, wav2, rank):
+    wav1, sr1 = sf.read(wav1)
+    wav2, sr2 = sf.read(wav2)
+
+    wav1 = torch.from_numpy(wav1).unsqueeze(0).float()
+    wav2 = torch.from_numpy(wav2).unsqueeze(0).float()
+    resample1 = Resample(orig_freq=sr1, new_freq=16000)
+    resample2 = Resample(orig_freq=sr2, new_freq=16000)
+    wav1 = resample1(wav1)
+    wav2 = resample2(wav2)
+
+    wav1 = wav1.cuda(f"cuda:{rank}")
+    wav2 = wav2.cuda(f"cuda:{rank}")
+
+    model.eval()
+    with torch.no_grad():
+        emb1 = model(wav1)
+        emb2 = model(wav2)
+
+    sim = F.cosine_similarity(emb1, emb2)
+    logging.info("The similarity score between two audios is %.4f (-1.0, 1.0)." % (sim[0].item()))
+    return sim[0].item()
+
+
+def eval_proc(model_path, task_queue, rank, sim_list):
+    model = None
+    assert MODEL_NAME in MODEL_LIST, 'The model_name should be in {}'.format(MODEL_LIST)
+    model = init_model(MODEL_NAME, model_path) if model is None else model
+    model.to(f"cuda:{rank}")
+    # sim_list = []
+    # for ref, gen in tqdm(ref_gen_list):
+    while True:
+        try:
+            new_record = task_queue.get()
+            if new_record is None:
+                logging.info("FINISH processing all inputs")
+                break
+
+            ref = new_record[0]
+            gen = new_record[1]
+            logging.info(f"eval SIM: {ref} v.s. {gen}")
+
+            if not os.path.exists(ref) or not os.path.exists(gen):
+                logging.info(f"MISSING: {ref} v.s. {gen}")
+                continue
+
+            sim = eval_speaker_similarity(model, ref, gen, rank)
+            sim_list.append((sim, ref, gen))
+        except:
+            logging.info(f"FAIL to eval SIM: {ref} v.s. {gen}")
+    
+
+def main(args):
+    handler = logging.FileHandler(filename=args.log_file, mode="w")
+    logging.root.addHandler(handler)
+
+    device_list = [0] 
+    if "CUDA_VISIBLE_DEVICES" in os.environ:
+        device_list = [int(x.strip()) for x in os.environ["CUDA_VISIBLE_DEVICES"].split(",")]
+
+    logging.info(f"Using devices: {device_list}")
+    n_procs = len(device_list)
+    ctx = mp.get_context('spawn')
+    with ctx.Manager() as manager:
+        sim_list = manager.list()
+        task_queue = manager.Queue()
+        get_ref_and_gen_files(args.test_lst, args.test_path, task_queue)
+
+        processes = []
+        for idx in range(n_procs):
+            task_queue.put(None)
+            rank = idx # device_list[idx]
+            p = ctx.Process(target=eval_proc, args=(args.model_path, task_queue, rank, sim_list))
+            processes.append(p)
+
+        for proc in processes:
+            proc.start()
+        
+        for proc in processes:
+            proc.join()
+   
+        sim_scores = []
+        for sim, ref, gen in sim_list:
+            logging.info(f"{ref} vs {gen} : {sim}")
+            sim_scores.append(sim)
+        avg_sim = round(np.mean(np.array(list(sim_scores))), 3)
+        logging.info("total evaluated wav pairs: %d" % (len(sim_list)))
+        logging.info("The average similarity score of %s is %.4f (-1.0, 1.0)." % (args.test_path, avg_sim))
+    return avg_sim
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--test-path",
+        required=True,
+        type=str,
+        help=f"folder of wav files",
+    )
+    parser.add_argument(
+        "--test-lst",
+        required=True,
+        type=str,
+        help=f"path to test file lst",
+    )
+    parser.add_argument(
+        "--log-file",
+        required=False,
+        type=str,
+        default=None,
+        help=f"path to test file lst",
+    )
+    parser.add_argument(
+        "--model-path",
+        type=str,
+        default="./wavlm-sv",
+        help=f"path to sv model",
+    )
+
+    args = parser.parse_args()
+    main(args)

evaluation/patch_unispeech.py

@@ -0,0 +1,131 @@
+# Copyright (C) 2025. Huawei Technologies Co., Ltd. All Rights Reserved. (authors: Xiao Chen)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from patch_utils import MindSpeedPatchesManager as aspm
+import os
+import torch
+import torch.nn as nn
+import logging
+import torchaudio.transforms as trans
+from s3prl.upstream.wavlm.expert import UpstreamExpert as s3prl_UpstreamExpert
+from models.ecapa_tdnn import Conv1dReluBn, SE_Res2Block, AttentiveStatsPool 
+from models.ecapa_tdnn import ECAPA_TDNN_SMALL, ECAPA_TDNN
+
+def init_model_patched(model_name, checkpoint=None):
+    S3PRL_PATH = os.environ.get("S3PRL_PATH")
+    if model_name == 'unispeech_sat':
+        config_path = 'config/unispeech_sat.th'
+        model = ECAPA_TDNN_SMALL(feat_dim=1024, feat_type='unispeech_sat', config_path=config_path)
+    elif model_name == 'wavlm_base_plus':
+        config_path = None
+        model = ECAPA_TDNN_SMALL(feat_dim=768, feat_type='wavlm_base_plus', config_path=config_path)
+    elif model_name == 'wavlm_large':
+        config_path = S3PRL_PATH
+        model = ECAPA_TDNN_SMALL(feat_dim=1024, feat_type='wavlm_large', config_path=config_path)
+    elif model_name == 'hubert_large':
+        config_path = None
+        model = ECAPA_TDNN_SMALL(feat_dim=1024, feat_type='hubert_large_ll60k', config_path=config_path)
+    elif model_name == 'wav2vec2_xlsr':
+        config_path = None
+        model = ECAPA_TDNN_SMALL(feat_dim=1024, feat_type='wav2vec2_xlsr', config_path=config_path)
+    else:
+        model = ECAPA_TDNN_SMALL(feat_dim=40, feat_type='fbank')
+
+    if checkpoint is not None:
+        state_dict = torch.load(checkpoint, map_location=lambda storage, loc: storage)
+        model.load_state_dict(state_dict['model'], strict=False)
+    return model
+
+
+class patched_ECAPA_TDNN(ECAPA_TDNN):
+    def __init__(self, feat_dim=80, channels=512, emb_dim=192, global_context_att=False,
+                    feat_type='fbank', sr=16000, feature_selection="hidden_states", update_extract=False, config_path=None):
+        super(ECAPA_TDNN, self).__init__()
+
+        self.feat_type = feat_type
+        self.feature_selection = feature_selection
+        self.update_extract = update_extract
+        self.sr = sr
+
+        if feat_type == "fbank" or feat_type == "mfcc":
+            self.update_extract = False
+
+        win_len = int(sr * 0.025)
+        hop_len = int(sr * 0.01)
+
+        if feat_type == 'fbank':
+            self.feature_extract = trans.MelSpectrogram(sample_rate=sr, n_fft=512, win_length=win_len,
+                                                        hop_length=hop_len, f_min=0.0, f_max=sr // 2,
+                                                        pad=0, n_mels=feat_dim)
+        elif feat_type == 'mfcc':
+            melkwargs = {
+                'n_fft': 512,
+                'win_length': win_len,
+                'hop_length': hop_len,
+                'f_min': 0.0,
+                'f_max': sr // 2,
+                'pad': 0
+            }
+            self.feature_extract = trans.MFCC(sample_rate=sr, n_mfcc=feat_dim, log_mels=False,
+                                                melkwargs=melkwargs)
+        else:
+            if config_path is None:
+                self.feature_extract = torch.hub.load('s3prl/s3prl', feat_type)
+            else:
+                self.feature_extract = s3prl_UpstreamExpert(config_path)
+            if len(self.feature_extract.model.encoder.layers) == 24 and hasattr(self.feature_extract.model.encoder.layers[23].self_attn, "fp32_attention"):
+                self.feature_extract.model.encoder.layers[23].self_attn.fp32_attention = False
+            if len(self.feature_extract.model.encoder.layers) == 24 and hasattr(self.feature_extract.model.encoder.layers[11].self_attn, "fp32_attention"):
+                self.feature_extract.model.encoder.layers[11].self_attn.fp32_attention = False
+
+            self.feat_num = self.get_feat_num()
+            self.feature_weight = nn.Parameter(torch.zeros(self.feat_num))
+
+        if feat_type != 'fbank' and feat_type != 'mfcc':
+            freeze_list = ['final_proj', 'label_embs_concat', 'mask_emb', 'project_q', 'quantizer']
+            for name, param in self.feature_extract.named_parameters():
+                for freeze_val in freeze_list:
+                    if freeze_val in name:
+                        param.requires_grad = False
+                        break
+
+        if not self.update_extract:
+            for param in self.feature_extract.parameters():
+                param.requires_grad = False
+
+        self.instance_norm = nn.InstanceNorm1d(feat_dim)
+        # self.channels = [channels] * 4 + [channels * 3]
+        self.channels = [channels] * 4 + [1536]
+
+        self.layer1 = Conv1dReluBn(feat_dim, self.channels[0], kernel_size=5, padding=2)
+        self.layer2 = SE_Res2Block(self.channels[0], self.channels[1], kernel_size=3, stride=1, padding=2, dilation=2, scale=8, se_bottleneck_dim=128)
+        self.layer3 = SE_Res2Block(self.channels[1], self.channels[2], kernel_size=3, stride=1, padding=3, dilation=3, scale=8, se_bottleneck_dim=128)
+        self.layer4 = SE_Res2Block(self.channels[2], self.channels[3], kernel_size=3, stride=1, padding=4, dilation=4, scale=8, se_bottleneck_dim=128)
+
+        # self.conv = nn.Conv1d(self.channels[-1], self.channels[-1], kernel_size=1)
+        cat_channels = channels * 3
+        self.conv = nn.Conv1d(cat_channels, self.channels[-1], kernel_size=1)
+        self.pooling = AttentiveStatsPool(self.channels[-1], attention_channels=128, global_context_att=global_context_att)
+        self.bn = nn.BatchNorm1d(self.channels[-1] * 2)
+        self.linear = nn.Linear(self.channels[-1] * 2, emb_dim)
+
+
+def patched_ECAPA_TDNN_SMALL(feat_dim, emb_dim=256, feat_type='fbank', sr=16000, feature_selection="hidden_states", update_extract=False, config_path=None):
+    return patched_ECAPA_TDNN(feat_dim=feat_dim, channels=512, emb_dim=emb_dim,
+                      feat_type=feat_type, sr=sr, feature_selection=feature_selection, update_extract=update_extract, config_path=config_path)
+
+def patch_for_npu():
+    aspm.register_patch('models.ecapa_tdnn.ECAPA_TDNN_SMALL', patched_ECAPA_TDNN_SMALL)
+    aspm.register_patch('verification.init_model', init_model_patched)
+    aspm.apply_patches()

evaluation/patch_utils.py

@@ -0,0 +1,122 @@
+# copied from https://gitee.com/ascend/MindSpeed/blob/master/mindspeed/patch_utils.py
+
+import importlib
+import sys
+import types
+
+
+def get_func_name(func):
+    if isinstance(func, str):
+        return func
+    return '.'.join((func.__module__, func.__qualname__))
+
+
+def dummy_function_wrapper(func_name):
+    def dummy_function(*args, **kwargs):
+        raise RuntimeError('function {} no exist'.format(func_name))
+
+    return dummy_function
+
+
+class Patch:
+    def __init__(self, orig_func_name, new_func, create_dummy):
+        split_name = orig_func_name.rsplit('.', 1)
+        if len(split_name) == 1:
+            self.orig_module_name, self.orig_func_name = orig_func_name, None
+        else:
+            self.orig_module_name, self.orig_func_name = split_name
+        self.orig_module = None
+        self.orig_func = None
+
+        self.patch_func = None
+        self.wrappers = []
+        if new_func is None:
+            new_func = dummy_function_wrapper(orig_func_name)
+        self.set_patch_func(new_func)
+        self.is_applied = False
+        self.create_dummy = create_dummy
+
+    @property
+    def orig_func_id(self):
+        return id(self.orig_func)
+
+    @property
+    def patch_func_id(self):
+        return id(self.patch_func)
+
+    def set_patch_func(self, new_func, force_patch=False):
+        if hasattr(new_func, '__name__') and new_func.__name__.endswith(('wrapper', 'decorator')):
+            self.wrappers.append(new_func)
+        else:
+            if self.patch_func and not force_patch:
+                raise RuntimeError('the patch of {} exist !'.format(self.orig_func_name))
+            self.patch_func = new_func
+        self.is_applied = False
+
+    def apply_patch(self):
+        if self.is_applied:
+            return
+
+        self.orig_module, self.orig_func = Patch.parse_path(self.orig_module_name, self.orig_func_name, self.create_dummy)
+
+        final_patch_func = self.orig_func
+        if self.patch_func is not None:
+            final_patch_func = self.patch_func
+
+        for wrapper in self.wrappers:
+            final_patch_func = wrapper(final_patch_func)
+
+        if self.orig_func_name is not None:
+            setattr(self.orig_module, self.orig_func_name, final_patch_func)
+        for key, value in sys.modules.copy().items():
+            if self.orig_func_name is not None and hasattr(value, self.orig_func_name) \
+                    and id(getattr(value, self.orig_func_name)) == self.orig_func_id:
+                setattr(value, self.orig_func_name, final_patch_func)
+        self.is_applied = True
+
+    @staticmethod
+    def parse_path(module_path, function_name, create_dummy):
+        from importlib.machinery import ModuleSpec
+        modules = module_path.split('.')
+        for i in range(1, len(modules) + 1):
+            parent = '.'.join(modules[:i - 1])
+            path = '.'.join(modules[:i])
+            try:
+                importlib.import_module(path)
+            except ModuleNotFoundError as e:
+                if not parent or not hasattr(importlib.import_module(parent), modules[i - 1]):
+                    if not create_dummy:
+                        raise ModuleNotFoundError(e) from e
+                    sys.modules[path] = types.ModuleType(path)
+                    sys.modules[path].__file__ = 'mindspeed.dummy_module.py'
+                    sys.modules[path].__spec__ = ModuleSpec(path, None)
+                    if parent:
+                        setattr(importlib.import_module(parent), modules[i - 1], sys.modules[path])
+                else:
+                    module = getattr(importlib.import_module(parent), modules[i - 1])
+                    if hasattr(module, function_name):
+                        return module, getattr(module, function_name)
+                    elif create_dummy:
+                        return module, dummy_function_wrapper(function_name)
+                    else:
+                        raise RuntimeError('no exist {} of {}'.format(function_name, module))
+
+        if function_name is not None and not hasattr(sys.modules[module_path], function_name):
+            setattr(sys.modules[module_path], function_name, None)
+        return sys.modules[module_path], getattr(sys.modules[module_path], function_name) if function_name is not None else None
+
+
+class MindSpeedPatchesManager:
+    patches_info = {}
+
+    @staticmethod
+    def register_patch(orig_func_name, new_func=None, force_patch=False, create_dummy=False):
+        if orig_func_name not in MindSpeedPatchesManager.patches_info:
+            MindSpeedPatchesManager.patches_info[orig_func_name] = Patch(orig_func_name, new_func, create_dummy)
+        else:
+            MindSpeedPatchesManager.patches_info.get(orig_func_name).set_patch_func(new_func, force_patch)
+
+    @staticmethod
+    def apply_patches():
+        for patch in MindSpeedPatchesManager.patches_info.values():
+            patch.apply_patch()

evaluation/requirements_sim.txt

@@ -0,0 +1,7 @@
+fire
+omegaconf==2.0.6
+s3prl==0.4.1
+numpy==1.21
+fairseq==0.12.2
+torch==2.1.0
+torchaudio==2.1.0

evaluation/requirements_wer.txt

@@ -0,0 +1,12 @@
+#fire
+#s3prl==0.4.1
+#hydra-core==1.0.7
+#fairseq==0.12.2
+transformers
+funasr==1.2.7
+zhconv
+zhon
+whisper_normalizer
+torch==2.1.0
+torchaudio==2.1.0
+lingvo

install_requirements.sh

@@ -0,0 +1,5 @@
+# to resolve deps conflicts of fairseq and f5-tts
+pip install pip==23.1.2
+pip install setuptools==65.5.1
+pip install f5-tts==0.3.4
+pip install -r requirements_npu.txt

reconstuction_example.py

@@ -0,0 +1,178 @@
+# Copyright (C) 2025. Huawei Technologies Co., Ltd. All Rights Reserved. (authors: Xiao Chen)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import librosa
+import logging
+import soundfile as sf
+import sys
+from pathlib import Path
+
+
+sub_modules = ["", "semantic_tokenizer/f40ms", "semantic_detokenizer"]
+for sub in sub_modules:
+    sys.path.append(str((Path(__file__).parent / sub).absolute()))
+
+from semantic_tokenizer.f40ms.simple_tokenizer_infer import SpeechTokenizer, TOKENIZER_CFG_NAME
+from semantic_detokenizer.chunk_infer import SpeechDetokenizer
+
+
+class ReconstructionPipeline:
+    def __init__(
+        self,
+        detok_vocoder: str,
+        tokenizer_cfg_name: str = TOKENIZER_CFG_NAME,
+        tokenizer_cfg_path: str = str(
+            (Path(__file__).parent / "semantic_tokenizer/f40ms/config").absolute()
+        ),
+        tokenizer_ckpt: str = str(
+            (
+                Path(__file__).parent / "semantic_tokenizer/f40ms/ckpt/model.pt"
+            ).absolute()
+        ),
+        detok_model_cfg: str = str(
+            (Path(__file__).parent / "semantic_detokenizer/ckpt/model.yaml").absolute()
+        ),
+        detok_ckpt: str = str(
+            (Path(__file__).parent / "semantic_detokenizer/ckpt/model.pt").absolute()
+        ),
+        detok_vocab: str = str(
+            (
+                Path(__file__).parent / "semantic_detokenizer/ckpt/vocab_4096.txt"
+            ).absolute()
+        ),
+    ):
+        self.tokenizer_cfg_name = tokenizer_cfg_name
+        self.tokenizer = SpeechTokenizer(
+            ckpt_path=tokenizer_ckpt,
+            cfg_path=tokenizer_cfg_path,
+            cfg_name=self.tokenizer_cfg_name,
+        )
+
+        self.device = "cuda:0"
+        self.detoker = SpeechDetokenizer(
+            vocoder_path=detok_vocoder,
+            model_cfg=detok_model_cfg,
+            ckpt_file=detok_ckpt,
+            vocab_file=detok_vocab,
+            device=self.device,
+        )
+
+        self.token_chunk_len = 75
+        self.chunk_cond_proportion = 0.3
+        self.chunk_look_ahead = 10
+        self.max_ref_duration = 4.5
+        self.ref_audio_cut_from_head = False
+
+    def reconstruct(self, ref_wav, input_wav):
+        ref_wavs_list = []
+        raw_ref_wav, sr = librosa.load(ref_wav, sr=16000)
+        ref_wavs_list.append(raw_ref_wav)
+
+        raw_input_wav, sr = librosa.load(input_wav, sr=16000)
+        ref_wavs_list.append(raw_input_wav)
+
+        token_list, token_info_list = self.tokenizer.extract(
+            ref_wavs_list
+        )
+        ref_tokens = token_info_list[0]["reduced_unit_sequence"]
+        input_tokens = token_info_list[1]["reduced_unit_sequence"]
+        logging.info("tokens for ref wav: %s are [%s]" % (ref_wav, ref_tokens))
+        logging.info("tokens for input wav: %s are [%s]" % (input_wav, input_tokens))
+
+        generated_wave, target_sample_rate = self.detoker.chunk_generate(
+            ref_wav,
+            ref_tokens.split(),
+            input_tokens.split(),
+            self.token_chunk_len,
+            self.chunk_cond_proportion,
+            self.chunk_look_ahead,
+            self.max_ref_duration,
+            self.ref_audio_cut_from_head,
+        )
+
+        if generated_wave is None:
+            logging.info("generation FAILED")
+            return None, None
+        return generated_wave, target_sample_rate
+
+
+def main(args):
+    # initialize
+    reconsturctor = ReconstructionPipeline(
+        detok_vocoder=args.detok_vocoder,
+    )
+
+    generated_wave, target_sample_rate = reconsturctor.reconstruct(args.ref_wav, args.input_wav)
+    with open(args.output_wav, "wb") as f:
+        sf.write(f.name, generated_wave, target_sample_rate)
+        logging.info(f"write output to: {f.name}")
+
+    logging.info("Finished")
+    return
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--tokenizer-ckpt",
+        required=False,
+        help="path to ckpt",
+    )
+    parser.add_argument(
+        "--tokenizer-cfg-path",
+        required=False,
+        default="semantic_tokenizer/f40ms/config",
+        help="path to config",
+    )
+    parser.add_argument(
+        "--detok-ckpt",
+        required=False,
+        help="path to ckpt",
+    )
+    parser.add_argument(
+        "--detok-model-cfg",
+        required=False,
+        help="path to model_cfg",
+    )
+    parser.add_argument(
+        "--detok-vocab",
+        required=False,
+        help="path to vocab",
+    )
+    parser.add_argument(
+        "--detok-vocoder",
+        required=True,
+        help="path to vocoder",
+    )
+    parser.add_argument(
+        "--ref-wav",
+        required=True,
+        help="path to ref wav",
+    )
+    parser.add_argument(
+        "--output-wav",
+        required=True,
+        help="path to output reconstructed wav",
+    )
+    parser.add_argument(
+        "--input-wav",
+        required=True,
+        help="input wav to reconstruction",
+    )
+
+    args = parser.parse_args()
+
+    main(args)

requirements_npu.txt

@@ -0,0 +1,18 @@
+librosa==0.9.2
+torch==2.5.1
+torch-npu==2.5.1
+torchaudio==2.5.1
+torchvision==0.20.1
+omegaconf==2.0.6
+fairseq==0.12.2
+Cython
+numpy==1.23.4
+g2p_en
+jieba_fast
+jieba
+LangSegment>=0.2.0
+wordsegment
+pypinyin
+cn2an
+g2pM
+WeTextProcessing

semantic_detokenizer/chunk_infer.py

@@ -0,0 +1,478 @@
+# Copyright (C) 2025. Huawei Technologies Co., Ltd. All Rights Reserved. (authors: Dehua Tao,
+#                                                                                  Xiao Chen)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import os
+from datetime import datetime
+from importlib.resources import files
+from pathlib import Path
+import tqdm
+
+import soundfile as sf
+import time
+
+from omegaconf import OmegaConf
+import torchaudio
+import torch.multiprocessing as mp
+
+from f5_tts.infer.utils_infer import (
+    load_model,
+    load_vocoder,
+    remove_silence_for_generated_wav,
+)
+from utils_infer import (
+    mel_spec_type,
+    target_rms,
+    nfe_step,
+    cfg_strength,
+    sway_sampling_coef,
+    speed,
+    fix_duration,
+    chunk_infer_batch_process
+)
+from model.cadit import CADiT
+
+import logging
+console_format = logging.Formatter(
+    "[%(asctime)s][%(filename)s:%(levelname)s][%(process)d:%(threadName)s]%(message)s"
+)
+console_handler = logging.StreamHandler()
+console_handler.setFormatter(console_format)
+console_handler.setLevel(logging.INFO)
+if len(logging.root.handlers) > 0:
+    for handler in logging.root.handlers:
+        logging.root.removeHandler(handler)
+logging.root.addHandler(console_handler)
+logging.root.setLevel(logging.INFO)
+
+
+TOKENIZE_ON_NPU = os.environ.get("TOKENIZE_ON_NPU")
+if TOKENIZE_ON_NPU is not None and TOKENIZE_ON_NPU == "1":
+    import torch_npu
+    import f5tts_npu_patch
+    from torch_npu.contrib import transfer_to_npu
+
+    logging.info("Applying Patches for NPU!!!")
+    f5tts_npu_patch.patch_for_npu()
+
+
+class SpeechDetokenizer:
+    def __init__(self, 
+                 vocoder_path:str, 
+                 model_cfg:str = str((Path(__file__).parent / "ckpt/model.yaml").absolute()), 
+                 ckpt_file:str = str((Path(__file__).parent / "ckpt/model.pt").absolute()), 
+                 vocab_file:str = str((Path(__file__).parent / "ckpt/vocab_4096.txt").absolute()), 
+                 device="cuda:0"):
+        self.model_cfg = model_cfg
+        self.ckpt_file = ckpt_file
+        self.vocab_file = vocab_file
+        self.vocoder_path = vocoder_path
+        self.device = device
+
+        self.cross_fade_duration = 0
+        self.initialize()
+
+    def initialize(self):
+        self.model = "CADiT"  # "F5TTS"
+        load_vocoder_from_local = True
+
+        self.vocoder_name = mel_spec_type
+        # Vocoder
+        vocoder_local_path = self.vocoder_path # "/home/ma-user/work/chenxiao/workspace/model/vocos_mel_24khz/"
+
+        # TTS model
+        model_cls = CADiT
+        model_cfg = OmegaConf.load(self.model_cfg).model.arch
+        logging.info(f"Using {self.model}...")
+
+        # Load vocoder
+        self.vocoder = load_vocoder(
+            vocoder_name=self.vocoder_name,
+            is_local=load_vocoder_from_local,
+            local_path=vocoder_local_path,
+            device=self.device,
+        )
+
+        # Load TTS model
+        self.ema_model = load_model(
+            model_cls,
+            model_cfg,
+            self.ckpt_file,
+            mel_spec_type=self.vocoder_name,
+            vocab_file=self.vocab_file,
+            device=self.device,
+        )
+
+    # def chunk_text(self, text_list, chunk_len=135, merge_short_last=False):
+    #     """
+    #     Splits the input text into chunks, each with a maximum number of characters.
+
+    #     Args:
+    #         text (str): The text to be split.
+    #         max_chars (int): The maximum number of characters per chunk.
+
+    #     Returns:
+    #         List[str]: A list of text chunks.
+    #     """
+    #     chunks = []
+
+    #     # if isinstance(text, list):
+    #     for i in range(0, len(text_list), chunk_len):
+    #         chunks.append(text_list[i : i + chunk_len])
+
+    #     if merge_short_last and len(chunks) >= 2 and len(chunks[-1]) < chunk_len:
+    #         # Merge the last two chunks
+    #         last = chunks.pop()
+    #         second_last = chunks.pop()
+    #         chunks.append(second_last + last)
+
+    #     return chunks
+
+
+    def chunk_text_with_look_ahead(
+        self, text_list, chunk_look_ahead_len, chunk_len=135, merge_short_last=False
+    ):
+        chunks = []
+
+        stride = chunk_len - chunk_look_ahead_len
+        for i in range(0, len(text_list), stride):
+            chk = text_list[i : i + chunk_len]
+            chunks.append(chk)
+            if i + chunk_len >= len(text_list):
+                break
+
+        if (
+            merge_short_last
+            and len(chunks) >= 2
+            and len(chunks[-1]) < stride # chunk_len * 4 / 5
+        ):
+            # Merge the last two chunks
+            last = chunks.pop()
+            second_last = chunks.pop()
+            if chunk_look_ahead_len <= 0:
+                chunks.append(second_last + last)
+            else:
+                chunks.append(second_last[:-chunk_look_ahead_len] + last)
+
+        actual_chunks = []
+        for idx in range(len(chunks)):
+            chk = chunks[idx]
+            if chunk_look_ahead_len <= 0:
+                actual_chunks.extend(chk)
+            else:
+                if idx < len(chunks) - 1:
+                    actual_chunks.extend(chk[:-chunk_look_ahead_len])
+                else:
+                    actual_chunks.extend(chk)
+
+        assert(len(actual_chunks) == len(text_list))
+        assert(actual_chunks == text_list)
+        return chunks
+
+    def chunk_generate(
+        self,
+        ref_audio,
+        ref_text_list,
+        gen_text_list,
+        token_chunk_len,
+        chunk_cond_proportion,
+        chunk_look_ahead_len=0,
+        max_ref_duration=4.5,
+        ref_head_cut=False,
+    ):
+
+        gen_text_batches = self.chunk_text_with_look_ahead(
+            gen_text_list,
+            chunk_look_ahead_len,
+            chunk_len=token_chunk_len,
+            merge_short_last=True,
+        )
+
+        if len(gen_text_batches) == 0:
+            return None, None
+
+        for i, gen_text in enumerate(gen_text_batches):
+            logging.info(f"gen_text {i} with {len(gen_text)} tokens : {gen_text}")
+
+        audio, sr = torchaudio.load(ref_audio)
+        logging.info(f"Generating audio in {len(gen_text_batches)} batches...")
+
+        target_wave, target_sample_rate, combined_spectrogram = chunk_infer_batch_process(
+            (audio, sr),
+            ref_text_list,
+            gen_text_batches,
+            self.ema_model,
+            self.vocoder,
+            mel_spec_type=mel_spec_type,
+            progress=tqdm,
+            target_rms=target_rms,
+            cross_fade_duration=self.cross_fade_duration,
+            nfe_step=nfe_step,
+            cfg_strength=cfg_strength,
+            sway_sampling_coef=sway_sampling_coef,
+            speed=speed,
+            fix_duration=fix_duration,
+            device=self.device,
+            chunk_cond_proportion=chunk_cond_proportion,
+            chunk_look_ahead=chunk_look_ahead_len,
+            max_ref_duration=max_ref_duration,
+            ref_head_cut=ref_head_cut,
+        )
+        return target_wave, target_sample_rate
+
+
+def get_audio_duration(audio_path):
+    audio, sample_rate = torchaudio.load(audio_path)
+    return audio.shape[1] / sample_rate
+
+
+def get_test_list(testset_path):
+    testset_file_path = testset_path
+    testset_list = []  # list of [ref_text, ref_audio, gen_text, gen_audio(optional)]
+
+    with open(testset_file_path, "r") as f:
+        for line in f:
+            content = line.strip().split("|")
+            if len(content) == 2 or len(content) == 3:
+                testset_list.append([content[1], content[0], content[1]])
+            elif len(content) == 4 or len(content) == 5:
+                testset_list.append([content[1], content[0], content[3], content[2]])
+    return testset_list
+
+
+def infer(args, task_queue, rank=0):
+    device_spec = f"cuda:{rank}"
+
+    if args.model_cfg is None or args.ckpt is None or args.vocab is None:
+        detoker = SpeechDetokenizer(
+            vocoder_path=args.vocoder,
+            device=device_spec,
+        )
+    else:
+        detoker = SpeechDetokenizer(
+            vocoder_path=args.vocoder,
+            model_cfg=args.model_cfg,
+            ckpt_file=args.ckpt,
+            vocab_file=args.vocab,
+            device=device_spec,
+        )
+
+    token_chunk_len = args.chunk_token
+    chunk_cond_proportion = args.chunk_cond_portion
+    if chunk_cond_proportion > 1 or chunk_cond_proportion <= 0:
+        chunk_cond_proportion = 0.5 # set default
+
+    chunk_look_ahead = args.chunk_look_ahead
+    if chunk_look_ahead >= token_chunk_len:
+        chunk_look_ahead = 0
+
+    remove_silence = False
+
+    output_dir = args.output
+    if not os.path.exists(Path(output_dir)):
+        os.makedirs(Path(output_dir))
+
+    # logging.info(f"Using {model}...")
+    logging.info(f"infer with chunk of {token_chunk_len} tokens")
+    logging.info(f"the last {chunk_cond_proportion} of each chunk added into condition")
+    logging.info(f"Using the last {chunk_look_ahead} tokens as look ahead")
+
+    gen_nums = 0
+    while True:
+        try:
+            _tst = task_queue.get()
+            if _tst is None:
+                logging.info("FINISH processing all inputs")
+                break
+
+            ref_text_list = _tst[0].split()
+            ref_audio = _tst[1]
+            gen_text_list = _tst[2].split()
+
+            if len(_tst) == 4:
+                gen_audio = _tst[3]
+            else:
+                gen_audio = None
+
+            ref_wave_path = (
+                Path(output_dir) / f"{ref_audio.split('/')[-1].split('.')[0]}_ref.wav"
+            )
+            if gen_audio is None:
+                gen_wave_path = (
+                    Path(output_dir)
+                    / f"{ref_audio.split('/')[-1].split('.')[0]}_gen.wav"
+                )
+                orig_wave_path = None
+            else:
+                gen_wave_path = (
+                    Path(output_dir)
+                    / f"{gen_audio.split('/')[-1].split('.')[0]}_gen.wav"
+                )
+                orig_wave_path = (
+                    Path(output_dir)
+                    / f"{gen_audio.split('/')[-1].split('.')[0]}_orig.wav"
+                )
+
+            if os.path.exists(gen_wave_path):
+                logging.info(f"{gen_wave_path} already exist, skip")
+                continue
+
+            if not os.path.exists(ref_wave_path):
+                os.system(f"cp {ref_audio} {ref_wave_path}")
+
+            if gen_audio is not None and os.path.exists(gen_audio) and orig_wave_path:
+                os.system(f"cp {gen_audio} {orig_wave_path}")
+
+            generated_wave, target_sample_rate = detoker.chunk_generate(
+                ref_audio,
+                ref_text_list,
+                gen_text_list,
+                token_chunk_len,
+                chunk_cond_proportion,
+                chunk_look_ahead,
+                args.max_ref_duration,
+                args.ref_audio_cut_from_head,
+            )
+
+            if generated_wave is None:
+                continue
+
+            with open(gen_wave_path, "wb") as f:
+                sf.write(f.name, generated_wave, target_sample_rate)
+                # Remove silence
+                if remove_silence:
+                    remove_silence_for_generated_wav(f.name)
+                logging.info(f"write output to: {f.name}")
+
+            gen_nums += 1
+        # if gen_nums >= 10:
+        #     break
+        except:
+            logging.info(f"Fail to get new task")
+
+
+def run_infer_mp(args):
+
+    device_list = [0] 
+    if "CUDA_VISIBLE_DEVICES" in os.environ:
+        device_list = [int(x.strip()) for x in os.environ["CUDA_VISIBLE_DEVICES"].split(",")]
+    elif "ASCEND_RT_VISIBLE_DEVICES" in os.environ:
+        device_list = [int(x.strip()) for x in os.environ["ASCEND_RT_VISIBLE_DEVICES"].split(",")]
+
+    logging.info(f"Using devices: {device_list}")
+    n_procs = len(device_list)
+
+    # load testset
+    testset_list = get_test_list(args.testset_path)
+
+    ctx = mp.get_context("spawn")
+    with ctx.Manager() as manager:
+        task_queue = manager.Queue()
+        for task in testset_list:
+            task_queue.put(task)
+
+        processes = []
+        for idx in range(n_procs):
+            task_queue.put(None)
+            rank = idx  # device_list[idx]
+            p = mp.Process(target=infer, args=(args, task_queue, rank))
+            p.start()
+            processes.append(p)
+
+        for p in processes:
+            p.join()
+
+    os.system(f"cp {args.testset_path} {args.output}")
+    logging.info(f"Finish processing of {n_procs}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--ckpt",
+        required=False,
+        help="path to ckpt",
+    )
+    parser.add_argument(
+        "--model-cfg",
+        required=False,
+        help="path to model_cfg",
+    )
+    parser.add_argument(
+        "--vocab",
+        required=False,
+        help="path to vocab",
+    )
+    parser.add_argument(
+        "--vocoder",
+        required=True,
+        help="path to vocoder",
+    )
+    parser.add_argument(
+        "--testset",
+        dest="testset_path",
+        required=True,
+        help="path of testset file",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="path to output generated audio",
+    )
+    parser.add_argument(
+        "--chunk-token",
+        required=True,
+        type=int,
+        default=25,
+        help="max number of tokens in a chunk",
+    )
+    parser.add_argument(
+        "--chunk-look-ahead",
+        required=False,
+        type=int,
+        default=0,
+        help="number of tokens in a chunk as look ahead",
+    )
+    parser.add_argument(
+        "--chunk-cond-portion",
+        required=True,
+        type=float,
+        default=25,
+        help="the portion at the tail of the prev chunk as condition",
+    )
+    parser.add_argument(
+        "--max-ref-duration",
+        required=False,
+        type=float,
+        default=4.5,
+        help="the max duration of ref audio in seconds",
+    )
+    parser.add_argument(
+        "--ref-audio-cut-from-head",
+        default=False,
+        action="store_true",
+        help="cut ref audio from head, if not set, from tail by default",
+    )
+
+    args = parser.parse_args()
+
+    start_time = time.perf_counter()
+
+    run_infer_mp(args)
+
+    end_time = time.perf_counter()
+    logging.info("processig time: %f sec\n" % (end_time - start_time))
+    logging.info(f"Finished! output to : {args.output}")

semantic_detokenizer/ckpt/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f99c218589e92c399190175273fa179d3e0ce6661469e6c19cce1fd4cab6abc
+size 6507564685

semantic_detokenizer/ckpt/model.pt.md5

@@ -0,0 +1 @@
+7c64bfb2fefb99ac48e7d6bfd72732e3  model.pt

semantic_detokenizer/ckpt/model.yaml

@@ -0,0 +1,51 @@
+hydra:
+  run:
+    # _${datasets.name}/
+    dir: ckpts/${model.name}_${model.mel_spec.mel_spec_type}_${model.tokenizer}/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    
+datasets:
+  name: LibriTTS_960_WenetSpeech4TTS_Premium_Standard_TTSDatawithMultiStyleEmotion_Datatang_Haitianruisheng_Qingshu_XiaoyiF143_Bani  # dataset name
+  path: /home/ma-user/work/dehua/unit2speech/u2s_training_data/40ms_imedia_45wh_hubert_large_FSQ_8888_CTC_sampled_EN_CH_4000h_wulan_H20_mt_1280k_lr_3e-4_d2v_phase_nomask_20241018_8p_fromDaxin/LibriTTS_WenetSpeech4TTS_TTSDatawithMultiStyleEmotion_XiaoyiF143_Bani/train-960_Premium_Standard_datatang_haitianruisheng_qingshu_freetalk_style1_all-three
+  batch_size_per_gpu: 2000 # 38400  # 8 GPUs, 8 * 38400 = 307200
+  batch_size_type: frame  # "frame" or "sample"
+  max_samples: 4 # 64  # max sequences per batch if use frame-wise batch_size. we set 32 for small models, 64 for base models
+  num_workers: 2
+
+optim:
+  epochs: 1000 # 15
+  learning_rate: 7.5e-5
+  num_warmup_updates: 20000  # warmup steps
+  grad_accumulation_steps: 1  # note: updates = steps / grad_accumulation_steps
+  max_grad_norm: 1.0  # gradient clipping
+  bnb_optimizer: False  # use bnb 8bit AdamW optimizer or not
+
+model:
+  name: CADiT_Base_train_40ms_imedia_45wh_HuBERT_large_FSQ_8888_CTC_sampled_EN_CH_4000h_CTC_LibriTTS_960_WenetSpeech4TTS_Premium_Standard_TTSDatawithMultiStyleEmotion_Datatang_Haitianruisheng_Qingshu_XiaoyiF143_Bani_Shanghai_V100_5n8g_20250414  # model name
+  tokenizer: custom  # pinyin; tokenizer type
+  # if tokenizer = 'custom', define the path to the tokenizer you want to use (should be vocab.txt)
+  tokenizer_path: /home/ma-user/work/dehua/unit2speech/u2s_training_data/40ms_imedia_45wh_hubert_large_FSQ_8888_CTC_sampled_EN_CH_4000h_wulan_H20_mt_1280k_lr_3e-4_d2v_phase_nomask_20241018_8p_fromDaxin/LibriTTS_WenetSpeech4TTS_TTSDatawithMultiStyleEmotion_XiaoyiF143_Bani/train-960_Premium_Standard_datatang_haitianruisheng_qingshu_freetalk_style1_all-three/vocab.txt
+  arch:
+    dim: 1024
+    depth: 22
+    heads: 16
+    ff_mult: 2
+    text_dim: 512
+    should_extend_text: True
+    conv_layers: 4
+    checkpoint_activations: False  # recompute activations and save memory for extra compute
+  mel_spec:
+    target_sample_rate: 24000
+    n_mel_channels: 100
+    hop_length: 256
+    win_length: 1024
+    n_fft: 1024
+    mel_spec_type: vocos  # 'vocos' or 'bigvgan'
+  vocoder:
+    is_local: True  # use local offline ckpt or not
+    local_path: /home/ma-user/work/dehua/unit2speech/CA-F5-TTS/pretrained_vocoder/charactr/vocos-mel-24khz  # local vocoder path
+
+ckpts:
+  logger: tensorboard  # wandb | tensorboard | None
+  save_per_updates: 50000 # save checkpoint per steps
+  last_per_steps: 2000 # 5000 # save last checkpoint per steps
+  save_dir: ckpts/${model.name}_${model.mel_spec.mel_spec_type}_${model.tokenizer} # _${datasets.name}

semantic_detokenizer/ckpt/vocab_4096.txt

@@ -0,0 +1,4096 @@
+0
+1
+10
+100
+1000
+1001
+1002
+1003
+1004
+1005
+1006
+1007
+1008
+1009
+101
+1010
+1011
+1012
+1013
+1014
+1015
+1016
+1017
+1018
+1019
+102
+1020
+1021
+1022
+1023
+1024
+1025
+1026
+1027
+1028
+1029
+103
+1030
+1031
+1032
+1033
+1034
+1035
+1036
+1037
+1038
+1039
+104
+1040
+1041
+1042
+1043
+1044
+1045
+1046
+1047
+1048
+1049
+105
+1050
+1051
+1052
+1053
+1054
+1055
+1056
+1057
+1058
+1059
+106
+1060
+1061
+1062
+1063
+1064
+1065
+1066
+1067
+1068
+1069
+107
+1070
+1071
+1072
+1073
+1074
+1075
+1076
+1077
+1078
+1079
+108
+1080
+1081
+1082
+1083
+1084
+1085
+1086
+1087
+1088
+1089
+109
+1090
+1091
+1092
+1093
+1094
+1095
+1096
+1097
+1098
+1099
+11
+110
+1100
+1101
+1102
+1103
+1104
+1105
+1106
+1107
+1108
+1109
+111
+1110
+1111
+1112
+1113
+1114
+1115
+1116
+1117
+1118
+1119
+112
+1120
+1121
+1122
+1123
+1124
+1125
+1126
+1127
+1128
+1129
+113
+1130
+1131
+1132
+1133
+1134
+1135
+1136
+1137
+1138
+1139
+114
+1140
+1141
+1142
+1143
+1144
+1145
+1146
+1147
+1148
+1149
+115
+1150
+1151
+1152
+1153
+1154
+1155
+1156
+1157
+1158
+1159
+116
+1160
+1161
+1162
+1163
+1164
+1165
+1166
+1167
+1168
+1169
+117
+1170
+1171
+1172
+1173
+1174
+1175
+1176
+1177
+1178
+1179
+118
+1180
+1181
+1182
+1183
+1184
+1185
+1186
+1187
+1188
+1189
+119
+1190
+1191
+1192
+1193
+1194
+1195
+1196
+1197
+1198
+1199
+12
+120
+1200
+1201
+1202
+1203
+1204
+1205
+1206
+1207
+1208
+1209
+121
+1210
+1211
+1212
+1213
+1214
+1215
+1216
+1217
+1218
+1219
+122
+1220
+1221
+1222
+1223
+1224
+1225
+1226
+1227
+1228
+1229
+123
+1230
+1231
+1232
+1233
+1234
+1235
+1236
+1237
+1238
+1239
+124
+1240
+1241
+1242
+1243
+1244
+1245
+1246
+1247
+1248
+1249
+125
+1250
+1251
+1252
+1253
+1254
+1255
+1256
+1257
+1258
+1259
+126
+1260
+1261
+1262
+1263
+1264
+1265
+1266
+1267
+1268
+1269
+127
+1270
+1271
+1272
+1273
+1274
+1275
+1276
+1277
+1278
+1279
+128
+1280
+1281
+1282
+1283
+1284
+1285
+1286
+1287
+1288
+1289
+129
+1290
+1291
+1292
+1293
+1294
+1295
+1296
+1297
+1298
+1299
+13
+130
+1300
+1301
+1302
+1303
+1304
+1305
+1306
+1307
+1308
+1309
+131
+1310
+1311
+1312
+1313
+1314
+1315
+1316
+1317
+1318
+1319
+132
+1320
+1321
+1322
+1323
+1324
+1325
+1326
+1327
+1328
+1329
+133
+1330
+1331
+1332
+1333
+1334
+1335
+1336
+1337
+1338
+1339
+134
+1340
+1341
+1342
+1343
+1344
+1345
+1346
+1347
+1348
+1349
+135
+1350
+1351
+1352
+1353
+1354
+1355
+1356
+1357
+1358
+1359
+136
+1360
+1361
+1362
+1363
+1364
+1365
+1366
+1367
+1368
+1369
+137
+1370
+1371
+1372
+1373
+1374
+1375
+1376
+1377
+1378
+1379
+138
+1380
+1381
+1382
+1383
+1384
+1385
+1386
+1387
+1388
+1389
+139
+1390
+1391
+1392
+1393
+1394
+1395
+1396
+1397
+1398
+1399
+14
+140
+1400
+1401
+1402
+1403
+1404
+1405
+1406
+1407
+1408
+1409
+141
+1410
+1411
+1412
+1413
+1414
+1415
+1416
+1417
+1418
+1419
+142
+1420
+1421
+1422
+1423
+1424
+1425
+1426
+1427
+1428
+1429
+143
+1430
+1431
+1432
+1433
+1434
+1435
+1436
+1437
+1438
+1439
+144
+1440
+1441
+1442
+1443
+1444
+1445
+1446
+1447
+1448
+1449
+145
+1450
+1451
+1452
+1453
+1454
+1455
+1456
+1457
+1458
+1459
+146
+1460
+1461
+1462
+1463
+1464
+1465
+1466
+1467
+1468
+1469
+147
+1470
+1471
+1472
+1473
+1474
+1475
+1476
+1477
+1478
+1479
+148
+1480
+1481
+1482
+1483
+1484
+1485
+1486
+1487
+1488
+1489
+149
+1490
+1491
+1492
+1493
+1494
+1495
+1496
+1497
+1498
+1499
+15
+150
+1500
+1501
+1502
+1503
+1504
+1505
+1506
+1507
+1508
+1509
+151
+1510
+1511
+1512
+1513
+1514
+1515
+1516
+1517
+1518
+1519
+152
+1520
+1521
+1522
+1523
+1524
+1525
+1526
+1527
+1528
+1529
+153
+1530
+1531
+1532
+1533
+1534
+1535
+1536
+1537
+1538
+1539
+154
+1540
+1541
+1542
+1543
+1544
+1545
+1546
+1547
+1548
+1549
+155
+1550
+1551
+1552
+1553
+1554
+1555
+1556
+1557
+1558
+1559
+156
+1560
+1561
+1562
+1563
+1564
+1565
+1566
+1567
+1568
+1569
+157
+1570
+1571
+1572
+1573
+1574
+1575
+1576
+1577
+1578
+1579
+158
+1580
+1581
+1582
+1583
+1584
+1585
+1586
+1587
+1588
+1589
+159
+1590
+1591
+1592
+1593
+1594
+1595
+1596
+1597
+1598
+1599
+16
+160
+1600
+1601
+1602
+1603
+1604
+1605
+1606
+1607
+1608
+1609
+161
+1610
+1611
+1612
+1613
+1614
+1615
+1616
+1617
+1618
+1619
+162
+1620
+1621
+1622
+1623
+1624
+1625
+1626
+1627
+1628
+1629
+163
+1630
+1631
+1632
+1633
+1634
+1635
+1636
+1637
+1638
+1639
+164
+1640
+1641
+1642
+1643
+1644
+1645
+1646
+1647
+1648
+1649
+165
+1650
+1651
+1652
+1653
+1654
+1655
+1656
+1657
+1658
+1659
+166
+1660
+1661
+1662
+1663
+1664
+1665
+1666
+1667
+1668
+1669
+167
+1670
+1671
+1672
+1673
+1674
+1675
+1676
+1677
+1678
+1679
+168
+1680
+1681
+1682
+1683
+1684
+1685
+1686
+1687
+1688
+1689
+169
+1690
+1691
+1692
+1693
+1694
+1695
+1696
+1697
+1698
+1699
+17
+170
+1700
+1701
+1702
+1703
+1704
+1705
+1706
+1707
+1708
+1709
+171
+1710
+1711
+1712
+1713
+1714
+1715
+1716
+1717
+1718
+1719
+172
+1720
+1721
+1722
+1723
+1724
+1725
+1726
+1727
+1728
+1729
+173
+1730
+1731
+1732
+1733
+1734
+1735
+1736
+1737
+1738
+1739
+174
+1740
+1741
+1742
+1743
+1744
+1745
+1746
+1747
+1748
+1749
+175
+1750
+1751
+1752
+1753
+1754
+1755
+1756
+1757
+1758
+1759
+176
+1760
+1761
+1762
+1763
+1764
+1765
+1766
+1767
+1768
+1769
+177
+1770
+1771
+1772
+1773
+1774
+1775
+1776
+1777
+1778
+1779
+178
+1780
+1781
+1782
+1783
+1784
+1785
+1786
+1787
+1788
+1789
+179
+1790
+1791
+1792
+1793
+1794
+1795
+1796
+1797
+1798
+1799
+18
+180
+1800
+1801
+1802
+1803
+1804
+1805
+1806
+1807
+1808
+1809
+181
+1810
+1811
+1812
+1813
+1814
+1815
+1816
+1817
+1818
+1819
+182
+1820
+1821
+1822
+1823
+1824
+1825
+1826
+1827
+1828
+1829
+183
+1830
+1831
+1832
+1833
+1834
+1835
+1836
+1837
+1838
+1839
+184
+1840
+1841
+1842
+1843
+1844
+1845
+1846
+1847
+1848
+1849
+185
+1850
+1851
+1852
+1853
+1854
+1855
+1856
+1857
+1858
+1859
+186
+1860
+1861
+1862
+1863
+1864
+1865
+1866
+1867
+1868
+1869
+187
+1870
+1871
+1872
+1873
+1874
+1875
+1876
+1877
+1878
+1879
+188
+1880
+1881
+1882
+1883
+1884
+1885
+1886
+1887
+1888
+1889
+189
+1890
+1891
+1892
+1893
+1894
+1895
+1896
+1897
+1898
+1899
+19
+190
+1900
+1901
+1902
+1903
+1904
+1905
+1906
+1907
+1908
+1909
+191
+1910
+1911
+1912
+1913
+1914
+1915
+1916
+1917
+1918
+1919
+192
+1920
+1921
+1922
+1923
+1924
+1925
+1926
+1927
+1928
+1929
+193
+1930
+1931
+1932
+1933
+1934
+1935
+1936
+1937
+1938
+1939
+194
+1940
+1941
+1942
+1943
+1944
+1945
+1946
+1947
+1948
+1949
+195
+1950
+1951
+1952
+1953
+1954
+1955
+1956
+1957
+1958
+1959
+196
+1960
+1961
+1962
+1963
+1964
+1965
+1966
+1967
+1968
+1969
+197
+1970
+1971
+1972
+1973
+1974
+1975
+1976
+1977
+1978
+1979
+198
+1980
+1981
+1982
+1983
+1984
+1985
+1986
+1987
+1988
+1989
+199
+1990
+1991
+1992
+1993
+1994
+1995
+1996
+1997
+1998
+1999
+2
+20
+200
+2000
+2001
+2002
+2003
+2004
+2005
+2006
+2007
+2008
+2009
+201
+2010
+2011
+2012
+2013
+2014
+2015
+2016
+2017
+2018
+2019
+202
+2020
+2021
+2022
+2023
+2024
+2025
+2026
+2027
+2028
+2029
+203
+2030
+2031
+2032
+2033
+2034
+2035
+2036
+2037
+2038
+2039
+204
+2040
+2041
+2042
+2043
+2044
+2045
+2046
+2047
+2048
+2049
+205
+2050
+2051
+2052
+2053
+2054
+2055
+2056
+2057
+2058
+2059
+206
+2060
+2061
+2062
+2063
+2064
+2065
+2066
+2067
+2068
+2069
+207
+2070
+2071
+2072
+2073
+2074
+2075
+2076
+2077
+2078
+2079
+208
+2080
+2081
+2082
+2083
+2084
+2085
+2086
+2087
+2088
+2089
+209
+2090
+2091
+2092
+2093
+2094
+2095
+2096
+2097
+2098
+2099
+21
+210
+2100
+2101
+2102
+2103
+2104
+2105
+2106
+2107
+2108
+2109
+211
+2110
+2111
+2112
+2113
+2114
+2115
+2116
+2117
+2118
+2119
+212
+2120
+2121
+2122
+2123
+2124
+2125
+2126
+2127
+2128
+2129
+213
+2130
+2131
+2132
+2133
+2134
+2135
+2136
+2137
+2138
+2139
+214
+2140
+2141
+2142
+2143
+2144
+2145
+2146
+2147
+2148
+2149
+215
+2150
+2151
+2152
+2153
+2154
+2155
+2156
+2157
+2158
+2159
+216
+2160
+2161
+2162
+2163
+2164
+2165
+2166
+2167
+2168
+2169
+217
+2170
+2171
+2172
+2173
+2174
+2175
+2176
+2177
+2178
+2179
+218
+2180
+2181
+2182
+2183
+2184
+2185
+2186
+2187
+2188
+2189
+219
+2190
+2191
+2192
+2193
+2194
+2195
+2196
+2197
+2198
+2199
+22
+220
+2200
+2201
+2202
+2203
+2204
+2205
+2206
+2207
+2208
+2209
+221
+2210
+2211
+2212
+2213
+2214
+2215
+2216
+2217
+2218
+2219
+222
+2220
+2221
+2222
+2223
+2224
+2225
+2226
+2227
+2228
+2229
+223
+2230
+2231
+2232
+2233
+2234
+2235
+2236
+2237
+2238
+2239
+224
+2240
+2241
+2242
+2243
+2244
+2245
+2246
+2247
+2248
+2249
+225
+2250
+2251
+2252
+2253
+2254
+2255
+2256
+2257
+2258
+2259
+226
+2260
+2261
+2262
+2263
+2264
+2265
+2266
+2267
+2268
+2269
+227
+2270
+2271
+2272
+2273
+2274
+2275
+2276
+2277
+2278
+2279
+228
+2280
+2281
+2282
+2283
+2284
+2285
+2286
+2287
+2288
+2289
+229
+2290
+2291
+2292
+2293
+2294
+2295
+2296
+2297
+2298
+2299
+23
+230
+2300
+2301
+2302
+2303
+2304
+2305
+2306
+2307
+2308
+2309
+231
+2310
+2311
+2312
+2313
+2314
+2315
+2316
+2317
+2318
+2319
+232
+2320
+2321
+2322
+2323
+2324
+2325
+2326
+2327
+2328
+2329
+233
+2330
+2331
+2332
+2333
+2334
+2335
+2336
+2337
+2338
+2339
+234
+2340
+2341
+2342
+2343
+2344
+2345
+2346
+2347
+2348
+2349
+235
+2350
+2351
+2352
+2353
+2354
+2355
+2356
+2357
+2358
+2359
+236
+2360
+2361
+2362
+2363
+2364
+2365
+2366
+2367
+2368
+2369
+237
+2370
+2371
+2372
+2373
+2374
+2375
+2376
+2377
+2378
+2379
+238
+2380
+2381
+2382
+2383
+2384
+2385
+2386
+2387
+2388
+2389
+239
+2390
+2391
+2392
+2393
+2394
+2395
+2396
+2397
+2398
+2399
+24
+240
+2400
+2401
+2402
+2403
+2404
+2405
+2406
+2407
+2408
+2409
+241
+2410
+2411
+2412
+2413
+2414
+2415
+2416
+2417
+2418
+2419
+242
+2420
+2421
+2422
+2423
+2424
+2425
+2426
+2427
+2428
+2429
+243
+2430
+2431
+2432
+2433
+2434
+2435
+2436
+2437
+2438
+2439
+244
+2440
+2441
+2442
+2443
+2444
+2445
+2446
+2447
+2448
+2449
+245
+2450
+2451
+2452
+2453
+2454
+2455
+2456
+2457
+2458
+2459
+246
+2460
+2461
+2462
+2463
+2464
+2465
+2466
+2467
+2468
+2469
+247
+2470
+2471
+2472
+2473
+2474
+2475
+2476
+2477
+2478
+2479
+248
+2480
+2481
+2482
+2483
+2484
+2485
+2486
+2487
+2488
+2489
+249
+2490
+2491
+2492
+2493
+2494
+2495
+2496
+2497
+2498
+2499
+25
+250
+2500
+2501
+2502
+2503
+2504
+2505
+2506
+2507
+2508
+2509
+251
+2510
+2511
+2512
+2513
+2514
+2515
+2516
+2517
+2518
+2519
+252
+2520
+2521
+2522
+2523
+2524
+2525
+2526
+2527
+2528
+2529
+253
+2530
+2531
+2532
+2533
+2534
+2535
+2536
+2537
+2538
+2539
+254
+2540
+2541
+2542
+2543
+2544
+2545
+2546
+2547
+2548
+2549
+255
+2550
+2551
+2552
+2553
+2554
+2555
+2556
+2557
+2558
+2559
+256
+2560
+2561
+2562
+2563
+2564
+2565
+2566
+2567
+2568
+2569
+257
+2570
+2571
+2572
+2573
+2574
+2575
+2576
+2577
+2578
+2579
+258
+2580
+2581
+2582
+2583
+2584
+2585
+2586
+2587
+2588
+2589
+259
+2590
+2591
+2592
+2593
+2594
+2595
+2596
+2597
+2598
+2599
+26
+260
+2600
+2601
+2602
+2603
+2604
+2605
+2606
+2607
+2608
+2609
+261
+2610
+2611
+2612
+2613
+2614
+2615
+2616
+2617
+2618
+2619
+262
+2620
+2621
+2622
+2623
+2624
+2625
+2626
+2627
+2628
+2629
+263
+2630
+2631
+2632
+2633
+2634
+2635
+2636
+2637
+2638
+2639
+264
+2640
+2641
+2642
+2643
+2644
+2645
+2646
+2647
+2648
+2649
+265
+2650
+2651
+2652
+2653
+2654
+2655
+2656
+2657
+2658
+2659
+266
+2660
+2661
+2662
+2663
+2664
+2665
+2666
+2667
+2668
+2669
+267
+2670
+2671
+2672
+2673
+2674
+2675
+2676
+2677
+2678
+2679
+268
+2680
+2681
+2682
+2683
+2684
+2685
+2686
+2687
+2688
+2689
+269
+2690
+2691
+2692
+2693
+2694
+2695
+2696
+2697
+2698
+2699
+27
+270
+2700
+2701
+2702
+2703
+2704
+2705
+2706
+2707
+2708
+2709
+271
+2710
+2711
+2712
+2713
+2714
+2715
+2716
+2717
+2718
+2719
+272
+2720
+2721
+2722
+2723
+2724
+2725
+2726
+2727
+2728
+2729
+273
+2730
+2731
+2732
+2733
+2734
+2735
+2736
+2737
+2738
+2739
+274
+2740
+2741
+2742
+2743
+2744
+2745
+2746
+2747
+2748
+2749
+275
+2750
+2751
+2752
+2753
+2754
+2755
+2756
+2757
+2758
+2759
+276
+2760
+2761
+2762
+2763
+2764
+2765
+2766
+2767
+2768
+2769
+277
+2770
+2771
+2772
+2773
+2774
+2775
+2776
+2777
+2778
+2779
+278
+2780
+2781
+2782
+2783
+2784
+2785
+2786
+2787
+2788
+2789
+279
+2790
+2791
+2792
+2793
+2794
+2795
+2796
+2797
+2798
+2799
+28
+280
+2800
+2801
+2802
+2803
+2804
+2805
+2806
+2807
+2808
+2809
+281
+2810
+2811
+2812
+2813
+2814
+2815
+2816
+2817
+2818
+2819
+282
+2820
+2821
+2822
+2823
+2824
+2825
+2826
+2827
+2828
+2829
+283
+2830
+2831
+2832
+2833
+2834
+2835
+2836
+2837
+2838
+2839
+284
+2840
+2841
+2842
+2843
+2844
+2845
+2846
+2847
+2848
+2849
+285
+2850
+2851
+2852
+2853
+2854
+2855
+2856
+2857
+2858
+2859
+286
+2860
+2861
+2862
+2863
+2864
+2865
+2866
+2867
+2868
+2869
+287
+2870
+2871
+2872
+2873
+2874
+2875
+2876
+2877
+2878
+2879
+288
+2880
+2881
+2882
+2883
+2884
+2885
+2886
+2887
+2888
+2889
+289
+2890
+2891
+2892
+2893
+2894
+2895
+2896
+2897
+2898
+2899
+29
+290
+2900
+2901
+2902
+2903
+2904
+2905
+2906
+2907
+2908
+2909
+291
+2910
+2911
+2912
+2913
+2914
+2915
+2916
+2917
+2918
+2919
+292
+2920
+2921
+2922
+2923
+2924
+2925
+2926
+2927
+2928
+2929
+293
+2930
+2931
+2932
+2933
+2934
+2935
+2936
+2937
+2938
+2939
+294
+2940
+2941
+2942
+2943
+2944
+2945
+2946
+2947
+2948
+2949
+295
+2950
+2951
+2952
+2953
+2954
+2955
+2956
+2957
+2958
+2959
+296
+2960
+2961
+2962
+2963
+2964
+2965
+2966
+2967
+2968
+2969
+297
+2970
+2971
+2972
+2973
+2974
+2975
+2976
+2977
+2978
+2979
+298
+2980
+2981
+2982
+2983
+2984
+2985
+2986
+2987
+2988
+2989
+299
+2990
+2991
+2992
+2993
+2994
+2995
+2996
+2997
+2998
+2999
+3
+30
+300
+3000
+3001
+3002
+3003
+3004
+3005
+3006
+3007
+3008
+3009
+301
+3010
+3011
+3012
+3013
+3014
+3015
+3016
+3017
+3018
+3019
+302
+3020
+3021
+3022
+3023
+3024
+3025
+3026
+3027
+3028
+3029
+303
+3030
+3031
+3032
+3033
+3034
+3035
+3036
+3037
+3038
+3039
+304
+3040
+3041
+3042
+3043
+3044
+3045
+3046
+3047
+3048
+3049
+305
+3050
+3051
+3052
+3053
+3054
+3055
+3056
+3057
+3058
+3059
+306
+3060
+3061
+3062
+3063
+3064
+3065
+3066
+3067
+3068
+3069
+307
+3070
+3071
+3072
+3073
+3074
+3075
+3076
+3077
+3078
+3079
+308
+3080
+3081
+3082
+3083
+3084
+3085
+3086
+3087
+3088
+3089
+309
+3090
+3091
+3092
+3093
+3094
+3095
+3096
+3097
+3098
+3099
+31
+310
+3100
+3101
+3102
+3103
+3104
+3105
+3106
+3107
+3108
+3109
+311
+3110
+3111
+3112
+3113
+3114
+3115
+3116
+3117
+3118
+3119
+312
+3120
+3121
+3122
+3123
+3124
+3125
+3126
+3127
+3128
+3129
+313
+3130
+3131
+3132
+3133
+3134
+3135
+3136
+3137
+3138
+3139
+314
+3140
+3141
+3142
+3143
+3144
+3145
+3146
+3147
+3148
+3149
+315
+3150
+3151
+3152
+3153
+3154
+3155
+3156
+3157
+3158
+3159
+316
+3160
+3161
+3162
+3163
+3164
+3165
+3166
+3167
+3168
+3169
+317
+3170
+3171
+3172
+3173
+3174
+3175
+3176
+3177
+3178
+3179
+318
+3180
+3181
+3182
+3183
+3184
+3185
+3186
+3187
+3188
+3189
+319
+3190
+3191
+3192
+3193
+3194
+3195
+3196
+3197
+3198
+3199
+32
+320
+3200
+3201
+3202
+3203
+3204
+3205
+3206
+3207
+3208
+3209
+321
+3210
+3211
+3212
+3213
+3214
+3215
+3216
+3217
+3218
+3219
+322
+3220
+3221
+3222
+3223
+3224
+3225
+3226
+3227
+3228
+3229
+323
+3230
+3231
+3232
+3233
+3234
+3235
+3236
+3237
+3238
+3239
+324
+3240
+3241
+3242
+3243
+3244
+3245
+3246
+3247
+3248
+3249
+325
+3250
+3251
+3252
+3253
+3254
+3255
+3256
+3257
+3258
+3259
+326
+3260
+3261
+3262
+3263
+3264
+3265
+3266
+3267
+3268
+3269
+327
+3270
+3271
+3272
+3273
+3274
+3275
+3276
+3277
+3278
+3279
+328
+3280
+3281
+3282
+3283
+3284
+3285
+3286
+3287
+3288
+3289
+329
+3290
+3291
+3292
+3293
+3294
+3295
+3296
+3297
+3298
+3299
+33
+330
+3300
+3301
+3302
+3303
+3304
+3305
+3306
+3307
+3308
+3309
+331
+3310
+3311
+3312
+3313
+3314
+3315
+3316
+3317
+3318
+3319
+332
+3320
+3321
+3322
+3323
+3324
+3325
+3326
+3327
+3328
+3329
+333
+3330
+3331
+3332
+3333
+3334
+3335
+3336
+3337
+3338
+3339
+334
+3340
+3341
+3342
+3343
+3344
+3345
+3346
+3347
+3348
+3349
+335
+3350
+3351
+3352
+3353
+3354
+3355
+3356
+3357
+3358
+3359
+336
+3360
+3361
+3362
+3363
+3364
+3365
+3366
+3367
+3368
+3369
+337
+3370
+3371
+3372
+3373
+3374
+3375
+3376
+3377
+3378
+3379
+338
+3380
+3381
+3382
+3383
+3384
+3385
+3386
+3387
+3388
+3389
+339
+3390
+3391
+3392
+3393
+3394
+3395
+3396
+3397
+3398
+3399
+34
+340
+3400
+3401
+3402
+3403
+3404
+3405
+3406
+3407
+3408
+3409
+341
+3410
+3411
+3412
+3413
+3414
+3415
+3416
+3417
+3418
+3419
+342
+3420
+3421
+3422
+3423
+3424
+3425
+3426
+3427
+3428
+3429
+343
+3430
+3431
+3432
+3433
+3434
+3435
+3436
+3437
+3438
+3439
+344
+3440
+3441
+3442
+3443
+3444
+3445
+3446
+3447
+3448
+3449
+345
+3450
+3451
+3452
+3453
+3454
+3455
+3456
+3457
+3458
+3459
+346
+3460
+3461
+3462
+3463
+3464
+3465
+3466
+3467
+3468
+3469
+347
+3470
+3471
+3472
+3473
+3474
+3475
+3476
+3477
+3478
+3479
+348
+3480
+3481
+3482
+3483
+3484
+3485
+3486
+3487
+3488
+3489
+349
+3490
+3491
+3492
+3493
+3494
+3495
+3496
+3497
+3498
+3499
+35
+350
+3500
+3501
+3502
+3503
+3504
+3505
+3506
+3507
+3508
+3509
+351
+3510
+3511
+3512
+3513
+3514
+3515
+3516
+3517
+3518
+3519
+352
+3520
+3521
+3522
+3523
+3524
+3525
+3526
+3527
+3528
+3529
+353
+3530
+3531
+3532
+3533
+3534
+3535
+3536
+3537
+3538
+3539
+354
+3540
+3541
+3542
+3543
+3544
+3545
+3546
+3547
+3548
+3549
+355
+3550
+3551
+3552
+3553
+3554
+3555
+3556
+3557
+3558
+3559
+356
+3560
+3561
+3562
+3563
+3564
+3565
+3566
+3567
+3568
+3569
+357
+3570
+3571
+3572
+3573
+3574
+3575
+3576
+3577
+3578
+3579
+358
+3580
+3581
+3582
+3583
+3584
+3585
+3586
+3587
+3588
+3589
+359
+3590
+3591
+3592
+3593
+3594
+3595
+3596
+3597
+3598
+3599
+36
+360
+3600
+3601
+3602
+3603
+3604
+3605
+3606
+3607
+3608
+3609
+361
+3610
+3611
+3612
+3613
+3614
+3615
+3616
+3617
+3618
+3619
+362
+3620
+3621
+3622
+3623
+3624
+3625
+3626
+3627
+3628
+3629
+363
+3630
+3631
+3632
+3633
+3634
+3635
+3636
+3637
+3638
+3639
+364
+3640
+3641
+3642
+3643
+3644
+3645
+3646
+3647
+3648
+3649
+365
+3650
+3651
+3652
+3653
+3654
+3655
+3656
+3657
+3658
+3659
+366
+3660
+3661
+3662
+3663
+3664
+3665
+3666
+3667
+3668
+3669
+367
+3670
+3671
+3672
+3673
+3674
+3675
+3676
+3677
+3678
+3679
+368
+3680
+3681
+3682
+3683
+3684
+3685
+3686
+3687
+3688
+3689
+369
+3690
+3691
+3692
+3693
+3694
+3695
+3696
+3697
+3698
+3699
+37
+370
+3700
+3701
+3702
+3703
+3704
+3705
+3706
+3707
+3708
+3709
+371
+3710
+3711
+3712
+3713
+3714
+3715
+3716
+3717
+3718
+3719
+372
+3720
+3721
+3722
+3723
+3724
+3725
+3726
+3727
+3728
+3729
+373
+3730
+3731
+3732
+3733
+3734
+3735
+3736
+3737
+3738
+3739
+374
+3740
+3741
+3742
+3743
+3744
+3745
+3746
+3747
+3748
+3749
+375
+3750
+3751
+3752
+3753
+3754
+3755
+3756
+3757
+3758
+3759
+376
+3760
+3761
+3762
+3763
+3764
+3765
+3766
+3767
+3768
+3769
+377
+3770
+3771
+3772
+3773
+3774
+3775
+3776
+3777
+3778
+3779
+378
+3780
+3781
+3782
+3783
+3784
+3785
+3786
+3787
+3788
+3789
+379
+3790
+3791
+3792
+3793
+3794
+3795
+3796
+3797
+3798
+3799
+38
+380
+3800
+3801
+3802
+3803
+3804
+3805
+3806
+3807
+3808
+3809
+381
+3810
+3811
+3812
+3813
+3814
+3815
+3816
+3817
+3818
+3819
+382
+3820
+3821
+3822
+3823
+3824
+3825
+3826
+3827
+3828
+3829
+383
+3830
+3831
+3832
+3833
+3834
+3835
+3836
+3837
+3838
+3839
+384
+3840
+3841
+3842
+3843
+3844
+3845
+3846
+3847
+3848
+3849
+385
+3850
+3851
+3852
+3853
+3854
+3855
+3856
+3857
+3858
+3859
+386
+3860
+3861
+3862
+3863
+3864
+3865
+3866
+3867
+3868
+3869
+387
+3870
+3871
+3872
+3873
+3874
+3875
+3876
+3877
+3878
+3879
+388
+3880
+3881
+3882
+3883
+3884
+3885
+3886
+3887
+3888
+3889
+389
+3890
+3891
+3892
+3893
+3894
+3895
+3896
+3897
+3898
+3899
+39
+390
+3900
+3901
+3902
+3903
+3904
+3905
+3906
+3907
+3908
+3909
+391
+3910
+3911
+3912
+3913
+3914
+3915
+3916
+3917
+3918
+3919
+392
+3920
+3921
+3922
+3923
+3924
+3925
+3926
+3927
+3928
+3929
+393
+3930
+3931
+3932
+3933
+3934
+3935
+3936
+3937
+3938
+3939
+394
+3940
+3941
+3942
+3943
+3944
+3945
+3946
+3947
+3948
+3949
+395
+3950
+3951
+3952
+3953
+3954
+3955
+3956
+3957
+3958
+3959
+396
+3960
+3961
+3962
+3963
+3964
+3965
+3966
+3967
+3968
+3969
+397
+3970
+3971
+3972
+3973
+3974
+3975
+3976
+3977
+3978
+3979
+398
+3980
+3981
+3982
+3983
+3984
+3985
+3986
+3987
+3988
+3989
+399
+3990
+3991
+3992
+3993
+3994
+3995
+3996
+3997
+3998
+3999
+4
+40
+400
+4000
+4001
+4002
+4003
+4004
+4005
+4006
+4007
+4008
+4009
+401
+4010
+4011
+4012
+4013
+4014
+4015
+4016
+4017
+4018
+4019
+402
+4020
+4021
+4022
+4023
+4024
+4025
+4026
+4027
+4028
+4029
+403
+4030
+4031
+4032
+4033
+4034
+4035
+4036
+4037
+4038
+4039
+404
+4040
+4041
+4042
+4043
+4044
+4045
+4046
+4047
+4048
+4049
+405
+4050
+4051
+4052
+4053
+4054
+4055
+4056
+4057
+4058
+4059
+406
+4060
+4061
+4062
+4063
+4064
+4065
+4066
+4067
+4068
+4069
+407
+4070
+4071
+4072
+4073
+4074
+4075
+4076
+4077
+4078
+4079
+408
+4080
+4081
+4082
+4083
+4084
+4085
+4086
+4087
+4088
+4089
+409
+4090
+4091
+4092
+4093
+4094
+4095
+41
+410
+411
+412
+413
+414
+415
+416
+417
+418
+419
+42
+420
+421
+422
+423
+424
+425
+426
+427
+428
+429
+43
+430
+431
+432
+433
+434
+435
+436
+437
+438
+439
+44
+440
+441
+442
+443
+444
+445
+446
+447
+448
+449
+45
+450
+451
+452
+453
+454
+455
+456
+457
+458
+459
+46
+460
+461
+462
+463
+464
+465
+466
+467
+468
+469
+47
+470
+471
+472
+473
+474
+475
+476
+477
+478
+479
+48
+480
+481
+482
+483
+484
+485
+486
+487
+488
+489
+49
+490
+491
+492
+493
+494
+495
+496
+497
+498
+499
+5
+50
+500
+501
+502
+503
+504
+505
+506
+507
+508
+509
+51
+510
+511
+512
+513
+514
+515
+516
+517
+518
+519
+52
+520
+521
+522
+523
+524
+525
+526
+527
+528
+529
+53
+530
+531
+532
+533
+534
+535
+536
+537
+538
+539
+54
+540
+541
+542
+543
+544
+545
+546
+547
+548
+549
+55
+550
+551
+552
+553
+554
+555
+556
+557
+558
+559
+56
+560
+561
+562
+563
+564
+565
+566
+567
+568
+569
+57
+570
+571
+572
+573
+574
+575
+576
+577
+578
+579
+58
+580
+581
+582
+583
+584
+585
+586
+587
+588
+589
+59
+590
+591
+592
+593
+594
+595
+596
+597
+598
+599
+6
+60
+600
+601
+602
+603
+604
+605
+606
+607
+608
+609
+61
+610
+611
+612
+613
+614
+615
+616
+617
+618
+619
+62
+620
+621
+622
+623
+624
+625
+626
+627
+628
+629
+63
+630
+631
+632
+633
+634
+635
+636
+637
+638
+639
+64
+640
+641
+642
+643
+644
+645
+646
+647
+648
+649
+65
+650
+651
+652
+653
+654
+655
+656
+657
+658
+659
+66
+660
+661
+662
+663
+664
+665
+666
+667
+668
+669
+67
+670
+671
+672
+673
+674
+675
+676
+677
+678
+679
+68
+680
+681
+682
+683
+684
+685
+686
+687
+688
+689
+69
+690
+691
+692
+693
+694
+695
+696
+697
+698
+699
+7
+70
+700
+701
+702
+703
+704
+705
+706
+707
+708
+709
+71
+710
+711
+712
+713
+714
+715
+716
+717
+718
+719
+72
+720
+721
+722
+723
+724
+725
+726
+727
+728
+729
+73
+730
+731
+732
+733
+734
+735
+736
+737
+738
+739
+74
+740
+741
+742
+743
+744
+745
+746
+747
+748
+749
+75
+750
+751
+752
+753
+754
+755
+756
+757
+758
+759
+76
+760
+761
+762
+763
+764
+765
+766
+767
+768
+769
+77
+770
+771
+772
+773
+774
+775
+776
+777
+778
+779
+78
+780
+781
+782
+783
+784
+785
+786
+787
+788
+789
+79
+790
+791
+792
+793
+794
+795
+796
+797
+798
+799
+8
+80
+800
+801
+802
+803
+804
+805
+806
+807
+808
+809
+81
+810
+811
+812
+813
+814
+815
+816
+817
+818
+819
+82
+820
+821
+822
+823
+824
+825
+826
+827
+828
+829
+83
+830
+831
+832
+833
+834
+835
+836
+837
+838
+839
+84
+840
+841
+842
+843
+844
+845
+846
+847
+848
+849
+85
+850
+851
+852
+853
+854
+855
+856
+857
+858
+859
+86
+860
+861
+862
+863
+864
+865
+866
+867
+868
+869
+87
+870
+871
+872
+873
+874
+875
+876
+877
+878
+879
+88
+880
+881
+882
+883
+884
+885
+886
+887
+888
+889
+89
+890
+891
+892
+893
+894
+895
+896
+897
+898
+899
+9
+90
+900
+901
+902
+903
+904
+905
+906
+907
+908
+909
+91
+910
+911
+912
+913
+914
+915
+916
+917
+918
+919
+92
+920
+921
+922
+923
+924
+925
+926
+927
+928
+929
+93
+930
+931
+932
+933
+934
+935
+936
+937
+938
+939
+94
+940
+941
+942
+943
+944
+945
+946
+947
+948
+949
+95
+950
+951
+952
+953
+954
+955
+956
+957
+958
+959
+96
+960
+961
+962
+963
+964
+965
+966
+967
+968
+969
+97
+970
+971
+972
+973
+974
+975
+976
+977
+978
+979
+98
+980
+981
+982
+983
+984
+985
+986
+987
+988
+989
+99
+990
+991
+992
+993
+994
+995
+996
+997
+998
+999

semantic_detokenizer/f5tts_npu_patch.py

@@ -0,0 +1,106 @@
+# Copyright (C) 2025. Huawei Technologies Co., Ltd. All Rights Reserved. (authors: Xiao Chen)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from patch_utils import MindSpeedPatchesManager as aspm
+import torchaudio
+import torch
+import logging
+
+def get_vocos_mel_spectrogram_cpu(
+    waveform,
+    n_fft=1024,
+    n_mel_channels=100,
+    target_sample_rate=24000,
+    hop_length=256,
+    win_length=1024,
+):
+    wave_device = waveform.device
+    waveform = waveform.cpu()
+    mel_stft = torchaudio.transforms.MelSpectrogram(
+        sample_rate=target_sample_rate,
+        n_fft=n_fft,
+        win_length=win_length,
+        hop_length=hop_length,
+        n_mels=n_mel_channels,
+        power=1,
+        center=True,
+        normalized=False,
+        norm=None,
+    ).to(waveform.device)
+    if len(waveform.shape) == 3:
+        waveform = waveform.squeeze(1)  # 'b 1 nw -> b nw'
+
+    assert len(waveform.shape) == 2
+
+    mel = mel_stft(waveform)
+    mel = mel.clamp(min=1e-5).log()
+    waveform = waveform.to(wave_device)
+    mel = mel.to(wave_device)
+    return mel
+
+
+def load_checkpoint_npu(model, ckpt_path, device: str, dtype=None, use_ema=True):
+    logging.info(f"Load checkpoint {ckpt_path}")
+    if dtype is None:
+        dtype = (
+            torch.float16
+            if "cuda" in device or "npu" in device
+            and torch.cuda.get_device_properties(device).major >= 6
+            and not torch.cuda.get_device_name().endswith("[ZLUDA]")
+            else torch.float32
+        )
+    model = model.to(dtype)
+
+    ckpt_type = ckpt_path.split(".")[-1]
+    if ckpt_type == "safetensors":
+        from safetensors.torch import load_file
+
+        checkpoint = load_file(ckpt_path, device=device)
+    else:
+        checkpoint = torch.load(ckpt_path, map_location=device, weights_only=True)
+
+    if use_ema:
+        if ckpt_type == "safetensors":
+            checkpoint = {"ema_model_state_dict": checkpoint}
+        checkpoint["model_state_dict"] = {
+            k.replace("ema_model.", ""): v
+            for k, v in checkpoint["ema_model_state_dict"].items()
+            if k not in ["initted", "step"]
+        }
+
+        # patch for backward compatibility, 305e3ea
+        for key in [
+            "mel_spec.mel_stft.mel_scale.fb",
+            "mel_spec.mel_stft.spectrogram.window",
+        ]:
+            if key in checkpoint["model_state_dict"]:
+                del checkpoint["model_state_dict"][key]
+
+        model.load_state_dict(checkpoint["model_state_dict"])
+    else:
+        if ckpt_type == "safetensors":
+            checkpoint = {"model_state_dict": checkpoint}
+        model.load_state_dict(checkpoint["model_state_dict"])
+
+    del checkpoint
+    torch.cuda.empty_cache()
+
+    return model.to(device)
+
+def patch_for_npu():
+    # replace torch.cuda.get_device_capability with implementation from MindSpeed
+    aspm.register_patch('f5_tts.infer.utils_infer.load_checkpoint', load_checkpoint_npu)
+    aspm.register_patch('f5_tts.model.modules.get_vocos_mel_spectrogram', get_vocos_mel_spectrogram_cpu)
+    aspm.apply_patches()

semantic_detokenizer/model/cadit.py

@@ -0,0 +1,263 @@
+# Copyright (C) 2025. Huawei Technologies Co., Ltd. All Rights Reserved. (authors: Dehua Tao)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+ein notation:
+b - batch
+n - sequence
+nt - text sequence
+nw - raw wave length
+d - dimension
+"""
+
+from __future__ import annotations
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+from torchaudio.models import Conformer
+
+from x_transformers.x_transformers import RotaryEmbedding
+
+from f5_tts.model.modules import (
+    TimestepEmbedding,
+    ConvNeXtV2Block,
+    ConvPositionEmbedding,
+    AdaLayerNormZero_Final,
+    precompute_freqs_cis,
+    get_pos_embed_indices,
+)
+
+from model.modules import CADiTBlock
+
+import logging
+
+# Text embedding
+class TextEmbedding(nn.Module):
+    def __init__(
+        self,
+        text_num_embeds,
+        text_dim,
+        should_extend_text=True,
+        conv_layers=0,
+        conv_mult=2,
+    ):
+        super().__init__()
+        self.text_embed = nn.Embedding(
+            text_num_embeds + 1, text_dim
+        )  # use 0 as filler token
+
+        self.should_extend_text = should_extend_text
+        logging.info(f"should_extend_text={should_extend_text}")
+
+        if conv_layers > 0:
+            self.extra_modeling = True
+            self.precompute_max_pos = 4096  # ~44s of 24khz audio
+            self.register_buffer(
+                "freqs_cis",
+                precompute_freqs_cis(text_dim, self.precompute_max_pos),
+                persistent=False,
+            )
+            self.text_blocks = nn.Sequential(
+                *[
+                    ConvNeXtV2Block(text_dim, text_dim * conv_mult)
+                    for _ in range(conv_layers)
+                ]
+            )
+            # # Can be deleted
+            # self.text_blocks = Conformer(
+            #     input_dim=text_dim,
+            #     num_heads=8,  # Not sure it is good
+            #     ffn_dim=text_dim * conv_mult,
+            #     num_layers=conv_layers,
+            #     depthwise_conv_kernel_size=7,  # See ConvNeXtV2Block
+            # )
+        else:
+            self.extra_modeling = False
+
+    def forward(self, text: int["b nt"], seq_len, drop_text=False):  # noqa: F722
+        text = (
+            text + 1
+        )  # use 0 as filler token. preprocess of batch pad -1, see list_str_to_idx()
+        text = text[
+            :, :seq_len
+        ]  # curtail if character tokens are more than the mel spec tokens
+        batch, text_len = text.shape[0], text.shape[1]
+
+        if self.should_extend_text:
+            text = F.pad(text, (0, seq_len - text_len), value=0)
+        else:
+            seq_len = text_len
+
+        if drop_text:  # cfg for text
+            text = torch.zeros_like(text)
+
+        text = self.text_embed(text)  # b n -> b n d
+
+        # possible extra modeling
+        if self.extra_modeling:
+            # sinus pos emb
+            batch_start = torch.zeros((batch,), dtype=torch.long)
+            pos_idx = get_pos_embed_indices(
+                batch_start, seq_len, max_pos=self.precompute_max_pos
+            )
+            text_pos_embed = self.freqs_cis[pos_idx]
+            text = text + text_pos_embed
+
+            # convnextv2 blocks
+            text = self.text_blocks(text)
+
+            # # Can be deleted
+            # # conformer blocks
+            # lengths = torch.Tensor([text.size(1)] * text.size(0)).to(text.device)
+            # text, _ = self.text_blocks(text, lengths)
+
+        return text
+
+
+# noised input audio embedding
+
+
+class InputAudioEmbedding(nn.Module):
+    def __init__(self, mel_dim, out_dim):
+        super().__init__()
+        self.proj = nn.Linear(mel_dim * 2, out_dim)
+        self.conv_pos_embed = ConvPositionEmbedding(dim=out_dim)
+
+    def forward(
+        self,
+        x: float["b n d"],
+        cond: float["b n d"],
+        drop_audio_cond=False,
+    ):  # noqa: F722
+        if drop_audio_cond:  # cfg for cond audio
+            cond = torch.zeros_like(cond)
+
+        x = self.proj(torch.cat((x, cond), dim=-1))
+        x = self.conv_pos_embed(x) + x
+        return x
+
+
+# Transformer backbone using cross-attention DiT blocks
+
+
+class CADiT(nn.Module):
+    def __init__(
+        self,
+        *,
+        dim,
+        depth=8,
+        heads=8,
+        dim_head=64,
+        dropout=0.1,
+        ff_mult=4,
+        mel_dim=100,
+        text_num_embeds=256,
+        text_dim=None,
+        should_extend_text=True,
+        conv_layers=0,
+        long_skip_connection=False,
+        checkpoint_activations=False,
+    ):
+        super().__init__()
+
+        self.time_embed = TimestepEmbedding(dim)
+        if text_dim is None:
+            text_dim = mel_dim
+        self.text_embed = TextEmbedding(
+            text_num_embeds,
+            text_dim,
+            should_extend_text=should_extend_text,
+            conv_layers=conv_layers,
+        )
+
+        # Modification: only concatenate noisy and masked speech
+        self.input_embed = InputAudioEmbedding(mel_dim, dim)
+
+        self.rotary_embed = RotaryEmbedding(dim_head)
+
+        self.dim = dim
+        self.depth = depth
+
+        # Modification: use cross-attention DiT block
+        self.transformer_blocks = nn.ModuleList(
+            [
+                CADiTBlock(
+                    dim=dim,
+                    text_dim=text_dim,
+                    heads=heads,
+                    dim_head=dim_head,
+                    ff_mult=ff_mult,
+                    dropout=dropout,
+                )
+                for _ in range(depth)
+            ]
+        )
+        self.long_skip_connection = (
+            nn.Linear(dim * 2, dim, bias=False) if long_skip_connection else None
+        )
+
+        self.norm_out = AdaLayerNormZero_Final(dim)  # final modulation
+        self.proj_out = nn.Linear(dim, mel_dim)
+
+        self.checkpoint_activations = checkpoint_activations
+
+    def ckpt_wrapper(self, module):
+        # https://github.com/chuanyangjin/fast-DiT/blob/main/models.py
+        def ckpt_forward(*inputs):
+            outputs = module(*inputs)
+            return outputs
+
+        return ckpt_forward
+
+    def forward(
+        self,
+        x: float["b n d"],  # nosied input audio  # noqa: F722
+        cond: float["b n d"],  # masked cond audio  # noqa: F722
+        text: int["b nt"],  # text  # noqa: F722
+        time: float["b"] | float[""],  # time step  # noqa: F821 F722
+        drop_audio_cond,  # cfg for cond audio
+        drop_text,  # cfg for text
+        mask: bool["b n"] | None = None,  # noqa: F722
+    ):
+        batch, seq_len = x.shape[0], x.shape[1]
+        if time.ndim == 0:
+            time = time.repeat(batch)
+
+        # t: conditioning time, x: noised input audio
+        t = self.time_embed(time)
+        text_embed = self.text_embed(text, seq_len, drop_text=drop_text)
+        x = self.input_embed(x, cond, drop_audio_cond=drop_audio_cond)
+
+        rope = self.rotary_embed.forward_from_seq_len(seq_len)
+
+        if self.long_skip_connection is not None:
+            residual = x
+
+        for block in self.transformer_blocks:
+            if self.checkpoint_activations:
+                x = torch.utils.checkpoint.checkpoint(
+                    self.ckpt_wrapper(block), x, text_embed, t, mask, rope
+                )
+            else:
+                x = block(x, text_embed, t, mask=mask, rope=rope)
+
+        if self.long_skip_connection is not None:
+            x = self.long_skip_connection(torch.cat((x, residual), dim=-1))
+
+        x = self.norm_out(x, t)
+        output = self.proj_out(x)
+
+        return output

semantic_detokenizer/model/modules.py

@@ -0,0 +1,223 @@
+# Copyright (C) 2025. Huawei Technologies Co., Ltd. All Rights Reserved. (authors: Dehua Tao)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+ein notation:
+b - batch
+n - sequence
+nt - text sequence
+nw - raw wave length
+d - dimension
+"""
+
+from __future__ import annotations
+
+# import math
+# from typing import Optional
+
+import torch
+import torch.nn.functional as F
+# import torchaudio
+from librosa.filters import mel as librosa_mel_fn
+from torch import nn
+from x_transformers.x_transformers import apply_rotary_pos_emb
+
+mel_basis_cache = {}
+hann_window_cache = {}
+
+from f5_tts.model.modules import AdaLayerNormZero, Attention, AttnProcessor, FeedForward
+
+
+# Cross-attention with audio as query and text as key/value
+
+
+class CrossAttention(nn.Module):
+    def __init__(
+        self,
+        processor: CrossAttnProcessor,
+        dim: int,
+        dim_to_k: int,
+        heads: int = 8,
+        dim_head: int = 64,
+        dropout: float = 0.0,
+    ):
+        super().__init__()
+
+        if not hasattr(F, "scaled_dot_product_attention"):
+            raise ImportError(
+                "Attention equires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0."
+            )
+
+        self.processor = processor
+
+        self.dim = dim
+        self.heads = heads
+        self.inner_dim = dim_head * heads
+        self.dropout = dropout
+
+        self.to_q = nn.Linear(dim, self.inner_dim)
+        self.to_k = nn.Linear(dim_to_k, self.inner_dim)
+        self.to_v = nn.Linear(dim_to_k, self.inner_dim)
+
+        self.to_out = nn.ModuleList([])
+        self.to_out.append(nn.Linear(self.inner_dim, dim))
+        self.to_out.append(nn.Dropout(dropout))
+
+    def forward(
+        self,
+        x_for_q: float["b n d"],  # (noisy + masked) audio input, x_for_q  # noqa: F722
+        x_for_k: float["b n d"] = None,  # text input, x_for_k  # noqa: F722
+        mask: bool["b n"] | None = None,  # noqa: F722
+        rope=None,  # rotary position embedding for x
+    ) -> torch.Tensor:
+        return self.processor(
+            self,
+            x_for_q,
+            x_for_k,
+            mask=mask,
+            rope=rope,
+        )
+
+
+# Cross-attention processor
+
+
+class CrossAttnProcessor:
+    def __init__(self):
+        pass
+
+    def __call__(
+        self,
+        attn: CrossAttention,
+        x_for_q: float["b n d"],  # (noisy + masked) audio input, x_for_q  # noqa: F722
+        x_for_k: float["b n d"],  # text input, x_for_k  # noqa: F722
+        mask: bool["b n"] | None = None,  # noqa: F722
+        rope=None,  # rotary position embedding
+    ) -> torch.FloatTensor:
+        batch_size = x_for_q.shape[0]
+
+        # `sample` projections.
+        query = attn.to_q(x_for_q)
+        key = attn.to_k(x_for_k)
+        value = attn.to_v(x_for_k)
+
+        # apply rotary position embedding
+        if rope is not None:
+            freqs, xpos_scale = rope
+            q_xpos_scale, k_xpos_scale = (
+                (xpos_scale, xpos_scale**-1.0) if xpos_scale is not None else (1.0, 1.0)
+            )
+
+            query = apply_rotary_pos_emb(query, freqs, q_xpos_scale)
+            key = apply_rotary_pos_emb(key, freqs, k_xpos_scale)
+
+        # attention
+        inner_dim = key.shape[-1]
+        head_dim = inner_dim // attn.heads
+        query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+        value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
+
+        # mask. e.g. inference got a batch with different target durations, mask out the padding
+        if mask is not None:
+            attn_mask = mask
+            attn_mask = attn_mask.unsqueeze(1).unsqueeze(1)  # 'b n -> b 1 1 n'
+            attn_mask = attn_mask.expand(
+                batch_size, attn.heads, query.shape[-2], key.shape[-2]
+            )
+        else:
+            attn_mask = None
+
+        x = F.scaled_dot_product_attention(
+            query, key, value, attn_mask=attn_mask, dropout_p=0.0, is_causal=False
+        )
+        x = x.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
+        x = x.to(query.dtype)
+
+        # linear proj
+        x = attn.to_out[0](x)
+        # dropout
+        x = attn.to_out[1](x)
+
+        if mask is not None:
+            mask = mask.unsqueeze(-1)
+            x = x.masked_fill(~mask, 0.0)
+
+        return x
+
+
+# Cross-attention DiT Block
+
+
+class CADiTBlock(nn.Module):
+    def __init__(self, dim, text_dim, heads, dim_head, ff_mult=4, dropout=0.1):
+        super().__init__()
+
+        self.attn_norm = AdaLayerNormZero(dim)
+        self.attn = Attention(
+            processor=AttnProcessor(),
+            dim=dim,
+            heads=heads,
+            dim_head=dim_head,
+            dropout=dropout,
+        )
+
+        self.cross_attn_norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
+        self.cross_attn = CrossAttention(
+            processor=CrossAttnProcessor(),
+            dim=dim,
+            dim_to_k=text_dim,
+            heads=heads,
+            dim_head=dim_head,
+            dropout=dropout,
+        )
+
+        self.ff_norm = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
+        self.ff = FeedForward(
+            dim=dim, mult=ff_mult, dropout=dropout, approximate="tanh"
+        )
+
+    def forward(
+        self,
+        x,
+        y,
+        t,
+        mask=None,
+        rope=None,
+    ):  # x: audio input, y: text input, t: time embedding
+
+        ## for self-attention
+
+        # pre-norm & modulation for attention input
+        norm, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.attn_norm(x, emb=t)
+
+        # attention
+        attn_output = self.attn(x=norm, mask=mask, rope=rope)
+
+        # process attention output for input x
+        x = x + gate_msa.unsqueeze(1) * attn_output
+
+        ## for cross-attention
+        ca_norm = self.cross_attn_norm(x)
+        cross_attn_output = self.cross_attn(ca_norm, y, mask=mask, rope=rope)
+        x = x + cross_attn_output
+
+        norm = self.ff_norm(x) * (1 + scale_mlp[:, None]) + shift_mlp[:, None]
+        ff_output = self.ff(norm)
+        x = x + gate_mlp.unsqueeze(1) * ff_output
+
+        return x
+
+

semantic_detokenizer/patch_utils.py

@@ -0,0 +1,122 @@
+# copied from https://gitee.com/ascend/MindSpeed/blob/master/mindspeed/patch_utils.py
+
+import importlib
+import sys
+import types
+
+
+def get_func_name(func):
+    if isinstance(func, str):
+        return func
+    return '.'.join((func.__module__, func.__qualname__))
+
+
+def dummy_function_wrapper(func_name):
+    def dummy_function(*args, **kwargs):
+        raise RuntimeError('function {} no exist'.format(func_name))
+
+    return dummy_function
+
+
+class Patch:
+    def __init__(self, orig_func_name, new_func, create_dummy):
+        split_name = orig_func_name.rsplit('.', 1)
+        if len(split_name) == 1:
+            self.orig_module_name, self.orig_func_name = orig_func_name, None
+        else:
+            self.orig_module_name, self.orig_func_name = split_name
+        self.orig_module = None
+        self.orig_func = None
+
+        self.patch_func = None
+        self.wrappers = []
+        if new_func is None:
+            new_func = dummy_function_wrapper(orig_func_name)
+        self.set_patch_func(new_func)
+        self.is_applied = False
+        self.create_dummy = create_dummy
+
+    @property
+    def orig_func_id(self):
+        return id(self.orig_func)
+
+    @property
+    def patch_func_id(self):
+        return id(self.patch_func)
+
+    def set_patch_func(self, new_func, force_patch=False):
+        if hasattr(new_func, '__name__') and new_func.__name__.endswith(('wrapper', 'decorator')):
+            self.wrappers.append(new_func)
+        else:
+            if self.patch_func and not force_patch:
+                raise RuntimeError('the patch of {} exist !'.format(self.orig_func_name))
+            self.patch_func = new_func
+        self.is_applied = False
+
+    def apply_patch(self):
+        if self.is_applied:
+            return
+
+        self.orig_module, self.orig_func = Patch.parse_path(self.orig_module_name, self.orig_func_name, self.create_dummy)
+
+        final_patch_func = self.orig_func
+        if self.patch_func is not None:
+            final_patch_func = self.patch_func
+
+        for wrapper in self.wrappers:
+            final_patch_func = wrapper(final_patch_func)
+
+        if self.orig_func_name is not None:
+            setattr(self.orig_module, self.orig_func_name, final_patch_func)
+        for key, value in sys.modules.copy().items():
+            if self.orig_func_name is not None and hasattr(value, self.orig_func_name) \
+                    and id(getattr(value, self.orig_func_name)) == self.orig_func_id:
+                setattr(value, self.orig_func_name, final_patch_func)
+        self.is_applied = True
+
+    @staticmethod
+    def parse_path(module_path, function_name, create_dummy):
+        from importlib.machinery import ModuleSpec
+        modules = module_path.split('.')
+        for i in range(1, len(modules) + 1):
+            parent = '.'.join(modules[:i - 1])
+            path = '.'.join(modules[:i])
+            try:
+                importlib.import_module(path)
+            except ModuleNotFoundError as e:
+                if not parent or not hasattr(importlib.import_module(parent), modules[i - 1]):
+                    if not create_dummy:
+                        raise ModuleNotFoundError(e) from e
+                    sys.modules[path] = types.ModuleType(path)
+                    sys.modules[path].__file__ = 'mindspeed.dummy_module.py'
+                    sys.modules[path].__spec__ = ModuleSpec(path, None)
+                    if parent:
+                        setattr(importlib.import_module(parent), modules[i - 1], sys.modules[path])
+                else:
+                    module = getattr(importlib.import_module(parent), modules[i - 1])
+                    if hasattr(module, function_name):
+                        return module, getattr(module, function_name)
+                    elif create_dummy:
+                        return module, dummy_function_wrapper(function_name)
+                    else:
+                        raise RuntimeError('no exist {} of {}'.format(function_name, module))
+
+        if function_name is not None and not hasattr(sys.modules[module_path], function_name):
+            setattr(sys.modules[module_path], function_name, None)
+        return sys.modules[module_path], getattr(sys.modules[module_path], function_name) if function_name is not None else None
+
+
+class MindSpeedPatchesManager:
+    patches_info = {}
+
+    @staticmethod
+    def register_patch(orig_func_name, new_func=None, force_patch=False, create_dummy=False):
+        if orig_func_name not in MindSpeedPatchesManager.patches_info:
+            MindSpeedPatchesManager.patches_info[orig_func_name] = Patch(orig_func_name, new_func, create_dummy)
+        else:
+            MindSpeedPatchesManager.patches_info.get(orig_func_name).set_patch_func(new_func, force_patch)
+
+    @staticmethod
+    def apply_patches():
+        for patch in MindSpeedPatchesManager.patches_info.values():
+            patch.apply_patch()

semantic_detokenizer/requirements.txt

@@ -0,0 +1,13 @@
+Cython
+numpy==1.23.4
+g2p_en
+jieba_fast
+jieba
+LangSegment>=0.2.0
+wordsegment
+pypinyin
+cn2an
+g2pM
+#torch==2.1.0
+WeTextProcessing
+f5-tts==0.3.4

semantic_detokenizer/utils_infer.py

@@ -0,0 +1,298 @@
+# Copyright (C) 2025. Huawei Technologies Co., Ltd. All Rights Reserved. (authors: Dehua Tao,
+#                                                                                  Xiao Chen)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import sys
+
+os.environ["PYTOCH_ENABLE_MPS_FALLBACK"] = "1"  # for MPS device compatibility
+from importlib.resources import files
+import matplotlib
+
+matplotlib.use("Agg")
+
+
+import numpy as np
+import torch
+import torchaudio
+import tqdm
+import logging
+# torch.set_printoptions(profile="full")
+# from f5_tts.model import CFM
+from f5_tts.model.utils import (
+    get_tokenizer,
+    convert_char_to_pinyin,
+)
+from f5_tts.model.modules import MelSpec
+
+
+device = (
+    "cuda"
+    if torch.cuda.is_available()
+    else (
+        "xpu"
+        if torch.xpu.is_available()
+        else "mps" if torch.backends.mps.is_available() else "cpu"
+    )
+)
+
+# -----------------------------------------
+
+target_sample_rate = 24000
+n_mel_channels = 100
+hop_length = 256
+win_length = 1024
+n_fft = 1024
+mel_spec_type = "vocos"
+target_rms = 0.1
+cross_fade_duration = 0.15
+ode_method = "euler"
+nfe_step = 32  # 16, 32
+cfg_strength = 2.0
+sway_sampling_coef = -1.0
+speed = 1.0
+fix_duration = None
+seed = 3214
+
+# -----------------------------------------
+def chunk_infer_batch_process(
+    ref_audio,
+    ref_text,
+    gen_text_batches,
+    model_obj,
+    vocoder,
+    mel_spec_type="vocos",
+    progress=tqdm,
+    target_rms=0.1,
+    cross_fade_duration=0.15,
+    nfe_step=32,
+    cfg_strength=2.0,
+    sway_sampling_coef=-1.0,
+    speed=1.0,
+    fix_duration=None,
+    device=None,
+    chunk_cond_proportion=0.5,
+    chunk_look_ahead=0,
+    max_ref_duration=4.5,
+    ref_head_cut=False,
+):
+    audio, sr = ref_audio
+    if audio.shape[0] > 1:
+        audio = torch.mean(audio, dim=0, keepdim=True)
+
+    rms = torch.sqrt(torch.mean(torch.square(audio)))
+    if rms < target_rms:
+        audio = audio * target_rms / rms
+    if sr != target_sample_rate:
+        resampler = torchaudio.transforms.Resample(sr, target_sample_rate)
+        audio = resampler(audio)
+
+    logging.info(
+        "audio shape:" + str(audio.shape) + "; ref_text shape:" + str(len(ref_text))
+    )
+    ref_duration = audio.shape[1] / target_sample_rate
+    if ref_duration > max_ref_duration:
+        reserved_ref_audio_len = round(max_ref_duration * target_sample_rate)
+        if ref_head_cut:
+            logging.info(f"Using the first {max_ref_duration} seconds as ref audio")
+            audio = audio[:, :reserved_ref_audio_len]
+            ref_text = ref_text[
+                : round(max_ref_duration * len(ref_text) / ref_duration)
+            ]
+        else:
+            logging.info(f"Using the last {max_ref_duration} seconds as ref audio")
+            audio = audio[:, -reserved_ref_audio_len:]
+            ref_text = ref_text[
+                -round(max_ref_duration * len(ref_text) / ref_duration) :
+            ]
+    logging.info(
+        "audio shape:" + str(audio.shape) + "; ref_text shape:" + str(len(ref_text))
+    )
+    audio = audio.to(device)
+
+    generated_waves = []
+    spectrograms = []
+
+    # fixed_ref_audio_len = audio.shape[-1] // hop_length
+    mel_spec_module = MelSpec(mel_spec_type=mel_spec_type)
+    fixed_ref_audio_mel_spec = mel_spec_module(audio)
+    # The last dim should be num_channels
+    fixed_ref_audio_mel_cond = fixed_ref_audio_mel_spec.permute(0, 2, 1)
+    fixed_ref_audio_len = fixed_ref_audio_mel_cond.shape[1]
+
+    assert isinstance(ref_text, list) is True
+    fixed_ref_text = ref_text[:]
+    fixed_ref_text_len = len(fixed_ref_text)
+
+    mel_cond = fixed_ref_audio_mel_cond.clone()
+
+    prev_chunk_audio_len = 0
+
+    for i, gen_text in enumerate(progress.tqdm(gen_text_batches)):
+        # Prepare the text
+        final_text_list = [ref_text + gen_text]
+        logging.info(f"final_text_list: {final_text_list}")
+
+        if fix_duration is not None:
+            duration = int(fix_duration * target_sample_rate / hop_length)
+        else:
+            # Calculate duration
+            assert isinstance(gen_text, list) is True
+            gen_text_len = len(gen_text)
+            duration = (
+                fixed_ref_audio_len
+                + prev_chunk_audio_len
+                + int(fixed_ref_audio_len / fixed_ref_text_len * gen_text_len / speed)
+            )
+        logging.info(f"Duration: {duration}")
+
+        # inference
+        with torch.inference_mode():
+            logging.info(f"generate with nfe_step:{nfe_step}, cfg_strength:{cfg_strength}, sway_sampling_coef:{sway_sampling_coef}")
+            # logging.info("mel_cond: " + str(mel_cond))
+            generated, _ = model_obj.sample(
+                cond=mel_cond,
+                text=final_text_list,
+                duration=duration,
+                steps=nfe_step,
+                cfg_strength=cfg_strength,
+                sway_sampling_coef=sway_sampling_coef,
+                seed=seed,
+            )
+            generated = generated.to(torch.float32)
+            logging.info("gen mel shape: " + str(generated.shape))
+
+            # try to remove condition mel
+            stripped_generated = generated[
+                :, (fixed_ref_audio_len + prev_chunk_audio_len) :, :
+            ]
+
+            # remove chunk_look_ahead from the tail of each generated mel
+            look_ahead_mel_len = round(
+                (duration - fixed_ref_audio_len - prev_chunk_audio_len)
+                * chunk_look_ahead
+                / len(gen_text)
+            )
+            if look_ahead_mel_len > 0 and i < len(gen_text_batches) - 1:
+                stripped_generated_without_look_ahead = stripped_generated[
+                    :,
+                    :(-look_ahead_mel_len),
+                    :,
+                ]
+                # try to remove the chunk_look_ahead from the tail of gen_text
+                gen_text = gen_text[:-chunk_look_ahead]
+            else:
+                stripped_generated_without_look_ahead = stripped_generated
+            logging.info("gen mel shape: %s, gen text len: %d" % (str(stripped_generated_without_look_ahead.shape), len(gen_text)))
+            # logging.info("generated mel: " + str(generated))
+
+            # prev chunk audio len is the length without fixed condition and chunk look ahead
+            prev_chunk_audio_len = stripped_generated_without_look_ahead.shape[1]
+            # prev_chunk_audio_len_with_look_ahead = stripped_generated.shape[1]
+
+            # generate wav with look ahead
+            generated_mel_spec = stripped_generated_without_look_ahead.permute(0, 2, 1)
+            # generated_mel_spec = stripped_generated.permute(0, 2, 1)
+
+            if mel_spec_type == "vocos":
+                generated_wave = vocoder.decode(generated_mel_spec)
+            elif mel_spec_type == "bigvgan":
+                generated_wave = vocoder(generated_mel_spec)
+
+            # strip look ahead wav from generated wav
+            # if look_ahead_mel_len > 0 and i < len(gen_text_batches) - 1:
+            #     look_ahead_wav_len = round(
+            #         look_ahead_mel_len
+            #         * generated_wave.shape[1]
+            #         / prev_chunk_audio_len_with_look_ahead
+            #     )
+            #     generated_wave = generated_wave[:, :-look_ahead_wav_len]
+
+            if rms < target_rms:
+                generated_wave = generated_wave * rms / target_rms
+
+            logging.info("gen wav shape: " + str(generated_wave.shape))
+            # logging.info("generated wav: " + str(generated_wave))
+
+            # wav -> numpy
+            generated_wave = generated_wave.squeeze().cpu().numpy()
+
+            generated_waves.append(generated_wave)
+            spectrograms.append(generated_mel_spec[0].cpu().numpy())
+
+            prev_chunk_cond_audio_len = round(chunk_cond_proportion * prev_chunk_audio_len)
+            if prev_chunk_audio_len > prev_chunk_cond_audio_len:
+                gen_text_cond = gen_text[-round(chunk_cond_proportion * len(gen_text)):]
+                prev_chunk_audio_len = prev_chunk_cond_audio_len
+                generated_cond = stripped_generated_without_look_ahead[:, (-prev_chunk_audio_len):, :]
+            else:
+                generated_cond = stripped_generated_without_look_ahead
+                gen_text_cond = gen_text
+
+            logging.info("gen text cond len: %d, gen mel cond len: %d" % (len(gen_text_cond), len(generated_cond)))
+
+        ref_text = fixed_ref_text + gen_text_cond
+        mel_cond = torch.cat([fixed_ref_audio_mel_cond, generated_cond], dim=1)
+
+    # Combine all generated waves with cross-fading
+    if cross_fade_duration <= 0:
+        # Simply concatenate
+        logging.info("simply concatenate")
+        final_wave = np.concatenate(generated_waves)
+    else:
+        final_wave = generated_waves[0]
+        for i in range(1, len(generated_waves)):
+            prev_wave = final_wave
+            next_wave = generated_waves[i]
+
+            # Calculate cross-fade samples, ensuring it does not exceed wave lengths
+            cross_fade_samples = int(cross_fade_duration * target_sample_rate)
+            cross_fade_samples = min(cross_fade_samples, len(prev_wave), len(next_wave))
+
+            if cross_fade_samples <= 0:
+                # No overlap possible, concatenate
+                final_wave = np.concatenate([prev_wave, next_wave])
+                continue
+
+            # Overlapping parts
+            prev_overlap = prev_wave[-cross_fade_samples:]
+            next_overlap = next_wave[:cross_fade_samples]
+
+            # Fade out and fade in
+            # fade_out = np.linspace(1, 0, cross_fade_samples)
+            # fade_in = np.linspace(0, 1, cross_fade_samples)
+
+            wave_window = np.hamming(2 * cross_fade_samples)
+            fade_out = wave_window[cross_fade_samples:]
+            fade_in = wave_window[:cross_fade_samples]
+
+            # Cross-faded overlap
+            cross_faded_overlap = prev_overlap * fade_out + next_overlap * fade_in
+
+            # Combine
+            new_wave = np.concatenate(
+                [
+                    prev_wave[:-cross_fade_samples],
+                    cross_faded_overlap,
+                    next_wave[cross_fade_samples:],
+                ]
+            )
+
+            final_wave = new_wave
+
+    # Create a combined spectrogram
+    combined_spectrogram = np.concatenate(spectrograms, axis=1)
+
+    return final_wave, target_sample_rate, combined_spectrogram

semantic_tokenizer/f40ms/README.md

@@ -0,0 +1,7 @@
+# SpeechTokenizerInference
+
+# 25Hz(80ms) speech semantic tokenizer based on fairseq
+
+pip install -r requirements_npu.txt
+
+

semantic_tokenizer/f40ms/ckpt/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c65b209e916297d6b1b2692c04ebc765da894e71beab290da7131b7892d9262d
+size 3866980554

semantic_tokenizer/f40ms/ckpt/model.pt.md5

@@ -0,0 +1 @@
+960e4618b84d277d646c22e5f6e9754d  model.pt

semantic_tokenizer/f40ms/config/dict.km.txt

@@ -0,0 +1,1000 @@
+0 1
+1 1
+2 1
+3 1
+4 1
+5 1
+6 1
+7 1
+8 1
+9 1
+10 1
+11 1
+12 1
+13 1
+14 1
+15 1
+16 1
+17 1
+18 1
+19 1
+20 1
+21 1
+22 1
+23 1
+24 1
+25 1
+26 1
+27 1
+28 1
+29 1
+30 1
+31 1
+32 1
+33 1
+34 1
+35 1
+36 1
+37 1
+38 1
+39 1
+40 1
+41 1
+42 1
+43 1
+44 1
+45 1
+46 1
+47 1
+48 1
+49 1
+50 1
+51 1
+52 1
+53 1
+54 1
+55 1
+56 1
+57 1
+58 1
+59 1
+60 1
+61 1
+62 1
+63 1
+64 1
+65 1
+66 1
+67 1
+68 1
+69 1
+70 1
+71 1
+72 1
+73 1
+74 1
+75 1
+76 1
+77 1
+78 1
+79 1
+80 1
+81 1
+82 1
+83 1
+84 1
+85 1
+86 1
+87 1
+88 1
+89 1
+90 1
+91 1
+92 1
+93 1
+94 1
+95 1
+96 1
+97 1
+98 1
+99 1
+100 1
+101 1
+102 1
+103 1
+104 1
+105 1
+106 1
+107 1
+108 1
+109 1
+110 1
+111 1
+112 1
+113 1
+114 1
+115 1
+116 1
+117 1
+118 1
+119 1
+120 1
+121 1
+122 1
+123 1
+124 1
+125 1
+126 1
+127 1
+128 1
+129 1
+130 1
+131 1
+132 1
+133 1
+134 1
+135 1
+136 1
+137 1
+138 1
+139 1
+140 1
+141 1
+142 1
+143 1
+144 1
+145 1
+146 1
+147 1
+148 1
+149 1
+150 1
+151 1
+152 1
+153 1
+154 1
+155 1
+156 1
+157 1
+158 1
+159 1
+160 1
+161 1
+162 1
+163 1
+164 1
+165 1
+166 1
+167 1
+168 1
+169 1
+170 1
+171 1
+172 1
+173 1
+174 1
+175 1
+176 1
+177 1
+178 1
+179 1
+180 1
+181 1
+182 1
+183 1
+184 1
+185 1
+186 1
+187 1
+188 1
+189 1
+190 1
+191 1
+192 1
+193 1
+194 1
+195 1
+196 1
+197 1
+198 1
+199 1
+200 1
+201 1
+202 1
+203 1
+204 1
+205 1
+206 1
+207 1
+208 1
+209 1
+210 1
+211 1
+212 1
+213 1
+214 1
+215 1
+216 1
+217 1
+218 1
+219 1
+220 1
+221 1
+222 1
+223 1
+224 1
+225 1
+226 1
+227 1
+228 1
+229 1
+230 1
+231 1
+232 1
+233 1
+234 1
+235 1
+236 1
+237 1
+238 1
+239 1
+240 1
+241 1
+242 1
+243 1
+244 1
+245 1
+246 1
+247 1
+248 1
+249 1
+250 1
+251 1
+252 1
+253 1
+254 1
+255 1
+256 1
+257 1
+258 1
+259 1
+260 1
+261 1
+262 1
+263 1
+264 1
+265 1
+266 1
+267 1
+268 1
+269 1
+270 1
+271 1
+272 1
+273 1
+274 1
+275 1
+276 1
+277 1
+278 1
+279 1
+280 1
+281 1
+282 1
+283 1
+284 1
+285 1
+286 1
+287 1
+288 1
+289 1
+290 1
+291 1
+292 1
+293 1
+294 1
+295 1
+296 1
+297 1
+298 1
+299 1
+300 1
+301 1
+302 1
+303 1
+304 1
+305 1
+306 1
+307 1
+308 1
+309 1
+310 1
+311 1
+312 1
+313 1
+314 1
+315 1
+316 1
+317 1
+318 1
+319 1
+320 1
+321 1
+322 1
+323 1
+324 1
+325 1
+326 1
+327 1
+328 1
+329 1
+330 1
+331 1
+332 1
+333 1
+334 1
+335 1
+336 1
+337 1
+338 1
+339 1
+340 1
+341 1
+342 1
+343 1
+344 1
+345 1
+346 1
+347 1
+348 1
+349 1
+350 1
+351 1
+352 1
+353 1
+354 1
+355 1
+356 1
+357 1
+358 1
+359 1
+360 1
+361 1
+362 1
+363 1
+364 1
+365 1
+366 1
+367 1
+368 1
+369 1
+370 1
+371 1
+372 1
+373 1
+374 1
+375 1
+376 1
+377 1
+378 1
+379 1
+380 1
+381 1
+382 1
+383 1
+384 1
+385 1
+386 1
+387 1
+388 1
+389 1
+390 1
+391 1
+392 1
+393 1
+394 1
+395 1
+396 1
+397 1
+398 1
+399 1
+400 1
+401 1
+402 1
+403 1
+404 1
+405 1
+406 1
+407 1
+408 1
+409 1
+410 1
+411 1
+412 1
+413 1
+414 1
+415 1
+416 1
+417 1
+418 1
+419 1
+420 1
+421 1
+422 1
+423 1
+424 1
+425 1
+426 1
+427 1
+428 1
+429 1
+430 1
+431 1
+432 1
+433 1
+434 1
+435 1
+436 1
+437 1
+438 1
+439 1
+440 1
+441 1
+442 1
+443 1
+444 1
+445 1
+446 1
+447 1
+448 1
+449 1
+450 1
+451 1
+452 1
+453 1
+454 1
+455 1
+456 1
+457 1
+458 1
+459 1
+460 1
+461 1
+462 1
+463 1
+464 1
+465 1
+466 1
+467 1
+468 1
+469 1
+470 1
+471 1
+472 1
+473 1
+474 1
+475 1
+476 1
+477 1
+478 1
+479 1
+480 1
+481 1
+482 1
+483 1
+484 1
+485 1
+486 1
+487 1
+488 1
+489 1
+490 1
+491 1
+492 1
+493 1
+494 1
+495 1
+496 1
+497 1
+498 1
+499 1
+500 1
+501 1
+502 1
+503 1
+504 1
+505 1
+506 1
+507 1
+508 1
+509 1
+510 1
+511 1
+512 1
+513 1
+514 1
+515 1
+516 1
+517 1
+518 1
+519 1
+520 1
+521 1
+522 1
+523 1
+524 1
+525 1
+526 1
+527 1
+528 1
+529 1
+530 1
+531 1
+532 1
+533 1
+534 1
+535 1
+536 1
+537 1
+538 1
+539 1
+540 1
+541 1
+542 1
+543 1
+544 1
+545 1
+546 1
+547 1
+548 1
+549 1
+550 1
+551 1
+552 1
+553 1
+554 1
+555 1
+556 1
+557 1
+558 1
+559 1
+560 1
+561 1
+562 1
+563 1
+564 1
+565 1
+566 1
+567 1
+568 1
+569 1
+570 1
+571 1
+572 1
+573 1
+574 1
+575 1
+576 1
+577 1
+578 1
+579 1
+580 1
+581 1
+582 1
+583 1
+584 1
+585 1
+586 1
+587 1
+588 1
+589 1
+590 1
+591 1
+592 1
+593 1
+594 1
+595 1
+596 1
+597 1
+598 1
+599 1
+600 1
+601 1
+602 1
+603 1
+604 1
+605 1
+606 1
+607 1
+608 1
+609 1
+610 1
+611 1
+612 1
+613 1
+614 1
+615 1
+616 1
+617 1
+618 1
+619 1
+620 1
+621 1
+622 1
+623 1
+624 1
+625 1
+626 1
+627 1
+628 1
+629 1
+630 1
+631 1
+632 1
+633 1
+634 1
+635 1
+636 1
+637 1
+638 1
+639 1
+640 1
+641 1
+642 1
+643 1
+644 1
+645 1
+646 1
+647 1
+648 1
+649 1
+650 1
+651 1
+652 1
+653 1
+654 1
+655 1
+656 1
+657 1
+658 1
+659 1
+660 1
+661 1
+662 1
+663 1
+664 1
+665 1
+666 1
+667 1
+668 1
+669 1
+670 1
+671 1
+672 1
+673 1
+674 1
+675 1
+676 1
+677 1
+678 1
+679 1
+680 1
+681 1
+682 1
+683 1
+684 1
+685 1
+686 1
+687 1
+688 1
+689 1
+690 1
+691 1
+692 1
+693 1
+694 1
+695 1
+696 1
+697 1
+698 1
+699 1
+700 1
+701 1
+702 1
+703 1
+704 1
+705 1
+706 1
+707 1
+708 1
+709 1
+710 1
+711 1
+712 1
+713 1
+714 1
+715 1
+716 1
+717 1
+718 1
+719 1
+720 1
+721 1
+722 1
+723 1
+724 1
+725 1
+726 1
+727 1
+728 1
+729 1
+730 1
+731 1
+732 1
+733 1
+734 1
+735 1
+736 1
+737 1
+738 1
+739 1
+740 1
+741 1
+742 1
+743 1
+744 1
+745 1
+746 1
+747 1
+748 1
+749 1
+750 1
+751 1
+752 1
+753 1
+754 1
+755 1
+756 1
+757 1
+758 1
+759 1
+760 1
+761 1
+762 1
+763 1
+764 1
+765 1
+766 1
+767 1
+768 1
+769 1
+770 1
+771 1
+772 1
+773 1
+774 1
+775 1
+776 1
+777 1
+778 1
+779 1
+780 1
+781 1
+782 1
+783 1
+784 1
+785 1
+786 1
+787 1
+788 1
+789 1
+790 1
+791 1
+792 1
+793 1
+794 1
+795 1
+796 1
+797 1
+798 1
+799 1
+800 1
+801 1
+802 1
+803 1
+804 1
+805 1
+806 1
+807 1
+808 1
+809 1
+810 1
+811 1
+812 1
+813 1
+814 1
+815 1
+816 1
+817 1
+818 1
+819 1
+820 1
+821 1
+822 1
+823 1
+824 1
+825 1
+826 1
+827 1
+828 1
+829 1
+830 1
+831 1
+832 1
+833 1
+834 1
+835 1
+836 1
+837 1
+838 1
+839 1
+840 1
+841 1
+842 1
+843 1
+844 1
+845 1
+846 1
+847 1
+848 1
+849 1
+850 1
+851 1
+852 1
+853 1
+854 1
+855 1
+856 1
+857 1
+858 1
+859 1
+860 1
+861 1
+862 1
+863 1
+864 1
+865 1
+866 1
+867 1
+868 1
+869 1
+870 1
+871 1
+872 1
+873 1
+874 1
+875 1
+876 1
+877 1
+878 1
+879 1
+880 1
+881 1
+882 1
+883 1
+884 1
+885 1
+886 1
+887 1
+888 1
+889 1
+890 1
+891 1
+892 1
+893 1
+894 1
+895 1
+896 1
+897 1
+898 1
+899 1
+900 1
+901 1
+902 1
+903 1
+904 1
+905 1
+906 1
+907 1
+908 1
+909 1
+910 1
+911 1
+912 1
+913 1
+914 1
+915 1
+916 1
+917 1
+918 1
+919 1
+920 1
+921 1
+922 1
+923 1
+924 1
+925 1
+926 1
+927 1
+928 1
+929 1
+930 1
+931 1
+932 1
+933 1
+934 1
+935 1
+936 1
+937 1
+938 1
+939 1
+940 1
+941 1
+942 1
+943 1
+944 1
+945 1
+946 1
+947 1
+948 1
+949 1
+950 1
+951 1
+952 1
+953 1
+954 1
+955 1
+956 1
+957 1
+958 1
+959 1
+960 1
+961 1
+962 1
+963 1
+964 1
+965 1
+966 1
+967 1
+968 1
+969 1
+970 1
+971 1
+972 1
+973 1
+974 1
+975 1
+976 1
+977 1
+978 1
+979 1
+980 1
+981 1
+982 1
+983 1
+984 1
+985 1
+986 1
+987 1
+988 1
+989 1
+990 1
+991 1
+992 1
+993 1
+994 1
+995 1
+996 1
+997 1
+998 1
+999 1

semantic_tokenizer/f40ms/config/hubert_config.yaml

@@ -0,0 +1,344 @@
+_name: null
+common:
+  _name: null
+  no_progress_bar: false
+  log_interval: 100
+  log_format: json
+  log_file: null
+  aim_repo: null
+  aim_run_hash: null
+  tensorboard_logdir: tblog
+  wandb_project: null
+  azureml_logging: false
+  seed: 1337
+  cpu: false
+  tpu: false
+  bf16: false
+  memory_efficient_bf16: false
+  fp16: true
+  memory_efficient_fp16: true
+  fp16_no_flatten_grads: false
+  fp16_init_scale: 128
+  fp16_scale_window: null
+  fp16_scale_tolerance: 0.0
+  on_cpu_convert_precision: false
+  min_loss_scale: 0.0001
+  threshold_loss_scale: null
+  amp: false
+  amp_batch_retries: 2
+  amp_init_scale: 128
+  amp_scale_window: null
+  user_dir: null
+  empty_cache_freq: 0
+  all_gather_list_size: 16384
+  model_parallel_size: 1
+  quantization_config_path: null
+  profile: false
+  reset_logging: false
+  suppress_crashes: false
+  use_plasma_view: false
+  plasma_path: /tmp/plasma
+common_eval:
+  _name: null
+  path: null
+  post_process: null
+  quiet: false
+  model_overrides: '{}'
+  results_path: null
+distributed_training:
+  _name: null
+  distributed_world_size: 64
+  distributed_num_procs: 8
+  distributed_rank: 0
+  distributed_backend: nccl
+  distributed_init_method: tcp://modelarts-job-4e3029c5-de6d-4973-85ce-9be855ccbfcf:6000
+  distributed_port: -1
+  device_id: 0
+  distributed_no_spawn: false
+  ddp_backend: no_c10d
+  ddp_comm_hook: none
+  bucket_cap_mb: 25
+  fix_batches_to_gpus: false
+  find_unused_parameters: true
+  gradient_as_bucket_view: false
+  fast_stat_sync: false
+  heartbeat_timeout: -1
+  broadcast_buffers: false
+  slowmo_momentum: null
+  slowmo_base_algorithm: localsgd
+  localsgd_frequency: 3
+  nprocs_per_node: 8
+  pipeline_model_parallel: false
+  pipeline_balance: null
+  pipeline_devices: null
+  pipeline_chunks: 0
+  pipeline_encoder_balance: null
+  pipeline_encoder_devices: null
+  pipeline_decoder_balance: null
+  pipeline_decoder_devices: null
+  pipeline_checkpoint: never
+  zero_sharding: none
+  fp16: true
+  memory_efficient_fp16: true
+  tpu: false
+  no_reshard_after_forward: false
+  fp32_reduce_scatter: false
+  cpu_offload: false
+  use_sharded_state: false
+  not_fsdp_flatten_parameters: false
+dataset:
+  _name: null
+  num_workers: 1
+  skip_invalid_size_inputs_valid_test: true
+  max_tokens: 450000
+  batch_size: null
+  required_batch_size_multiple: 8
+  required_seq_len_multiple: 1
+  dataset_impl: null
+  data_buffer_size: 10
+  train_subset: train
+  valid_subset: valid
+  combine_valid_subsets: null
+  ignore_unused_valid_subsets: false
+  validate_interval: 1
+  validate_interval_updates: 1000000
+  validate_after_updates: 0
+  fixed_validation_seed: null
+  disable_validation: false
+  max_tokens_valid: 450000
+  batch_size_valid: null
+  max_valid_steps: null
+  curriculum: 0
+  gen_subset: test
+  num_shards: 1
+  shard_id: 0
+  grouped_shuffling: false
+  update_epoch_batch_itr: false
+  update_ordered_indices_seed: false
+optimization:
+  _name: null
+  max_epoch: 0
+  max_update: 700000
+  stop_time_hours: 0.0
+  clip_norm: 1.0
+  sentence_avg: false
+  update_freq:
+  - 4
+  lr:
+  - 0.0015
+  stop_min_lr: -1.0
+  use_bmuf: false
+  skip_remainder_batch: false
+checkpoint:
+  _name: null
+  save_dir: checkpoints
+  restore_file: checkpoint_last.pt
+  continue_once: null
+  finetune_from_model: null
+  reset_dataloader: false
+  reset_lr_scheduler: false
+  reset_meters: false
+  reset_optimizer: false
+  optimizer_overrides: '{}'
+  save_interval: 1
+  save_interval_updates: 1000000
+  keep_interval_updates: 100
+  keep_interval_updates_pattern: -1
+  keep_last_epochs: -1
+  keep_best_checkpoints: -1
+  no_save: false
+  no_epoch_checkpoints: false
+  no_last_checkpoints: false
+  no_save_optimizer_state: false
+  best_checkpoint_metric: loss
+  maximize_best_checkpoint_metric: false
+  patience: -1
+  checkpoint_suffix: ''
+  checkpoint_shard_count: 1
+  load_checkpoint_on_all_dp_ranks: false
+  write_checkpoints_asynchronously: false
+  model_parallel_size: 1
+bmuf:
+  _name: null
+  block_lr: 1.0
+  block_momentum: 0.875
+  global_sync_iter: 50
+  warmup_iterations: 500
+  use_nbm: false
+  average_sync: false
+  distributed_world_size: 64
+generation:
+  _name: null
+  beam: 5
+  nbest: 1
+  max_len_a: 0.0
+  max_len_b: 200
+  min_len: 1
+  match_source_len: false
+  unnormalized: false
+  no_early_stop: false
+  no_beamable_mm: false
+  lenpen: 1.0
+  unkpen: 0.0
+  replace_unk: null
+  sacrebleu: false
+  score_reference: false
+  prefix_size: 0
+  no_repeat_ngram_size: 0
+  sampling: false
+  sampling_topk: -1
+  sampling_topp: -1.0
+  constraints: null
+  temperature: 1.0
+  diverse_beam_groups: -1
+  diverse_beam_strength: 0.5
+  diversity_rate: -1.0
+  print_alignment: null
+  print_step: false
+  lm_path: null
+  lm_weight: 0.0
+  iter_decode_eos_penalty: 0.0
+  iter_decode_max_iter: 10
+  iter_decode_force_max_iter: false
+  iter_decode_with_beam: 1
+  iter_decode_with_external_reranker: false
+  retain_iter_history: false
+  retain_dropout: false
+  retain_dropout_modules: null
+  decoding_format: null
+  no_seed_provided: false
+  eos_token: null
+eval_lm:
+  _name: null
+  output_word_probs: false
+  output_word_stats: false
+  context_window: 0
+  softmax_batch: 9223372036854775807
+interactive:
+  _name: null
+  buffer_size: 0
+  input: '-'
+model:
+  _name: hubert
+  label_rate: 50.0
+  extractor_mode: layer_norm
+  encoder_layers: 24
+  encoder_embed_dim: 1024
+  encoder_ffn_embed_dim: 4096
+  encoder_attention_heads: 16
+  activation_fn: gelu
+  layer_type: transformer
+  dropout: 0.0
+  attention_dropout: 0.0
+  activation_dropout: 0.0
+  encoder_layerdrop: 0.0
+  dropout_input: 0.0
+  dropout_features: 0.0
+  final_dim: 768
+  untie_final_proj: true
+  layer_norm_first: true
+  conv_feature_layers: '[(512,10,5)] + [(512,3,2)] * 4 + [(512,2,2)] * 2'
+  conv_bias: false
+  logit_temp: 0.1
+  target_glu: false
+  feature_grad_mult: 1.0
+  mask_length: 10
+  mask_prob: 0.8
+  mask_selection: static
+  mask_other: 0.0
+  no_mask_overlap: false
+  mask_min_space: 1
+  mask_channel_length: 10
+  mask_channel_prob: 0.0
+  mask_channel_selection: static
+  mask_channel_other: 0.0
+  no_mask_channel_overlap: false
+  mask_channel_min_space: 1
+  conv_pos: 128
+  conv_pos_groups: 16
+  latent_temp:
+  - 2.0
+  - 0.5
+  - 0.999995
+  skip_masked: false
+  skip_nomask: false
+  checkpoint_activations: false
+  required_seq_len_multiple: 2
+  depthwise_conv_kernel_size: 31
+  attn_type: ''
+  pos_enc_type: abs
+  fp16: false
+task:
+  _name: hubert_pretraining
+  data: data
+  fine_tuning: false
+  labels:
+  - km
+  label_dir: config
+  label_rate: 50.0
+  sample_rate: 16000
+  normalize: true
+  enable_padding: false
+  max_keep_size: 320000
+  max_sample_size: 320000
+  min_sample_size: 16000
+  single_target: false
+  random_crop: true
+  pad_audio: false
+criterion:
+  _name: hubert
+  pred_masked_weight: 1.0
+  pred_nomask_weight: 0.0
+  loss_weights:
+  - 10.0
+  log_keys: []
+optimizer:
+  _name: adam
+  adam_betas: (0.9,0.98)
+  adam_eps: 1.0e-06
+  weight_decay: 0.01
+  use_old_adam: false
+  fp16_adam_stats: false
+  tpu: false
+  lr:
+  - 0.0015
+lr_scheduler:
+  _name: polynomial_decay
+  warmup_updates: 32000
+  force_anneal: null
+  end_learning_rate: 0.0
+  power: 1.0
+  total_num_update: 700000.0
+  lr:
+  - 0.0015
+scoring: null
+bpe: null
+tokenizer: null
+ema:
+  _name: null
+  store_ema: false
+  ema_decay: 0.9999
+  ema_start_update: 0
+  ema_seed_model: null
+  ema_update_freq: 1
+  ema_fp32: false
+job_logging_cfg:
+  version: 1
+  formatters:
+    simple:
+      format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+  handlers:
+    console:
+      class: logging.StreamHandler
+      formatter: simple
+      stream: ext://sys.stdout
+    file:
+      class: logging.FileHandler
+      formatter: simple
+      filename: hydra_train.log
+  root:
+    level: INFO
+    handlers:
+    - console
+    - file
+  disable_existing_loggers: false

semantic_tokenizer/f40ms/fairseq_npu_patch.py

@@ -0,0 +1,34 @@
+# Copyright (C) 2025. Huawei Technologies Co., Ltd. All Rights Reserved. (authors: Xiao Chen)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from patch_utils import MindSpeedPatchesManager as aspm
+import torch_npu
+
+def cnn_feature_forward_npu(self, x):
+
+    # BxT -> BxCxT
+    x = x.unsqueeze(1)
+
+    for conv in self.conv_layers:
+        # x = conv(x)
+        x_tmp = conv[:-1](x)
+        x = torch_npu.npu_gelu(x_tmp)
+
+    return x
+
+def patch_for_npu():
+    # replace torch.cuda.get_device_capability with implementation from MindSpeed
+    aspm.register_patch('fairseq.models.wav2vec.wav2vec2.ConvFeatureExtractionModel.forward', cnn_feature_forward_npu)
+    aspm.apply_patches()

semantic_tokenizer/f40ms/infer_for_eval.py

@@ -0,0 +1,140 @@
+# Copyright (C) 2025. Huawei Technologies Co., Ltd. All Rights Reserved. (authors: Daxin Tan,
+#                                                                                  Xiao Chen)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from simple_tokenizer_infer import SpeechTokenizer
+import argparse
+import librosa
+import logging
+from pathlib import Path
+
+
+def main(args):
+    ref_wav_file_list = []
+    line_info_list = []
+    reconstruct_wav_file_list = []
+
+    logging.info(f"loading eval file list")
+    base_path = Path(args.input_list).parent
+    with open(args.input_list, "r") as input_file:
+        for line in input_file:
+            fields = line.strip().split("|")
+            if args.input_type == "tts":
+                ref_wav_file_list.append(fields[2])
+            else:
+                reconstruct_wav_file_list.append(fields[4])
+                ref_wav_file_list.append(fields[2])
+            line_info_list.append([fields[2], fields[0], fields[3]]) # ref wav path, gen wav id, text
+
+    logging.info(f"loading ref audio")
+    raw_ref_wavs_list = []  # 用librosa 加载后的raw wave 波形数据
+    for file_path in ref_wav_file_list:
+        # 加载波形数据
+        raw_wav, sr = librosa.load(
+            (base_path / file_path), sr=16000
+        )  # sr=None 保留原始采样率
+        raw_ref_wavs_list.append(raw_wav)
+
+    logging.info(f"extracting token for ref audio")
+    if args.ckpt is not None:
+        tokenizer = SpeechTokenizer(
+            ckpt_path=args.ckpt, cfg_path=args.cfg_path, cfg_name=args.cfg_name
+        )
+    else:
+        tokenizer = SpeechTokenizer()
+    ref_token_list, ref_token_info_list = tokenizer.extract(raw_ref_wavs_list)
+
+    if args.input_type == "reconstruct":
+        logging.info(f"loading reconstruct audio")
+        raw_reconstruct_wav_list = []  # 用librosa 加载后的raw wave 波形数据
+        for file_path in reconstruct_wav_file_list:
+            # 加载波形数据
+            raw_wav, sr = librosa.load(
+                (base_path / file_path), sr=16000
+            )  # sr=None 保留原始采样率
+            raw_reconstruct_wav_list.append(raw_wav)
+
+        logging.info(f"extracting token for reconstruct audio")
+        recon_token_list, recon_token_info_list = tokenizer.extract(raw_reconstruct_wav_list)
+        assert(len(ref_token_info_list) == len(recon_token_info_list))
+
+    assert(len(ref_token_info_list) == len(line_info_list))
+    with open(args.output_file, "w") as output_file:
+        logging.info(f"writing output file")
+        if args.input_type == "tts":
+            for ref, line_info in zip(ref_token_info_list, line_info_list):
+                ref_units = ref["reduced_unit_sequence"]
+                # logging.info(ref_units)
+                ref_path = str((base_path / line_info[0]))
+                output_file.write(f"{ref_path}|{ref_units}|{line_info[1]}|{line_info[2]}\n")
+        else:
+            for ref, recon, line_info in zip(ref_token_info_list, recon_token_info_list, line_info_list):
+                ref_units = ref["reduced_unit_sequence"]
+                recon_units = recon["reduced_unit_sequence"]
+                # logging.info(ref_units)
+                ref_path = str((base_path / line_info[0]))
+                output_file.write(f"{ref_path}|{ref_units}|{line_info[1]}|{recon_units}|{line_info[2]}\n")
+    output_file.close()
+    logging.info("Finished")
+    return
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--ckpt",
+        dest="ckpt",
+        required=False,
+        help="path to ckpt",
+    )
+    parser.add_argument(
+        "--cfg-path",
+        dest="cfg_path",
+        required=False,
+        default="config",
+        help="path to config",
+    )
+    parser.add_argument(
+        "--cfg-name",
+        dest="cfg_name",
+        required=False,
+        default="hubert_config",
+        help="name of config",
+    )
+    parser.add_argument(
+        "--input-list",
+        dest="input_list",
+        required=True,
+        help="list of input wavform",
+    )
+    parser.add_argument(
+        "--output-file",
+        dest="output_file",
+        required=True,
+        help="file to output speech tokens",
+    )
+    parser.add_argument(
+        "--input-type",
+        default="tts",
+        type=str,
+        required=True,
+        help=f"test fil list type: tts or reconstruct, seedtts format",
+    )
+    args = parser.parse_args()
+
+    if args.input_type not in {"tts", "reconstruct"}:
+        logging.info(f"Input type must be tts or reconstruct")
+        exit()
+    main(args)