Upload 22 files

.gitattributes

@@ -47,3 +47,5 @@ ebook2audiobook.egg-info/assets/gui_1.png filter=lfs diff=lfs merge=lfs -text
 ebook2audiobook.egg-info/assets/gui_2.png filter=lfs diff=lfs merge=lfs -text
 ebook2audiobook.egg-info/assets/gui_3.png filter=lfs diff=lfs merge=lfs -text
 ebook2audiobook.egg-info/assets/Rainy_Day_voice_Demo.mp4 filter=lfs diff=lfs merge=lfs -text
+lib/__pycache__/functions.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text
+lib/__pycache__/lang.cpython-312.pyc filter=lfs diff=lfs merge=lfs -text

lib/__init__.py

@@ -0,0 +1,57 @@
+from .models import (
+    TTS_ENGINES, TTS_VOICE_CONVERSION, TTS_SML, default_fine_tuned, default_tts_engine, 
+    default_engine_settings, default_vc_model, default_voice_detection_model,
+    loaded_tts, max_custom_model, max_custom_voices,
+    max_tts_in_memory, max_upload_size, models, os, voices_dir
+)
+
+from .conf import (
+    FULL_DOCKER, NATIVE, audiobooks_cli_dir, audiobooks_gradio_dir,
+    audiobooks_host_dir, debug_mode, default_audio_proc_samplerate, 
+    default_audio_proc_format, default_device, default_gpu_wiki, 
+    default_output_format, device_list, ebook_formats,
+    ebooks_dir, interface_component_options, interface_concurrency_limit,
+    interface_host, interface_port, interface_shared_tmp_expire,
+    max_python_version, min_python_version, models_dir, os,
+    output_formats, platform, prog_version, python_env_dir,
+    requirements_file, tmp_dir, tmp_expire, tts_dir, voice_formats,
+    voices_dir, default_output_split, default_output_split_hours
+)
+
+from .lang import (
+    abbreviations_mapping, chapter_word_mapping, default_language_code,
+    roman_numbers_tuples, emojis_list, install_info, language_mapping,
+    language_math_phonemes, language_clock, language_tts, os, punctuation_list, 
+    punctuation_list_set, punctuation_split_hard, punctuation_split_hard_set,
+    punctuation_split_soft, punctuation_split_soft_set, punctuation_switch,
+    specialchars_mapping, specialchars_remove, year_to_decades_languages
+)
+
+__all__ = [
+    # from models
+    "TTS_ENGINES", "TTS_VOICE_CONVERSION", "TTS_SML", "default_fine_tuned", "default_tts_engine",
+    "default_engine_settings", "default_vc_model", "default_voice_detection_model",
+    "loaded_tts", "max_custom_model",
+    "max_custom_voices", "max_tts_in_memory", "max_upload_size",
+    "models", "os", "voices_dir",
+
+    # from conf
+    "FULL_DOCKER", "NATIVE", "audiobooks_cli_dir", "audiobooks_gradio_dir",
+    "audiobooks_host_dir", "debug_mode", "default_audio_proc_samplerate",
+    "default_audio_proc_format", "default_device", "default_gpu_wiki",
+    "default_output_format", "device_list", "ebook_formats", "ebooks_dir",
+    "interface_component_options", "interface_concurrency_limit",
+    "interface_host", "interface_port", "interface_shared_tmp_expire",
+    "max_python_version", "min_python_version", "models_dir", "os",
+    "output_formats", "platform", "prog_version", "python_env_dir",
+    "requirements_file", "tmp_dir", "tmp_expire", "tts_dir",
+    "voice_formats", "voices_dir", "default_output_split", "default_output_split_hours",
+
+    # from lang
+    "abbreviations_mapping", "chapter_word_mapping", "default_language_code",
+    "roman_numbers_tuples", "emojis_list", "install_info", "language_mapping",
+    "language_math_phonemes", "language_clock", "language_tts", "os", "punctuation_list", 
+    "punctuation_list_set", "punctuation_split_hard", "punctuation_split_hard_set",
+    "punctuation_split_soft", "punctuation_split_soft_set", "punctuation_switch",
+    "specialchars_mapping", "specialchars_remove", "year_to_decades_languages"
+]

lib/__pycache__/functions.cpython-312.pyc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59b1809dd2e4e86864d8ff51fbdade7548389b92cd6f3b24d9e9a54235eb0de2
+size 236223

lib/__pycache__/lang.cpython-312.pyc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff1e8d413d7881648a9aa7ffae42617ebc430ee61b2523706c9eb8315889c86e
+size 228874

lib/classes/argos_translator.py

@@ -0,0 +1,122 @@
+import os
+import tempfile
+import argostranslate.package
+import argostranslate.translate
+
+from iso639 import languages
+from lib.conf import models_dir
+from lib.lang import language_mapping
+
+# NOTE: source_lang and target_lang must be iso639-1 (2 letters)
+
+class ArgosTranslator:
+
+    def __init__(self, neural_machine="argostranslate"):
+        self.neural_machine = neural_machine
+        self.translation = None
+        
+    def get_language_iso3(self, lang_iso1):
+        lang = lang_iso1
+        try:  
+            lang_array = languages.get(part1=lang_iso1)
+            if lang_array:
+                lang = lang_array.part3
+        except Exception:
+            pass
+        return lang
+
+    def get_all_sources_lang(self):
+        available_packages = argostranslate.package.get_available_packages()
+        return sorted(set(pkg.from_code for pkg in available_packages))
+
+    def get_all_targets_lang(self, source_lang):
+        available_packages = argostranslate.package.get_available_packages()
+        list_iso1 = sorted(set(pkg.to_code for pkg in available_packages if pkg.from_code == source_lang))
+        language_translate_mapping = {}
+        for iso1 in list_iso1:
+            try:
+                iso3 = self.get_language_iso3(iso1)
+                if iso3 in language_mapping:
+                    language_translate_mapping[iso3] = dict(language_mapping[iso3])
+                    language_translate_mapping[iso3]["iso1"] = iso1
+            except KeyError:
+                pass
+        language_translate_options = [
+            (
+                f"{details['name']} - {details['native_name']}" if details['name'] != details['native_name'] else details['name'],
+                lang
+            )
+            for lang, details in language_translate_mapping.items()
+        ]
+        return language_translate_options
+        
+    def get_all_target_packages(self, source_lang):
+        available_packages = argostranslate.package.get_available_packages()
+        return [pkg for pkg in available_packages if pkg.from_code == source_lang]
+
+    def is_package_installed(self, source_lang, target_lang):
+        try:
+            installed_languages = argostranslate.translate.get_installed_languages()
+            source_language = next((lang for lang in installed_languages if lang.code == source_lang), None)
+            target_language = next((lang for lang in installed_languages if lang.code == target_lang), None)
+            return source_language is not None and target_language is not None
+        except Exception as e:
+            error = f'is_package_installed() error: {e}'
+            return False
+
+    def download_and_install_argos_package(self, source_lang, target_lang):
+        try:
+            if self.is_package_installed(source_lang, target_lang):
+                print(f"Package for translation from {source_lang} to {target_lang} is already installed.")
+                print(msg)                
+                return msg, True
+            available_packages = self.get_all_target_packages(source_lang)
+            target_package = None
+            for pkg in available_packages:
+                if pkg.from_code == source_lang and pkg.to_code == target_lang:
+                    target_package = pkg
+                    break
+            if target_package:
+                with tempfile.TemporaryDirectory() as tmpdirname:
+                    print(f"Downloading package for translation from {source_lang} to {target_lang}...")
+                    package_path = target_package.download()
+                    argostranslate.package.install_from_path(package_path)
+                    print(f"Package installed for translation from {source_lang} to {target_lang}")
+                    return None, True
+            else:
+                msg = f"No available package found for translation from {source_lang} to {target_lang}."
+                return msg, False
+        except Exception as e:
+            error = f'download_and_install_argos_package() error: {e}'
+            return error, False
+
+    def process(self, text):
+        try:
+            return self.translation.translate(text), True
+        except Exception as e:
+            error = f'AgrosTranslator.process() error: {e}'
+            return error, False
+
+    def start(self, source_lang, target_lang):
+        try:
+            if self.neural_machine != "argostranslate":
+                error = f"Neural machine '{self.neural_machine}' is not supported."
+                return error, False
+            status = True
+            if not self.is_package_installed(source_lang, target_lang):
+                error, status = self.download_and_install_argos_package(source_lang, target_lang)
+            if status:
+                installed_languages = argostranslate.translate.get_installed_languages()
+                source_language = next((lang for lang in installed_languages if lang.code == source_lang), None)
+                target_language = next((lang for lang in installed_languages if lang.code == target_lang), None)
+
+                if not source_language or not target_language:
+                    error = f"Translation languages not installed: {source_lang} to {target_lang}"
+                    return error, False
+
+                self.translation = source_language.get_translation(target_language)
+                return None, True
+            return error, status
+        except Exception as e:
+            error = f'AgrosTranslator.process() error: {e}'
+            return error, False

lib/classes/background_detector.py

@@ -0,0 +1,37 @@
+import os
+import numpy as np
+import librosa
+
+from pyannote.audio import Model
+from pyannote.audio.pipelines import VoiceActivityDetection
+from lib.conf import tts_dir
+from lib.models import default_voice_detection_model
+
+class BackgroundDetector:
+
+    def __init__(self, wav_file: str):
+        self.wav_file   = wav_file
+        model = Model.from_pretrained(default_voice_detection_model, cache_dir=tts_dir)
+        self.pipeline = VoiceActivityDetection(segmentation=model)
+        hyper_params = {
+          # onset/offset activation thresholds
+          "onset": 0.5, "offset": 0.5,
+          # remove speech regions shorter than that many seconds.
+          "min_duration_on": 0.0,
+          # fill non-speech regions shorter than that many seconds.
+          "min_duration_off": 0.0
+        }
+        self.pipeline.instantiate(hyper_params)
+
+    def detect(self, vad_ratio_thresh: float=0.05):
+        diarization     = self.pipeline(self.wav_file)
+        speech_segments = [(s.start, s.end) for s in diarization.get_timeline()]
+        total_duration  = librosa.get_duration(path=self.wav_file)
+        speech_time     = sum(end - start for start, end in speech_segments)
+        non_speech_ratio = 1 - (speech_time / total_duration)
+        status = non_speech_ratio > vad_ratio_thresh
+        report = {
+            'non_speech_ratio': non_speech_ratio,
+            'background_detected': status
+        }
+        return status, report

lib/classes/redirect_console.py

@@ -0,0 +1,51 @@
+from queue import Queue, Empty
+import time
+import logging
+
+
+class RedirectConsole:
+    def __init__(self, log_buffer: Queue, real_output):
+        self.log_buffer = log_buffer  # Queue buffer for the log
+        self.real_output = real_output  # Real terminal (sys.__stdout__ or sys.__stderr__)
+
+        # Setup for transformers logging
+        self.setup_transformers_logger()
+
+    def write(self, message: str):
+        # Write to the real terminal
+        self.real_output.write(message)
+        self.real_output.flush()
+
+        # Write to the log buffer
+        self.log_buffer.put(message)
+
+    def flush(self):
+        self.real_output.flush()
+
+    def isatty(self) -> bool:
+        return self.real_output.isatty()
+
+    def poll_logs(self, stop_event):
+        logs = ""
+        errors = ""
+        while not stop_event.is_set() or not self.log_buffer.empty():
+            try:
+                # Read logs from the buffer without blocking
+                log = self.log_buffer.get_nowait()
+                if "An error occurred" in log:
+                    errors += log  # Capture error messages separately
+                logs += log
+            except Empty:
+                pass  # No logs in the buffer
+            yield logs, errors  # Yield updated logs and errors
+            time.sleep(0.1)  # Prevent tight looping
+
+    def setup_transformers_logger(self):
+        # Configure the `transformers` logger
+        transformers_logger = logging.getLogger("transformers")
+        transformers_logger.setLevel(logging.WARNING)  # Capture warnings and above
+
+        # Create a handler that writes to this instance
+        handler = logging.StreamHandler(self)
+        handler.setFormatter(logging.Formatter("%(message)s"))  # Simplified format
+        transformers_logger.addHandler(handler)

lib/classes/tts_engines/.template.py

@@ -0,0 +1,232 @@
+import hashlib
+import math
+import os
+import shutil
+import subprocess
+import tempfile
+import threading
+import uuid
+
+import numpy as np
+import regex as re
+import soundfile as sf
+import torch
+import torchaudio
+
+from huggingface_hub import hf_hub_download
+from pathlib import Path
+from pprint import pprint
+
+from lib import *
+from lib.classes.tts_engines.common.utils import unload_tts, append_sentence2vtt
+from lib.classes.tts_engines.common.audio_filters import detect_gender, trim_audio, normalize_audio, is_audio_data_valid
+
+#import logging
+#logging.basicConfig(level=logging.DEBUG)
+
+lock = threading.Lock()
+
+class Coqui:
+
+    def __init__(self, session):
+        try:
+            self.session = session
+            self.cache_dir = tts_dir
+            self.speakers_path = None
+            self.tts_key = f"{self.session['tts_engine']}-{self.session['fine_tuned']}"
+            self.tts_vc_key = default_vc_model.rsplit('/', 1)[-1]
+            self.is_bf16 = True if self.session['device'] == 'cuda' and torch.cuda.is_bf16_supported() == True else False
+            self.npz_path = None
+            self.npz_data = None
+            self.sentences_total_time = 0.0
+            self.sentence_idx = 1
+            self.params = {TTS_ENGINES['NEW_TTS']: {}}
+            self.params[self.session['tts_engine']]['samplerate'] = models[self.session['tts_engine']][self.session['fine_tuned']]['samplerate']
+            self.vtt_path = os.path.join(self.session['process_dir'], os.path.splitext(self.session['final_name'])[0] + '.vtt')    
+            self.resampler_cache = {}
+            self.audio_segments = []
+            self._build()
+        except Exception as e:
+            error = f'__init__() error: {e}'
+            print(error)
+            return None
+
+    def _build(self):
+        try:
+            tts = (loaded_tts.get(self.tts_key) or {}).get('engine', False)
+            if not tts:
+                if self.session['tts_engine'] == TTS_ENGINES['NEW_TTS']:
+                    if self.session['custom_model'] is not None:
+                        msg = f"{self.session['tts_engine']} custom model not implemented yet!"
+                        print(msg)
+                        return False
+                    else:
+                        model_path = models[self.session['tts_engine']][self.session['fine_tuned']]['repo']
+                        tts = self._load_api(self.tts_key, model_path, self.session['device'])
+            return (loaded_tts.get(self.tts_key) or {}).get('engine', False)
+        except Exception as e:
+            error = f'build() error: {e}'
+            print(error)
+            return False
+
+    def _load_api(self, key, model_path, device):
+        global lock
+        try:
+            if key in loaded_tts.keys():
+                return loaded_tts[key]['engine']
+            unload_tts(device, [self.tts_key, self.tts_vc_key])
+            with lock:
+                tts = NEW_TTS(model_path)
+                if tts
+                    if device == 'cuda':
+                        NEW_TTS.WITH_CUDA
+                    else:
+                        NEW_TTS.WITHOUT_CUDA
+                    loaded_tts[key] = {"engine": tts, "config": None} 
+                    msg = f'{model_path} Loaded!'
+                    print(msg)
+                    return tts
+                else:
+                    error = 'TTS engine could not be created!'
+                    print(error)
+        except Exception as e:
+            error = f'_load_api() error: {e}'
+            print(error)
+        return False
+
+    def _load_checkpoint(self, **kwargs):
+        global lock
+        try:
+            key = kwargs.get('key')
+            if key in loaded_tts.keys():
+                return loaded_tts[key]['engine']
+            tts_engine = kwargs.get('tts_engine')
+            device = kwargs.get('device')
+            unload_tts(device, [self.tts_key])
+            with lock:
+                checkpoint_dir = kwargs.get('checkpoint_dir')
+                NEW_TTS.LOAD_CHECKPOINT(
+                    config,
+                    checkpoint_dir=checkpoint_dir,
+                    eval=True
+                )                    
+            if tts:
+                if device == 'cuda':
+                    NEW_TTS.WITH_CUDA
+                else:
+                    NEW_TTS.WITHOUT_CUDA
+                loaded_tts[key] = {"engine": tts, "config": config}
+                msg = f'{tts_engine} Loaded!'
+                print(msg)
+                return tts
+            else:
+                error = 'TTS engine could not be created!'
+                print(error)
+        except Exception as e:
+            error = f'_load_checkpoint() error: {e}'
+        return False
+        
+    def _tensor_type(self, audio_data):
+        if isinstance(audio_data, torch.Tensor):
+            return audio_data
+        elif isinstance(audio_data, np.ndarray):  
+            return torch.from_numpy(audio_data).float()
+        elif isinstance(audio_data, list):  
+            return torch.tensor(audio_data, dtype=torch.float32)
+        else:
+            raise TypeError(f"Unsupported type for audio_data: {type(audio_data)}")
+            
+    def _get_resampler(self, orig_sr, target_sr):
+        key = (orig_sr, target_sr)
+        if key not in self.resampler_cache:
+            self.resampler_cache[key] = torchaudio.transforms.Resample(
+                orig_freq=orig_sr, new_freq=target_sr
+            )
+        return self.resampler_cache[key]
+
+    def _resample_wav(self, wav_path, expected_sr):
+        waveform, orig_sr = torchaudio.load(wav_path)
+        if orig_sr == expected_sr and waveform.size(0) == 1:
+            return wav_path
+        if waveform.size(0) > 1:
+            waveform = waveform.mean(dim=0, keepdim=True)
+        if orig_sr != expected_sr:
+            resampler = self._get_resampler(orig_sr, expected_sr)
+            waveform = resampler(waveform)
+        wav_tensor = waveform.squeeze(0)
+        wav_numpy = wav_tensor.cpu().numpy()
+        tmp_fh = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
+        tmp_path = tmp_fh.name
+        tmp_fh.close()
+        sf.write(tmp_path, wav_numpy, expected_sr, subtype="PCM_16")
+        return tmp_path
+
+    def convert(self, sentence_number, sentence):
+        global xtts_builtin_speakers_list
+        try:
+            speaker = None
+            audio_data = False
+            trim_audio_buffer = 0.004
+            settings = self.params[self.session['tts_engine']]
+            final_sentence_file = os.path.join(self.session['chapters_dir_sentences'], f'{sentence_number}.{default_audio_proc_format}')
+            sentence = sentence.strip()
+            settings['voice_path'] = (
+                self.session['voice'] if self.session['voice'] is not None 
+                else os.path.join(self.session['custom_model_dir'], self.session['tts_engine'], self.session['custom_model'], 'ref.wav') if self.session['custom_model'] is not None
+                else models[self.session['tts_engine']][self.session['fine_tuned']]['voice']
+            )
+            if settings['voice_path'] is not None:
+                speaker = re.sub(r'\.wav$', '', os.path.basename(settings['voice_path']))
+            tts = (loaded_tts.get(self.tts_key) or {}).get('engine', False)
+            if tts:
+                if sentence[-1].isalnum():
+                    sentence = f'{sentence} —'
+                if sentence == TTS_SML['break']:
+                    break_tensor = torch.zeros(1, int(settings['samplerate'] * (int(np.random.uniform(0.3, 0.6) * 100) / 100))) # 0.4 to 0.7 seconds
+                    self.audio_segments.append(break_tensor.clone())
+                    return True
+                elif sentence == TTS_SML['pause']:
+                    pause_tensor = torch.zeros(1, int(settings['samplerate'] * (int(np.random.uniform(1.0, 1.8) * 100) / 100))) # 1.0 to 1.8 seconds
+                    self.audio_segments.append(pause_tensor.clone())
+                    return True
+                else:
+                    if self.session['tts_engine'] == TTS_ENGINES['NEW_TTS']:
+                        audio_sentence = NEW_TTS.CONVERT() # audio_sentence must be torch.Tensor or (list, tuple) or np.ndarray
+                    if is_audio_data_valid(audio_sentence):
+                        sourceTensor = self._tensor_type(audio_sentence)
+                        audio_tensor = sourceTensor.clone().detach().unsqueeze(0).cpu()
+                        if sentence[-1].isalnum() or sentence[-1] == '—':
+                            audio_tensor = trim_audio(audio_tensor.squeeze(), settings['samplerate'], 0.003, trim_audio_buffer).unsqueeze(0)
+                        self.audio_segments.append(audio_tensor)
+                        if not re.search(r'\w$', sentence, flags=re.UNICODE):
+                            break_tensor = torch.zeros(1, int(settings['samplerate'] * (int(np.random.uniform(0.3, 0.6) * 100) / 100)))
+                            self.audio_segments.append(break_tensor.clone())
+                        if self.audio_segments:
+                            audio_tensor = torch.cat(self.audio_segments, dim=-1)
+                            start_time = self.sentences_total_time
+                            duration = audio_tensor.shape[-1] / settings['samplerate']
+                            end_time = start_time + duration
+                            self.sentences_total_time = end_time
+                            sentence_obj = {
+                                "start": start_time,
+                                "end": end_time,
+                                "text": sentence,
+                                "resume_check": self.sentence_idx
+                            }
+                            self.sentence_idx = append_sentence2vtt(sentence_obj, self.vtt_path)
+                            if self.sentence_idx:
+                                torchaudio.save(final_sentence_file, audio_tensor, settings['samplerate'], format=default_audio_proc_format)
+                                del audio_tensor
+                        self.audio_segments = []
+                        if os.path.exists(final_sentence_file):
+                            return True
+                        else:
+                            error = f"Cannot create {final_sentence_file}"
+                            print(error)
+            else:
+                error = f"convert() error: {self.session['tts_engine']} is None"
+                print(error)
+        except Exception as e:
+            error = f'Coquit.convert(): {e}'
+            raise ValueError(e)
+        return False

lib/classes/tts_engines/common/audio_filters.py

@@ -0,0 +1,107 @@
+import numpy as np
+import torch
+import subprocess
+import shutil
+
+from scipy.io import wavfile as wav
+from scipy.signal import find_peaks
+
+def detect_gender(voice_path):
+    try:
+        samplerate, signal = wav.read(voice_path)
+        # Convert stereo to mono if needed
+        if len(signal.shape) > 1:
+            signal = np.mean(signal, axis=1)
+        # Compute FFT
+        fft_spectrum = np.abs(np.fft.fft(signal))
+        freqs = np.fft.fftfreq(len(fft_spectrum), d=1/samplerate)
+        # Consider only positive frequencies
+        positive_freqs = freqs[:len(freqs)//2]
+        positive_magnitude = fft_spectrum[:len(fft_spectrum)//2]
+        # Find peaks in frequency spectrum
+        peaks, _ = find_peaks(positive_magnitude, height=np.max(positive_magnitude) * 0.2)
+        if len(peaks) == 0:
+            return None 
+        # Find the first strong peak within the human voice range (75Hz - 300Hz)
+        for peak in peaks:
+            if 75 <= positive_freqs[peak] <= 300:
+                pitch = positive_freqs[peak]
+                gender = "female" if pitch > 135 else "male"
+                return gender
+                break     
+        return None
+    except Exception as e:
+        error = f"_detect_gender() error: {voice_path}: {e}"
+        print(error)
+        return None
+
+def trim_audio(audio_data, samplerate, silence_threshold=0.003, buffer_sec=0.005):
+	# Ensure audio_data is a PyTorch tensor
+	if isinstance(audio_data, list):  
+		audio_data = torch.tensor(audio_data, dtype=torch.float32)  # Ensure dtype and always float32 for audio
+	if isinstance(audio_data, torch.Tensor):
+		if audio_data.ndim != 1:
+			error = "audio_data must be a 1D tensor (mono audio)."
+			raise ValueError(error)
+		if audio_data.is_cuda:
+			audio_data = audio_data.cpu()           
+		# Detect non-silent indices
+		non_silent_indices = torch.where(audio_data.abs() > silence_threshold)[0]
+		if len(non_silent_indices) == 0:
+			return torch.tensor([], dtype=audio_data.dtype)  # Preserves dtype
+		# Calculate start and end trimming indices with buffer
+		start_index = max(non_silent_indices[0].item() - int(buffer_sec * samplerate), 0)
+		end_index = min(non_silent_indices[-1].item() + int(buffer_sec * samplerate), audio_data.size(0))  # Clamp end to signal length
+		trimmed_audio = audio_data[start_index:end_index]
+		return trimmed_audio
+	error = "audio_data must be a PyTorch tensor or a list of numerical values."
+	raise TypeError(error)
+
+def normalize_audio(input_file, output_file, samplerate):
+    filter_complex = (
+        'agate=threshold=-25dB:ratio=1.4:attack=10:release=250,'
+        'afftdn=nf=-70,'
+        'acompressor=threshold=-20dB:ratio=2:attack=80:release=200:makeup=1dB,'
+        'loudnorm=I=-14:TP=-3:LRA=7:linear=true,'
+        'equalizer=f=150:t=q:w=2:g=1,'
+        'equalizer=f=250:t=q:w=2:g=-3,'
+        'equalizer=f=3000:t=q:w=2:g=2,'
+        'equalizer=f=5500:t=q:w=2:g=-4,'
+        'equalizer=f=9000:t=q:w=2:g=-2,'
+        'highpass=f=63[audio]'
+    )
+    ffmpeg_cmd = [shutil.which('ffmpeg'), '-hide_banner', '-nostats', '-i', input_file]
+    ffmpeg_cmd += [
+        '-filter_complex', filter_complex,
+        '-map', '[audio]',
+        '-ar', str(samplerate),
+        '-y', output_file
+    ]
+    try:
+        subprocess.run(
+            ffmpeg_cmd,
+            env={},
+            stdout=subprocess.PIPE, 
+            stderr=subprocess.PIPE,
+            encoding='utf-8',
+            errors='ignore'
+        )
+        return True
+    except subprocess.CalledProcessError as e:
+        error = f"normalize_audio() error: {input_file}: {e}"
+        print(error)
+        return False
+
+def is_audio_data_valid(audio_data):
+    if audio_data is None:
+        return False
+    if isinstance(audio_data, torch.Tensor):
+        return audio_data.numel() > 0
+    if isinstance(audio_data, (list, tuple)):
+        return len(audio_data) > 0
+    try:
+        if isinstance(audio_data, np.ndarray):
+            return audio_data.size > 0
+    except ImportError:
+        pass
+    return False

lib/classes/tts_engines/common/utils.py

@@ -0,0 +1,57 @@
+import os
+import torch
+import regex as re
+import stanza
+
+from lib.models import loaded_tts, max_tts_in_memory, TTS_ENGINES
+
+def unload_tts(device, reserved_keys=None, tts_key=None):
+    try:
+        if len(loaded_tts) >= max_tts_in_memory:
+            if reserved_keys is None:
+                reserved_keys = []
+            if tts_key is not None:
+                if tts_key in loaded_tts.keys():
+                    del loaded_tts[tts_key]
+                if device == 'cuda':
+                    torch.cuda.empty_cache()
+                    torch.cuda.ipc_collect()
+            else:
+                for key in list(loaded_tts.keys()):
+                    if key not in reserved_keys:
+                        del loaded_tts[key]
+    except Exception as e:
+        error = f'unload_tts() error: {e}'
+        print(error)
+        return False
+        
+def append_sentence2vtt(sentence_obj, path):
+
+    def format_timestamp(seconds):
+        m, s = divmod(seconds, 60)
+        h, m = divmod(m, 60)
+        return f"{int(h):02}:{int(m):02}:{s:06.3f}"
+
+    try:
+        index = 1
+        if os.path.exists(path):
+            with open(path, "r", encoding="utf-8") as f:
+                lines = f.readlines()
+                for line in lines:
+                    if "-->" in line:
+                        index += 1
+        if index > 1 and "resume_check" in sentence_obj and sentence_obj["resume_check"] < index:
+            return index  # Already written
+        if not os.path.exists(path):
+            with open(path, "w", encoding="utf-8") as f:
+                f.write("WEBVTT\n\n")
+        with open(path, "a", encoding="utf-8") as f:
+            start = format_timestamp(sentence_obj["start"])
+            end = format_timestamp(sentence_obj["end"])
+            text = re.sub(r'[\r\n]+', ' ', sentence_obj["text"]).strip()
+            f.write(f"{start} --> {end}\n{text}\n\n")
+        return index + 1
+    except Exception as e:
+        error = f'append_sentence2vtt() error: {e}'
+        print(error)
+        return False

lib/classes/tts_engines/coqui.py

@@ -0,0 +1,810 @@
+import hashlib, math, os, shutil, subprocess, tempfile, threading, uuid
+import numpy as np, regex as re, soundfile as sf, torch, torchaudio
+
+from huggingface_hub import hf_hub_download
+from pathlib import Path
+from pprint import pprint
+
+from lib import *
+from lib.classes.tts_engines.common.utils import unload_tts, append_sentence2vtt
+from lib.classes.tts_engines.common.audio_filters import detect_gender, trim_audio, normalize_audio, is_audio_data_valid
+
+#import logging
+#logging.basicConfig(level=logging.DEBUG)
+
+lock = threading.Lock()
+xtts_builtin_speakers_list = None
+
+class Coqui:
+
+    def __init__(self, session):
+        try:
+            self.session = session
+            self.cache_dir = tts_dir
+            self.speakers_path = None
+            self.tts_key = f"{self.session['tts_engine']}-{self.session['fine_tuned']}"
+            self.tts_vc_key = default_vc_model.rsplit('/', 1)[-1]
+            self.is_bf16 = True if self.session['device'] == 'cuda' and torch.cuda.is_bf16_supported() == True else False
+            self.npz_path = None
+            self.npz_data = None
+            self.sentences_total_time = 0.0
+            self.sentence_idx = 1
+            self.params = {TTS_ENGINES['XTTSv2']: {"latent_embedding":{}}, TTS_ENGINES['BARK']: {},TTS_ENGINES['VITS']: {"semitones": {}}, TTS_ENGINES['FAIRSEQ']: {"semitones": {}}, TTS_ENGINES['TACOTRON2']: {"semitones": {}}, TTS_ENGINES['YOURTTS']: {}}  
+            self.params[self.session['tts_engine']]['samplerate'] = models[self.session['tts_engine']][self.session['fine_tuned']]['samplerate']
+            self.vtt_path = os.path.join(self.session['process_dir'], Path(self.session['final_name']).stem + '.vtt')    
+            self.resampler_cache = {}
+            self.audio_segments = []
+            self._build()
+        except Exception as e:
+            error = f'__init__() error: {e}'
+            print(error)
+            return None
+
+    def _build(self):
+        try:
+            global xtts_builtin_speakers_list
+            load_zeroshot = True if self.session['tts_engine'] in [TTS_ENGINES['VITS'], TTS_ENGINES['FAIRSEQ'], TTS_ENGINES['TACOTRON2']] else False
+            tts = (loaded_tts.get(self.tts_key) or {}).get('engine', False)
+            if not tts:
+                if xtts_builtin_speakers_list is None:
+                    self.speakers_path = hf_hub_download(repo_id=models[TTS_ENGINES['XTTSv2']]['internal']['repo'], filename=default_engine_settings[TTS_ENGINES['XTTSv2']]['files'][4], cache_dir=self.cache_dir)
+                    xtts_builtin_speakers_list = torch.load(self.speakers_path)
+                if self.session['tts_engine'] == TTS_ENGINES['XTTSv2']:
+                    msg = f"Loading TTS {self.session['tts_engine']} model, it takes a while, please be patient..."
+                    print(msg)
+                    if self.session['custom_model'] is not None:
+                        config_path = os.path.join(self.session['custom_model_dir'], self.session['tts_engine'], self.session['custom_model'], default_engine_settings[TTS_ENGINES['XTTSv2']]['files'][0])
+                        checkpoint_path = os.path.join(self.session['custom_model_dir'], self.session['tts_engine'], self.session['custom_model'], default_engine_settings[TTS_ENGINES['XTTSv2']]['files'][1])
+                        vocab_path = os.path.join(self.session['custom_model_dir'], self.session['tts_engine'], self.session['custom_model'],default_engine_settings[TTS_ENGINES['XTTSv2']]['files'][2])
+                        self.tts_key = f"{self.session['tts_engine']}-{self.session['custom_model']}"
+                        tts = self._load_checkpoint(tts_engine=self.session['tts_engine'], key=self.tts_key, checkpoint_path=checkpoint_path, config_path=config_path, vocab_path=vocab_path, device=self.session['device'])
+                    else:
+                        hf_repo = models[self.session['tts_engine']][self.session['fine_tuned']]['repo']
+                        if self.session['fine_tuned'] == 'internal':
+                            hf_sub = ''
+                            if self.speakers_path is None:
+                                self.speakers_path = hf_hub_download(repo_id=hf_repo, filename=default_engine_settings[TTS_ENGINES['XTTSv2']]['files'][4], cache_dir=self.cache_dir)
+                        else:
+                            hf_sub = models[self.session['tts_engine']][self.session['fine_tuned']]['sub']
+                        config_path = hf_hub_download(repo_id=hf_repo, filename=f"{hf_sub}{models[self.session['tts_engine']][self.session['fine_tuned']]['files'][0]}", cache_dir=self.cache_dir)
+                        checkpoint_path = hf_hub_download(repo_id=hf_repo, filename=f"{hf_sub}{models[self.session['tts_engine']][self.session['fine_tuned']]['files'][1]}", cache_dir=self.cache_dir)
+                        vocab_path = hf_hub_download(repo_id=hf_repo, filename=f"{hf_sub}{models[self.session['tts_engine']][self.session['fine_tuned']]['files'][2]}", cache_dir=self.cache_dir)
+                        tts = self._load_checkpoint(tts_engine=self.session['tts_engine'], key=self.tts_key, checkpoint_path=checkpoint_path, config_path=config_path, vocab_path=vocab_path, device=self.session['device'])
+                elif self.session['tts_engine'] == TTS_ENGINES['BARK']:      
+                    if self.session['custom_model'] is not None:
+                        msg = f"{self.session['tts_engine']} custom model not implemented yet!"
+                        print(msg)
+                        return False
+                    else:
+                        hf_repo = models[self.session['tts_engine']][self.session['fine_tuned']]['repo']
+                        hf_sub = models[self.session['tts_engine']][self.session['fine_tuned']]['sub']
+                        text_model_path = hf_hub_download(repo_id=hf_repo, filename=f"{hf_sub}{models[self.session['tts_engine']][self.session['fine_tuned']]['files'][0]}", cache_dir=self.cache_dir)
+                        coarse_model_path = hf_hub_download(repo_id=hf_repo, filename=f"{hf_sub}{models[self.session['tts_engine']][self.session['fine_tuned']]['files'][1]}", cache_dir=self.cache_dir)
+                        fine_model_path = hf_hub_download(repo_id=hf_repo, filename=f"{hf_sub}{models[self.session['tts_engine']][self.session['fine_tuned']]['files'][2]}", cache_dir=self.cache_dir)
+                        checkpoint_dir = os.path.dirname(text_model_path)
+                        tts = self._load_checkpoint(tts_engine=self.session['tts_engine'], key=self.tts_key, checkpoint_dir=checkpoint_dir, device=self.session['device'])
+                elif self.session['tts_engine'] == TTS_ENGINES['VITS']:
+                    if self.session['custom_model'] is not None:
+                        msg = f"{self.session['tts_engine']} custom model not implemented yet!"
+                        print(msg)     
+                        return False
+                    else:
+                        iso_dir = language_tts[self.session['tts_engine']][self.session['language']]
+                        sub_dict = models[self.session['tts_engine']][self.session['fine_tuned']]['sub']
+                        sub = next((key for key, lang_list in sub_dict.items() if iso_dir in lang_list), None)  
+                        if sub is not None:
+                            self.params[self.session['tts_engine']]['samplerate'] = models[TTS_ENGINES['VITS']][self.session['fine_tuned']]['samplerate'][sub]
+                            model_path = models[self.session['tts_engine']][self.session['fine_tuned']]['repo'].replace("[lang_iso1]", iso_dir).replace("[xxx]", sub)
+                            msg = f"Loading TTS {model_path} model, it takes a while, please be patient..."
+                            print(msg)
+                            self.tts_key = model_path
+                            tts = self._load_api(self.tts_key, model_path, self.session['device'])
+                        else:
+                            msg = f"{self.session['tts_engine']} checkpoint for {self.session['language']} not found!"
+                            print(msg)
+                            return False
+                elif self.session['tts_engine'] == TTS_ENGINES['FAIRSEQ']:
+                    if self.session['custom_model'] is not None:
+                        msg = f"{self.session['tts_engine']} custom model not implemented yet!"
+                        print(msg)
+                        return False
+                    else:
+                        model_path = models[self.session['tts_engine']][self.session['fine_tuned']]['repo'].replace("[lang]", self.session['language'])
+                        self.tts_key = model_path
+                        tts = self._load_api(self.tts_key, model_path, self.session['device'])
+                elif self.session['tts_engine'] == TTS_ENGINES['TACOTRON2']:
+                    if self.session['custom_model'] is not None:
+                        msg = f"{self.session['tts_engine']} custom model not implemented yet!"
+                        print(msg)     
+                        return False
+                    else:
+                        iso_dir = language_tts[self.session['tts_engine']][self.session['language']]
+                        sub_dict = models[self.session['tts_engine']][self.session['fine_tuned']]['sub']
+                        sub = next((key for key, lang_list in sub_dict.items() if iso_dir in lang_list), None)
+                        self.params[self.session['tts_engine']]['samplerate'] = models[TTS_ENGINES['TACOTRON2']][self.session['fine_tuned']]['samplerate'][sub]
+                        if sub is None:
+                            iso_dir = self.session['language']
+                            sub = next((key for key, lang_list in sub_dict.items() if iso_dir in lang_list), None)
+                        if sub is not None:
+                            model_path = models[self.session['tts_engine']][self.session['fine_tuned']]['repo'].replace("[lang_iso1]", iso_dir).replace("[xxx]", sub)
+                            msg = f"Loading TTS {model_path} model, it takes a while, please be patient..."
+                            print(msg)
+                            self.tts_key = model_path
+                            tts = self._load_api(self.tts_key, model_path, self.session['device'])
+                        else:
+                            msg = f"{self.session['tts_engine']} checkpoint for {self.session['language']} not found!"
+                            print(msg)
+                            return False
+                elif self.session['tts_engine'] == TTS_ENGINES['YOURTTS']:
+                    if self.session['custom_model'] is not None:
+                        msg = f"{self.session['tts_engine']} custom model not implemented yet!"
+                        print(msg)
+                        return False
+                    else:
+                        model_path = models[self.session['tts_engine']][self.session['fine_tuned']]['repo']
+                        tts = self._load_api(self.tts_key, model_path, self.session['device'])
+            if load_zeroshot:
+                tts_vc = (loaded_tts.get(self.tts_vc_key) or {}).get('engine', False)
+                if not tts_vc:
+                    if self.session['voice'] is not None:
+                        msg = f"Loading TTS {self.tts_vc_key} zeroshot model, it takes a while, please be patient..."
+                        print(msg)
+                        tts_vc = self._load_api(self.tts_vc_key, default_vc_model, self.session['device'])
+            return (loaded_tts.get(self.tts_key) or {}).get('engine', False)
+        except Exception as e:
+            error = f'build() error: {e}'
+            print(error)
+            return False
+
+    def _load_api(self, key, model_path, device):
+        global lock
+        try:
+            if key in loaded_tts.keys():
+                return loaded_tts[key]['engine']
+            unload_tts(device, [self.tts_key, self.tts_vc_key])
+            from TTS.api import TTS as coquiAPI
+            with lock:
+                tts = coquiAPI(model_path)
+                if tts:
+                    if device == 'cuda':
+                        tts.cuda()
+                    else:
+                        tts.to(device)
+                    loaded_tts[key] = {"engine": tts, "config": None} 
+                    msg = f'{model_path} Loaded!'
+                    print(msg)
+                    return tts
+                else:
+                    error = 'TTS engine could not be created!'
+                    print(error)
+        except Exception as e:
+            error = f'_load_api() error: {e}'
+            print(error)
+        return False
+
+    def _load_checkpoint(self, **kwargs):
+        global lock
+        try:
+            key = kwargs.get('key')
+            if key in loaded_tts.keys():
+                return loaded_tts[key]['engine']
+            tts_engine = kwargs.get('tts_engine')
+            device = kwargs.get('device')
+            unload_tts(device, [self.tts_key, self.tts_vc_key])
+            with lock:
+                if tts_engine == TTS_ENGINES['XTTSv2']:
+                    from TTS.tts.configs.xtts_config import XttsConfig
+                    from TTS.tts.models.xtts import Xtts
+                    checkpoint_path = kwargs.get('checkpoint_path')
+                    config_path = kwargs.get('config_path', None)
+                    vocab_path = kwargs.get('vocab_path', None)
+                    config = XttsConfig()
+                    config.models_dir = os.path.join("models", "tts")
+                    config.load_json(config_path)
+                    tts = Xtts.init_from_config(config)
+                    tts.load_checkpoint(
+                        config,
+                        checkpoint_path=checkpoint_path,
+                        vocab_path=vocab_path,
+                        use_deepspeed=default_engine_settings[TTS_ENGINES['XTTSv2']]['use_deepspeed'],
+                        eval=True
+                    )
+                elif tts_engine == TTS_ENGINES['BARK']:
+                    from TTS.tts.configs.bark_config import BarkConfig
+                    from TTS.tts.models.bark import Bark
+                    checkpoint_dir = kwargs.get('checkpoint_dir')
+                    config = BarkConfig()
+                    config.CACHE_DIR = self.cache_dir
+                    config.USE_SMALLER_MODELS = os.environ.get('SUNO_USE_SMALL_MODELS', '').lower() == 'true'
+                    tts = Bark.init_from_config(config)
+                    tts.load_checkpoint(
+                        config,
+                        checkpoint_dir=checkpoint_dir,
+                        eval=True
+                    )                    
+            if tts:
+                if device == 'cuda':
+                    tts.cuda()
+                else:
+                    tts.to(device)
+                loaded_tts[key] = {"engine": tts, "config": config}
+                msg = f'{tts_engine} Loaded!'
+                print(msg)
+                return tts
+            else:
+                error = 'TTS engine could not be created!'
+                print(error)
+        except Exception as e:
+            error = f'_load_checkpoint() error: {e}'
+        return False
+
+    def _check_xtts_builtin_speakers(self, voice_path, speaker, device):
+        try:
+            voice_parts = Path(voice_path).parts
+            if self.session['language'] not in voice_parts and speaker not in default_engine_settings[TTS_ENGINES['BARK']]['voices'].keys() and self.session['language'] != 'eng':
+                if self.session['language'] in language_tts[TTS_ENGINES['XTTSv2']].keys():
+                    default_text_file = os.path.join(voices_dir, self.session['language'], 'default.txt')
+                    if os.path.exists(default_text_file):
+                        msg = f"Converting builtin eng voice to {self.session['language']}..."
+                        print(msg)
+                        tts_internal_key = f"{TTS_ENGINES['XTTSv2']}-internal"
+                        default_text = Path(default_text_file).read_text(encoding="utf-8")
+                        hf_repo = models[TTS_ENGINES['XTTSv2']]['internal']['repo']
+                        hf_sub = ''
+                        tts = (loaded_tts.get(tts_internal_key) or {}).get('engine', False)
+                        if not tts:
+                            for key in list(loaded_tts.keys()): unload_tts(device, None, key)
+                            config_path = hf_hub_download(repo_id=hf_repo, filename=f"{hf_sub}{models[TTS_ENGINES['XTTSv2']]['internal']['files'][0]}", cache_dir=self.cache_dir)
+                            checkpoint_path = hf_hub_download(repo_id=hf_repo, filename=f"{hf_sub}{models[TTS_ENGINES['XTTSv2']]['internal']['files'][1]}", cache_dir=self.cache_dir)
+                            vocab_path = hf_hub_download(repo_id=hf_repo, filename=f"{hf_sub}{models[TTS_ENGINES['XTTSv2']]['internal']['files'][2]}", cache_dir=self.cache_dir)
+                            tts = self._load_checkpoint(tts_engine=TTS_ENGINES['XTTSv2'], key=tts_internal_key, checkpoint_path=checkpoint_path, config_path=config_path, vocab_path=vocab_path, device=device)
+                        if tts:
+                            if speaker in default_engine_settings[TTS_ENGINES['XTTSv2']]['voices'].keys():
+                                gpt_cond_latent, speaker_embedding = xtts_builtin_speakers_list[default_engine_settings[TTS_ENGINES['XTTSv2']]['voices'][speaker]].values()
+                            else:
+                                gpt_cond_latent, speaker_embedding = tts.get_conditioning_latents(audio_path=[voice_path])
+                            fine_tuned_params = {
+                                key: cast_type(self.session[key])
+                                for key, cast_type in {
+                                    "temperature": float,
+                                    "length_penalty": float,
+                                    "num_beams": int,
+                                    "repetition_penalty": float,
+                                    "top_k": int,
+                                    "top_p": float,
+                                    "speed": float,
+                                    "enable_text_splitting": bool
+                                }.items()
+                                if self.session.get(key) is not None
+                            }
+                            with torch.no_grad():
+                                result = tts.inference(
+                                    text=default_text,
+                                    language=self.session['language_iso1'],
+                                    gpt_cond_latent=gpt_cond_latent,
+                                    speaker_embedding=speaker_embedding,
+                                    **fine_tuned_params
+                                )
+                            audio_data = result.get('wav')
+                            if audio_data is not None:
+                                audio_data = audio_data.tolist()
+                                sourceTensor = self._tensor_type(audio_data)
+                                audio_tensor = sourceTensor.clone().detach().unsqueeze(0).cpu()
+                                lang_dir = 'con-' if self.session['language'] == 'con' else self.session['language']
+                                new_voice_path = re.sub(r'([\\/])eng([\\/])', rf'\1{lang_dir}\2', voice_path)
+                                proc_voice_path = new_voice_path.replace('.wav', '_temp.wav')
+                                torchaudio.save(proc_voice_path, audio_tensor, default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate'], format='wav')
+                                if normalize_audio(proc_voice_path, new_voice_path, default_audio_proc_samplerate):
+                                    del audio_data, sourceTensor, audio_tensor  
+                                    if self.session['tts_engine'] != TTS_ENGINES['XTTSv2']:
+                                        del tts
+                                        unload_tts(device, None, tts_internal_key)
+                                    return new_voice_path
+                                else:
+                                    error = 'normalize_audio() error:' 
+                            else:
+                                error = f'No audio waveform found in _check_xtts_builtin_speakers() result: {result}'
+                        else:
+                            error = f"_check_xtts_builtin_speakers() error: {TTS_ENGINES['XTTSv2']} is False"
+                    else:
+                        error = f'The translated {default_text_file} could not be found! Voice cloning file will stay in English.'
+                    print(error)
+                else:
+                    return voice_path
+            else:
+                return voice_path
+        except Exception as e:
+            error = f'_check_xtts_builtin_speakers() error: {e}'
+            print(error)
+        return False
+
+    def _check_bark_npz(self, voice_path, bark_dir, speaker, device):
+        try:
+            if self.session['language'] in language_tts[TTS_ENGINES['BARK']].keys():
+                npz_dir = os.path.join(bark_dir, speaker)
+                npz_file = os.path.join(npz_dir, f'{speaker}.npz')
+                if os.path.exists(npz_file):
+                    return True
+                else:
+                    os.makedirs(npz_dir, exist_ok=True)
+                    tts_internal_key = f"{TTS_ENGINES['BARK']}-internal"
+                    hf_repo = models[TTS_ENGINES['BARK']]['internal']['repo']
+                    hf_sub = models[TTS_ENGINES['BARK']]['internal']['sub']
+                    tts = (loaded_tts.get(tts_internal_key) or {}).get('engine', False)
+                    if not tts:
+                        for key in list(loaded_tts.keys()): unload_tts(device, None, key)
+                        text_model_path = hf_hub_download(repo_id=hf_repo, filename=f"{hf_sub}{models[TTS_ENGINES['BARK']]['internal']['files'][0]}", cache_dir=self.cache_dir)
+                        coarse_model_path = hf_hub_download(repo_id=hf_repo, filename=f"{hf_sub}{models[TTS_ENGINES['BARK']]['internal']['files'][1]}", cache_dir=self.cache_dir)
+                        fine_model_path = hf_hub_download(repo_id=hf_repo, filename=f"{hf_sub}{models[TTS_ENGINES['BARK']]['internal']['files'][2]}", cache_dir=self.cache_dir)
+                        checkpoint_dir = os.path.dirname(text_model_path)
+                        tts = self._load_checkpoint(tts_engine=TTS_ENGINES['BARK'], key=tts_internal_key, checkpoint_dir=checkpoint_dir, device=device)
+                    if tts:
+                        voice_temp = os.path.splitext(npz_file)[0]+'.wav'
+                        shutil.copy(voice_path, voice_temp)
+                        default_text_file = os.path.join(voices_dir, self.session['language'], 'default.txt')
+                        default_text = Path(default_text_file).read_text(encoding="utf-8")
+                        fine_tuned_params = {
+                            key: cast_type(self.session[key])
+                            for key, cast_type in {
+                                "text_temp": float,
+                                "waveform_temp": float
+                            }.items()
+                            if self.session.get(key) is not None
+                        }
+                        with torch.no_grad():
+                            torch.manual_seed(67878789)
+                            audio_data = tts.synthesize(
+                                default_text,
+                                loaded_tts[tts_internal_key]['config'],
+                                speaker_id=speaker,
+                                voice_dirs=bark_dir,
+                                silent=True,
+                                **fine_tuned_params
+                            )
+                        os.remove(voice_temp)
+                        del audio_data
+                        if self.session['tts_engine'] != TTS_ENGINES['BARK']:
+                            del tts
+                            unload_tts(device, None, tts_internal_key)
+                        msg = f"Saved NPZ file: {npz_file}"
+                        print(msg)
+                        return True
+                    else:
+                        error = f'_check_bark_npz() error: {tts_internal_key} is False'
+                        print(error)
+            else:
+                return True
+        except Exception as e:
+            error = f'_check_bark_npz() error: {e}'
+            print(error)
+        return False
+        
+    def _tensor_type(self, audio_data):
+        if isinstance(audio_data, torch.Tensor):
+            return audio_data
+        elif isinstance(audio_data, np.ndarray):  
+            return torch.from_numpy(audio_data).float()
+        elif isinstance(audio_data, list):  
+            return torch.tensor(audio_data, dtype=torch.float32)
+        else:
+            raise TypeError(f"Unsupported type for audio_data: {type(audio_data)}")
+            
+    def _get_resampler(self, orig_sr, target_sr):
+        key = (orig_sr, target_sr)
+        if key not in self.resampler_cache:
+            self.resampler_cache[key] = torchaudio.transforms.Resample(
+                orig_freq=orig_sr, new_freq=target_sr
+            )
+        return self.resampler_cache[key]
+
+    def _resample_wav(self, wav_path, expected_sr):
+        waveform, orig_sr = torchaudio.load(wav_path)
+        if orig_sr == expected_sr and waveform.size(0) == 1:
+            return wav_path
+        if waveform.size(0) > 1:
+            waveform = waveform.mean(dim=0, keepdim=True)
+        if orig_sr != expected_sr:
+            resampler = self._get_resampler(orig_sr, expected_sr)
+            waveform = resampler(waveform)
+        wav_tensor = waveform.squeeze(0)
+        wav_numpy = wav_tensor.cpu().numpy()
+        tmp_fh = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
+        tmp_path = tmp_fh.name
+        tmp_fh.close()
+        sf.write(tmp_path, wav_numpy, expected_sr, subtype="PCM_16")
+        return tmp_path
+
+    def convert(self, s_n, s):
+        global xtts_builtin_speakers_list
+        try:
+            sentence_number = s_n
+            sentence = s
+            speaker = None
+            audio_data = False
+            trim_audio_buffer = 0.004
+            settings = self.params[self.session['tts_engine']]
+            final_sentence_file = os.path.join(self.session['chapters_dir_sentences'], f'{sentence_number}.{default_audio_proc_format}')
+            settings['voice_path'] = (
+                self.session['voice'] if self.session['voice'] is not None 
+                else os.path.join(self.session['custom_model_dir'], self.session['tts_engine'], self.session['custom_model'], 'ref.wav') if self.session['custom_model'] is not None
+                else models[self.session['tts_engine']][self.session['fine_tuned']]['voice']
+            )
+            if settings['voice_path'] is not None:
+                speaker = re.sub(r'\.wav$', '', os.path.basename(settings['voice_path']))
+                if settings['voice_path'] not in default_engine_settings[TTS_ENGINES['BARK']]['voices'].keys() and os.path.basename(settings['voice_path']) != 'ref.wav':
+                    self.session['voice'] = settings['voice_path'] = self._check_xtts_builtin_speakers(settings['voice_path'], speaker, self.session['device'])
+                    if not settings['voice_path']:
+                        msg = f"Could not create the builtin speaker selected voice in {self.session['language']}"
+                        print(msg)
+                        return False
+            tts = (loaded_tts.get(self.tts_key) or {}).get('engine', False)
+            if tts:
+                if sentence == TTS_SML['break']:
+                    silence_time = int(np.random.uniform(0.3, 0.6) * 100) / 100
+                    break_tensor = torch.zeros(1, int(settings['samplerate'] * silence_time)) # 0.4 to 0.7 seconds
+                    self.audio_segments.append(break_tensor.clone())
+                    return True
+                elif sentence == TTS_SML['pause']:
+                    silence_time = int(np.random.uniform(1.0, 1.8) * 100) / 100
+                    pause_tensor = torch.zeros(1, int(settings['samplerate'] * silence_time)) # 1.0 to 1.8 seconds
+                    self.audio_segments.append(pause_tensor.clone())
+                    return True
+                else:
+                    if sentence[-1].isalnum():
+                        sentence = f'{sentence} —'
+                    if self.session['tts_engine'] == TTS_ENGINES['XTTSv2']:
+                        trim_audio_buffer = 0.008
+                        if settings['voice_path'] is not None and settings['voice_path'] in settings['latent_embedding'].keys():
+                            settings['gpt_cond_latent'], settings['speaker_embedding'] = settings['latent_embedding'][settings['voice_path']]
+                        else:
+                            msg = 'Computing speaker latents...'
+                            print(msg)
+                            if speaker in default_engine_settings[TTS_ENGINES['XTTSv2']]['voices'].keys():
+                                settings['gpt_cond_latent'], settings['speaker_embedding'] = xtts_builtin_speakers_list[default_engine_settings[TTS_ENGINES['XTTSv2']]['voices'][speaker]].values()
+                            else:
+                                settings['gpt_cond_latent'], settings['speaker_embedding'] = tts.get_conditioning_latents(audio_path=[settings['voice_path']])  
+                            settings['latent_embedding'][settings['voice_path']] = settings['gpt_cond_latent'], settings['speaker_embedding']
+                        fine_tuned_params = {
+                            key: cast_type(self.session[key])
+                            for key, cast_type in {
+                                "temperature": float,
+                                "length_penalty": float,
+                                "num_beams": int,
+                                "repetition_penalty": float,
+                                "top_k": int,
+                                "top_p": float,
+                                "speed": float,
+                                "enable_text_splitting": bool
+                            }.items()
+                            if self.session.get(key) is not None
+                        }
+                        with torch.no_grad():
+                            result = tts.inference(
+                                text=sentence.replace('.', ' —'),
+                                language=self.session['language_iso1'],
+                                gpt_cond_latent=settings['gpt_cond_latent'],
+                                speaker_embedding=settings['speaker_embedding'],
+                                **fine_tuned_params
+                            )
+                        audio_sentence = result.get('wav')
+                        if is_audio_data_valid(audio_sentence):
+                            audio_sentence = audio_sentence.tolist()
+                    elif self.session['tts_engine'] == TTS_ENGINES['BARK']:
+                        trim_audio_buffer = 0.002
+                        '''
+                            [laughter]
+                            [laughs]
+                            [sighs]
+                            [music]
+                            [gasps]
+                            [clears throat]
+                            — or ... for hesitations
+                            ♪ for song lyrics
+                            CAPITALIZATION for emphasis of a word
+                            [MAN] and [WOMAN] to bias Bark toward male and female speakers, respectively
+                        '''
+                        if speaker in default_engine_settings[self.session['tts_engine']]['voices'].keys():
+                            bark_dir = default_engine_settings[self.session['tts_engine']]['speakers_path']
+                        else:
+                            bark_dir = os.path.join(os.path.dirname(settings['voice_path']), 'bark')       
+                            if not self._check_bark_npz(settings['voice_path'], bark_dir, speaker, self.session['device']):
+                                error = 'Could not create npz file!'
+                                print(error)
+                                return False
+                        npz_file = os.path.join(bark_dir, speaker, f'{speaker}.npz')
+                        fine_tuned_params = {
+                            key: cast_type(self.session[key])
+                            for key, cast_type in {
+                                "text_temp": float,
+                                "waveform_temp": float
+                            }.items()
+                            if self.session.get(key) is not None
+                        }
+                        if self.npz_path is None or self.npz_path != npz_file:
+                            self.npz_path = npz_file
+                            self.npz_data = np.load(self.npz_path, allow_pickle=True)
+                        history_prompt = [
+                                self.npz_data["semantic_prompt"],
+                                self.npz_data["coarse_prompt"],
+                                self.npz_data["fine_prompt"]
+                        ]
+                        with torch.no_grad():
+                            torch.manual_seed(67878789)
+                            audio_sentence, _ = tts.generate_audio(
+                                sentence,
+                                history_prompt=history_prompt,
+                                silent=True,
+                                **fine_tuned_params
+                            )
+                        if is_audio_data_valid(audio_sentence):
+                            audio_sentence = audio_sentence.tolist()
+                    elif self.session['tts_engine'] == TTS_ENGINES['VITS']:
+                        speaker_argument = {}
+                        if self.session['language'] == 'eng' and 'vctk/vits' in models[self.session['tts_engine']]['internal']['sub']:
+                            if self.session['language'] in models[self.session['tts_engine']]['internal']['sub']['vctk/vits'] or self.session['language_iso1'] in models[self.session['tts_engine']]['internal']['sub']['vctk/vits']:
+                                speaker_argument = {"speaker": 'p262'}
+                        elif self.session['language'] == 'cat' and 'custom/vits' in models[self.session['tts_engine']]['internal']['sub']:
+                            if self.session['language'] in models[self.session['tts_engine']]['internal']['sub']['custom/vits'] or self.session['language_iso1'] in models[self.session['tts_engine']]['internal']['sub']['custom/vits']:
+                                speaker_argument = {"speaker": '09901'}
+                        if settings['voice_path'] is not None:
+                            proc_dir = os.path.join(self.session['voice_dir'], 'proc')
+                            os.makedirs(proc_dir, exist_ok=True)
+                            tmp_in_wav = os.path.join(proc_dir, f"{uuid.uuid4()}.wav")
+                            tmp_out_wav = os.path.join(proc_dir, f"{uuid.uuid4()}.wav")
+                            tts.tts_to_file(
+                                text=sentence,
+                                file_path=tmp_in_wav,
+                                **speaker_argument
+                            )
+                            if settings['voice_path'] in settings['semitones'].keys():
+                                semitones = settings['semitones'][settings['voice_path']]
+                            else:
+                                voice_path_gender = detect_gender(settings['voice_path'])
+                                voice_builtin_gender = detect_gender(tmp_in_wav)
+                                msg = f"Cloned voice seems to be {voice_path_gender}\nBuiltin voice seems to be {voice_builtin_gender}"
+                                print(msg)
+                                if voice_builtin_gender != voice_path_gender:
+                                    semitones = -4 if voice_path_gender == 'male' else 4
+                                    msg = f"Adapting builtin voice frequencies from the clone voice..."
+                                    print(msg)
+                                else:
+                                    semitones = 0
+                                settings['semitones'][settings['voice_path']] = semitones
+                            if semitones > 0:
+                                try:
+                                    cmd = [
+                                        shutil.which('sox'), tmp_in_wav,
+                                        "-r", str(settings['samplerate']), tmp_out_wav,
+                                        "pitch", str(semitones * 100)
+                                    ]
+                                    subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+                                except subprocess.CalledProcessError as e:
+                                    error = f"Subprocess error: {e.stderr}"
+                                    print(error)
+                                    DependencyError(e)
+                                    return False
+                                except FileNotFoundError as e:
+                                    error = f"File not found: {e}"
+                                    print(error)
+                                    DependencyError(e)
+                                    return False
+                            else:
+                                tmp_out_wav = tmp_in_wav
+                            tts_vc = (loaded_tts.get(self.tts_vc_key) or {}).get('engine', False)
+                            if tts_vc:
+                                settings['samplerate'] = TTS_VOICE_CONVERSION[self.tts_vc_key]['samplerate']
+                                source_wav = self._resample_wav(tmp_out_wav, settings['samplerate'])
+                                target_wav = self._resample_wav(settings['voice_path'], settings['samplerate'])
+                                audio_sentence = tts_vc.voice_conversion(
+                                    source_wav=source_wav,
+                                    target_wav=target_wav
+                                )
+                            else:
+                                error = f'Engine {self.tts_vc_key} is None'
+                                print(error)
+                                return False
+                            if os.path.exists(tmp_in_wav):
+                                os.remove(tmp_in_wav)
+                            if os.path.exists(tmp_out_wav):
+                                os.remove(tmp_out_wav)
+                            if os.path.exists(source_wav):
+                                os.remove(source_wav)
+                        else:
+                            audio_sentence = tts.tts(
+                                text=sentence,
+                                **speaker_argument
+                            )
+                    elif self.session['tts_engine'] == TTS_ENGINES['FAIRSEQ']:
+                        speaker_argument = {}
+                        not_supported_punc_pattern = re.compile(r"[.:—]")
+                        if settings['voice_path'] is not None:
+                            proc_dir = os.path.join(self.session['voice_dir'], 'proc')
+                            os.makedirs(proc_dir, exist_ok=True)
+                            tmp_in_wav = os.path.join(proc_dir, f"{uuid.uuid4()}.wav")
+                            tmp_out_wav = os.path.join(proc_dir, f"{uuid.uuid4()}.wav")
+                            tts.tts_to_file(
+                                text=re.sub(not_supported_punc_pattern, ' ', sentence),
+                                file_path=tmp_in_wav,
+                                **speaker_argument
+                            )
+                            if settings['voice_path'] in settings['semitones'].keys():
+                                semitones = settings['semitones'][settings['voice_path']]
+                            else:
+                                voice_path_gender = detect_gender(settings['voice_path'])
+                                voice_builtin_gender = detect_gender(tmp_in_wav)
+                                msg = f"Cloned voice seems to be {voice_path_gender}\nBuiltin voice seems to be {voice_builtin_gender}"
+                                print(msg)
+                                if voice_builtin_gender != voice_path_gender:
+                                    semitones = -4 if voice_path_gender == 'male' else 4
+                                    msg = f"Adapting builtin voice frequencies from the clone voice..."
+                                    print(msg)
+                                else:
+                                    semitones = 0
+                                settings['semitones'][settings['voice_path']] = semitones
+                            if semitones > 0:
+                                try:
+                                    cmd = [
+                                        shutil.which('sox'), tmp_in_wav,
+                                        "-r", str(settings['samplerate']), tmp_out_wav,
+                                        "pitch", str(semitones * 100)
+                                    ]
+                                    subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+                                except subprocess.CalledProcessError as e:
+                                    print(f"Subprocess error: {e.stderr}")
+                                    DependencyError(e)
+                                    return False
+                                except FileNotFoundError as e:
+                                    print(f"File not found: {e}")
+                                    DependencyError(e)
+                                    return False
+                            else:
+                                tmp_out_wav = tmp_in_wav
+                            tts_vc = (loaded_tts.get(self.tts_vc_key) or {}).get('engine', False)
+                            if tts_vc:
+                                settings['samplerate'] = TTS_VOICE_CONVERSION[self.tts_vc_key]['samplerate']
+                                source_wav = self._resample_wav(tmp_out_wav, settings['samplerate'])
+                                target_wav = self._resample_wav(settings['voice_path'], settings['samplerate'])
+                                audio_sentence = tts_vc.voice_conversion(
+                                    source_wav=source_wav,
+                                    target_wav=target_wav
+                                )
+                            else:
+                                error = f'Engine {self.tts_vc_key} is None'
+                                print(error)
+                                return False
+                            if os.path.exists(tmp_in_wav):
+                                os.remove(tmp_in_wav)
+                            if os.path.exists(tmp_out_wav):
+                                os.remove(tmp_out_wav)
+                            if os.path.exists(source_wav):
+                                os.remove(source_wav)
+                        else:
+                            audio_sentence = tts.tts(
+                                text=re.sub(not_supported_punc_pattern, ' ', sentence),
+                                **speaker_argument
+                            )
+                    elif self.session['tts_engine'] == TTS_ENGINES['TACOTRON2']:
+                        speaker_argument = {}
+                        not_supported_punc_pattern = re.compile(r'["—]')
+                        if settings['voice_path'] is not None:
+                            proc_dir = os.path.join(self.session['voice_dir'], 'proc')
+                            os.makedirs(proc_dir, exist_ok=True)
+                            tmp_in_wav = os.path.join(proc_dir, f"{uuid.uuid4()}.wav")
+                            tmp_out_wav = os.path.join(proc_dir, f"{uuid.uuid4()}.wav")
+                            tts.tts_to_file(
+                                text=re.sub(not_supported_punc_pattern, '', sentence),
+                                file_path=tmp_in_wav,
+                                **speaker_argument
+                            )
+                            if settings['voice_path'] in settings['semitones'].keys():
+                                semitones = settings['semitones'][settings['voice_path']]
+                            else:
+                                voice_path_gender = detect_gender(settings['voice_path'])
+                                voice_builtin_gender = detect_gender(tmp_in_wav)
+                                msg = f"Cloned voice seems to be {voice_path_gender}\nBuiltin voice seems to be {voice_builtin_gender}"
+                                print(msg)
+                                if voice_builtin_gender != voice_path_gender:
+                                    semitones = -4 if voice_path_gender == 'male' else 4
+                                    msg = f"Adapting builtin voice frequencies from the clone voice..."
+                                    print(msg)
+                                else:
+                                    semitones = 0
+                                settings['semitones'][settings['voice_path']] = semitones
+                            if semitones > 0:
+                                try:
+                                    cmd = [
+                                        shutil.which('sox'), tmp_in_wav,
+                                        "-r", str(settings['samplerate']), tmp_out_wav,
+                                        "pitch", str(semitones * 100)
+                                    ]
+                                    subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+                                except subprocess.CalledProcessError as e:
+                                    error = f"Subprocess error: {e.stderr}"
+                                    print(error)
+                                    DependencyError(e)
+                                    return False
+                                except FileNotFoundError as e:
+                                    error = f"File not found: {e}"
+                                    print(error)
+                                    DependencyError(e)
+                                    return False
+                            else:
+                                tmp_out_wav = tmp_in_wav
+                            tts_vc = (loaded_tts.get(self.tts_vc_key) or {}).get('engine', False)
+                            if tts_vc:
+                                settings['samplerate'] = TTS_VOICE_CONVERSION[self.tts_vc_key]['samplerate']
+                                source_wav = self._resample_wav(tmp_out_wav, settings['samplerate'])
+                                target_wav = self._resample_wav(settings['voice_path'], settings['samplerate'])
+                                audio_sentence = tts_vc.voice_conversion(
+                                    source_wav=source_wav,
+                                    target_wav=target_wav
+                                )
+                            else:
+                                error = f'Engine {self.tts_vc_key} is None'
+                                print(error)
+                                return False
+                            if os.path.exists(tmp_in_wav):
+                                os.remove(tmp_in_wav)
+                            if os.path.exists(tmp_out_wav):
+                                os.remove(tmp_out_wav)
+                            if os.path.exists(source_wav):
+                                os.remove(source_wav)
+                        else:
+                            audio_sentence = tts.tts(
+                                text=re.sub(not_supported_punc_pattern, '', sentence),
+                                **speaker_argument
+                            )
+                    elif self.session['tts_engine'] == TTS_ENGINES['YOURTTS']:
+                        speaker_argument = {}
+                        language = self.session['language_iso1'] if self.session['language_iso1'] == 'en' else 'fr-fr' if self.session['language_iso1'] == 'fr' else 'pt-br' if self.session['language_iso1'] == 'pt' else 'en'
+                        if settings['voice_path'] is not None:
+                            speaker_wav = settings['voice_path']
+                            speaker_argument = {"speaker_wav": speaker_wav}
+                        else:
+                            voice_key = default_engine_settings[TTS_ENGINES['YOURTTS']]['voices']['ElectroMale-2']
+                            speaker_argument = {"speaker": voice_key}
+                        with torch.no_grad():
+                            audio_sentence = tts.tts(
+                                text=sentence.replace('—', '').strip(),
+                                language=language,
+                                **speaker_argument
+                            )
+                    if is_audio_data_valid(audio_sentence):
+                        sourceTensor = self._tensor_type(audio_sentence)
+                        audio_tensor = sourceTensor.clone().detach().unsqueeze(0).cpu()
+                        if sentence[-1].isalnum() or sentence[-1] == '—':
+                            audio_tensor = trim_audio(audio_tensor.squeeze(), settings['samplerate'], 0.003, trim_audio_buffer).unsqueeze(0)
+                        self.audio_segments.append(audio_tensor)
+                        if not re.search(r'\w$', sentence, flags=re.UNICODE):
+                            silence_time = int(np.random.uniform(0.3, 0.6) * 100) / 100
+                            break_tensor = torch.zeros(1, int(settings['samplerate'] * silence_time))
+                            self.audio_segments.append(break_tensor.clone())
+                        if self.audio_segments:
+                            audio_tensor = torch.cat(self.audio_segments, dim=-1)
+                            start_time = self.sentences_total_time
+                            duration = round((audio_tensor.shape[-1] / settings['samplerate']), 2)
+                            end_time = start_time + duration
+                            self.sentences_total_time = end_time
+                            sentence_obj = {
+                                "start": start_time,
+                                "end": end_time,
+                                "text": sentence,
+                                "resume_check": self.sentence_idx
+                            }
+                            self.sentence_idx = append_sentence2vtt(sentence_obj, self.vtt_path)
+                            if self.sentence_idx:
+                                torchaudio.save(final_sentence_file, audio_tensor, settings['samplerate'], format=default_audio_proc_format)
+                                del audio_tensor
+                        self.audio_segments = []
+                        if os.path.exists(final_sentence_file):
+                            return True
+                        else:
+                            error = f"Cannot create {final_sentence_file}"
+                            print(error)
+            else:
+                error = f"convert() error: {self.session['tts_engine']} is None"
+                print(error)
+        except Exception as e:
+            error = f'Coquit.convert(): {e}'
+            raise ValueError(e)
+        return False

lib/classes/tts_manager.py

@@ -0,0 +1,37 @@
+import os
+
+from lib.models import TTS_ENGINES
+
+class TTSManager:
+    def __init__(self, session):   
+        self.session = session
+        self.tts = None
+        self._build()
+ 
+    def _build(self):
+        if self.session['tts_engine'] in TTS_ENGINES.values():
+            if self.session['tts_engine'] in [TTS_ENGINES['XTTSv2'], TTS_ENGINES['BARK'], TTS_ENGINES['VITS'], TTS_ENGINES['FAIRSEQ'], TTS_ENGINES['TACOTRON2'], TTS_ENGINES['YOURTTS']]:
+                from lib.classes.tts_engines.coqui import Coqui
+                self.tts = Coqui(self.session)
+            #elif self.session['tts_engine'] in [TTS_ENGINES['NEW_TTS']]:
+            #    from lib.classes.tts_engines.new_tts import NewTts
+            #    self.tts = NewTts(self.session)
+            if self.tts:
+                return True
+            else:
+                error = 'TTS engine could not be created!'
+                print(error)
+        else:
+            print('Other TTS engines coming soon!')
+        return False
+
+    def convert_sentence2audio(self, sentence_number, sentence):
+        try:
+            if self.session['tts_engine'] in TTS_ENGINES.values():
+                return self.tts.convert(sentence_number, sentence)
+            else:
+                print('Other TTS engines coming soon!')    
+        except Exception as e:
+            error = f'convert_sentence2audio(): {e}'
+            raise ValueError(e)
+        return False

lib/classes/voice_extractor.py

@@ -0,0 +1,286 @@
+import os
+import numpy as np
+import regex as re
+import scipy.fftpack
+import soundfile as sf
+import subprocess
+import shutil
+
+from io import BytesIO
+from pydub import AudioSegment, silence
+from pydub.silence import detect_silence
+
+from lib.conf import voice_formats, default_audio_proc_samplerate
+from lib.models import TTS_ENGINES, models
+from lib.classes.background_detector import BackgroundDetector
+
+class VoiceExtractor:
+
+    def __init__(self, session, voice_file, voice_name):
+        self.wav_file = None
+        self.session = session
+        self.voice_file = voice_file
+        self.voice_name = voice_name
+        self.voice_track = 'vocals.wav'
+        self.samplerate = models[session['tts_engine']][session['fine_tuned']]['samplerate']
+        self.output_dir = self.session['voice_dir']
+        self.demucs_dir = os.path.join(self.output_dir, 'htdemucs', voice_name)
+        self.silence_threshold = -60
+
+    def _validate_format(self):
+        file_extension = os.path.splitext(self.voice_file)[1].lower()
+        if file_extension in voice_formats:
+            msg = 'Input file valid'
+            return True, msg
+        error = f'Unsupported file format: {file_extension}. Supported formats are: {", ".join(voice_formats)}'
+        return False, error
+
+    def _convert2wav(self):
+        try:
+            self.wav_file = os.path.join(self.session['voice_dir'], f'{self.voice_name}.wav')
+            ffmpeg_cmd = [
+                shutil.which('ffmpeg'), '-hide_banner', '-nostats', '-i', self.voice_file,
+                '-ac', '1',
+                '-y', self.wav_file
+            ]
+            process = subprocess.Popen(
+                ffmpeg_cmd,
+                env={},
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                universal_newlines=True,
+                encoding='utf-8'
+            )
+            for line in process.stdout:
+                print(line, end='')  # Print each line of stdout
+            process.wait()
+            if process.returncode != 0:
+                error = f'_convert2wav(): process.returncode: {process.returncode}'
+            elif not os.path.exists(self.wav_file) or os.path.getsize(self.wav_file) == 0:
+                error = f'_convert2wav output error: {self.wav_file} was not created or is empty.'                
+            else:
+                msg = 'Conversion to .wav format for processing successful'
+                return True, msg
+        except subprocess.CalledProcessError as e:
+            error = f'convert2wav fmpeg.Error: {e.stderr.decode()}'
+            raise ValueError(error)
+        except Exception as e:
+            error = f'_convert2wav() error: {e}'
+            raise ValueError(error)
+        return False, error
+
+    def _detect_background(self):
+        try:
+            msg = 'Detecting any background noise or music...'
+            print(msg)
+            detector = BackgroundDetector(wav_file=self.wav_file)
+            status, report = detector.detect(vad_ratio_thresh=0.15)
+            print(report)
+            if status:
+                msg = 'Background noise or music detected. Proceeding voice extraction...'
+            else:
+                msg = 'No background noise or music detected. Skipping separation...'
+            return True, status, msg
+        except Exception as e:
+            error = f'_detect_background() error: {e}'
+            raise ValueError(error)
+            return False, False, error
+
+    def _demucs_voice(self):
+        try:             
+            cmd = [
+                "demucs",
+                "--verbose",
+                "--two-stems=vocals",
+                "--out", self.output_dir,
+                self.wav_file
+            ]
+            try:
+                process = subprocess.run(cmd, check=True)
+                self.voice_track = os.path.join(self.demucs_dir, self.voice_track)
+                msg = 'Voice track isolation successful'
+                return True, msg
+            except subprocess.CalledProcessError as e:
+                error = (
+                    f'_demucs_voice() subprocess CalledProcessError error: {e.returncode}\n\n'
+                    f'stdout: {e.output}\n\n'
+                    f'stderr: {e.stderr}'
+                )
+                raise ValueError(error)
+            except FileNotFoundError:
+                error = f'_demucs_voice() subprocess FileNotFoundError error: The "demucs" command was not found. Ensure it is installed and in PATH.'
+                raise ValueError(error)
+            except Exception as e:              
+                error = f'_demucs_voice() subprocess Exception error: {str(e)}'
+                raise ValueError(error)
+        except Exception as e:
+            error = f'_demucs_voice() error: {e}'
+            raise ValueError(error)
+        return False, error
+
+    def _remove_silences(self, audio, silence_threshold, min_silence_len=200, keep_silence=300):
+        final_audio = AudioSegment.silent(duration=0)
+        chunks = silence.split_on_silence(
+            audio,
+            min_silence_len=min_silence_len,
+            silence_thresh=silence_threshold,
+            keep_silence=keep_silence
+        )
+        for chunk in chunks:
+            final_audio += chunk
+        final_audio.export(self.voice_track, format='wav')
+    
+    def _trim_and_clean(self,silence_threshold, min_silence_len=200, chunk_size=100):
+        try:
+            audio = AudioSegment.from_file(self.voice_track)
+            total_duration = len(audio)  # Total duration in milliseconds
+            min_required_duration = 20000 if self.session['tts_engine'] == TTS_ENGINES['BARK'] else 12000
+            msg = f"Removing long pauses..."
+            print(msg)
+            self._remove_silences(audio, silence_threshold)
+            if total_duration <= min_required_duration:
+                msg = f"Audio is only {total_duration/1000:.2f}s long; skipping audio trimming..."
+                return True, msg
+            else:
+                if total_duration > (min_required_duration * 2):
+                    msg = f"Audio longer than the max allowed. Proceeding to audio trimming..."       
+                    print(msg)
+                    window = min_required_duration
+                    hop = max(1, window // 4)
+                    best_var   = -float("inf")
+                    best_start = 0
+                    sr = audio.frame_rate
+                    for start in range(0, total_duration - window + 1, hop):
+                        chunk   = audio[start : start + window]
+                        samples = np.array(chunk.get_array_of_samples()).astype(float)
+                        # 1) FFT + magnitude
+                        spectrum = np.abs(scipy.fftpack.fft(samples))
+                        # 2) turn into a probability distribution
+                        p = spectrum / (np.sum(spectrum) + 1e-10)
+                        # 3) spectral entropy
+                        entropy = -np.sum(p * np.log2(p + 1e-10))
+                        if entropy > best_var:
+                            best_var   = entropy
+                            best_start = start
+                    best_end = best_start + window
+                    msg = (
+                        f"Selected most‐diverse‐spectrum window "
+                        f"{best_start/1000:.2f}s–{best_end/1000:.2f}s "
+                        f"(@ entropy {best_var:.2f} bits)"
+                    )
+                    print(msg)
+                    # 1) find all silent spans in the file
+                    silence_spans = detect_silence(
+                        audio,
+                        min_silence_len=min_silence_len,
+                        silence_thresh=silence_threshold
+                    )
+                    # silence_spans = [ [start_ms, end_ms], … ]
+                    # 2) snap best_start *backward* to the end of the last silence before it
+                    prev_ends = [end for (start, end) in silence_spans if end <= best_start]
+                    if prev_ends:
+                        new_start = max(prev_ends)
+                    else:
+                        new_start = 0
+                    # 3) snap best_end *forward* to the start of the first silence after it
+                    next_starts = [start for (start, end) in silence_spans if start >= best_end]
+                    if next_starts:
+                        new_end = min(next_starts)
+                    else:
+                        new_end = total_duration
+                    # 4) update your slice bounds
+                    best_start, best_end = new_start, new_end
+                else:
+                    best_start = 0
+                    best_end = total_duration
+            trimmed_audio = audio[best_start:best_end]
+            trimmed_audio.export(self.voice_track, format='wav')
+            msg = 'Audio trimmed and cleaned!'
+            return True, msg
+        except Exception as e:
+            error = f'_trim_and_clean() error: {e}'
+            raise ValueError(error)
+
+    def _normalize_audio(self):
+        error = ''
+        try:
+            proc_voice_file = os.path.join(self.session['voice_dir'], f'{self.voice_name}_proc.wav')
+            final_voice_file = os.path.join(self.session['voice_dir'], f'{self.voice_name}.wav')
+            ffmpeg_cmd = [shutil.which('ffmpeg'), '-hide_banner', '-nostats', '-i', self.voice_track]
+            filter_complex = (
+                'agate=threshold=-25dB:ratio=1.4:attack=10:release=250,'
+                'afftdn=nf=-70,'
+                'acompressor=threshold=-20dB:ratio=2:attack=80:release=200:makeup=1dB,'
+                'loudnorm=I=-14:TP=-3:LRA=7:linear=true,'
+                'equalizer=f=150:t=q:w=2:g=1,'
+                'equalizer=f=250:t=q:w=2:g=-3,'
+                'equalizer=f=3000:t=q:w=2:g=2,'
+                'equalizer=f=5500:t=q:w=2:g=-4,'
+                'equalizer=f=9000:t=q:w=2:g=-2,'
+                'highpass=f=63[audio]'
+            )
+            ffmpeg_cmd += [
+                '-filter_complex', filter_complex,
+                '-map', '[audio]',
+                '-ar', f'{default_audio_proc_samplerate}',
+                '-y', proc_voice_file
+            ]
+            try:
+                process = subprocess.Popen(
+                    ffmpeg_cmd,
+                    env={},
+                    stdout=subprocess.PIPE, 
+                    stderr=subprocess.PIPE,
+                    encoding='utf-8',
+                    errors='ignore'
+                )
+                for line in process.stdout:
+                    print(line, end='')  # Print each line of stdout
+                process.wait()
+                if process.returncode != 0:
+                    error = f'_normalize_audio(): process.returncode: {process.returncode}'
+                elif not os.path.exists(proc_voice_file) or os.path.getsize(proc_voice_file) == 0:
+                    error = f'_normalize_audio() error: {proc_voice_file} was not created or is empty.'
+                else:
+                    os.replace(proc_voice_file, final_voice_file)
+                    shutil.rmtree(self.demucs_dir, ignore_errors=True)
+                    msg = 'Audio normalization successful!'
+                    return True, msg
+            except subprocess.CalledProcessError as e:
+                error = f'_normalize_audio() ffmpeg.Error: {e.stderr.decode()}'
+        except FileNotFoundError as e:
+            error = '_normalize_audio() FileNotFoundError: {e} Input file or FFmpeg PATH not found!'
+        except Exception as e:
+            error = f'_normalize_audio() error: {e}'
+        return False, error
+
+    def extract_voice(self):
+        success = False
+        msg = None
+        try:
+            success, msg = self._validate_format()
+            print(msg)
+            if success:
+                success, msg = self._convert2wav()
+                print(msg)
+                if success:
+                    success, status, msg = self._detect_background()
+                    print(msg)
+                    if success:
+                        if status:
+                            success, msg = self._demucs_voice()
+                            print(msg)
+                        else:
+                            self.voice_track = self.wav_file
+                        if success:
+                            success, msg = self._trim_and_clean(self.silence_threshold)
+                            print(msg)
+                            if success:
+                                success, msg = self._normalize_audio()
+                                print(msg)
+        except Exception as e:
+            msg = f'extract_voice() error: {e}'
+            raise ValueError(msg)
+        shutil.rmtree(self.demucs_dir, ignore_errors=True)
+        return success, msg

lib/conf.py

@@ -0,0 +1,78 @@
+import os
+import platform
+
+tmp_dir = os.path.abspath('tmp')
+tmp_expire = 7 # days
+
+models_dir = os.path.abspath('models')
+ebooks_dir = os.path.abspath('ebooks')
+voices_dir = os.path.abspath('voices')
+tts_dir = os.path.join(models_dir, 'tts')
+
+os.environ['PYTHONUTF8'] = '1'
+os.environ['PYTHONIOENCODING'] = 'utf-8'
+os.environ['COQUI_TOS_AGREED'] = '1'
+os.environ['PYTHONIOENCODING'] = 'utf-8'
+os.environ['CALIBRE_NO_NATIVE_FILEDIALOGS'] = '1'
+os.environ['GRADIO_DEBUG'] = '1'
+os.environ['DO_NOT_TRACK'] = 'true'
+os.environ['CALIBRE_TEMP_DIR'] = tmp_dir
+os.environ['CALIBRE_CACHE_DIRECTORY'] = tmp_dir
+os.environ['HUGGINGFACE_HUB_CACHE'] = tts_dir
+os.environ['HF_HOME'] = tts_dir
+os.environ['HF_DATASETS_CACHE'] = tts_dir
+os.environ['BARK_CACHE_DIR'] = tts_dir
+os.environ['TTS_CACHE'] = tts_dir
+os.environ['TORCH_HOME'] = tts_dir
+os.environ['TTS_HOME'] = models_dir
+os.environ['XDG_CACHE_HOME'] = models_dir
+os.environ['STANZA_RESOURCES_DIR'] = os.path.join(models_dir, 'stanza')
+os.environ['ARGOS_TRANSLATE_PACKAGE_PATH'] = os.path.join(models_dir, 'argostranslate')
+os.environ['TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD'] = '1'
+os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1'
+os.environ['SUNO_OFFLOAD_CPU'] = 'False' # BARK option: False needs A GPU
+os.environ['SUNO_USE_SMALL_MODELS'] = 'False' # BARK option: False needs a GPU with VRAM > 4GB
+if platform.system() == 'Windows':
+    os.environ['ESPEAK_DATA_PATH'] = os.path.expandvars(r"%USERPROFILE%\scoop\apps\espeak-ng\current\eSpeak NG\espeak-ng-data")
+
+prog_version = (lambda: open('VERSION.txt').read().strip())()
+
+min_python_version = (3,10)
+max_python_version = (3,12)
+
+NATIVE = 'native'
+FULL_DOCKER = 'full_docker'
+
+debug_mode = True
+
+device_list = ['cpu', 'gpu', 'mps']
+default_device = 'cpu'
+default_gpu_wiki = '<a href="https://github.com/DrewThomasson/ebook2audiobook/wiki/GPU-ISSUES">howto wiki</a>'
+
+python_env_dir = os.path.abspath(os.path.join('.','python_env'))
+requirements_file = os.path.abspath(os.path.join('.','requirements.txt'))
+
+interface_host = '0.0.0.0'
+interface_port = 7860
+interface_shared_tmp_expire = 3 # in days
+interface_concurrency_limit = 1 # or None for unlimited
+
+interface_component_options = {
+    "gr_tab_xtts_params": True,
+    "gr_tab_bark_params": True,
+    "gr_group_voice_file": True,
+    "gr_group_custom_model": True
+}
+
+audiobooks_gradio_dir = os.path.abspath(os.path.join('audiobooks','gui','gradio'))
+audiobooks_host_dir = os.path.abspath(os.path.join('audiobooks','gui','host'))
+audiobooks_cli_dir = os.path.abspath(os.path.join('audiobooks','cli'))
+
+ebook_formats = ['.epub', '.mobi', '.azw3', '.fb2', '.lrf', '.rb', '.snb', '.tcr', '.pdf', '.txt', '.rtf', '.doc', '.docx', '.html', '.odt', '.azw'] # Add or remove the format you accept as input
+voice_formats = ['.mp4', '.m4b', '.m4a', '.mp3', '.wav', '.aac', '.flac', '.alac', '.ogg', '.aiff', '.aif', '.wma', '.dsd', '.opus', '.pcmu', '.pcma', '.gsm'] # Add or remove the format you accept as input
+output_formats = ['aac', 'flac', 'mp3', 'm4b', 'm4a', 'mp4', 'mov', 'ogg', 'wav', 'webm']
+default_audio_proc_samplerate = 24000
+default_audio_proc_format = 'flac' # or 'mp3', 'aac', 'm4a', 'm4b', 'amr', '3gp', 'alac'. 'wav' format is ok but limited to process files < 4GB
+default_output_format = 'm4b'
+default_output_split = False
+default_output_split_hours = '6' # if the final ouput esceed outpout_split_hours * 2 hours the final file will be splitted by outpout_split_hours + the end if any.

lib/models.py

@@ -0,0 +1,493 @@
+import os
+
+from lib.conf import tts_dir, voices_dir
+loaded_tts = {}
+
+TTS_ENGINES = {
+    "XTTSv2": "xtts", 
+    "BARK": "bark", 
+    "VITS": "vits", 
+    "FAIRSEQ": "fairseq", 
+    "TACOTRON2": "tacotron", 
+    "YOURTTS": "yourtts"
+}
+
+TTS_VOICE_CONVERSION = {
+    "freevc24": {"path": "voice_conversion_models/multilingual/vctk/freevc24", "samplerate": 24000},
+    "knnvc": {"path": "voice_conversion_models/multilingual/multi-dataset/knnvc", "samplerate": 16000},
+    "openvoice_v1": {"path": "voice_conversion_models/multilingual/multi-dataset/openvoice_v1", "samplerate": 22050},
+    "openvoice_v2": {"path": "voice_conversion_models/multilingual/multi-dataset/openvoice_v2", "samplerate": 22050}
+}
+
+TTS_SML = {
+    "break": "‡break‡",
+    "pause": "‡pause‡",
+    "###": "‡pause‡"
+}
+
+default_tts_engine = TTS_ENGINES['XTTSv2']
+default_fine_tuned = 'internal'
+default_vc_model = TTS_VOICE_CONVERSION['knnvc']['path']
+default_voice_detection_model = 'drewThomasson/segmentation'
+
+max_tts_in_memory = 2 # TTS engines to keep in memory (1 tts engine ~= 4GB to 8GB RAM).
+max_custom_model = 100
+max_custom_voices = 1000
+max_upload_size = '6GB'
+
+default_engine_settings = {
+    TTS_ENGINES['XTTSv2']: {
+        "samplerate": 24000,
+        "temperature": 0.75,
+        "length_penalty": 1.0,
+        "num_beams": 1,
+        "repetition_penalty": 3.0,
+        "top_k": 50,
+        "top_p": 0.85,
+        "speed": 1.0,
+        "enable_text_splitting": False,
+        # to enable deepspeed, you must install it first:
+        # conda activate ./python_env (linux/mac) or .\python_env (windows)
+        # pip install deepspeed
+        # conda deactivate
+        "use_deepspeed": False,
+        "files": ['config.json', 'model.pth', 'vocab.json', 'ref.wav', 'speakers_xtts.pth'],
+        "voices": {
+            "ClaribelDervla": "Claribel Dervla", "DaisyStudious": "Daisy Studious", "GracieWise": "Gracie Wise",
+            "TammieEma": "Tammie Ema", "AlisonDietlinde": "Alison Dietlinde", "AnaFlorence": "Ana Florence",
+            "AnnmarieNele": "Annmarie Nele", "AsyaAnara": "Asya Anara", "BrendaStern": "Brenda Stern",
+            "GittaNikolina": "Gitta Nikolina", "HenrietteUsha": "Henriette Usha", "SofiaHellen": "Sofia Hellen",
+            "TammyGrit": "Tammy Grit", "TanjaAdelina": "Tanja Adelina", "VjollcaJohnnie": "Vjollca Johnnie",
+            "AndrewChipper": "Andrew Chipper", "BadrOdhiambo": "Badr Odhiambo", "DionisioSchuyler": "Dionisio Schuyler",
+            "RoystonMin": "Royston Min", "ViktorEka": "Viktor Eka", "AbrahanMack": "Abrahan Mack",
+            "AddeMichal": "Adde Michal", "BaldurSanjin": "Baldur Sanjin", "CraigGutsy": "Craig Gutsy",
+            "DamienBlack": "Damien Black", "GilbertoMathias": "Gilberto Mathias", "IlkinUrbano": "Ilkin Urbano",
+            "KazuhikoAtallah": "Kazuhiko Atallah", "LudvigMilivoj": "Ludvig Milivoj", "SuadQasim": "Suad Qasim",
+            "TorcullDiarmuid": "Torcull Diarmuid", "ViktorMenelaos": "Viktor Menelaos", "ZacharieAimilios": "Zacharie Aimilios",
+            "NovaHogarth": "Nova Hogarth", "MajaRuoho": "Maja Ruoho", "UtaObando": "Uta Obando",
+            "LidiyaSzekeres": "Lidiya Szekeres", "ChandraMacFarland": "Chandra MacFarland", "SzofiGranger": "Szofi Granger",
+            "CamillaHolmström": "Camilla Holmström", "LilyaStainthorpe": "Lilya Stainthorpe", "ZofijaKendrick": "Zofija Kendrick",
+            "NarelleMoon": "Narelle Moon", "BarboraMacLean": "Barbora MacLean", "AlexandraHisakawa": "Alexandra Hisakawa",
+            "AlmaMaría": "Alma María", "RosemaryOkafor": "Rosemary Okafor", "IgeBehringer": "Ige Behringer",
+            "FilipTraverse": "Filip Traverse", "DamjanChapman": "Damjan Chapman", "WulfCarlevaro": "Wulf Carlevaro",
+            "AaronDreschner": "Aaron Dreschner", "KumarDahl": "Kumar Dahl", "EugenioMataracı": "Eugenio Mataracı",
+            "FerranSimen": "Ferran Simen", "XavierHayasaka": "Xavier Hayasaka", "LuisMoray": "Luis Moray",
+            "MarcosRudaski": "Marcos Rudaski"
+        },
+        "rating": {"GPU VRAM": 4, "CPU": 3, "RAM": 8, "Realism": 4}
+    },
+    TTS_ENGINES['BARK']: {
+        "samplerate": 24000,
+        "text_temp": 0.50,
+        "waveform_temp": 0.50,
+        "files": ["text_2.pt", "coarse_2.pt", "fine_2.pt"],
+        "speakers_path": os.path.join(voices_dir, '__bark'),
+        "voices": {
+            "de_speaker_0": "Speaker 0", "de_speaker_1": "Speaker 1", "de_speaker_2": "Speaker 2",
+            "de_speaker_3": "Speaker 3", "de_speaker_4": "Speaker 4", "de_speaker_5": "Speaker 5",
+            "de_speaker_6": "Speaker 6", "de_speaker_7": "Speaker 7", "de_speaker_8": "Speaker 8",
+            "de_speaker_9": "Speaker 9", "en_speaker_0": "Speaker 0", "en_speaker_1": "Speaker 1",
+            "en_speaker_2": "Speaker 2", "en_speaker_3": "Speaker 3", "en_speaker_4": "Speaker 4",
+            "en_speaker_5": "Speaker 5", "en_speaker_6": "Speaker 6", "en_speaker_7": "Speaker 7",
+            "en_speaker_8": "Speaker 8", "en_speaker_9": "Speaker 9", "es_speaker_0": "Speaker 0",
+            "es_speaker_1": "Speaker 1", "es_speaker_2": "Speaker 2", "es_speaker_3": "Speaker 3",
+            "es_speaker_4": "Speaker 4", "es_speaker_5": "Speaker 5", "es_speaker_6": "Speaker 6",
+            "es_speaker_7": "Speaker 7", "es_speaker_8": "Speaker 8", "es_speaker_9": "Speaker 9",
+            "fr_speaker_0": "Speaker 0", "fr_speaker_1": "Speaker 1", "fr_speaker_2": "Speaker 2",
+            "fr_speaker_3": "Speaker 3", "fr_speaker_4": "Speaker 4", "fr_speaker_5": "Speaker 5",
+            "fr_speaker_6": "Speaker 6", "fr_speaker_7": "Speaker 7", "fr_speaker_8": "Speaker 8",
+            "fr_speaker_9": "Speaker 9", "hi_speaker_0": "Speaker 0", "hi_speaker_1": "Speaker 1",
+            "hi_speaker_2": "Speaker 2", "hi_speaker_3": "Speaker 3", "hi_speaker_4": "Speaker 4",
+            "hi_speaker_5": "Speaker 5", "hi_speaker_6": "Speaker 6", "hi_speaker_7": "Speaker 7",
+            "hi_speaker_8": "Speaker 8", "hi_speaker_9": "Speaker 9", "it_speaker_0": "Speaker 0",
+            "it_speaker_1": "Speaker 1", "it_speaker_2": "Speaker 2", "it_speaker_3": "Speaker 3",
+            "it_speaker_4": "Speaker 4", "it_speaker_5": "Speaker 5", "it_speaker_6": "Speaker 6",
+            "it_speaker_7": "Speaker 7", "it_speaker_8": "Speaker 8", "it_speaker_9": "Speaker 9",
+            "ja_speaker_0": "Speaker 0", "ja_speaker_1": "Speaker 1", "ja_speaker_2": "Speaker 2",
+            "ja_speaker_3": "Speaker 3", "ja_speaker_4": "Speaker 4", "ja_speaker_5": "Speaker 5",
+            "ja_speaker_6": "Speaker 6", "ja_speaker_7": "Speaker 7", "ja_speaker_8": "Speaker 8",
+            "ja_speaker_9": "Speaker 9", "ko_speaker_0": "Speaker 0", "ko_speaker_1": "Speaker 1",
+            "ko_speaker_2": "Speaker 2", "ko_speaker_3": "Speaker 3", "ko_speaker_4": "Speaker 4",
+            "ko_speaker_5": "Speaker 5", "ko_speaker_6": "Speaker 6", "ko_speaker_7": "Speaker 7",
+            "ko_speaker_8": "Speaker 8", "ko_speaker_9": "Speaker 9", "pl_speaker_0": "Speaker 0",
+            "pl_speaker_1": "Speaker 1", "pl_speaker_2": "Speaker 2", "pl_speaker_3": "Speaker 3",
+            "pl_speaker_4": "Speaker 4", "pl_speaker_5": "Speaker 5", "pl_speaker_6": "Speaker 6",
+            "pl_speaker_7": "Speaker 7", "pl_speaker_8": "Speaker 8", "pl_speaker_9": "Speaker 9",
+            "pt_speaker_0": "Speaker 0", "pt_speaker_1": "Speaker 1", "pt_speaker_2": "Speaker 2",
+            "pt_speaker_3": "Speaker 3", "pt_speaker_4": "Speaker 4", "pt_speaker_5": "Speaker 5",
+            "pt_speaker_6": "Speaker 6", "pt_speaker_7": "Speaker 7", "pt_speaker_8": "Speaker 8",
+            "pt_speaker_9": "Speaker 9", "ru_speaker_0": "Speaker 0", "ru_speaker_1": "Speaker 1",
+            "ru_speaker_2": "Speaker 2", "ru_speaker_3": "Speaker 3", "ru_speaker_4": "Speaker 4",
+            "ru_speaker_5": "Speaker 5", "ru_speaker_6": "Speaker 6", "ru_speaker_7": "Speaker 7",
+            "ru_speaker_8": "Speaker 8", "ru_speaker_9": "Speaker 9", "tr_speaker_0": "Speaker 0",
+            "tr_speaker_1": "Speaker 1", "tr_speaker_2": "Speaker 2", "tr_speaker_3": "Speaker 3",
+            "tr_speaker_4": "Speaker 4", "tr_speaker_5": "Speaker 5", "tr_speaker_6": "Speaker 6",
+            "tr_speaker_7": "Speaker 7", "tr_speaker_8": "Speaker 8", "tr_speaker_9": "Speaker 9",
+            "zh_speaker_0": "Speaker 0", "zh_speaker_1": "Speaker 1", "zh_speaker_2": "Speaker 2",
+            "zh_speaker_3": "Speaker 3", "zh_speaker_4": "Speaker 4", "zh_speaker_5": "Speaker 5",
+            "zh_speaker_6": "Speaker 6", "zh_speaker_7": "Speaker 7", "zh_speaker_8": "Speaker 8",
+            "zh_speaker_9": "Speaker 9"
+        },
+        "rating": {"GPU VRAM": 4, "CPU": 1, "RAM": 16, "Realism": 3}
+    },
+    TTS_ENGINES['VITS']: {
+        "samplerate": 22050,
+        "files": ['config.json', 'model_file.pth', 'language_ids.json'],
+        "voices": {},
+        "rating": {"GPU VRAM": 2, "CPU": 3, "RAM": 4, "Realism": 2}
+    },
+    TTS_ENGINES['FAIRSEQ']: {
+        "samplerate": 16000,
+        "files": ['config.json', 'G_100000.pth', 'vocab.json'],
+        "voices": {},
+        "rating": {"GPU VRAM": 2, "CPU": 3, "RAM": 4, "Realism": 2}
+    },
+    TTS_ENGINES['TACOTRON2']: {
+        "samplerate": 22050,
+        "files": ['config.json', 'best_model.pth', 'vocoder_config.json', 'vocoder_model.pth'],
+        "voices": {},
+        "rating": {"GPU VRAM": 2, "CPU": 3, "RAM": 4, "Realism": 2}
+    },
+    TTS_ENGINES['YOURTTS']: {
+        "samplerate": 16000,
+        "files": ['config.json', 'model_file.pth'],
+        "voices": {"Machinella-5": "female-en-5", "ElectroMale-2": "male-en-2", 'Machinella-4': 'female-pt-4\n', 'ElectroMale-3': 'male-pt-3\n'},
+        "rating": {"GPU VRAM": 1, "CPU": 5, "RAM": 4, "Realism": 1}
+    }
+}
+models = {
+    TTS_ENGINES['XTTSv2']: {
+        "internal": {
+            "lang": "multi",
+            "repo": "coqui/XTTS-v2",
+            "sub": "tts_models/multilingual/multi-dataset/xtts_v2/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'KumarDahl.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "AiExplained": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/AiExplained/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'AiExplained.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "AsmrRacoon": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/AsmrRacoon/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'AsmrRacoon.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "Awkwafina": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/Awkwafina/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'female', 'Awkwafina.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "BobOdenkirk": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/BobOdenkirk/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'BobOdenkirk.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "BobRoss": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/BobRoss/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'BobRoss.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "BrinaPalencia": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/BrinaPalencia/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'female', 'BrinaPalencia.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "BryanCranston": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/BryanCranston/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'BryanCranston.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "DavidAttenborough": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/DavidAttenborough/",
+            "voice": os.path.join(voices_dir, 'eng', 'elder', 'male', 'DavidAttenborough.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "DeathPussInBoots": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/DeathPussInBoots/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'DeathPussInBoots.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "DermotCrowley": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/DermotCrowley/",
+            "voice": os.path.join(voices_dir, 'eng', 'elder', 'male', 'DermotCrowley.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "EvaSeymour": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/EvaSeymour/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'female', 'EvaSeymour.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "GideonOfnirEldenRing": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/GideonOfnirEldenRing/",
+            "voice": os.path.join(voices_dir, 'eng', 'elder', 'male', 'GideonOfnirEldenRing.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "GhostMW2": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/GhostMW2/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'GhostMW2.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "JhonButlerASMR": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/JhonButlerASMR/",
+            "voice": os.path.join(voices_dir, 'eng', 'elder', 'male', 'JhonButlerASMR.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "JhonMulaney": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/JhonMulaney/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'JhonMulaney.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "JillRedfield": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/JillRedfield/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'female', 'JillRedfield.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "JuliaWhenlan": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/JuliaWhenlan/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'female', 'JuliaWhenlan.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "LeeHorsley": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/LeeHorsley/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'LeeHorsley.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "MelinaEldenRing": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/MelinaEldenRing/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'female', 'MelinaEldenRing.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "MorganFreeman": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/MorganFreeman/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'MorganFreeman.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "NeilGaiman": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/NeilGaiman/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'NeilGaiman.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "RainyDayHeadSpace": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/RainyDayHeadSpace/",
+            "voice": os.path.join(voices_dir, 'eng', 'elder', 'male', 'RainyDayHeadSpace.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "RayPorter": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/RayPorter/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'RayPorter.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "RelaxForAWhile": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/RelaxForAWhile/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'female', 'RelaxForAWhile.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "RosamundPike": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/RosamundPike/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'female', 'RosamundPike.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "ScarlettJohansson": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/ScarlettJohansson/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'female', 'ScarlettJohansson.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "SladeTeenTitans": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/SladeTeenTitans/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'SladeTeenTitans.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "StanleyParable": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/StanleyParable/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'StanleyParable.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "WhisperSalemASMR": {
+            "lang": "eng",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/eng/WhisperSalemASMR/",
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'WhisperSalemASMR.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        },
+        "Konishev": {
+            "lang": "rus",
+            "repo": "drewThomasson/fineTunedTTSModels",
+            "sub": "xtts-v2/rus/Konishev/",
+            "voice": os.path.join(voices_dir, 'rus', 'adult', 'male', 'Konishev.wav'),
+            "files": default_engine_settings[TTS_ENGINES['XTTSv2']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['XTTSv2']]['samplerate']
+        }
+    },
+    TTS_ENGINES['BARK']: {
+        "internal": {
+            "lang": "multi",
+            "repo": "erogol/bark", # suno/bark, rsxdalv/suno, tts_models/multilingual/multi-dataset/bark
+            "sub": "", # {"big-bf16": "big-bf16/", "small-bf16": "small-bf16/", "big": "big/", "small": "small/"}
+            "voice": os.path.join(voices_dir, 'eng', 'adult', 'male', 'KumarDahl.wav'),
+            "files": default_engine_settings[TTS_ENGINES['BARK']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['BARK']]['samplerate']
+        }
+    },
+    TTS_ENGINES['VITS']: {
+        "internal": {
+            "lang": "multi",
+            "repo": "tts_models/[lang_iso1]/[xxx]",
+            "sub": {
+                "css10/vits": ['es','hu','fi','fr','nl','ru','el'],
+                "custom/vits": ['ca'],
+                "custom/vits-female": ['bn', 'fa'],
+                "cv/vits": ['bg','cs','da','et','ga','hr','lt','lv','mt','pt','ro','sk','sl','sv'],
+                "mai/vits": ['uk'],
+                "mai_female/vits": ['pl'],
+                "mai_male/vits": ['it'],
+                "openbible/vits": ['ewe','hau','lin','tw_akuapem','tw_asante','yor'],
+                "vctk/vits": ['en'],
+                "thorsten/vits": ['de']
+            },
+            "voice": None,
+            "files": default_engine_settings[TTS_ENGINES['VITS']]['files'],
+            "samplerate": {
+                "css10/vits": default_engine_settings[TTS_ENGINES['VITS']]['samplerate'],
+                "custom/vits": default_engine_settings[TTS_ENGINES['VITS']]['samplerate'],
+                "custom/vits-female": default_engine_settings[TTS_ENGINES['VITS']]['samplerate'],
+                "cv/vits": default_engine_settings[TTS_ENGINES['VITS']]['samplerate'],
+                "mai/vits": default_engine_settings[TTS_ENGINES['VITS']]['samplerate'],
+                "mai_female/vits": 24000,
+                "mai_male/vits": 16000,
+                "openbible/vits": default_engine_settings[TTS_ENGINES['VITS']]['samplerate'],
+                "vctk/vits": default_engine_settings[TTS_ENGINES['VITS']]['samplerate'],
+                "thorsten/vits": default_engine_settings[TTS_ENGINES['VITS']]['samplerate']
+            }
+        }
+    },
+    TTS_ENGINES['FAIRSEQ']: {
+        "internal": {
+            "lang": "multi",
+            "repo": "tts_models/[lang]/fairseq/vits",
+            "sub": "",
+            "voice": None,
+            "files": default_engine_settings[TTS_ENGINES['FAIRSEQ']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['FAIRSEQ']]['samplerate']
+        }
+    },
+    TTS_ENGINES['TACOTRON2']: {
+       "internal": {
+            "lang": "multi",
+            "repo": "tts_models/[lang_iso1]/[xxx]",
+            "sub": {
+                "mai/tacotron2-DDC": ['fr', 'es', 'nl'],
+                "thorsten/tacotron2-DDC": ['de'],
+                "kokoro/tacotron2-DDC": ['ja'],
+                "ljspeech/tacotron2-DDC": ['en'],
+                "baker/tacotron2-DDC-GST": ['zh-CN']              
+            },
+            "voice": None,
+            "files": default_engine_settings[TTS_ENGINES['TACOTRON2']]['files'],
+            "samplerate": {
+                "mai/tacotron2-DDC": default_engine_settings[TTS_ENGINES['TACOTRON2']]['samplerate'],
+                "thorsten/tacotron2-DDC": default_engine_settings[TTS_ENGINES['TACOTRON2']]['samplerate'],
+                "kokoro/tacotron2-DDC": default_engine_settings[TTS_ENGINES['TACOTRON2']]['samplerate'],
+                "ljspeech/tacotron2-DDC": default_engine_settings[TTS_ENGINES['TACOTRON2']]['samplerate'],
+                "baker/tacotron2-DDC-GST": default_engine_settings[TTS_ENGINES['TACOTRON2']]['samplerate']
+            },
+        }
+    },
+    TTS_ENGINES['YOURTTS']: {
+        "internal": {
+            "lang": "multi",
+            "repo": "tts_models/multilingual/multi-dataset/your_tts",
+            "sub": "",
+            "voice": None,
+            "files": default_engine_settings[TTS_ENGINES['YOURTTS']]['files'],
+            "samplerate": default_engine_settings[TTS_ENGINES['YOURTTS']]['samplerate']
+        }
+    }
+}