Spaces:
Runtime error
Runtime error
Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- .gitignore +162 -0
- Data/configs/haruka.json +99 -0
- Data/models/G_82000.pth +3 -0
- __pycache__/attentions.cpython-310.pyc +0 -0
- __pycache__/commons.cpython-310.pyc +0 -0
- __pycache__/config.cpython-310.pyc +0 -0
- __pycache__/emo_gen.cpython-310.pyc +0 -0
- __pycache__/infer.cpython-310.pyc +0 -0
- __pycache__/models.cpython-310.pyc +0 -0
- __pycache__/modules.cpython-310.pyc +0 -0
- __pycache__/re_matching.cpython-310.pyc +0 -0
- __pycache__/transforms.cpython-310.pyc +0 -0
- __pycache__/utils.cpython-310.pyc +0 -0
- app.py +485 -0
- attentions.py +464 -0
- bert/bert_models.json +14 -0
- commons.py +166 -0
- config.py +243 -0
- config.yml +174 -0
- emo_gen.py +162 -0
- infer.py +261 -0
- models.py +1044 -0
- modules.py +597 -0
- monotonic_align/__init__.py +16 -0
- monotonic_align/__pycache__/__init__.cpython-310.pyc +0 -0
- monotonic_align/__pycache__/core.cpython-310.pyc +0 -0
- monotonic_align/core.py +46 -0
- re_matching.py +82 -0
- requirements.txt +33 -0
- text/__init__.py +51 -0
- text/__pycache__/__init__.cpython-310.pyc +0 -0
- text/__pycache__/bert_utils.cpython-310.pyc +0 -0
- text/__pycache__/chinese.cpython-310.pyc +0 -0
- text/__pycache__/chinese_bert.cpython-310.pyc +0 -0
- text/__pycache__/cleaner.cpython-310.pyc +0 -0
- text/__pycache__/english.cpython-310.pyc +0 -0
- text/__pycache__/english_bert_mock.cpython-310.pyc +0 -0
- text/__pycache__/japanese.cpython-310.pyc +0 -0
- text/__pycache__/japanese_bert.cpython-310.pyc +0 -0
- text/__pycache__/symbols.cpython-310.pyc +0 -0
- text/__pycache__/tone_sandhi.cpython-310.pyc +0 -0
- text/bert_utils.py +23 -0
- text/chinese.py +199 -0
- text/chinese_bert.py +102 -0
- text/cleaner.py +28 -0
- text/cmudict.rep +0 -0
- text/cmudict_cache.pickle +3 -0
- text/english.py +454 -0
- text/english_bert_mock.py +43 -0
- text/japanese.py +432 -0
.gitignore
ADDED
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Byte-compiled / optimized / DLL files
|
2 |
+
__pycache__/
|
3 |
+
*.py[cod]
|
4 |
+
*$py.class
|
5 |
+
|
6 |
+
# C extensions
|
7 |
+
*.so
|
8 |
+
|
9 |
+
# Distribution / packaging
|
10 |
+
.Python
|
11 |
+
build/
|
12 |
+
develop-eggs/
|
13 |
+
dist/
|
14 |
+
downloads/
|
15 |
+
eggs/
|
16 |
+
.eggs/
|
17 |
+
lib/
|
18 |
+
lib64/
|
19 |
+
parts/
|
20 |
+
sdist/
|
21 |
+
var/
|
22 |
+
wheels/
|
23 |
+
share/python-wheels/
|
24 |
+
*.egg-info/
|
25 |
+
.installed.cfg
|
26 |
+
*.egg
|
27 |
+
MANIFEST
|
28 |
+
|
29 |
+
# PyInstaller
|
30 |
+
# Usually these files are written by a python script from a template
|
31 |
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
32 |
+
*.manifest
|
33 |
+
*.spec
|
34 |
+
|
35 |
+
# Installer logs
|
36 |
+
pip-log.txt
|
37 |
+
pip-delete-this-directory.txt
|
38 |
+
|
39 |
+
# Unit test / coverage reports
|
40 |
+
htmlcov/
|
41 |
+
.tox/
|
42 |
+
.nox/
|
43 |
+
.coverage
|
44 |
+
.coverage.*
|
45 |
+
.cache
|
46 |
+
nosetests.xml
|
47 |
+
coverage.xml
|
48 |
+
*.cover
|
49 |
+
*.py,cover
|
50 |
+
.hypothesis/
|
51 |
+
.pytest_cache/
|
52 |
+
cover/
|
53 |
+
|
54 |
+
# Translations
|
55 |
+
*.mo
|
56 |
+
*.pot
|
57 |
+
|
58 |
+
# Django stuff:
|
59 |
+
*.log
|
60 |
+
local_settings.py
|
61 |
+
db.sqlite3
|
62 |
+
db.sqlite3-journal
|
63 |
+
|
64 |
+
# Flask stuff:
|
65 |
+
instance/
|
66 |
+
.webassets-cache
|
67 |
+
|
68 |
+
# Scrapy stuff:
|
69 |
+
.scrapy
|
70 |
+
|
71 |
+
# Sphinx documentation
|
72 |
+
docs/_build/
|
73 |
+
|
74 |
+
# PyBuilder
|
75 |
+
.pybuilder/
|
76 |
+
target/
|
77 |
+
|
78 |
+
# Jupyter Notebook
|
79 |
+
.ipynb_checkpoints
|
80 |
+
|
81 |
+
# IPython
|
82 |
+
profile_default/
|
83 |
+
ipython_config.py
|
84 |
+
|
85 |
+
# pyenv
|
86 |
+
# For a library or package, you might want to ignore these files since the code is
|
87 |
+
# intended to run in multiple environments; otherwise, check them in:
|
88 |
+
# .python-version
|
89 |
+
|
90 |
+
# pipenv
|
91 |
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
92 |
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
93 |
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
94 |
+
# install all needed dependencies.
|
95 |
+
#Pipfile.lock
|
96 |
+
|
97 |
+
# poetry
|
98 |
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
99 |
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
100 |
+
# commonly ignored for libraries.
|
101 |
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
102 |
+
#poetry.lock
|
103 |
+
|
104 |
+
# pdm
|
105 |
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
106 |
+
#pdm.lock
|
107 |
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
108 |
+
# in version control.
|
109 |
+
# https://pdm.fming.dev/#use-with-ide
|
110 |
+
.pdm.toml
|
111 |
+
|
112 |
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
113 |
+
__pypackages__/
|
114 |
+
|
115 |
+
# Celery stuff
|
116 |
+
celerybeat-schedule
|
117 |
+
celerybeat.pid
|
118 |
+
|
119 |
+
# SageMath parsed files
|
120 |
+
*.sage.py
|
121 |
+
|
122 |
+
# Environments
|
123 |
+
.env
|
124 |
+
.venv
|
125 |
+
env/
|
126 |
+
venv/
|
127 |
+
ENV/
|
128 |
+
env.bak/
|
129 |
+
venv.bak/
|
130 |
+
|
131 |
+
# Spyder project settings
|
132 |
+
.spyderproject
|
133 |
+
.spyproject
|
134 |
+
|
135 |
+
# Rope project settings
|
136 |
+
.ropeproject
|
137 |
+
|
138 |
+
# mkdocs documentation
|
139 |
+
/site
|
140 |
+
|
141 |
+
# mypy
|
142 |
+
.mypy_cache/
|
143 |
+
.dmypy.json
|
144 |
+
dmypy.json
|
145 |
+
|
146 |
+
# Pyre type checker
|
147 |
+
.pyre/
|
148 |
+
|
149 |
+
# pytype static type analyzer
|
150 |
+
.pytype/
|
151 |
+
|
152 |
+
# Cython debug symbols
|
153 |
+
cython_debug/
|
154 |
+
|
155 |
+
# PyCharm
|
156 |
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
157 |
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
158 |
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
159 |
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
160 |
+
#.idea/
|
161 |
+
|
162 |
+
.DS_Store
|
Data/configs/haruka.json
ADDED
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"train": {
|
3 |
+
"log_interval": 200,
|
4 |
+
"eval_interval": 1000,
|
5 |
+
"seed": 42,
|
6 |
+
"epochs": 1000,
|
7 |
+
"learning_rate": 0.0002,
|
8 |
+
"betas": [
|
9 |
+
0.8,
|
10 |
+
0.99
|
11 |
+
],
|
12 |
+
"eps": 1e-09,
|
13 |
+
"batch_size": 4,
|
14 |
+
"fp16_run": false,
|
15 |
+
"lr_decay": 0.99995,
|
16 |
+
"segment_size": 16384,
|
17 |
+
"init_lr_ratio": 1,
|
18 |
+
"warmup_epochs": 0,
|
19 |
+
"c_mel": 45,
|
20 |
+
"c_kl": 1.0,
|
21 |
+
"skip_optimizer": true
|
22 |
+
},
|
23 |
+
"data": {
|
24 |
+
"training_files": "Data/filelists/train.list",
|
25 |
+
"validation_files": "Data/filelists/val.list",
|
26 |
+
"max_wav_value": 32768.0,
|
27 |
+
"sampling_rate": 44100,
|
28 |
+
"filter_length": 2048,
|
29 |
+
"hop_length": 512,
|
30 |
+
"win_length": 2048,
|
31 |
+
"n_mel_channels": 128,
|
32 |
+
"mel_fmin": 0.0,
|
33 |
+
"mel_fmax": null,
|
34 |
+
"add_blank": true,
|
35 |
+
"n_speakers": 4,
|
36 |
+
"cleaned_text": true,
|
37 |
+
"spk2id": {
|
38 |
+
"希亜": 0,
|
39 |
+
"天": 1,
|
40 |
+
"春風": 2,
|
41 |
+
"都": 3
|
42 |
+
}
|
43 |
+
},
|
44 |
+
"model": {
|
45 |
+
"use_spk_conditioned_encoder": true,
|
46 |
+
"use_noise_scaled_mas": true,
|
47 |
+
"use_mel_posterior_encoder": false,
|
48 |
+
"use_duration_discriminator": true,
|
49 |
+
"inter_channels": 192,
|
50 |
+
"hidden_channels": 192,
|
51 |
+
"filter_channels": 768,
|
52 |
+
"n_heads": 2,
|
53 |
+
"n_layers": 6,
|
54 |
+
"kernel_size": 3,
|
55 |
+
"p_dropout": 0.1,
|
56 |
+
"resblock": "1",
|
57 |
+
"resblock_kernel_sizes": [
|
58 |
+
3,
|
59 |
+
7,
|
60 |
+
11
|
61 |
+
],
|
62 |
+
"resblock_dilation_sizes": [
|
63 |
+
[
|
64 |
+
1,
|
65 |
+
3,
|
66 |
+
5
|
67 |
+
],
|
68 |
+
[
|
69 |
+
1,
|
70 |
+
3,
|
71 |
+
5
|
72 |
+
],
|
73 |
+
[
|
74 |
+
1,
|
75 |
+
3,
|
76 |
+
5
|
77 |
+
]
|
78 |
+
],
|
79 |
+
"upsample_rates": [
|
80 |
+
8,
|
81 |
+
8,
|
82 |
+
2,
|
83 |
+
2,
|
84 |
+
2
|
85 |
+
],
|
86 |
+
"upsample_initial_channel": 512,
|
87 |
+
"upsample_kernel_sizes": [
|
88 |
+
16,
|
89 |
+
16,
|
90 |
+
8,
|
91 |
+
2,
|
92 |
+
2
|
93 |
+
],
|
94 |
+
"n_layers_q": 3,
|
95 |
+
"use_spectral_norm": false,
|
96 |
+
"gin_channels": 256
|
97 |
+
},
|
98 |
+
"version": "2.1"
|
99 |
+
}
|
Data/models/G_82000.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9cff7a84b5bc7fbb44df429e1737132eebe0a01588acb0e7c2a30cb638ac08f5
|
3 |
+
size 718770997
|
__pycache__/attentions.cpython-310.pyc
ADDED
Binary file (11.1 kB). View file
|
|
__pycache__/commons.cpython-310.pyc
ADDED
Binary file (5.82 kB). View file
|
|
__pycache__/config.cpython-310.pyc
ADDED
Binary file (7.38 kB). View file
|
|
__pycache__/emo_gen.cpython-310.pyc
ADDED
Binary file (5.08 kB). View file
|
|
__pycache__/infer.cpython-310.pyc
ADDED
Binary file (5.09 kB). View file
|
|
__pycache__/models.cpython-310.pyc
ADDED
Binary file (21.6 kB). View file
|
|
__pycache__/modules.cpython-310.pyc
ADDED
Binary file (12.5 kB). View file
|
|
__pycache__/re_matching.cpython-310.pyc
ADDED
Binary file (2.53 kB). View file
|
|
__pycache__/transforms.cpython-310.pyc
ADDED
Binary file (3.89 kB). View file
|
|
__pycache__/utils.cpython-310.pyc
ADDED
Binary file (13.4 kB). View file
|
|
app.py
ADDED
@@ -0,0 +1,485 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# flake8: noqa: E402
|
2 |
+
import os
|
3 |
+
import logging
|
4 |
+
import re_matching
|
5 |
+
from tools.sentence import split_by_language
|
6 |
+
|
7 |
+
logging.getLogger("numba").setLevel(logging.WARNING)
|
8 |
+
logging.getLogger("markdown_it").setLevel(logging.WARNING)
|
9 |
+
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
10 |
+
logging.getLogger("matplotlib").setLevel(logging.WARNING)
|
11 |
+
|
12 |
+
logging.basicConfig(
|
13 |
+
level=logging.INFO, format="| %(name)s | %(levelname)s | %(message)s"
|
14 |
+
)
|
15 |
+
|
16 |
+
logger = logging.getLogger(__name__)
|
17 |
+
|
18 |
+
import torch
|
19 |
+
import utils
|
20 |
+
from infer import infer, latest_version, get_net_g, infer_multilang
|
21 |
+
import gradio as gr
|
22 |
+
import webbrowser
|
23 |
+
import numpy as np
|
24 |
+
from config import config
|
25 |
+
from tools.translate import translate
|
26 |
+
import librosa
|
27 |
+
|
28 |
+
net_g = None
|
29 |
+
|
30 |
+
device = config.webui_config.device
|
31 |
+
if device == "mps":
|
32 |
+
os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
|
33 |
+
|
34 |
+
|
35 |
+
def generate_audio(
|
36 |
+
slices,
|
37 |
+
sdp_ratio,
|
38 |
+
noise_scale,
|
39 |
+
noise_scale_w,
|
40 |
+
length_scale,
|
41 |
+
speaker,
|
42 |
+
language,
|
43 |
+
reference_audio,
|
44 |
+
emotion,
|
45 |
+
skip_start=False,
|
46 |
+
skip_end=False,
|
47 |
+
):
|
48 |
+
audio_list = []
|
49 |
+
# silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
|
50 |
+
with torch.no_grad():
|
51 |
+
for idx, piece in enumerate(slices):
|
52 |
+
skip_start = (idx != 0) and skip_start
|
53 |
+
skip_end = (idx != len(slices) - 1) and skip_end
|
54 |
+
audio = infer(
|
55 |
+
piece,
|
56 |
+
reference_audio=reference_audio,
|
57 |
+
emotion=emotion,
|
58 |
+
sdp_ratio=sdp_ratio,
|
59 |
+
noise_scale=noise_scale,
|
60 |
+
noise_scale_w=noise_scale_w,
|
61 |
+
length_scale=length_scale,
|
62 |
+
sid=speaker,
|
63 |
+
language=language,
|
64 |
+
hps=hps,
|
65 |
+
net_g=net_g,
|
66 |
+
device=device,
|
67 |
+
skip_start=skip_start,
|
68 |
+
skip_end=skip_end,
|
69 |
+
)
|
70 |
+
audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
|
71 |
+
audio_list.append(audio16bit)
|
72 |
+
# audio_list.append(silence) # 将静音添加到列表中
|
73 |
+
return audio_list
|
74 |
+
|
75 |
+
|
76 |
+
def generate_audio_multilang(
|
77 |
+
slices,
|
78 |
+
sdp_ratio,
|
79 |
+
noise_scale,
|
80 |
+
noise_scale_w,
|
81 |
+
length_scale,
|
82 |
+
speaker,
|
83 |
+
language,
|
84 |
+
reference_audio,
|
85 |
+
emotion,
|
86 |
+
skip_start=False,
|
87 |
+
skip_end=False,
|
88 |
+
):
|
89 |
+
audio_list = []
|
90 |
+
# silence = np.zeros(hps.data.sampling_rate // 2, dtype=np.int16)
|
91 |
+
with torch.no_grad():
|
92 |
+
for idx, piece in enumerate(slices):
|
93 |
+
skip_start = (idx != 0) and skip_start
|
94 |
+
skip_end = (idx != len(slices) - 1) and skip_end
|
95 |
+
audio = infer_multilang(
|
96 |
+
piece,
|
97 |
+
reference_audio=reference_audio,
|
98 |
+
emotion=emotion,
|
99 |
+
sdp_ratio=sdp_ratio,
|
100 |
+
noise_scale=noise_scale,
|
101 |
+
noise_scale_w=noise_scale_w,
|
102 |
+
length_scale=length_scale,
|
103 |
+
sid=speaker,
|
104 |
+
language=language[idx],
|
105 |
+
hps=hps,
|
106 |
+
net_g=net_g,
|
107 |
+
device=device,
|
108 |
+
skip_start=skip_start,
|
109 |
+
skip_end=skip_end,
|
110 |
+
)
|
111 |
+
audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
|
112 |
+
audio_list.append(audio16bit)
|
113 |
+
# audio_list.append(silence) # 将静音添加到列表中
|
114 |
+
return audio_list
|
115 |
+
|
116 |
+
|
117 |
+
def tts_split(
|
118 |
+
text: str,
|
119 |
+
speaker,
|
120 |
+
sdp_ratio,
|
121 |
+
noise_scale,
|
122 |
+
noise_scale_w,
|
123 |
+
length_scale,
|
124 |
+
language,
|
125 |
+
cut_by_sent,
|
126 |
+
interval_between_para,
|
127 |
+
interval_between_sent,
|
128 |
+
reference_audio,
|
129 |
+
emotion,
|
130 |
+
):
|
131 |
+
if language == "mix":
|
132 |
+
return ("invalid", None)
|
133 |
+
while text.find("\n\n") != -1:
|
134 |
+
text = text.replace("\n\n", "\n")
|
135 |
+
para_list = re_matching.cut_para(text)
|
136 |
+
audio_list = []
|
137 |
+
if not cut_by_sent:
|
138 |
+
for idx, p in enumerate(para_list):
|
139 |
+
skip_start = idx != 0
|
140 |
+
skip_end = idx != len(para_list) - 1
|
141 |
+
audio = infer(
|
142 |
+
p,
|
143 |
+
reference_audio=reference_audio,
|
144 |
+
emotion=emotion,
|
145 |
+
sdp_ratio=sdp_ratio,
|
146 |
+
noise_scale=noise_scale,
|
147 |
+
noise_scale_w=noise_scale_w,
|
148 |
+
length_scale=length_scale,
|
149 |
+
sid=speaker,
|
150 |
+
language=language,
|
151 |
+
hps=hps,
|
152 |
+
net_g=net_g,
|
153 |
+
device=device,
|
154 |
+
skip_start=skip_start,
|
155 |
+
skip_end=skip_end,
|
156 |
+
)
|
157 |
+
audio16bit = gr.processing_utils.convert_to_16_bit_wav(audio)
|
158 |
+
audio_list.append(audio16bit)
|
159 |
+
silence = np.zeros((int)(44100 * interval_between_para), dtype=np.int16)
|
160 |
+
audio_list.append(silence)
|
161 |
+
else:
|
162 |
+
for idx, p in enumerate(para_list):
|
163 |
+
skip_start = idx != 0
|
164 |
+
skip_end = idx != len(para_list) - 1
|
165 |
+
audio_list_sent = []
|
166 |
+
sent_list = re_matching.cut_sent(p)
|
167 |
+
for idx, s in enumerate(sent_list):
|
168 |
+
skip_start = (idx != 0) and skip_start
|
169 |
+
skip_end = (idx != len(sent_list) - 1) and skip_end
|
170 |
+
audio = infer(
|
171 |
+
s,
|
172 |
+
reference_audio=reference_audio,
|
173 |
+
emotion=emotion,
|
174 |
+
sdp_ratio=sdp_ratio,
|
175 |
+
noise_scale=noise_scale,
|
176 |
+
noise_scale_w=noise_scale_w,
|
177 |
+
length_scale=length_scale,
|
178 |
+
sid=speaker,
|
179 |
+
language=language,
|
180 |
+
hps=hps,
|
181 |
+
net_g=net_g,
|
182 |
+
device=device,
|
183 |
+
skip_start=skip_start,
|
184 |
+
skip_end=skip_end,
|
185 |
+
)
|
186 |
+
audio_list_sent.append(audio)
|
187 |
+
silence = np.zeros((int)(44100 * interval_between_sent))
|
188 |
+
audio_list_sent.append(silence)
|
189 |
+
if (interval_between_para - interval_between_sent) > 0:
|
190 |
+
silence = np.zeros(
|
191 |
+
(int)(44100 * (interval_between_para - interval_between_sent))
|
192 |
+
)
|
193 |
+
audio_list_sent.append(silence)
|
194 |
+
audio16bit = gr.processing_utils.convert_to_16_bit_wav(
|
195 |
+
np.concatenate(audio_list_sent)
|
196 |
+
) # 对完整句子做音量归一
|
197 |
+
audio_list.append(audio16bit)
|
198 |
+
audio_concat = np.concatenate(audio_list)
|
199 |
+
return ("Success", (44100, audio_concat))
|
200 |
+
|
201 |
+
|
202 |
+
def tts_fn(
|
203 |
+
text: str,
|
204 |
+
speaker,
|
205 |
+
sdp_ratio,
|
206 |
+
noise_scale,
|
207 |
+
noise_scale_w,
|
208 |
+
length_scale,
|
209 |
+
language,
|
210 |
+
reference_audio,
|
211 |
+
emotion,
|
212 |
+
):
|
213 |
+
audio_list = []
|
214 |
+
if language == "mix":
|
215 |
+
bool_valid, str_valid = re_matching.validate_text(text)
|
216 |
+
if not bool_valid:
|
217 |
+
return str_valid, (
|
218 |
+
hps.data.sampling_rate,
|
219 |
+
np.concatenate([np.zeros(hps.data.sampling_rate // 2)]),
|
220 |
+
)
|
221 |
+
result = []
|
222 |
+
for slice in re_matching.text_matching(text):
|
223 |
+
_speaker = slice.pop()
|
224 |
+
temp_contant = []
|
225 |
+
temp_lang = []
|
226 |
+
for lang, content in slice:
|
227 |
+
if "|" in content:
|
228 |
+
temp = []
|
229 |
+
temp_ = []
|
230 |
+
for i in content.split("|"):
|
231 |
+
if i != "":
|
232 |
+
temp.append([i])
|
233 |
+
temp_.append([lang])
|
234 |
+
else:
|
235 |
+
temp.append([])
|
236 |
+
temp_.append([])
|
237 |
+
temp_contant += temp
|
238 |
+
temp_lang += temp_
|
239 |
+
else:
|
240 |
+
if len(temp_contant) == 0:
|
241 |
+
temp_contant.append([])
|
242 |
+
temp_lang.append([])
|
243 |
+
temp_contant[-1].append(content)
|
244 |
+
temp_lang[-1].append(lang)
|
245 |
+
for i, j in zip(temp_lang, temp_contant):
|
246 |
+
result.append([*zip(i, j), _speaker])
|
247 |
+
for i, one in enumerate(result):
|
248 |
+
skip_start = i != 0
|
249 |
+
skip_end = i != len(result) - 1
|
250 |
+
_speaker = one.pop()
|
251 |
+
idx = 0
|
252 |
+
while idx < len(one):
|
253 |
+
text_to_generate = []
|
254 |
+
lang_to_generate = []
|
255 |
+
while True:
|
256 |
+
lang, content = one[idx]
|
257 |
+
temp_text = [content]
|
258 |
+
if len(text_to_generate) > 0:
|
259 |
+
text_to_generate[-1] += [temp_text.pop(0)]
|
260 |
+
lang_to_generate[-1] += [lang]
|
261 |
+
if len(temp_text) > 0:
|
262 |
+
text_to_generate += [[i] for i in temp_text]
|
263 |
+
lang_to_generate += [[lang]] * len(temp_text)
|
264 |
+
if idx + 1 < len(one):
|
265 |
+
idx += 1
|
266 |
+
else:
|
267 |
+
break
|
268 |
+
skip_start = (idx != 0) and skip_start
|
269 |
+
skip_end = (idx != len(one) - 1) and skip_end
|
270 |
+
print(text_to_generate, lang_to_generate)
|
271 |
+
audio_list.extend(
|
272 |
+
generate_audio_multilang(
|
273 |
+
text_to_generate,
|
274 |
+
sdp_ratio,
|
275 |
+
noise_scale,
|
276 |
+
noise_scale_w,
|
277 |
+
length_scale,
|
278 |
+
speaker,
|
279 |
+
lang_to_generate,
|
280 |
+
reference_audio,
|
281 |
+
emotion,
|
282 |
+
skip_start,
|
283 |
+
skip_end,
|
284 |
+
)
|
285 |
+
)
|
286 |
+
idx += 1
|
287 |
+
elif language.lower() == "auto":
|
288 |
+
for idx, slice in enumerate(text.split("|")):
|
289 |
+
if slice == "":
|
290 |
+
continue
|
291 |
+
skip_start = idx != 0
|
292 |
+
skip_end = idx != len(text.split("|")) - 1
|
293 |
+
sentences_list = split_by_language(
|
294 |
+
slice, target_languages=["zh", "ja", "en"]
|
295 |
+
)
|
296 |
+
idx = 0
|
297 |
+
while idx < len(sentences_list):
|
298 |
+
text_to_generate = []
|
299 |
+
lang_to_generate = []
|
300 |
+
while True:
|
301 |
+
content, lang = sentences_list[idx]
|
302 |
+
temp_text = [content]
|
303 |
+
lang = lang.upper()
|
304 |
+
if lang == "JA":
|
305 |
+
lang = "JP"
|
306 |
+
if len(text_to_generate) > 0:
|
307 |
+
text_to_generate[-1] += [temp_text.pop(0)]
|
308 |
+
lang_to_generate[-1] += [lang]
|
309 |
+
if len(temp_text) > 0:
|
310 |
+
text_to_generate += [[i] for i in temp_text]
|
311 |
+
lang_to_generate += [[lang]] * len(temp_text)
|
312 |
+
if idx + 1 < len(sentences_list):
|
313 |
+
idx += 1
|
314 |
+
else:
|
315 |
+
break
|
316 |
+
skip_start = (idx != 0) and skip_start
|
317 |
+
skip_end = (idx != len(sentences_list) - 1) and skip_end
|
318 |
+
print(text_to_generate, lang_to_generate)
|
319 |
+
audio_list.extend(
|
320 |
+
generate_audio_multilang(
|
321 |
+
text_to_generate,
|
322 |
+
sdp_ratio,
|
323 |
+
noise_scale,
|
324 |
+
noise_scale_w,
|
325 |
+
length_scale,
|
326 |
+
speaker,
|
327 |
+
lang_to_generate,
|
328 |
+
reference_audio,
|
329 |
+
emotion,
|
330 |
+
skip_start,
|
331 |
+
skip_end,
|
332 |
+
)
|
333 |
+
)
|
334 |
+
idx += 1
|
335 |
+
else:
|
336 |
+
audio_list.extend(
|
337 |
+
generate_audio(
|
338 |
+
text.split("|"),
|
339 |
+
sdp_ratio,
|
340 |
+
noise_scale,
|
341 |
+
noise_scale_w,
|
342 |
+
length_scale,
|
343 |
+
speaker,
|
344 |
+
language,
|
345 |
+
reference_audio,
|
346 |
+
emotion,
|
347 |
+
)
|
348 |
+
)
|
349 |
+
|
350 |
+
audio_concat = np.concatenate(audio_list)
|
351 |
+
return "Success", (hps.data.sampling_rate, audio_concat)
|
352 |
+
|
353 |
+
|
354 |
+
if __name__ == "__main__":
|
355 |
+
if config.webui_config.debug:
|
356 |
+
logger.info("Enable DEBUG-LEVEL log")
|
357 |
+
logging.basicConfig(level=logging.DEBUG)
|
358 |
+
hps = utils.get_hparams_from_file(config.webui_config.config_path)
|
359 |
+
# 若config.json中未指定版本则默认为最新版本
|
360 |
+
version = hps.version if hasattr(hps, "version") else latest_version
|
361 |
+
net_g = get_net_g(
|
362 |
+
model_path=config.webui_config.model, version=version, device=device, hps=hps
|
363 |
+
)
|
364 |
+
speaker_ids = hps.data.spk2id
|
365 |
+
speakers = list(speaker_ids.keys())
|
366 |
+
languages = ["ZH", "JP", "EN", "mix", "auto"]
|
367 |
+
with gr.Blocks() as app:
|
368 |
+
with gr.Row():
|
369 |
+
with gr.Column():
|
370 |
+
text = gr.TextArea(
|
371 |
+
label="输入文本内容",
|
372 |
+
placeholder="""
|
373 |
+
如果你选择语言为\'mix\',必须按照格式输入,否则报错:
|
374 |
+
格式举例(zh是中文,jp是日语,不区分大小写;说话人举例:gongzi):
|
375 |
+
[说话人1]<zh>你好,こんにちは! <jp>こんにちは,世界。
|
376 |
+
[说话人2]<zh>你好吗?<jp>元気ですか?
|
377 |
+
[说话人3]<zh>谢谢。<jp>どういたしまして。
|
378 |
+
...
|
379 |
+
另外,所有的语言选项都可以用'|'分割长段实现分句生成。
|
380 |
+
""",
|
381 |
+
)
|
382 |
+
trans = gr.Button("中翻日", variant="primary")
|
383 |
+
slicer = gr.Button("快速切分", variant="primary")
|
384 |
+
speaker = gr.Dropdown(
|
385 |
+
choices=speakers, value=speakers[0], label="Speaker"
|
386 |
+
)
|
387 |
+
emotion = gr.Slider(
|
388 |
+
minimum=0, maximum=9, value=0, step=1, label="Emotion"
|
389 |
+
)
|
390 |
+
sdp_ratio = gr.Slider(
|
391 |
+
minimum=0, maximum=1, value=0.2, step=0.1, label="SDP Ratio"
|
392 |
+
)
|
393 |
+
noise_scale = gr.Slider(
|
394 |
+
minimum=0.1, maximum=2, value=0.6, step=0.1, label="Noise"
|
395 |
+
)
|
396 |
+
noise_scale_w = gr.Slider(
|
397 |
+
minimum=0.1, maximum=2, value=0.8, step=0.1, label="Noise_W"
|
398 |
+
)
|
399 |
+
length_scale = gr.Slider(
|
400 |
+
minimum=0.1, maximum=2, value=1.0, step=0.1, label="Length"
|
401 |
+
)
|
402 |
+
language = gr.Dropdown(
|
403 |
+
choices=languages, value=languages[0], label="Language"
|
404 |
+
)
|
405 |
+
btn = gr.Button("生成音频!", variant="primary")
|
406 |
+
with gr.Column():
|
407 |
+
with gr.Row():
|
408 |
+
with gr.Column():
|
409 |
+
interval_between_sent = gr.Slider(
|
410 |
+
minimum=0,
|
411 |
+
maximum=5,
|
412 |
+
value=0.2,
|
413 |
+
step=0.1,
|
414 |
+
label="句间停顿(秒),勾选按句切分才生效",
|
415 |
+
)
|
416 |
+
interval_between_para = gr.Slider(
|
417 |
+
minimum=0,
|
418 |
+
maximum=10,
|
419 |
+
value=1,
|
420 |
+
step=0.1,
|
421 |
+
label="段间停顿(秒),需要大于句间停顿才有效",
|
422 |
+
)
|
423 |
+
opt_cut_by_sent = gr.Checkbox(
|
424 |
+
label="按句切分 在按段落切分的基础上再按句子切分文本"
|
425 |
+
)
|
426 |
+
slicer = gr.Button("切分生成", variant="primary")
|
427 |
+
text_output = gr.Textbox(label="状态信息")
|
428 |
+
audio_output = gr.Audio(label="输出音频")
|
429 |
+
# explain_image = gr.Image(
|
430 |
+
# label="参数解释信息",
|
431 |
+
# show_label=True,
|
432 |
+
# show_share_button=False,
|
433 |
+
# show_download_button=False,
|
434 |
+
# value=os.path.abspath("./img/参数说明.png"),
|
435 |
+
# )
|
436 |
+
reference_text = gr.Markdown(value="## 情感参考音频(WAV 格式):用于生成语音的情感参考。")
|
437 |
+
reference_audio = gr.Audio(label="情感参考音频(WAV 格式)", type="filepath")
|
438 |
+
btn.click(
|
439 |
+
tts_fn,
|
440 |
+
inputs=[
|
441 |
+
text,
|
442 |
+
speaker,
|
443 |
+
sdp_ratio,
|
444 |
+
noise_scale,
|
445 |
+
noise_scale_w,
|
446 |
+
length_scale,
|
447 |
+
language,
|
448 |
+
reference_audio,
|
449 |
+
emotion,
|
450 |
+
],
|
451 |
+
outputs=[text_output, audio_output],
|
452 |
+
)
|
453 |
+
|
454 |
+
trans.click(
|
455 |
+
translate,
|
456 |
+
inputs=[text],
|
457 |
+
outputs=[text],
|
458 |
+
)
|
459 |
+
slicer.click(
|
460 |
+
tts_split,
|
461 |
+
inputs=[
|
462 |
+
text,
|
463 |
+
speaker,
|
464 |
+
sdp_ratio,
|
465 |
+
noise_scale,
|
466 |
+
noise_scale_w,
|
467 |
+
length_scale,
|
468 |
+
language,
|
469 |
+
opt_cut_by_sent,
|
470 |
+
interval_between_para,
|
471 |
+
interval_between_sent,
|
472 |
+
reference_audio,
|
473 |
+
emotion,
|
474 |
+
],
|
475 |
+
outputs=[text_output, audio_output],
|
476 |
+
)
|
477 |
+
|
478 |
+
reference_audio.upload(
|
479 |
+
lambda x: librosa.load(x, 16000)[::-1],
|
480 |
+
inputs=[reference_audio],
|
481 |
+
outputs=[reference_audio],
|
482 |
+
)
|
483 |
+
print("推理页面已开启!")
|
484 |
+
webbrowser.open(f"http://127.0.0.1:{config.webui_config.port}")
|
485 |
+
app.launch(share=config.webui_config.share, server_port=config.webui_config.port)
|
attentions.py
ADDED
@@ -0,0 +1,464 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import math
|
2 |
+
import torch
|
3 |
+
from torch import nn
|
4 |
+
from torch.nn import functional as F
|
5 |
+
|
6 |
+
import commons
|
7 |
+
import logging
|
8 |
+
|
9 |
+
logger = logging.getLogger(__name__)
|
10 |
+
|
11 |
+
|
12 |
+
class LayerNorm(nn.Module):
|
13 |
+
def __init__(self, channels, eps=1e-5):
|
14 |
+
super().__init__()
|
15 |
+
self.channels = channels
|
16 |
+
self.eps = eps
|
17 |
+
|
18 |
+
self.gamma = nn.Parameter(torch.ones(channels))
|
19 |
+
self.beta = nn.Parameter(torch.zeros(channels))
|
20 |
+
|
21 |
+
def forward(self, x):
|
22 |
+
x = x.transpose(1, -1)
|
23 |
+
x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
|
24 |
+
return x.transpose(1, -1)
|
25 |
+
|
26 |
+
|
27 |
+
@torch.jit.script
|
28 |
+
def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
|
29 |
+
n_channels_int = n_channels[0]
|
30 |
+
in_act = input_a + input_b
|
31 |
+
t_act = torch.tanh(in_act[:, :n_channels_int, :])
|
32 |
+
s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
|
33 |
+
acts = t_act * s_act
|
34 |
+
return acts
|
35 |
+
|
36 |
+
|
37 |
+
class Encoder(nn.Module):
|
38 |
+
def __init__(
|
39 |
+
self,
|
40 |
+
hidden_channels,
|
41 |
+
filter_channels,
|
42 |
+
n_heads,
|
43 |
+
n_layers,
|
44 |
+
kernel_size=1,
|
45 |
+
p_dropout=0.0,
|
46 |
+
window_size=4,
|
47 |
+
isflow=True,
|
48 |
+
**kwargs
|
49 |
+
):
|
50 |
+
super().__init__()
|
51 |
+
self.hidden_channels = hidden_channels
|
52 |
+
self.filter_channels = filter_channels
|
53 |
+
self.n_heads = n_heads
|
54 |
+
self.n_layers = n_layers
|
55 |
+
self.kernel_size = kernel_size
|
56 |
+
self.p_dropout = p_dropout
|
57 |
+
self.window_size = window_size
|
58 |
+
# if isflow:
|
59 |
+
# cond_layer = torch.nn.Conv1d(256, 2*hidden_channels*n_layers, 1)
|
60 |
+
# self.cond_pre = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, 1)
|
61 |
+
# self.cond_layer = weight_norm(cond_layer, name='weight')
|
62 |
+
# self.gin_channels = 256
|
63 |
+
self.cond_layer_idx = self.n_layers
|
64 |
+
if "gin_channels" in kwargs:
|
65 |
+
self.gin_channels = kwargs["gin_channels"]
|
66 |
+
if self.gin_channels != 0:
|
67 |
+
self.spk_emb_linear = nn.Linear(self.gin_channels, self.hidden_channels)
|
68 |
+
# vits2 says 3rd block, so idx is 2 by default
|
69 |
+
self.cond_layer_idx = (
|
70 |
+
kwargs["cond_layer_idx"] if "cond_layer_idx" in kwargs else 2
|
71 |
+
)
|
72 |
+
logging.debug(self.gin_channels, self.cond_layer_idx)
|
73 |
+
assert (
|
74 |
+
self.cond_layer_idx < self.n_layers
|
75 |
+
), "cond_layer_idx should be less than n_layers"
|
76 |
+
self.drop = nn.Dropout(p_dropout)
|
77 |
+
self.attn_layers = nn.ModuleList()
|
78 |
+
self.norm_layers_1 = nn.ModuleList()
|
79 |
+
self.ffn_layers = nn.ModuleList()
|
80 |
+
self.norm_layers_2 = nn.ModuleList()
|
81 |
+
for i in range(self.n_layers):
|
82 |
+
self.attn_layers.append(
|
83 |
+
MultiHeadAttention(
|
84 |
+
hidden_channels,
|
85 |
+
hidden_channels,
|
86 |
+
n_heads,
|
87 |
+
p_dropout=p_dropout,
|
88 |
+
window_size=window_size,
|
89 |
+
)
|
90 |
+
)
|
91 |
+
self.norm_layers_1.append(LayerNorm(hidden_channels))
|
92 |
+
self.ffn_layers.append(
|
93 |
+
FFN(
|
94 |
+
hidden_channels,
|
95 |
+
hidden_channels,
|
96 |
+
filter_channels,
|
97 |
+
kernel_size,
|
98 |
+
p_dropout=p_dropout,
|
99 |
+
)
|
100 |
+
)
|
101 |
+
self.norm_layers_2.append(LayerNorm(hidden_channels))
|
102 |
+
|
103 |
+
def forward(self, x, x_mask, g=None):
|
104 |
+
attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
|
105 |
+
x = x * x_mask
|
106 |
+
for i in range(self.n_layers):
|
107 |
+
if i == self.cond_layer_idx and g is not None:
|
108 |
+
g = self.spk_emb_linear(g.transpose(1, 2))
|
109 |
+
g = g.transpose(1, 2)
|
110 |
+
x = x + g
|
111 |
+
x = x * x_mask
|
112 |
+
y = self.attn_layers[i](x, x, attn_mask)
|
113 |
+
y = self.drop(y)
|
114 |
+
x = self.norm_layers_1[i](x + y)
|
115 |
+
|
116 |
+
y = self.ffn_layers[i](x, x_mask)
|
117 |
+
y = self.drop(y)
|
118 |
+
x = self.norm_layers_2[i](x + y)
|
119 |
+
x = x * x_mask
|
120 |
+
return x
|
121 |
+
|
122 |
+
|
123 |
+
class Decoder(nn.Module):
|
124 |
+
def __init__(
|
125 |
+
self,
|
126 |
+
hidden_channels,
|
127 |
+
filter_channels,
|
128 |
+
n_heads,
|
129 |
+
n_layers,
|
130 |
+
kernel_size=1,
|
131 |
+
p_dropout=0.0,
|
132 |
+
proximal_bias=False,
|
133 |
+
proximal_init=True,
|
134 |
+
**kwargs
|
135 |
+
):
|
136 |
+
super().__init__()
|
137 |
+
self.hidden_channels = hidden_channels
|
138 |
+
self.filter_channels = filter_channels
|
139 |
+
self.n_heads = n_heads
|
140 |
+
self.n_layers = n_layers
|
141 |
+
self.kernel_size = kernel_size
|
142 |
+
self.p_dropout = p_dropout
|
143 |
+
self.proximal_bias = proximal_bias
|
144 |
+
self.proximal_init = proximal_init
|
145 |
+
|
146 |
+
self.drop = nn.Dropout(p_dropout)
|
147 |
+
self.self_attn_layers = nn.ModuleList()
|
148 |
+
self.norm_layers_0 = nn.ModuleList()
|
149 |
+
self.encdec_attn_layers = nn.ModuleList()
|
150 |
+
self.norm_layers_1 = nn.ModuleList()
|
151 |
+
self.ffn_layers = nn.ModuleList()
|
152 |
+
self.norm_layers_2 = nn.ModuleList()
|
153 |
+
for i in range(self.n_layers):
|
154 |
+
self.self_attn_layers.append(
|
155 |
+
MultiHeadAttention(
|
156 |
+
hidden_channels,
|
157 |
+
hidden_channels,
|
158 |
+
n_heads,
|
159 |
+
p_dropout=p_dropout,
|
160 |
+
proximal_bias=proximal_bias,
|
161 |
+
proximal_init=proximal_init,
|
162 |
+
)
|
163 |
+
)
|
164 |
+
self.norm_layers_0.append(LayerNorm(hidden_channels))
|
165 |
+
self.encdec_attn_layers.append(
|
166 |
+
MultiHeadAttention(
|
167 |
+
hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout
|
168 |
+
)
|
169 |
+
)
|
170 |
+
self.norm_layers_1.append(LayerNorm(hidden_channels))
|
171 |
+
self.ffn_layers.append(
|
172 |
+
FFN(
|
173 |
+
hidden_channels,
|
174 |
+
hidden_channels,
|
175 |
+
filter_channels,
|
176 |
+
kernel_size,
|
177 |
+
p_dropout=p_dropout,
|
178 |
+
causal=True,
|
179 |
+
)
|
180 |
+
)
|
181 |
+
self.norm_layers_2.append(LayerNorm(hidden_channels))
|
182 |
+
|
183 |
+
def forward(self, x, x_mask, h, h_mask):
|
184 |
+
"""
|
185 |
+
x: decoder input
|
186 |
+
h: encoder output
|
187 |
+
"""
|
188 |
+
self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(
|
189 |
+
device=x.device, dtype=x.dtype
|
190 |
+
)
|
191 |
+
encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
|
192 |
+
x = x * x_mask
|
193 |
+
for i in range(self.n_layers):
|
194 |
+
y = self.self_attn_layers[i](x, x, self_attn_mask)
|
195 |
+
y = self.drop(y)
|
196 |
+
x = self.norm_layers_0[i](x + y)
|
197 |
+
|
198 |
+
y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
|
199 |
+
y = self.drop(y)
|
200 |
+
x = self.norm_layers_1[i](x + y)
|
201 |
+
|
202 |
+
y = self.ffn_layers[i](x, x_mask)
|
203 |
+
y = self.drop(y)
|
204 |
+
x = self.norm_layers_2[i](x + y)
|
205 |
+
x = x * x_mask
|
206 |
+
return x
|
207 |
+
|
208 |
+
|
209 |
+
class MultiHeadAttention(nn.Module):
|
210 |
+
def __init__(
|
211 |
+
self,
|
212 |
+
channels,
|
213 |
+
out_channels,
|
214 |
+
n_heads,
|
215 |
+
p_dropout=0.0,
|
216 |
+
window_size=None,
|
217 |
+
heads_share=True,
|
218 |
+
block_length=None,
|
219 |
+
proximal_bias=False,
|
220 |
+
proximal_init=False,
|
221 |
+
):
|
222 |
+
super().__init__()
|
223 |
+
assert channels % n_heads == 0
|
224 |
+
|
225 |
+
self.channels = channels
|
226 |
+
self.out_channels = out_channels
|
227 |
+
self.n_heads = n_heads
|
228 |
+
self.p_dropout = p_dropout
|
229 |
+
self.window_size = window_size
|
230 |
+
self.heads_share = heads_share
|
231 |
+
self.block_length = block_length
|
232 |
+
self.proximal_bias = proximal_bias
|
233 |
+
self.proximal_init = proximal_init
|
234 |
+
self.attn = None
|
235 |
+
|
236 |
+
self.k_channels = channels // n_heads
|
237 |
+
self.conv_q = nn.Conv1d(channels, channels, 1)
|
238 |
+
self.conv_k = nn.Conv1d(channels, channels, 1)
|
239 |
+
self.conv_v = nn.Conv1d(channels, channels, 1)
|
240 |
+
self.conv_o = nn.Conv1d(channels, out_channels, 1)
|
241 |
+
self.drop = nn.Dropout(p_dropout)
|
242 |
+
|
243 |
+
if window_size is not None:
|
244 |
+
n_heads_rel = 1 if heads_share else n_heads
|
245 |
+
rel_stddev = self.k_channels**-0.5
|
246 |
+
self.emb_rel_k = nn.Parameter(
|
247 |
+
torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
|
248 |
+
* rel_stddev
|
249 |
+
)
|
250 |
+
self.emb_rel_v = nn.Parameter(
|
251 |
+
torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels)
|
252 |
+
* rel_stddev
|
253 |
+
)
|
254 |
+
|
255 |
+
nn.init.xavier_uniform_(self.conv_q.weight)
|
256 |
+
nn.init.xavier_uniform_(self.conv_k.weight)
|
257 |
+
nn.init.xavier_uniform_(self.conv_v.weight)
|
258 |
+
if proximal_init:
|
259 |
+
with torch.no_grad():
|
260 |
+
self.conv_k.weight.copy_(self.conv_q.weight)
|
261 |
+
self.conv_k.bias.copy_(self.conv_q.bias)
|
262 |
+
|
263 |
+
def forward(self, x, c, attn_mask=None):
|
264 |
+
q = self.conv_q(x)
|
265 |
+
k = self.conv_k(c)
|
266 |
+
v = self.conv_v(c)
|
267 |
+
|
268 |
+
x, self.attn = self.attention(q, k, v, mask=attn_mask)
|
269 |
+
|
270 |
+
x = self.conv_o(x)
|
271 |
+
return x
|
272 |
+
|
273 |
+
def attention(self, query, key, value, mask=None):
|
274 |
+
# reshape [b, d, t] -> [b, n_h, t, d_k]
|
275 |
+
b, d, t_s, t_t = (*key.size(), query.size(2))
|
276 |
+
query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
|
277 |
+
key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
|
278 |
+
value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
|
279 |
+
|
280 |
+
scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
|
281 |
+
if self.window_size is not None:
|
282 |
+
assert (
|
283 |
+
t_s == t_t
|
284 |
+
), "Relative attention is only available for self-attention."
|
285 |
+
key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
|
286 |
+
rel_logits = self._matmul_with_relative_keys(
|
287 |
+
query / math.sqrt(self.k_channels), key_relative_embeddings
|
288 |
+
)
|
289 |
+
scores_local = self._relative_position_to_absolute_position(rel_logits)
|
290 |
+
scores = scores + scores_local
|
291 |
+
if self.proximal_bias:
|
292 |
+
assert t_s == t_t, "Proximal bias is only available for self-attention."
|
293 |
+
scores = scores + self._attention_bias_proximal(t_s).to(
|
294 |
+
device=scores.device, dtype=scores.dtype
|
295 |
+
)
|
296 |
+
if mask is not None:
|
297 |
+
scores = scores.masked_fill(mask == 0, -1e4)
|
298 |
+
if self.block_length is not None:
|
299 |
+
assert (
|
300 |
+
t_s == t_t
|
301 |
+
), "Local attention is only available for self-attention."
|
302 |
+
block_mask = (
|
303 |
+
torch.ones_like(scores)
|
304 |
+
.triu(-self.block_length)
|
305 |
+
.tril(self.block_length)
|
306 |
+
)
|
307 |
+
scores = scores.masked_fill(block_mask == 0, -1e4)
|
308 |
+
p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s]
|
309 |
+
p_attn = self.drop(p_attn)
|
310 |
+
output = torch.matmul(p_attn, value)
|
311 |
+
if self.window_size is not None:
|
312 |
+
relative_weights = self._absolute_position_to_relative_position(p_attn)
|
313 |
+
value_relative_embeddings = self._get_relative_embeddings(
|
314 |
+
self.emb_rel_v, t_s
|
315 |
+
)
|
316 |
+
output = output + self._matmul_with_relative_values(
|
317 |
+
relative_weights, value_relative_embeddings
|
318 |
+
)
|
319 |
+
output = (
|
320 |
+
output.transpose(2, 3).contiguous().view(b, d, t_t)
|
321 |
+
) # [b, n_h, t_t, d_k] -> [b, d, t_t]
|
322 |
+
return output, p_attn
|
323 |
+
|
324 |
+
def _matmul_with_relative_values(self, x, y):
|
325 |
+
"""
|
326 |
+
x: [b, h, l, m]
|
327 |
+
y: [h or 1, m, d]
|
328 |
+
ret: [b, h, l, d]
|
329 |
+
"""
|
330 |
+
ret = torch.matmul(x, y.unsqueeze(0))
|
331 |
+
return ret
|
332 |
+
|
333 |
+
def _matmul_with_relative_keys(self, x, y):
|
334 |
+
"""
|
335 |
+
x: [b, h, l, d]
|
336 |
+
y: [h or 1, m, d]
|
337 |
+
ret: [b, h, l, m]
|
338 |
+
"""
|
339 |
+
ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
|
340 |
+
return ret
|
341 |
+
|
342 |
+
def _get_relative_embeddings(self, relative_embeddings, length):
|
343 |
+
2 * self.window_size + 1
|
344 |
+
# Pad first before slice to avoid using cond ops.
|
345 |
+
pad_length = max(length - (self.window_size + 1), 0)
|
346 |
+
slice_start_position = max((self.window_size + 1) - length, 0)
|
347 |
+
slice_end_position = slice_start_position + 2 * length - 1
|
348 |
+
if pad_length > 0:
|
349 |
+
padded_relative_embeddings = F.pad(
|
350 |
+
relative_embeddings,
|
351 |
+
commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]),
|
352 |
+
)
|
353 |
+
else:
|
354 |
+
padded_relative_embeddings = relative_embeddings
|
355 |
+
used_relative_embeddings = padded_relative_embeddings[
|
356 |
+
:, slice_start_position:slice_end_position
|
357 |
+
]
|
358 |
+
return used_relative_embeddings
|
359 |
+
|
360 |
+
def _relative_position_to_absolute_position(self, x):
|
361 |
+
"""
|
362 |
+
x: [b, h, l, 2*l-1]
|
363 |
+
ret: [b, h, l, l]
|
364 |
+
"""
|
365 |
+
batch, heads, length, _ = x.size()
|
366 |
+
# Concat columns of pad to shift from relative to absolute indexing.
|
367 |
+
x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, 1]]))
|
368 |
+
|
369 |
+
# Concat extra elements so to add up to shape (len+1, 2*len-1).
|
370 |
+
x_flat = x.view([batch, heads, length * 2 * length])
|
371 |
+
x_flat = F.pad(
|
372 |
+
x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [0, length - 1]])
|
373 |
+
)
|
374 |
+
|
375 |
+
# Reshape and slice out the padded elements.
|
376 |
+
x_final = x_flat.view([batch, heads, length + 1, 2 * length - 1])[
|
377 |
+
:, :, :length, length - 1 :
|
378 |
+
]
|
379 |
+
return x_final
|
380 |
+
|
381 |
+
def _absolute_position_to_relative_position(self, x):
|
382 |
+
"""
|
383 |
+
x: [b, h, l, l]
|
384 |
+
ret: [b, h, l, 2*l-1]
|
385 |
+
"""
|
386 |
+
batch, heads, length, _ = x.size()
|
387 |
+
# pad along column
|
388 |
+
x = F.pad(
|
389 |
+
x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length - 1]])
|
390 |
+
)
|
391 |
+
x_flat = x.view([batch, heads, length**2 + length * (length - 1)])
|
392 |
+
# add 0's in the beginning that will skew the elements after reshape
|
393 |
+
x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
|
394 |
+
x_final = x_flat.view([batch, heads, length, 2 * length])[:, :, :, 1:]
|
395 |
+
return x_final
|
396 |
+
|
397 |
+
def _attention_bias_proximal(self, length):
|
398 |
+
"""Bias for self-attention to encourage attention to close positions.
|
399 |
+
Args:
|
400 |
+
length: an integer scalar.
|
401 |
+
Returns:
|
402 |
+
a Tensor with shape [1, 1, length, length]
|
403 |
+
"""
|
404 |
+
r = torch.arange(length, dtype=torch.float32)
|
405 |
+
diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
|
406 |
+
return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
|
407 |
+
|
408 |
+
|
409 |
+
class FFN(nn.Module):
|
410 |
+
def __init__(
|
411 |
+
self,
|
412 |
+
in_channels,
|
413 |
+
out_channels,
|
414 |
+
filter_channels,
|
415 |
+
kernel_size,
|
416 |
+
p_dropout=0.0,
|
417 |
+
activation=None,
|
418 |
+
causal=False,
|
419 |
+
):
|
420 |
+
super().__init__()
|
421 |
+
self.in_channels = in_channels
|
422 |
+
self.out_channels = out_channels
|
423 |
+
self.filter_channels = filter_channels
|
424 |
+
self.kernel_size = kernel_size
|
425 |
+
self.p_dropout = p_dropout
|
426 |
+
self.activation = activation
|
427 |
+
self.causal = causal
|
428 |
+
|
429 |
+
if causal:
|
430 |
+
self.padding = self._causal_padding
|
431 |
+
else:
|
432 |
+
self.padding = self._same_padding
|
433 |
+
|
434 |
+
self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
|
435 |
+
self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
|
436 |
+
self.drop = nn.Dropout(p_dropout)
|
437 |
+
|
438 |
+
def forward(self, x, x_mask):
|
439 |
+
x = self.conv_1(self.padding(x * x_mask))
|
440 |
+
if self.activation == "gelu":
|
441 |
+
x = x * torch.sigmoid(1.702 * x)
|
442 |
+
else:
|
443 |
+
x = torch.relu(x)
|
444 |
+
x = self.drop(x)
|
445 |
+
x = self.conv_2(self.padding(x * x_mask))
|
446 |
+
return x * x_mask
|
447 |
+
|
448 |
+
def _causal_padding(self, x):
|
449 |
+
if self.kernel_size == 1:
|
450 |
+
return x
|
451 |
+
pad_l = self.kernel_size - 1
|
452 |
+
pad_r = 0
|
453 |
+
padding = [[0, 0], [0, 0], [pad_l, pad_r]]
|
454 |
+
x = F.pad(x, commons.convert_pad_shape(padding))
|
455 |
+
return x
|
456 |
+
|
457 |
+
def _same_padding(self, x):
|
458 |
+
if self.kernel_size == 1:
|
459 |
+
return x
|
460 |
+
pad_l = (self.kernel_size - 1) // 2
|
461 |
+
pad_r = self.kernel_size // 2
|
462 |
+
padding = [[0, 0], [0, 0], [pad_l, pad_r]]
|
463 |
+
x = F.pad(x, commons.convert_pad_shape(padding))
|
464 |
+
return x
|
bert/bert_models.json
ADDED
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"deberta-v2-large-japanese-char-wwm": {
|
3 |
+
"repo_id": "ku-nlp/deberta-v2-large-japanese-char-wwm",
|
4 |
+
"files": ["pytorch_model.bin"]
|
5 |
+
},
|
6 |
+
"chinese-roberta-wwm-ext-large": {
|
7 |
+
"repo_id": "hfl/chinese-roberta-wwm-ext-large",
|
8 |
+
"files": ["pytorch_model.bin"]
|
9 |
+
},
|
10 |
+
"deberta-v3-large": {
|
11 |
+
"repo_id": "microsoft/deberta-v3-large",
|
12 |
+
"files": ["spm.model", "pytorch_model.bin"]
|
13 |
+
}
|
14 |
+
}
|
commons.py
ADDED
@@ -0,0 +1,166 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import math
|
2 |
+
import torch
|
3 |
+
from torch.nn import functional as F
|
4 |
+
|
5 |
+
|
6 |
+
def init_weights(m, mean=0.0, std=0.01):
|
7 |
+
classname = m.__class__.__name__
|
8 |
+
if classname.find("Conv") != -1:
|
9 |
+
m.weight.data.normal_(mean, std)
|
10 |
+
|
11 |
+
|
12 |
+
def get_padding(kernel_size, dilation=1):
|
13 |
+
return int((kernel_size * dilation - dilation) / 2)
|
14 |
+
|
15 |
+
|
16 |
+
def convert_pad_shape(pad_shape):
|
17 |
+
layer = pad_shape[::-1]
|
18 |
+
pad_shape = [item for sublist in layer for item in sublist]
|
19 |
+
return pad_shape
|
20 |
+
|
21 |
+
|
22 |
+
def intersperse(lst, item):
|
23 |
+
result = [item] * (len(lst) * 2 + 1)
|
24 |
+
result[1::2] = lst
|
25 |
+
return result
|
26 |
+
|
27 |
+
|
28 |
+
def kl_divergence(m_p, logs_p, m_q, logs_q):
|
29 |
+
"""KL(P||Q)"""
|
30 |
+
kl = (logs_q - logs_p) - 0.5
|
31 |
+
kl += (
|
32 |
+
0.5 * (torch.exp(2.0 * logs_p) + ((m_p - m_q) ** 2)) * torch.exp(-2.0 * logs_q)
|
33 |
+
)
|
34 |
+
return kl
|
35 |
+
|
36 |
+
|
37 |
+
def rand_gumbel(shape):
|
38 |
+
"""Sample from the Gumbel distribution, protect from overflows."""
|
39 |
+
uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
|
40 |
+
return -torch.log(-torch.log(uniform_samples))
|
41 |
+
|
42 |
+
|
43 |
+
def rand_gumbel_like(x):
|
44 |
+
g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
|
45 |
+
return g
|
46 |
+
|
47 |
+
|
48 |
+
def slice_segments(x, ids_str, segment_size=4):
|
49 |
+
ret = torch.zeros_like(x[:, :, :segment_size])
|
50 |
+
for i in range(x.size(0)):
|
51 |
+
idx_str = ids_str[i]
|
52 |
+
idx_end = idx_str + segment_size
|
53 |
+
if idx_str < 0:
|
54 |
+
i1 = x.size(2) + idx_str
|
55 |
+
r1 = x[i, :, i1:]
|
56 |
+
r2 = x[i, :, :idx_end]
|
57 |
+
ret[i] = torch.cat([r1, r2], dim=1)
|
58 |
+
else:
|
59 |
+
ret[i] = x[i, :, idx_str:idx_end]
|
60 |
+
return ret
|
61 |
+
|
62 |
+
|
63 |
+
def rand_slice_segments(x, x_lengths=None, segment_size=4):
|
64 |
+
b, d, t = x.size()
|
65 |
+
if x_lengths is None:
|
66 |
+
x_lengths = t
|
67 |
+
ids_str_max = x_lengths - segment_size + 1
|
68 |
+
ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
|
69 |
+
ret = slice_segments(x, ids_str, segment_size)
|
70 |
+
return ret, ids_str
|
71 |
+
|
72 |
+
|
73 |
+
def get_timing_signal_1d(length, channels, min_timescale=1.0, max_timescale=1.0e4):
|
74 |
+
position = torch.arange(length, dtype=torch.float)
|
75 |
+
num_timescales = channels // 2
|
76 |
+
log_timescale_increment = math.log(float(max_timescale) / float(min_timescale)) / (
|
77 |
+
num_timescales - 1
|
78 |
+
)
|
79 |
+
inv_timescales = min_timescale * torch.exp(
|
80 |
+
torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment
|
81 |
+
)
|
82 |
+
scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
|
83 |
+
signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
|
84 |
+
signal = F.pad(signal, [0, 0, 0, channels % 2])
|
85 |
+
signal = signal.view(1, channels, length)
|
86 |
+
return signal
|
87 |
+
|
88 |
+
|
89 |
+
def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
|
90 |
+
b, channels, length = x.size()
|
91 |
+
signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
|
92 |
+
return x + signal.to(dtype=x.dtype, device=x.device)
|
93 |
+
|
94 |
+
|
95 |
+
def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
|
96 |
+
b, channels, length = x.size()
|
97 |
+
signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
|
98 |
+
return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
|
99 |
+
|
100 |
+
|
101 |
+
def subsequent_mask(length):
|
102 |
+
mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
|
103 |
+
return mask
|
104 |
+
|
105 |
+
|
106 |
+
@torch.jit.script
|
107 |
+
def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
|
108 |
+
n_channels_int = n_channels[0]
|
109 |
+
in_act = input_a + input_b
|
110 |
+
t_act = torch.tanh(in_act[:, :n_channels_int, :])
|
111 |
+
s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
|
112 |
+
acts = t_act * s_act
|
113 |
+
return acts
|
114 |
+
|
115 |
+
|
116 |
+
def convert_pad_shape(pad_shape):
|
117 |
+
layer = pad_shape[::-1]
|
118 |
+
pad_shape = [item for sublist in layer for item in sublist]
|
119 |
+
return pad_shape
|
120 |
+
|
121 |
+
|
122 |
+
def shift_1d(x):
|
123 |
+
x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
|
124 |
+
return x
|
125 |
+
|
126 |
+
|
127 |
+
def sequence_mask(length, max_length=None):
|
128 |
+
if max_length is None:
|
129 |
+
max_length = length.max()
|
130 |
+
x = torch.arange(max_length, dtype=length.dtype, device=length.device)
|
131 |
+
return x.unsqueeze(0) < length.unsqueeze(1)
|
132 |
+
|
133 |
+
|
134 |
+
def generate_path(duration, mask):
|
135 |
+
"""
|
136 |
+
duration: [b, 1, t_x]
|
137 |
+
mask: [b, 1, t_y, t_x]
|
138 |
+
"""
|
139 |
+
|
140 |
+
b, _, t_y, t_x = mask.shape
|
141 |
+
cum_duration = torch.cumsum(duration, -1)
|
142 |
+
|
143 |
+
cum_duration_flat = cum_duration.view(b * t_x)
|
144 |
+
path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
|
145 |
+
path = path.view(b, t_x, t_y)
|
146 |
+
path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
|
147 |
+
path = path.unsqueeze(1).transpose(2, 3) * mask
|
148 |
+
return path
|
149 |
+
|
150 |
+
|
151 |
+
def clip_grad_value_(parameters, clip_value, norm_type=2):
|
152 |
+
if isinstance(parameters, torch.Tensor):
|
153 |
+
parameters = [parameters]
|
154 |
+
parameters = list(filter(lambda p: p.grad is not None, parameters))
|
155 |
+
norm_type = float(norm_type)
|
156 |
+
if clip_value is not None:
|
157 |
+
clip_value = float(clip_value)
|
158 |
+
|
159 |
+
total_norm = 0
|
160 |
+
for p in parameters:
|
161 |
+
param_norm = p.grad.data.norm(norm_type)
|
162 |
+
total_norm += param_norm.item() ** norm_type
|
163 |
+
if clip_value is not None:
|
164 |
+
p.grad.data.clamp_(min=-clip_value, max=clip_value)
|
165 |
+
total_norm = total_norm ** (1.0 / norm_type)
|
166 |
+
return total_norm
|
config.py
ADDED
@@ -0,0 +1,243 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"""
|
2 |
+
@Desc: 全局配置文件读取
|
3 |
+
"""
|
4 |
+
import argparse
|
5 |
+
import yaml
|
6 |
+
from typing import Dict, List
|
7 |
+
import os
|
8 |
+
import shutil
|
9 |
+
import sys
|
10 |
+
|
11 |
+
|
12 |
+
class Resample_config:
|
13 |
+
"""重采样配置"""
|
14 |
+
|
15 |
+
def __init__(self, in_dir: str, out_dir: str, sampling_rate: int = 44100):
|
16 |
+
self.sampling_rate: int = sampling_rate # 目标采样率
|
17 |
+
self.in_dir: str = in_dir # 待处理音频目录路径
|
18 |
+
self.out_dir: str = out_dir # 重采样输出路径
|
19 |
+
|
20 |
+
@classmethod
|
21 |
+
def from_dict(cls, dataset_path: str, data: Dict[str, any]):
|
22 |
+
"""从字典中生成实例"""
|
23 |
+
|
24 |
+
# 不检查路径是否有效,此逻辑在resample.py中处理
|
25 |
+
data["in_dir"] = os.path.join(dataset_path, data["in_dir"])
|
26 |
+
data["out_dir"] = os.path.join(dataset_path, data["out_dir"])
|
27 |
+
|
28 |
+
return cls(**data)
|
29 |
+
|
30 |
+
|
31 |
+
class Preprocess_text_config:
|
32 |
+
"""数据预处理配置"""
|
33 |
+
|
34 |
+
def __init__(
|
35 |
+
self,
|
36 |
+
transcription_path: str,
|
37 |
+
cleaned_path: str,
|
38 |
+
train_path: str,
|
39 |
+
val_path: str,
|
40 |
+
config_path: str,
|
41 |
+
val_per_spk: int = 5,
|
42 |
+
max_val_total: int = 10000,
|
43 |
+
clean: bool = True,
|
44 |
+
):
|
45 |
+
self.transcription_path: str = transcription_path # 原始文本文件路径,文本格式应为{wav_path}|{speaker_name}|{language}|{text}。
|
46 |
+
self.cleaned_path: str = cleaned_path # 数据清洗后文本路径,可以不填。不填则将在原始文本目录生成
|
47 |
+
self.train_path: str = train_path # 训练集路径,可以不填。不填则将在原始文本目录生成
|
48 |
+
self.val_path: str = val_path # 验证集路径,可以不填。不填则将在原始文本目录生成
|
49 |
+
self.config_path: str = config_path # 配置文件路径
|
50 |
+
self.val_per_spk: int = val_per_spk # 每个speaker的验证集条数
|
51 |
+
self.max_val_total: int = max_val_total # 验证集最大条数,多于的会被截断并放到训练集中
|
52 |
+
self.clean: bool = clean # 是否进行数据清洗
|
53 |
+
|
54 |
+
@classmethod
|
55 |
+
def from_dict(cls, dataset_path: str, data: Dict[str, any]):
|
56 |
+
"""从字典中生成实例"""
|
57 |
+
|
58 |
+
data["transcription_path"] = os.path.join(
|
59 |
+
dataset_path, data["transcription_path"]
|
60 |
+
)
|
61 |
+
if data["cleaned_path"] == "" or data["cleaned_path"] is None:
|
62 |
+
data["cleaned_path"] = None
|
63 |
+
else:
|
64 |
+
data["cleaned_path"] = os.path.join(dataset_path, data["cleaned_path"])
|
65 |
+
data["train_path"] = os.path.join(dataset_path, data["train_path"])
|
66 |
+
data["val_path"] = os.path.join(dataset_path, data["val_path"])
|
67 |
+
data["config_path"] = os.path.join(dataset_path, data["config_path"])
|
68 |
+
|
69 |
+
return cls(**data)
|
70 |
+
|
71 |
+
|
72 |
+
class Bert_gen_config:
|
73 |
+
"""bert_gen 配置"""
|
74 |
+
|
75 |
+
def __init__(
|
76 |
+
self,
|
77 |
+
config_path: str,
|
78 |
+
num_processes: int = 2,
|
79 |
+
device: str = "cuda",
|
80 |
+
use_multi_device: bool = False,
|
81 |
+
):
|
82 |
+
self.config_path = config_path
|
83 |
+
self.num_processes = num_processes
|
84 |
+
self.device = device
|
85 |
+
self.use_multi_device = use_multi_device
|
86 |
+
|
87 |
+
@classmethod
|
88 |
+
def from_dict(cls, dataset_path: str, data: Dict[str, any]):
|
89 |
+
data["config_path"] = os.path.join(dataset_path, data["config_path"])
|
90 |
+
|
91 |
+
return cls(**data)
|
92 |
+
|
93 |
+
|
94 |
+
class Emo_gen_config:
|
95 |
+
"""emo_gen 配置"""
|
96 |
+
|
97 |
+
def __init__(
|
98 |
+
self,
|
99 |
+
config_path: str,
|
100 |
+
num_processes: int = 2,
|
101 |
+
device: str = "cuda",
|
102 |
+
):
|
103 |
+
self.config_path = config_path
|
104 |
+
self.num_processes = num_processes
|
105 |
+
self.device = device
|
106 |
+
|
107 |
+
@classmethod
|
108 |
+
def from_dict(cls, dataset_path: str, data: Dict[str, any]):
|
109 |
+
data["config_path"] = os.path.join(dataset_path, data["config_path"])
|
110 |
+
|
111 |
+
return cls(**data)
|
112 |
+
|
113 |
+
|
114 |
+
class Train_ms_config:
|
115 |
+
"""训练配置"""
|
116 |
+
|
117 |
+
def __init__(
|
118 |
+
self,
|
119 |
+
config_path: str,
|
120 |
+
env: Dict[str, any],
|
121 |
+
base: Dict[str, any],
|
122 |
+
model: str,
|
123 |
+
num_workers: int,
|
124 |
+
spec_cache: bool,
|
125 |
+
keep_ckpts: int,
|
126 |
+
):
|
127 |
+
self.env = env # 需要加载的环境变量
|
128 |
+
self.base = base # 底模配置
|
129 |
+
self.model = model # 训练模型存储目录,该路径为相对于dataset_path的路径,而非项目根目录
|
130 |
+
self.config_path = config_path # 配置文件路径
|
131 |
+
self.num_workers = num_workers # worker数量
|
132 |
+
self.spec_cache = spec_cache # 是否启用spec缓存
|
133 |
+
self.keep_ckpts = keep_ckpts # ckpt数量
|
134 |
+
|
135 |
+
@classmethod
|
136 |
+
def from_dict(cls, dataset_path: str, data: Dict[str, any]):
|
137 |
+
# data["model"] = os.path.join(dataset_path, data["model"])
|
138 |
+
data["config_path"] = os.path.join(dataset_path, data["config_path"])
|
139 |
+
|
140 |
+
return cls(**data)
|
141 |
+
|
142 |
+
|
143 |
+
class Webui_config:
|
144 |
+
"""webui 配置"""
|
145 |
+
|
146 |
+
def __init__(
|
147 |
+
self,
|
148 |
+
device: str,
|
149 |
+
model: str,
|
150 |
+
config_path: str,
|
151 |
+
language_identification_library: str,
|
152 |
+
port: int = 7860,
|
153 |
+
share: bool = False,
|
154 |
+
debug: bool = False,
|
155 |
+
):
|
156 |
+
self.device: str = device
|
157 |
+
self.model: str = model # 端口号
|
158 |
+
self.config_path: str = config_path # 是否公开部署,对外网开放
|
159 |
+
self.port: int = port # 是否开启debug模式
|
160 |
+
self.share: bool = share # 模型路径
|
161 |
+
self.debug: bool = debug # 配置文件路径
|
162 |
+
self.language_identification_library: str = (
|
163 |
+
language_identification_library # 语种识别库
|
164 |
+
)
|
165 |
+
|
166 |
+
@classmethod
|
167 |
+
def from_dict(cls, dataset_path: str, data: Dict[str, any]):
|
168 |
+
data["config_path"] = os.path.join(dataset_path, data["config_path"])
|
169 |
+
data["model"] = os.path.join(dataset_path, data["model"])
|
170 |
+
return cls(**data)
|
171 |
+
|
172 |
+
|
173 |
+
class Server_config:
|
174 |
+
def __init__(
|
175 |
+
self, models: List[Dict[str, any]], port: int = 5000, device: str = "cuda"
|
176 |
+
):
|
177 |
+
self.models: List[Dict[str, any]] = models # 需要加载的所有模型的配置
|
178 |
+
self.port: int = port # 端口号
|
179 |
+
self.device: str = device # 模型默认使用设备
|
180 |
+
|
181 |
+
@classmethod
|
182 |
+
def from_dict(cls, data: Dict[str, any]):
|
183 |
+
return cls(**data)
|
184 |
+
|
185 |
+
|
186 |
+
class Translate_config:
|
187 |
+
"""翻译api配置"""
|
188 |
+
|
189 |
+
def __init__(self, app_key: str, secret_key: str):
|
190 |
+
self.app_key = app_key
|
191 |
+
self.secret_key = secret_key
|
192 |
+
|
193 |
+
@classmethod
|
194 |
+
def from_dict(cls, data: Dict[str, any]):
|
195 |
+
return cls(**data)
|
196 |
+
|
197 |
+
|
198 |
+
class Config:
|
199 |
+
def __init__(self, config_path: str):
|
200 |
+
if not os.path.isfile(config_path) and os.path.isfile("default_config.yml"):
|
201 |
+
shutil.copy(src="default_config.yml", dst=config_path)
|
202 |
+
print(
|
203 |
+
f"已根据默认配置文件default_config.yml生成配置文件{config_path}。请按该配置文件的说明进行配置后重新运行。"
|
204 |
+
)
|
205 |
+
print("如无特殊需求,请勿修改default_config.yml或备份该文件。")
|
206 |
+
sys.exit(0)
|
207 |
+
with open(file=config_path, mode="r", encoding="utf-8") as file:
|
208 |
+
yaml_config: Dict[str, any] = yaml.safe_load(file.read())
|
209 |
+
dataset_path: str = yaml_config["dataset_path"]
|
210 |
+
openi_token: str = yaml_config["openi_token"]
|
211 |
+
self.dataset_path: str = dataset_path
|
212 |
+
self.mirror: str = yaml_config["mirror"]
|
213 |
+
self.openi_token: str = openi_token
|
214 |
+
self.resample_config: Resample_config = Resample_config.from_dict(
|
215 |
+
dataset_path, yaml_config["resample"]
|
216 |
+
)
|
217 |
+
self.preprocess_text_config: Preprocess_text_config = (
|
218 |
+
Preprocess_text_config.from_dict(
|
219 |
+
dataset_path, yaml_config["preprocess_text"]
|
220 |
+
)
|
221 |
+
)
|
222 |
+
self.bert_gen_config: Bert_gen_config = Bert_gen_config.from_dict(
|
223 |
+
dataset_path, yaml_config["bert_gen"]
|
224 |
+
)
|
225 |
+
self.train_ms_config: Train_ms_config = Train_ms_config.from_dict(
|
226 |
+
dataset_path, yaml_config["train_ms"]
|
227 |
+
)
|
228 |
+
self.webui_config: Webui_config = Webui_config.from_dict(
|
229 |
+
dataset_path, yaml_config["webui"]
|
230 |
+
)
|
231 |
+
self.server_config: Server_config = Server_config.from_dict(
|
232 |
+
yaml_config["server"]
|
233 |
+
)
|
234 |
+
self.translate_config: Translate_config = Translate_config.from_dict(
|
235 |
+
yaml_config["translate"]
|
236 |
+
)
|
237 |
+
|
238 |
+
|
239 |
+
parser = argparse.ArgumentParser()
|
240 |
+
# 为避免与以前的config.json起冲突,将其更名如下
|
241 |
+
parser.add_argument("-y", "--yml_config", type=str, default="config.yml")
|
242 |
+
args, _ = parser.parse_known_args()
|
243 |
+
config = Config(args.yml_config)
|
config.yml
ADDED
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# 全局配置
|
2 |
+
# 对于希望在同一时间使用多个配置文件的情况,例如两个GPU同时跑两个训练集:通过环境变量指定配置文件,不指定则默认为./config.yml
|
3 |
+
|
4 |
+
# 拟提供通用路径配置,统一存放数据,避免数据放得很乱
|
5 |
+
# 每个数据集与其对应的模型存放至统一路径下,后续所有的路径配置均为相对于datasetPath的路径
|
6 |
+
# 不填或者填空则路径为相对于项目根目录的路径
|
7 |
+
dataset_path: "Data/"
|
8 |
+
|
9 |
+
# 模型镜像源,默认huggingface,使用openi镜像源需指定openi_token
|
10 |
+
mirror: ""
|
11 |
+
openi_token: "" # openi token
|
12 |
+
|
13 |
+
# resample 音频重采样配置
|
14 |
+
# 注意, “:” 后需要加空格
|
15 |
+
resample:
|
16 |
+
# 目标重采样率
|
17 |
+
sampling_rate: 44100
|
18 |
+
# 音频文件输入路径,重采样会将该路径下所有.wav音频文件重采样
|
19 |
+
# 请填入相对于datasetPath的相对路径
|
20 |
+
in_dir: "audios/raw" # 相对于根目录的路径为 /datasetPath/in_dir
|
21 |
+
# 音频文件重采样后输出路径
|
22 |
+
out_dir: "audios/wavs"
|
23 |
+
|
24 |
+
|
25 |
+
# preprocess_text 数据集预处理相关配置
|
26 |
+
# 注意, “:” 后需要加空格
|
27 |
+
preprocess_text:
|
28 |
+
# 原始文本文件路径,文本格式应为{wav_path}|{speaker_name}|{language}|{text}。
|
29 |
+
transcription_path: "filelists/haruka.list"
|
30 |
+
# 数据清洗后文本路径,可以不填。不填则将在原始文本目录生成
|
31 |
+
cleaned_path: ""
|
32 |
+
# 训练集路径
|
33 |
+
train_path: "filelists/train.list"
|
34 |
+
# 验证集路径
|
35 |
+
val_path: "filelists/val.list"
|
36 |
+
# 配置文件路径
|
37 |
+
config_path: "haruka.json"
|
38 |
+
# 每个speaker的验证集条数
|
39 |
+
val_per_spk: 40
|
40 |
+
# 验证集最大条数,多于的会被截断并放到训练集中
|
41 |
+
max_val_total: 200
|
42 |
+
# 是否进行数据清洗
|
43 |
+
clean: true
|
44 |
+
|
45 |
+
|
46 |
+
# bert_gen 相关配置
|
47 |
+
# 注意, “:” 后需要加空格
|
48 |
+
bert_gen:
|
49 |
+
# 训练数据集配置文件路径
|
50 |
+
config_path: "haruka.json"
|
51 |
+
# 并行数
|
52 |
+
num_processes: 2
|
53 |
+
# 使用设备:可选项 "cuda" 显卡推理,"cpu" cpu推理
|
54 |
+
# 该选项同时决定了get_bert_feature的默认设备
|
55 |
+
device: "cuda"
|
56 |
+
# 使用多卡推理
|
57 |
+
use_multi_device: false
|
58 |
+
|
59 |
+
# emo_gen 相关配置
|
60 |
+
# 注意, “:” 后需要加空格
|
61 |
+
emo_gen:
|
62 |
+
# 训练数据集配置文件路径
|
63 |
+
config_path: "haruka.json"
|
64 |
+
# 并行数
|
65 |
+
num_processes: 2
|
66 |
+
# 使用设备:可选项 "cuda" 显卡推理,"cpu" cpu推理
|
67 |
+
device: "cuda"
|
68 |
+
|
69 |
+
# train 训练配置
|
70 |
+
# 注意, “:” 后需要加空格
|
71 |
+
train_ms:
|
72 |
+
env:
|
73 |
+
MASTER_ADDR: "localhost"
|
74 |
+
MASTER_PORT: 10086
|
75 |
+
WORLD_SIZE: 1
|
76 |
+
LOCAL_RANK: 0
|
77 |
+
RANK: 0
|
78 |
+
# 可以填写任意名的环境变量
|
79 |
+
# THE_ENV_VAR_YOU_NEED_TO_USE: "1234567"
|
80 |
+
# 底模设置
|
81 |
+
base:
|
82 |
+
use_base_model: false
|
83 |
+
repo_id: "Stardust_minus/Bert-VITS2"
|
84 |
+
model_image: "Bert-VITS2_2.1-Emo底模" # openi网页的模型名
|
85 |
+
# 训练模型存储目录:与旧版本的区别,原先数据集是存放在logs/model_name下的,现在改为统一存放在Data/你的数据集/models下
|
86 |
+
model: "models"
|
87 |
+
# 配置文件路径
|
88 |
+
config_path: "haruka.json"
|
89 |
+
# 训练使用的worker,不建议超过CPU核心数
|
90 |
+
num_workers: 16
|
91 |
+
# 关闭此项可以节约接近50%的磁盘空间,但是可能导致实际训练速度变慢和更高的CPU使用率。
|
92 |
+
spec_cache: True
|
93 |
+
# 保存的检查点数量,多于此数目的权重会被删除来节省空间。
|
94 |
+
keep_ckpts: 8
|
95 |
+
|
96 |
+
|
97 |
+
# webui webui配置
|
98 |
+
# 注意, “:” 后需要加空格
|
99 |
+
webui:
|
100 |
+
# 推理设备
|
101 |
+
device: "cuda"
|
102 |
+
# 模型路径
|
103 |
+
model: "models/G_82000.pth"
|
104 |
+
# 配置文件路径
|
105 |
+
config_path: "configs/haruka.json"
|
106 |
+
# 端口号
|
107 |
+
port: 7860
|
108 |
+
# 是否公开部署,对外网开放
|
109 |
+
share: false
|
110 |
+
# 是否开启debug模式
|
111 |
+
debug: false
|
112 |
+
# 语种识别库,可选langid, fastlid
|
113 |
+
language_identification_library: "langid"
|
114 |
+
|
115 |
+
|
116 |
+
# server api配置
|
117 |
+
# 注意, “:” 后需要加空格
|
118 |
+
# 注意,本配置下的所有配置均为相对于根目录的路径
|
119 |
+
server:
|
120 |
+
# 端口号
|
121 |
+
port: 5000
|
122 |
+
# 模型默认使用设备:但是当前并没有实现这个配置。
|
123 |
+
device: "cuda"
|
124 |
+
# 需要加载的所有模型的配置
|
125 |
+
# 注意,所有模型都必须正确配置model与config的路径,空路径会导致加载错误。
|
126 |
+
models:
|
127 |
+
- # 模型的路径
|
128 |
+
model: ""
|
129 |
+
# 模型config.json的路径
|
130 |
+
config: ""
|
131 |
+
# 模型使用设备,若填写则会覆盖默认配置
|
132 |
+
device: "cuda"
|
133 |
+
# 模型默认使用的语言
|
134 |
+
language: "ZH"
|
135 |
+
# 模型人物默认参数
|
136 |
+
# 不必填写所有人物,不填的使用默认值
|
137 |
+
# 暂时不用填写,当前尚未实现按人区分配置
|
138 |
+
speakers:
|
139 |
+
- speaker: "科比"
|
140 |
+
sdp_ratio: 0.2
|
141 |
+
noise_scale: 0.6
|
142 |
+
noise_scale_w: 0.8
|
143 |
+
length_scale: 1
|
144 |
+
- speaker: "五条悟"
|
145 |
+
sdp_ratio: 0.3
|
146 |
+
noise_scale: 0.7
|
147 |
+
noise_scale_w: 0.8
|
148 |
+
length_scale: 0.5
|
149 |
+
- speaker: "安倍晋三"
|
150 |
+
sdp_ratio: 0.2
|
151 |
+
noise_scale: 0.6
|
152 |
+
noise_scale_w: 0.8
|
153 |
+
length_scale: 1.2
|
154 |
+
- # 模型的路径
|
155 |
+
model: ""
|
156 |
+
# 模型config.json的路径
|
157 |
+
config: ""
|
158 |
+
# 模型使用设备,若填写则会覆盖默认配置
|
159 |
+
device: "cpu"
|
160 |
+
# 模型默认使用的语言
|
161 |
+
language: "JP"
|
162 |
+
# 模型人物默认参数
|
163 |
+
# 不必填写所有人物,不填的使用默认值
|
164 |
+
speakers: [ ] # 也可以不填
|
165 |
+
|
166 |
+
|
167 |
+
# 百度翻译开放平台 api配置
|
168 |
+
# api接入文档 https://api.fanyi.baidu.com/doc/21
|
169 |
+
# 请不要在github等网站公开分享你的app id 与 key
|
170 |
+
translate:
|
171 |
+
# 你的APPID
|
172 |
+
"app_key": ""
|
173 |
+
# 你的密钥
|
174 |
+
"secret_key": ""
|
emo_gen.py
ADDED
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import argparse
|
2 |
+
import os
|
3 |
+
from pathlib import Path
|
4 |
+
|
5 |
+
import librosa
|
6 |
+
import numpy as np
|
7 |
+
import torch
|
8 |
+
import torch.nn as nn
|
9 |
+
from torch.utils.data import DataLoader, Dataset
|
10 |
+
from tqdm import tqdm
|
11 |
+
from transformers import Wav2Vec2Processor
|
12 |
+
from transformers.models.wav2vec2.modeling_wav2vec2 import (
|
13 |
+
Wav2Vec2Model,
|
14 |
+
Wav2Vec2PreTrainedModel,
|
15 |
+
)
|
16 |
+
|
17 |
+
import utils
|
18 |
+
from config import config
|
19 |
+
|
20 |
+
|
21 |
+
class RegressionHead(nn.Module):
|
22 |
+
r"""Classification head."""
|
23 |
+
|
24 |
+
def __init__(self, config):
|
25 |
+
super().__init__()
|
26 |
+
|
27 |
+
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
|
28 |
+
self.dropout = nn.Dropout(config.final_dropout)
|
29 |
+
self.out_proj = nn.Linear(config.hidden_size, config.num_labels)
|
30 |
+
|
31 |
+
def forward(self, features, **kwargs):
|
32 |
+
x = features
|
33 |
+
x = self.dropout(x)
|
34 |
+
x = self.dense(x)
|
35 |
+
x = torch.tanh(x)
|
36 |
+
x = self.dropout(x)
|
37 |
+
x = self.out_proj(x)
|
38 |
+
|
39 |
+
return x
|
40 |
+
|
41 |
+
|
42 |
+
class EmotionModel(Wav2Vec2PreTrainedModel):
|
43 |
+
r"""Speech emotion classifier."""
|
44 |
+
|
45 |
+
def __init__(self, config):
|
46 |
+
super().__init__(config)
|
47 |
+
|
48 |
+
self.config = config
|
49 |
+
self.wav2vec2 = Wav2Vec2Model(config)
|
50 |
+
self.classifier = RegressionHead(config)
|
51 |
+
self.init_weights()
|
52 |
+
|
53 |
+
def forward(
|
54 |
+
self,
|
55 |
+
input_values,
|
56 |
+
):
|
57 |
+
outputs = self.wav2vec2(input_values)
|
58 |
+
hidden_states = outputs[0]
|
59 |
+
hidden_states = torch.mean(hidden_states, dim=1)
|
60 |
+
logits = self.classifier(hidden_states)
|
61 |
+
|
62 |
+
return hidden_states, logits
|
63 |
+
|
64 |
+
|
65 |
+
class AudioDataset(Dataset):
|
66 |
+
def __init__(self, list_of_wav_files, sr, processor):
|
67 |
+
self.list_of_wav_files = list_of_wav_files
|
68 |
+
self.processor = processor
|
69 |
+
self.sr = sr
|
70 |
+
|
71 |
+
def __len__(self):
|
72 |
+
return len(self.list_of_wav_files)
|
73 |
+
|
74 |
+
def __getitem__(self, idx):
|
75 |
+
wav_file = self.list_of_wav_files[idx]
|
76 |
+
audio_data, _ = librosa.load(wav_file, sr=self.sr)
|
77 |
+
processed_data = self.processor(audio_data, sampling_rate=self.sr)[
|
78 |
+
"input_values"
|
79 |
+
][0]
|
80 |
+
return torch.from_numpy(processed_data)
|
81 |
+
|
82 |
+
|
83 |
+
def process_func(
|
84 |
+
x: np.ndarray,
|
85 |
+
sampling_rate: int,
|
86 |
+
model: EmotionModel,
|
87 |
+
processor: Wav2Vec2Processor,
|
88 |
+
device: str,
|
89 |
+
embeddings: bool = False,
|
90 |
+
) -> np.ndarray:
|
91 |
+
r"""Predict emotions or extract embeddings from raw audio signal."""
|
92 |
+
model = model.to(device)
|
93 |
+
y = processor(x, sampling_rate=sampling_rate)
|
94 |
+
y = y["input_values"][0]
|
95 |
+
y = torch.from_numpy(y).unsqueeze(0).to(device)
|
96 |
+
|
97 |
+
# run through model
|
98 |
+
with torch.no_grad():
|
99 |
+
y = model(y)[0 if embeddings else 1]
|
100 |
+
|
101 |
+
# convert to numpy
|
102 |
+
y = y.detach().cpu().numpy()
|
103 |
+
|
104 |
+
return y
|
105 |
+
|
106 |
+
|
107 |
+
def get_emo(path):
|
108 |
+
wav, sr = librosa.load(path, 16000)
|
109 |
+
device = config.bert_gen_config.device
|
110 |
+
return process_func(
|
111 |
+
np.expand_dims(wav, 0).astype(np.float64),
|
112 |
+
sr,
|
113 |
+
model,
|
114 |
+
processor,
|
115 |
+
device,
|
116 |
+
embeddings=True,
|
117 |
+
).squeeze(0)
|
118 |
+
|
119 |
+
|
120 |
+
if __name__ == "__main__":
|
121 |
+
parser = argparse.ArgumentParser()
|
122 |
+
parser.add_argument(
|
123 |
+
"-c", "--config", type=str, default=config.bert_gen_config.config_path
|
124 |
+
)
|
125 |
+
parser.add_argument(
|
126 |
+
"--num_processes", type=int, default=config.bert_gen_config.num_processes
|
127 |
+
)
|
128 |
+
args, _ = parser.parse_known_args()
|
129 |
+
config_path = args.config
|
130 |
+
hps = utils.get_hparams_from_file(config_path)
|
131 |
+
|
132 |
+
device = config.bert_gen_config.device
|
133 |
+
|
134 |
+
model_name = "./emotional/wav2vec2-large-robust-12-ft-emotion-msp-dim"
|
135 |
+
REPO_ID = "audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim"
|
136 |
+
if not Path(model_name).joinpath("pytorch_model.bin").exists():
|
137 |
+
utils.download_emo_models(config.mirror, model_name, REPO_ID)
|
138 |
+
|
139 |
+
processor = Wav2Vec2Processor.from_pretrained(model_name)
|
140 |
+
model = EmotionModel.from_pretrained(model_name).to(device)
|
141 |
+
|
142 |
+
lines = []
|
143 |
+
with open(hps.data.training_files, encoding="utf-8") as f:
|
144 |
+
lines.extend(f.readlines())
|
145 |
+
|
146 |
+
with open(hps.data.validation_files, encoding="utf-8") as f:
|
147 |
+
lines.extend(f.readlines())
|
148 |
+
|
149 |
+
wavnames = [line.split("|")[0] for line in lines]
|
150 |
+
dataset = AudioDataset(wavnames, 16000, processor)
|
151 |
+
data_loader = DataLoader(dataset, batch_size=1, shuffle=False, num_workers=16)
|
152 |
+
|
153 |
+
with torch.no_grad():
|
154 |
+
for i, data in tqdm(enumerate(data_loader), total=len(data_loader)):
|
155 |
+
wavname = wavnames[i]
|
156 |
+
emo_path = wavname.replace(".wav", ".emo.npy")
|
157 |
+
if os.path.exists(emo_path):
|
158 |
+
continue
|
159 |
+
emb = model(data.to(device))[0].detach().cpu().numpy()
|
160 |
+
np.save(emo_path, emb)
|
161 |
+
|
162 |
+
print("Emo vec 生成完毕!")
|
infer.py
ADDED
@@ -0,0 +1,261 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"""
|
2 |
+
版本管理、兼容推理及模型加载实现。
|
3 |
+
版本说明:
|
4 |
+
1. 版本号与github的release版本号对应,使用哪个release版本训练的模型即对应其版本号
|
5 |
+
2. 请在模型的config.json中显示声明版本号,添加一个字段"version" : "你的版本号"
|
6 |
+
特殊版本说明:
|
7 |
+
1.1.1-fix: 1.1.1版本训练的模型,但是在推理时使用dev的日语修复
|
8 |
+
1.1.1-dev: dev开发
|
9 |
+
2.1:当前版本
|
10 |
+
"""
|
11 |
+
import torch
|
12 |
+
import commons
|
13 |
+
from text import cleaned_text_to_sequence, get_bert
|
14 |
+
from emo_gen import get_emo
|
15 |
+
from text.cleaner import clean_text
|
16 |
+
import utils
|
17 |
+
|
18 |
+
from models import SynthesizerTrn
|
19 |
+
from text.symbols import symbols
|
20 |
+
|
21 |
+
# 当前版本信息
|
22 |
+
latest_version = "2.1"
|
23 |
+
|
24 |
+
|
25 |
+
|
26 |
+
def get_net_g(model_path: str, version: str, device: str, hps):
|
27 |
+
if version != latest_version:
|
28 |
+
pass
|
29 |
+
else:
|
30 |
+
# 当前版本模型 net_g
|
31 |
+
net_g = SynthesizerTrn(
|
32 |
+
len(symbols),
|
33 |
+
hps.data.filter_length // 2 + 1,
|
34 |
+
hps.train.segment_size // hps.data.hop_length,
|
35 |
+
n_speakers=hps.data.n_speakers,
|
36 |
+
**hps.model,
|
37 |
+
).to(device)
|
38 |
+
_ = net_g.eval()
|
39 |
+
_ = utils.load_checkpoint(model_path, net_g, None, skip_optimizer=True)
|
40 |
+
return net_g
|
41 |
+
|
42 |
+
|
43 |
+
def get_text(text, reference_audio, emotion, language_str, hps, device):
|
44 |
+
# 在此处实现当前版本的get_text
|
45 |
+
norm_text, phone, tone, word2ph = clean_text(text, language_str)
|
46 |
+
phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)
|
47 |
+
|
48 |
+
if hps.data.add_blank:
|
49 |
+
phone = commons.intersperse(phone, 0)
|
50 |
+
tone = commons.intersperse(tone, 0)
|
51 |
+
language = commons.intersperse(language, 0)
|
52 |
+
for i in range(len(word2ph)):
|
53 |
+
word2ph[i] = word2ph[i] * 2
|
54 |
+
word2ph[0] += 1
|
55 |
+
bert_ori = get_bert(norm_text, word2ph, language_str, device)
|
56 |
+
del word2ph
|
57 |
+
assert bert_ori.shape[-1] == len(phone), phone
|
58 |
+
|
59 |
+
if language_str == "ZH":
|
60 |
+
bert = bert_ori
|
61 |
+
ja_bert = torch.zeros(1024, len(phone))
|
62 |
+
en_bert = torch.zeros(1024, len(phone))
|
63 |
+
elif language_str == "JP":
|
64 |
+
bert = torch.zeros(1024, len(phone))
|
65 |
+
ja_bert = bert_ori
|
66 |
+
en_bert = torch.zeros(1024, len(phone))
|
67 |
+
elif language_str == "EN":
|
68 |
+
bert = torch.zeros(1024, len(phone))
|
69 |
+
ja_bert = torch.zeros(1024, len(phone))
|
70 |
+
en_bert = bert_ori
|
71 |
+
else:
|
72 |
+
raise ValueError("language_str should be ZH, JP or EN")
|
73 |
+
|
74 |
+
emo = (
|
75 |
+
torch.from_numpy(get_emo(reference_audio))
|
76 |
+
if reference_audio
|
77 |
+
else torch.Tensor([emotion])
|
78 |
+
)
|
79 |
+
|
80 |
+
assert bert.shape[-1] == len(
|
81 |
+
phone
|
82 |
+
), f"Bert seq len {bert.shape[-1]} != {len(phone)}"
|
83 |
+
|
84 |
+
phone = torch.LongTensor(phone)
|
85 |
+
tone = torch.LongTensor(tone)
|
86 |
+
language = torch.LongTensor(language)
|
87 |
+
return bert, ja_bert, en_bert, emo, phone, tone, language
|
88 |
+
|
89 |
+
|
90 |
+
def infer(
|
91 |
+
text,
|
92 |
+
sdp_ratio,
|
93 |
+
noise_scale,
|
94 |
+
noise_scale_w,
|
95 |
+
length_scale,
|
96 |
+
sid,
|
97 |
+
language,
|
98 |
+
hps,
|
99 |
+
net_g,
|
100 |
+
device,
|
101 |
+
reference_audio=None,
|
102 |
+
emotion=None,
|
103 |
+
skip_start=False,
|
104 |
+
skip_end=False,
|
105 |
+
):
|
106 |
+
version = hps.version if hasattr(hps, "version") else latest_version
|
107 |
+
# 非当前版本,根据版本号选择合适的infer
|
108 |
+
if version != latest_version:
|
109 |
+
pass
|
110 |
+
# 在此处实现当前版本的推理
|
111 |
+
bert, ja_bert, en_bert, emo, phones, tones, lang_ids = get_text(
|
112 |
+
text, reference_audio, emotion, language, hps, device
|
113 |
+
)
|
114 |
+
if skip_start:
|
115 |
+
phones = phones[1:]
|
116 |
+
tones = tones[1:]
|
117 |
+
lang_ids = lang_ids[1:]
|
118 |
+
bert = bert[:, 1:]
|
119 |
+
ja_bert = ja_bert[:, 1:]
|
120 |
+
en_bert = en_bert[:, 1:]
|
121 |
+
if skip_end:
|
122 |
+
phones = phones[:-1]
|
123 |
+
tones = tones[:-1]
|
124 |
+
lang_ids = lang_ids[:-1]
|
125 |
+
bert = bert[:, :-1]
|
126 |
+
ja_bert = ja_bert[:, :-1]
|
127 |
+
en_bert = en_bert[:, :-1]
|
128 |
+
with torch.no_grad():
|
129 |
+
x_tst = phones.to(device).unsqueeze(0)
|
130 |
+
tones = tones.to(device).unsqueeze(0)
|
131 |
+
lang_ids = lang_ids.to(device).unsqueeze(0)
|
132 |
+
bert = bert.to(device).unsqueeze(0)
|
133 |
+
ja_bert = ja_bert.to(device).unsqueeze(0)
|
134 |
+
en_bert = en_bert.to(device).unsqueeze(0)
|
135 |
+
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
|
136 |
+
emo = emo.to(device).unsqueeze(0)
|
137 |
+
del phones
|
138 |
+
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
|
139 |
+
audio = (
|
140 |
+
net_g.infer(
|
141 |
+
x_tst,
|
142 |
+
x_tst_lengths,
|
143 |
+
speakers,
|
144 |
+
tones,
|
145 |
+
lang_ids,
|
146 |
+
bert,
|
147 |
+
ja_bert,
|
148 |
+
en_bert,
|
149 |
+
emo,
|
150 |
+
sdp_ratio=sdp_ratio,
|
151 |
+
noise_scale=noise_scale,
|
152 |
+
noise_scale_w=noise_scale_w,
|
153 |
+
length_scale=length_scale,
|
154 |
+
)[0][0, 0]
|
155 |
+
.data.cpu()
|
156 |
+
.float()
|
157 |
+
.numpy()
|
158 |
+
)
|
159 |
+
del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
|
160 |
+
if torch.cuda.is_available():
|
161 |
+
torch.cuda.empty_cache()
|
162 |
+
return audio
|
163 |
+
|
164 |
+
|
165 |
+
def infer_multilang(
|
166 |
+
text,
|
167 |
+
sdp_ratio,
|
168 |
+
noise_scale,
|
169 |
+
noise_scale_w,
|
170 |
+
length_scale,
|
171 |
+
sid,
|
172 |
+
language,
|
173 |
+
hps,
|
174 |
+
net_g,
|
175 |
+
device,
|
176 |
+
reference_audio=None,
|
177 |
+
emotion=None,
|
178 |
+
skip_start=False,
|
179 |
+
skip_end=False,
|
180 |
+
):
|
181 |
+
bert, ja_bert, en_bert, emo, phones, tones, lang_ids = [], [], [], [], [], [], []
|
182 |
+
# bert, ja_bert, en_bert, phones, tones, lang_ids = get_text(
|
183 |
+
# text, language, hps, device
|
184 |
+
# )
|
185 |
+
for idx, (txt, lang) in enumerate(zip(text, language)):
|
186 |
+
skip_start = (idx != 0) or (skip_start and idx == 0)
|
187 |
+
skip_end = (idx != len(text) - 1) or (skip_end and idx == len(text) - 1)
|
188 |
+
(
|
189 |
+
temp_bert,
|
190 |
+
temp_ja_bert,
|
191 |
+
temp_en_bert,
|
192 |
+
temp_emo,
|
193 |
+
temp_phones,
|
194 |
+
temp_tones,
|
195 |
+
temp_lang_ids,
|
196 |
+
) = get_text(txt, reference_audio, emotion, language, hps, device)
|
197 |
+
if skip_start:
|
198 |
+
temp_bert = temp_bert[:, 1:]
|
199 |
+
temp_ja_bert = temp_ja_bert[:, 1:]
|
200 |
+
temp_en_bert = temp_en_bert[:, 1:]
|
201 |
+
temp_emo = temp_emo[:, 1:]
|
202 |
+
temp_phones = temp_phones[1:]
|
203 |
+
temp_tones = temp_tones[1:]
|
204 |
+
temp_lang_ids = temp_lang_ids[1:]
|
205 |
+
if skip_end:
|
206 |
+
temp_bert = temp_bert[:, :-1]
|
207 |
+
temp_ja_bert = temp_ja_bert[:, :-1]
|
208 |
+
temp_en_bert = temp_en_bert[:, :-1]
|
209 |
+
temp_emo = temp_emo[:, :-1]
|
210 |
+
temp_phones = temp_phones[:-1]
|
211 |
+
temp_tones = temp_tones[:-1]
|
212 |
+
temp_lang_ids = temp_lang_ids[:-1]
|
213 |
+
bert.append(temp_bert)
|
214 |
+
ja_bert.append(temp_ja_bert)
|
215 |
+
en_bert.append(temp_en_bert)
|
216 |
+
emo.append(temp_emo)
|
217 |
+
phones.append(temp_phones)
|
218 |
+
tones.append(temp_tones)
|
219 |
+
lang_ids.append(temp_lang_ids)
|
220 |
+
bert = torch.concatenate(bert, dim=1)
|
221 |
+
ja_bert = torch.concatenate(ja_bert, dim=1)
|
222 |
+
en_bert = torch.concatenate(en_bert, dim=1)
|
223 |
+
emo = torch.concatenate(emo, dim=1)
|
224 |
+
phones = torch.concatenate(phones, dim=0)
|
225 |
+
tones = torch.concatenate(tones, dim=0)
|
226 |
+
lang_ids = torch.concatenate(lang_ids, dim=0)
|
227 |
+
with torch.no_grad():
|
228 |
+
x_tst = phones.to(device).unsqueeze(0)
|
229 |
+
tones = tones.to(device).unsqueeze(0)
|
230 |
+
lang_ids = lang_ids.to(device).unsqueeze(0)
|
231 |
+
bert = bert.to(device).unsqueeze(0)
|
232 |
+
ja_bert = ja_bert.to(device).unsqueeze(0)
|
233 |
+
en_bert = en_bert.to(device).unsqueeze(0)
|
234 |
+
emo = emo.to(device).unsqueeze(0)
|
235 |
+
x_tst_lengths = torch.LongTensor([phones.size(0)]).to(device)
|
236 |
+
del phones
|
237 |
+
speakers = torch.LongTensor([hps.data.spk2id[sid]]).to(device)
|
238 |
+
audio = (
|
239 |
+
net_g.infer(
|
240 |
+
x_tst,
|
241 |
+
x_tst_lengths,
|
242 |
+
speakers,
|
243 |
+
tones,
|
244 |
+
lang_ids,
|
245 |
+
bert,
|
246 |
+
ja_bert,
|
247 |
+
en_bert,
|
248 |
+
emo,
|
249 |
+
sdp_ratio=sdp_ratio,
|
250 |
+
noise_scale=noise_scale,
|
251 |
+
noise_scale_w=noise_scale_w,
|
252 |
+
length_scale=length_scale,
|
253 |
+
)[0][0, 0]
|
254 |
+
.data.cpu()
|
255 |
+
.float()
|
256 |
+
.numpy()
|
257 |
+
)
|
258 |
+
del x_tst, tones, lang_ids, bert, x_tst_lengths, speakers, ja_bert, en_bert, emo
|
259 |
+
if torch.cuda.is_available():
|
260 |
+
torch.cuda.empty_cache()
|
261 |
+
return audio
|
models.py
ADDED
@@ -0,0 +1,1044 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import math
|
2 |
+
import torch
|
3 |
+
from torch import nn
|
4 |
+
from torch.nn import functional as F
|
5 |
+
|
6 |
+
import commons
|
7 |
+
import modules
|
8 |
+
import attentions
|
9 |
+
import monotonic_align
|
10 |
+
|
11 |
+
from torch.nn import Conv1d, ConvTranspose1d, Conv2d
|
12 |
+
from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
|
13 |
+
from vector_quantize_pytorch import VectorQuantize
|
14 |
+
|
15 |
+
from commons import init_weights, get_padding
|
16 |
+
from text import symbols, num_tones, num_languages
|
17 |
+
|
18 |
+
|
19 |
+
class DurationDiscriminator(nn.Module): # vits2
|
20 |
+
def __init__(
|
21 |
+
self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0
|
22 |
+
):
|
23 |
+
super().__init__()
|
24 |
+
|
25 |
+
self.in_channels = in_channels
|
26 |
+
self.filter_channels = filter_channels
|
27 |
+
self.kernel_size = kernel_size
|
28 |
+
self.p_dropout = p_dropout
|
29 |
+
self.gin_channels = gin_channels
|
30 |
+
|
31 |
+
self.drop = nn.Dropout(p_dropout)
|
32 |
+
self.conv_1 = nn.Conv1d(
|
33 |
+
in_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
34 |
+
)
|
35 |
+
self.norm_1 = modules.LayerNorm(filter_channels)
|
36 |
+
self.conv_2 = nn.Conv1d(
|
37 |
+
filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
38 |
+
)
|
39 |
+
self.norm_2 = modules.LayerNorm(filter_channels)
|
40 |
+
self.dur_proj = nn.Conv1d(1, filter_channels, 1)
|
41 |
+
|
42 |
+
self.pre_out_conv_1 = nn.Conv1d(
|
43 |
+
2 * filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
44 |
+
)
|
45 |
+
self.pre_out_norm_1 = modules.LayerNorm(filter_channels)
|
46 |
+
self.pre_out_conv_2 = nn.Conv1d(
|
47 |
+
filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
48 |
+
)
|
49 |
+
self.pre_out_norm_2 = modules.LayerNorm(filter_channels)
|
50 |
+
|
51 |
+
if gin_channels != 0:
|
52 |
+
self.cond = nn.Conv1d(gin_channels, in_channels, 1)
|
53 |
+
|
54 |
+
self.output_layer = nn.Sequential(nn.Linear(filter_channels, 1), nn.Sigmoid())
|
55 |
+
|
56 |
+
def forward_probability(self, x, x_mask, dur, g=None):
|
57 |
+
dur = self.dur_proj(dur)
|
58 |
+
x = torch.cat([x, dur], dim=1)
|
59 |
+
x = self.pre_out_conv_1(x * x_mask)
|
60 |
+
x = torch.relu(x)
|
61 |
+
x = self.pre_out_norm_1(x)
|
62 |
+
x = self.drop(x)
|
63 |
+
x = self.pre_out_conv_2(x * x_mask)
|
64 |
+
x = torch.relu(x)
|
65 |
+
x = self.pre_out_norm_2(x)
|
66 |
+
x = self.drop(x)
|
67 |
+
x = x * x_mask
|
68 |
+
x = x.transpose(1, 2)
|
69 |
+
output_prob = self.output_layer(x)
|
70 |
+
return output_prob
|
71 |
+
|
72 |
+
def forward(self, x, x_mask, dur_r, dur_hat, g=None):
|
73 |
+
x = torch.detach(x)
|
74 |
+
if g is not None:
|
75 |
+
g = torch.detach(g)
|
76 |
+
x = x + self.cond(g)
|
77 |
+
x = self.conv_1(x * x_mask)
|
78 |
+
x = torch.relu(x)
|
79 |
+
x = self.norm_1(x)
|
80 |
+
x = self.drop(x)
|
81 |
+
x = self.conv_2(x * x_mask)
|
82 |
+
x = torch.relu(x)
|
83 |
+
x = self.norm_2(x)
|
84 |
+
x = self.drop(x)
|
85 |
+
|
86 |
+
output_probs = []
|
87 |
+
for dur in [dur_r, dur_hat]:
|
88 |
+
output_prob = self.forward_probability(x, x_mask, dur, g)
|
89 |
+
output_probs.append(output_prob)
|
90 |
+
|
91 |
+
return output_probs
|
92 |
+
|
93 |
+
|
94 |
+
class TransformerCouplingBlock(nn.Module):
|
95 |
+
def __init__(
|
96 |
+
self,
|
97 |
+
channels,
|
98 |
+
hidden_channels,
|
99 |
+
filter_channels,
|
100 |
+
n_heads,
|
101 |
+
n_layers,
|
102 |
+
kernel_size,
|
103 |
+
p_dropout,
|
104 |
+
n_flows=4,
|
105 |
+
gin_channels=0,
|
106 |
+
share_parameter=False,
|
107 |
+
):
|
108 |
+
super().__init__()
|
109 |
+
self.channels = channels
|
110 |
+
self.hidden_channels = hidden_channels
|
111 |
+
self.kernel_size = kernel_size
|
112 |
+
self.n_layers = n_layers
|
113 |
+
self.n_flows = n_flows
|
114 |
+
self.gin_channels = gin_channels
|
115 |
+
|
116 |
+
self.flows = nn.ModuleList()
|
117 |
+
|
118 |
+
self.wn = (
|
119 |
+
attentions.FFT(
|
120 |
+
hidden_channels,
|
121 |
+
filter_channels,
|
122 |
+
n_heads,
|
123 |
+
n_layers,
|
124 |
+
kernel_size,
|
125 |
+
p_dropout,
|
126 |
+
isflow=True,
|
127 |
+
gin_channels=self.gin_channels,
|
128 |
+
)
|
129 |
+
if share_parameter
|
130 |
+
else None
|
131 |
+
)
|
132 |
+
|
133 |
+
for i in range(n_flows):
|
134 |
+
self.flows.append(
|
135 |
+
modules.TransformerCouplingLayer(
|
136 |
+
channels,
|
137 |
+
hidden_channels,
|
138 |
+
kernel_size,
|
139 |
+
n_layers,
|
140 |
+
n_heads,
|
141 |
+
p_dropout,
|
142 |
+
filter_channels,
|
143 |
+
mean_only=True,
|
144 |
+
wn_sharing_parameter=self.wn,
|
145 |
+
gin_channels=self.gin_channels,
|
146 |
+
)
|
147 |
+
)
|
148 |
+
self.flows.append(modules.Flip())
|
149 |
+
|
150 |
+
def forward(self, x, x_mask, g=None, reverse=False):
|
151 |
+
if not reverse:
|
152 |
+
for flow in self.flows:
|
153 |
+
x, _ = flow(x, x_mask, g=g, reverse=reverse)
|
154 |
+
else:
|
155 |
+
for flow in reversed(self.flows):
|
156 |
+
x = flow(x, x_mask, g=g, reverse=reverse)
|
157 |
+
return x
|
158 |
+
|
159 |
+
|
160 |
+
class StochasticDurationPredictor(nn.Module):
|
161 |
+
def __init__(
|
162 |
+
self,
|
163 |
+
in_channels,
|
164 |
+
filter_channels,
|
165 |
+
kernel_size,
|
166 |
+
p_dropout,
|
167 |
+
n_flows=4,
|
168 |
+
gin_channels=0,
|
169 |
+
):
|
170 |
+
super().__init__()
|
171 |
+
filter_channels = in_channels # it needs to be removed from future version.
|
172 |
+
self.in_channels = in_channels
|
173 |
+
self.filter_channels = filter_channels
|
174 |
+
self.kernel_size = kernel_size
|
175 |
+
self.p_dropout = p_dropout
|
176 |
+
self.n_flows = n_flows
|
177 |
+
self.gin_channels = gin_channels
|
178 |
+
|
179 |
+
self.log_flow = modules.Log()
|
180 |
+
self.flows = nn.ModuleList()
|
181 |
+
self.flows.append(modules.ElementwiseAffine(2))
|
182 |
+
for i in range(n_flows):
|
183 |
+
self.flows.append(
|
184 |
+
modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3)
|
185 |
+
)
|
186 |
+
self.flows.append(modules.Flip())
|
187 |
+
|
188 |
+
self.post_pre = nn.Conv1d(1, filter_channels, 1)
|
189 |
+
self.post_proj = nn.Conv1d(filter_channels, filter_channels, 1)
|
190 |
+
self.post_convs = modules.DDSConv(
|
191 |
+
filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout
|
192 |
+
)
|
193 |
+
self.post_flows = nn.ModuleList()
|
194 |
+
self.post_flows.append(modules.ElementwiseAffine(2))
|
195 |
+
for i in range(4):
|
196 |
+
self.post_flows.append(
|
197 |
+
modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3)
|
198 |
+
)
|
199 |
+
self.post_flows.append(modules.Flip())
|
200 |
+
|
201 |
+
self.pre = nn.Conv1d(in_channels, filter_channels, 1)
|
202 |
+
self.proj = nn.Conv1d(filter_channels, filter_channels, 1)
|
203 |
+
self.convs = modules.DDSConv(
|
204 |
+
filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout
|
205 |
+
)
|
206 |
+
if gin_channels != 0:
|
207 |
+
self.cond = nn.Conv1d(gin_channels, filter_channels, 1)
|
208 |
+
|
209 |
+
def forward(self, x, x_mask, w=None, g=None, reverse=False, noise_scale=1.0):
|
210 |
+
x = torch.detach(x)
|
211 |
+
x = self.pre(x)
|
212 |
+
if g is not None:
|
213 |
+
g = torch.detach(g)
|
214 |
+
x = x + self.cond(g)
|
215 |
+
x = self.convs(x, x_mask)
|
216 |
+
x = self.proj(x) * x_mask
|
217 |
+
|
218 |
+
if not reverse:
|
219 |
+
flows = self.flows
|
220 |
+
assert w is not None
|
221 |
+
|
222 |
+
logdet_tot_q = 0
|
223 |
+
h_w = self.post_pre(w)
|
224 |
+
h_w = self.post_convs(h_w, x_mask)
|
225 |
+
h_w = self.post_proj(h_w) * x_mask
|
226 |
+
e_q = (
|
227 |
+
torch.randn(w.size(0), 2, w.size(2)).to(device=x.device, dtype=x.dtype)
|
228 |
+
* x_mask
|
229 |
+
)
|
230 |
+
z_q = e_q
|
231 |
+
for flow in self.post_flows:
|
232 |
+
z_q, logdet_q = flow(z_q, x_mask, g=(x + h_w))
|
233 |
+
logdet_tot_q += logdet_q
|
234 |
+
z_u, z1 = torch.split(z_q, [1, 1], 1)
|
235 |
+
u = torch.sigmoid(z_u) * x_mask
|
236 |
+
z0 = (w - u) * x_mask
|
237 |
+
logdet_tot_q += torch.sum(
|
238 |
+
(F.logsigmoid(z_u) + F.logsigmoid(-z_u)) * x_mask, [1, 2]
|
239 |
+
)
|
240 |
+
logq = (
|
241 |
+
torch.sum(-0.5 * (math.log(2 * math.pi) + (e_q**2)) * x_mask, [1, 2])
|
242 |
+
- logdet_tot_q
|
243 |
+
)
|
244 |
+
|
245 |
+
logdet_tot = 0
|
246 |
+
z0, logdet = self.log_flow(z0, x_mask)
|
247 |
+
logdet_tot += logdet
|
248 |
+
z = torch.cat([z0, z1], 1)
|
249 |
+
for flow in flows:
|
250 |
+
z, logdet = flow(z, x_mask, g=x, reverse=reverse)
|
251 |
+
logdet_tot = logdet_tot + logdet
|
252 |
+
nll = (
|
253 |
+
torch.sum(0.5 * (math.log(2 * math.pi) + (z**2)) * x_mask, [1, 2])
|
254 |
+
- logdet_tot
|
255 |
+
)
|
256 |
+
return nll + logq # [b]
|
257 |
+
else:
|
258 |
+
flows = list(reversed(self.flows))
|
259 |
+
flows = flows[:-2] + [flows[-1]] # remove a useless vflow
|
260 |
+
z = (
|
261 |
+
torch.randn(x.size(0), 2, x.size(2)).to(device=x.device, dtype=x.dtype)
|
262 |
+
* noise_scale
|
263 |
+
)
|
264 |
+
for flow in flows:
|
265 |
+
z = flow(z, x_mask, g=x, reverse=reverse)
|
266 |
+
z0, z1 = torch.split(z, [1, 1], 1)
|
267 |
+
logw = z0
|
268 |
+
return logw
|
269 |
+
|
270 |
+
|
271 |
+
class DurationPredictor(nn.Module):
|
272 |
+
def __init__(
|
273 |
+
self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0
|
274 |
+
):
|
275 |
+
super().__init__()
|
276 |
+
|
277 |
+
self.in_channels = in_channels
|
278 |
+
self.filter_channels = filter_channels
|
279 |
+
self.kernel_size = kernel_size
|
280 |
+
self.p_dropout = p_dropout
|
281 |
+
self.gin_channels = gin_channels
|
282 |
+
|
283 |
+
self.drop = nn.Dropout(p_dropout)
|
284 |
+
self.conv_1 = nn.Conv1d(
|
285 |
+
in_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
286 |
+
)
|
287 |
+
self.norm_1 = modules.LayerNorm(filter_channels)
|
288 |
+
self.conv_2 = nn.Conv1d(
|
289 |
+
filter_channels, filter_channels, kernel_size, padding=kernel_size // 2
|
290 |
+
)
|
291 |
+
self.norm_2 = modules.LayerNorm(filter_channels)
|
292 |
+
self.proj = nn.Conv1d(filter_channels, 1, 1)
|
293 |
+
|
294 |
+
if gin_channels != 0:
|
295 |
+
self.cond = nn.Conv1d(gin_channels, in_channels, 1)
|
296 |
+
|
297 |
+
def forward(self, x, x_mask, g=None):
|
298 |
+
x = torch.detach(x)
|
299 |
+
if g is not None:
|
300 |
+
g = torch.detach(g)
|
301 |
+
x = x + self.cond(g)
|
302 |
+
x = self.conv_1(x * x_mask)
|
303 |
+
x = torch.relu(x)
|
304 |
+
x = self.norm_1(x)
|
305 |
+
x = self.drop(x)
|
306 |
+
x = self.conv_2(x * x_mask)
|
307 |
+
x = torch.relu(x)
|
308 |
+
x = self.norm_2(x)
|
309 |
+
x = self.drop(x)
|
310 |
+
x = self.proj(x * x_mask)
|
311 |
+
return x * x_mask
|
312 |
+
|
313 |
+
|
314 |
+
class TextEncoder(nn.Module):
|
315 |
+
def __init__(
|
316 |
+
self,
|
317 |
+
n_vocab,
|
318 |
+
out_channels,
|
319 |
+
hidden_channels,
|
320 |
+
filter_channels,
|
321 |
+
n_heads,
|
322 |
+
n_layers,
|
323 |
+
kernel_size,
|
324 |
+
p_dropout,
|
325 |
+
n_speakers,
|
326 |
+
gin_channels=0,
|
327 |
+
):
|
328 |
+
super().__init__()
|
329 |
+
self.n_vocab = n_vocab
|
330 |
+
self.out_channels = out_channels
|
331 |
+
self.hidden_channels = hidden_channels
|
332 |
+
self.filter_channels = filter_channels
|
333 |
+
self.n_heads = n_heads
|
334 |
+
self.n_layers = n_layers
|
335 |
+
self.kernel_size = kernel_size
|
336 |
+
self.p_dropout = p_dropout
|
337 |
+
self.gin_channels = gin_channels
|
338 |
+
self.emb = nn.Embedding(len(symbols), hidden_channels)
|
339 |
+
nn.init.normal_(self.emb.weight, 0.0, hidden_channels**-0.5)
|
340 |
+
self.tone_emb = nn.Embedding(num_tones, hidden_channels)
|
341 |
+
nn.init.normal_(self.tone_emb.weight, 0.0, hidden_channels**-0.5)
|
342 |
+
self.language_emb = nn.Embedding(num_languages, hidden_channels)
|
343 |
+
nn.init.normal_(self.language_emb.weight, 0.0, hidden_channels**-0.5)
|
344 |
+
self.bert_proj = nn.Conv1d(1024, hidden_channels, 1)
|
345 |
+
self.ja_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
|
346 |
+
self.en_bert_proj = nn.Conv1d(1024, hidden_channels, 1)
|
347 |
+
self.emo_proj = nn.Linear(1024, 1024)
|
348 |
+
self.emo_quantizer = [
|
349 |
+
VectorQuantize(
|
350 |
+
dim=1024,
|
351 |
+
codebook_size=10,
|
352 |
+
decay=0.8,
|
353 |
+
commitment_weight=1.0,
|
354 |
+
learnable_codebook=True,
|
355 |
+
ema_update=False,
|
356 |
+
)
|
357 |
+
] * n_speakers
|
358 |
+
self.emo_q_proj = nn.Linear(1024, hidden_channels)
|
359 |
+
|
360 |
+
self.encoder = attentions.Encoder(
|
361 |
+
hidden_channels,
|
362 |
+
filter_channels,
|
363 |
+
n_heads,
|
364 |
+
n_layers,
|
365 |
+
kernel_size,
|
366 |
+
p_dropout,
|
367 |
+
gin_channels=self.gin_channels,
|
368 |
+
)
|
369 |
+
self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
|
370 |
+
|
371 |
+
def forward(
|
372 |
+
self, x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=None
|
373 |
+
):
|
374 |
+
sid = sid.cpu()
|
375 |
+
bert_emb = self.bert_proj(bert).transpose(1, 2)
|
376 |
+
ja_bert_emb = self.ja_bert_proj(ja_bert).transpose(1, 2)
|
377 |
+
en_bert_emb = self.en_bert_proj(en_bert).transpose(1, 2)
|
378 |
+
if emo.size(-1) == 1024:
|
379 |
+
emo_emb = self.emo_proj(emo.unsqueeze(1))
|
380 |
+
emo_commit_loss = torch.zeros(1)
|
381 |
+
emo_emb_ = []
|
382 |
+
for i in range(emo_emb.size(0)):
|
383 |
+
temp_emo_emb, _, temp_emo_commit_loss = self.emo_quantizer[sid[i]](
|
384 |
+
emo_emb[i].unsqueeze(0).cpu()
|
385 |
+
)
|
386 |
+
emo_commit_loss += temp_emo_commit_loss
|
387 |
+
emo_emb_.append(temp_emo_emb)
|
388 |
+
emo_emb = torch.cat(emo_emb_, dim=0).to(emo_emb.device)
|
389 |
+
emo_commit_loss = emo_commit_loss.to(emo_emb.device)
|
390 |
+
else:
|
391 |
+
emo_emb = (
|
392 |
+
self.emo_quantizer[sid[0]]
|
393 |
+
.get_output_from_indices(emo.to(torch.int).cpu())
|
394 |
+
.unsqueeze(0)
|
395 |
+
.to(emo.device)
|
396 |
+
)
|
397 |
+
emo_commit_loss = torch.zeros(1)
|
398 |
+
x = (
|
399 |
+
self.emb(x)
|
400 |
+
+ self.tone_emb(tone)
|
401 |
+
+ self.language_emb(language)
|
402 |
+
+ bert_emb
|
403 |
+
+ ja_bert_emb
|
404 |
+
+ en_bert_emb
|
405 |
+
+ self.emo_q_proj(emo_emb)
|
406 |
+
) * math.sqrt(
|
407 |
+
self.hidden_channels
|
408 |
+
) # [b, t, h]
|
409 |
+
x = torch.transpose(x, 1, -1) # [b, h, t]
|
410 |
+
x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(
|
411 |
+
x.dtype
|
412 |
+
)
|
413 |
+
|
414 |
+
x = self.encoder(x * x_mask, x_mask, g=g)
|
415 |
+
stats = self.proj(x) * x_mask
|
416 |
+
|
417 |
+
m, logs = torch.split(stats, self.out_channels, dim=1)
|
418 |
+
return x, m, logs, x_mask, emo_commit_loss
|
419 |
+
|
420 |
+
|
421 |
+
class ResidualCouplingBlock(nn.Module):
|
422 |
+
def __init__(
|
423 |
+
self,
|
424 |
+
channels,
|
425 |
+
hidden_channels,
|
426 |
+
kernel_size,
|
427 |
+
dilation_rate,
|
428 |
+
n_layers,
|
429 |
+
n_flows=4,
|
430 |
+
gin_channels=0,
|
431 |
+
):
|
432 |
+
super().__init__()
|
433 |
+
self.channels = channels
|
434 |
+
self.hidden_channels = hidden_channels
|
435 |
+
self.kernel_size = kernel_size
|
436 |
+
self.dilation_rate = dilation_rate
|
437 |
+
self.n_layers = n_layers
|
438 |
+
self.n_flows = n_flows
|
439 |
+
self.gin_channels = gin_channels
|
440 |
+
|
441 |
+
self.flows = nn.ModuleList()
|
442 |
+
for i in range(n_flows):
|
443 |
+
self.flows.append(
|
444 |
+
modules.ResidualCouplingLayer(
|
445 |
+
channels,
|
446 |
+
hidden_channels,
|
447 |
+
kernel_size,
|
448 |
+
dilation_rate,
|
449 |
+
n_layers,
|
450 |
+
gin_channels=gin_channels,
|
451 |
+
mean_only=True,
|
452 |
+
)
|
453 |
+
)
|
454 |
+
self.flows.append(modules.Flip())
|
455 |
+
|
456 |
+
def forward(self, x, x_mask, g=None, reverse=False):
|
457 |
+
if not reverse:
|
458 |
+
for flow in self.flows:
|
459 |
+
x, _ = flow(x, x_mask, g=g, reverse=reverse)
|
460 |
+
else:
|
461 |
+
for flow in reversed(self.flows):
|
462 |
+
x = flow(x, x_mask, g=g, reverse=reverse)
|
463 |
+
return x
|
464 |
+
|
465 |
+
|
466 |
+
class PosteriorEncoder(nn.Module):
|
467 |
+
def __init__(
|
468 |
+
self,
|
469 |
+
in_channels,
|
470 |
+
out_channels,
|
471 |
+
hidden_channels,
|
472 |
+
kernel_size,
|
473 |
+
dilation_rate,
|
474 |
+
n_layers,
|
475 |
+
gin_channels=0,
|
476 |
+
):
|
477 |
+
super().__init__()
|
478 |
+
self.in_channels = in_channels
|
479 |
+
self.out_channels = out_channels
|
480 |
+
self.hidden_channels = hidden_channels
|
481 |
+
self.kernel_size = kernel_size
|
482 |
+
self.dilation_rate = dilation_rate
|
483 |
+
self.n_layers = n_layers
|
484 |
+
self.gin_channels = gin_channels
|
485 |
+
|
486 |
+
self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
|
487 |
+
self.enc = modules.WN(
|
488 |
+
hidden_channels,
|
489 |
+
kernel_size,
|
490 |
+
dilation_rate,
|
491 |
+
n_layers,
|
492 |
+
gin_channels=gin_channels,
|
493 |
+
)
|
494 |
+
self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
|
495 |
+
|
496 |
+
def forward(self, x, x_lengths, g=None):
|
497 |
+
x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(
|
498 |
+
x.dtype
|
499 |
+
)
|
500 |
+
x = self.pre(x) * x_mask
|
501 |
+
x = self.enc(x, x_mask, g=g)
|
502 |
+
stats = self.proj(x) * x_mask
|
503 |
+
m, logs = torch.split(stats, self.out_channels, dim=1)
|
504 |
+
z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
|
505 |
+
return z, m, logs, x_mask
|
506 |
+
|
507 |
+
|
508 |
+
class Generator(torch.nn.Module):
|
509 |
+
def __init__(
|
510 |
+
self,
|
511 |
+
initial_channel,
|
512 |
+
resblock,
|
513 |
+
resblock_kernel_sizes,
|
514 |
+
resblock_dilation_sizes,
|
515 |
+
upsample_rates,
|
516 |
+
upsample_initial_channel,
|
517 |
+
upsample_kernel_sizes,
|
518 |
+
gin_channels=0,
|
519 |
+
):
|
520 |
+
super(Generator, self).__init__()
|
521 |
+
self.num_kernels = len(resblock_kernel_sizes)
|
522 |
+
self.num_upsamples = len(upsample_rates)
|
523 |
+
self.conv_pre = Conv1d(
|
524 |
+
initial_channel, upsample_initial_channel, 7, 1, padding=3
|
525 |
+
)
|
526 |
+
resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2
|
527 |
+
|
528 |
+
self.ups = nn.ModuleList()
|
529 |
+
for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
|
530 |
+
self.ups.append(
|
531 |
+
weight_norm(
|
532 |
+
ConvTranspose1d(
|
533 |
+
upsample_initial_channel // (2**i),
|
534 |
+
upsample_initial_channel // (2 ** (i + 1)),
|
535 |
+
k,
|
536 |
+
u,
|
537 |
+
padding=(k - u) // 2,
|
538 |
+
)
|
539 |
+
)
|
540 |
+
)
|
541 |
+
|
542 |
+
self.resblocks = nn.ModuleList()
|
543 |
+
for i in range(len(self.ups)):
|
544 |
+
ch = upsample_initial_channel // (2 ** (i + 1))
|
545 |
+
for j, (k, d) in enumerate(
|
546 |
+
zip(resblock_kernel_sizes, resblock_dilation_sizes)
|
547 |
+
):
|
548 |
+
self.resblocks.append(resblock(ch, k, d))
|
549 |
+
|
550 |
+
self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
|
551 |
+
self.ups.apply(init_weights)
|
552 |
+
|
553 |
+
if gin_channels != 0:
|
554 |
+
self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
|
555 |
+
|
556 |
+
def forward(self, x, g=None):
|
557 |
+
x = self.conv_pre(x)
|
558 |
+
if g is not None:
|
559 |
+
x = x + self.cond(g)
|
560 |
+
|
561 |
+
for i in range(self.num_upsamples):
|
562 |
+
x = F.leaky_relu(x, modules.LRELU_SLOPE)
|
563 |
+
x = self.ups[i](x)
|
564 |
+
xs = None
|
565 |
+
for j in range(self.num_kernels):
|
566 |
+
if xs is None:
|
567 |
+
xs = self.resblocks[i * self.num_kernels + j](x)
|
568 |
+
else:
|
569 |
+
xs += self.resblocks[i * self.num_kernels + j](x)
|
570 |
+
x = xs / self.num_kernels
|
571 |
+
x = F.leaky_relu(x)
|
572 |
+
x = self.conv_post(x)
|
573 |
+
x = torch.tanh(x)
|
574 |
+
|
575 |
+
return x
|
576 |
+
|
577 |
+
def remove_weight_norm(self):
|
578 |
+
print("Removing weight norm...")
|
579 |
+
for layer in self.ups:
|
580 |
+
remove_weight_norm(layer)
|
581 |
+
for layer in self.resblocks:
|
582 |
+
layer.remove_weight_norm()
|
583 |
+
|
584 |
+
|
585 |
+
class DiscriminatorP(torch.nn.Module):
|
586 |
+
def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
|
587 |
+
super(DiscriminatorP, self).__init__()
|
588 |
+
self.period = period
|
589 |
+
self.use_spectral_norm = use_spectral_norm
|
590 |
+
norm_f = weight_norm if use_spectral_norm is False else spectral_norm
|
591 |
+
self.convs = nn.ModuleList(
|
592 |
+
[
|
593 |
+
norm_f(
|
594 |
+
Conv2d(
|
595 |
+
1,
|
596 |
+
32,
|
597 |
+
(kernel_size, 1),
|
598 |
+
(stride, 1),
|
599 |
+
padding=(get_padding(kernel_size, 1), 0),
|
600 |
+
)
|
601 |
+
),
|
602 |
+
norm_f(
|
603 |
+
Conv2d(
|
604 |
+
32,
|
605 |
+
128,
|
606 |
+
(kernel_size, 1),
|
607 |
+
(stride, 1),
|
608 |
+
padding=(get_padding(kernel_size, 1), 0),
|
609 |
+
)
|
610 |
+
),
|
611 |
+
norm_f(
|
612 |
+
Conv2d(
|
613 |
+
128,
|
614 |
+
512,
|
615 |
+
(kernel_size, 1),
|
616 |
+
(stride, 1),
|
617 |
+
padding=(get_padding(kernel_size, 1), 0),
|
618 |
+
)
|
619 |
+
),
|
620 |
+
norm_f(
|
621 |
+
Conv2d(
|
622 |
+
512,
|
623 |
+
1024,
|
624 |
+
(kernel_size, 1),
|
625 |
+
(stride, 1),
|
626 |
+
padding=(get_padding(kernel_size, 1), 0),
|
627 |
+
)
|
628 |
+
),
|
629 |
+
norm_f(
|
630 |
+
Conv2d(
|
631 |
+
1024,
|
632 |
+
1024,
|
633 |
+
(kernel_size, 1),
|
634 |
+
1,
|
635 |
+
padding=(get_padding(kernel_size, 1), 0),
|
636 |
+
)
|
637 |
+
),
|
638 |
+
]
|
639 |
+
)
|
640 |
+
self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
|
641 |
+
|
642 |
+
def forward(self, x):
|
643 |
+
fmap = []
|
644 |
+
|
645 |
+
# 1d to 2d
|
646 |
+
b, c, t = x.shape
|
647 |
+
if t % self.period != 0: # pad first
|
648 |
+
n_pad = self.period - (t % self.period)
|
649 |
+
x = F.pad(x, (0, n_pad), "reflect")
|
650 |
+
t = t + n_pad
|
651 |
+
x = x.view(b, c, t // self.period, self.period)
|
652 |
+
|
653 |
+
for layer in self.convs:
|
654 |
+
x = layer(x)
|
655 |
+
x = F.leaky_relu(x, modules.LRELU_SLOPE)
|
656 |
+
fmap.append(x)
|
657 |
+
x = self.conv_post(x)
|
658 |
+
fmap.append(x)
|
659 |
+
x = torch.flatten(x, 1, -1)
|
660 |
+
|
661 |
+
return x, fmap
|
662 |
+
|
663 |
+
|
664 |
+
class DiscriminatorS(torch.nn.Module):
|
665 |
+
def __init__(self, use_spectral_norm=False):
|
666 |
+
super(DiscriminatorS, self).__init__()
|
667 |
+
norm_f = weight_norm if use_spectral_norm is False else spectral_norm
|
668 |
+
self.convs = nn.ModuleList(
|
669 |
+
[
|
670 |
+
norm_f(Conv1d(1, 16, 15, 1, padding=7)),
|
671 |
+
norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)),
|
672 |
+
norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)),
|
673 |
+
norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)),
|
674 |
+
norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)),
|
675 |
+
norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
|
676 |
+
]
|
677 |
+
)
|
678 |
+
self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
|
679 |
+
|
680 |
+
def forward(self, x):
|
681 |
+
fmap = []
|
682 |
+
|
683 |
+
for layer in self.convs:
|
684 |
+
x = layer(x)
|
685 |
+
x = F.leaky_relu(x, modules.LRELU_SLOPE)
|
686 |
+
fmap.append(x)
|
687 |
+
x = self.conv_post(x)
|
688 |
+
fmap.append(x)
|
689 |
+
x = torch.flatten(x, 1, -1)
|
690 |
+
|
691 |
+
return x, fmap
|
692 |
+
|
693 |
+
|
694 |
+
class MultiPeriodDiscriminator(torch.nn.Module):
|
695 |
+
def __init__(self, use_spectral_norm=False):
|
696 |
+
super(MultiPeriodDiscriminator, self).__init__()
|
697 |
+
periods = [2, 3, 5, 7, 11]
|
698 |
+
|
699 |
+
discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]
|
700 |
+
discs = discs + [
|
701 |
+
DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods
|
702 |
+
]
|
703 |
+
self.discriminators = nn.ModuleList(discs)
|
704 |
+
|
705 |
+
def forward(self, y, y_hat):
|
706 |
+
y_d_rs = []
|
707 |
+
y_d_gs = []
|
708 |
+
fmap_rs = []
|
709 |
+
fmap_gs = []
|
710 |
+
for i, d in enumerate(self.discriminators):
|
711 |
+
y_d_r, fmap_r = d(y)
|
712 |
+
y_d_g, fmap_g = d(y_hat)
|
713 |
+
y_d_rs.append(y_d_r)
|
714 |
+
y_d_gs.append(y_d_g)
|
715 |
+
fmap_rs.append(fmap_r)
|
716 |
+
fmap_gs.append(fmap_g)
|
717 |
+
|
718 |
+
return y_d_rs, y_d_gs, fmap_rs, fmap_gs
|
719 |
+
|
720 |
+
|
721 |
+
class ReferenceEncoder(nn.Module):
|
722 |
+
"""
|
723 |
+
inputs --- [N, Ty/r, n_mels*r] mels
|
724 |
+
outputs --- [N, ref_enc_gru_size]
|
725 |
+
"""
|
726 |
+
|
727 |
+
def __init__(self, spec_channels, gin_channels=0):
|
728 |
+
super().__init__()
|
729 |
+
self.spec_channels = spec_channels
|
730 |
+
ref_enc_filters = [32, 32, 64, 64, 128, 128]
|
731 |
+
K = len(ref_enc_filters)
|
732 |
+
filters = [1] + ref_enc_filters
|
733 |
+
convs = [
|
734 |
+
weight_norm(
|
735 |
+
nn.Conv2d(
|
736 |
+
in_channels=filters[i],
|
737 |
+
out_channels=filters[i + 1],
|
738 |
+
kernel_size=(3, 3),
|
739 |
+
stride=(2, 2),
|
740 |
+
padding=(1, 1),
|
741 |
+
)
|
742 |
+
)
|
743 |
+
for i in range(K)
|
744 |
+
]
|
745 |
+
self.convs = nn.ModuleList(convs)
|
746 |
+
# self.wns = nn.ModuleList([weight_norm(num_features=ref_enc_filters[i]) for i in range(K)]) # noqa: E501
|
747 |
+
|
748 |
+
out_channels = self.calculate_channels(spec_channels, 3, 2, 1, K)
|
749 |
+
self.gru = nn.GRU(
|
750 |
+
input_size=ref_enc_filters[-1] * out_channels,
|
751 |
+
hidden_size=256 // 2,
|
752 |
+
batch_first=True,
|
753 |
+
)
|
754 |
+
self.proj = nn.Linear(128, gin_channels)
|
755 |
+
|
756 |
+
def forward(self, inputs, mask=None):
|
757 |
+
N = inputs.size(0)
|
758 |
+
out = inputs.view(N, 1, -1, self.spec_channels) # [N, 1, Ty, n_freqs]
|
759 |
+
for conv in self.convs:
|
760 |
+
out = conv(out)
|
761 |
+
# out = wn(out)
|
762 |
+
out = F.relu(out) # [N, 128, Ty//2^K, n_mels//2^K]
|
763 |
+
|
764 |
+
out = out.transpose(1, 2) # [N, Ty//2^K, 128, n_mels//2^K]
|
765 |
+
T = out.size(1)
|
766 |
+
N = out.size(0)
|
767 |
+
out = out.contiguous().view(N, T, -1) # [N, Ty//2^K, 128*n_mels//2^K]
|
768 |
+
|
769 |
+
self.gru.flatten_parameters()
|
770 |
+
memory, out = self.gru(out) # out --- [1, N, 128]
|
771 |
+
|
772 |
+
return self.proj(out.squeeze(0))
|
773 |
+
|
774 |
+
def calculate_channels(self, L, kernel_size, stride, pad, n_convs):
|
775 |
+
for i in range(n_convs):
|
776 |
+
L = (L - kernel_size + 2 * pad) // stride + 1
|
777 |
+
return L
|
778 |
+
|
779 |
+
|
780 |
+
class SynthesizerTrn(nn.Module):
|
781 |
+
"""
|
782 |
+
Synthesizer for Training
|
783 |
+
"""
|
784 |
+
|
785 |
+
def __init__(
|
786 |
+
self,
|
787 |
+
n_vocab,
|
788 |
+
spec_channels,
|
789 |
+
segment_size,
|
790 |
+
inter_channels,
|
791 |
+
hidden_channels,
|
792 |
+
filter_channels,
|
793 |
+
n_heads,
|
794 |
+
n_layers,
|
795 |
+
kernel_size,
|
796 |
+
p_dropout,
|
797 |
+
resblock,
|
798 |
+
resblock_kernel_sizes,
|
799 |
+
resblock_dilation_sizes,
|
800 |
+
upsample_rates,
|
801 |
+
upsample_initial_channel,
|
802 |
+
upsample_kernel_sizes,
|
803 |
+
n_speakers=256,
|
804 |
+
gin_channels=256,
|
805 |
+
use_sdp=True,
|
806 |
+
n_flow_layer=4,
|
807 |
+
n_layers_trans_flow=4,
|
808 |
+
flow_share_parameter=False,
|
809 |
+
use_transformer_flow=True,
|
810 |
+
**kwargs
|
811 |
+
):
|
812 |
+
super().__init__()
|
813 |
+
self.n_vocab = n_vocab
|
814 |
+
self.spec_channels = spec_channels
|
815 |
+
self.inter_channels = inter_channels
|
816 |
+
self.hidden_channels = hidden_channels
|
817 |
+
self.filter_channels = filter_channels
|
818 |
+
self.n_heads = n_heads
|
819 |
+
self.n_layers = n_layers
|
820 |
+
self.kernel_size = kernel_size
|
821 |
+
self.p_dropout = p_dropout
|
822 |
+
self.resblock = resblock
|
823 |
+
self.resblock_kernel_sizes = resblock_kernel_sizes
|
824 |
+
self.resblock_dilation_sizes = resblock_dilation_sizes
|
825 |
+
self.upsample_rates = upsample_rates
|
826 |
+
self.upsample_initial_channel = upsample_initial_channel
|
827 |
+
self.upsample_kernel_sizes = upsample_kernel_sizes
|
828 |
+
self.segment_size = segment_size
|
829 |
+
self.n_speakers = n_speakers
|
830 |
+
self.gin_channels = gin_channels
|
831 |
+
self.n_layers_trans_flow = n_layers_trans_flow
|
832 |
+
self.use_spk_conditioned_encoder = kwargs.get(
|
833 |
+
"use_spk_conditioned_encoder", True
|
834 |
+
)
|
835 |
+
self.use_sdp = use_sdp
|
836 |
+
self.use_noise_scaled_mas = kwargs.get("use_noise_scaled_mas", False)
|
837 |
+
self.mas_noise_scale_initial = kwargs.get("mas_noise_scale_initial", 0.01)
|
838 |
+
self.noise_scale_delta = kwargs.get("noise_scale_delta", 2e-6)
|
839 |
+
self.current_mas_noise_scale = self.mas_noise_scale_initial
|
840 |
+
if self.use_spk_conditioned_encoder and gin_channels > 0:
|
841 |
+
self.enc_gin_channels = gin_channels
|
842 |
+
self.enc_p = TextEncoder(
|
843 |
+
n_vocab,
|
844 |
+
inter_channels,
|
845 |
+
hidden_channels,
|
846 |
+
filter_channels,
|
847 |
+
n_heads,
|
848 |
+
n_layers,
|
849 |
+
kernel_size,
|
850 |
+
p_dropout,
|
851 |
+
self.n_speakers,
|
852 |
+
gin_channels=self.enc_gin_channels,
|
853 |
+
)
|
854 |
+
self.dec = Generator(
|
855 |
+
inter_channels,
|
856 |
+
resblock,
|
857 |
+
resblock_kernel_sizes,
|
858 |
+
resblock_dilation_sizes,
|
859 |
+
upsample_rates,
|
860 |
+
upsample_initial_channel,
|
861 |
+
upsample_kernel_sizes,
|
862 |
+
gin_channels=gin_channels,
|
863 |
+
)
|
864 |
+
self.enc_q = PosteriorEncoder(
|
865 |
+
spec_channels,
|
866 |
+
inter_channels,
|
867 |
+
hidden_channels,
|
868 |
+
5,
|
869 |
+
1,
|
870 |
+
16,
|
871 |
+
gin_channels=gin_channels,
|
872 |
+
)
|
873 |
+
if use_transformer_flow:
|
874 |
+
self.flow = TransformerCouplingBlock(
|
875 |
+
inter_channels,
|
876 |
+
hidden_channels,
|
877 |
+
filter_channels,
|
878 |
+
n_heads,
|
879 |
+
n_layers_trans_flow,
|
880 |
+
5,
|
881 |
+
p_dropout,
|
882 |
+
n_flow_layer,
|
883 |
+
gin_channels=gin_channels,
|
884 |
+
share_parameter=flow_share_parameter,
|
885 |
+
)
|
886 |
+
else:
|
887 |
+
self.flow = ResidualCouplingBlock(
|
888 |
+
inter_channels,
|
889 |
+
hidden_channels,
|
890 |
+
5,
|
891 |
+
1,
|
892 |
+
n_flow_layer,
|
893 |
+
gin_channels=gin_channels,
|
894 |
+
)
|
895 |
+
self.sdp = StochasticDurationPredictor(
|
896 |
+
hidden_channels, 192, 3, 0.5, 4, gin_channels=gin_channels
|
897 |
+
)
|
898 |
+
self.dp = DurationPredictor(
|
899 |
+
hidden_channels, 256, 3, 0.5, gin_channels=gin_channels
|
900 |
+
)
|
901 |
+
|
902 |
+
if n_speakers >= 1:
|
903 |
+
self.emb_g = nn.Embedding(n_speakers, gin_channels)
|
904 |
+
else:
|
905 |
+
self.ref_enc = ReferenceEncoder(spec_channels, gin_channels)
|
906 |
+
|
907 |
+
def forward(
|
908 |
+
self,
|
909 |
+
x,
|
910 |
+
x_lengths,
|
911 |
+
y,
|
912 |
+
y_lengths,
|
913 |
+
sid,
|
914 |
+
tone,
|
915 |
+
language,
|
916 |
+
bert,
|
917 |
+
ja_bert,
|
918 |
+
en_bert,
|
919 |
+
emo=None,
|
920 |
+
):
|
921 |
+
if self.n_speakers > 0:
|
922 |
+
g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
|
923 |
+
else:
|
924 |
+
g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
|
925 |
+
x, m_p, logs_p, x_mask, loss_commit = self.enc_p(
|
926 |
+
x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=g
|
927 |
+
)
|
928 |
+
z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
|
929 |
+
z_p = self.flow(z, y_mask, g=g)
|
930 |
+
|
931 |
+
with torch.no_grad():
|
932 |
+
# negative cross-entropy
|
933 |
+
s_p_sq_r = torch.exp(-2 * logs_p) # [b, d, t]
|
934 |
+
neg_cent1 = torch.sum(
|
935 |
+
-0.5 * math.log(2 * math.pi) - logs_p, [1], keepdim=True
|
936 |
+
) # [b, 1, t_s]
|
937 |
+
neg_cent2 = torch.matmul(
|
938 |
+
-0.5 * (z_p**2).transpose(1, 2), s_p_sq_r
|
939 |
+
) # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]
|
940 |
+
neg_cent3 = torch.matmul(
|
941 |
+
z_p.transpose(1, 2), (m_p * s_p_sq_r)
|
942 |
+
) # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]
|
943 |
+
neg_cent4 = torch.sum(
|
944 |
+
-0.5 * (m_p**2) * s_p_sq_r, [1], keepdim=True
|
945 |
+
) # [b, 1, t_s]
|
946 |
+
neg_cent = neg_cent1 + neg_cent2 + neg_cent3 + neg_cent4
|
947 |
+
if self.use_noise_scaled_mas:
|
948 |
+
epsilon = (
|
949 |
+
torch.std(neg_cent)
|
950 |
+
* torch.randn_like(neg_cent)
|
951 |
+
* self.current_mas_noise_scale
|
952 |
+
)
|
953 |
+
neg_cent = neg_cent + epsilon
|
954 |
+
|
955 |
+
attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
|
956 |
+
attn = (
|
957 |
+
monotonic_align.maximum_path(neg_cent, attn_mask.squeeze(1))
|
958 |
+
.unsqueeze(1)
|
959 |
+
.detach()
|
960 |
+
)
|
961 |
+
|
962 |
+
w = attn.sum(2)
|
963 |
+
|
964 |
+
l_length_sdp = self.sdp(x, x_mask, w, g=g)
|
965 |
+
l_length_sdp = l_length_sdp / torch.sum(x_mask)
|
966 |
+
|
967 |
+
logw_ = torch.log(w + 1e-6) * x_mask
|
968 |
+
logw = self.dp(x, x_mask, g=g)
|
969 |
+
l_length_dp = torch.sum((logw - logw_) ** 2, [1, 2]) / torch.sum(
|
970 |
+
x_mask
|
971 |
+
) # for averaging
|
972 |
+
|
973 |
+
l_length = l_length_dp + l_length_sdp
|
974 |
+
|
975 |
+
# expand prior
|
976 |
+
m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1, 2)).transpose(1, 2)
|
977 |
+
logs_p = torch.matmul(attn.squeeze(1), logs_p.transpose(1, 2)).transpose(1, 2)
|
978 |
+
|
979 |
+
z_slice, ids_slice = commons.rand_slice_segments(
|
980 |
+
z, y_lengths, self.segment_size
|
981 |
+
)
|
982 |
+
o = self.dec(z_slice, g=g)
|
983 |
+
return (
|
984 |
+
o,
|
985 |
+
l_length,
|
986 |
+
attn,
|
987 |
+
ids_slice,
|
988 |
+
x_mask,
|
989 |
+
y_mask,
|
990 |
+
(z, z_p, m_p, logs_p, m_q, logs_q),
|
991 |
+
(x, logw, logw_),
|
992 |
+
loss_commit,
|
993 |
+
)
|
994 |
+
|
995 |
+
def infer(
|
996 |
+
self,
|
997 |
+
x,
|
998 |
+
x_lengths,
|
999 |
+
sid,
|
1000 |
+
tone,
|
1001 |
+
language,
|
1002 |
+
bert,
|
1003 |
+
ja_bert,
|
1004 |
+
en_bert,
|
1005 |
+
emo=None,
|
1006 |
+
noise_scale=0.667,
|
1007 |
+
length_scale=1,
|
1008 |
+
noise_scale_w=0.8,
|
1009 |
+
max_len=None,
|
1010 |
+
sdp_ratio=0,
|
1011 |
+
y=None,
|
1012 |
+
):
|
1013 |
+
# x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths, tone, language, bert)
|
1014 |
+
# g = self.gst(y)
|
1015 |
+
if self.n_speakers > 0:
|
1016 |
+
g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
|
1017 |
+
else:
|
1018 |
+
g = self.ref_enc(y.transpose(1, 2)).unsqueeze(-1)
|
1019 |
+
x, m_p, logs_p, x_mask, _ = self.enc_p(
|
1020 |
+
x, x_lengths, tone, language, bert, ja_bert, en_bert, emo, sid, g=g
|
1021 |
+
)
|
1022 |
+
logw = self.sdp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w) * (
|
1023 |
+
sdp_ratio
|
1024 |
+
) + self.dp(x, x_mask, g=g) * (1 - sdp_ratio)
|
1025 |
+
w = torch.exp(logw) * x_mask * length_scale
|
1026 |
+
w_ceil = torch.ceil(w)
|
1027 |
+
y_lengths = torch.clamp_min(torch.sum(w_ceil, [1, 2]), 1).long()
|
1028 |
+
y_mask = torch.unsqueeze(commons.sequence_mask(y_lengths, None), 1).to(
|
1029 |
+
x_mask.dtype
|
1030 |
+
)
|
1031 |
+
attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
|
1032 |
+
attn = commons.generate_path(w_ceil, attn_mask)
|
1033 |
+
|
1034 |
+
m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1, 2)).transpose(
|
1035 |
+
1, 2
|
1036 |
+
) # [b, t', t], [b, t, d] -> [b, d, t']
|
1037 |
+
logs_p = torch.matmul(attn.squeeze(1), logs_p.transpose(1, 2)).transpose(
|
1038 |
+
1, 2
|
1039 |
+
) # [b, t', t], [b, t, d] -> [b, d, t']
|
1040 |
+
|
1041 |
+
z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale
|
1042 |
+
z = self.flow(z_p, y_mask, g=g, reverse=True)
|
1043 |
+
o = self.dec((z * y_mask)[:, :, :max_len], g=g)
|
1044 |
+
return o, attn, y_mask, (z, z_p, m_p, logs_p)
|
modules.py
ADDED
@@ -0,0 +1,597 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import math
|
2 |
+
import torch
|
3 |
+
from torch import nn
|
4 |
+
from torch.nn import functional as F
|
5 |
+
|
6 |
+
from torch.nn import Conv1d
|
7 |
+
from torch.nn.utils import weight_norm, remove_weight_norm
|
8 |
+
|
9 |
+
import commons
|
10 |
+
from commons import init_weights, get_padding
|
11 |
+
from transforms import piecewise_rational_quadratic_transform
|
12 |
+
from attentions import Encoder
|
13 |
+
|
14 |
+
LRELU_SLOPE = 0.1
|
15 |
+
|
16 |
+
|
17 |
+
class LayerNorm(nn.Module):
|
18 |
+
def __init__(self, channels, eps=1e-5):
|
19 |
+
super().__init__()
|
20 |
+
self.channels = channels
|
21 |
+
self.eps = eps
|
22 |
+
|
23 |
+
self.gamma = nn.Parameter(torch.ones(channels))
|
24 |
+
self.beta = nn.Parameter(torch.zeros(channels))
|
25 |
+
|
26 |
+
def forward(self, x):
|
27 |
+
x = x.transpose(1, -1)
|
28 |
+
x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
|
29 |
+
return x.transpose(1, -1)
|
30 |
+
|
31 |
+
|
32 |
+
class ConvReluNorm(nn.Module):
|
33 |
+
def __init__(
|
34 |
+
self,
|
35 |
+
in_channels,
|
36 |
+
hidden_channels,
|
37 |
+
out_channels,
|
38 |
+
kernel_size,
|
39 |
+
n_layers,
|
40 |
+
p_dropout,
|
41 |
+
):
|
42 |
+
super().__init__()
|
43 |
+
self.in_channels = in_channels
|
44 |
+
self.hidden_channels = hidden_channels
|
45 |
+
self.out_channels = out_channels
|
46 |
+
self.kernel_size = kernel_size
|
47 |
+
self.n_layers = n_layers
|
48 |
+
self.p_dropout = p_dropout
|
49 |
+
assert n_layers > 1, "Number of layers should be larger than 0."
|
50 |
+
|
51 |
+
self.conv_layers = nn.ModuleList()
|
52 |
+
self.norm_layers = nn.ModuleList()
|
53 |
+
self.conv_layers.append(
|
54 |
+
nn.Conv1d(
|
55 |
+
in_channels, hidden_channels, kernel_size, padding=kernel_size // 2
|
56 |
+
)
|
57 |
+
)
|
58 |
+
self.norm_layers.append(LayerNorm(hidden_channels))
|
59 |
+
self.relu_drop = nn.Sequential(nn.ReLU(), nn.Dropout(p_dropout))
|
60 |
+
for _ in range(n_layers - 1):
|
61 |
+
self.conv_layers.append(
|
62 |
+
nn.Conv1d(
|
63 |
+
hidden_channels,
|
64 |
+
hidden_channels,
|
65 |
+
kernel_size,
|
66 |
+
padding=kernel_size // 2,
|
67 |
+
)
|
68 |
+
)
|
69 |
+
self.norm_layers.append(LayerNorm(hidden_channels))
|
70 |
+
self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
|
71 |
+
self.proj.weight.data.zero_()
|
72 |
+
self.proj.bias.data.zero_()
|
73 |
+
|
74 |
+
def forward(self, x, x_mask):
|
75 |
+
x_org = x
|
76 |
+
for i in range(self.n_layers):
|
77 |
+
x = self.conv_layers[i](x * x_mask)
|
78 |
+
x = self.norm_layers[i](x)
|
79 |
+
x = self.relu_drop(x)
|
80 |
+
x = x_org + self.proj(x)
|
81 |
+
return x * x_mask
|
82 |
+
|
83 |
+
|
84 |
+
class DDSConv(nn.Module):
|
85 |
+
"""
|
86 |
+
Dialted and Depth-Separable Convolution
|
87 |
+
"""
|
88 |
+
|
89 |
+
def __init__(self, channels, kernel_size, n_layers, p_dropout=0.0):
|
90 |
+
super().__init__()
|
91 |
+
self.channels = channels
|
92 |
+
self.kernel_size = kernel_size
|
93 |
+
self.n_layers = n_layers
|
94 |
+
self.p_dropout = p_dropout
|
95 |
+
|
96 |
+
self.drop = nn.Dropout(p_dropout)
|
97 |
+
self.convs_sep = nn.ModuleList()
|
98 |
+
self.convs_1x1 = nn.ModuleList()
|
99 |
+
self.norms_1 = nn.ModuleList()
|
100 |
+
self.norms_2 = nn.ModuleList()
|
101 |
+
for i in range(n_layers):
|
102 |
+
dilation = kernel_size**i
|
103 |
+
padding = (kernel_size * dilation - dilation) // 2
|
104 |
+
self.convs_sep.append(
|
105 |
+
nn.Conv1d(
|
106 |
+
channels,
|
107 |
+
channels,
|
108 |
+
kernel_size,
|
109 |
+
groups=channels,
|
110 |
+
dilation=dilation,
|
111 |
+
padding=padding,
|
112 |
+
)
|
113 |
+
)
|
114 |
+
self.convs_1x1.append(nn.Conv1d(channels, channels, 1))
|
115 |
+
self.norms_1.append(LayerNorm(channels))
|
116 |
+
self.norms_2.append(LayerNorm(channels))
|
117 |
+
|
118 |
+
def forward(self, x, x_mask, g=None):
|
119 |
+
if g is not None:
|
120 |
+
x = x + g
|
121 |
+
for i in range(self.n_layers):
|
122 |
+
y = self.convs_sep[i](x * x_mask)
|
123 |
+
y = self.norms_1[i](y)
|
124 |
+
y = F.gelu(y)
|
125 |
+
y = self.convs_1x1[i](y)
|
126 |
+
y = self.norms_2[i](y)
|
127 |
+
y = F.gelu(y)
|
128 |
+
y = self.drop(y)
|
129 |
+
x = x + y
|
130 |
+
return x * x_mask
|
131 |
+
|
132 |
+
|
133 |
+
class WN(torch.nn.Module):
|
134 |
+
def __init__(
|
135 |
+
self,
|
136 |
+
hidden_channels,
|
137 |
+
kernel_size,
|
138 |
+
dilation_rate,
|
139 |
+
n_layers,
|
140 |
+
gin_channels=0,
|
141 |
+
p_dropout=0,
|
142 |
+
):
|
143 |
+
super(WN, self).__init__()
|
144 |
+
assert kernel_size % 2 == 1
|
145 |
+
self.hidden_channels = hidden_channels
|
146 |
+
self.kernel_size = (kernel_size,)
|
147 |
+
self.dilation_rate = dilation_rate
|
148 |
+
self.n_layers = n_layers
|
149 |
+
self.gin_channels = gin_channels
|
150 |
+
self.p_dropout = p_dropout
|
151 |
+
|
152 |
+
self.in_layers = torch.nn.ModuleList()
|
153 |
+
self.res_skip_layers = torch.nn.ModuleList()
|
154 |
+
self.drop = nn.Dropout(p_dropout)
|
155 |
+
|
156 |
+
if gin_channels != 0:
|
157 |
+
cond_layer = torch.nn.Conv1d(
|
158 |
+
gin_channels, 2 * hidden_channels * n_layers, 1
|
159 |
+
)
|
160 |
+
self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name="weight")
|
161 |
+
|
162 |
+
for i in range(n_layers):
|
163 |
+
dilation = dilation_rate**i
|
164 |
+
padding = int((kernel_size * dilation - dilation) / 2)
|
165 |
+
in_layer = torch.nn.Conv1d(
|
166 |
+
hidden_channels,
|
167 |
+
2 * hidden_channels,
|
168 |
+
kernel_size,
|
169 |
+
dilation=dilation,
|
170 |
+
padding=padding,
|
171 |
+
)
|
172 |
+
in_layer = torch.nn.utils.weight_norm(in_layer, name="weight")
|
173 |
+
self.in_layers.append(in_layer)
|
174 |
+
|
175 |
+
# last one is not necessary
|
176 |
+
if i < n_layers - 1:
|
177 |
+
res_skip_channels = 2 * hidden_channels
|
178 |
+
else:
|
179 |
+
res_skip_channels = hidden_channels
|
180 |
+
|
181 |
+
res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1)
|
182 |
+
res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name="weight")
|
183 |
+
self.res_skip_layers.append(res_skip_layer)
|
184 |
+
|
185 |
+
def forward(self, x, x_mask, g=None, **kwargs):
|
186 |
+
output = torch.zeros_like(x)
|
187 |
+
n_channels_tensor = torch.IntTensor([self.hidden_channels])
|
188 |
+
|
189 |
+
if g is not None:
|
190 |
+
g = self.cond_layer(g)
|
191 |
+
|
192 |
+
for i in range(self.n_layers):
|
193 |
+
x_in = self.in_layers[i](x)
|
194 |
+
if g is not None:
|
195 |
+
cond_offset = i * 2 * self.hidden_channels
|
196 |
+
g_l = g[:, cond_offset : cond_offset + 2 * self.hidden_channels, :]
|
197 |
+
else:
|
198 |
+
g_l = torch.zeros_like(x_in)
|
199 |
+
|
200 |
+
acts = commons.fused_add_tanh_sigmoid_multiply(x_in, g_l, n_channels_tensor)
|
201 |
+
acts = self.drop(acts)
|
202 |
+
|
203 |
+
res_skip_acts = self.res_skip_layers[i](acts)
|
204 |
+
if i < self.n_layers - 1:
|
205 |
+
res_acts = res_skip_acts[:, : self.hidden_channels, :]
|
206 |
+
x = (x + res_acts) * x_mask
|
207 |
+
output = output + res_skip_acts[:, self.hidden_channels :, :]
|
208 |
+
else:
|
209 |
+
output = output + res_skip_acts
|
210 |
+
return output * x_mask
|
211 |
+
|
212 |
+
def remove_weight_norm(self):
|
213 |
+
if self.gin_channels != 0:
|
214 |
+
torch.nn.utils.remove_weight_norm(self.cond_layer)
|
215 |
+
for l in self.in_layers:
|
216 |
+
torch.nn.utils.remove_weight_norm(l)
|
217 |
+
for l in self.res_skip_layers:
|
218 |
+
torch.nn.utils.remove_weight_norm(l)
|
219 |
+
|
220 |
+
|
221 |
+
class ResBlock1(torch.nn.Module):
|
222 |
+
def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
|
223 |
+
super(ResBlock1, self).__init__()
|
224 |
+
self.convs1 = nn.ModuleList(
|
225 |
+
[
|
226 |
+
weight_norm(
|
227 |
+
Conv1d(
|
228 |
+
channels,
|
229 |
+
channels,
|
230 |
+
kernel_size,
|
231 |
+
1,
|
232 |
+
dilation=dilation[0],
|
233 |
+
padding=get_padding(kernel_size, dilation[0]),
|
234 |
+
)
|
235 |
+
),
|
236 |
+
weight_norm(
|
237 |
+
Conv1d(
|
238 |
+
channels,
|
239 |
+
channels,
|
240 |
+
kernel_size,
|
241 |
+
1,
|
242 |
+
dilation=dilation[1],
|
243 |
+
padding=get_padding(kernel_size, dilation[1]),
|
244 |
+
)
|
245 |
+
),
|
246 |
+
weight_norm(
|
247 |
+
Conv1d(
|
248 |
+
channels,
|
249 |
+
channels,
|
250 |
+
kernel_size,
|
251 |
+
1,
|
252 |
+
dilation=dilation[2],
|
253 |
+
padding=get_padding(kernel_size, dilation[2]),
|
254 |
+
)
|
255 |
+
),
|
256 |
+
]
|
257 |
+
)
|
258 |
+
self.convs1.apply(init_weights)
|
259 |
+
|
260 |
+
self.convs2 = nn.ModuleList(
|
261 |
+
[
|
262 |
+
weight_norm(
|
263 |
+
Conv1d(
|
264 |
+
channels,
|
265 |
+
channels,
|
266 |
+
kernel_size,
|
267 |
+
1,
|
268 |
+
dilation=1,
|
269 |
+
padding=get_padding(kernel_size, 1),
|
270 |
+
)
|
271 |
+
),
|
272 |
+
weight_norm(
|
273 |
+
Conv1d(
|
274 |
+
channels,
|
275 |
+
channels,
|
276 |
+
kernel_size,
|
277 |
+
1,
|
278 |
+
dilation=1,
|
279 |
+
padding=get_padding(kernel_size, 1),
|
280 |
+
)
|
281 |
+
),
|
282 |
+
weight_norm(
|
283 |
+
Conv1d(
|
284 |
+
channels,
|
285 |
+
channels,
|
286 |
+
kernel_size,
|
287 |
+
1,
|
288 |
+
dilation=1,
|
289 |
+
padding=get_padding(kernel_size, 1),
|
290 |
+
)
|
291 |
+
),
|
292 |
+
]
|
293 |
+
)
|
294 |
+
self.convs2.apply(init_weights)
|
295 |
+
|
296 |
+
def forward(self, x, x_mask=None):
|
297 |
+
for c1, c2 in zip(self.convs1, self.convs2):
|
298 |
+
xt = F.leaky_relu(x, LRELU_SLOPE)
|
299 |
+
if x_mask is not None:
|
300 |
+
xt = xt * x_mask
|
301 |
+
xt = c1(xt)
|
302 |
+
xt = F.leaky_relu(xt, LRELU_SLOPE)
|
303 |
+
if x_mask is not None:
|
304 |
+
xt = xt * x_mask
|
305 |
+
xt = c2(xt)
|
306 |
+
x = xt + x
|
307 |
+
if x_mask is not None:
|
308 |
+
x = x * x_mask
|
309 |
+
return x
|
310 |
+
|
311 |
+
def remove_weight_norm(self):
|
312 |
+
for l in self.convs1:
|
313 |
+
remove_weight_norm(l)
|
314 |
+
for l in self.convs2:
|
315 |
+
remove_weight_norm(l)
|
316 |
+
|
317 |
+
|
318 |
+
class ResBlock2(torch.nn.Module):
|
319 |
+
def __init__(self, channels, kernel_size=3, dilation=(1, 3)):
|
320 |
+
super(ResBlock2, self).__init__()
|
321 |
+
self.convs = nn.ModuleList(
|
322 |
+
[
|
323 |
+
weight_norm(
|
324 |
+
Conv1d(
|
325 |
+
channels,
|
326 |
+
channels,
|
327 |
+
kernel_size,
|
328 |
+
1,
|
329 |
+
dilation=dilation[0],
|
330 |
+
padding=get_padding(kernel_size, dilation[0]),
|
331 |
+
)
|
332 |
+
),
|
333 |
+
weight_norm(
|
334 |
+
Conv1d(
|
335 |
+
channels,
|
336 |
+
channels,
|
337 |
+
kernel_size,
|
338 |
+
1,
|
339 |
+
dilation=dilation[1],
|
340 |
+
padding=get_padding(kernel_size, dilation[1]),
|
341 |
+
)
|
342 |
+
),
|
343 |
+
]
|
344 |
+
)
|
345 |
+
self.convs.apply(init_weights)
|
346 |
+
|
347 |
+
def forward(self, x, x_mask=None):
|
348 |
+
for c in self.convs:
|
349 |
+
xt = F.leaky_relu(x, LRELU_SLOPE)
|
350 |
+
if x_mask is not None:
|
351 |
+
xt = xt * x_mask
|
352 |
+
xt = c(xt)
|
353 |
+
x = xt + x
|
354 |
+
if x_mask is not None:
|
355 |
+
x = x * x_mask
|
356 |
+
return x
|
357 |
+
|
358 |
+
def remove_weight_norm(self):
|
359 |
+
for l in self.convs:
|
360 |
+
remove_weight_norm(l)
|
361 |
+
|
362 |
+
|
363 |
+
class Log(nn.Module):
|
364 |
+
def forward(self, x, x_mask, reverse=False, **kwargs):
|
365 |
+
if not reverse:
|
366 |
+
y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask
|
367 |
+
logdet = torch.sum(-y, [1, 2])
|
368 |
+
return y, logdet
|
369 |
+
else:
|
370 |
+
x = torch.exp(x) * x_mask
|
371 |
+
return x
|
372 |
+
|
373 |
+
|
374 |
+
class Flip(nn.Module):
|
375 |
+
def forward(self, x, *args, reverse=False, **kwargs):
|
376 |
+
x = torch.flip(x, [1])
|
377 |
+
if not reverse:
|
378 |
+
logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)
|
379 |
+
return x, logdet
|
380 |
+
else:
|
381 |
+
return x
|
382 |
+
|
383 |
+
|
384 |
+
class ElementwiseAffine(nn.Module):
|
385 |
+
def __init__(self, channels):
|
386 |
+
super().__init__()
|
387 |
+
self.channels = channels
|
388 |
+
self.m = nn.Parameter(torch.zeros(channels, 1))
|
389 |
+
self.logs = nn.Parameter(torch.zeros(channels, 1))
|
390 |
+
|
391 |
+
def forward(self, x, x_mask, reverse=False, **kwargs):
|
392 |
+
if not reverse:
|
393 |
+
y = self.m + torch.exp(self.logs) * x
|
394 |
+
y = y * x_mask
|
395 |
+
logdet = torch.sum(self.logs * x_mask, [1, 2])
|
396 |
+
return y, logdet
|
397 |
+
else:
|
398 |
+
x = (x - self.m) * torch.exp(-self.logs) * x_mask
|
399 |
+
return x
|
400 |
+
|
401 |
+
|
402 |
+
class ResidualCouplingLayer(nn.Module):
|
403 |
+
def __init__(
|
404 |
+
self,
|
405 |
+
channels,
|
406 |
+
hidden_channels,
|
407 |
+
kernel_size,
|
408 |
+
dilation_rate,
|
409 |
+
n_layers,
|
410 |
+
p_dropout=0,
|
411 |
+
gin_channels=0,
|
412 |
+
mean_only=False,
|
413 |
+
):
|
414 |
+
assert channels % 2 == 0, "channels should be divisible by 2"
|
415 |
+
super().__init__()
|
416 |
+
self.channels = channels
|
417 |
+
self.hidden_channels = hidden_channels
|
418 |
+
self.kernel_size = kernel_size
|
419 |
+
self.dilation_rate = dilation_rate
|
420 |
+
self.n_layers = n_layers
|
421 |
+
self.half_channels = channels // 2
|
422 |
+
self.mean_only = mean_only
|
423 |
+
|
424 |
+
self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
|
425 |
+
self.enc = WN(
|
426 |
+
hidden_channels,
|
427 |
+
kernel_size,
|
428 |
+
dilation_rate,
|
429 |
+
n_layers,
|
430 |
+
p_dropout=p_dropout,
|
431 |
+
gin_channels=gin_channels,
|
432 |
+
)
|
433 |
+
self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
|
434 |
+
self.post.weight.data.zero_()
|
435 |
+
self.post.bias.data.zero_()
|
436 |
+
|
437 |
+
def forward(self, x, x_mask, g=None, reverse=False):
|
438 |
+
x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
|
439 |
+
h = self.pre(x0) * x_mask
|
440 |
+
h = self.enc(h, x_mask, g=g)
|
441 |
+
stats = self.post(h) * x_mask
|
442 |
+
if not self.mean_only:
|
443 |
+
m, logs = torch.split(stats, [self.half_channels] * 2, 1)
|
444 |
+
else:
|
445 |
+
m = stats
|
446 |
+
logs = torch.zeros_like(m)
|
447 |
+
|
448 |
+
if not reverse:
|
449 |
+
x1 = m + x1 * torch.exp(logs) * x_mask
|
450 |
+
x = torch.cat([x0, x1], 1)
|
451 |
+
logdet = torch.sum(logs, [1, 2])
|
452 |
+
return x, logdet
|
453 |
+
else:
|
454 |
+
x1 = (x1 - m) * torch.exp(-logs) * x_mask
|
455 |
+
x = torch.cat([x0, x1], 1)
|
456 |
+
return x
|
457 |
+
|
458 |
+
|
459 |
+
class ConvFlow(nn.Module):
|
460 |
+
def __init__(
|
461 |
+
self,
|
462 |
+
in_channels,
|
463 |
+
filter_channels,
|
464 |
+
kernel_size,
|
465 |
+
n_layers,
|
466 |
+
num_bins=10,
|
467 |
+
tail_bound=5.0,
|
468 |
+
):
|
469 |
+
super().__init__()
|
470 |
+
self.in_channels = in_channels
|
471 |
+
self.filter_channels = filter_channels
|
472 |
+
self.kernel_size = kernel_size
|
473 |
+
self.n_layers = n_layers
|
474 |
+
self.num_bins = num_bins
|
475 |
+
self.tail_bound = tail_bound
|
476 |
+
self.half_channels = in_channels // 2
|
477 |
+
|
478 |
+
self.pre = nn.Conv1d(self.half_channels, filter_channels, 1)
|
479 |
+
self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.0)
|
480 |
+
self.proj = nn.Conv1d(
|
481 |
+
filter_channels, self.half_channels * (num_bins * 3 - 1), 1
|
482 |
+
)
|
483 |
+
self.proj.weight.data.zero_()
|
484 |
+
self.proj.bias.data.zero_()
|
485 |
+
|
486 |
+
def forward(self, x, x_mask, g=None, reverse=False):
|
487 |
+
x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
|
488 |
+
h = self.pre(x0)
|
489 |
+
h = self.convs(h, x_mask, g=g)
|
490 |
+
h = self.proj(h) * x_mask
|
491 |
+
|
492 |
+
b, c, t = x0.shape
|
493 |
+
h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2) # [b, cx?, t] -> [b, c, t, ?]
|
494 |
+
|
495 |
+
unnormalized_widths = h[..., : self.num_bins] / math.sqrt(self.filter_channels)
|
496 |
+
unnormalized_heights = h[..., self.num_bins : 2 * self.num_bins] / math.sqrt(
|
497 |
+
self.filter_channels
|
498 |
+
)
|
499 |
+
unnormalized_derivatives = h[..., 2 * self.num_bins :]
|
500 |
+
|
501 |
+
x1, logabsdet = piecewise_rational_quadratic_transform(
|
502 |
+
x1,
|
503 |
+
unnormalized_widths,
|
504 |
+
unnormalized_heights,
|
505 |
+
unnormalized_derivatives,
|
506 |
+
inverse=reverse,
|
507 |
+
tails="linear",
|
508 |
+
tail_bound=self.tail_bound,
|
509 |
+
)
|
510 |
+
|
511 |
+
x = torch.cat([x0, x1], 1) * x_mask
|
512 |
+
logdet = torch.sum(logabsdet * x_mask, [1, 2])
|
513 |
+
if not reverse:
|
514 |
+
return x, logdet
|
515 |
+
else:
|
516 |
+
return x
|
517 |
+
|
518 |
+
|
519 |
+
class TransformerCouplingLayer(nn.Module):
|
520 |
+
def __init__(
|
521 |
+
self,
|
522 |
+
channels,
|
523 |
+
hidden_channels,
|
524 |
+
kernel_size,
|
525 |
+
n_layers,
|
526 |
+
n_heads,
|
527 |
+
p_dropout=0,
|
528 |
+
filter_channels=0,
|
529 |
+
mean_only=False,
|
530 |
+
wn_sharing_parameter=None,
|
531 |
+
gin_channels=0,
|
532 |
+
):
|
533 |
+
assert channels % 2 == 0, "channels should be divisible by 2"
|
534 |
+
super().__init__()
|
535 |
+
self.channels = channels
|
536 |
+
self.hidden_channels = hidden_channels
|
537 |
+
self.kernel_size = kernel_size
|
538 |
+
self.n_layers = n_layers
|
539 |
+
self.half_channels = channels // 2
|
540 |
+
self.mean_only = mean_only
|
541 |
+
|
542 |
+
self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
|
543 |
+
self.enc = (
|
544 |
+
Encoder(
|
545 |
+
hidden_channels,
|
546 |
+
filter_channels,
|
547 |
+
n_heads,
|
548 |
+
n_layers,
|
549 |
+
kernel_size,
|
550 |
+
p_dropout,
|
551 |
+
isflow=True,
|
552 |
+
gin_channels=gin_channels,
|
553 |
+
)
|
554 |
+
if wn_sharing_parameter is None
|
555 |
+
else wn_sharing_parameter
|
556 |
+
)
|
557 |
+
self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
|
558 |
+
self.post.weight.data.zero_()
|
559 |
+
self.post.bias.data.zero_()
|
560 |
+
|
561 |
+
def forward(self, x, x_mask, g=None, reverse=False):
|
562 |
+
x0, x1 = torch.split(x, [self.half_channels] * 2, 1)
|
563 |
+
h = self.pre(x0) * x_mask
|
564 |
+
h = self.enc(h, x_mask, g=g)
|
565 |
+
stats = self.post(h) * x_mask
|
566 |
+
if not self.mean_only:
|
567 |
+
m, logs = torch.split(stats, [self.half_channels] * 2, 1)
|
568 |
+
else:
|
569 |
+
m = stats
|
570 |
+
logs = torch.zeros_like(m)
|
571 |
+
|
572 |
+
if not reverse:
|
573 |
+
x1 = m + x1 * torch.exp(logs) * x_mask
|
574 |
+
x = torch.cat([x0, x1], 1)
|
575 |
+
logdet = torch.sum(logs, [1, 2])
|
576 |
+
return x, logdet
|
577 |
+
else:
|
578 |
+
x1 = (x1 - m) * torch.exp(-logs) * x_mask
|
579 |
+
x = torch.cat([x0, x1], 1)
|
580 |
+
return x
|
581 |
+
|
582 |
+
x1, logabsdet = piecewise_rational_quadratic_transform(
|
583 |
+
x1,
|
584 |
+
unnormalized_widths,
|
585 |
+
unnormalized_heights,
|
586 |
+
unnormalized_derivatives,
|
587 |
+
inverse=reverse,
|
588 |
+
tails="linear",
|
589 |
+
tail_bound=self.tail_bound,
|
590 |
+
)
|
591 |
+
|
592 |
+
x = torch.cat([x0, x1], 1) * x_mask
|
593 |
+
logdet = torch.sum(logabsdet * x_mask, [1, 2])
|
594 |
+
if not reverse:
|
595 |
+
return x, logdet
|
596 |
+
else:
|
597 |
+
return x
|
monotonic_align/__init__.py
ADDED
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from numpy import zeros, int32, float32
|
2 |
+
from torch import from_numpy
|
3 |
+
|
4 |
+
from .core import maximum_path_jit
|
5 |
+
|
6 |
+
|
7 |
+
def maximum_path(neg_cent, mask):
|
8 |
+
device = neg_cent.device
|
9 |
+
dtype = neg_cent.dtype
|
10 |
+
neg_cent = neg_cent.data.cpu().numpy().astype(float32)
|
11 |
+
path = zeros(neg_cent.shape, dtype=int32)
|
12 |
+
|
13 |
+
t_t_max = mask.sum(1)[:, 0].data.cpu().numpy().astype(int32)
|
14 |
+
t_s_max = mask.sum(2)[:, 0].data.cpu().numpy().astype(int32)
|
15 |
+
maximum_path_jit(path, neg_cent, t_t_max, t_s_max)
|
16 |
+
return from_numpy(path).to(device=device, dtype=dtype)
|
monotonic_align/__pycache__/__init__.cpython-310.pyc
ADDED
Binary file (727 Bytes). View file
|
|
monotonic_align/__pycache__/core.cpython-310.pyc
ADDED
Binary file (980 Bytes). View file
|
|
monotonic_align/core.py
ADDED
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numba
|
2 |
+
|
3 |
+
|
4 |
+
@numba.jit(
|
5 |
+
numba.void(
|
6 |
+
numba.int32[:, :, ::1],
|
7 |
+
numba.float32[:, :, ::1],
|
8 |
+
numba.int32[::1],
|
9 |
+
numba.int32[::1],
|
10 |
+
),
|
11 |
+
nopython=True,
|
12 |
+
nogil=True,
|
13 |
+
)
|
14 |
+
def maximum_path_jit(paths, values, t_ys, t_xs):
|
15 |
+
b = paths.shape[0]
|
16 |
+
max_neg_val = -1e9
|
17 |
+
for i in range(int(b)):
|
18 |
+
path = paths[i]
|
19 |
+
value = values[i]
|
20 |
+
t_y = t_ys[i]
|
21 |
+
t_x = t_xs[i]
|
22 |
+
|
23 |
+
v_prev = v_cur = 0.0
|
24 |
+
index = t_x - 1
|
25 |
+
|
26 |
+
for y in range(t_y):
|
27 |
+
for x in range(max(0, t_x + y - t_y), min(t_x, y + 1)):
|
28 |
+
if x == y:
|
29 |
+
v_cur = max_neg_val
|
30 |
+
else:
|
31 |
+
v_cur = value[y - 1, x]
|
32 |
+
if x == 0:
|
33 |
+
if y == 0:
|
34 |
+
v_prev = 0.0
|
35 |
+
else:
|
36 |
+
v_prev = max_neg_val
|
37 |
+
else:
|
38 |
+
v_prev = value[y - 1, x - 1]
|
39 |
+
value[y, x] += max(v_prev, v_cur)
|
40 |
+
|
41 |
+
for y in range(t_y - 1, -1, -1):
|
42 |
+
path[y, index] = 1
|
43 |
+
if index != 0 and (
|
44 |
+
index == y or value[y - 1, index] < value[y - 1, index - 1]
|
45 |
+
):
|
46 |
+
index = index - 1
|
re_matching.py
ADDED
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import re
|
2 |
+
|
3 |
+
|
4 |
+
def extract_language_and_text_updated(speaker, dialogue):
|
5 |
+
# 使用正则表达式匹配<语言>标签和其后的文本
|
6 |
+
pattern_language_text = r"<(\S+?)>([^<]+)"
|
7 |
+
matches = re.findall(pattern_language_text, dialogue, re.DOTALL)
|
8 |
+
speaker = speaker[1:-1]
|
9 |
+
# 清理文本:去除两边的空白字符
|
10 |
+
matches_cleaned = [(lang.upper(), text.strip()) for lang, text in matches]
|
11 |
+
matches_cleaned.append(speaker)
|
12 |
+
return matches_cleaned
|
13 |
+
|
14 |
+
|
15 |
+
def validate_text(input_text):
|
16 |
+
# 验证说话人的正则表达式
|
17 |
+
pattern_speaker = r"(\[\S+?\])((?:\s*<\S+?>[^<\[\]]+?)+)"
|
18 |
+
|
19 |
+
# 使用re.DOTALL标志使.匹配包括换行符在内的所有字符
|
20 |
+
matches = re.findall(pattern_speaker, input_text, re.DOTALL)
|
21 |
+
|
22 |
+
# 对每个匹配到的说话人内容进行进一步验证
|
23 |
+
for _, dialogue in matches:
|
24 |
+
language_text_matches = extract_language_and_text_updated(_, dialogue)
|
25 |
+
if not language_text_matches:
|
26 |
+
return (
|
27 |
+
False,
|
28 |
+
"Error: Invalid format detected in dialogue content. Please check your input.",
|
29 |
+
)
|
30 |
+
|
31 |
+
# 如果输入的文本中没有找到任何匹配项
|
32 |
+
if not matches:
|
33 |
+
return (
|
34 |
+
False,
|
35 |
+
"Error: No valid speaker format detected. Please check your input.",
|
36 |
+
)
|
37 |
+
|
38 |
+
return True, "Input is valid."
|
39 |
+
|
40 |
+
|
41 |
+
def text_matching(text: str) -> list:
|
42 |
+
speaker_pattern = r"(\[\S+?\])(.+?)(?=\[\S+?\]|$)"
|
43 |
+
matches = re.findall(speaker_pattern, text, re.DOTALL)
|
44 |
+
result = []
|
45 |
+
for speaker, dialogue in matches:
|
46 |
+
result.append(extract_language_and_text_updated(speaker, dialogue))
|
47 |
+
print(result)
|
48 |
+
return result
|
49 |
+
|
50 |
+
|
51 |
+
def cut_para(text):
|
52 |
+
splitted_para = re.split("[\n]", text) # 按段分
|
53 |
+
splitted_para = [
|
54 |
+
sentence.strip() for sentence in splitted_para if sentence.strip()
|
55 |
+
] # 删除空字符串
|
56 |
+
return splitted_para
|
57 |
+
|
58 |
+
|
59 |
+
def cut_sent(para):
|
60 |
+
para = re.sub("([。!;?\?])([^”’])", r"\1\n\2", para) # 单字符断句符
|
61 |
+
para = re.sub("(\.{6})([^”’])", r"\1\n\2", para) # 英文省略号
|
62 |
+
para = re.sub("(\…{2})([^”’])", r"\1\n\2", para) # 中文省略号
|
63 |
+
para = re.sub("([。!?\?][”’])([^,。!?\?])", r"\1\n\2", para)
|
64 |
+
para = para.rstrip() # 段尾如果有多余的\n就去掉它
|
65 |
+
return para.split("\n")
|
66 |
+
|
67 |
+
|
68 |
+
if __name__ == "__main__":
|
69 |
+
text = """
|
70 |
+
[说话人1]
|
71 |
+
[说话人2]<zh>你好吗?<jp>元気ですか?<jp>こんにちは,世界。<zh>你好吗?
|
72 |
+
[说话人3]<zh>谢谢。<jp>どういたしまして。
|
73 |
+
"""
|
74 |
+
text_matching(text)
|
75 |
+
# 测试函数
|
76 |
+
test_text = """
|
77 |
+
[说话人1]<zh>你好,こんにちは!<jp>こんにちは,世界。
|
78 |
+
[说话人2]<zh>你好吗?
|
79 |
+
"""
|
80 |
+
text_matching(test_text)
|
81 |
+
res = validate_text(test_text)
|
82 |
+
print(res)
|
requirements.txt
ADDED
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
librosa==0.9.1
|
2 |
+
matplotlib
|
3 |
+
numpy
|
4 |
+
numba
|
5 |
+
phonemizer
|
6 |
+
scipy
|
7 |
+
tensorboard
|
8 |
+
Unidecode
|
9 |
+
amfm_decompy
|
10 |
+
jieba
|
11 |
+
transformers
|
12 |
+
pypinyin
|
13 |
+
cn2an
|
14 |
+
gradio
|
15 |
+
av
|
16 |
+
mecab-python3
|
17 |
+
loguru
|
18 |
+
unidic-lite
|
19 |
+
cmudict
|
20 |
+
fugashi
|
21 |
+
num2words
|
22 |
+
PyYAML
|
23 |
+
requests
|
24 |
+
pyopenjtalk; sys_platform == 'linux'
|
25 |
+
openjtalk; sys_platform != 'linux'
|
26 |
+
jaconv
|
27 |
+
psutil
|
28 |
+
GPUtil
|
29 |
+
vector_quantize_pytorch
|
30 |
+
g2p_en
|
31 |
+
sentencepiece
|
32 |
+
pykakasi
|
33 |
+
langid
|
text/__init__.py
ADDED
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from text.symbols import *
|
2 |
+
|
3 |
+
_symbol_to_id = {s: i for i, s in enumerate(symbols)}
|
4 |
+
|
5 |
+
|
6 |
+
def cleaned_text_to_sequence(cleaned_text, tones, language):
|
7 |
+
"""Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
|
8 |
+
Args:
|
9 |
+
text: string to convert to a sequence
|
10 |
+
Returns:
|
11 |
+
List of integers corresponding to the symbols in the text
|
12 |
+
"""
|
13 |
+
phones = [_symbol_to_id[symbol] for symbol in cleaned_text]
|
14 |
+
tone_start = language_tone_start_map[language]
|
15 |
+
tones = [i + tone_start for i in tones]
|
16 |
+
lang_id = language_id_map[language]
|
17 |
+
lang_ids = [lang_id for i in phones]
|
18 |
+
return phones, tones, lang_ids
|
19 |
+
|
20 |
+
|
21 |
+
def get_bert(norm_text, word2ph, language, device):
|
22 |
+
from .chinese_bert import get_bert_feature as zh_bert
|
23 |
+
from .english_bert_mock import get_bert_feature as en_bert
|
24 |
+
from .japanese_bert import get_bert_feature as jp_bert
|
25 |
+
|
26 |
+
lang_bert_func_map = {"ZH": zh_bert, "EN": en_bert, "JP": jp_bert}
|
27 |
+
bert = lang_bert_func_map[language](norm_text, word2ph, device)
|
28 |
+
return bert
|
29 |
+
|
30 |
+
|
31 |
+
def check_bert_models():
|
32 |
+
import json
|
33 |
+
from pathlib import Path
|
34 |
+
|
35 |
+
from config import config
|
36 |
+
from .bert_utils import _check_bert
|
37 |
+
|
38 |
+
if config.mirror.lower() == "openi":
|
39 |
+
import openi
|
40 |
+
|
41 |
+
kwargs = {"token": config.openi_token} if config.openi_token else {}
|
42 |
+
openi.login(**kwargs)
|
43 |
+
|
44 |
+
with open("./bert/bert_models.json", "r") as fp:
|
45 |
+
models = json.load(fp)
|
46 |
+
for k, v in models.items():
|
47 |
+
local_path = Path("./bert").joinpath(k)
|
48 |
+
_check_bert(v["repo_id"], v["files"], local_path)
|
49 |
+
|
50 |
+
|
51 |
+
#check_bert_models()
|
text/__pycache__/__init__.cpython-310.pyc
ADDED
Binary file (2.19 kB). View file
|
|
text/__pycache__/bert_utils.cpython-310.pyc
ADDED
Binary file (732 Bytes). View file
|
|
text/__pycache__/chinese.cpython-310.pyc
ADDED
Binary file (4.6 kB). View file
|
|
text/__pycache__/chinese_bert.cpython-310.pyc
ADDED
Binary file (1.74 kB). View file
|
|
text/__pycache__/cleaner.cpython-310.pyc
ADDED
Binary file (979 Bytes). View file
|
|
text/__pycache__/english.cpython-310.pyc
ADDED
Binary file (9.49 kB). View file
|
|
text/__pycache__/english_bert_mock.cpython-310.pyc
ADDED
Binary file (1.24 kB). View file
|
|
text/__pycache__/japanese.cpython-310.pyc
ADDED
Binary file (11.4 kB). View file
|
|
text/__pycache__/japanese_bert.cpython-310.pyc
ADDED
Binary file (1.32 kB). View file
|
|
text/__pycache__/symbols.cpython-310.pyc
ADDED
Binary file (1.48 kB). View file
|
|
text/__pycache__/tone_sandhi.cpython-310.pyc
ADDED
Binary file (13.4 kB). View file
|
|
text/bert_utils.py
ADDED
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from pathlib import Path
|
2 |
+
|
3 |
+
from huggingface_hub import hf_hub_download
|
4 |
+
|
5 |
+
from config import config
|
6 |
+
|
7 |
+
|
8 |
+
MIRROR: str = config.mirror
|
9 |
+
|
10 |
+
|
11 |
+
def _check_bert(repo_id, files, local_path):
|
12 |
+
for file in files:
|
13 |
+
if not Path(local_path).joinpath(file).exists():
|
14 |
+
if MIRROR.lower() == "openi":
|
15 |
+
import openi
|
16 |
+
|
17 |
+
openi.model.download_model(
|
18 |
+
"Stardust_minus/Bert-VITS2", repo_id.split("/")[-1], "./bert"
|
19 |
+
)
|
20 |
+
else:
|
21 |
+
hf_hub_download(
|
22 |
+
repo_id, file, local_dir=local_path, local_dir_use_symlinks=False
|
23 |
+
)
|
text/chinese.py
ADDED
@@ -0,0 +1,199 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
import re
|
3 |
+
|
4 |
+
import cn2an
|
5 |
+
from pypinyin import lazy_pinyin, Style
|
6 |
+
|
7 |
+
from text.symbols import punctuation
|
8 |
+
from text.tone_sandhi import ToneSandhi
|
9 |
+
|
10 |
+
current_file_path = os.path.dirname(__file__)
|
11 |
+
pinyin_to_symbol_map = {
|
12 |
+
line.split("\t")[0]: line.strip().split("\t")[1]
|
13 |
+
for line in open(os.path.join(current_file_path, "opencpop-strict.txt")).readlines()
|
14 |
+
}
|
15 |
+
|
16 |
+
import jieba.posseg as psg
|
17 |
+
|
18 |
+
|
19 |
+
rep_map = {
|
20 |
+
":": ",",
|
21 |
+
";": ",",
|
22 |
+
",": ",",
|
23 |
+
"。": ".",
|
24 |
+
"!": "!",
|
25 |
+
"?": "?",
|
26 |
+
"\n": ".",
|
27 |
+
"·": ",",
|
28 |
+
"、": ",",
|
29 |
+
"...": "…",
|
30 |
+
"$": ".",
|
31 |
+
"“": "'",
|
32 |
+
"”": "'",
|
33 |
+
'"': "'",
|
34 |
+
"‘": "'",
|
35 |
+
"’": "'",
|
36 |
+
"(": "'",
|
37 |
+
")": "'",
|
38 |
+
"(": "'",
|
39 |
+
")": "'",
|
40 |
+
"《": "'",
|
41 |
+
"》": "'",
|
42 |
+
"【": "'",
|
43 |
+
"】": "'",
|
44 |
+
"[": "'",
|
45 |
+
"]": "'",
|
46 |
+
"—": "-",
|
47 |
+
"~": "-",
|
48 |
+
"~": "-",
|
49 |
+
"「": "'",
|
50 |
+
"」": "'",
|
51 |
+
}
|
52 |
+
|
53 |
+
tone_modifier = ToneSandhi()
|
54 |
+
|
55 |
+
|
56 |
+
def replace_punctuation(text):
|
57 |
+
text = text.replace("嗯", "恩").replace("呣", "母")
|
58 |
+
pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
|
59 |
+
|
60 |
+
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
|
61 |
+
|
62 |
+
replaced_text = re.sub(
|
63 |
+
r"[^\u4e00-\u9fa5" + "".join(punctuation) + r"]+", "", replaced_text
|
64 |
+
)
|
65 |
+
|
66 |
+
return replaced_text
|
67 |
+
|
68 |
+
|
69 |
+
def g2p(text):
|
70 |
+
pattern = r"(?<=[{0}])\s*".format("".join(punctuation))
|
71 |
+
sentences = [i for i in re.split(pattern, text) if i.strip() != ""]
|
72 |
+
phones, tones, word2ph = _g2p(sentences)
|
73 |
+
assert sum(word2ph) == len(phones)
|
74 |
+
assert len(word2ph) == len(text) # Sometimes it will crash,you can add a try-catch.
|
75 |
+
phones = ["_"] + phones + ["_"]
|
76 |
+
tones = [0] + tones + [0]
|
77 |
+
word2ph = [1] + word2ph + [1]
|
78 |
+
return phones, tones, word2ph
|
79 |
+
|
80 |
+
|
81 |
+
def _get_initials_finals(word):
|
82 |
+
initials = []
|
83 |
+
finals = []
|
84 |
+
orig_initials = lazy_pinyin(word, neutral_tone_with_five=True, style=Style.INITIALS)
|
85 |
+
orig_finals = lazy_pinyin(
|
86 |
+
word, neutral_tone_with_five=True, style=Style.FINALS_TONE3
|
87 |
+
)
|
88 |
+
for c, v in zip(orig_initials, orig_finals):
|
89 |
+
initials.append(c)
|
90 |
+
finals.append(v)
|
91 |
+
return initials, finals
|
92 |
+
|
93 |
+
|
94 |
+
def _g2p(segments):
|
95 |
+
phones_list = []
|
96 |
+
tones_list = []
|
97 |
+
word2ph = []
|
98 |
+
for seg in segments:
|
99 |
+
# Replace all English words in the sentence
|
100 |
+
seg = re.sub("[a-zA-Z]+", "", seg)
|
101 |
+
seg_cut = psg.lcut(seg)
|
102 |
+
initials = []
|
103 |
+
finals = []
|
104 |
+
seg_cut = tone_modifier.pre_merge_for_modify(seg_cut)
|
105 |
+
for word, pos in seg_cut:
|
106 |
+
if pos == "eng":
|
107 |
+
continue
|
108 |
+
sub_initials, sub_finals = _get_initials_finals(word)
|
109 |
+
sub_finals = tone_modifier.modified_tone(word, pos, sub_finals)
|
110 |
+
initials.append(sub_initials)
|
111 |
+
finals.append(sub_finals)
|
112 |
+
|
113 |
+
# assert len(sub_initials) == len(sub_finals) == len(word)
|
114 |
+
initials = sum(initials, [])
|
115 |
+
finals = sum(finals, [])
|
116 |
+
#
|
117 |
+
for c, v in zip(initials, finals):
|
118 |
+
raw_pinyin = c + v
|
119 |
+
# NOTE: post process for pypinyin outputs
|
120 |
+
# we discriminate i, ii and iii
|
121 |
+
if c == v:
|
122 |
+
assert c in punctuation
|
123 |
+
phone = [c]
|
124 |
+
tone = "0"
|
125 |
+
word2ph.append(1)
|
126 |
+
else:
|
127 |
+
v_without_tone = v[:-1]
|
128 |
+
tone = v[-1]
|
129 |
+
|
130 |
+
pinyin = c + v_without_tone
|
131 |
+
assert tone in "12345"
|
132 |
+
|
133 |
+
if c:
|
134 |
+
# 多音节
|
135 |
+
v_rep_map = {
|
136 |
+
"uei": "ui",
|
137 |
+
"iou": "iu",
|
138 |
+
"uen": "un",
|
139 |
+
}
|
140 |
+
if v_without_tone in v_rep_map.keys():
|
141 |
+
pinyin = c + v_rep_map[v_without_tone]
|
142 |
+
else:
|
143 |
+
# 单音节
|
144 |
+
pinyin_rep_map = {
|
145 |
+
"ing": "ying",
|
146 |
+
"i": "yi",
|
147 |
+
"in": "yin",
|
148 |
+
"u": "wu",
|
149 |
+
}
|
150 |
+
if pinyin in pinyin_rep_map.keys():
|
151 |
+
pinyin = pinyin_rep_map[pinyin]
|
152 |
+
else:
|
153 |
+
single_rep_map = {
|
154 |
+
"v": "yu",
|
155 |
+
"e": "e",
|
156 |
+
"i": "y",
|
157 |
+
"u": "w",
|
158 |
+
}
|
159 |
+
if pinyin[0] in single_rep_map.keys():
|
160 |
+
pinyin = single_rep_map[pinyin[0]] + pinyin[1:]
|
161 |
+
|
162 |
+
assert pinyin in pinyin_to_symbol_map.keys(), (pinyin, seg, raw_pinyin)
|
163 |
+
phone = pinyin_to_symbol_map[pinyin].split(" ")
|
164 |
+
word2ph.append(len(phone))
|
165 |
+
|
166 |
+
phones_list += phone
|
167 |
+
tones_list += [int(tone)] * len(phone)
|
168 |
+
return phones_list, tones_list, word2ph
|
169 |
+
|
170 |
+
|
171 |
+
def text_normalize(text):
|
172 |
+
numbers = re.findall(r"\d+(?:\.?\d+)?", text)
|
173 |
+
for number in numbers:
|
174 |
+
text = text.replace(number, cn2an.an2cn(number), 1)
|
175 |
+
text = replace_punctuation(text)
|
176 |
+
return text
|
177 |
+
|
178 |
+
|
179 |
+
def get_bert_feature(text, word2ph):
|
180 |
+
from text import chinese_bert
|
181 |
+
|
182 |
+
return chinese_bert.get_bert_feature(text, word2ph)
|
183 |
+
|
184 |
+
|
185 |
+
if __name__ == "__main__":
|
186 |
+
from text.chinese_bert import get_bert_feature
|
187 |
+
|
188 |
+
text = "啊!但是《原神》是由,米哈\游自主, [研发]的一款全.新开放世界.冒险游戏"
|
189 |
+
text = text_normalize(text)
|
190 |
+
print(text)
|
191 |
+
phones, tones, word2ph = g2p(text)
|
192 |
+
bert = get_bert_feature(text, word2ph)
|
193 |
+
|
194 |
+
print(phones, tones, word2ph, bert.shape)
|
195 |
+
|
196 |
+
|
197 |
+
# # 示例用法
|
198 |
+
# text = "这是一个示例文本:,你好!这是一个测试...."
|
199 |
+
# print(g2p_paddle(text)) # 输出: 这是一个示例文本你好这是一个测试
|
text/chinese_bert.py
ADDED
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import sys
|
2 |
+
|
3 |
+
import torch
|
4 |
+
from transformers import AutoModelForMaskedLM, AutoTokenizer
|
5 |
+
|
6 |
+
from config import config
|
7 |
+
|
8 |
+
#LOCAL_PATH = "./bert/chinese-roberta-wwm-ext-large"
|
9 |
+
LOCAL_PATH = 'hfl/chinese-roberta-wwm-ext-large'
|
10 |
+
|
11 |
+
tokenizer = AutoTokenizer.from_pretrained(LOCAL_PATH)
|
12 |
+
|
13 |
+
models = dict()
|
14 |
+
|
15 |
+
|
16 |
+
def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
|
17 |
+
if (
|
18 |
+
sys.platform == "darwin"
|
19 |
+
and torch.backends.mps.is_available()
|
20 |
+
and device == "cpu"
|
21 |
+
):
|
22 |
+
device = "mps"
|
23 |
+
if not device:
|
24 |
+
device = "cuda"
|
25 |
+
if device not in models.keys():
|
26 |
+
models[device] = AutoModelForMaskedLM.from_pretrained(LOCAL_PATH).to(device)
|
27 |
+
with torch.no_grad():
|
28 |
+
inputs = tokenizer(text, return_tensors="pt")
|
29 |
+
for i in inputs:
|
30 |
+
inputs[i] = inputs[i].to(device)
|
31 |
+
res = models[device](**inputs, output_hidden_states=True)
|
32 |
+
res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
|
33 |
+
|
34 |
+
assert len(word2ph) == len(text) + 2
|
35 |
+
word2phone = word2ph
|
36 |
+
phone_level_feature = []
|
37 |
+
for i in range(len(word2phone)):
|
38 |
+
repeat_feature = res[i].repeat(word2phone[i], 1)
|
39 |
+
phone_level_feature.append(repeat_feature)
|
40 |
+
|
41 |
+
phone_level_feature = torch.cat(phone_level_feature, dim=0)
|
42 |
+
|
43 |
+
return phone_level_feature.T
|
44 |
+
|
45 |
+
|
46 |
+
if __name__ == "__main__":
|
47 |
+
word_level_feature = torch.rand(38, 1024) # 12个词,每个词1024维特征
|
48 |
+
word2phone = [
|
49 |
+
1,
|
50 |
+
2,
|
51 |
+
1,
|
52 |
+
2,
|
53 |
+
2,
|
54 |
+
1,
|
55 |
+
2,
|
56 |
+
2,
|
57 |
+
1,
|
58 |
+
2,
|
59 |
+
2,
|
60 |
+
1,
|
61 |
+
2,
|
62 |
+
2,
|
63 |
+
2,
|
64 |
+
2,
|
65 |
+
2,
|
66 |
+
1,
|
67 |
+
1,
|
68 |
+
2,
|
69 |
+
2,
|
70 |
+
1,
|
71 |
+
2,
|
72 |
+
2,
|
73 |
+
2,
|
74 |
+
2,
|
75 |
+
1,
|
76 |
+
2,
|
77 |
+
2,
|
78 |
+
2,
|
79 |
+
2,
|
80 |
+
2,
|
81 |
+
1,
|
82 |
+
2,
|
83 |
+
2,
|
84 |
+
2,
|
85 |
+
2,
|
86 |
+
1,
|
87 |
+
]
|
88 |
+
|
89 |
+
# 计算总帧数
|
90 |
+
total_frames = sum(word2phone)
|
91 |
+
print(word_level_feature.shape)
|
92 |
+
print(word2phone)
|
93 |
+
phone_level_feature = []
|
94 |
+
for i in range(len(word2phone)):
|
95 |
+
print(word_level_feature[i].shape)
|
96 |
+
|
97 |
+
# 对每个词重复word2phone[i]次
|
98 |
+
repeat_feature = word_level_feature[i].repeat(word2phone[i], 1)
|
99 |
+
phone_level_feature.append(repeat_feature)
|
100 |
+
|
101 |
+
phone_level_feature = torch.cat(phone_level_feature, dim=0)
|
102 |
+
print(phone_level_feature.shape) # torch.Size([36, 1024])
|
text/cleaner.py
ADDED
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from text import chinese, japanese, english, cleaned_text_to_sequence
|
2 |
+
|
3 |
+
|
4 |
+
language_module_map = {"ZH": chinese, "JP": japanese, "EN": english}
|
5 |
+
|
6 |
+
|
7 |
+
def clean_text(text, language):
|
8 |
+
language_module = language_module_map[language]
|
9 |
+
norm_text = language_module.text_normalize(text)
|
10 |
+
phones, tones, word2ph = language_module.g2p(norm_text)
|
11 |
+
return norm_text, phones, tones, word2ph
|
12 |
+
|
13 |
+
|
14 |
+
def clean_text_bert(text, language):
|
15 |
+
language_module = language_module_map[language]
|
16 |
+
norm_text = language_module.text_normalize(text)
|
17 |
+
phones, tones, word2ph = language_module.g2p(norm_text)
|
18 |
+
bert = language_module.get_bert_feature(norm_text, word2ph)
|
19 |
+
return phones, tones, bert
|
20 |
+
|
21 |
+
|
22 |
+
def text_to_sequence(text, language):
|
23 |
+
norm_text, phones, tones, word2ph = clean_text(text, language)
|
24 |
+
return cleaned_text_to_sequence(phones, tones, language)
|
25 |
+
|
26 |
+
|
27 |
+
if __name__ == "__main__":
|
28 |
+
pass
|
text/cmudict.rep
ADDED
The diff for this file is too large to render.
See raw diff
|
|
text/cmudict_cache.pickle
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b9b21b20325471934ba92f2e4a5976989e7d920caa32e7a286eacb027d197949
|
3 |
+
size 6212655
|
text/english.py
ADDED
@@ -0,0 +1,454 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import pickle
|
2 |
+
import os
|
3 |
+
import re
|
4 |
+
from g2p_en import G2p
|
5 |
+
from transformers import DebertaV2Tokenizer
|
6 |
+
|
7 |
+
from text import symbols
|
8 |
+
|
9 |
+
current_file_path = os.path.dirname(__file__)
|
10 |
+
CMU_DICT_PATH = os.path.join(current_file_path, "cmudict.rep")
|
11 |
+
CACHE_PATH = os.path.join(current_file_path, "cmudict_cache.pickle")
|
12 |
+
_g2p = G2p()
|
13 |
+
#LOCAL_PATH = "./bert/deberta-v3-large"
|
14 |
+
LOCAL_PATH = 'microsoft/deberta-v3-large'
|
15 |
+
tokenizer = DebertaV2Tokenizer.from_pretrained(LOCAL_PATH)
|
16 |
+
|
17 |
+
arpa = {
|
18 |
+
"AH0",
|
19 |
+
"S",
|
20 |
+
"AH1",
|
21 |
+
"EY2",
|
22 |
+
"AE2",
|
23 |
+
"EH0",
|
24 |
+
"OW2",
|
25 |
+
"UH0",
|
26 |
+
"NG",
|
27 |
+
"B",
|
28 |
+
"G",
|
29 |
+
"AY0",
|
30 |
+
"M",
|
31 |
+
"AA0",
|
32 |
+
"F",
|
33 |
+
"AO0",
|
34 |
+
"ER2",
|
35 |
+
"UH1",
|
36 |
+
"IY1",
|
37 |
+
"AH2",
|
38 |
+
"DH",
|
39 |
+
"IY0",
|
40 |
+
"EY1",
|
41 |
+
"IH0",
|
42 |
+
"K",
|
43 |
+
"N",
|
44 |
+
"W",
|
45 |
+
"IY2",
|
46 |
+
"T",
|
47 |
+
"AA1",
|
48 |
+
"ER1",
|
49 |
+
"EH2",
|
50 |
+
"OY0",
|
51 |
+
"UH2",
|
52 |
+
"UW1",
|
53 |
+
"Z",
|
54 |
+
"AW2",
|
55 |
+
"AW1",
|
56 |
+
"V",
|
57 |
+
"UW2",
|
58 |
+
"AA2",
|
59 |
+
"ER",
|
60 |
+
"AW0",
|
61 |
+
"UW0",
|
62 |
+
"R",
|
63 |
+
"OW1",
|
64 |
+
"EH1",
|
65 |
+
"ZH",
|
66 |
+
"AE0",
|
67 |
+
"IH2",
|
68 |
+
"IH",
|
69 |
+
"Y",
|
70 |
+
"JH",
|
71 |
+
"P",
|
72 |
+
"AY1",
|
73 |
+
"EY0",
|
74 |
+
"OY2",
|
75 |
+
"TH",
|
76 |
+
"HH",
|
77 |
+
"D",
|
78 |
+
"ER0",
|
79 |
+
"CH",
|
80 |
+
"AO1",
|
81 |
+
"AE1",
|
82 |
+
"AO2",
|
83 |
+
"OY1",
|
84 |
+
"AY2",
|
85 |
+
"IH1",
|
86 |
+
"OW0",
|
87 |
+
"L",
|
88 |
+
"SH",
|
89 |
+
}
|
90 |
+
|
91 |
+
|
92 |
+
def post_replace_ph(ph):
|
93 |
+
rep_map = {
|
94 |
+
":": ",",
|
95 |
+
";": ",",
|
96 |
+
",": ",",
|
97 |
+
"。": ".",
|
98 |
+
"!": "!",
|
99 |
+
"?": "?",
|
100 |
+
"\n": ".",
|
101 |
+
"·": ",",
|
102 |
+
"、": ",",
|
103 |
+
"…": "...",
|
104 |
+
"···": "...",
|
105 |
+
"・・・": "...",
|
106 |
+
"v": "V",
|
107 |
+
}
|
108 |
+
if ph in rep_map.keys():
|
109 |
+
ph = rep_map[ph]
|
110 |
+
if ph in symbols:
|
111 |
+
return ph
|
112 |
+
if ph not in symbols:
|
113 |
+
ph = "UNK"
|
114 |
+
return ph
|
115 |
+
|
116 |
+
|
117 |
+
rep_map = {
|
118 |
+
":": ",",
|
119 |
+
";": ",",
|
120 |
+
",": ",",
|
121 |
+
"。": ".",
|
122 |
+
"!": "!",
|
123 |
+
"?": "?",
|
124 |
+
"\n": ".",
|
125 |
+
".": ".",
|
126 |
+
"…": "...",
|
127 |
+
"···": "...",
|
128 |
+
"・・・": "...",
|
129 |
+
"·": ",",
|
130 |
+
"・": ",",
|
131 |
+
"、": ",",
|
132 |
+
"$": ".",
|
133 |
+
"“": "'",
|
134 |
+
"”": "'",
|
135 |
+
'"': "'",
|
136 |
+
"‘": "'",
|
137 |
+
"’": "'",
|
138 |
+
"(": "'",
|
139 |
+
")": "'",
|
140 |
+
"(": "'",
|
141 |
+
")": "'",
|
142 |
+
"《": "'",
|
143 |
+
"》": "'",
|
144 |
+
"【": "'",
|
145 |
+
"】": "'",
|
146 |
+
"[": "'",
|
147 |
+
"]": "'",
|
148 |
+
"—": "-",
|
149 |
+
"−": "-",
|
150 |
+
"~": "-",
|
151 |
+
"~": "-",
|
152 |
+
"「": "'",
|
153 |
+
"」": "'",
|
154 |
+
}
|
155 |
+
|
156 |
+
|
157 |
+
def replace_punctuation(text):
|
158 |
+
pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
|
159 |
+
|
160 |
+
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
|
161 |
+
|
162 |
+
# replaced_text = re.sub(
|
163 |
+
# r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005"
|
164 |
+
# + "".join(punctuation)
|
165 |
+
# + r"]+",
|
166 |
+
# "",
|
167 |
+
# replaced_text,
|
168 |
+
# )
|
169 |
+
|
170 |
+
return replaced_text
|
171 |
+
|
172 |
+
|
173 |
+
def read_dict():
|
174 |
+
g2p_dict = {}
|
175 |
+
start_line = 49
|
176 |
+
with open(CMU_DICT_PATH) as f:
|
177 |
+
line = f.readline()
|
178 |
+
line_index = 1
|
179 |
+
while line:
|
180 |
+
if line_index >= start_line:
|
181 |
+
line = line.strip()
|
182 |
+
word_split = line.split(" ")
|
183 |
+
word = word_split[0]
|
184 |
+
|
185 |
+
syllable_split = word_split[1].split(" - ")
|
186 |
+
g2p_dict[word] = []
|
187 |
+
for syllable in syllable_split:
|
188 |
+
phone_split = syllable.split(" ")
|
189 |
+
g2p_dict[word].append(phone_split)
|
190 |
+
|
191 |
+
line_index = line_index + 1
|
192 |
+
line = f.readline()
|
193 |
+
|
194 |
+
return g2p_dict
|
195 |
+
|
196 |
+
|
197 |
+
def cache_dict(g2p_dict, file_path):
|
198 |
+
with open(file_path, "wb") as pickle_file:
|
199 |
+
pickle.dump(g2p_dict, pickle_file)
|
200 |
+
|
201 |
+
|
202 |
+
def get_dict():
|
203 |
+
if os.path.exists(CACHE_PATH):
|
204 |
+
with open(CACHE_PATH, "rb") as pickle_file:
|
205 |
+
g2p_dict = pickle.load(pickle_file)
|
206 |
+
else:
|
207 |
+
g2p_dict = read_dict()
|
208 |
+
cache_dict(g2p_dict, CACHE_PATH)
|
209 |
+
|
210 |
+
return g2p_dict
|
211 |
+
|
212 |
+
|
213 |
+
eng_dict = get_dict()
|
214 |
+
|
215 |
+
|
216 |
+
def refine_ph(phn):
|
217 |
+
tone = 0
|
218 |
+
if re.search(r"\d$", phn):
|
219 |
+
tone = int(phn[-1]) + 1
|
220 |
+
phn = phn[:-1]
|
221 |
+
return phn.lower(), tone
|
222 |
+
|
223 |
+
|
224 |
+
def refine_syllables(syllables):
|
225 |
+
tones = []
|
226 |
+
phonemes = []
|
227 |
+
for phn_list in syllables:
|
228 |
+
for i in range(len(phn_list)):
|
229 |
+
phn = phn_list[i]
|
230 |
+
phn, tone = refine_ph(phn)
|
231 |
+
phonemes.append(phn)
|
232 |
+
tones.append(tone)
|
233 |
+
return phonemes, tones
|
234 |
+
|
235 |
+
|
236 |
+
import re
|
237 |
+
import inflect
|
238 |
+
|
239 |
+
_inflect = inflect.engine()
|
240 |
+
_comma_number_re = re.compile(r"([0-9][0-9\,]+[0-9])")
|
241 |
+
_decimal_number_re = re.compile(r"([0-9]+\.[0-9]+)")
|
242 |
+
_pounds_re = re.compile(r"£([0-9\,]*[0-9]+)")
|
243 |
+
_dollars_re = re.compile(r"\$([0-9\.\,]*[0-9]+)")
|
244 |
+
_ordinal_re = re.compile(r"[0-9]+(st|nd|rd|th)")
|
245 |
+
_number_re = re.compile(r"[0-9]+")
|
246 |
+
|
247 |
+
# List of (regular expression, replacement) pairs for abbreviations:
|
248 |
+
_abbreviations = [
|
249 |
+
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1])
|
250 |
+
for x in [
|
251 |
+
("mrs", "misess"),
|
252 |
+
("mr", "mister"),
|
253 |
+
("dr", "doctor"),
|
254 |
+
("st", "saint"),
|
255 |
+
("co", "company"),
|
256 |
+
("jr", "junior"),
|
257 |
+
("maj", "major"),
|
258 |
+
("gen", "general"),
|
259 |
+
("drs", "doctors"),
|
260 |
+
("rev", "reverend"),
|
261 |
+
("lt", "lieutenant"),
|
262 |
+
("hon", "honorable"),
|
263 |
+
("sgt", "sergeant"),
|
264 |
+
("capt", "captain"),
|
265 |
+
("esq", "esquire"),
|
266 |
+
("ltd", "limited"),
|
267 |
+
("col", "colonel"),
|
268 |
+
("ft", "fort"),
|
269 |
+
]
|
270 |
+
]
|
271 |
+
|
272 |
+
|
273 |
+
# List of (ipa, lazy ipa) pairs:
|
274 |
+
_lazy_ipa = [
|
275 |
+
(re.compile("%s" % x[0]), x[1])
|
276 |
+
for x in [
|
277 |
+
("r", "ɹ"),
|
278 |
+
("æ", "e"),
|
279 |
+
("ɑ", "a"),
|
280 |
+
("ɔ", "o"),
|
281 |
+
("ð", "z"),
|
282 |
+
("θ", "s"),
|
283 |
+
("ɛ", "e"),
|
284 |
+
("ɪ", "i"),
|
285 |
+
("ʊ", "u"),
|
286 |
+
("ʒ", "ʥ"),
|
287 |
+
("ʤ", "ʥ"),
|
288 |
+
("ˈ", "↓"),
|
289 |
+
]
|
290 |
+
]
|
291 |
+
|
292 |
+
# List of (ipa, lazy ipa2) pairs:
|
293 |
+
_lazy_ipa2 = [
|
294 |
+
(re.compile("%s" % x[0]), x[1])
|
295 |
+
for x in [
|
296 |
+
("r", "ɹ"),
|
297 |
+
("ð", "z"),
|
298 |
+
("θ", "s"),
|
299 |
+
("ʒ", "ʑ"),
|
300 |
+
("ʤ", "dʑ"),
|
301 |
+
("ˈ", "↓"),
|
302 |
+
]
|
303 |
+
]
|
304 |
+
|
305 |
+
# List of (ipa, ipa2) pairs
|
306 |
+
_ipa_to_ipa2 = [
|
307 |
+
(re.compile("%s" % x[0]), x[1]) for x in [("r", "ɹ"), ("ʤ", "dʒ"), ("ʧ", "tʃ")]
|
308 |
+
]
|
309 |
+
|
310 |
+
|
311 |
+
def _expand_dollars(m):
|
312 |
+
match = m.group(1)
|
313 |
+
parts = match.split(".")
|
314 |
+
if len(parts) > 2:
|
315 |
+
return match + " dollars" # Unexpected format
|
316 |
+
dollars = int(parts[0]) if parts[0] else 0
|
317 |
+
cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
|
318 |
+
if dollars and cents:
|
319 |
+
dollar_unit = "dollar" if dollars == 1 else "dollars"
|
320 |
+
cent_unit = "cent" if cents == 1 else "cents"
|
321 |
+
return "%s %s, %s %s" % (dollars, dollar_unit, cents, cent_unit)
|
322 |
+
elif dollars:
|
323 |
+
dollar_unit = "dollar" if dollars == 1 else "dollars"
|
324 |
+
return "%s %s" % (dollars, dollar_unit)
|
325 |
+
elif cents:
|
326 |
+
cent_unit = "cent" if cents == 1 else "cents"
|
327 |
+
return "%s %s" % (cents, cent_unit)
|
328 |
+
else:
|
329 |
+
return "zero dollars"
|
330 |
+
|
331 |
+
|
332 |
+
def _remove_commas(m):
|
333 |
+
return m.group(1).replace(",", "")
|
334 |
+
|
335 |
+
|
336 |
+
def _expand_ordinal(m):
|
337 |
+
return _inflect.number_to_words(m.group(0))
|
338 |
+
|
339 |
+
|
340 |
+
def _expand_number(m):
|
341 |
+
num = int(m.group(0))
|
342 |
+
if num > 1000 and num < 3000:
|
343 |
+
if num == 2000:
|
344 |
+
return "two thousand"
|
345 |
+
elif num > 2000 and num < 2010:
|
346 |
+
return "two thousand " + _inflect.number_to_words(num % 100)
|
347 |
+
elif num % 100 == 0:
|
348 |
+
return _inflect.number_to_words(num // 100) + " hundred"
|
349 |
+
else:
|
350 |
+
return _inflect.number_to_words(
|
351 |
+
num, andword="", zero="oh", group=2
|
352 |
+
).replace(", ", " ")
|
353 |
+
else:
|
354 |
+
return _inflect.number_to_words(num, andword="")
|
355 |
+
|
356 |
+
|
357 |
+
def _expand_decimal_point(m):
|
358 |
+
return m.group(1).replace(".", " point ")
|
359 |
+
|
360 |
+
|
361 |
+
def normalize_numbers(text):
|
362 |
+
text = re.sub(_comma_number_re, _remove_commas, text)
|
363 |
+
text = re.sub(_pounds_re, r"\1 pounds", text)
|
364 |
+
text = re.sub(_dollars_re, _expand_dollars, text)
|
365 |
+
text = re.sub(_decimal_number_re, _expand_decimal_point, text)
|
366 |
+
text = re.sub(_ordinal_re, _expand_ordinal, text)
|
367 |
+
text = re.sub(_number_re, _expand_number, text)
|
368 |
+
return text
|
369 |
+
|
370 |
+
|
371 |
+
def text_normalize(text):
|
372 |
+
text = normalize_numbers(text)
|
373 |
+
text = replace_punctuation(text)
|
374 |
+
text = re.sub(r"([,;.\?\!])([\w])", r"\1 \2", text)
|
375 |
+
return text
|
376 |
+
|
377 |
+
|
378 |
+
def distribute_phone(n_phone, n_word):
|
379 |
+
phones_per_word = [0] * n_word
|
380 |
+
for task in range(n_phone):
|
381 |
+
min_tasks = min(phones_per_word)
|
382 |
+
min_index = phones_per_word.index(min_tasks)
|
383 |
+
phones_per_word[min_index] += 1
|
384 |
+
return phones_per_word
|
385 |
+
|
386 |
+
|
387 |
+
def sep_text(text):
|
388 |
+
words = re.split(r"([,;.\?\!\s+])", text)
|
389 |
+
words = [word for word in words if word.strip() != ""]
|
390 |
+
return words
|
391 |
+
|
392 |
+
|
393 |
+
def g2p(text):
|
394 |
+
phones = []
|
395 |
+
tones = []
|
396 |
+
# word2ph = []
|
397 |
+
words = sep_text(text)
|
398 |
+
tokens = [tokenizer.tokenize(i) for i in words]
|
399 |
+
for word in words:
|
400 |
+
if word.upper() in eng_dict:
|
401 |
+
phns, tns = refine_syllables(eng_dict[word.upper()])
|
402 |
+
phones.append([post_replace_ph(i) for i in phns])
|
403 |
+
tones.append(tns)
|
404 |
+
# word2ph.append(len(phns))
|
405 |
+
else:
|
406 |
+
phone_list = list(filter(lambda p: p != " ", _g2p(word)))
|
407 |
+
phns = []
|
408 |
+
tns = []
|
409 |
+
for ph in phone_list:
|
410 |
+
if ph in arpa:
|
411 |
+
ph, tn = refine_ph(ph)
|
412 |
+
phns.append(ph)
|
413 |
+
tns.append(tn)
|
414 |
+
else:
|
415 |
+
phns.append(ph)
|
416 |
+
tns.append(0)
|
417 |
+
phones.append([post_replace_ph(i) for i in phns])
|
418 |
+
tones.append(tns)
|
419 |
+
# word2ph.append(len(phns))
|
420 |
+
# phones = [post_replace_ph(i) for i in phones]
|
421 |
+
|
422 |
+
word2ph = []
|
423 |
+
for token, phoneme in zip(tokens, phones):
|
424 |
+
phone_len = len(phoneme)
|
425 |
+
word_len = len(token)
|
426 |
+
|
427 |
+
aaa = distribute_phone(phone_len, word_len)
|
428 |
+
word2ph += aaa
|
429 |
+
|
430 |
+
phones = ["_"] + [j for i in phones for j in i] + ["_"]
|
431 |
+
tones = [0] + [j for i in tones for j in i] + [0]
|
432 |
+
word2ph = [1] + word2ph + [1]
|
433 |
+
assert len(phones) == len(tones), text
|
434 |
+
assert len(phones) == sum(word2ph), text
|
435 |
+
|
436 |
+
return phones, tones, word2ph
|
437 |
+
|
438 |
+
|
439 |
+
def get_bert_feature(text, word2ph):
|
440 |
+
from text import english_bert_mock
|
441 |
+
|
442 |
+
return english_bert_mock.get_bert_feature(text, word2ph)
|
443 |
+
|
444 |
+
|
445 |
+
if __name__ == "__main__":
|
446 |
+
# print(get_dict())
|
447 |
+
# print(eng_word_to_phoneme("hello"))
|
448 |
+
print(g2p("In this paper, we propose 1 DSPGAN, a GAN-based universal vocoder."))
|
449 |
+
# all_phones = set()
|
450 |
+
# for k, syllables in eng_dict.items():
|
451 |
+
# for group in syllables:
|
452 |
+
# for ph in group:
|
453 |
+
# all_phones.add(ph)
|
454 |
+
# print(all_phones)
|
text/english_bert_mock.py
ADDED
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import sys
|
2 |
+
|
3 |
+
import torch
|
4 |
+
from transformers import DebertaV2Model, DebertaV2Tokenizer
|
5 |
+
|
6 |
+
from config import config
|
7 |
+
|
8 |
+
|
9 |
+
#LOCAL_PATH = "./bert/deberta-v3-large"
|
10 |
+
LOCAL_PATH = 'microsoft/deberta-v2-xlarge'
|
11 |
+
|
12 |
+
tokenizer = DebertaV2Tokenizer.from_pretrained(LOCAL_PATH)
|
13 |
+
|
14 |
+
models = dict()
|
15 |
+
|
16 |
+
|
17 |
+
def get_bert_feature(text, word2ph, device=config.bert_gen_config.device):
|
18 |
+
if (
|
19 |
+
sys.platform == "darwin"
|
20 |
+
and torch.backends.mps.is_available()
|
21 |
+
and device == "cpu"
|
22 |
+
):
|
23 |
+
device = "mps"
|
24 |
+
if not device:
|
25 |
+
device = "cuda"
|
26 |
+
if device not in models.keys():
|
27 |
+
models[device] = DebertaV2Model.from_pretrained(LOCAL_PATH).to(device)
|
28 |
+
with torch.no_grad():
|
29 |
+
inputs = tokenizer(text, return_tensors="pt")
|
30 |
+
for i in inputs:
|
31 |
+
inputs[i] = inputs[i].to(device)
|
32 |
+
res = models[device](**inputs, output_hidden_states=True)
|
33 |
+
res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()
|
34 |
+
assert len(word2ph) == res.shape[0], (text, res.shape[0], len(word2ph))
|
35 |
+
word2phone = word2ph
|
36 |
+
phone_level_feature = []
|
37 |
+
for i in range(len(word2phone)):
|
38 |
+
repeat_feature = res[i].repeat(word2phone[i], 1)
|
39 |
+
phone_level_feature.append(repeat_feature)
|
40 |
+
|
41 |
+
phone_level_feature = torch.cat(phone_level_feature, dim=0)
|
42 |
+
|
43 |
+
return phone_level_feature.T
|
text/japanese.py
ADDED
@@ -0,0 +1,432 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Convert Japanese text to phonemes which is
|
2 |
+
# compatible with Julius https://github.com/julius-speech/segmentation-kit
|
3 |
+
import re
|
4 |
+
import unicodedata
|
5 |
+
|
6 |
+
from transformers import AutoTokenizer
|
7 |
+
|
8 |
+
from text import punctuation, symbols
|
9 |
+
|
10 |
+
from num2words import num2words
|
11 |
+
|
12 |
+
import pyopenjtalk
|
13 |
+
import jaconv
|
14 |
+
|
15 |
+
|
16 |
+
def kata2phoneme(text: str) -> str:
|
17 |
+
"""Convert katakana text to phonemes."""
|
18 |
+
text = text.strip()
|
19 |
+
if text == "ー":
|
20 |
+
return ["ー"]
|
21 |
+
elif text.startswith("ー"):
|
22 |
+
return ["ー"] + kata2phoneme(text[1:])
|
23 |
+
res = []
|
24 |
+
prev = None
|
25 |
+
while text:
|
26 |
+
if re.match(_MARKS, text):
|
27 |
+
res.append(text)
|
28 |
+
text = text[1:]
|
29 |
+
continue
|
30 |
+
if text.startswith("ー"):
|
31 |
+
if prev:
|
32 |
+
res.append(prev[-1])
|
33 |
+
text = text[1:]
|
34 |
+
continue
|
35 |
+
res += pyopenjtalk.g2p(text).lower().replace("cl", "q").split(" ")
|
36 |
+
break
|
37 |
+
# res = _COLON_RX.sub(":", res)
|
38 |
+
return res
|
39 |
+
|
40 |
+
|
41 |
+
def hira2kata(text: str) -> str:
|
42 |
+
return jaconv.hira2kata(text)
|
43 |
+
|
44 |
+
|
45 |
+
_SYMBOL_TOKENS = set(list("・、。?!"))
|
46 |
+
_NO_YOMI_TOKENS = set(list("「」『』―()[][]"))
|
47 |
+
_MARKS = re.compile(
|
48 |
+
r"[^A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]"
|
49 |
+
)
|
50 |
+
|
51 |
+
|
52 |
+
def text2kata(text: str) -> str:
|
53 |
+
parsed = pyopenjtalk.run_frontend(text)
|
54 |
+
|
55 |
+
res = []
|
56 |
+
for parts in parsed:
|
57 |
+
word, yomi = replace_punctuation(parts["string"]), parts["pron"].replace(
|
58 |
+
"’", ""
|
59 |
+
)
|
60 |
+
if yomi:
|
61 |
+
if re.match(_MARKS, yomi):
|
62 |
+
if len(word) > 1:
|
63 |
+
word = [replace_punctuation(i) for i in list(word)]
|
64 |
+
yomi = word
|
65 |
+
res += yomi
|
66 |
+
sep += word
|
67 |
+
continue
|
68 |
+
elif word not in rep_map.keys() and word not in rep_map.values():
|
69 |
+
word = ","
|
70 |
+
yomi = word
|
71 |
+
res.append(yomi)
|
72 |
+
else:
|
73 |
+
if word in _SYMBOL_TOKENS:
|
74 |
+
res.append(word)
|
75 |
+
elif word in ("っ", "ッ"):
|
76 |
+
res.append("ッ")
|
77 |
+
elif word in _NO_YOMI_TOKENS:
|
78 |
+
pass
|
79 |
+
else:
|
80 |
+
res.append(word)
|
81 |
+
return hira2kata("".join(res))
|
82 |
+
|
83 |
+
|
84 |
+
def text2sep_kata(text: str) -> (list, list):
|
85 |
+
parsed = pyopenjtalk.run_frontend(text)
|
86 |
+
|
87 |
+
res = []
|
88 |
+
sep = []
|
89 |
+
for parts in parsed:
|
90 |
+
word, yomi = replace_punctuation(parts["string"]), parts["pron"].replace(
|
91 |
+
"’", ""
|
92 |
+
)
|
93 |
+
if yomi:
|
94 |
+
if re.match(_MARKS, yomi):
|
95 |
+
if len(word) > 1:
|
96 |
+
word = [replace_punctuation(i) for i in list(word)]
|
97 |
+
yomi = word
|
98 |
+
res += yomi
|
99 |
+
sep += word
|
100 |
+
continue
|
101 |
+
elif word not in rep_map.keys() and word not in rep_map.values():
|
102 |
+
word = ","
|
103 |
+
yomi = word
|
104 |
+
res.append(yomi)
|
105 |
+
else:
|
106 |
+
if word in _SYMBOL_TOKENS:
|
107 |
+
res.append(word)
|
108 |
+
elif word in ("っ", "ッ"):
|
109 |
+
res.append("ッ")
|
110 |
+
elif word in _NO_YOMI_TOKENS:
|
111 |
+
pass
|
112 |
+
else:
|
113 |
+
res.append(word)
|
114 |
+
sep.append(word)
|
115 |
+
return sep, [hira2kata(i) for i in res], get_accent(parsed)
|
116 |
+
|
117 |
+
|
118 |
+
def get_accent(parsed):
|
119 |
+
labels = pyopenjtalk.make_label(parsed)
|
120 |
+
|
121 |
+
phonemes = []
|
122 |
+
accents = []
|
123 |
+
for n, label in enumerate(labels):
|
124 |
+
phoneme = re.search(r"\-([^\+]*)\+", label).group(1)
|
125 |
+
if phoneme not in ["sil", "pau"]:
|
126 |
+
phonemes.append(phoneme.replace("cl", "q").lower())
|
127 |
+
else:
|
128 |
+
continue
|
129 |
+
a1 = int(re.search(r"/A:(\-?[0-9]+)\+", label).group(1))
|
130 |
+
a2 = int(re.search(r"\+(\d+)\+", label).group(1))
|
131 |
+
if re.search(r"\-([^\+]*)\+", labels[n + 1]).group(1) in ["sil", "pau"]:
|
132 |
+
a2_next = -1
|
133 |
+
else:
|
134 |
+
a2_next = int(re.search(r"\+(\d+)\+", labels[n + 1]).group(1))
|
135 |
+
# Falling
|
136 |
+
if a1 == 0 and a2_next == a2 + 1:
|
137 |
+
accents.append(-1)
|
138 |
+
# Rising
|
139 |
+
elif a2 == 1 and a2_next == 2:
|
140 |
+
accents.append(1)
|
141 |
+
else:
|
142 |
+
accents.append(0)
|
143 |
+
return list(zip(phonemes, accents))
|
144 |
+
|
145 |
+
|
146 |
+
_ALPHASYMBOL_YOMI = {
|
147 |
+
"#": "シャープ",
|
148 |
+
"%": "パーセント",
|
149 |
+
"&": "アンド",
|
150 |
+
"+": "プラス",
|
151 |
+
"-": "マイナス",
|
152 |
+
":": "コロン",
|
153 |
+
";": "セミコロン",
|
154 |
+
"<": "小なり",
|
155 |
+
"=": "イコール",
|
156 |
+
">": "大なり",
|
157 |
+
"@": "アット",
|
158 |
+
"a": "エー",
|
159 |
+
"b": "ビー",
|
160 |
+
"c": "シー",
|
161 |
+
"d": "ディー",
|
162 |
+
"e": "イー",
|
163 |
+
"f": "エフ",
|
164 |
+
"g": "ジー",
|
165 |
+
"h": "エイチ",
|
166 |
+
"i": "アイ",
|
167 |
+
"j": "ジェー",
|
168 |
+
"k": "ケー",
|
169 |
+
"l": "エル",
|
170 |
+
"m": "エム",
|
171 |
+
"n": "エヌ",
|
172 |
+
"o": "オー",
|
173 |
+
"p": "ピー",
|
174 |
+
"q": "キュー",
|
175 |
+
"r": "アール",
|
176 |
+
"s": "エス",
|
177 |
+
"t": "ティー",
|
178 |
+
"u": "ユー",
|
179 |
+
"v": "ブイ",
|
180 |
+
"w": "ダブリュー",
|
181 |
+
"x": "エックス",
|
182 |
+
"y": "ワイ",
|
183 |
+
"z": "ゼット",
|
184 |
+
"α": "アルファ",
|
185 |
+
"β": "ベータ",
|
186 |
+
"γ": "ガンマ",
|
187 |
+
"δ": "デルタ",
|
188 |
+
"ε": "イプシロン",
|
189 |
+
"ζ": "ゼータ",
|
190 |
+
"η": "イータ",
|
191 |
+
"θ": "シータ",
|
192 |
+
"ι": "イオタ",
|
193 |
+
"κ": "カッパ",
|
194 |
+
"λ": "ラムダ",
|
195 |
+
"μ": "ミュー",
|
196 |
+
"ν": "ニュー",
|
197 |
+
"ξ": "クサイ",
|
198 |
+
"ο": "オミクロン",
|
199 |
+
"π": "パイ",
|
200 |
+
"ρ": "ロー",
|
201 |
+
"σ": "シグマ",
|
202 |
+
"τ": "タウ",
|
203 |
+
"υ": "ウプシロン",
|
204 |
+
"φ": "ファイ",
|
205 |
+
"χ": "カイ",
|
206 |
+
"ψ": "プサイ",
|
207 |
+
"ω": "オメガ",
|
208 |
+
}
|
209 |
+
|
210 |
+
|
211 |
+
_NUMBER_WITH_SEPARATOR_RX = re.compile("[0-9]{1,3}(,[0-9]{3})+")
|
212 |
+
_CURRENCY_MAP = {"$": "ドル", "¥": "円", "£": "ポンド", "€": "ユーロ"}
|
213 |
+
_CURRENCY_RX = re.compile(r"([$¥£€])([0-9.]*[0-9])")
|
214 |
+
_NUMBER_RX = re.compile(r"[0-9]+(\.[0-9]+)?")
|
215 |
+
|
216 |
+
|
217 |
+
def japanese_convert_numbers_to_words(text: str) -> str:
|
218 |
+
res = _NUMBER_WITH_SEPARATOR_RX.sub(lambda m: m[0].replace(",", ""), text)
|
219 |
+
res = _CURRENCY_RX.sub(lambda m: m[2] + _CURRENCY_MAP.get(m[1], m[1]), res)
|
220 |
+
res = _NUMBER_RX.sub(lambda m: num2words(m[0], lang="ja"), res)
|
221 |
+
return res
|
222 |
+
|
223 |
+
|
224 |
+
def japanese_convert_alpha_symbols_to_words(text: str) -> str:
|
225 |
+
return "".join([_ALPHASYMBOL_YOMI.get(ch, ch) for ch in text.lower()])
|
226 |
+
|
227 |
+
|
228 |
+
def japanese_text_to_phonemes(text: str) -> str:
|
229 |
+
"""Convert Japanese text to phonemes."""
|
230 |
+
res = unicodedata.normalize("NFKC", text)
|
231 |
+
res = japanese_convert_numbers_to_words(res)
|
232 |
+
# res = japanese_convert_alpha_symbols_to_words(res)
|
233 |
+
res = text2kata(res)
|
234 |
+
res = kata2phoneme(res)
|
235 |
+
return res
|
236 |
+
|
237 |
+
|
238 |
+
def is_japanese_character(char):
|
239 |
+
# 定义日语文字系统的 Unicode 范围
|
240 |
+
japanese_ranges = [
|
241 |
+
(0x3040, 0x309F), # 平假名
|
242 |
+
(0x30A0, 0x30FF), # 片假名
|
243 |
+
(0x4E00, 0x9FFF), # 汉字 (CJK Unified Ideographs)
|
244 |
+
(0x3400, 0x4DBF), # 汉字扩展 A
|
245 |
+
(0x20000, 0x2A6DF), # 汉字扩展 B
|
246 |
+
# 可以根据需要添加其他汉字扩展范围
|
247 |
+
]
|
248 |
+
|
249 |
+
# 将字符的 Unicode 编码转换为整数
|
250 |
+
char_code = ord(char)
|
251 |
+
|
252 |
+
# 检查字符是否在任何一个日语范围内
|
253 |
+
for start, end in japanese_ranges:
|
254 |
+
if start <= char_code <= end:
|
255 |
+
return True
|
256 |
+
|
257 |
+
return False
|
258 |
+
|
259 |
+
|
260 |
+
rep_map = {
|
261 |
+
":": ",",
|
262 |
+
";": ",",
|
263 |
+
",": ",",
|
264 |
+
"。": ".",
|
265 |
+
"!": "!",
|
266 |
+
"?": "?",
|
267 |
+
"\n": ".",
|
268 |
+
".": ".",
|
269 |
+
"…": "...",
|
270 |
+
"···": "...",
|
271 |
+
"・・・": "...",
|
272 |
+
"·": ",",
|
273 |
+
"・": ",",
|
274 |
+
"、": ",",
|
275 |
+
"$": ".",
|
276 |
+
"“": "'",
|
277 |
+
"”": "'",
|
278 |
+
'"': "'",
|
279 |
+
"‘": "'",
|
280 |
+
"’": "'",
|
281 |
+
"(": "'",
|
282 |
+
")": "'",
|
283 |
+
"(": "'",
|
284 |
+
")": "'",
|
285 |
+
"《": "'",
|
286 |
+
"》": "'",
|
287 |
+
"【": "'",
|
288 |
+
"】": "'",
|
289 |
+
"[": "'",
|
290 |
+
"]": "'",
|
291 |
+
"—": "-",
|
292 |
+
"−": "-",
|
293 |
+
"~": "-",
|
294 |
+
"~": "-",
|
295 |
+
"「": "'",
|
296 |
+
"」": "'",
|
297 |
+
}
|
298 |
+
|
299 |
+
|
300 |
+
def replace_punctuation(text):
|
301 |
+
pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
|
302 |
+
|
303 |
+
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
|
304 |
+
|
305 |
+
replaced_text = re.sub(
|
306 |
+
r"[^\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\u3400-\u4DBF\u3005"
|
307 |
+
+ "".join(punctuation)
|
308 |
+
+ r"]+",
|
309 |
+
"",
|
310 |
+
replaced_text,
|
311 |
+
)
|
312 |
+
|
313 |
+
return replaced_text
|
314 |
+
|
315 |
+
|
316 |
+
def text_normalize(text):
|
317 |
+
res = unicodedata.normalize("NFKC", text)
|
318 |
+
res = japanese_convert_numbers_to_words(res)
|
319 |
+
# res = "".join([i for i in res if is_japanese_character(i)])
|
320 |
+
res = replace_punctuation(res)
|
321 |
+
res = res.replace("゙", "")
|
322 |
+
return res
|
323 |
+
|
324 |
+
|
325 |
+
def distribute_phone(n_phone, n_word):
|
326 |
+
phones_per_word = [0] * n_word
|
327 |
+
for task in range(n_phone):
|
328 |
+
min_tasks = min(phones_per_word)
|
329 |
+
min_index = phones_per_word.index(min_tasks)
|
330 |
+
phones_per_word[min_index] += 1
|
331 |
+
return phones_per_word
|
332 |
+
|
333 |
+
|
334 |
+
def handle_long(sep_phonemes):
|
335 |
+
for i in range(len(sep_phonemes)):
|
336 |
+
if sep_phonemes[i][0] == "ー":
|
337 |
+
sep_phonemes[i][0] = sep_phonemes[i - 1][-1]
|
338 |
+
if "ー" in sep_phonemes[i]:
|
339 |
+
for j in range(len(sep_phonemes[i])):
|
340 |
+
if sep_phonemes[i][j] == "ー":
|
341 |
+
sep_phonemes[i][j] = sep_phonemes[i][j - 1][-1]
|
342 |
+
return sep_phonemes
|
343 |
+
|
344 |
+
|
345 |
+
#tokenizer = AutoTokenizer.from_pretrained("./bert/deberta-v2-large-japanese-char-wwm")
|
346 |
+
tokenizer = AutoTokenizer.from_pretrained("ku-nlp/deberta-v2-large-japanese-char-wwm")
|
347 |
+
|
348 |
+
def align_tones(phones, tones):
|
349 |
+
res = []
|
350 |
+
for pho in phones:
|
351 |
+
temp = [0] * len(pho)
|
352 |
+
for idx, p in enumerate(pho):
|
353 |
+
if len(tones) == 0:
|
354 |
+
break
|
355 |
+
if p == tones[0][0]:
|
356 |
+
temp[idx] = tones[0][1]
|
357 |
+
if idx > 0:
|
358 |
+
temp[idx] += temp[idx - 1]
|
359 |
+
tones.pop(0)
|
360 |
+
temp = [0] + temp
|
361 |
+
temp = temp[:-1]
|
362 |
+
if -1 in temp:
|
363 |
+
temp = [i + 1 for i in temp]
|
364 |
+
res.append(temp)
|
365 |
+
res = [i for j in res for i in j]
|
366 |
+
assert not any([i < 0 for i in res]) and not any([i > 1 for i in res])
|
367 |
+
return res
|
368 |
+
|
369 |
+
|
370 |
+
def rearrange_tones(tones, phones):
|
371 |
+
res = [0] * len(tones)
|
372 |
+
for i in range(len(tones)):
|
373 |
+
if i == 0:
|
374 |
+
if tones[i] not in punctuation:
|
375 |
+
res[i] = 1
|
376 |
+
elif tones[i] == prev:
|
377 |
+
if phones[i] in punctuation:
|
378 |
+
res[i] = 0
|
379 |
+
else:
|
380 |
+
res[i] = 1
|
381 |
+
elif tones[i] > prev:
|
382 |
+
res[i] = 2
|
383 |
+
elif tones[i] < prev:
|
384 |
+
res[i - 1] = 3
|
385 |
+
res[i] = 1
|
386 |
+
prev = tones[i]
|
387 |
+
return res
|
388 |
+
|
389 |
+
|
390 |
+
def g2p(norm_text):
|
391 |
+
sep_text, sep_kata, acc = text2sep_kata(norm_text)
|
392 |
+
sep_tokenized = []
|
393 |
+
for i in sep_text:
|
394 |
+
if i not in punctuation:
|
395 |
+
sep_tokenized.append(tokenizer.tokenize(i))
|
396 |
+
else:
|
397 |
+
sep_tokenized.append([i])
|
398 |
+
|
399 |
+
sep_phonemes = handle_long([kata2phoneme(i) for i in sep_kata])
|
400 |
+
# 异常处理,MeCab不认识的词的话会一路传到这里来,然后炸掉。目前来看只有那些超级稀有的生僻词会出现这种情况
|
401 |
+
for i in sep_phonemes:
|
402 |
+
for j in i:
|
403 |
+
assert j in symbols, (sep_text, sep_kata, sep_phonemes)
|
404 |
+
tones = align_tones(sep_phonemes, acc)
|
405 |
+
|
406 |
+
word2ph = []
|
407 |
+
for token, phoneme in zip(sep_tokenized, sep_phonemes):
|
408 |
+
phone_len = len(phoneme)
|
409 |
+
word_len = len(token)
|
410 |
+
|
411 |
+
aaa = distribute_phone(phone_len, word_len)
|
412 |
+
word2ph += aaa
|
413 |
+
phones = ["_"] + [j for i in sep_phonemes for j in i] + ["_"]
|
414 |
+
# tones = [0] + rearrange_tones(tones, phones[1:-1]) + [0]
|
415 |
+
tones = [0] + tones + [0]
|
416 |
+
word2ph = [1] + word2ph + [1]
|
417 |
+
assert len(phones) == len(tones)
|
418 |
+
return phones, tones, word2ph
|
419 |
+
|
420 |
+
|
421 |
+
if __name__ == "__main__":
|
422 |
+
tokenizer = AutoTokenizer.from_pretrained("./bert/deberta-v2-large-japanese")
|
423 |
+
text = "hello,こんにちは、世界ー!……"
|
424 |
+
from text.japanese_bert import get_bert_feature
|
425 |
+
|
426 |
+
text = text_normalize(text)
|
427 |
+
print(text)
|
428 |
+
|
429 |
+
phones, tones, word2ph = g2p(text)
|
430 |
+
bert = get_bert_feature(text, word2ph)
|
431 |
+
|
432 |
+
print(phones, tones, word2ph, bert.shape)
|