在完成fastspeech论文学习后,对github上一个复现的仓库进行学习,帮助理解算法实现过程中的一些细节;所选择的仓库复现仓库是基于pytorch实现,链接为https://github.com/xcmyz/FastSpeech。该仓库使用的数据集为LJSpeech,与数据处理相关的主要是audio、data路径下的文件和prepreocess.py文件,本笔记主要针对相关文件进行解读。
audio路径下包含以下四个py文件,前三个是基础功能实现,最后的tools.py中实现主要的实际功能。
该文件是直接从tacotrons项目中复制过来,主要是为了使用其中的griffin_lim()函数通过音频的线性谱图重建音频文件
""" from https://github.com/NVIDIA/tacotron2 """
import torch
import numpy as np
from scipy.signal import get_window
import librosa.util as librosa_util
def window_sumsquare(window, n_frames, hop_length=200, win_length=800,
n_fft=800, dtype=np.float32, norm=None):
"""
# from librosa 0.6
Compute the sum-square envelope of a window function at a given hop length.
This is used to estimate modulation effects induced by windowing
observations in short-time fourier transforms.
Parameters
----------
window : string, tuple, number, callable, or list-like
Window specification, as in `get_window`
n_frames : int > 0
The number of analysis frames
hop_length : int > 0
The number of samples to advance between frames
win_length : [optional]
The length of the window function. By default, this matches `n_fft`.
n_fft : int > 0
The length of each analysis frame.
dtype : np.dtype
The data type of the output
Returns
-------
wss : np.ndarray, shape=`(n_fft + hop_length * (n_frames - 1))`
The sum-squared envelope of the window function
"""
if win_length is None:
win_length = n_fft
n = n_fft + hop_length * (n_frames - 1)
x = np.zeros(n, dtype=dtype)
# Compute the squared window at the desired length
win_sq = get_window(window, win_length, fftbins=True)
win_sq = librosa_util.normalize(win_sq, norm=norm)**2
win_sq = librosa_util.pad_center(win_sq, n_fft)
# Fill the envelope
for i in range(n_frames):
sample = i * hop_length
x[sample:min(n, sample + n_fft)
] += win_sq[:max(0, min(n_fft, n - sample))]
return x
def griffin_lim(magnitudes, stft_fn, n_iters=30):
"""
PARAMS
------
magnitudes: spectrogram magnitudes
stft_fn: STFT class with transform (STFT) and inverse (ISTFT) methods
"""
angles = np.angle(np.exp(2j * np.pi * np.random.rand(*magnitudes.size())))
angles = angles.astype(np.float32)
angles = torch.autograd.Variable(torch.from_numpy(angles))
signal = stft_fn.inverse(magnitudes, angles).squeeze(1)
for i in range(n_iters):
_, angles = stft_fn.transform(signal)
signal = stft_fn.inverse(magnitudes, angles).squeeze(1)
return signal
def dynamic_range_compression(x, C=1, clip_val=1e-5):
"""
PARAMS
------
C: compression factor
"""
return torch.log(torch.clamp(x, min=clip_val) * C)
def dynamic_range_decompression(x, C=1):
"""
PARAMS
------
C: compression factor used to compress
"""
return torch.exp(x) / C
该文件中设置初始化STFT模块的参数,始于音频文件和mel谱图相关的参数值
max_wav_value = 32768.0
sampling_rate = 22050
filter_length = 1024
hop_length = 256
win_length = 1024
n_mel_channels = 80
mel_fmin = 0.0
mel_fmax = 8000.0
此文件也是复制于tacotron2项目,其中将短时傅里叶变换封装成一个torch.module的模块,使用该模块可以从音频数据中提取对应的mel谱图
""" from https://github.com/NVIDIA/tacotron2 """
import torch
import torch.nn.functional as F
from torch.autograd import Variable
import numpy as np
from scipy.signal import get_window
from librosa.util import pad_center, tiny
from librosa.filters import mel as librosa_mel_fn
from audio.audio_processing import dynamic_range_compression
from audio.audio_processing import dynamic_range_decompression
from audio.audio_processing import window_sumsquare
class STFT(torch.nn.Module):
"""adapted from Prem Seetharaman's https://github.com/pseeth/pytorch-stft"""
def __init__(self, filter_length=800, hop_length=200, win_length=800,
window='hann'):
super(STFT, self).__init__()
self.filter_length = filter_length
self.hop_length = hop_length
self.win_length = win_length
self.window = window
self.forward_transform = None
scale = self.filter_length / self.hop_length
fourier_basis = np.fft.fft(np.eye(self.filter_length))
cutoff = int((self.filter_length / 2 + 1))
fourier_basis = np.vstack([np.real(fourier_basis[:cutoff, :]),
np.imag(fourier_basis[:cutoff, :])])
forward_basis = torch.FloatTensor(fourier_basis[:, None, :])
inverse_basis = torch.FloatTensor(
np.linalg.pinv(scale * fourier_basis).T[:, None, :])
if window is not None:
assert(filter_length >= win_length)
# get window and zero center pad it to filter_length
fft_window = get_window(window, win_length, fftbins=True)
fft_window = pad_center(fft_window, filter_length)
fft_window = torch.from_numpy(fft_window).float()
# window the bases
forward_basis *= fft_window
inverse_basis *= fft_window
self.register_buffer('forward_basis', forward_basis.float())
self.register_buffer('inverse_basis', inverse_basis.float())
def transform(self, input_data):
num_batches = input_data.size(0)
num_samples = input_data.size(1)
self.num_samples = num_samples
# similar to librosa, reflect-pad the input
input_data = input_data.view(num_batches, 1, num_samples)
input_data = F.pad(
input_data.unsqueeze(1),
(int(self.filter_length / 2), int(self.filter_length / 2), 0, 0),
mode='reflect')
input_data = input_data.squeeze(1)
forward_transform = F.conv1d(
input_data.cpu(),
Variable(self.forward_basis, requires_grad=False).cpu(),
stride=self.hop_length,
padding=0).cpu()
cutoff = int((self.filter_length / 2) + 1)
real_part = forward_transform[:, :cutoff, :]
imag_part = forward_transform[:, cutoff:, :]
magnitude = torch.sqrt(real_part**2 + imag_part**2)
phase = torch.autograd.Variable(
torch.atan2(imag_part.data, real_part.data))
return magnitude, phase
def inverse(self, magnitude, phase):
recombine_magnitude_phase = torch.cat(
[magnitude*torch.cos(phase), magnitude*torch.sin(phase)], dim=1)
inverse_transform = F.conv_transpose1d(
recombine_magnitude_phase,
Variable(self.inverse_basis, requires_grad=False),
stride=self.hop_length,
padding=0)
if self.window is not None:
window_sum = window_sumsquare(
self.window, magnitude.size(-1), hop_length=self.hop_length,
win_length=self.win_length, n_fft=self.filter_length,
dtype=np.float32)
# remove modulation effects
approx_nonzero_indices = torch.from_numpy(
np.where(window_sum > tiny(window_sum))[0])
window_sum = torch.autograd.Variable(
torch.from_numpy(window_sum), requires_grad=False)
window_sum = window_sum.cuda() if magnitude.is_cuda else window_sum
inverse_transform[:, :,
approx_nonzero_indices] /= window_sum[approx_nonzero_indices]
# scale by hop ratio
inverse_transform *= float(self.filter_length) / self.hop_length
inverse_transform = inverse_transform[:, :, int(self.filter_length/2):]
inverse_transform = inverse_transform[:,
:, :-int(self.filter_length/2):]
return inverse_transform
def forward(self, input_data):
self.magnitude, self.phase = self.transform(input_data)
reconstruction = self.inverse(self.magnitude, self.phase)
return reconstruction
class TacotronSTFT(torch.nn.Module):
def __init__(self, filter_length=1024, hop_length=256, win_length=1024,
n_mel_channels=80, sampling_rate=22050, mel_fmin=0.0,
mel_fmax=8000.0):
super(TacotronSTFT, self).__init__()
self.n_mel_channels = n_mel_channels
self.sampling_rate = sampling_rate
self.stft_fn = STFT(filter_length, hop_length, win_length)
mel_basis = librosa_mel_fn(
sampling_rate, filter_length, n_mel_channels, mel_fmin, mel_fmax)
mel_basis = torch.from_numpy(mel_basis).float()
self.register_buffer('mel_basis', mel_basis)
def spectral_normalize(self, magnitudes):
output = dynamic_range_compression(magnitudes)
return output
def spectral_de_normalize(self, magnitudes):
output = dynamic_range_decompression(magnitudes)
return output
# 抽取对应的mel谱图
def mel_spectrogram(self, y):
"""Computes mel-spectrograms from a batch of waves
PARAMS
------
y: Variable(torch.FloatTensor) with shape (B, T) in range [-1, 1]
RETURNS
-------
mel_output: torch.FloatTensor of shape (B, n_mel_channels, T)
"""
assert(torch.min(y.data) >= -1)
assert(torch.max(y.data) <= 1)
magnitudes, phases = self.stft_fn.transform(y)
magnitudes = magnitudes.data
mel_output = torch.matmul(self.mel_basis, magnitudes)
mel_output = self.spectral_normalize(mel_output)
return mel_output
本文件主要基于上述的三个文件实现mel数据抽取、保存的具体功能
""" from https://github.com/NVIDIA/tacotron2 """
import torch
import numpy as np
from scipy.io.wavfile import read
from scipy.io.wavfile import write
import audio.stft as stft
import audio.hparams_audio as hparams
from audio.audio_processing import griffin_lim
# 初始化stft.py中构建的短时傅里叶变化/STFT的模块
_stft = stft.TacotronSTFT(
hparams.filter_length, hparams.hop_length, hparams.win_length,
hparams.n_mel_channels, hparams.sampling_rate, hparams.mel_fmin,
hparams.mel_fmax)
# 根据音频文件路径加载数据
def load_wav_to_torch(full_path):
sampling_rate, data = read(full_path) # 调用scipy中的音频读取接口
return torch.FloatTensor(data.astype(np.float32)), sampling_rate
# 从音频文件中获取对应的mel谱图
def get_mel(filename):
audio, sampling_rate = load_wav_to_torch(filename) # 加载一条音频文件
if sampling_rate != _stft.sampling_rate: # 音频文件的采样频率要与设置的频率一致
raise ValueError("{} {} SR doesn't match target {} SR".format(
sampling_rate, _stft.sampling_rate))
audio_norm = audio / hparams.max_wav_value
audio_norm = audio_norm.unsqueeze(0)
# 将音频数据文件设置为不需要求梯度
audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False)
melspec = _stft.mel_spectrogram(audio_norm) # 获取mel谱图
melspec = torch.squeeze(melspec, 0)
# melspec = torch.from_numpy(_normalize(melspec.numpy()))
return melspec
# 直接从音频文件中获取对应的mel谱图
def get_mel_from_wav(audio):
sampling_rate = hparams.sampling_rate
if sampling_rate != _stft.sampling_rate:
raise ValueError("{} {} SR doesn't match target {} SR".format(
sampling_rate, _stft.sampling_rate))
audio_norm = audio / hparams.max_wav_value
audio_norm = audio_norm.unsqueeze(0)
audio_norm = torch.autograd.Variable(audio_norm, requires_grad=False)
melspec = _stft.mel_spectrogram(audio_norm)
melspec = torch.squeeze(melspec, 0)
return melspec
# 从mel谱图中重建音频文件
def inv_mel_spec(mel, out_filename, griffin_iters=60):
mel = torch.stack([mel])
# mel = torch.stack([torch.from_numpy(_denormalize(mel.numpy()))])
# 将mel谱图解压,抽取对应的线性谱图
mel_decompress = _stft.spectral_de_normalize(mel)
mel_decompress = mel_decompress.transpose(1, 2).data.cpu()
spec_from_mel_scaling = 1000
spec_from_mel = torch.mm(mel_decompress[0], _stft.mel_basis)
spec_from_mel = spec_from_mel.transpose(0, 1).unsqueeze(0)
spec_from_mel = spec_from_mel * spec_from_mel_scaling
# 基于线性谱图使用griffin_lim算法重建音频文件
audio = griffin_lim(torch.autograd.Variable(
spec_from_mel[:, :, :-1]), _stft.stft_fn, griffin_iters)
audio = audio.squeeze()
audio = audio.cpu().numpy()
audio_path = out_filename
write(audio_path, hparams.sampling_rate, audio) # 保存音频文件
本文件则是主要基于audio目录中文件从LJSpeech中提取数据
import numpy as np
import os
import audio
from tqdm import tqdm
from functools import partial
from concurrent.futures import ProcessPoolExecutor
def build_from_path(in_dir, out_dir):
index = 1
# executor = ProcessPoolExecutor(max_workers=4)
# futures = []
texts = []
with open(os.path.join(in_dir, 'metadata.csv'), encoding='utf-8') as f:
for line in f.readlines():
if index % 100 == 0:
print("{:d} Done".format(index))
parts = line.strip().split('|') # 使用|将文本分割
wav_path = os.path.join(in_dir, 'wavs', '%s.wav' % parts[0]) # 获取音频文件路径
text = parts[2] # 获取音频文件对应的文本
# futures.append(executor.submit(
# partial(_process_utterance, out_dir, index, wav_path, text)))
texts.append(_process_utterance(out_dir, index, wav_path, text))
index = index + 1
# return [future.result() for future in tqdm(futures)]
return texts # 返回的是所有音频文件对应的文本数据
# 调用tools.py中的get_mel函数获取音频文件的mel谱图
def _process_utterance(out_dir, index, wav_path, text):
# Compute a mel-scale spectrogram from the wav:
mel_spectrogram = audio.tools.get_mel(wav_path).numpy().astype(np.float32) # 使用audio中实现的get_mel函数从音频文件中获取mel谱图
# Write the spectrograms to disk:
mel_filename = 'ljspeech-mel-%05d.npy' % index
# 保存提取的mel谱图
np.save(os.path.join(out_dir, mel_filename),
mel_spectrogram.T, allow_pickle=False)
return text # 返回一个音频文件对应的文本内容
主要是调用ljspeech.py文件最先数据提取和保存
import torch
import numpy as np
import shutil
import os
from data import ljspeech
import hparams as hp # 导入所有的参数
def preprocess_ljspeech(filename):
in_dir = filename
out_dir = hp.mel_ground_truth # mel谱图的保存路径
if not os.path.exists(out_dir):
os.makedirs(out_dir, exist_ok=True)
metadata = ljspeech.build_from_path(in_dir, out_dir) # 从音频文件中抽取对应的mel谱图
write_metadata(metadata, out_dir) # 保存文本数据
# 将生成的train.txt文件移动到data路径下
shutil.move(os.path.join(hp.mel_ground_truth, "train.txt"),
os.path.join("data", "train.txt"))
# 将每个音频文件对应的文本数据进行保存
def write_metadata(metadata, out_dir):
with open(os.path.join(out_dir, 'train.txt'), 'w', encoding='utf-8') as f:
for m in metadata:
f.write(m + '\n')
def main():
path = os.path.join("data", "LJSpeech-1.1")
preprocess_ljspeech(path)
if __name__ == "__main__":
main()
本笔记主要记录所选择的fastspeech复现仓库中数据准备相关的代码,结合之前FastSppech论文阅读笔记笔记中的数据准备部分,因为所使用的都是LJSpeech数据集,可以发现整体逻辑和实现方式基本一致。理解该过程有助于后续在自己的项目中构建特定领域的数据集,不限于英文,对于中文、日文等数据集的构建也有较大的帮助。本笔记主要是对代码进行详细的注释,读者若发现问题或错误,请评论指出,互相学习。