利用循环神经网络,基于梅尔频率倒谱系数(MFCC)的语音信号处理技术来进行语音分类,由于只有两个分类,学习难度不算大。对语音分类后可以将语音传给百度不同类别的语音识别翻译出对应的文字。
本训练的数据可以通过收音机类的app获取普通话和广东话的语音资料。训练需要wav格式的单声道的语音,每个语音数据的时间长度可以选择5s、10s、15s等,但要统一长度,本次训练选择的10s的语音素材。收集到的数据可以利用ffmpeg工具进行格式转化和语音文件切割。
以下简单说说ffmpeg工具安装及使用
windows系统上的安装很简单,直接到官网下载windows系统安装包,解压后即可以使用
linux系统的安装可以通过编译安装,这种安装较复杂,也可以通过yum方式安装。
sudo rpm --import http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
sudo rpm -Uvh http://li.nux.ro/download/nux/dextop/el7/x86_64/nux-dextop-release-0-5.el7.nux.noarch.rpm
CentOS 6
sudo rpm --import http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
sudo rpm -Uvh http://li.nux.ro/download/nux/dextop/el6/x86_64/nux-dextop-release-0-2.el6.nux.noarch.rpm
sudo yum install ffmpeg ffmpeg-devel -y
ffmpeg
如果音频格式不是wav,那需要进行格式转换,格式转换命令如下:
./ffmpeg -i D:\data\guoyu\6.mp4 -ac 1 D:\data\guoyu\6.wav
以上命令是将D:\data\guoyu\6.mp4是MP4格式的音频资料,转换为wav格式并存储到D:\data\guoyu目录下,-ac 1表示转换后为单声道的音频。
ffmpeg工具的主要切割命令如下,可以通过脚本实现一个循环命令来进行音频文件的切割。
ffmpeg -i 6.wav -ss $startTime -to $endTime -acodec copy -vcodec copy 6-$i-0.wav
$startTime、$endTime是指切割文件的开始时间和结束时间,单位为秒。切割后的文件命名为6-$i-0.wav格式,其中$i是循环递增变量,0是训练用的标签,0表示普通话,1表示广东话。
本训练使用TensorFlow,语音为python,训练语音数据放到train目录下,测试数据放到test目录。训练数据1000个左右,测试数据200个左右。
# -*- coding: utf-8 -*-
import numpy as np
from python_speech_features import mfcc #需要pip install
import scipy.io.wavfile as wav
import os
'''读取wav文件对应的label'''
def get_wavs_lables1(wav_path):
#获得训练用的wav文件路径列表
wav_files = []
labels = []
for (dirpath, dirnames, filenames) in os.walk(wav_path):
for filename in filenames:
if filename.endswith('.wav') or filename.endswith('.WAV'):
filename_path = os.sep.join([dirpath, filename])
if os.stat(filename_path).st_size < 2400: # 剔除掉一些小文件
continue
wav_files.append(filename_path)
name = filename.split('.')[0]
label = name.split('-')[2]
labels.append(label)
return wav_files, labels
# Constants
SPACE_TOKEN = ''
SPACE_INDEX = 2
FIRST_INDEX = ord('a') - 1 # 0 is reserved to space
#n_input = 26 计算美尔倒谱系数的个数
# n_context = 9 对于每个时间点,要包含上下文样本的个数
def get_audio_and_transcriptch(txt_files, wav_files, n_input, n_context,word_num_map,txt_labels=None):
audio = []
audio_len = []
transcript = []
transcript_len = []
if txt_files!=None:
txt_labels = txt_files
for txt_obj, wav_file in zip(txt_labels, wav_files):
# load audio and convert to features
audio_data = audiofile_to_input_vector(wav_file, n_input, n_context)
audio_data = audio_data.astype('float32')
audio.append(audio_data)
audio_len.append(np.int32(len(audio_data)))
# load text transcription and convert to numerical array
target = []
if txt_files!=None:#txt_obj是文件
target = get_ch_lable_v(txt_obj,word_num_map)
else:
target = get_ch_lable_v(None,word_num_map,txt_obj)#txt_obj是labels
#target = text_to_char_array(target)
transcript.append(target)
transcript_len.append(len(target))
audio = np.asarray(audio)
audio_len = np.asarray(audio_len)
transcript = np.asarray(transcript)
transcript_len = np.asarray(transcript_len)
return audio, audio_len, transcript, transcript_len
#优先转文件里的字符到向量
def get_ch_lable_v(txt_file,word_num_map,txt_label=None):
words_size = len(word_num_map)
# dict.get(key, default=None) key -- 字典中要查找的键。default -- 如果指定键的值不存在时,返回该默认值值。
to_num = lambda word: word_num_map.get(word, words_size)
if txt_file!= None:
txt_label = get_ch_lable(txt_file)
#print(txt_label)
labels_vector = list(map(to_num, txt_label))
#print(labels_vector)
return labels_vector
def get_ch_lable(txt_file):
labels= ""
with open(txt_file, 'rb') as f:
for label in f:
#labels =label.decode('utf-8')
labels =labels+label.decode('gb2312')
#labels.append(label.decode('gb2312'))
return labels
# numcep参数是 n_input = 26 计算美尔倒谱系数的个数
# numcontext参数是 n_context = 9 对于每个时间点,要包含上下文样本的个数
def audiofile_to_input_vector(audio_filename, numcep, numcontext):
# Load wav files 读取.wav音频文件,返回一个元组,第一项为音频的采样率,第二项为音频数据的numpy数组。
fs, audio = wav.read(audio_filename)
# Get mfcc coefficients
orig_inputs = mfcc(audio, samplerate=fs, numcep=numcep)
#print(np.shape(orig_inputs))#(277, 26) list[start:end:step] start:起始位置 end:结束位置 step:步长
orig_inputs = orig_inputs[::2]#(139, 26)
train_inputs = np.array([], np.float32)
#ValueError: cannot resize an array that references or is referenced by another array in this way. Use the resize function
#train_inputs.resize((orig_inputs.shape[0], numcep + 2 * numcep * numcontext))
train_inputs = np.resize(train_inputs,(orig_inputs.shape[0], numcep + 2 * numcep * numcontext))
# or a.resize(new_shape, refcheck=False)
#print(np.shape(train_inputs))#)(139, 494)
# Prepare pre-fix post fix context
empty_mfcc = np.array([])
#empty_mfcc.resize((numcep))
empty_mfcc = np.resize(empty_mfcc,(numcep))
# Prepare train_inputs with past and future contexts
time_slices = range(train_inputs.shape[0])#139个切片
context_past_min = time_slices[0] + numcontext
context_future_max = time_slices[-1] - numcontext#[9,1,2...,137,129]
for time_slice in time_slices:
# 前9个补0,mfcc features
need_empty_past = max(0, (context_past_min - time_slice))
empty_source_past = list(empty_mfcc for empty_slots in range(need_empty_past))
data_source_past = orig_inputs[max(0, time_slice - numcontext):time_slice]
assert(len(empty_source_past) + len(data_source_past) == numcontext)
# 后9个补0,mfcc features
need_empty_future = max(0, (time_slice - context_future_max))
empty_source_future = list(empty_mfcc for empty_slots in range(need_empty_future))
data_source_future = orig_inputs[time_slice + 1:time_slice + numcontext + 1]
assert(len(empty_source_future) + len(data_source_future) == numcontext)
#np.concatenate联结函数,两个数组连接在一起
if need_empty_past:
past = np.concatenate((empty_source_past, data_source_past))
else:
past = data_source_past
if need_empty_future:
future = np.concatenate((data_source_future, empty_source_future))
else:
future = data_source_future
past = np.reshape(past, numcontext * numcep)
now = orig_inputs[time_slice]
future = np.reshape(future, numcontext * numcep)
train_inputs[time_slice] = np.concatenate((past, now, future))
assert(len(train_inputs[time_slice]) == numcep + 2 * numcep * numcontext)
# 将数据使用正太分布标准化,减去均值然后再除以方差
train_inputs = (train_inputs - np.mean(train_inputs)) / np.std(train_inputs)
return train_inputs
def pad_sequences(sequences, maxlen=None, dtype=np.float32,
padding='post', truncating='post', value=0.):
lengths = np.asarray([len(s) for s in sequences], dtype=np.int64)
nb_samples = len(sequences)
if maxlen is None:
maxlen = np.max(lengths)
# take the sample shape from the first non empty sequence
# checking for consistency in the main loop below.
sample_shape = tuple()
for s in sequences:
if len(s) > 0:
sample_shape = np.asarray(s).shape[1:]
break
x = (np.ones((nb_samples, maxlen) + sample_shape) * value).astype(dtype)
for idx, s in enumerate(sequences):
if len(s) == 0:
continue # empty list was found
if truncating == 'pre':
trunc = s[-maxlen:]
elif truncating == 'post':
trunc = s[:maxlen]
else:
raise ValueError('Truncating type "%s" not understood' % truncating)
# check `trunc` has expected shape
trunc = np.asarray(trunc, dtype=dtype)
if trunc.shape[1:] != sample_shape:
raise ValueError('Shape of sample %s of sequence at position %s is different from expected shape %s' %
(trunc.shape[1:], idx, sample_shape))
if padding == 'post':
x[idx, :len(trunc)] = trunc
elif padding == 'pre':
x[idx, -len(trunc):] = trunc
else:
raise ValueError('Padding type "%s" not understood' % padding)
return x, lengths
# -*- coding: utf-8 -*-
import numpy as np
import time
import tensorflow as tf
from tensorflow.python.ops import ctc_ops
from collections import Counter
import numpy as np
import time
import tensorflow as tf
from tensorflow.python.ops import ctc_ops
from collections import Counter
## 自定义
speechutils = __import__("speechutils")
sparse_tuple_to_texts_ch = speechutils.sparse_tuple_to_texts_ch
ndarray_to_text_ch = speechutils.ndarray_to_text_ch
get_audio_and_transcriptch = speechutils.get_audio_and_transcriptch
pad_sequences = speechutils.pad_sequences
sparse_tuple_from = speechutils.sparse_tuple_from
get_wavs_lables1 = speechutils.get_wavs_lables1
tf.reset_default_graph()
b_stddev = 0.046875
h_stddev = 0.046875
n_hidden = 1024
n_hidden_1 = 1024
n_hidden_2 =1024
n_hidden_5 = 1024
n_cell_dim = 1024
n_hidden_3 = 2 * 1024
keep_dropout_rate=0.95
relu_clip = 20
def BiRNN_model( batch_x, seq_length, n_input, n_context,n_character ,keep_dropout):
# batch_x_shape: [batch_size, n_steps, n_input + 2*n_input*n_context]
batch_x_shape = tf.shape(batch_x)
# 将输入转成时间序列优先,[1, 0, 2]将第一维和第二维交换
batch_x = tf.transpose(batch_x, [1, 0, 2])
# 再转成2维传入第一层
batch_x = tf.reshape(batch_x,
[-1, n_input + 2 * n_input * n_context]) # (n_steps*batch_size, n_input + 2*n_input*n_context)
# 使用clipped RELU activation and dropout.
# 1st layer
with tf.name_scope('fc1'):
b1 = variable_on_cpu('b1', [n_hidden_1], tf.random_normal_initializer(stddev=b_stddev))
h1 = variable_on_cpu('h1', [n_input + 2 * n_input * n_context, n_hidden_1],
tf.random_normal_initializer(stddev=h_stddev))
#tf.multiply()两个矩阵中对应元素各自相乘;tf.matmul()将矩阵a乘以矩阵b,生成a * b。
#tf.minimum(a,b)返回a,b之间的最小值
layer_1 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(batch_x, h1), b1)), relu_clip)
layer_1 = tf.nn.dropout(layer_1, keep_dropout)
# 2nd layer
with tf.name_scope('fc2'):
b2 = variable_on_cpu('b2', [n_hidden_2], tf.random_normal_initializer(stddev=b_stddev))
h2 = variable_on_cpu('h2', [n_hidden_1, n_hidden_2], tf.random_normal_initializer(stddev=h_stddev))
layer_2 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_1, h2), b2)), relu_clip)
layer_2 = tf.nn.dropout(layer_2, keep_dropout)
# 3rd layer
with tf.name_scope('fc3'):
b3 = variable_on_cpu('b3', [n_hidden_3], tf.random_normal_initializer(stddev=b_stddev))
h3 = variable_on_cpu('h3', [n_hidden_2, n_hidden_3], tf.random_normal_initializer(stddev=h_stddev))
layer_3 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_2, h3), b3)), relu_clip)
layer_3 = tf.nn.dropout(layer_3, keep_dropout)
# 双向rnn
with tf.name_scope('lstm'):
# Forward direction cell:
lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(n_cell_dim, forget_bias=1.0, state_is_tuple=True)
lstm_fw_cell = tf.contrib.rnn.DropoutWrapper(lstm_fw_cell,
input_keep_prob=keep_dropout)
# Backward direction cell:
lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(n_cell_dim, forget_bias=1.0, state_is_tuple=True)
lstm_bw_cell = tf.contrib.rnn.DropoutWrapper(lstm_bw_cell,
input_keep_prob=keep_dropout)
# `layer_3` `[n_steps, batch_size, 2*n_cell_dim]`
layer_3 = tf.reshape(layer_3, [-1, batch_x_shape[0], n_hidden_3])
outputs, output_states = tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fw_cell,
cell_bw=lstm_bw_cell,
inputs=layer_3,
dtype=tf.float32,
time_major=True,
sequence_length=seq_length)
# 连接正反向结果[n_steps, batch_size, 2*n_cell_dim]
outputs = tf.concat(outputs, 2)
# to a single tensor of shape [n_steps*batch_size, 2*n_cell_dim]
outputs = tf.reshape(outputs, [-1, 2 * n_cell_dim])
with tf.name_scope('fc5'):
b5 = variable_on_cpu('b5', [n_hidden_5], tf.random_normal_initializer(stddev=b_stddev))
h5 = variable_on_cpu('h5', [(2 * n_cell_dim), n_hidden_5], tf.random_normal_initializer(stddev=h_stddev))
layer_5 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(outputs, h5), b5)), relu_clip)
layer_5 = tf.nn.dropout(layer_5, keep_dropout)
with tf.name_scope('fc6'):
# 全连接层用于softmax分类
b6 = variable_on_cpu('b6', [n_character], tf.random_normal_initializer(stddev=b_stddev))
h6 = variable_on_cpu('h6', [n_hidden_5, n_character], tf.random_normal_initializer(stddev=h_stddev))
layer_6 = tf.add(tf.matmul(layer_5, h6), b6)
# 将2维[n_steps*batch_size, n_character]转成3维 time-major [n_steps, batch_size, n_character].
print(layer_6.get_shape().as_list())#[None, 2]
layer_6 = tf.reshape(layer_6, [-1, batch_x_shape[0], n_character])
print(layer_6.get_shape().as_list())#[None, None, 2]
layer_6 = tf.reduce_sum(layer_6, 0)#0按列相加,1按行相加
print(layer_6.get_shape().as_list())#[None, 2]
return layer_6
"""
used to create a variable in CPU memory.
"""
def variable_on_cpu(name, shape, initializer):
# Use the /cpu:0 device for scoped operations
with tf.device('/cpu:0'):
# Create or get apropos variable
var = tf.get_variable(name=name, shape=shape, initializer=initializer)
return var
wav_path='train'
test_path='test'
wav_files, labels = get_wavs_lables1(wav_path)
test_files, test_labels = get_wavs_lables1(test_path)
print(wav_files[0], labels[0])
print("wav:",len(wav_files),"label",len(labels))
# 字表
all_words = []
for label in labels:
#print(label)
all_words += [word for word in label]
counter = Counter(all_words)
words = sorted(counter)
words_size= len(words)
word_num_map = dict(zip(words, range(words_size)))
print('字表大小:', words_size)
n_input = 26#计算美尔倒谱系数的个数
n_context = 9#对于每个时间点,要包含上下文样本的个数
batch_size =8
def next_batch(labels, start_idx = 0,batch_size=1,wav_files = wav_files):
filesize = len(labels)
end_idx = min(filesize, start_idx + batch_size)
idx_list = range(start_idx, end_idx)
txt_labels = [labels[i] for i in idx_list]
wav_files = [wav_files[i] for i in idx_list]
(source, audio_len, target, transcript_len) = get_audio_and_transcriptch(None,
wav_files,
n_input,
n_context,word_num_map,txt_labels)
start_idx += batch_size
# Verify that the start_idx is not larger than total available sample size
if start_idx >= filesize:
start_idx = -1
# Pad input to max_time_step of this batch
source, source_lengths = pad_sequences(source)#如果多个文件将长度统一,支持按最大截断或补0
sparse_labels = [label[0] for label in target]
return start_idx,source, source_lengths, sparse_labels
#sparse_lab 为文字转化成向量后并生成的稀疏矩阵,所以长度为3,补0对齐后的音频数据的shape为(8,1168,494),
# 8代表batchsize;1168代表时序的总个数。494是组合好的MFCC特征数:取前9个时序的MFCC,当前MFCC再加上后9个
#MFCC,每个MFCC由26个数字组成。
#在矩阵中,若数值为0的元素数目远远多于非0元素的数目,并且非0元素分布没有规律时,则称该矩阵为稀疏矩阵;
# 与之相反,若非0元素数目占大多数时,则称该矩阵为稠密矩阵。
next_idx,source,source_len,sparse_lab = next_batch(labels,0,batch_size)
print(len(sparse_lab))
print(np.shape(source))
#print(sparse_lab)
#t = sparse_tuple_to_texts_ch(sparse_lab,words)
#print(t[0])
#source为具体的样本,每条样本的内容为19个时间序列,source已经将变为前9(不够补空)+本身+后9,
#每个时间序列有26个美尔倒谱系数。第一条的样本是从第10个时间序列开始
# shape = [batch_size, max_stepsize, n_input + (2 * n_input * n_context)]
# the batch_size and max_stepsize每步都是变长的。
input_tensor = tf.placeholder(tf.float32, [None, None, n_input + (2 * n_input * n_context)], name='input')#语音log filter bank or MFCC features
# Use sparse_placeholder; will generate a SparseTensor, required by ctc_loss op.
targets = tf.placeholder(tf.int64, shape=[None], name='targets')#文本
# 1d array of size [batch_size]
seq_length = tf.placeholder(tf.int32, [None], name='seq_length')#序列长
keep_dropout= tf.placeholder(tf.float32)
logits = BiRNN_model( input_tensor, tf.to_int64(seq_length), n_input, n_context,words_size,keep_dropout)
avg_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=targets, logits=logits))
#[optimizer]
learning_rate = 0.001
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(avg_loss)
with tf.name_scope("decode"):
decoded=tf.nn.softmax(logits)
with tf.name_scope("accuracy"):
ler = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), targets), tf.float32))
epochs = 20
savedir = "train"
saver = tf.train.Saver(max_to_keep=1) # 生成saver
# create the session
sess = tf.Session()
# 没有模型的话,就重新初始化
sess.run(tf.global_variables_initializer())
kpt = tf.train.latest_checkpoint(savedir)
print("kpt:",kpt)
startepo= 0
if kpt!=None:
saver.restore(sess, kpt)
ind = kpt.find("-")
startepo = int(kpt[ind+1:])
startepo +=1
print(startepo)
# 准备运行训练步骤
section = '\n{0:=^40}\n'
print(section.format('Run training epoch'))
train_start = time.time()
for epoch in range(epochs):#样本集迭代次数
epoch_start = time.time()
if epoch<startepo:
continue
print("epoch start:",epoch,"total epochs= ",epochs)
#######################run batch####
n_batches_per_epoch = int(np.ceil(len(labels) / batch_size))
print("total loop ",n_batches_per_epoch,"in one epoch,",batch_size,"items in one loop")
train_cost = 0
train_ler = 0
next_idx =0
test_next_idx=0
for batch in range(n_batches_per_epoch):#一次batch_size,取多少次
#取数据
next_idx,source,source_lengths,sparse_labels = \
next_batch(labels,next_idx ,batch_size)
feed = {input_tensor: source, targets: sparse_labels,seq_length: source_lengths,keep_dropout:keep_dropout_rate}
#计算 avg_loss optimizer ;
batch_cost, _ = sess.run([avg_loss, optimizer], feed_dict=feed )
train_cost += batch_cost
if (batch +1)%2 == 0:
print('Epoch/Epochs:',epoch,'/',epochs,'loop:',batch, 'Train cost: ', train_cost/(batch+1))
feed2 = {input_tensor: source, targets: sparse_labels,seq_length: source_lengths,keep_dropout:1.0}
d,train_ler = sess.run([decoded,ler], feed_dict=feed2)
dense_decoded = tf.argmax( d, 1).eval(session=sess)
counter =0
print('Label right rate: ', train_ler)
for orig, decoded_arr in zip(sparse_labels, dense_decoded):
print(' file {}'.format( counter))
print('Original: {}'.format(orig))
print('Decoded: {}'.format(decoded_arr))
counter=counter+1
# break
if (batch +1)%5 == 0:
print('Epoch/Epochs:',epoch,'/',epochs,'loop:',batch, 'Train cost: ', train_cost/(batch+1))
#测试数据
test_next_idx,test_source,test_source_lengths,test_sparse_labels = \
next_batch(test_labels,test_next_idx ,batch_size,wav_files=test_files)
feed2 = {input_tensor: test_source, targets: test_sparse_labels,seq_length: test_source_lengths,keep_dropout:1.0}
d,test_ler = sess.run([decoded,ler], feed_dict=feed2)
dense_decoded = tf.argmax( d, 1).eval(session=sess)
counter =0
print('test Label right rate: ', test_ler)
for orig, decoded_arr in zip(test_sparse_labels, dense_decoded):
print(' test file {}'.format( counter))
print('test Original: {}'.format(orig))
print('test Decoded: {}'.format(decoded_arr))
counter=counter+1
# break
epoch_duration = time.time() - epoch_start
log = 'Epoch {}/{}, train_cost: {:.3f}, train_ler: {:.3f}, time: {:.2f} sec'
print(log.format(epoch ,epochs, train_cost,train_ler,epoch_duration))
saver.save(sess, savedir+"speechclassify.gd", global_step=epoch)
train_duration = time.time() - train_start
print('Training complete, total duration: {:.2f} min'.format(train_duration / 60))
sess.close()
该语音分类需要保证语音文件的取样率一致,如果取样率不一致可以通过librosa重新设置语音文件的采样率。
wav语音文件的采样率通过UItraEdit查看,其中18H-1bH偏移地址的内容为采样率。
重置采样率代码示例:
def resample4wavs(frompath,topath,resamplerate):
'''
:param frompath: 源文件所在目录
:param topath: 重置采样率文件存放目录
:param resamplerate: 重置采样率
:return:
'''
fs=os.listdir(frompath)
for f in fs:
try:
fromfile = frompath+f
print(fromfile)
tofile = topath+f
y, sr = librosa.load(fromfile)
to_y = librosa.resample(y,sr,resamplerate)
librosa.output.write_wav(tofile, to_y, resamplerate)
except Exception as e:
print('Error:',e)