Android AudioRecord录音 并用 websocket实时传输，AudioTrack 播放wav 音频
在一家专注于AI音频公司做了一年，最近正处于预离职状态，正好刚刚给客户写了个关于android音频方面的demo，花了我足足一天赶出来的，感觉挺全面的决定再努力一点写个总结。
公司虽小，是和中科院声学所合作，也和讯飞一样也有自己关于音频的一系列语音识别/语音转写等引擎，麻雀虽小五脏俱全的感觉。
Android 音频这块其实也没那么神秘，神秘的地方有专门的C++/算法工程师等为我们负责，大家都懂得，我只是搬搬砖而已。
主要涉及3点
- SpeechToText(音频转文本：STT)： AudioRecord 录制音频 并用本地和webSocket方式上传 。
- TextToSpeech （文本转语音：TTS） API获取音频流并用AudioTrack 播放。
- Speex 加密
这里不讲TTS/STT底层原理，怎么实现的呆了这么久我也只是大概，涉及人耳听声相关函数/声波/傅里叶分析/一系列复杂算法， 感兴趣请大家自行Google 。，
AudioRecord 介绍
AudioRecord 过程是一个IPC过程，Java层通过JNI调用到native层的AudioRecord，后者通过IAudioRecord接口跨进程调用到 AudioFlinger，AudioFlinger负责启动录音线程，将从录音数据源里采集的音频数据填充到共享内存缓冲区，然后应用程序侧从其里面拷贝数据到自己的缓冲区。
public AudioRecord(int audioSource, //指定声音源 MediaRecorder.AudioSource.MIC;
int sampleRateInHz,//指定采样率 这里8000
int channelConfig,//指定声道数，单声道
int audioFormat, //指定8/16pcm 这里16bit 模拟信号转化为数字信号时的量化单位
int bufferSizeInBytes)//缓冲区大小 根据采样率 通道 量化参数决定
1. STT 之本地录完之后文件形式上传
第二步再与socket 上传比较
//参数初始化
// 音频输入-麦克风
public final static int AUDIO_INPUT = MediaRecorder.AudioSource.MIC;
public final static int AUDIO_SAMPLE_RATE = 8000; // 44.1KHz,普遍使用的频率
public final static int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO;
public final static int AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT;
private int bufferSizeInBytes = 0;//缓冲区字节大小
private AudioRecord audioRecord;
private volatile boolean isRecord = false;// volatile 可见性 设置正在录制的状态
//创建AudioRecord
private void creatAudioRecord() {
// 获得缓冲区字节大小
bufferSizeInBytes = AudioRecord.getMinBufferSize(AudioFileUtils.AUDIO_SAMPLE_RATE,
AudioFileUtils.CHANNEL_CONFIG, AudioFileUtils.AUDIO_FORMAT);
// MONO单声道
audioRecord = new AudioRecord(AudioFileUtils.AUDIO_INPUT, AudioFileUtils.AUDIO_SAMPLE_RATE,
AudioFileUtils.CHANNEL_CONFIG, AudioFileUtils.AUDIO_FORMAT, bufferSizeInBytes);
}
//
@Override
public boolean onTouch(View v, MotionEvent event) {
AudioRecordUtils utils = AudioRecordUtils.getInstance();
switch (event.getAction()) {
case MotionEvent.ACTION_DOWN:
utils.startRecordAndFile();
break;
case MotionEvent.ACTION_UP:
utils.stopRecordAndFile();
Log.d(TAG, "stopRecordAndFile");
stt();
break;
}
return false;
}
//开始录音
public int startRecordAndFile() {
Log.d("NLPService", "startRecordAndFile");
// 判断是否有外部存储设备sdcard
if (AudioFileUtils.isSdcardExit()) {
if (isRecord) {
return ErrorCode.E_STATE_RECODING;
} else {
if (audioRecord == null) {
creatAudioRecord();
}
audioRecord.startRecording();
// 让录制状态为true
isRecord = true;
// 开启音频文件写入线程
new Thread(new AudioRecordThread()).start();
return ErrorCode.SUCCESS;
}
} else {
return ErrorCode.E_NOSDCARD;
}
}
//录音线程
class AudioRecordThread implements Runnable {
@Override
public void run() {
writeDateTOFile();// 往文件中写入裸数据
AudioFileUtils.raw2Wav(mAudioRaw, mAudioWav, bufferSizeInBytes);// 给裸数据加上头文件
}
}
// 往文件中写入裸数据
private void writeDateTOFile() {
Log.d("NLPService", "writeDateTOFile");
// new一个byte数组用来存一些字节数据，大小为缓冲区大小
byte[] audiodata = new byte[bufferSizeInBytes];
FileOutputStream fos = null;
int readsize = 0;
try {
File file = new File(mAudioRaw);
if (file.exists()) {
file.delete();
}
fos = new FileOutputStream(file);// 建立一个可存取字节的文件
} catch (Exception e) {
e.printStackTrace();
}
while (isRecord) {
readsize = audioRecord.read(audiodata, 0, bufferSizeInBytes);
if (AudioRecord.ERROR_INVALID_OPERATION != readsize && fos != null) {
try {
fos.write(audiodata);
} catch (IOException e) {
e.printStackTrace();
}
}
}
try {
if (fos != null)
fos.close();// 关闭写入流
} catch (IOException e) {
e.printStackTrace();
}
}
//add wav header
public static void raw2Wav(String inFilename, String outFilename, int bufferSizeInBytes) {
Log.d("NLPService", "raw2Wav");
FileInputStream in = null;
RandomAccessFile out = null;
byte[] data = new byte[bufferSizeInBytes];
try {
in = new FileInputStream(inFilename);
out = new RandomAccessFile(outFilename, "rw");
fixWavHeader(out, AUDIO_SAMPLE_RATE, 1, AudioFormat.ENCODING_PCM_16BIT);
while (in.read(data) != -1) {
out.write(data);
}
in.close();
out.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
private static void fixWavHeader(RandomAccessFile file, int rate, int channels, int format) {
try {
int blockAlign;
if (format == AudioFormat.ENCODING_PCM_16BIT)
blockAlign = channels * 2;
else
blockAlign = channels;
int bitsPerSample;
if (format == AudioFormat.ENCODING_PCM_16BIT)
bitsPerSample = 16;
else
bitsPerSample = 8;
long dataLen = file.length() - 44;
// hard coding
byte[] header = new byte[44];
header[0] = 'R'; // RIFF/WAVE header
header[1] = 'I';
header[2] = 'F';
header[3] = 'F';
header[4] = (byte) ((dataLen + 36) & 0xff);
header[5] = (byte) (((dataLen + 36) >> 8) & 0xff);
header[6] = (byte) (((dataLen + 36) >> 16) & 0xff);
header[7] = (byte) (((dataLen + 36) >> 24) & 0xff);
header[8] = 'W';
header[9] = 'A';
header[10] = 'V';
header[11] = 'E';
header[12] = 'f'; // 'fmt ' chunk
header[13] = 'm';
header[14] = 't';
header[15] = ' ';
header[16] = 16; // 4 bytes: size of 'fmt ' chunk
header[17] = 0;
header[18] = 0;
header[19] = 0;
header[20] = 1; // format = 1
header[21] = 0;
header[22] = (byte) channels;
header[23] = 0;
header[24] = (byte) (rate & 0xff);
header[25] = (byte) ((rate >> 8) & 0xff);
header[26] = (byte) ((rate >> 16) & 0xff);
header[27] = (byte) ((rate >> 24) & 0xff);
header[28] = (byte) ((rate * blockAlign) & 0xff);
header[29] = (byte) (((rate * blockAlign) >> 8) & 0xff);
header[30] = (byte) (((rate * blockAlign) >> 16) & 0xff);
header[31] = (byte) (((rate * blockAlign) >> 24) & 0xff);
header[32] = (byte) (blockAlign); // block align
header[33] = 0;
header[34] = (byte) bitsPerSample; // bits per sample
header[35] = 0;
header[36] = 'd';
header[37] = 'a';
header[38] = 't';
header[39] = 'a';
header[40] = (byte) (dataLen & 0xff);
header[41] = (byte) ((dataLen >> 8) & 0xff);
header[42] = (byte) ((dataLen >> 16) & 0xff);
header[43] = (byte) ((dataLen >> 24) & 0xff);
file.seek(0);
file.write(header, 0, 44);
} catch (Exception e) {
} finally {
}
}
//文件上传 结果回调
public void stt() {
File voiceFile = new File(AudioFileUtils.getWavFilePath());
if (!voiceFile.exists()) {
return;
}
RequestBody requestBody = RequestBody.create(MediaType.parse("multipart/form-data"), voiceFile);
MultipartBody.Part file =
MultipartBody.Part.createFormData("file", voiceFile.getName(), requestBody);
NetRequest.sAPIClient.stt(RequestBodyUtil.getParams(), file)
.observeOn(AndroidSchedulers.mainThread())
.subscribe(new Action1() {
@Override
public void call(STT result) {
if (result != null && result.getCount() > 0) {
sttTv.setText("结果: " + result.getSegments().get(0).getContent());
}
}
});
}
//记得关闭AudioRecord
private void stopRecordAndFile() {
if (audioRecord != null) {
isRecord = false;// 停止文件写入
audioRecord.stop();
audioRecord.release();// 释放资源
audioRecord = null;
}
}
2. STT 之AudioRecord录制websocket 在线传输
WebSocket介绍： 我只记住一点点：它是应用层协议 ，就像http 也是，不过它是一种全双工通信，
socket 只是TCP/IP 的封装，不算协议。websocket 第一次需要以http 接口建立长连接,就这么点了。
//MyWebSocketListener Websocket 回调
class MyWebSocketListener extends WebSocketListener {
@Override
public void onOpen(WebSocket webSocket, Response response) {
output("onOpen: " + "webSocket connect success");
STTWebSocketActivity.this.webSocket = webSocket;
startRecordAndFile();
//看清楚了开始录音函数在这里，原因由于涉及回调，当分离时候 处理逻辑复杂
//，而且第二次录音时候由于服务端WebSocket已经关闭 ，录音数据不能正常传输，需要重新建立连接
}
@Override
public void onMessage(WebSocket webSocket, final String text) {
runOnUiThread(new Runnable() {
@Override
public void run() {
sttTv.setText("Stt result:" + text);
}
});
output("onMessage1: " + text);
}
@Override
public void onMessage(WebSocket webSocket, ByteString bytes) {
output("onMessage2 byteString: " + bytes);
}
@Override
public void onClosing(WebSocket webSocket, int code, String reason) {
output("onClosing: " + code + "/" + reason);
}
@Override
public void onClosed(WebSocket webSocket, int code, String reason) {
output("onClosed: " + code + "/" + reason);
}
@Override
public void onFailure(WebSocket webSocket, Throwable t, Response response) {
output("onFailure: " + t.getMessage());
}
private void output(String s) {
Log.d("NLPService", s);
}
}
补充：AudioRecord创建与前面相同
// okhttp 创建websocket 并设置监听
private void createWebSocket() {
Request request = new Request.Builder().url(sttApi).build();
NetRequest.getOkHttpClient().newWebSocket(request, socketListener);
}
class AudioRecordThread implements Runnable {
@Override
public void run() {
//byteBuffer 缓冲区 （内存地址以数组形式排列，一个基本数据类型的数组）
ByteBuffer audioBuffer = ByteBuffer.allocateDirect(bufferSizeInBytes).order(ByteOrder.LITTLE_ENDIAN);//小端模式
int readSize = 0;
Log.d(TAG, "isRecord=" + isRecord);
while (isRecord) {
readSize = audioRecord.read(audioBuffer, audioBuffer.capacity());
if (readSize == AudioRecord.ERROR_INVALID_OPERATION || readSize == AudioRecord.ERROR_BAD_VALUE) {
Log.d("NLPService", "Could not read audio data.");
break;
}
boolean send = webSocket.send(ByteString.of(audioBuffer));//就这么简单哈哈
Log.d("NLPService", "send=" + send);
audioBuffer.clear();//记住清空
}
webSocket.send("close");//录制完之后发送约定字段。通知服务端关闭。
}
}
......然后呢，然后就有数据了 ，就是这么简单
......然后老司机就要说了。。。你这没有加密啊，效率很低啊。在此陈述一点，这里是转写引擎，每次就一句话 ，传输数据量本身不大，后端大神们说没必要加密，然后我就照办了...当然也可以一边加密一边传输
3.TTS 之AudioTrack 播放wav文件
这里就比较简单了，okhttp 调用API 传递text 获取response 然后用之AudioTrack 播放。这里是原始音频流，mediaplayer播放就有点大才小用了（我没试过），不过 mediaplayer播放也是IPC过程，底层最终也是调用AudioTrack 进行播放的。
直接上代码 ：
public boolean request() {
OkHttpClient client = NetRequest.getOkHttpClient();
Request request = new Request.Builder().url(NetRequest.BASE_URL + "api/tts?text=今天是星期三").build();
client.newCall(request).enqueue(new Callback() {
@Override
public void onFailure(Call call, IOException e) {
}
@Override
public void onResponse(Call call, Response response) throws IOException {
play(response.body().bytes());
}
});
return true;
}
public void play( byte[] data) {
try {
Log.d(TAG, "audioTrack start ");
AudioTrack audioTrack = new AudioTrack(mOutput, mSamplingRate,
AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_16BIT,
data.length, AudioTrack.MODE_STATIC);
audioTrack.write(data, 0, data.length);
audioTrack.play();
while (audioTrack.getPlaybackHeadPosition() < (data.length / 2)) {
Thread.yield();//播放延迟处理......
}
audioTrack.stop();
audioTrack.release();
} catch (IllegalArgumentException e) {
} catch (IllegalStateException e) {
}
}
4.speex 加密
speex 是一个开源免费的音频加密库，C++ 写的。demo里面是编译好的so 文件，
，我亲自编译了好久各种坑，最后没成功，只能借用了。-_-||。
下面有个speexDemo整个项目在工程里，音频加密解密都正常，亲测可用。学习这块时候CSDN下来的， 搬过来凑合数。
public static void raw2spx(String inFileName, String outFileName) {
FileInputStream rawFileInputStream = null;
FileOutputStream fileOutputStream = null;
try {
rawFileInputStream = new FileInputStream(inFileName);
fileOutputStream = new FileOutputStream(outFileName);
byte[] rawbyte = new byte[320];
byte[] encoded = new byte[160];
//将原数据转换成spx压缩的文件，speex只能编码160字节的数据，需要使用一个循环
int readedtotal = 0;
int size = 0;
int encodedtotal = 0;
while ((size = rawFileInputStream.read(rawbyte, 0, 320)) != -1) {
readedtotal = readedtotal + size;
short[] rawdata = ShortByteUtil.byteArray2ShortArray(rawbyte);
int encodesize = SpeexUtil.getInstance().encode(rawdata, 0, encoded, rawdata.length);
fileOutputStream.write(encoded, 0, encodesize);
encodedtotal = encodedtotal + encodesize;
}
fileOutputStream.close();
rawFileInputStream.close();
} catch (Exception e) {
}
}
最后附上项目 NTSDemo地址 https://github.com/weiminsir/NTSDemo
Date:2018/10/17
Author:weimin