pjmedia系列之媒体设备pjmedia_snd_port
在simpleua.c文件，当协商成功call_on_media_update中，会创建音频设备对象。
static pjmedia_snd_port *g_snd_port; /* Sound device. */
static void call_on_media_update( pjsip_inv_session *inv, pj_status_t status)
{}
pjmedia_port *media_port;
/* Get the media port interface of the audio stream.
* Media port interface is basicly a struct containing get_frame() and
* put_frame() function. With this media port interface, we can attach
* the port interface to conference bridge, or directly to a sound
* player/recorder device.
*/
pjmedia_stream_get_port(g_med_stream, &media_port);
/* Create sound port */
pjmedia_snd_port_create(inv->pool,
PJMEDIA_AUD_DEFAULT_CAPTURE_DEV,
PJMEDIA_AUD_DEFAULT_PLAYBACK_DEV,
PJMEDIA_PIA_SRATE(&media_port->info),/* clock rate */
PJMEDIA_PIA_CCNT(&media_port->info),/* channel count */
PJMEDIA_PIA_SPF(&media_port->info), /* samples per frame*/
PJMEDIA_PIA_BITS(&media_port->info),/* bits per sample */
0,
&g_snd_port);
status = pjmedia_snd_port_connect(g_snd_port, media_port);
}首先从stream获取一个media_port的接口实例，这个实例是在创建流的时候创建的。
PJ_DEF(pj_status_t) pjmedia_stream_get_port( pjmedia_stream *stream,
pjmedia_port **p_port )
{
*p_port = &stream->port;
return PJ_SUCCESS;
}
pjmedia_stream_create()
{
...
stream->port.put_frame = &put_frame;
stream->port.get_frame = &get_frame;
...
}这里实际上是把stream的两个回调，设置到音频设备，当设备mic采集到数据时，最终调用put_frame，当要播放数据时，调用get_frame获取。stream中的两个回调稍后再分析，继续看音频设备。
结构体
struct pjmedia_snd_port
{
int rec_id;
int play_id;
pj_uint32_t aud_caps;
pjmedia_aud_param aud_param;
pjmedia_aud_stream *aud_stream;
pjmedia_dir dir;
pjmedia_port *port;
pjmedia_clock_src cap_clocksrc,
play_clocksrc;
unsigned clock_rate;
unsigned channel_count;
unsigned samples_per_frame;
unsigned bits_per_sample;
unsigned options;
unsigned prm_ec_options;
/* audio frame preview callbacks */
void *user_data;
pjmedia_aud_play_cb on_play_frame;
pjmedia_aud_rec_cb on_rec_frame;
};结构体里有两个回到函数指针on_play_frame、on_rec_frame，会指向前面讲的stream两个函数put_frame和get_frame。
创建
PJ_DEF(pj_status_t) pjmedia_snd_port_create( pj_pool_t *pool,
int rec_id,
int play_id,
unsigned clock_rate,
unsigned channel_count,
unsigned samples_per_frame,
unsigned bits_per_sample,
unsigned options,
pjmedia_snd_port **p_port)
{
pjmedia_snd_port_param param;
pj_status_t status;
pjmedia_snd_port_param_default(¶m);
/* Normalize rec_id & play_id */
if (rec_id < 0)
rec_id = PJMEDIA_AUD_DEFAULT_CAPTURE_DEV;
if (play_id < 0)
play_id = PJMEDIA_AUD_DEFAULT_PLAYBACK_DEV;
status = pjmedia_aud_dev_default_param(rec_id, ¶m.base);
if (status != PJ_SUCCESS)
return status;
param.base.dir = PJMEDIA_DIR_CAPTURE_PLAYBACK;
param.base.rec_id = rec_id;
param.base.play_id = play_id;
param.base.clock_rate = clock_rate;
param.base.channel_count = channel_count;
param.base.samples_per_frame = samples_per_frame;
param.base.bits_per_sample = bits_per_sample;
param.options = options;
param.ec_options = 0;
return pjmedia_snd_port_create2(pool, ¶m, p_port);
}初始化一些参数，然后调用 pjmedia_snd_port_create2。
/*
* Create sound port.
*/
PJ_DEF(pj_status_t) pjmedia_snd_port_create2(pj_pool_t *pool,
const pjmedia_snd_port_param *prm,
pjmedia_snd_port **p_port)
{
pjmedia_snd_port *snd_port;
pj_status_t status;
unsigned ptime_usec;
PJ_ASSERT_RETURN(pool && prm && p_port, PJ_EINVAL);
snd_port = PJ_POOL_ZALLOC_T(pool, pjmedia_snd_port);
PJ_ASSERT_RETURN(snd_port, PJ_ENOMEM);
snd_port->dir = prm->base.dir;
snd_port->rec_id = prm->base.rec_id;
snd_port->play_id = prm->base.play_id;
snd_port->clock_rate = prm->base.clock_rate;
snd_port->channel_count = prm->base.channel_count;
snd_port->samples_per_frame = prm->base.samples_per_frame;
snd_port->bits_per_sample = prm->base.bits_per_sample;
pj_memcpy(&snd_port->aud_param, &prm->base, sizeof(snd_port->aud_param));
snd_port->options = prm->options;
snd_port->prm_ec_options = prm->ec_options;
snd_port->user_data = prm->user_data;
snd_port->on_play_frame = prm->on_play_frame;
snd_port->on_rec_frame = prm->on_rec_frame;
ptime_usec = prm->base.samples_per_frame * 1000 / prm->base.channel_count /
prm->base.clock_rate * 1000;
pjmedia_clock_src_init(&snd_port->cap_clocksrc, PJMEDIA_TYPE_AUDIO,
snd_port->clock_rate, ptime_usec);
pjmedia_clock_src_init(&snd_port->play_clocksrc, PJMEDIA_TYPE_AUDIO,
snd_port->clock_rate, ptime_usec);
/* Start sound device immediately.
* If there's no port connected, the sound callback will return
* empty signal.
*/
status = start_sound_device( pool, snd_port );
if (status != PJ_SUCCESS) {
pjmedia_snd_port_destroy(snd_port);
return status;
}
*p_port = snd_port;
return PJ_SUCCESS;
}pjmedia_snd_port_create2同样是初始化一些参数，主要有时钟频率，最后调用start_sound_device。
static pj_status_t start_sound_device( pj_pool_t *pool,
pjmedia_snd_port *snd_port )
{
pjmedia_aud_rec_cb snd_rec_cb;
pjmedia_aud_play_cb snd_play_cb;
pjmedia_aud_param param_copy;
pj_status_t status;
...
/* Use different callback if format is not PCM */
if (snd_port->aud_param.ext_fmt.id == PJMEDIA_FORMAT_L16) {
snd_rec_cb = &rec_cb;
snd_play_cb = &play_cb;
} else {
snd_rec_cb = &rec_cb_ext;
snd_play_cb = &play_cb_ext;
}
/* Open the device */
status = pjmedia_aud_stream_create(¶m_copy,
snd_rec_cb,
snd_play_cb,
snd_port,
&snd_port->aud_stream);
...
/* Start sound stream. */
if (!(snd_port->options & PJMEDIA_SND_PORT_NO_AUTO_START)) {
status = pjmedia_aud_stream_start(snd_port->aud_stream);
}
return PJ_SUCCESS;
}由流程可知，真正到pjmedia_aud_stream_create函数才创建设备，并且传入两个回调rec_cb和play_cb，这两个回调并不是stream的那两个回调，而是sound_port.c这一层自己实现的回调，play_cb里面才会调用stream的回调，也就是多了一层，这点跟transport是一样的，都是在本层实现一个回调，然后在调用上层设置的回调。
pjmedia_aud_stream_create的实现在pjmedia/audiodev.c（注意不是pjmedia-audiodev/audiodev.c）。
/* API: Open audio stream object using the specified parameters. */
PJ_DEF(pj_status_t) pjmedia_aud_stream_create(const pjmedia_aud_param *prm,
pjmedia_aud_rec_cb rec_cb,
pjmedia_aud_play_cb play_cb,
void *user_data,
pjmedia_aud_stream **p_aud_strm)
{
pjmedia_aud_dev_factory *rec_f=NULL, *play_f=NULL, *f=NULL;
pjmedia_aud_param param;
pj_status_t status;
PJ_ASSERT_RETURN(prm && prm->dir && p_aud_strm, PJ_EINVAL);
PJ_ASSERT_RETURN(aud_subsys.pf, PJMEDIA_EAUD_INIT);
PJ_ASSERT_RETURN(prm->dir==PJMEDIA_DIR_CAPTURE ||
prm->dir==PJMEDIA_DIR_PLAYBACK ||
prm->dir==PJMEDIA_DIR_CAPTURE_PLAYBACK,
PJ_EINVAL);
status = lookup_dev(param.rec_id, &rec_f, &index);
status = lookup_dev(param.play_id, &play_f, &index);
/* Create the stream */
status = f->op->create_stream(f, ¶m, rec_cb, play_cb,
user_data, p_aud_strm);
if (status != PJ_SUCCESS)
return status;
/* Assign factory id to the stream */
(*p_aud_strm)->sys.drv_idx = f->sys.drv_idx;
return PJ_SUCCESS;
}先通过lookup_dev搜索设备工厂，然后通过工厂创建设备f->op->create_stream。关键点来了，如何搜索到设备。
/* Internal: lookup device id */
static pj_status_t lookup_dev(pjmedia_aud_dev_index id,
pjmedia_aud_dev_factory **p_f,
unsigned *p_local_index)
{
int f_id, index;
if (id < 0) {
unsigned i;
if (id == PJMEDIA_AUD_INVALID_DEV)
return PJMEDIA_EAUD_INVDEV;
for (i=0; idev_idx >= 0) {
id = drv->dev_idx;
make_global_index(i, &id);
break;
} else if (id==PJMEDIA_AUD_DEFAULT_CAPTURE_DEV &&
drv->rec_dev_idx >= 0)
{
id = drv->rec_dev_idx;
make_global_index(i, &id);
break;
} else if (id==PJMEDIA_AUD_DEFAULT_PLAYBACK_DEV &&
drv->play_dev_idx >= 0)
{
id = drv->play_dev_idx;
make_global_index(i, &id);
break;
}
}
if (id < 0) {
return PJMEDIA_EAUD_NODEFDEV;
}
}
f_id = GET_FID(aud_subsys.dev_list[id]);
index = GET_INDEX(aud_subsys.dev_list[id]);
if (f_id < 0 || f_id >= (int)aud_subsys.drv_cnt)
return PJMEDIA_EAUD_INVDEV;
if (index < 0 || index >= (int)aud_subsys.drv[f_id].dev_cnt)
return PJMEDIA_EAUD_INVDEV;
*p_f = aud_subsys.drv[f_id].f;
*p_local_index = (unsigned)index;
return PJ_SUCCESS;
} 从这里可以看出，全局变量aud_subsys已经存储了所有的音频设备，这里把它找出来即可。那音频设备是不是在初始化的时候就搜索完了呢？aud_subsys全局变量什么时候赋值的？继续分析
在pjmedia-audiodev/audiodev.c中，有个音频子系统初始化函数
PJ_DEF(pj_status_t) pjmedia_aud_subsys_init(pj_pool_factory *pf)
{
unsigned i;
pj_status_t status;
pjmedia_aud_subsys *aud_subsys = pjmedia_get_aud_subsys();
/* Allow init() to be called multiple times as long as there is matching
* number of shutdown().
*/
if (aud_subsys->init_count++ != 0) {
return PJ_SUCCESS;
}
/* Register error subsystem */
status = pj_register_strerror(PJMEDIA_AUDIODEV_ERRNO_START,
PJ_ERRNO_SPACE_SIZE,
&pjmedia_audiodev_strerror);
pj_assert(status == PJ_SUCCESS);
/* Init */
aud_subsys->pf = pf;
aud_subsys->drv_cnt = 0;
aud_subsys->dev_cnt = 0;
/* Register creation functions */
#if PJMEDIA_AUDIO_DEV_HAS_OPENSL
aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_opensl_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_ANDROID_JNI
aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_android_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_BB10
aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_bb10_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_ALSA
aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_alsa_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_COREAUDIO
aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_coreaudio_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_PORTAUDIO
aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_pa_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_WMME
aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_wmme_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_BDIMAD
aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_bdimad_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_SYMB_VAS
aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_symb_vas_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_SYMB_APS
aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_aps_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_SYMB_MDA
aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_symb_mda_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_WASAPI
aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_wasapi_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_NULL_AUDIO
aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_null_audio_factory;
#endif
/* Initialize each factory and build the device ID list */
for (i=0; idrv_cnt; ++i) {
status = pjmedia_aud_driver_init(i, PJ_FALSE);
if (status != PJ_SUCCESS) {
pjmedia_aud_driver_deinit(i);
continue;
}
}
return aud_subsys->dev_cnt ? PJ_SUCCESS : status;
} 也就是说，初始化的时候调用这个函数，就会根据根据编译宏，把各种音频类型工厂添加到子系统全局变量，其中我们关注pjmedia_alsa_factory。而音频子系统的初始化，是在创建媒体端点的时候。
PJ_INLINE(pj_status_t) pjmedia_endpt_create(pj_pool_factory *pf,
pj_ioqueue_t *ioqueue,
unsigned worker_cnt,
pjmedia_endpt **p_endpt)
{
/* This function is inlined to avoid build problem due to circular
* dependency, i.e: this function prevents pjmedia's dependency on
* pjmedia-audiodev.
*/
pj_status_t status;
/* Sound */
status = pjmedia_aud_subsys_init(pf);
if (status != PJ_SUCCESS)
return status;
status = pjmedia_endpt_create2(pf, ioqueue, worker_cnt, p_endpt);
if (status != PJ_SUCCESS) {
pjmedia_aud_subsys_shutdown();
}
return status;
}这样分析后，整个脉络就理清了 。初始化的时候创建媒体端点pjmedia_endpt，同时初始化了音频子系统，把各种类型的音频设备工厂添加到全局变量static pjmedia_aud_subsys aud_subsys;。这样当创建设备时，就可以遍历这些工厂，寻找合适的工厂，通过工厂创建设备实例。比如alsa类型的设备在alsa_dev.c
static pjmedia_aud_dev_factory_op alsa_factory_op =
{
&alsa_factory_init,
&alsa_factory_destroy,
&alsa_factory_get_dev_count,
&alsa_factory_get_dev_info,
&alsa_factory_default_param,
&alsa_factory_create_stream,
&alsa_factory_refresh
};到此，设备的创建流程基本分析完了。音频设备分三层，最底层的是各种设备类型，比如alsa，再上一层是抽象设备操作audiodev.c，最上面一层是设备接口sound_port。
数据流
创建完设备，我们再来分析数据流向，以alsa为例，在alsa_dev.c中，会创建播放和采集两条线程。
static pj_status_t alsa_stream_start (pjmedia_aud_stream *s)
{
struct alsa_stream *stream = (struct alsa_stream*)s;
pj_status_t status = PJ_SUCCESS;
stream->quit = 0;
if (stream->param.dir & PJMEDIA_DIR_PLAYBACK) {
status = pj_thread_create (stream->pool,
"alsasound_playback",
pb_thread_func,
stream,
0, //ZERO,
0,
&stream->pb_thread);
if (stream->param.dir & PJMEDIA_DIR_CAPTURE) {
status = pj_thread_create (stream->pool,
"alsasound_playback",
ca_thread_func,
stream,
0, //ZERO,
0,
&stream->ca_thread);
}
return status;
}以播放线程为例，采集线程一样。
static int pb_thread_func (void *arg)
{
struct alsa_stream* stream = (struct alsa_stream*) arg;
snd_pcm_t* pcm = stream->pb_pcm;
int size = stream->pb_buf_size;
snd_pcm_uframes_t nframes = stream->pb_frames;
void* user_data = stream->user_data;
char* buf = stream->pb_buf;
pj_timestamp tstamp;
int result;
pj_bzero (buf, size);
tstamp.u64 = 0;
snd_pcm_prepare (pcm);
while (!stream->quit) {
pjmedia_frame frame;
frame.type = PJMEDIA_FRAME_TYPE_AUDIO;
frame.buf = buf;
frame.size = size;
frame.timestamp.u64 = tstamp.u64;
frame.bit_info = 0;
result = stream->pb_cb (user_data, &frame);
if (result != PJ_SUCCESS || stream->quit)
break;
if (frame.type != PJMEDIA_FRAME_TYPE_AUDIO)
pj_bzero (buf, size);
result = snd_pcm_writei (pcm, buf, nframes);
if (result == -EPIPE) {
PJ_LOG (4,(THIS_FILE, "pb_thread_func: underrun!"));
snd_pcm_prepare (pcm);
} else if (result < 0) {
PJ_LOG (4,(THIS_FILE, "pb_thread_func: error writing data!"));
}
tstamp.u64 += nframes;
}
snd_pcm_drain (pcm);
TRACE_((THIS_FILE, "pb_thread_func: Stopped"));
return PJ_SUCCESS;
}播放线程先通过回调拿到待播放的音频数据stream->pb_cb ，然后写到声卡snd_pcm_writei。pb_cb就是sound_port.c中的play_cb，来看下play_cb的流程。
static pj_status_t play_cb(void *user_data, pjmedia_frame *frame)
{
pjmedia_snd_port *snd_port = (pjmedia_snd_port*) user_data;
pjmedia_port *port;
const unsigned required_size = (unsigned)frame->size;
pj_status_t status;
port = snd_port->port;
status = pjmedia_port_get_frame(port, frame);
/* Invoke preview callback */
if (snd_port->on_play_frame)
(*snd_port->on_play_frame)(snd_port->user_data, frame);
return PJ_SUCCESS;
}play_cb做了两件事
1、通过pjmedia_port* port获取一帧数据
port是snd_port的成员，这个成员是在什么时候赋值的？回到simpleua.c中，pjmedia_snd_port_create后，还调用了一个函数pjmedia_snd_port_connect(g_snd_port, media_port);把stream的media_port传进去。
PJ_DEF(pj_status_t) pjmedia_snd_port_connect( pjmedia_snd_port *snd_port,
pjmedia_port *port)
{
pjmedia_audio_format_detail *afd;
...
/* Port is okay. */
snd_port->port = port;
return PJ_SUCCESS;
}/**
* Get a frame from the port (and subsequent downstream ports).
*/
PJ_DEF(pj_status_t) pjmedia_port_get_frame( pjmedia_port *port,
pjmedia_frame *frame )
{
PJ_ASSERT_RETURN(port && frame, PJ_EINVAL);
if (port->get_frame)
return port->get_frame(port, frame);
else {
frame->type = PJMEDIA_FRAME_TYPE_NONE;
return PJ_EINVALIDOP;
}
}所以这个port->get_frame就是stream.c中的get_frame。
2、通过pjmedia_snd_port调用预览，实际上跟踪会发现，预览指针并没有赋值。
调用pjmedia_snd_port* snd_port的on_play_frame函数指针，通过user_data传递snd_port。从pjmedia_aud_stream_create函数可以知道，第4个参数就是user_data，这个参数是在start_sound_device传入的第二个参数pjmedia_snd_port *snd_port。跟踪发现，想创建设备的时候，为on_play_frame指针赋值为空，那什么时候有值？
这样整个播放数据流就清楚了。首先在alsa创建播放线程，播放线程通过一系列回调，从stream拿到一帧数据，然后写设备播放，关键是捋清楚多层回调的关系。