libevent（六）bufferevent

先来看一下bufferevent的创建和工作流程：

base=event_base_new();
bev=bufferevent_socket_new();
bufferevent_setcb(bev,do_read,do_write,do_event,(void *)base);
bufferevent_enable(bev, EV_READ|EV_WRITE);
bufferevent_socket_connect();
event_base_dispatch(base);

大致过程就是这样，创建一个event_base,然后调用bufferevent_socket_new为其分配内存，bufferevent_setcb设置回调函数，bufferevent_enable使能某些标志，最后循环event_base。event_base坚监听此bufferevent上enable的事件，事件触发，调用响应的回调函数去处理事件。

结构体bufferevent(include/event2/bufferevent_struct.h)

struct bufferevent {
    struct event_base *ev_base;//创建此缓冲区事件的事件库

    const struct bufferevent_ops *be_ops;

    struct event ev_read;//当超时或套接字发生时触发的读取事件准备读取数据。仅用于某些子类型bufferevent
    struct event ev_write;//当超时或套接字发生时触发的写入事件准备读取数据。仅用于某些子类型bufferevent


    struct evbuffer *input;//一个输入缓冲区。只允许bufferevent向其中添加数据这个缓冲区，允许用户将其耗尽
    struct evbuffer *output;//一个输入缓冲区。只有bufferevent允许清空数据，允许用户从这个缓冲区添加它

    struct event_watermark wm_read;
    struct event_watermark wm_write;//读取和写入水位设置结构体

    bufferevent_data_cb readcb;
    bufferevent_data_cb writecb;
    bufferevent_event_cb errorcb;//这应该被称为'eventcb
    //分别是可读可写出错事件发生时，调用的回调函数
    void *cbarg;//传给回调函数的参数
    
    struct timeval timeout_read;
    struct timeval timeout_write;//读写事件event的超时值
    
    short enabled;//当前支持的事件
};

结构体evbuffer

struct evbuffer {
    /** The first chain in this buffer's linked list of chains. */
    struct evbuffer_chain *first;
    /** The last chain in this buffer's linked list of chains. */
    struct evbuffer_chain *last;


    /** Pointer to the next pointer pointing at the 'last_with_data' chain.
     *
     * To unpack:
     *
     * The last_with_data chain is the last chain that has any data in it.
     * If all chains in the buffer are empty, it is the first chain.
     * If the buffer has no chains, it is NULL.
     *
     * The last_with_datap pointer points at _whatever 'next' pointer_
     * pointing at the last_with_data chain. If the last_with_data chain
     * is the first chain, or it is NULL, then the last_with_datap pointer
     * is &buf->first.
     */
    struct evbuffer_chain **last_with_datap;


    /** Total amount of bytes stored in all chains.*/
    size_t total_len;


    /** Number of bytes we have added to the buffer since we last tried to
     * invoke callbacks. */
    size_t n_add_for_cb;
    /** Number of bytes we have removed from the buffer since we last
     * tried to invoke callbacks. */
    size_t n_del_for_cb;


#ifndef EVENT__DISABLE_THREAD_SUPPORT
    /** A lock used to mediate access to this buffer. */
    void *lock;
#endif
    /** True iff we should free the lock field when we free this
     * evbuffer. */
    unsigned own_lock : 1;
    /** True iff we should not allow changes to the front of the buffer
     * (drains or prepends). */
    unsigned freeze_start : 1;
    /** True iff we should not allow changes to the end of the buffer
     * (appends) */
    unsigned freeze_end : 1;
    /** True iff this evbuffer's callbacks are not invoked immediately
     * upon a change in the buffer, but instead are deferred to be invoked
     * from the event_base's loop.  Useful for preventing enormous stack
     * overflows when we have mutually recursive callbacks, and for
     * serializing callbacks in a single thread. */
    unsigned deferred_cbs : 1;
#ifdef _WIN32
    /** True iff this buffer is set up for overlapped IO. */
    unsigned is_overlapped : 1;
#endif
    /** Zero or more EVBUFFER_FLAG_* bits */
    ev_uint32_t flags;


    /** Used to implement deferred callbacks. */
    struct event_base *cb_queue;


    /** A reference count on this evbuffer.  When the reference count
     * reaches 0, the buffer is destroyed.  Manipulated with
     * evbuffer_incref and evbuffer_decref_and_unlock and
     * evbuffer_free. */
    int refcnt;


    /** A struct event_callback handle to make all of this buffer's callbacks
     * invoked from the event loop. */
    struct event_callback deferred;


    /** A doubly-linked-list of callback functions */
    LIST_HEAD(evbuffer_cb_queue, evbuffer_cb_entry) callbacks;


    /** The parent bufferevent object this evbuffer belongs to.
     * NULL if the evbuffer stands alone. */
    struct bufferevent *parent;
};

结构体bufferevent_private(bufferevent_sock.c)

struct bufferevent_private
{
struct bufferevent bev;//底层bufferevent结构
struct evbuffer_cb_entry *read_watermarks_cb;//设置input evbuffer的高水位时，需要一个evbuffer回调函数配合工作
/** If set, we should free the lock when we free the bufferevent. */
unsigned own_lock : 1;

unsigned readcb_pending : 1;
unsigned writecb_pending : 1;

unsigned connecting : 1;

unsigned connection_refused : 1;

short eventcb_pending;

bufferevent_suspend_flags read_suspended;
bufferevent_suspend_flags write_suspended;

int errno_pending;
int dns_error;

struct event_callback deferred;

enum bufferevent_options options;

int refcnt;

void *lock;

ev_ssize_t max_single_read;
ev_ssize_t max_single_write;

struct bufferevent_rate_limit *rate_limiting;
union {
        struct sockaddr_in6 in6;
        struct sockaddr_in in;
    } conn_address;

    struct evdns_getaddrinfo_request *dns_request;
}

新建一个bufferevent

结构体bufferevent_ops(bufferevent-internal.h)

struct bufferevent_ops {
    /** The name of the bufferevent's type. */
    const char *type;
    /** At what offset into the implementation type will we find a
        bufferevent structure?


        Example: if the type is implemented as
        struct bufferevent_x {
           int extra_data;
           struct bufferevent bev;
        }
        then mem_offset should be offsetof(struct bufferevent_x, bev)
    */
    off_t mem_offset;
    
    /** Enables one or more of EV_READ|EV_WRITE on a bufferevent.  Does
        not need to adjust the 'enabled' field.  Returns 0 on success, -1
        on failure.
     */
    int (*enable)(struct bufferevent *, short);


    /** Disables one or more of EV_READ|EV_WRITE on a bufferevent.  Does
        not need to adjust the 'enabled' field.  Returns 0 on success, -1
        on failure.
     */
    int (*disable)(struct bufferevent *, short);


    /** Detatches the bufferevent from related data structures. Called as
     * soon as its reference count reaches 0. */
    void (*unlink)(struct bufferevent *);


    /** Free any storage and deallocate any extra data or structures used
        in this implementation. Called when the bufferevent is
        finalized.
     */
    void (*destruct)(struct bufferevent *);


    /** Called when the timeouts on the bufferevent have changed.*/
    int (*adj_timeouts)(struct bufferevent *);


    /** Called to flush data. */
    int (*flush)(struct bufferevent *, short, enum bufferevent_flush_mode);


    /** Called to access miscellaneous fields. */
    int (*ctrl)(struct bufferevent *, enum bufferevent_ctrl_op, union bufferevent_ctrl_data *);


};

const struct bufferevent_ops bufferevent_ops_socket = {
    "socket",
    evutil_offsetof(struct bufferevent_private, bev),
    be_socket_enable,
    be_socket_disable,
    NULL, /* unlink */
    be_socket_destruct,
    bufferevent_generic_adj_existing_timeouts_,
    be_socket_flush,
    be_socket_ctrl,
};

struct bufferevent *bufferevent_socket_new(struct event_base *base, evutil_socket_t fd,
    int options)
{
    struct bufferevent_private *bufev_p;
    struct bufferevent *bufev;


#ifdef _WIN32
    if (base && event_base_get_iocp_(base))
        return bufferevent_async_new_(base, fd, options);
#endif


    if ((bufev_p = mm_calloc(1, sizeof(struct bufferevent_private)))== NULL)
        return NULL;//为bufev_p分配内存


    if (bufferevent_init_common_(bufev_p, base, &bufferevent_ops_socket,
                    options) < 0) {//初始化一些公共选项
        mm_free(bufev_p);
        return NULL;
    }
    bufev = &bufev_p->bev;//获取初始化的bufferevent

    evbuffer_set_flags(bufev->output, EVBUFFER_FLAG_DRAINS_TO_FD);//设置输出缓冲区的标志


    event_assign(&bufev->ev_read, bufev->ev_base, fd,
        EV_READ|EV_PERSIST|EV_FINALIZE, bufferevent_readcb, bufev);//注册读事件到event_base，当内核缓冲区有数据到达时，回调函数会被调用
    event_assign(&bufev->ev_write, bufev->ev_base, fd,
        EV_WRITE|EV_PERSIST|EV_FINALIZE, bufferevent_writecb, bufev);//注册写事件到event_base,当输出缓冲区有数据时，该函数会被调用


    evbuffer_add_cb(bufev->output, bufferevent_socket_outbuf_cb, bufev);//设置输出缓冲区的回调函数


    evbuffer_freeze(bufev->input, 0);//冻结输入缓冲区evbuffer->freeze_end = 1;
    evbuffer_freeze(bufev->output, 1);//冻结输出缓冲区evbuffer->freeze_start = 1;


    return bufev;
}

函数bufferevent_init_common_(bufev_p, base, &bufferevent_ops_socket,options)

int bufferevent_init_common_(struct bufferevent_private *bufev_private,
    struct event_base *base,
    const struct bufferevent_ops *ops,
    enum bufferevent_options options)
{
    struct bufferevent *bufev = &bufev_private->bev;//取出private中的bufferevent结构

    if (!bufev->input) {
        if ((bufev->input = evbuffer_new()) == NULL)
            goto err;
    }//为输入（读）缓冲区分配内存


    if (!bufev->output) {
        if ((bufev->output = evbuffer_new()) == NULL)
            goto err;
    }//为输出（写）缓冲区分配内存


    bufev_private->refcnt = 1;
    bufev->ev_base = base;//设置其所属的event_base


    /* Disable timeouts. */
    evutil_timerclear(&bufev->timeout_read);
    evutil_timerclear(&bufev->timeout_write);//清空读写时间超时值，默认为0即代表disable超时
    
    bufev->be_ops = ops;  //上面的全局结构体bufferevent_ops                    
    
    if (bufferevent_ratelim_init_(bufev_private))//设置bufferevent_private结构的一些限制的变量
        goto err;
    
    /*
     * Set to EV_WRITE so that using bufferevent_write is going to
     * trigger a callback.  Reading needs to be explicitly enabled
     * because otherwise no data will be available.
     */
    bufev->enabled = EV_WRITE;//bufferevent默认使能读事件


#ifndef EVENT__DISABLE_THREAD_SUPPORT
    if (options & BEV_OPT_THREADSAFE) {//为bufferevent分配线程锁
        if (bufferevent_enable_locking_(bufev, NULL) < 0)
            goto err;
    }
#endif
    if ((options & (BEV_OPT_DEFER_CALLBACKS|BEV_OPT_UNLOCK_CALLBACKS))
        == BEV_OPT_UNLOCK_CALLBACKS) {
        event_warnx("UNLOCK_CALLBACKS requires DEFER_CALLBACKS");
        goto err;
    }
    if (options & BEV_OPT_UNLOCK_CALLBACKS)
        event_deferred_cb_init_(
            &bufev_private->deferred,
            event_base_get_npriorities(base) / 2,
            bufferevent_run_deferred_callbacks_unlocked,
            bufev_private);
    else
        event_deferred_cb_init_(
            &bufev_private->deferred,
            event_base_get_npriorities(base) / 2,
            bufferevent_run_deferred_callbacks_locked,
            bufev_private);//延迟回调的初始化


    bufev_private->options = options;


    evbuffer_set_parent_(bufev->input, bufev);
    evbuffer_set_parent_(bufev->output, bufev);//关联evbuffer和其所属的bufferevent

    return 0;

err:
    if (bufev->input) {
        evbuffer_free(bufev->input);
        bufev->input = NULL;
    }
    if (bufev->output) {
        evbuffer_free(bufev->output);
        bufev->output = NULL;
    }
    return -1;
}

设置bufferevent回调函数

设置bufferevent回调函数
void    
bufferevent_setcb(struct bufferevent *bufev,
    bufferevent_data_cb readcb, bufferevent_data_cb writecb,
    bufferevent_event_cb eventcb, void *cbarg)
{           
    BEV_LOCK(bufev);
    
    bufev->readcb = readcb;
    bufev->writecb = writecb;
    bufev->errorcb = eventcb;//回调函数


    bufev->cbarg = cbarg;//回调参数
    BEV_UNLOCK(bufev);
}

使能bufferevent的输入和输出回调函数

int bufferevent_enable(struct bufferevent *bufev, short event)
{   
    struct bufferevent_private *bufev_private = BEV_UPCAST(bufev);//类似container_of宏，结构体bufferevent属于结构体bufferevent_private，根据bufferevent所在bufferevent_private的位置找到bufferevent_private的首地址
    short impl_events = event;
    int r = 0;
    
    bufferevent_incref_and_lock_(bufev);//上锁
    if (bufev_private->read_suspended)
        impl_events &= ~EV_READ;//如果读事件被挂起，则清除EV_READ标志
    if (bufev_private->write_suspended)
        impl_events &= ~EV_WRITE;//如果写事件被挂起，则清除EV_WRITE标志
        
    bufev->enabled |= event;//设置buffer_event的使能事件标志
    
    if (impl_events && bufev->be_ops->enable(bufev, impl_events) < 0)
        r = -1;
    if (r)
        event_debug(("%s: cannot enable 0x%hx on %p", __func__, event, bufev));
        
    bufferevent_decref_and_unlock_(bufev);//解锁
    return r;
}   

bufferevent->be_ops->enable函数

static int
be_socket_enable(struct bufferevent *bufev, short event)
{
    if (event & EV_READ &&
        bufferevent_add_event_(&bufev->ev_read, &bufev->timeout_read) == -1)
            return -1;
    if (event & EV_WRITE &&
        bufferevent_add_event_(&bufev->ev_write, &bufev->timeout_write) == -1)
            return -1;
    return 0;
}

int
bufferevent_add_event_(struct event *ev, const struct timeval *tv)
{
    if (!evutil_timerisset(tv))
        return event_add(ev, NULL);
    else
        return event_add(ev, tv);//添加事件到event_base的事件链表中，如果在bufferevent设置了超时，会添加超时
}

手动添加bufferevent的事件的超时值
在前面的bufferevent初始化中，设置的默认超时值为0，我们可以通过调用函数来设置超时值。

void
bufferevent_settimeout(struct bufferevent *bufev,
               int timeout_read, int timeout_write)
{
    struct timeval tv_read, tv_write;
    struct timeval *ptv_read = NULL, *ptv_write = NULL;


    memset(&tv_read, 0, sizeof(tv_read));
    memset(&tv_write, 0, sizeof(tv_write));


    if (timeout_read) {
        tv_read.tv_sec = timeout_read;
        ptv_read = &tv_read;
    }
    if (timeout_write) {
        tv_write.tv_sec = timeout_write;
        ptv_write = &tv_write;
    }


    bufferevent_set_timeouts(bufev, ptv_read, ptv_write);
}

int
bufferevent_set_timeouts(struct bufferevent *bufev,
             const struct timeval *tv_read,
             const struct timeval *tv_write)
{
    int r = 0;
    BEV_LOCK(bufev);
    if (tv_read) {
        bufev->timeout_read = *tv_read;
    } else {
        evutil_timerclear(&bufev->timeout_read);
    }
    if (tv_write) {
        bufev->timeout_write = *tv_write;
    } else {
        evutil_timerclear(&bufev->timeout_write);
    }


    if (bufev->be_ops->adj_timeouts)
        r = bufev->be_ops->adj_timeouts(bufev);
    BEV_UNLOCK(bufev);


    return r;
}

到此，bufferevent的初始化就结束了，接下来用户调用event_base_loop等函数，就可以启动监听了。

bufferevent初始化回调函数bufferevent_readcb

static void bufferevent_readcb(evutil_socket_t fd, short event, void *arg)
{
    struct bufferevent *bufev = arg;//回调参数为bufferevent
    struct bufferevent_private *bufev_p = BEV_UPCAST(bufev);//获取private
    struct evbuffer *input;
    int res = 0;
    short what = BEV_EVENT_READING;
    ev_ssize_t howmuch = -1, readmax=-1;


    bufferevent_incref_and_lock_(bufev);//上锁


    if (event == EV_TIMEOUT) {
        /* Note that we only check for event==EV_TIMEOUT. If
         * event==EV_TIMEOUT|EV_READ, we can safely ignore the
         * timeout, since a read has occurred */
        what |= BEV_EVENT_TIMEOUT;
        goto error;
    }//如果只设置了超时，回调函数不做任何事，直接返回


    input = bufev->input;//获取bufferevent的输入缓冲区


    /*
     * If we have a high watermark configured then we don't want to
     * read more data than would make us reach the watermark.
     */
    if (bufev->wm_read.high != 0) {//默认高水位为0，如果用户修改高水位不为0
        howmuch = bufev->wm_read.high - evbuffer_get_length(input);//查看缓冲区数据是否大于高水位
        /* we somehow lowered the watermark, stop reading */
        if (howmuch <= 0) {
            bufferevent_wm_suspend_read(bufev);
            goto done;//如果输入缓冲区数据大于高水位，就挂起读事件
        }
    }
    readmax = bufferevent_get_read_max_(bufev_p);/获取private的read_max值，默认为16384（在bufferevent_socket_new中被初始化
    if (howmuch < 0 || howmuch > readmax) /* The use of -1 for "unlimited"
                           * uglifies this code. XXXX */
        howmuch = readmax;//将howmuch设置为最大容量
    if (bufev_p->read_suspended)//挂起
        goto done;


    evbuffer_unfreeze(input, 0);//"解冻"输入缓冲区
    res = evbuffer_read(input, fd, (int)howmuch);//从内核缓冲区将数据读入到输入缓冲区，要求一次直接读取最大容量的数据，（对于边沿触发的多路复用IO如epoll，这样是最好的
    evbuffer_freeze(input, 0);//再次冻结输入缓冲区


    if (res == -1) {
        int err = evutil_socket_geterror(fd);//读取出错可能是文件描述符被关闭或网络断开连接
        if (EVUTIL_ERR_RW_RETRIABLE(err))
            goto reschedule;
        if (EVUTIL_ERR_CONNECT_REFUSED(err)) {
            bufev_p->connection_refused = 1;
            goto done;//判断错误和退出
        }
        /* error case */
        what |= BEV_EVENT_ERROR;//其他错误
    } else if (res == 0) {
        /* eof case */
        what |= BEV_EVENT_EOF;//读取完成
    }


    if (res <= 0)
        goto error;//其他错误，goto error


    bufferevent_decrement_read_buckets_(bufev_p, res);


    /* Invoke the user callback - must always be called last */
    bufferevent_trigger_nolock_(bufev, EV_READ, 0);//调用用户回调函数处理数据


    goto done;


reschedule:
    goto done;


error:
    bufferevent_disable(bufev, EV_READ);//清除bufferevent的EV_READ标志
    bufferevent_run_eventcb_(bufev, what, 0);//调用用户的错误处理回调


done:
    bufferevent_decref_and_unlock_(bufev);
}

函数bufferevent_trigger_nolock_

static inline void
bufferevent_trigger_nolock_(struct bufferevent *bufev, short iotype, int options)
{   
    if ((iotype & EV_READ) && ((options & BEV_TRIG_IGNORE_WATERMARKS) ||
        evbuffer_get_length(bufev->input) >= bufev->wm_read.low))
        bufferevent_run_readcb_(bufev, options);
    if ((iotype & EV_WRITE) && ((options & BEV_TRIG_IGNORE_WATERMARKS) ||
        evbuffer_get_length(bufev->output) <= bufev->wm_write.low))
        bufferevent_run_writecb_(bufev, options);
}

函数bufferevent_ratelim_init_

int
bufferevent_ratelim_init_(struct bufferevent_private *bev)
{       
    bev->rate_limiting = NULL;
    bev->max_single_read = MAX_SINGLE_READ_DEFAULT;
    bev->max_single_write = MAX_SINGLE_WRITE_DEFAULT;


    return 0;
}

void
bufferevent_run_readcb_(struct bufferevent *bufev, int options)
{   
    /* Requires that we hold the lock and a reference */
    struct bufferevent_private *p = BEV_UPCAST(bufev);
    if (bufev->readcb == NULL)
        return;
    if ((p->options|options) & BEV_OPT_DEFER_CALLBACKS) {
        p->readcb_pending = 1;
        SCHEDULE_DEFERRED(p);
    } else {
        bufev->readcb(bufev, bufev->cbarg);//调用用户回调函数
        bufferevent_inbuf_wm_check(bufev);
    }
}

函数bufferevent_wm_suspend_read

#define bufferevent_wm_suspend_read(b) \
    bufferevent_suspend_read_((b), BEV_SUSPEND_WM)
#define bufferevent_wm_unsuspend_read(b) \
    bufferevent_unsuspend_read_((b), BEV_SUSPEND_WM)

void
bufferevent_suspend_read_(struct bufferevent *bufev, bufferevent_suspend_flags what)
{
    struct bufferevent_private *bufev_private = BEV_UPCAST(bufev);
    BEV_LOCK(bufev);
    if (!bufev_private->read_suspended)
        bufev->be_ops->disable(bufev, EV_READ);
    bufev_private->read_suspended |= what;//设置挂起成员变量值
    BEV_UNLOCK(bufev);
}  

void
bufferevent_unsuspend_read_(struct bufferevent *bufev, bufferevent_suspend_flags what)
{
    struct bufferevent_private *bufev_private = BEV_UPCAST(bufev);
    BEV_LOCK(bufev);
    bufev_private->read_suspended &= ~what;
    if (!bufev_private->read_suspended && (bufev->enabled & EV_READ))
        bufev->be_ops->enable(bufev, EV_READ);
    BEV_UNLOCK(bufev);
}

static int
be_socket_disable(struct bufferevent *bufev, short event)
{
    struct bufferevent_private *bufev_p = BEV_UPCAST(bufev);
    if (event & EV_READ) {
        if (event_del(&bufev->ev_read) == -1)
            return -1;
    }//直接使bufferevent的事件ev_read变为非未决状态
    /* Don't actually disable the write if we are trying to connect. */
    if ((event & EV_WRITE) && ! bufev_p->connecting) {
        if (event_del(&bufev->ev_write) == -1)
            return -1;
    }
    return 0;
}

函数bufferevent_disable

int
bufferevent_disable(struct bufferevent *bufev, short event)
{   
    int r = 0;
    BEV_LOCK(bufev);
    bufev->enabled &= ~event;//清除EV_READ标志
    
    if (bufev->be_ops->disable(bufev, event) < 0)
        r = -1;
    if (r)
        event_debug(("%s: cannot disable 0x%hx on %p", __func__, event, bufev));
    
    BEV_UNLOCK(bufev);
    return r;
}   

函数bufferevent_run_eventcb_

void
bufferevent_run_eventcb_(struct bufferevent *bufev, short what, int options)
{   
    /* Requires that we hold the lock and a reference */
    struct bufferevent_private *p = BEV_UPCAST(bufev);
    if (bufev->errorcb == NULL)
        return;
    if ((p->options|options) & BEV_OPT_DEFER_CALLBACKS) {
        p->eventcb_pending |= what;
        p->errno_pending = EVUTIL_SOCKET_ERROR();
        SCHEDULE_DEFERRED(p);
    } else {
        bufev->errorcb(bufev, what, bufev->cbarg);
    }
}  

总结函数bufferevent_readcb：
如果该回调的event只有EV_TIMEOUT，即默认，就算被触发，也什么也不做，直接返回。
如果设置了高水位，判断输入缓冲区的数据量是否大于高水位，如果大于了，就进入挂起状态并退出回调，该状态会使事件ev_read变为非未决状态。
否则就直接将内核收到的数据写入到输入缓冲区，最后调用用户的read回调函数处理数据。

现在，只有一点，用户设置高水位，当数据量过大时，变为挂起，如何退出挂起状态并恢复ev_read回调函数？

函数bufferevent_setwatermark

void
bufferevent_setwatermark(struct bufferevent *bufev, short events,
    size_t lowmark, size_t highmark)
{
    struct bufferevent_private *bufev_private = BEV_UPCAST(bufev);


    BEV_LOCK(bufev);//上锁
    if (events & EV_WRITE) {
        bufev->wm_write.low = lowmark;
        bufev->wm_write.high = highmark;
    }


    if (events & EV_READ) {
        bufev->wm_read.low = lowmark;
        bufev->wm_read.high = highmark;//设置水位


        if (highmark) {//如果高水位不为0
            /* There is now a new high-water mark for read.
               enable the callback if needed, and see if we should
               suspend/bufferevent_wm_unsuspend. */


            if (bufev_private->read_watermarks_cb == NULL) {//如果该回调不存在
                bufev_private->read_watermarks_cb =
                    evbuffer_add_cb(bufev->input,
                            bufferevent_inbuf_wm_cb,
                            bufev);//设置输入缓冲区的回调函数
            }
            evbuffer_cb_set_flags(bufev->input,
                      bufev_private->read_watermarks_cb,
                      EVBUFFER_CB_ENABLED|EVBUFFER_CB_NODEFER);//设置标志


            if (evbuffer_get_length(bufev->input) >= highmark)
                bufferevent_wm_suspend_read(bufev);
            else if (evbuffer_get_length(bufev->input) < highmark)
                bufferevent_wm_unsuspend_read(bufev);//由于水位设置可能在运行过程中，过程中缓冲区也可能高于高水位，也可能由于水位的设置使其低于高水位，因此调用相应的挂起和恢复操作
        } else {
            /* There is now no high-water mark for read. */
            if (bufev_private->read_watermarks_cb)
                evbuffer_cb_clear_flags(bufev->input,
                    bufev_private->read_watermarks_cb,
                    EVBUFFER_CB_ENABLED);
            bufferevent_wm_unsuspend_read(bufev);
        }
    }
    BEV_UNLOCK(bufev);//释放锁
}

函数evbuffer_add_cb

struct evbuffer_cb_entry *
evbuffer_add_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg)
{                   
    struct evbuffer_cb_entry *e;
    if (! (e = mm_calloc(1, sizeof(struct evbuffer_cb_entry))))
        return NULL;
    EVBUFFER_LOCK(buffer);
    e->cb.cb_func = cb;
    e->cbarg = cbarg;
    e->flags = EVBUFFER_CB_ENABLED;
    LIST_INSERT_HEAD(&buffer->callbacks, e, next);
    EVBUFFER_UNLOCK(buffer);
    return e;
}   

函数bufferevent_inbuf_wm_cb

static void
bufferevent_inbuf_wm_cb(struct evbuffer *buf,
    const struct evbuffer_cb_info *cbinfo,
    void *arg)
{
    struct bufferevent *bufev = arg;
    size_t size;

    size = evbuffer_get_length(buf);

    if (size >= bufev->wm_read.high)
        bufferevent_wm_suspend_read(bufev);
    else
        bufferevent_wm_unsuspend_read(bufev);
}

针对此处的情况，只有设置了高水位的情况下，事件才会被挂起。
高水位的设置，回向输入缓冲区结构添加一个回调函数，该函数在缓冲区中数据量发生变化时，会被调用，如果数据超过高水位值，则事件被挂起，只有再到数据小于高水位值时，ev_read事件才会再次被添加到event_base上运行。

函数bufferevent_writecb

static void
bufferevent_writecb(evutil_socket_t fd, short event, void *arg)
{
    struct bufferevent *bufev = arg;
    struct bufferevent_private *bufev_p = BEV_UPCAST(bufev);
    int res = 0;
    short what = BEV_EVENT_WRITING;
    int connected = 0;
    ev_ssize_t atmost = -1;


    bufferevent_incref_and_lock_(bufev);


    if (event == EV_TIMEOUT) {
        /* Note that we only check for event==EV_TIMEOUT. If
         * event==EV_TIMEOUT|EV_WRITE, we can safely ignore the
         * timeout, since a read has occurred */
        what |= BEV_EVENT_TIMEOUT;
        goto error;
    }
    if (bufev_p->connecting) {
        int c = evutil_socket_finished_connecting_(fd);//c为0，说明未连接上，c为1，说明连接成功，c为-1说明连接出错或断开
        /* we need to fake the error if the connection was refused
         * immediately - usually connection to localhost on BSD */
        if (bufev_p->connection_refused) {
            bufev_p->connection_refused = 0;
            c = -1;//拒绝连接
        }


        if (c == 0)
            goto done;//未连接上，继续连接


        bufev_p->connecting = 0;
        if (c < 0) {
            event_del(&bufev->ev_write);
            event_del(&bufev->ev_read);
            bufferevent_run_eventcb_(bufev, BEV_EVENT_ERROR, 0);
            goto done;//连接出错移除事件并调用错误处理回调
        } else {
            connected = 1;
            bufferevent_socket_set_conn_address_fd_(bufev, fd);
#ifdef _WIN32
            if (BEV_IS_ASYNC(bufev)) {
                event_del(&bufev->ev_write);
                bufferevent_async_set_connected_(bufev);
                bufferevent_run_eventcb_(bufev,
                        BEV_EVENT_CONNECTED, 0);
                goto done;
            }
#endif
            bufferevent_run_eventcb_(bufev,
                    BEV_EVENT_CONNECTED, 0);
            if (!(bufev->enabled & EV_WRITE) ||
                bufev_p->write_suspended) {
                event_del(&bufev->ev_write);
                goto done;//连接成功，也会调用错误处理函数，如果事件处于挂起状态，撤下事件
            }
        }
    }


    atmost = bufferevent_get_write_max_(bufev_p);//获取最大容量


    if (bufev_p->write_suspended)
        goto done;//处于挂起状态goto done


    if (evbuffer_get_length(bufev->output)) {//输出缓冲区不为空
        evbuffer_unfreeze(bufev->output, 1);//解冻
        res = evbuffer_write_atmost(bufev->output, fd, atmost);//尽可能写入数据到socket
        evbuffer_freeze(bufev->output, 1);//冻结
        if (res == -1) {
            int err = evutil_socket_geterror(fd);
            if (EVUTIL_ERR_RW_RETRIABLE(err))
                goto reschedule;
            what |= BEV_EVENT_ERROR;
        } else if (res == 0) {
            /* eof case
               XXXX Actually, a 0 on write doesn't indicate
               an EOF. An ECONNRESET might be more typical.
             */
            what |= BEV_EVENT_EOF;
        }
        if (res <= 0)
            goto error;
        bufferevent_decrement_write_buckets_(bufev_p, res);
    }


    if (evbuffer_get_length(bufev->output) == 0) {
        event_del(&bufev->ev_write);
    }//输入缓冲区为空，撤销写事件


    /*
     * Invoke the user callback if our buffer is drained or below the
     * low watermark.
     */
    if (res || !connected) {
        bufferevent_trigger_nolock_(bufev, EV_WRITE, 0);
    }//调用用户写回调函数


    goto done;


reschedule:
    if (evbuffer_get_length(bufev->output) == 0) {
        event_del(&bufev->ev_write);
    }
    goto done;


error:
    bufferevent_disable(bufev, EV_WRITE);//撤销写事件
    bufferevent_run_eventcb_(bufev, what, 0);//调用用户的错误回调


done:
    bufferevent_decref_and_unlock_(bufev);
}

函数evutil_socket_finished_connecting_

/* Check whether a socket on which we called connect() is done
   connecting. Return 1 for connected, 0 for not yet, -1 for error.  In the
   error case, set the current socket errno to the error that happened during
   the connect operation. */
int
evutil_socket_finished_connecting_(evutil_socket_t fd)
{       
    int e;
    ev_socklen_t elen = sizeof(e);
        
    if (getsockopt(fd, SOL_SOCKET, SO_ERROR, (void*)&e, &elen) < 0)
        return -1;
        
    if (e) {
        if (EVUTIL_ERR_CONNECT_RETRIABLE(e))
            return 0;
        EVUTIL_SET_SOCKET_ERROR(e);
        return -1;
    }       

    return 1;
}

函数bufferevent_socket_set_conn_address_fd_

void
bufferevent_socket_set_conn_address_fd_(struct bufferevent *bev,
    evutil_socket_t fd)
{
    struct bufferevent_private *bev_p = BEV_UPCAST(bev);


    socklen_t len = sizeof(bev_p->conn_address);


    struct sockaddr *addr = (struct sockaddr *)&bev_p->conn_address;
    if (addr->sa_family != AF_UNSPEC)
        getpeername(fd, addr, &len);
}

读事件和写事件的触发是不一样的，读事件在内核缓冲区收到数据时，就会被触发。
而event_base上注册的写事件则是不断的被触发，这样会消耗大量的时间，而libevent会这样做：写事件会被不断触发，但是要向缓冲区写入数据，条件是网络已经连接，所以写事件首先判断网络是否连接，只有在连接成功的情况下，并且输出缓冲区的evbuffer中有用户添加的数据时，才会进行像socket写入数据的操作。如果还没连接上网络，则继续进行，如果连接出错，释放所有资源。而当输出缓冲区evbuffer中并不存在数据时，会进行一个操作：将写事件从event_base的循环中取消。那接下来还有一个问题，当输出缓冲区没有数据时，写事件被从event_base删除，当用户向输出缓冲区添加数据时，此时写事件如何再次被添加到event_base中继续运行呢？

在bufferevent_socket_new进行初始化的过程中，有一个类似读事件设置高水位时为输入缓冲区添加一个回调的操作：evbuffer_add_cb(bufev->output, bufferevent_socket_outbuf_cb, bufev);

static void
bufferevent_socket_outbuf_cb(struct evbuffer *buf,
    const struct evbuffer_cb_info *cbinfo,
    void *arg)
{
    struct bufferevent *bufev = arg;
    struct bufferevent_private *bufev_p = BEV_UPCAST(bufev);


    if (cbinfo->n_added &&
        (bufev->enabled & EV_WRITE) &&
        !event_pending(&bufev->ev_write, EV_WRITE, NULL) &&
        !bufev_p->write_suspended) {
        /* Somebody added data to the buffer, and we would like to
         * write, and we were not writing.  So, start writing. */
        if (bufferevent_add_event_(&bufev->ev_write, &bufev->timeout_write) == -1) {
            /* Should we log this? */
        }
    }
}

当输出缓冲区有数据时，并且buffevent使能了读标志，处于非挂起状态，将会添加写事件到event_base。

函数bufferevent_socket_connect

int
bufferevent_socket_connect(struct bufferevent *bev,
    const struct sockaddr *sa, int socklen)
{   
    struct bufferevent_private *bufev_p = BEV_UPCAST(bev);
        
    evutil_socket_t fd;
    int r = 0;
    int result=-1;
    int ownfd = 0;
    
    bufferevent_incref_and_lock_(bev);//上锁
    
    fd = bufferevent_getfd(bev);//获取传入的文件描述符
    if (fd < 0) {
        if (!sa)
            goto done;
        fd = evutil_socket_(sa->sa_family,
            SOCK_STREAM|EVUTIL_SOCK_NONBLOCK, 0);
        if (fd < 0)
            goto freesock;
        ownfd = 1;
    }//如果传入的文件描述符为负，创建文件描述符，并设置非阻塞
    if (sa) {
#ifdef _WIN32
        if (bufferevent_async_can_connect_(bev)) {
            bufferevent_setfd(bev, fd);
            r = bufferevent_async_connect_(bev, fd, sa, socklen);
            if (r < 0)
                goto freesock;
            bufev_p->connecting = 1;
            result = 0;
            goto done;
        } else
#endif
        r = evutil_socket_connect_(&fd, sa, socklen);
        if (r < 0)
            goto freesock;
    }//建立socket连接
#ifdef _WIN32
    /* ConnectEx() isn't always around, even when IOCP is enabled.
     * Here, we borrow the socket object's write handler to fall back
     * on a non-blocking connect() when ConnectEx() is unavailable. */
    if (BEV_IS_ASYNC(bev)) {
        event_assign(&bev->ev_write, bev->ev_base, fd,
            EV_WRITE|EV_PERSIST|EV_FINALIZE, bufferevent_writecb, bev);
    }
#endif
    bufferevent_setfd(bev, fd);//添加文件描述符到bufferevent的两个event设置监听的fd
    if (r == 0) {
        if (! be_socket_enable(bev, EV_WRITE)) {
            bufev_p->connecting = 1;
            result = 0;
            goto done;
        }//未连接上，添加可写事件到event_base
    } else if (r == 1) {
        /* The connect succeeded already. How very BSD of it. */
        result = 0;
        bufev_p->connecting = 1;
        bufferevent_trigger_nolock_(bev, EV_WRITE, BEV_OPT_DEFER_CALLBACKS);//连接成功，手动激活写事件
    } else {
        /* The connect failed already.  How very BSD of it. */
        result = 0;
        bufferevent_run_eventcb_(bev, BEV_EVENT_ERROR, BEV_OPT_DEFER_CALLBACKS);
        bufferevent_disable(bev, EV_WRITE|EV_READ);//连接出错，调用错误处理回调，并禁用读写事件
    }


    goto done;


freesock:
    if (ownfd)
        evutil_closesocket(fd);
done:
    bufferevent_decref_and_unlock_(bev);
    return result;
}