物理PC一台(记作A),安装QAT加密卡硬件和CentOS 7系统。
登录A,执行:
# yum install -y kernel kernel-devel openssl-devel pciutils zlib-devel gcc libudev-devel boost-devel
如果内核有更新,重启。
https://01.org/sites/default/files/downloads/intelr-quickassist-technology/qat1.7.upstream.l.1.0.3-42.tar.gz
https://github.com/intel/QAT_Engine/archive/v0.5.34.tar.gz
https://github.com/openssl/openssl/archive/OpenSSL_1_1_0g.tar.gz
http://nginx.org/download/nginx-1.12.2.tar.gz
http://www.haproxy.org/download/1.8/src/haproxy-1.8.4.tar.gz
只能用浏览器下载。
登录A,执行:
# mkdir /tmp/QAT
# tar zxvf qat1.7.upstream.l.*.tar.gz -C /tmp/QAT
# cd /tmp/QAT
# chmod -R o-rwx *
# ./configure
# make
# make install
登录A,执行:
# tar xzf OpenSSL_1_1_0g.tar.gz
# cd openssl-OpenSSL_1_1_0g
# ./config --prefix=/usr/local/ssl
# make depend (ifrecommended by the OpenSSL* build system)
# make
# make install
# export OPENSSL_ENGINES=/usr/local/ssl/lib/engines-1.1
# echo /usr/local/ssl/lib > /etc/ld.so.conf.d/qat.conf
# ldconfig
# tar zxf QAT_Engine-0.5.34.tar.gz
# cd QAT_Engine-0.5.34/qat_contig_mem
# make
# make load
# make test
# cd QAT_Engine-0.5.34
# ./configure --with-qat_dir=/tmp/QAT\
--with-openssl_dir=/path/to/openssl-OpenSSL_1_1_0g\
--with-openssl_install_dir=/usr/local/ssl\
--enable-upstream_driver \
--enable-usdm
# make
# make install
# cp /path/to/QAT_Engine-0.5.34/qat/config/c6xx/multi_process_event-driven_optimized/*/etc
# service qat_servicestop
# service qat_servicestart
nginx-qat.patch:
diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c
index a1baefb..e783b5f 100644
--- a/src/core/ngx_connection.c
+++ b/src/core/ngx_connection.c
@@ -957,6 +957,18 @@ ngx_configure_listening_sockets(ngx_cycle_t *cycle)
return;
}
+#if (NGX_SSL)
+void
+ngx_asynch_del_conn(ngx_connection_t *c)
+{
+ if (ngx_del_async_conn) {
+ if (c->num_async_fds) {
+ ngx_del_async_conn(c, NGX_DISABLE_EVENT);
+ c->num_async_fds--;
+ }
+ }
+}
+#endif
void
ngx_close_listening_sockets(ngx_cycle_t *cycle)
@@ -986,7 +998,11 @@ ngx_close_listening_sockets(ngx_cycle_t *cycle)
* for closed shared listening sockets unless
* the events was explicitly deleted
*/
-
+#if (NGX_SSL)
+ if (c->asynch) {
+ ngx_asynch_del_conn(c);
+ }
+#endif
ngx_del_event(c->read, NGX_READ_EVENT, 0);
} else {
@@ -1035,6 +1051,9 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log)
{
ngx_uint_t instance;
ngx_event_t *rev, *wev;
+#if (NGX_SSL)
+ ngx_event_t *aev;
+#endif
ngx_connection_t *c;
/* disable warning: Win32 SOCKET is u_int while UNIX socket is int */
@@ -1071,11 +1090,17 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log)
rev = c->read;
wev = c->write;
+#if (NGX_SSL)
+ aev = c->async;
+#endif
ngx_memzero(c, sizeof(ngx_connection_t));
c->read = rev;
c->write = wev;
+#if (NGX_SSL)
+ c->async = aev;
+#endif
c->fd = s;
c->log = log;
@@ -1083,17 +1108,32 @@ ngx_get_connection(ngx_socket_t s, ngx_log_t *log)
ngx_memzero(rev, sizeof(ngx_event_t));
ngx_memzero(wev, sizeof(ngx_event_t));
+#if (NGX_SSL)
+ ngx_memzero(aev, sizeof(ngx_event_t));
+#endif
rev->instance = !instance;
wev->instance = !instance;
+#if (NGX_SSL)
+ aev->instance = !instance;
+#endif
rev->index = NGX_INVALID_INDEX;
wev->index = NGX_INVALID_INDEX;
+#if (NGX_SSL)
+ aev->index = NGX_INVALID_INDEX;
+#endif
rev->data = c;
wev->data = c;
+#if (NGX_SSL)
+ aev->data = c;
+#endif
wev->write = 1;
+#if (NGX_SSL)
+ aev->async = 1;
+#endif
return c;
}
@@ -1132,11 +1172,26 @@ ngx_close_connection(ngx_connection_t *c)
ngx_del_timer(c->write);
}
+#if (NGX_SSL)
+ if (c->async->timer_set) {
+ ngx_del_timer(c->async);
+ }
+
+ if (c->asynch) {
+ ngx_asynch_del_conn(c);
+ }
+#endif
if (!c->shared) {
if (ngx_del_conn) {
ngx_del_conn(c, NGX_CLOSE_EVENT);
} else {
+#if (NGX_SSL)
+ if (c->asynch) {
+ ngx_asynch_del_conn(c);
+ }
+#endif
+
if (c->read->active || c->read->disabled) {
ngx_del_event(c->read, NGX_READ_EVENT, NGX_CLOSE_EVENT);
}
@@ -1155,8 +1210,17 @@ ngx_close_connection(ngx_connection_t *c)
ngx_delete_posted_event(c->write);
}
+#if (NGX_SSL)
+ if (c->async->posted) {
+ ngx_delete_posted_event(c->async);
+ }
+#endif
+
c->read->closed = 1;
c->write->closed = 1;
+#if (NGX_SSL)
+ c->async->closed = 1;
+#endif
ngx_reusable_connection(c, 0);
diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h
index 1d3e3a3..3a481d9 100644
--- a/src/core/ngx_connection.h
+++ b/src/core/ngx_connection.h
@@ -122,8 +122,14 @@ struct ngx_connection_s {
void *data;
ngx_event_t *read;
ngx_event_t *write;
+#if (NGX_SSL)
+ ngx_event_t *async;
+#endif
ngx_socket_t fd;
+#if (NGX_SSL)
+ ngx_socket_t async_fd;
+#endif
ngx_recv_pt recv;
ngx_send_pt send;
@@ -149,6 +155,7 @@ struct ngx_connection_s {
#if (NGX_SSL || NGX_COMPAT)
ngx_ssl_connection_t *ssl;
+ ngx_flag_t asynch;
#endif
struct sockaddr *local_sockaddr;
@@ -181,6 +188,9 @@ struct ngx_connection_s {
unsigned tcp_nopush:2; /* ngx_connection_tcp_nopush_e */
unsigned need_last_buf:1;
+#if (NGX_SSL)
+ unsigned char num_async_fds;
+#endif
#if (NGX_HAVE_AIO_SENDFILE || NGX_COMPAT)
unsigned busy_count:2;
@@ -205,6 +215,7 @@ struct ngx_connection_s {
ngx_listening_t *ngx_create_listening(ngx_conf_t *cf, struct sockaddr *sockaddr,
socklen_t socklen);
+void ngx_asynch_del_conn(ngx_connection_t *c);
ngx_int_t ngx_clone_listening(ngx_conf_t *cf, ngx_listening_t *ls);
ngx_int_t ngx_set_inherited_sockets(ngx_cycle_t *cycle);
ngx_int_t ngx_open_listening_sockets(ngx_cycle_t *cycle);
diff --git a/src/core/ngx_cycle.h b/src/core/ngx_cycle.h
index 2b48ccb..f79da12 100644
--- a/src/core/ngx_cycle.h
+++ b/src/core/ngx_cycle.h
@@ -71,7 +71,9 @@ struct ngx_cycle_s {
ngx_connection_t *connections;
ngx_event_t *read_events;
ngx_event_t *write_events;
-
+#if (NGX_SSL)
+ ngx_event_t *async_events;
+#endif
ngx_cycle_t *old_cycle;
ngx_str_t conf_file;
diff --git a/src/event/modules/ngx_epoll_module.c b/src/event/modules/ngx_epoll_module.c
index 76aee08..b45202e 100644
--- a/src/event/modules/ngx_epoll_module.c
+++ b/src/event/modules/ngx_epoll_module.c
@@ -122,6 +122,11 @@ static ngx_int_t ngx_epoll_notify(ngx_event_handler_pt handler);
#endif
static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
ngx_uint_t flags);
+#if (NGX_SSL)
+static ngx_int_t ngx_epoll_add_async_connection(ngx_connection_t *c);
+static ngx_int_t ngx_epoll_del_async_connection(ngx_connection_t *c,
+ ngx_uint_t flags);
+#endif
#if (NGX_HAVE_FILE_AIO)
static void ngx_epoll_eventfd_handler(ngx_event_t *ev);
@@ -196,6 +201,13 @@ static ngx_event_module_t ngx_epoll_module_ctx = {
ngx_epoll_process_events, /* process the events */
ngx_epoll_init, /* init the events */
ngx_epoll_done, /* done the events */
+#if (NGX_SSL)
+ ngx_epoll_add_async_connection, /* add an async conn */
+ ngx_epoll_del_async_connection /* del an async conn */
+#else
+ NULL,
+ NULL
+#endif
}
};
@@ -758,6 +770,54 @@ ngx_epoll_del_connection(ngx_connection_t *c, ngx_uint_t flags)
return NGX_OK;
}
+#if (NGX_SSL)
+static ngx_int_t
+ngx_epoll_add_async_connection(ngx_connection_t *c)
+{
+ struct epoll_event ee;
+
+ ee.events = EPOLLIN|EPOLLOUT|EPOLLET|EPOLLRDHUP;
+ ee.data.ptr = (void *) ((uintptr_t) c | (c->async->async << 1) | c->async->instance);
+
+ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
+ "epoll add async connection: fd:%d ev:%08XD", c->async_fd, ee.events);
+ if (epoll_ctl(ep, EPOLL_CTL_ADD, c->async_fd, &ee) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
+ "async add conn epoll_ctl(EPOLL_CTL_ADD, %d) failed", c->async_fd);
+ return NGX_ERROR;
+ }
+
+ c->async->active = 1;
+
+ return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_epoll_del_async_connection(ngx_connection_t *c, ngx_uint_t flags)
+{
+ int op;
+ struct epoll_event ee;
+
+ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+ "epoll del async connection: fd:%d", c->async_fd);
+
+ op = EPOLL_CTL_DEL;
+ ee.events = 0;
+ ee.data.ptr = NULL;
+ if (epoll_ctl(ep, op, c->async_fd, &ee) == -1) {
+ ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno,
+ "async del conn epoll_ctl(%d, %d) failed", op, c->async_fd);
+ c->async_fd = -1;
+ return NGX_ERROR;
+ }
+ c->async_fd = -1;
+ c->async->active = 0;
+
+ return NGX_OK;
+}
+#endif
+
#if (NGX_HAVE_EVENTFD)
@@ -791,6 +851,10 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
ngx_event_t *rev, *wev;
ngx_queue_t *queue;
ngx_connection_t *c;
+#if (NGX_SSL)
+ ngx_int_t async;
+ ngx_event_t *aev;
+#endif
/* NGX_TIMER_INFINITE == INFTIM */
@@ -837,7 +901,12 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
c = event_list[i].data.ptr;
instance = (uintptr_t) c & 1;
+#if (NGX_SSL)
+ async = ((uintptr_t) c & 2) >> 1;
+ c = (ngx_connection_t *) ((uintptr_t) c & (uintptr_t) ~3);
+#else
c = (ngx_connection_t *) ((uintptr_t) c & (uintptr_t) ~1);
+#endif
rev = c->read;
@@ -880,7 +949,11 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
}
#endif
+#if (NGX_SSL)
+ if ((revents & EPOLLIN) && rev->active && !async) {
+#else
if ((revents & EPOLLIN) && rev->active) {
+#endif
#if (NGX_HAVE_EPOLLRDHUP)
if (revents & EPOLLRDHUP) {
@@ -905,7 +978,11 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
wev = c->write;
+#if (NGX_SSL)
+ if ((revents & EPOLLOUT) && wev->active && !async) {
+#else
if ((revents & EPOLLOUT) && wev->active) {
+#endif
if (c->fd == -1 || wev->instance != instance) {
@@ -931,6 +1008,32 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags)
wev->handler(wev);
}
}
+#if (NGX_SSL)
+ aev = c->async;
+
+ if ((revents & EPOLLIN) && aev->active && async) {
+
+ if (c->async_fd == -1 || aev->instance!= instance) {
+ /*
+ * the stale event from a file descriptor
+ * that was just closed in this iteration
+ */
+
+ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+ "epoll: stale event %p", c);
+ continue;
+ }
+
+ aev->ready = 1;
+
+ if (flags & NGX_POST_EVENTS) {
+ ngx_post_event(aev, &ngx_posted_events);
+
+ } else {
+ aev->handler(aev);
+ }
+ }
+#endif
}
return NGX_OK;
diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c
index 57af813..ed973d2 100644
--- a/src/event/ngx_event.c
+++ b/src/event/ngx_event.c
@@ -170,7 +170,7 @@ static ngx_event_module_t ngx_event_core_module_ctx = {
ngx_event_core_create_conf, /* create configuration */
ngx_event_core_init_conf, /* init configuration */
- { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
+ { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}
};
@@ -569,6 +569,9 @@ ngx_event_process_init(ngx_cycle_t *cycle)
{
ngx_uint_t m, i;
ngx_event_t *rev, *wev;
+#if (NGX_SSL)
+ ngx_event_t *aev;
+#endif
ngx_listening_t *ls;
ngx_connection_t *c, *next, *old;
ngx_core_conf_t *ccf;
@@ -711,6 +714,19 @@ ngx_event_process_init(ngx_cycle_t *cycle)
wev[i].closed = 1;
}
+#if (NGX_SSL)
+ cycle->async_events = ngx_alloc(sizeof(ngx_event_t) * cycle->connection_n,
+ cycle->log);
+ if (cycle->async_events == NULL) {
+ return NGX_ERROR;
+ }
+
+ aev = cycle->async_events;
+ for (i = 0; i < cycle->connection_n; i++) {
+ aev[i].closed = 1;
+ aev[i].instance = 1;
+ }
+#endif
i = cycle->connection_n;
next = NULL;
@@ -721,6 +737,10 @@ ngx_event_process_init(ngx_cycle_t *cycle)
c[i].read = &cycle->read_events[i];
c[i].write = &cycle->write_events[i];
c[i].fd = (ngx_socket_t) -1;
+#if (NGX_SSL)
+ c[i].async = &cycle->async_events[i];
+ c[i].async_fd = (ngx_socket_t) -1;
+#endif
next = &c[i];
} while (i);
diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h
index 053bd16..6a9ded8 100644
--- a/src/event/ngx_event.h
+++ b/src/event/ngx_event.h
@@ -32,6 +32,10 @@ struct ngx_event_s {
unsigned write:1;
+#if (NGX_SSL)
+ unsigned async:1;
+#endif
+
unsigned accept:1;
/* used to detect the stale events in kqueue and epoll */
@@ -108,6 +112,9 @@ struct ngx_event_s {
#endif
ngx_event_handler_pt handler;
+#if (NGX_SSL)
+ ngx_event_handler_pt saved_handler;
+#endif
#if (NGX_HAVE_IOCP)
@@ -191,6 +198,8 @@ typedef struct {
ngx_int_t (*init)(ngx_cycle_t *cycle, ngx_msec_t timer);
void (*done)(ngx_cycle_t *cycle);
+ ngx_int_t (*add_async_conn)(ngx_connection_t *c);
+ ngx_int_t (*del_async_conn)(ngx_connection_t *c, ngx_uint_t flags);
} ngx_event_actions_t;
@@ -415,6 +424,8 @@ extern ngx_uint_t ngx_use_epoll_rdhup;
#define ngx_del_event ngx_event_actions.del
#define ngx_add_conn ngx_event_actions.add_conn
#define ngx_del_conn ngx_event_actions.del_conn
+#define ngx_add_async_conn ngx_event_actions.add_async_conn
+#define ngx_del_async_conn ngx_event_actions.del_async_conn
#define ngx_notify ngx_event_actions.notify
diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c
index 7756370..f8cec14 100644
--- a/src/event/ngx_event_accept.c
+++ b/src/event/ngx_event_accept.c
@@ -249,6 +249,9 @@ ngx_event_accept(ngx_event_t *ev)
rev->log = log;
wev->log = log;
+#if (NGX_SSL)
+ c->async->log = log;
+#endif
/*
* TODO: MT: - ngx_atomic_fetch_add()
@@ -735,6 +738,11 @@ ngx_disable_accept_events(ngx_cycle_t *cycle, ngx_uint_t all)
#endif
+#if (NGX_SSL)
+ if (c->asynch) {
+ ngx_asynch_del_conn(c);
+ }
+#endif
if (ngx_del_event(c->read, NGX_READ_EVENT, NGX_DISABLE_EVENT)
== NGX_ERROR)
{
diff --git a/src/event/ngx_event_openssl.c b/src/event/ngx_event_openssl.c
index 8c7c677..a581967 100644
--- a/src/event/ngx_event_openssl.c
+++ b/src/event/ngx_event_openssl.c
@@ -115,6 +115,13 @@ int ngx_ssl_next_certificate_index;
int ngx_ssl_certificate_name_index;
int ngx_ssl_stapling_index;
+static void
+ngx_ssl_empty_handler(ngx_event_t *aev)
+{
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, aev->log, 0, "ssl empty handler");
+
+ return;
+}
ngx_int_t
ngx_ssl_init(ngx_log_t *log)
@@ -336,6 +343,10 @@ ngx_ssl_create(ngx_ssl_t *ssl, ngx_uint_t protocols, void *data)
SSL_CTX_set_mode(ssl->ctx, SSL_MODE_NO_AUTO_CHAIN);
#endif
+ if (ssl->asynch) {
+ SSL_CTX_set_mode(ssl->ctx, SSL_MODE_ASYNC);
+ }
+
SSL_CTX_set_read_ahead(ssl->ctx, 1);
SSL_CTX_set_info_callback(ssl->ctx, ngx_ssl_info_callback);
@@ -366,6 +377,26 @@ ngx_ssl_certificates(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_array_t *certs,
return NGX_OK;
}
+static ENGINE *
+ngx_ssl_setup_engine(const char *engine)
+{
+ ENGINE *e = NULL;
+
+#ifndef OPENSSL_NO_ENGINE
+ if (engine) {
+ if ((e = ENGINE_by_id(engine)) == NULL) {
+ return NULL;
+ }
+
+ ENGINE_ctrl_cmd(e, "SET_USER_INTERFACE", 0, NULL, 0, 1);
+ if (!ENGINE_set_default(e, ENGINE_METHOD_ALL)) {
+ ENGINE_free(e);
+ return NULL;
+ }
+ }
+#endif
+ return e;
+}
ngx_int_t
ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,
@@ -375,6 +406,7 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,
X509 *x509;
u_long n;
ngx_str_t *pwd;
+ char *qat_engine = "qat";
ngx_uint_t tries;
if (ngx_conf_full_name(cf->cycle, cert, 1) != NGX_OK) {
@@ -494,6 +526,12 @@ ngx_ssl_certificate(ngx_conf_t *cf, ngx_ssl_t *ssl, ngx_str_t *cert,
BIO_free(bio);
+ ssl->engine = ngx_ssl_setup_engine(qat_engine);
+ if (!ssl->engine) {
+ ngx_ssl_error(NGX_LOG_NOTICE, ssl->log, 0,
+ "Load ENGINE(\"%s\") failed", qat_engine);
+ }
+
if (ngx_strncmp(key->data, "engine:", sizeof("engine:") - 1) == 0) {
#ifndef OPENSSL_NO_ENGINE
@@ -1193,6 +1231,7 @@ ngx_ssl_create_connection(ngx_ssl_t *ssl, ngx_connection_t *c, ngx_uint_t flags)
}
c->ssl = sc;
+ c->asynch = ssl->asynch;
return NGX_OK;
}
@@ -1211,6 +1250,100 @@ ngx_ssl_set_session(ngx_connection_t *c, ngx_ssl_session_t *session)
return NGX_OK;
}
+ngx_int_t
+ngx_ssl_async_process_fds(ngx_connection_t *c)
+{
+ OSSL_ASYNC_FD *add_fds = NULL;
+ OSSL_ASYNC_FD *del_fds = NULL;
+ size_t num_add_fds = 0;
+ size_t num_del_fds = 0;
+ unsigned loop = 0;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+ "ngx_ssl_async_process_fds called");
+
+ if (!ngx_del_async_conn || !ngx_add_async_conn) {
+ ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+ "Async notifications not supported");
+ return 0;
+ }
+
+ SSL_get_changed_async_fds(c->ssl->connection, NULL, &num_add_fds,
+ NULL, &num_del_fds);
+
+ if (num_add_fds) {
+ add_fds = ngx_alloc(num_add_fds * sizeof(OSSL_ASYNC_FD), c->log);
+ if (add_fds == NULL) {
+ ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+ "Memory Allocation Error");
+ return 0;
+ }
+ }
+
+ if (num_del_fds) {
+ del_fds = ngx_alloc(num_del_fds * sizeof(OSSL_ASYNC_FD), c->log);
+ if (del_fds == NULL) {
+ ngx_ssl_error(NGX_LOG_ALERT, c->log, 0,
+ "Memory Allocation Error");
+ if (add_fds)
+ ngx_free(add_fds);
+ return 0;
+ }
+ }
+
+ SSL_get_changed_async_fds(c->ssl->connection, add_fds, &num_add_fds,
+ del_fds, &num_del_fds);
+
+ if (num_del_fds) {
+ for (loop = 0; loop < num_del_fds; loop++) {
+ c->async_fd = del_fds[loop];
+ if (c->num_async_fds) {
+ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "%s: deleting fd = %d", __func__, c->async_fd);
+ ngx_del_async_conn(c, NGX_DISABLE_EVENT);
+ c->num_async_fds--;
+ }
+ }
+ }
+ if (num_add_fds) {
+ for (loop = 0; loop < num_add_fds; loop++) {
+ if (c->num_async_fds == 0) {
+ c->num_async_fds++;
+ c->async_fd = add_fds[loop];
+ ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, "%s: adding fd = %d", __func__, c->async_fd);
+ ngx_add_async_conn(c);
+ }
+ }
+ }
+
+ if (add_fds)
+ ngx_free(add_fds);
+ if (del_fds)
+ ngx_free(del_fds);
+
+ return 1;
+}
+
+static void
+ngx_ssl_handshake_async_handler(ngx_event_t *aev)
+{
+ ngx_connection_t *c;
+
+ c = aev->data;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+ "SSL handshake async handler");
+
+ aev->ready = 0;
+ aev->handler = ngx_ssl_empty_handler;
+ c->read->handler = c->read->saved_handler;
+
+ if (ngx_ssl_handshake(c) == NGX_AGAIN) {
+ return;
+ }
+
+ c->ssl->handler(c);
+}
+
ngx_int_t
ngx_ssl_handshake(ngx_connection_t *c)
@@ -1226,6 +1359,10 @@ ngx_ssl_handshake(ngx_connection_t *c)
if (n == 1) {
+ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
+
if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
return NGX_ERROR;
}
@@ -1307,6 +1444,10 @@ ngx_ssl_handshake(ngx_connection_t *c)
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d", sslerr);
if (sslerr == SSL_ERROR_WANT_READ) {
+ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
+
c->read->ready = 0;
c->read->handler = ngx_ssl_handshake_handler;
c->write->handler = ngx_ssl_handshake_handler;
@@ -1323,6 +1464,10 @@ ngx_ssl_handshake(ngx_connection_t *c)
}
if (sslerr == SSL_ERROR_WANT_WRITE) {
+ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
+
c->write->ready = 0;
c->read->handler = ngx_ssl_handshake_handler;
c->write->handler = ngx_ssl_handshake_handler;
@@ -1338,6 +1483,21 @@ ngx_ssl_handshake(ngx_connection_t *c)
return NGX_AGAIN;
}
+ if (c->asynch && sslerr == SSL_ERROR_WANT_ASYNC) {
+ c->async->handler = ngx_ssl_handshake_async_handler;
+ c->read->saved_handler = c->read->handler;
+ c->read->handler = ngx_ssl_empty_handler;
+
+ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+ "SSL ASYNC WANT recieved: \"%s\"", __func__);
+
+ if (ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
+
+ return NGX_AGAIN;
+ }
+
err = (sslerr == SSL_ERROR_SYSCALL) ? ngx_errno : 0;
c->ssl->no_wait_shutdown = 1;
@@ -1378,6 +1538,11 @@ ngx_ssl_handshake_handler(ngx_event_t *ev)
return;
}
+ /*
+ * empty the handler of async event to avoid
+ * going back to previous ssl shutdown state
+ */
+ c->async->handler = ngx_ssl_empty_handler;
c->ssl->handler(c);
}
@@ -1518,6 +1683,22 @@ ngx_ssl_recv(ngx_connection_t *c, u_char *buf, size_t size)
}
}
+static void
+ngx_ssl_read_async_handler(ngx_event_t *aev)
+{
+ ngx_connection_t *c;
+
+ c = aev->data;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+ "SSL read async handler");
+
+ aev->ready = 0;
+ aev->handler = ngx_ssl_empty_handler;
+ c->read->handler = c->read->saved_handler;
+
+ c->read->handler(c->read);
+}
static ngx_int_t
ngx_ssl_handle_recv(ngx_connection_t *c, int n)
@@ -1549,6 +1730,10 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n)
if (n > 0) {
+ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
+
if (c->ssl->saved_write_handler) {
c->write->handler = c->ssl->saved_write_handler;
@@ -1572,6 +1757,10 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n)
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d", sslerr);
if (sslerr == SSL_ERROR_WANT_READ) {
+ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
+
c->read->ready = 0;
return NGX_AGAIN;
}
@@ -1581,6 +1770,10 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n)
ngx_log_error(NGX_LOG_INFO, c->log, 0,
"peer started SSL renegotiation");
+ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
+
c->write->ready = 0;
if (ngx_handle_write_event(c->write, 0) != NGX_OK) {
@@ -1599,6 +1792,21 @@ ngx_ssl_handle_recv(ngx_connection_t *c, int n)
return NGX_AGAIN;
}
+ if (c->asynch && sslerr == SSL_ERROR_WANT_ASYNC) {
+ c->async->handler = ngx_ssl_read_async_handler;
+ c->read->saved_handler = c->read->handler;
+ c->read->handler = ngx_ssl_empty_handler;
+
+ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+ "SSL ASYNC WANT recieved: \"%s\"", __func__);
+
+ if (ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
+
+ return NGX_AGAIN;
+ }
+
c->ssl->no_wait_shutdown = 1;
c->ssl->no_send_shutdown = 1;
@@ -1787,6 +1995,22 @@ ngx_ssl_send_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
return in;
}
+static void
+ngx_ssl_write_async_handler(ngx_event_t *aev)
+{
+ ngx_connection_t *c;
+
+ c = aev->data;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, 0,
+ "SSL write async handler");
+
+ aev->ready = 0;
+ aev->handler = ngx_ssl_empty_handler;
+ c->read->handler = c->read->saved_handler;
+
+ c->write->handler(c->write);
+}
ssize_t
ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size)
@@ -1804,6 +2028,10 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size)
if (n > 0) {
+ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
+
if (c->ssl->saved_read_handler) {
c->read->handler = c->ssl->saved_read_handler;
@@ -1829,6 +2057,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size)
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0, "SSL_get_error: %d", sslerr);
if (sslerr == SSL_ERROR_WANT_WRITE) {
+ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
c->write->ready = 0;
return NGX_AGAIN;
}
@@ -1838,6 +2069,9 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size)
ngx_log_error(NGX_LOG_INFO, c->log, 0,
"peer started SSL renegotiation");
+ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
c->read->ready = 0;
if (ngx_handle_read_event(c->read, 0) != NGX_OK) {
@@ -1857,6 +2091,21 @@ ngx_ssl_write(ngx_connection_t *c, u_char *data, size_t size)
return NGX_AGAIN;
}
+ if (c->asynch && sslerr == SSL_ERROR_WANT_ASYNC) {
+ c->async->handler = ngx_ssl_write_async_handler;
+ c->read->saved_handler = c->read->handler;
+ c->read->handler = ngx_ssl_empty_handler;
+
+ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+ "SSL ASYNC WANT recieved: \"%s\"", __func__);
+
+ if (ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
+
+ return NGX_AGAIN;
+ }
+
c->ssl->no_wait_shutdown = 1;
c->ssl->no_send_shutdown = 1;
c->write->error = 1;
@@ -1888,6 +2137,40 @@ ngx_ssl_free_buffer(ngx_connection_t *c)
}
}
+static void
+ngs_ssl_shutdown_async_del_conn(ngx_connection_t *c)
+{
+ ngx_ssl_async_process_fds(c);
+ ngx_asynch_del_conn(c);
+ ngx_del_conn(c, NGX_DISABLE_EVENT);
+}
+
+static void
+ngx_ssl_shutdown_async_handler(ngx_event_t *aev)
+{
+ ngx_connection_t *c;
+ ngx_connection_handler_pt handler;
+
+ c = aev->data;
+ handler = c->ssl->handler;
+
+ if (aev->timedout) {
+ c->timedout = 1;
+ }
+
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, aev->log, 0,
+ "SSL shutdown async handler");
+
+ aev->ready = 0;
+ aev->handler = ngx_ssl_empty_handler;
+ c->read->handler = c->read->saved_handler;
+
+ if (ngx_ssl_shutdown(c) == NGX_AGAIN) {
+ return;
+ }
+
+ handler(c);
+}
ngx_int_t
ngx_ssl_shutdown(ngx_connection_t *c)
@@ -1902,6 +2185,21 @@ ngx_ssl_shutdown(ngx_connection_t *c)
* Avoid calling SSL_shutdown() if handshake wasn't completed.
*/
+ if (c->asynch) {
+ /* Check if there is inflight request */
+ if (SSL_want_async(c->ssl->connection) && !c->timedout) {
+ c->async->handler = ngx_ssl_shutdown_async_handler;
+ ngx_ssl_async_process_fds(c);
+ ngx_add_timer(c->async, 300);
+ return NGX_AGAIN;
+ }
+
+ /* Ignore errors from ngx_ssl_async_process_fds as
+ * we want to carry on and close the SSL connection
+ * anyway. */
+ ngs_ssl_shutdown_async_del_conn(c);
+ }
+
SSL_free(c->ssl->connection);
c->ssl = NULL;
@@ -1941,13 +2239,26 @@ ngx_ssl_shutdown(ngx_connection_t *c)
/* before 0.9.8m SSL_shutdown() returned 0 instead of -1 on errors */
if (n != 1 && ERR_peek_error()) {
+ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
sslerr = SSL_get_error(c->ssl->connection, n);
-
ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
"SSL_get_error: %d", sslerr);
}
+ else if (c->asynch && n == -1) {
+ sslerr = SSL_get_error(c->ssl->connection, n);
+ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+ "SSL_get_error async: %d", sslerr);
+ }
if (n == 1 || sslerr == 0 || sslerr == SSL_ERROR_ZERO_RETURN) {
+ if (c->asynch) {
+ /* Ignore errors from ngx_ssl_async_process_fds as
+ * we want to carry on and close the SSL connection
+ * anyway. */
+ ngs_ssl_shutdown_async_del_conn(c);
+ }
SSL_free(c->ssl->connection);
c->ssl = NULL;
@@ -1955,6 +2266,9 @@ ngx_ssl_shutdown(ngx_connection_t *c)
}
if (sslerr == SSL_ERROR_WANT_READ || sslerr == SSL_ERROR_WANT_WRITE) {
+ if (c->asynch && ngx_ssl_async_process_fds(c) == 0) {
+ return NGX_ERROR;
+ }
c->read->handler = ngx_ssl_shutdown_handler;
c->write->handler = ngx_ssl_shutdown_handler;
@@ -1973,6 +2287,28 @@ ngx_ssl_shutdown(ngx_connection_t *c)
return NGX_AGAIN;
}
+ if (c->asynch) {
+ if (sslerr == SSL_ERROR_WANT_ASYNC) {
+ c->async->handler = ngx_ssl_shutdown_async_handler;
+ c->read->saved_handler = ngx_ssl_shutdown_handler;
+ c->read->handler = ngx_ssl_empty_handler;
+ c->write->handler = ngx_ssl_shutdown_handler;
+
+ ngx_log_debug1(NGX_LOG_DEBUG_EVENT, c->log, 0,
+ "SSL ASYNC WANT recieved: \"%s\"", __func__);
+
+ /* Ignore errors from ngx_ssl_async_process_fds as
+ * we want to carry on anyway. */
+ ngx_ssl_async_process_fds(c);
+ return NGX_AGAIN;
+ }
+
+ /* Ignore errors from ngx_ssl_async_process_fds as
+ * we want to carry on and close the SSL connection
+ * anyway. */
+ ngs_ssl_shutdown_async_del_conn(c);
+ }
+
err = (sslerr == SSL_ERROR_SYSCALL) ? ngx_errno : 0;
ngx_ssl_connection_error(c, sslerr, err, "SSL_shutdown() failed");
@@ -2003,6 +2339,11 @@ ngx_ssl_shutdown_handler(ngx_event_t *ev)
return;
}
+ /*
+ * empty the handler of async event to avoid
+ * going back to previous ssl shutdown state
+ */
+ c->async->handler = ngx_ssl_empty_handler;
handler(c);
}
@@ -3145,6 +3486,9 @@ ngx_ssl_cleanup_ctx(void *data)
}
SSL_CTX_free(ssl->ctx);
+ if (ssl->engine) {
+ ENGINE_free(ssl->engine);
+ }
}
diff --git a/src/event/ngx_event_openssl.h b/src/event/ngx_event_openssl.h
index e093e10..3629d07 100644
--- a/src/event/ngx_event_openssl.h
+++ b/src/event/ngx_event_openssl.h
@@ -57,7 +57,9 @@
struct ngx_ssl_s {
SSL_CTX *ctx;
ngx_log_t *log;
+ ENGINE *engine;
size_t buffer_size;
+ ngx_flag_t asynch;
};
@@ -187,6 +189,7 @@ ngx_int_t ngx_ssl_set_session(ngx_connection_t *c, ngx_ssl_session_t *session);
ngx_int_t ngx_ssl_check_host(ngx_connection_t *c, ngx_str_t *name);
+#define ngx_ssl_waiting_for_async(c) SSL_waiting_for_async(c->ssl->connection)
ngx_int_t ngx_ssl_get_protocol(ngx_connection_t *c, ngx_pool_t *pool,
ngx_str_t *s);
@@ -239,7 +242,7 @@ ngx_int_t ngx_ssl_shutdown(ngx_connection_t *c);
void ngx_cdecl ngx_ssl_error(ngx_uint_t level, ngx_log_t *log, ngx_err_t err,
char *fmt, ...);
void ngx_ssl_cleanup_ctx(void *data);
-
+ngx_int_t ngx_ssl_async_process_fds(ngx_connection_t *c);
extern int ngx_ssl_connection_index;
extern int ngx_ssl_server_conf_index;
diff --git a/src/http/modules/ngx_http_ssl_module.c b/src/http/modules/ngx_http_ssl_module.c
index 2771ac1..d893ee7 100644
--- a/src/http/modules/ngx_http_ssl_module.c
+++ b/src/http/modules/ngx_http_ssl_module.c
@@ -43,6 +43,8 @@ static char *ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf,
static char *ngx_http_ssl_enable(ngx_conf_t *cf, ngx_command_t *cmd,
void *conf);
+static char *ngx_http_ssl_enable_asynch(ngx_conf_t *cf, ngx_command_t *cmd,
+ void *conf);
static char *ngx_http_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd,
void *conf);
static char *ngx_http_ssl_session_cache(ngx_conf_t *cf, ngx_command_t *cmd,
@@ -79,6 +81,13 @@ static ngx_command_t ngx_http_ssl_commands[] = {
offsetof(ngx_http_ssl_srv_conf_t, enable),
NULL },
+ { ngx_string("ssl_asynch"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_FLAG,
+ ngx_http_ssl_enable_asynch,
+ NGX_HTTP_SRV_CONF_OFFSET,
+ offsetof(ngx_http_ssl_srv_conf_t, enable_asynch),
+ NULL },
+
{ ngx_string("ssl_certificate"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_CONF_TAKE1,
ngx_conf_set_str_array_slot,
@@ -541,6 +550,7 @@ ngx_http_ssl_create_srv_conf(ngx_conf_t *cf)
*/
sscf->enable = NGX_CONF_UNSET;
+ sscf->enable_asynch = NGX_CONF_UNSET;
sscf->prefer_server_ciphers = NGX_CONF_UNSET;
sscf->buffer_size = NGX_CONF_UNSET_SIZE;
sscf->verify = NGX_CONF_UNSET_UINT;
@@ -578,6 +588,17 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child)
}
}
+ if (conf->enable_asynch == NGX_CONF_UNSET) {
+ if (prev->enable_asynch == NGX_CONF_UNSET) {
+ conf->enable_asynch = 0;
+
+ } else {
+ conf->enable_asynch = prev->enable_asynch;
+ conf->file = prev->file;
+ conf->line = prev->line;
+ }
+ }
+
ngx_conf_merge_value(conf->session_timeout,
prev->session_timeout, 300);
@@ -650,6 +671,7 @@ ngx_http_ssl_merge_srv_conf(ngx_conf_t *cf, void *parent, void *child)
return NGX_CONF_ERROR;
}
+ conf->ssl.asynch = conf->enable_asynch;
} else {
if (conf->certificates == NULL) {
@@ -822,6 +844,36 @@ ngx_http_ssl_enable(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
return NGX_CONF_OK;
}
+static char *
+ngx_http_ssl_enable_asynch(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
+{
+ ngx_http_ssl_srv_conf_t *sscf = conf;
+
+ char *rv;
+
+ ngx_flag_t *pssl, *pssl_asynch;
+
+ rv = ngx_conf_set_flag_slot(cf, cmd, conf);
+
+ if (rv != NGX_CONF_OK) {
+ return rv;
+ }
+
+ /* If ssl_asynch on is configured, then ssl on is configured by default
+ * This will align 'ssl_asynch on;' and 'listen port ssl' diretives
+ * */
+ pssl = (ngx_flag_t *) ((char *)conf + offsetof(ngx_http_ssl_srv_conf_t, enable));
+ pssl_asynch = (ngx_flag_t *) ((char *)conf + cmd->offset);
+
+ if(*pssl_asynch && *pssl != 1) {
+ *pssl = *pssl_asynch;
+ }
+
+ sscf->file = cf->conf_file->file.name.data;
+ sscf->line = cf->conf_file->line;
+
+ return NGX_CONF_OK;
+}
static char *
ngx_http_ssl_password_file(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
diff --git a/src/http/modules/ngx_http_ssl_module.h b/src/http/modules/ngx_http_ssl_module.h
index 57f5941..b97417a 100644
--- a/src/http/modules/ngx_http_ssl_module.h
+++ b/src/http/modules/ngx_http_ssl_module.h
@@ -17,6 +17,8 @@
typedef struct {
ngx_flag_t enable;
+ ngx_flag_t enable_asynch;
+
ngx_ssl_t ssl;
ngx_flag_t prefer_server_ciphers;
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c
index 476f039..693c4f0 100644
--- a/src/http/ngx_http_request.c
+++ b/src/http/ngx_http_request.c
@@ -447,9 +447,21 @@ ngx_http_wait_request_handler(ngx_event_t *rev)
* We are trying to not hold c->buffer's memory for an idle connection.
*/
- if (ngx_pfree(c->pool, b->start) == NGX_OK) {
- b->start = NULL;
+ /* For the Async implementation we need the same buffer to be used
+ * again on any async calls that have not completed.
+ * As such we need to turn off this optimisation if an async request
+ * is still in progress.
+ */
+
+#if (NGX_HTTP_SSL)
+ if ((c->asynch && !ngx_ssl_waiting_for_async(c)) || !c->asynch) {
+#endif
+ if (ngx_pfree(c->pool, b->start) == NGX_OK) {
+ b->start = NULL;
+ }
+#if (NGX_HTTP_SSL)
}
+#endif
return;
}
@@ -1385,12 +1397,22 @@ ngx_http_read_request_header(ngx_http_request_t *r)
return n;
}
- if (rev->ready) {
+#if (NGX_HTTP_SSL)
+ if (c->asynch) {
n = c->recv(c, r->header_in->last,
+ r->header_in->end - r->header_in->last);
+ }
+ else {
+#endif
+ if (rev->ready) {
+ n = c->recv(c, r->header_in->last,
r->header_in->end - r->header_in->last);
- } else {
- n = NGX_AGAIN;
+ } else {
+ n = NGX_AGAIN;
+ }
+#if (NGX_HTTP_SSL)
}
+#endif
if (n == NGX_AGAIN) {
if (!rev->timer_set) {
@@ -2954,52 +2976,62 @@ ngx_http_set_keepalive(ngx_http_request_t *r)
* c->pool and are freed too.
*/
- b = c->buffer;
-
- if (ngx_pfree(c->pool, b->start) == NGX_OK) {
+ /* For the Async implementation we need the same buffer to be used
+ * again on any async calls that have not completed.
+ * As such we need to turn off this optimisation if an async request
+ * is still in progress.
+ */
- /*
- * the special note for ngx_http_keepalive_handler() that
- * c->buffer's memory was freed
- */
+#if (NGX_HTTP_SSL)
+ if ((c->asynch && !ngx_ssl_waiting_for_async(c)) || !c->asynch) {
+#endif
+ b = c->buffer;
- b->pos = NULL;
+ if (ngx_pfree(c->pool, b->start) == NGX_OK) {
- } else {
- b->pos = b->start;
- b->last = b->start;
- }
+ /*
+ * the special note for ngx_http_keepalive_handler() that
+ * c->buffer's memory was freed
+ */
- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "hc free: %p",
- hc->free);
+ b->pos = NULL;
- if (hc->free) {
- for (cl = hc->free; cl; /* void */) {
- ln = cl;
- cl = cl->next;
- ngx_pfree(c->pool, ln->buf->start);
- ngx_free_chain(c->pool, ln);
+ } else {
+ b->pos = b->start;
+ b->last = b->start;
}
- hc->free = NULL;
- }
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, "hc free: %p",
+ hc->free);
- ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "hc busy: %p %i",
- hc->busy, hc->nbusy);
+ if (hc->free) {
+ for (cl = hc->free; cl; /* void */) {
+ ln = cl;
+ cl = cl->next;
+ ngx_pfree(c->pool, ln->buf->start);
+ ngx_free_chain(c->pool, ln);
+ }
- if (hc->busy) {
- for (cl = hc->busy; cl; /* void */) {
- ln = cl;
- cl = cl->next;
- ngx_pfree(c->pool, ln->buf->start);
- ngx_free_chain(c->pool, ln);
+ hc->free = NULL;
}
- hc->busy = NULL;
- hc->nbusy = 0;
- }
+ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "hc busy: %p %i",
+ hc->busy, hc->nbusy);
+ if (hc->busy) {
+ for (cl = hc->busy; cl; /* void */) {
+ ln = cl;
+ cl = cl->next;
+ ngx_pfree(c->pool, ln->buf->start);
+ ngx_free_chain(c->pool, ln);
+ }
+
+ hc->busy = NULL;
+ hc->nbusy = 0;
+ }
#if (NGX_HTTP_SSL)
+ }
+
if (c->ssl) {
ngx_ssl_free_buffer(c);
}
@@ -3008,6 +3040,11 @@ ngx_http_set_keepalive(ngx_http_request_t *r)
rev->handler = ngx_http_keepalive_handler;
if (wev->active && (ngx_event_flags & NGX_USE_LEVEL_EVENT)) {
+#if (NGX_HTTP_SSL)
+ if (c->asynch) {
+ ngx_asynch_del_conn(c);
+ }
+#endif
if (ngx_del_event(wev, NGX_WRITE_EVENT, 0) != NGX_OK) {
ngx_http_close_connection(c);
return;
@@ -3152,15 +3189,27 @@ ngx_http_keepalive_handler(ngx_event_t *rev)
* Like ngx_http_set_keepalive() we are trying to not hold
* c->buffer's memory for a keepalive connection.
*/
+
+ /* For the Async implementation we need the same buffer to be used
+ * again on any async calls that have not completed.
+ * As such we need to turn off this optimisation if an async request
+ * is still in progress.
+ */
- if (ngx_pfree(c->pool, b->start) == NGX_OK) {
+#if (NGX_HTTP_SSL)
+ if ((c->asynch && !ngx_ssl_waiting_for_async(c)) || !c->asynch) {
+#endif
+ if (ngx_pfree(c->pool, b->start) == NGX_OK) {
- /*
- * the special note that c->buffer's memory was freed
- */
+ /*
+ * the special note that c->buffer's memory was freed
+ */
- b->pos = NULL;
+ b->pos = NULL;
+ }
+#if (NGX_HTTP_SSL)
}
+#endif
return;
}
@@ -3229,6 +3278,11 @@ ngx_http_set_lingering_close(ngx_http_request_t *r)
wev->handler = ngx_http_empty_handler;
if (wev->active && (ngx_event_flags & NGX_USE_LEVEL_EVENT)) {
+#if (NGX_HTTP_SSL)
+ if (c->asynch) {
+ ngx_asynch_del_conn(c);
+ }
+#endif
if (ngx_del_event(wev, NGX_WRITE_EVENT, 0) != NGX_OK) {
ngx_http_close_request(r, 0);
return;
diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
index 3695286..79bc8e9 100644
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -1285,7 +1285,11 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r,
if ((ngx_event_flags & NGX_USE_LEVEL_EVENT) && ev->active) {
event = ev->write ? NGX_WRITE_EVENT : NGX_READ_EVENT;
-
+#if (NGX_HTTP_SSL)
+ if (c->asynch) {
+ ngx_asynch_del_conn(c);
+ }
+#endif
if (ngx_del_event(ev, event, 0) != NGX_OK) {
ngx_http_upstream_finalize_request(r, u,
NGX_HTTP_INTERNAL_SERVER_ERROR);
@@ -1412,7 +1416,11 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r,
if ((ngx_event_flags & NGX_USE_LEVEL_EVENT) && ev->active) {
event = ev->write ? NGX_WRITE_EVENT : NGX_READ_EVENT;
-
+#if (NGX_HTTP_SSL)
+ if (c->asynch) {
+ ngx_asynch_del_conn(c);
+ }
+#endif
if (ngx_del_event(ev, event, 0) != NGX_OK) {
ngx_http_upstream_finalize_request(r, u,
NGX_HTTP_INTERNAL_SERVER_ERROR);
diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c
index 1710ea8..7228198 100644
--- a/src/os/unix/ngx_process_cycle.c
+++ b/src/os/unix/ngx_process_cycle.c
@@ -1027,6 +1027,11 @@ ngx_channel_handler(ngx_event_t *ev)
if (n == NGX_ERROR) {
if (ngx_event_flags & NGX_USE_EPOLL_EVENT) {
+#if (NGX_HTTP_SSL)
+ if (c->asynch) {
+ ngx_asynch_del_conn(c);
+ }
+#endif
ngx_del_conn(c, 0);
}
# tar xzf nginx-1.12.2.tar.gz
# cd nginx-1.12.2/
# patch –Np1 –i /path/to/nginx-qat.patch
# OPENSSL_LIB=/usr/local/ssl
# ./configure --with-http_ssl_module \
--conf-path=/etc/nginx/nginx.conf \
--http-log-path=/var/log/nginx/access.log \
--error-log-path=/var/log/nginx/error.log \
--with-debug \
--with-cc-opt="-I$OPENSSL_LIB/include" \
--with-ld-opt="-Wl,-rpath=$OPENSSL_LIB/lib -L$OPENSSL_LIB/lib"
# make
# make install
# tar xzf haproxy-1.8.4.tar.gz
# cd haproxy-1.8.4
#make TARGET=linux26 USE_OPENSSL=1SSL_INC=/usr/local/ssl/include SSL_LIB=/usr/local/ssl/lib ADDLIB=-ldl
# cd /path/to/openssl-OpenSSL_1_1_0g/apps
# ./openssl engine -t -c-vvvv qat
(qat) Reference implementation of QAT crypto engine
[RSA, DSA, DH, AES-128-CBC-HMAC-SHA1, AES-128-CBC-HMAC-SHA256, AES-256-CBC-HMAC-SHA1, AES-256-CBC-HMAC-SHA256, TLS1-PRF]
[ available ]
ENABLE_EXTERNAL_POLLING: Enables the external polling interface to the engine.
(input flags): NO_INPUT
POLL: Polls the engine for any completed requests
(input flags): NO_INPUT
SET_INSTANCE_FOR_THREAD: Set instance to be used by this thread
(input flags): NUMERIC
GET_NUM_OP_RETRIES: Get number of retries
(input flags): NO_INPUT
SET_MAX_RETRY_COUNT: Set maximum retry count
(input flags): NUMERIC
SET_INTERNAL_POLL_INTERVAL: Set internal polling interval
(input flags): NUMERIC
GET_EXTERNAL_POLLING_FD: Returns non blocking fd for crypto engine
(input flags): NO_INPUT
ENABLE_EVENT_DRIVEN_POLLING_MODE: Set event driven polling mode
(input flags): NO_INPUT
GET_NUM_CRYPTO_INSTANCES: Get the number of crypto instances
(input flags): NO_INPUT
DISABLE_EVENT_DRIVEN_POLLING_MODE: Unset event driven polling mode
(input flags): NO_INPUT
SET_EPOLL_TIMEOUT: Set epoll_wait timeout
(input flags): NUMERIC
SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD: Set QAT small packet threshold
(input flags): STRING
ENABLE_INLINE_POLLING: Enables the inline polling mode.
(input flags): NO_INPUT
[ available ]
ENABLE_EXTERNAL_POLLING: Enables the external polling interface to the engine.
(input flags): NO_INPUT
POLL: Polls the engine for any completed requests
(input flags): NO_INPUT
SET_INSTANCE_FOR_THREAD: Set instance to be used by this thread
(input flags): NUMERIC
GET_NUM_OP_RETRIES: Get number of retries
(input flags): NO_INPUT
SET_MAX_RETRY_COUNT: Set maximum retry count
(input flags): NUMERIC
SET_INTERNAL_POLL_INTERVAL: Set internal polling interval
(input flags): NUMERIC
GET_EXTERNAL_POLLING_FD: Returns non blocking fd for crypto engine
(input flags): NO_INPUT
ENABLE_EVENT_DRIVEN_POLLING_MODE: Set event driven polling mode
(input flags): NO_INPUT
GET_NUM_CRYPTO_INSTANCES: Get the number of crypto instances
(input flags): NO_INPUT
DISABLE_EVENT_DRIVEN_POLLING_MODE: Unset event driven polling mode
(input flags): NO_INPUT
SET_EPOLL_TIMEOUT: Set epoll_wait timeout
(input flags): NUMERIC
SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD: Set QAT small packet threshold
(input flags): STRING
ENABLE_INLINE_POLLING: Enables the inline polling mode.
(input flags): NO_INPUT
软件同步:
# ./openssl speed-elapsed rsa2048
硬件同步:
# ./openssl speed-engine qat -elapsed rsa2048
硬件异步:
# ./openssl speed-engine qat -elapsed -async_jobs 72 rsa2048
软件同步:
# ./openssl speed-elapsed ecdh
硬件同步:
# ./openssl speed-engine qat -elapsed ecdh
硬件异步:
# ./openssl speed-engine qat -elapsed -async_jobs 36 ecdh
软件同步:
# ./openssl speed-elapsed ecdsa
硬件同步:
# ./openssl speed-engine qat -elapsed ecdsa
硬件异步:
# ./openssl speed-engine qat -elapsed -async_jobs 36 ecdsa
软件同步:
# ./openssl speed-elapsed -multi 2 -evp aes-128-cbc-hmac-sha256
硬件同步:
# ./openssl speed-engine qat -elapsed -multi 2 -evp aes-128-cbc-hmac-sha256
硬件异步:
# ./openssl speed-engine qat -elapsed -async_jobs 128 -multi 2 -evp aes-128-cbc-hmac-sha256
/etc/nginx/nginx.conf:
user root;
worker_processes auto;
pid /run/nginx.pid;
events {
worker_connections 768;
}
http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
gzip on;
gzip_disable "msie6";
include /etc/nginx/conf.d/*.conf;
}
/etc/nginx/conf.d/v.conf:
upstream domain1 {
server 181.1.1.1;
}
server {
listen 447 ssl;
ssl_asynch on;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_certificate /root/cert/ecdsa.pem;
ssl_certificate_key /root/cert/ecdsa.key;
location / {
proxy_pass http://domain1;
}
}
# /usr/local/nginx/sbin/nginx
ssl.cfg:
global
maxconn 100
ssl-mode-async
ssl-engine qat
defaults
mode http
timeout connect 5s
timeout client 5s
timeout server 5s
frontend myfrontend
bind :447 ssl crt /root/hap-cert/cert.pem crt /root/hap-cert/
default_backend mybackend
backend mybackend
server s3 181.1.1.1:80
# cd haproxy-1.8.4
# ./haproxy –f ssl.cfg
# cat /sys/kernel/debug/qat_c6xx_04\:00.0/fw_counters
跑流量。完毕后再次执行:
# cat /sys/kernel/debug/qat_c6xx_04\:00.0/fw_counters
如果数据发送变化,说明QAT生效。