diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 43e881b..9846697 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -374,7 +374,7 @@ IF(WITH_GNUTLS) ENDIF(WITH_GNUTLS) IF(WITH_OPENSSL) - ADD_AND_INSTALL_LIBRARY(mod_openssl "modules/mod_openssl.c") + ADD_AND_INSTALL_LIBRARY(mod_openssl "modules/mod_openssl.c;modules/openssl_filter.c") TARGET_LINK_LIBRARIES(mod_openssl ssl) TARGET_LINK_LIBRARIES(mod_openssl crypto) ENDIF(WITH_OPENSSL) diff --git a/src/modules/Makefile.am b/src/modules/Makefile.am index 74478e6..1e2eede 100644 --- a/src/modules/Makefile.am +++ b/src/modules/Makefile.am @@ -103,10 +103,11 @@ libmod_memcached_la_LIBADD = $(common_libadd) if USE_OPENSSL install_libs += libmod_openssl.la libmod_openssl_la_CPPFLAGS = $(AM_CPPFLAGS) $(OPENSSL_CFLAGS) -libmod_openssl_la_SOURCES = mod_openssl.c +libmod_openssl_la_SOURCES = mod_openssl.c openssl_filter.c libmod_openssl_la_LDFLAGS = $(common_ldflags) libmod_openssl_la_LIBADD = $(common_libadd) $(OPENSSL_LIBS) endif +EXTRA_DIST += openssl_filter.h install_libs += libmod_progress.la libmod_progress_la_SOURCES = mod_progress.c diff --git a/src/modules/mod_openssl.c b/src/modules/mod_openssl.c index 886262a..fbce976 100644 --- a/src/modules/mod_openssl.c +++ b/src/modules/mod_openssl.c @@ -34,16 +34,19 @@ * openssl.setenv "client"; * * Author: - * Copyright (c) 2009-2011 Stefan Bühler, Joe Presbrey + * Copyright (c) 2009-2013 Stefan Bühler, Joe Presbrey */ #include #include +#include "openssl_filter.h" + #include #include #include + LI_API gboolean mod_openssl_init(liModules *mods, liModule *mod); LI_API gboolean mod_openssl_free(liModules *mods, liModule *mod); @@ -52,14 +55,11 @@ typedef struct openssl_connection_ctx openssl_connection_ctx; typedef struct openssl_context openssl_context; struct openssl_connection_ctx { - SSL *ssl; - liIOStream *sock_stream; - liBuffer *raw_in_buffer; - - gboolean write_blocked_by_read, read_blocked_by_write; + liConnection *con; + liOpenSSLFilter *ssl_filter; - unsigned int initial_handshaked_finished; - unsigned int client_initiated_renegotiation; + liIOStream *sock_stream; + gpointer simple_socket_data; }; struct openssl_context { @@ -73,409 +73,98 @@ enum { SE_SERVER_CERT = 0x8 }; -static const char openssl_setenv_config_error[] = "openssl.setenv expects a string or a list of strings consisting of: client, client-cert, server, server-cert"; - -static void openssl_tcp_free(liConnection *con, gboolean aborted) { - openssl_connection_ctx *conctx = con->con_sock.data; - if (NULL == conctx) return; - - con->con_sock.raw_out = con->con_sock.raw_in = NULL; - con->con_sock.data = NULL; - con->con_sock.callbacks = NULL; +static void tcp_io_cb(liIOStream *stream, liIOStreamEvent event) { + openssl_connection_ctx *conctx = stream->data; + assert(NULL == conctx->sock_stream || conctx->sock_stream == stream); - if (conctx->sock_stream) { - liIOStream *stream = conctx->sock_stream; - int fd; - conctx->sock_stream = NULL; - fd = li_iostream_reset(stream); - if (-1 != fd) { - if (aborted) { - shutdown(fd, SHUT_RDWR); - close(fd); - } else { - li_worker_add_closing_socket(con->wrk, fd); - } - } + if (LI_IOSTREAM_DESTROY == event) { + li_stream_simple_socket_close(stream, TRUE); /* kill it, ssl sent an close alert message */ } - if (conctx->raw_in_buffer) { - li_buffer_release(conctx->raw_in_buffer); - conctx->raw_in_buffer = NULL; - } + li_connection_simple_tcp(&conctx->con, stream, &conctx->simple_socket_data, event); - if (NULL != conctx->ssl) { - SSL_shutdown(conctx->ssl); /* TODO: wait for something??? */ - SSL_free(conctx->ssl); - conctx->ssl = NULL; + if (NULL != conctx->con && conctx->con->out_has_all_data + && (NULL == stream->stream_out.out || 0 == stream->stream_out.out->length) + && li_streams_empty(conctx->con->con_sock.raw_out, NULL)) { + li_connection_request_done(conctx->con); } - con->info.is_ssl = FALSE; - - g_slice_free(openssl_connection_ctx, conctx); -} - -static liNetworkStatus openssl_handle_error(liConnection *con, openssl_connection_ctx *conctx, const char *sslfunc, off_t len, int r) { - int oerrno = errno, err; - gboolean was_fatal; - - err = SSL_get_error(conctx->ssl, r); - - switch (err) { - case SSL_ERROR_WANT_READ: - conctx->write_blocked_by_read = TRUE; - conctx->sock_stream->can_read = FALSE; - /* ignore requirement that we should pass the same buffer again */ - return (len > 0) ? LI_NETWORK_STATUS_SUCCESS : LI_NETWORK_STATUS_WAIT_FOR_EVENT; - case SSL_ERROR_WANT_WRITE: - conctx->read_blocked_by_write = TRUE; - conctx->sock_stream->can_write = FALSE; - /* ignore requirement that we should pass the same buffer again */ - return (len > 0) ? LI_NETWORK_STATUS_SUCCESS : LI_NETWORK_STATUS_WAIT_FOR_EVENT; - case SSL_ERROR_SYSCALL: - /** - * man SSL_get_error() - * - * SSL_ERROR_SYSCALL - * Some I/O error occurred. The OpenSSL error queue may contain more - * information on the error. If the error queue is empty (i.e. - * ERR_get_error() returns 0), ret can be used to find out more about - * the error: If ret == 0, an EOF was observed that violates the - * protocol. If ret == -1, the underlying BIO reported an I/O error - * (for socket I/O on Unix systems, consult errno for details). - * - */ - while (0 != (err = ERR_get_error())) { - VR_ERROR(con->mainvr, "%s(%i): %s", sslfunc, - li_event_io_fd(&conctx->sock_stream->io_watcher), - ERR_error_string(err, NULL)); - } - - switch (oerrno) { - case EPIPE: - case ECONNRESET: - return LI_NETWORK_STATUS_CONNECTION_CLOSE; - } - - if (0 != r || oerrno != 0) { - VR_ERROR(con->mainvr, "%s(%i) returned %i: %s", sslfunc, - li_event_io_fd(&conctx->sock_stream->io_watcher), - r, - g_strerror(oerrno)); - return LI_NETWORK_STATUS_FATAL_ERROR; - } else { - return LI_NETWORK_STATUS_CONNECTION_CLOSE; - } - - break; - case SSL_ERROR_ZERO_RETURN: - /* clean shutdown on the remote side */ - return LI_NETWORK_STATUS_CONNECTION_CLOSE; + switch (event) { + case LI_IOSTREAM_DESTROY: + assert(NULL == conctx->sock_stream); + assert(NULL == conctx->ssl_filter); + assert(NULL == conctx->con); + stream->data = NULL; + g_slice_free(openssl_connection_ctx, conctx); + return; default: - was_fatal = FALSE; - - while((err = ERR_get_error())) { - switch (ERR_GET_REASON(err)) { - case SSL_R_SSL_HANDSHAKE_FAILURE: - case SSL_R_TLSV1_ALERT_UNKNOWN_CA: - case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: - case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE: - case SSL_R_NO_SHARED_CIPHER: - case SSL_R_UNKNOWN_PROTOCOL: - /* TODO: if (!con->conf.log_ssl_noise) */ continue; - break; - default: - was_fatal = TRUE; - break; - } - /* get all errors from the error-queue */ - VR_ERROR(con->mainvr, "%s(%i): %s", sslfunc, - li_event_io_fd(&conctx->sock_stream->io_watcher), - ERR_error_string(err, NULL)); - } - if (!was_fatal) return LI_NETWORK_STATUS_CONNECTION_CLOSE; - return LI_NETWORK_STATUS_FATAL_ERROR; - } -} - -static liNetworkStatus openssl_do_handshake(liConnection *con, openssl_connection_ctx *conctx) { - int r = SSL_do_handshake(conctx->ssl); - if (1 == r) { - conctx->initial_handshaked_finished = 1; - conctx->ssl->s3->flags |= SSL3_FLAGS_NO_RENEGOTIATE_CIPHERS; - return LI_NETWORK_STATUS_SUCCESS; - } else { - return openssl_handle_error(con, conctx, "SSL_do_handshake", 0, r); + break; } } -static liNetworkStatus openssl_con_write(liConnection *con, liChunkQueue *cq, goffset write_max) { - const ssize_t blocksize = 16*1024; /* 16k */ - char *block_data; - off_t block_len; - ssize_t r; - liChunkIter ci; - openssl_connection_ctx *conctx = con->con_sock.data; +static void handshake_cb(liOpenSSLFilter *f, gpointer data, liStream *plain_source, liStream *plain_drain) { + openssl_connection_ctx *conctx = data; + liConnection *con = conctx->con; + UNUSED(f); - if (!conctx->initial_handshaked_finished) { - liNetworkStatus res = openssl_do_handshake(con, conctx); - if (res != LI_NETWORK_STATUS_SUCCESS) return res; + if (NULL != con) { + li_stream_connect(plain_source, con->con_sock.raw_in); + li_stream_connect(con->con_sock.raw_out, plain_drain); + } else { + li_stream_reset(plain_source); + li_stream_reset(plain_drain); } - - do { - GError *err = NULL; - - if (0 == cq->length) { - return LI_NETWORK_STATUS_SUCCESS; - } - - ci = li_chunkqueue_iter(cq); - switch (li_chunkiter_read(ci, 0, blocksize, &block_data, &block_len, &err)) { - case LI_HANDLER_GO_ON: - break; - case LI_HANDLER_ERROR: - if (NULL != err) { - VR_ERROR(con->mainvr, "Couldn't read data from chunkqueue: %s", err->message); - g_error_free(err); - } - default: - return LI_NETWORK_STATUS_FATAL_ERROR; - } - - /** - * SSL_write man-page - * - * WARNING - * When an SSL_write() operation has to be repeated because of - * SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE, it must be - * repeated with the same arguments. - * - */ - - ERR_clear_error(); - r = SSL_write(conctx->ssl, block_data, block_len); - if (conctx->client_initiated_renegotiation) { - VR_ERROR(con->mainvr, "%s", "SSL: client initiated renegotitation, closing connection"); - return LI_NETWORK_STATUS_FATAL_ERROR; - } - if (r <= 0) { - return openssl_handle_error(con, conctx, "SSL_write", 0, r); - } - - li_chunkqueue_skip(cq, r); - write_max -= r; - } while (r == block_len && write_max > 0); - - return LI_NETWORK_STATUS_SUCCESS; } -static void openssl_tcp_write(liConnection *con, openssl_connection_ctx *data) { - liNetworkStatus res; - liChunkQueue *raw_out = data->sock_stream->stream_out.out; - liChunkQueue *from = data->sock_stream->stream_out.source->out; +static void close_cb(liOpenSSLFilter *f, gpointer data) { + openssl_connection_ctx *conctx = data; + liConnection *con = conctx->con; + assert(conctx->ssl_filter == f); - li_chunkqueue_steal_all(raw_out, from); + conctx->ssl_filter = NULL; + li_openssl_filter_free(f); - if (raw_out->length > 0) { - goffset transferred; - static const goffset WRITE_MAX = 256*1024; /* 256kB */ - goffset write_max; - GError *err = NULL; - - write_max = WRITE_MAX; - - transferred = raw_out->length; - - res = openssl_con_write(con, raw_out, write_max); - - transferred = transferred - raw_out->length; - con->info.out_queue_length = raw_out->length; - if (transferred > 0) { - li_connection_update_io_timeout(con); - li_vrequest_update_stats_out(con->mainvr, transferred); - } - - switch (res) { - case LI_NETWORK_STATUS_SUCCESS: - break; - case LI_NETWORK_STATUS_FATAL_ERROR: - _VR_ERROR(con->srv, con->mainvr, "network write fatal error: %s", NULL != err ? err->message : "(unknown)"); - g_error_free(err); - openssl_tcp_free(con, TRUE); - break; - case LI_NETWORK_STATUS_CONNECTION_CLOSE: - openssl_tcp_free(con, TRUE); - break; - case LI_NETWORK_STATUS_WAIT_FOR_EVENT: - data->sock_stream->can_write = FALSE; - break; - } + if (NULL != conctx->con) { + liStream *raw_out = con->con_sock.raw_out, *raw_in = con->con_sock.raw_in; + assert(con->con_sock.data == conctx); + conctx->con = NULL; + con->con_sock.data = NULL; + li_stream_acquire(raw_in); + li_stream_reset(raw_out); + li_stream_reset(raw_in); + li_stream_release(raw_in); } - if (0 == raw_out->length && from->is_closed) { - li_stream_disconnect(&data->sock_stream->stream_out); + if (NULL != conctx->sock_stream) { + liIOStream *stream = conctx->sock_stream; + conctx->sock_stream = NULL; + li_iostream_release(stream); } } -static liNetworkStatus openssl_con_read(liConnection *con, liChunkQueue *cq) { - openssl_connection_ctx *conctx = con->con_sock.data; - - const ssize_t blocksize = 16*1024; /* 16k */ - off_t max_read = 16 * blocksize; /* 256k */ - ssize_t r; - off_t len = 0; - - if (!conctx->initial_handshaked_finished) { - liNetworkStatus res = openssl_do_handshake(con, conctx); - if (res != LI_NETWORK_STATUS_SUCCESS) return res; - } - - do { - liBuffer *buf; - gboolean cq_buf_append; - - ERR_clear_error(); - - buf = li_chunkqueue_get_last_buffer(cq, 1024); - cq_buf_append = (buf != NULL); - - if (buf != NULL) { - /* use last buffer as raw_in_buffer; they should be the same anyway */ - if (G_UNLIKELY(buf != conctx->raw_in_buffer)) { - li_buffer_acquire(buf); - li_buffer_release(conctx->raw_in_buffer); - conctx->raw_in_buffer = buf; - } - } else { - buf = conctx->raw_in_buffer; - if (buf != NULL && buf->alloc_size - buf->used < 1024) { - /* release *buffer */ - li_buffer_release(buf); - conctx->raw_in_buffer = buf = NULL; - } - if (buf == NULL) { - conctx->raw_in_buffer = buf = li_buffer_new(blocksize); - } - } - assert(conctx->raw_in_buffer == buf); - - r = SSL_read(conctx->ssl, buf->addr + buf->used, buf->alloc_size - buf->used); - if (conctx->client_initiated_renegotiation) { - VR_ERROR(con->mainvr, "%s", "SSL: client initiated renegotitation, closing connection"); - return LI_NETWORK_STATUS_FATAL_ERROR; - } - if (r < 0) { - return openssl_handle_error(con, conctx, "SSL_read", len, r); - } else if (r == 0) { - return LI_NETWORK_STATUS_CONNECTION_CLOSE; - } - - if (cq_buf_append) { - li_chunkqueue_update_last_buffer_size(cq, r); - } else { - gsize offset; - - li_buffer_acquire(buf); - - offset = buf->used; - buf->used += r; - li_chunkqueue_append_buffer2(cq, buf, offset, r); - } - if (buf->alloc_size - buf->used < 1024) { - /* release *buffer */ - li_buffer_release(buf); - conctx->raw_in_buffer = buf = NULL; - } - len += r; - } while (len < max_read); - - return LI_NETWORK_STATUS_SUCCESS; -} - -static void openssl_tcp_read(liConnection *con, openssl_connection_ctx *data) { - liNetworkStatus res; - GError *err = NULL; - - liChunkQueue *raw_in = data->sock_stream->stream_in.out; - goffset transferred; - transferred = raw_in->length; - - if (NULL == data->raw_in_buffer && NULL != con->wrk->network_read_buf) { - /* reuse worker buf if needed */ - data->raw_in_buffer = con->wrk->network_read_buf; - con->wrk->network_read_buf = NULL; - } - - res = openssl_con_read(con, raw_in); - - if (NULL == con->wrk->network_read_buf && NULL != data->raw_in_buffer - && 1 == g_atomic_int_get(&data->raw_in_buffer->refcount)) { - /* move buffer back to worker if we didn't use it */ - con->wrk->network_read_buf = data->raw_in_buffer; - data->raw_in_buffer = NULL; - } - - transferred = raw_in->length - transferred; - if (transferred > 0) { - li_connection_update_io_timeout(con); - li_vrequest_update_stats_in(con->mainvr, transferred); - } - - switch (res) { - case LI_NETWORK_STATUS_SUCCESS: - break; - case LI_NETWORK_STATUS_FATAL_ERROR: - _VR_ERROR(con->srv, con->mainvr, "network read fatal error: %s", NULL != err ? err->message : "(unknown)"); - g_error_free(err); - openssl_tcp_free(con, TRUE); - break; - case LI_NETWORK_STATUS_CONNECTION_CLOSE: - openssl_tcp_free(con, TRUE); - break; - case LI_NETWORK_STATUS_WAIT_FOR_EVENT: - data->sock_stream->can_read = FALSE; - break; - } -} +static const liOpenSSLFilterCallbacks filter_callbacks = { + handshake_cb, + close_cb +}; -static void openssl_tcp_io_cb(liIOStream *stream, liIOStreamEvent event) { - liConnection *con; - openssl_connection_ctx *conctx; +static void openssl_tcp_finished(liConnection *con, gboolean aborted) { + openssl_connection_ctx *conctx = con->con_sock.data; + UNUSED(aborted); - con = stream->data; - if (NULL == con) return; - conctx = con->con_sock.data; + con->info.is_ssl = FALSE; + con->con_sock.callbacks = NULL; - switch (event) { - case LI_IOSTREAM_READ: - openssl_tcp_read(con, conctx); - conctx = con->con_sock.data; - if (NULL != conctx && conctx->write_blocked_by_read) { - conctx->write_blocked_by_read = FALSE; - openssl_tcp_write(con, conctx); - } - break; - case LI_IOSTREAM_WRITE: - openssl_tcp_write(con, conctx); - conctx = con->con_sock.data; - if (NULL != conctx && conctx->read_blocked_by_write) { - conctx->read_blocked_by_write = FALSE; - openssl_tcp_read(con, conctx); - } - break; - default: - break; + if (NULL != conctx) { + assert(con == conctx->con); + close_cb(conctx->ssl_filter, conctx); } -} -static void openssl_tcp_finished(liConnection *con, gboolean aborted) { - openssl_connection_ctx *data = con->con_sock.data; - if (NULL == data) return; - - if (!aborted && NULL != data->sock_stream && 0 == data->sock_stream->stream_out.out->length) { - openssl_tcp_free(con, FALSE); - } else { - openssl_tcp_free(con, TRUE); + { + liStream *raw_out = con->con_sock.raw_out, *raw_in = con->con_sock.raw_in; + con->con_sock.raw_out = con->con_sock.raw_in = NULL; + if (NULL != raw_out) { li_stream_reset(raw_out); li_stream_release(raw_out); } + if (NULL != raw_in) { li_stream_reset(raw_in); li_stream_release(raw_in); } } } @@ -484,56 +173,31 @@ static const liConnectionSocketCallbacks openssl_tcp_cbs = { }; static gboolean openssl_con_new(liConnection *con, int fd) { + liEventLoop *loop = &con->wrk->loop; liServer *srv = con->srv; openssl_context *ctx = con->srv_sock->data; openssl_connection_ctx *conctx = g_slice_new0(openssl_connection_ctx); - conctx->sock_stream = li_iostream_new(con->wrk, fd, openssl_tcp_io_cb, con); - conctx->raw_in_buffer = NULL; - con->con_sock.data = conctx; - con->con_sock.callbacks = &openssl_tcp_cbs; - con->con_sock.raw_out = &conctx->sock_stream->stream_out; - con->con_sock.raw_in = &conctx->sock_stream->stream_in; + conctx->sock_stream = li_iostream_new(con->wrk, fd, tcp_io_cb, conctx); - conctx->ssl = SSL_new(ctx->ssl_ctx); - conctx->read_blocked_by_write = conctx->write_blocked_by_read = FALSE; - SSL_set_app_data(conctx->ssl, conctx); - conctx->initial_handshaked_finished = 0; - conctx->client_initiated_renegotiation = 0; + conctx->ssl_filter = li_openssl_filter_new(srv, con->wrk, &filter_callbacks, conctx, ctx->ssl_ctx, + &conctx->sock_stream->stream_in, &conctx->sock_stream->stream_out); - if (NULL == conctx->ssl) { + if (NULL == conctx->ssl_filter) { ERROR(srv, "SSL_new: %s", ERR_error_string(ERR_get_error(), NULL)); - goto fail; - } - - SSL_set_accept_state(conctx->ssl); - - if (1 != (SSL_set_fd(conctx->ssl, fd))) { - ERROR(srv, "SSL_set_fd: %s", ERR_error_string(ERR_get_error(), NULL)); - goto fail; + li_iostream_reset(conctx->sock_stream); + g_slice_free(openssl_connection_ctx, conctx); + return FALSE; } + conctx->con = con; con->con_sock.data = conctx; + con->con_sock.callbacks = &openssl_tcp_cbs; + con->con_sock.raw_out = li_stream_plug_new(loop); + con->con_sock.raw_in = li_stream_plug_new(loop); con->info.is_ssl = TRUE; return TRUE; - -fail: - openssl_tcp_free(con, TRUE); - - return FALSE; -} - - -static void openssl_info_callback(const SSL *ssl, int where, int ret) { - UNUSED(ret); - - if (0 != (where & SSL_CB_HANDSHAKE_START)) { - openssl_connection_ctx *conctx = SSL_get_app_data(ssl); - if (conctx->initial_handshaked_finished) { - conctx->client_initiated_renegotiation = TRUE; - } - } } @@ -594,7 +258,7 @@ static liHandlerResult openssl_setenv(liVRequest *vr, gpointer param, gpointer * if (!(con = li_connection_from_vrequest(vr)) || !(con->srv_sock && con->srv_sock->new_cb == openssl_con_new) || !(conctx = con->con_sock.data) - || !(ssl = conctx->ssl)) + || !(ssl = li_openssl_filter_ssl(conctx->ssl_filter))) return LI_HANDLER_GO_ON; if ((params & SE_CLIENT) && (x1 || (x1 = SSL_get_peer_certificate(ssl)))) @@ -612,6 +276,7 @@ static liHandlerResult openssl_setenv(liVRequest *vr, gpointer param, gpointer * return LI_HANDLER_GO_ON; } +static const char openssl_setenv_config_error[] = "openssl.setenv expects a string or a list of strings consisting of: client, client-cert, server, server-cert"; static liAction* openssl_setenv_create(liServer *srv, liWorker *wrk, liPlugin* p, liValue *val, gpointer userdata) { guint i; liValue *v; @@ -891,8 +556,6 @@ static gboolean openssl_setup(liServer *srv, liPlugin* p, liValue *val, gpointer goto error_free_socket; } - SSL_CTX_set_info_callback(ctx->ssl_ctx, openssl_info_callback); - if (!SSL_CTX_set_options(ctx->ssl_ctx, options)) { ERROR(srv, "SSL_CTX_set_options(%lx): %s", options, ERR_error_string(ERR_get_error(), NULL)); goto error_free_socket; diff --git a/src/modules/openssl_filter.c b/src/modules/openssl_filter.c new file mode 100644 index 0000000..5fad76b --- /dev/null +++ b/src/modules/openssl_filter.c @@ -0,0 +1,699 @@ + +#include "openssl_filter.h" + +#include +#include +#include + + +struct liOpenSSLFilter { + int refcount; + const liOpenSSLFilterCallbacks *callbacks; + gpointer callback_data; + + liServer *srv; + liWorker *wrk; + liLogContext *log_context; /* TODO: support setting this to VR context */ + + SSL *ssl; + BIO *bio; + liStream crypt_source; + liStream crypt_drain; + liStream plain_source; + liStream plain_drain; + + liBuffer *raw_in_buffer; /* for SSL_read */ + + unsigned int initial_handshaked_finished:1; + unsigned int client_initiated_renegotiation:1; + unsigned int closing:1, aborted:1; + unsigned int write_wants_read:1; +}; + +#define BIO_TYPE_LI_STREAM (127|BIO_TYPE_SOURCE_SINK) + +static int stream_bio_write(BIO *bio, const char *buf, int len); +static int stream_bio_read(BIO *bio, char *buf, int len); +static int stream_bio_puts(BIO *bio, const char *str); +static int stream_bio_gets(BIO *bio, char *str, int len); +static long stream_bio_ctrl(BIO *bio, int cmd, long num, void *ptr); +static int stream_bio_create(BIO *bio); +static int stream_bio_destroy(BIO *bio); + +static BIO_METHOD chunkqueue_bio_method = { + BIO_TYPE_LI_STREAM, + "lighttpd stream glue", + stream_bio_write, + stream_bio_read, + stream_bio_puts, + stream_bio_gets, + stream_bio_ctrl, + stream_bio_create, + stream_bio_destroy, + NULL +}; + +static int stream_bio_write(BIO *bio, const char *buf, int len) { + liOpenSSLFilter *f = bio->ptr; + liChunkQueue *cq; + + errno = ECONNRESET; + + if (NULL == f || NULL == f->crypt_source.out) return -1; + cq = f->crypt_source.out; + if (cq->is_closed) return -1; + + li_chunkqueue_append_mem(cq, buf, len); + li_stream_notify_later(&f->crypt_source); + + errno = 0; + return len; +} +static int stream_bio_read(BIO *bio, char *buf, int len) { + liOpenSSLFilter *f = bio->ptr; + liChunkQueue *cq; + + errno = ECONNRESET; + BIO_clear_retry_flags(bio); + + if (NULL == f || NULL == f->crypt_drain.out) return -1; + + cq = f->crypt_drain.out; + + if (0 == cq->length) { + if (cq->is_closed) { + errno = 0; + return 0; + } else { + errno = EAGAIN; + BIO_set_retry_read(bio); + return -1; + } + } + + if (len > cq->length) len = cq->length; + if (!li_chunkqueue_extract_to_memory(cq, len, buf, NULL)) return -1; + li_chunkqueue_skip(cq, len); + + errno = 0; + return len; +} +static int stream_bio_puts(BIO *bio, const char *str) { + return stream_bio_write(bio, str, strlen(str)); +} +static int stream_bio_gets(BIO *bio, char *str, int len) { + UNUSED(bio); UNUSED(str); UNUSED(len); + return -1; +} +static long stream_bio_ctrl(BIO *bio, int cmd, long num, void *ptr) { + liOpenSSLFilter *f = bio->ptr; + UNUSED(num); UNUSED(ptr); + + switch (cmd) { + case BIO_CTRL_FLUSH: + return 1; + case BIO_CTRL_PENDING: + if (NULL == f || NULL == f->crypt_drain.out) return 0; + return f->crypt_drain.out->length; + default: + return 0; + } +} +static int stream_bio_create(BIO *bio) { + bio->ptr = NULL; + bio->init = 1; + bio->shutdown = 1; + bio->num = 0; + bio->flags = 0; + return 1; +} +static int stream_bio_destroy(BIO *bio) { + liOpenSSLFilter *f = bio->ptr; + bio->ptr = NULL; + if (NULL != f) f->bio = NULL; + bio->init = 0; + return 1; +} + +static void f_close_ssl(liOpenSSLFilter *f) { + if (NULL != f->ssl && !f->closing) { + SSL *ssl; + liCQLimit *limit; + + f->closing = TRUE; + + assert(NULL != f->crypt_source.out); + assert(NULL != f->crypt_source.out->limit); + limit = f->crypt_source.out->limit; + limit->notify = NULL; + limit->context = NULL; + + li_stream_disconnect(&f->plain_source); + + li_stream_disconnect(&f->plain_drain); + li_stream_disconnect_dest(&f->plain_source); + + f->log_context = NULL; + if (NULL != f->callbacks && NULL != f->callbacks->closed_cb) { + f->callbacks->closed_cb(f, f->callback_data); + } + ssl = f->ssl; + f->ssl = NULL; + if (NULL != ssl) SSL_free(ssl); + } +} +static void f_acquire(liOpenSSLFilter *f) { + assert(f->refcount > 0); + ++f->refcount; +} +static void f_release(liOpenSSLFilter *f) { + assert(f->refcount > 0); + if (0 == --f->refcount) { + f->refcount = 1; + f_close_ssl(f); + if (NULL != f->bio) { + BIO_free(f->bio); + f->bio = NULL; + } + + g_slice_free(liOpenSSLFilter, f); + } +} +static void f_abort_ssl(liOpenSSLFilter *f) { + if (f->aborted) return; + f->aborted = TRUE; + f_acquire(f); + f_close_ssl(f); + li_stream_disconnect(&f->crypt_drain); + li_stream_disconnect_dest(&f->crypt_source); + f_release(f); +} + +static void do_handle_error(liOpenSSLFilter *f, const char *sslfunc, int r, gboolean writing) { + int oerrno = errno, err; + gboolean was_fatal; + + err = SSL_get_error(f->ssl, r); + + switch (err) { + case SSL_ERROR_WANT_READ: + if (writing) f->write_wants_read = TRUE; + break; + /* + case SSL_ERROR_WANT_WRITE: + we buffer all writes, can't happen! - handle as fatal error below + break; + */ + case SSL_ERROR_SYSCALL: + /** + * man SSL_get_error() + * + * SSL_ERROR_SYSCALL + * Some I/O error occurred. The OpenSSL error queue may contain more + * information on the error. If the error queue is empty (i.e. + * ERR_get_error() returns 0), ret can be used to find out more about + * the error: If ret == 0, an EOF was observed that violates the + * protocol. If ret == -1, the underlying BIO reported an I/O error + * (for socket I/O on Unix systems, consult errno for details). + * + */ + while (0 != (err = ERR_get_error())) { + _ERROR(f->srv, f->wrk, f->log_context, "%s: %s", sslfunc, + ERR_error_string(err, NULL)); + } + + if (0 != r || (0 != oerrno && ECONNRESET != oerrno)) { + _ERROR(f->srv, f->wrk, f->log_context, "%s returned %i: %s", sslfunc, + r, + g_strerror(oerrno)); + } + f_abort_ssl(f); + + break; + case SSL_ERROR_ZERO_RETURN: + /* clean shutdown on the remote side */ + f->plain_source.out->is_closed = TRUE; + li_stream_notify(&f->plain_source); + li_stream_disconnect(&f->crypt_drain); + li_stream_disconnect_dest(&f->crypt_source); + break; + default: + was_fatal = FALSE; + + while((err = ERR_get_error())) { + switch (ERR_GET_REASON(err)) { + case SSL_R_SSL_HANDSHAKE_FAILURE: + case SSL_R_TLSV1_ALERT_UNKNOWN_CA: + case SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN: + case SSL_R_SSLV3_ALERT_BAD_CERTIFICATE: + case SSL_R_NO_SHARED_CIPHER: + case SSL_R_UNKNOWN_PROTOCOL: + /* TODO: if (!con->conf.log_ssl_noise) */ continue; + break; + default: + was_fatal = TRUE; + break; + } + /* get all errors from the error-queue */ + _ERROR(f->srv, f->wrk, f->log_context, "%s: %s", sslfunc, + ERR_error_string(err, NULL)); + } + if (!was_fatal) f_abort_ssl(f); + } + +} + +static gboolean do_ssl_handshake(liOpenSSLFilter *f, gboolean writing) { + int r = SSL_do_handshake(f->ssl); + if (1 == r) { + f->initial_handshaked_finished = 1; + f->ssl->s3->flags |= SSL3_FLAGS_NO_RENEGOTIATE_CIPHERS; + li_stream_acquire(&f->plain_source); + li_stream_acquire(&f->plain_drain); + f->callbacks->handshake_cb(f, f->callback_data, &f->plain_source, &f->plain_drain); + li_stream_release(&f->plain_source); + li_stream_release(&f->plain_drain); + return TRUE; + } else { + do_handle_error(f, "SSL_do_handshake", r, writing); + return FALSE; + } +} + +static void do_ssl_read(liOpenSSLFilter *f) { + const ssize_t blocksize = 16*1024; /* 16k */ + off_t max_read = 4 * blocksize; /* 64k */ + ssize_t r; + off_t len = 0; + liChunkQueue *cq = f->plain_source.out; + + f_acquire(f); + + if (NULL != f->ssl && !f->initial_handshaked_finished && !do_ssl_handshake(f, FALSE)) goto out; + if (NULL == f->ssl) { + f_abort_ssl(f); + goto out; + } + + do { + liBuffer *buf; + gboolean cq_buf_append; + + ERR_clear_error(); + + buf = li_chunkqueue_get_last_buffer(cq, 1024); + cq_buf_append = (buf != NULL); + + if (buf != NULL) { + /* use last buffer as raw_in_buffer; they should be the same anyway */ + if (G_UNLIKELY(buf != f->raw_in_buffer)) { + li_buffer_acquire(buf); + li_buffer_release(f->raw_in_buffer); + f->raw_in_buffer = buf; + } + } else { + buf = f->raw_in_buffer; + if (buf != NULL && buf->alloc_size - buf->used < 1024) { + /* release *buffer */ + li_buffer_release(buf); + f->raw_in_buffer = buf = NULL; + } + if (buf == NULL) { + f->raw_in_buffer = buf = li_buffer_new(blocksize); + } + } + assert(f->raw_in_buffer == buf); + + r = SSL_read(f->ssl, buf->addr + buf->used, buf->alloc_size - buf->used); + if (f->client_initiated_renegotiation) { + _ERROR(f->srv, f->wrk, f->log_context, "%s", "SSL: client initiated renegotitation, closing connection"); + f_abort_ssl(f); + goto out; + } + if (r < 0) { + do_handle_error(f, "SSL_read", r, FALSE); + goto out; + } else if (r == 0) { + /* clean shutdown? */ + r = SSL_shutdown(f->ssl); + switch (r) { + case 0: /* don't care about bidirectional shutdown */ + case 1: /* bidirectional shutdown finished */ + f->plain_source.out->is_closed = TRUE; + f->plain_drain.out->is_closed = TRUE; + f->crypt_source.out->is_closed = TRUE; + f->crypt_drain.out->is_closed = TRUE; + li_stream_disconnect(&f->crypt_drain); + li_stream_disconnect_dest(&f->crypt_source); + f_close_ssl(f); + break; + default: + do_handle_error(f, "SSL_shutdown", r, TRUE); + f_abort_ssl(f); + break; + } + goto out; + } + + if (cq_buf_append) { + li_chunkqueue_update_last_buffer_size(cq, r); + } else { + gsize offset; + + li_buffer_acquire(buf); + + offset = buf->used; + buf->used += r; + li_chunkqueue_append_buffer2(cq, buf, offset, r); + } + if (buf->alloc_size - buf->used < 1024) { + /* release *buffer */ + li_buffer_release(buf); + f->raw_in_buffer = buf = NULL; + } + len += r; + } while (len < max_read); + +out: + f_release(f); +} + +static void do_ssl_write(liOpenSSLFilter *f) { + const ssize_t blocksize = 16*1024; /* 16k */ + char *block_data; + off_t block_len; + ssize_t r; + off_t write_max = 64*1024; + liChunkQueue *cq = f->plain_drain.out; + + f_acquire(f); + + f->write_wants_read = FALSE; + + /* use space in (encrypted) outgoing buffer as amounts of bytes we try to write from (plain) output + * don't care if we write a little bit more than the limit allowed */ + write_max = li_chunkqueue_limit_available(f->crypt_source.out); + assert(write_max >= 0); /* we set a limit! */ + if (0 == write_max) goto out; + /* if we start writing, try to write at least blocksize bytes */ + if (write_max < blocksize) write_max = blocksize; + + if (NULL != f->ssl && !f->initial_handshaked_finished && !do_ssl_handshake(f, TRUE)) goto out; + if (NULL == f->ssl) { + f_abort_ssl(f); + goto out; + } + + do { + GError *err = NULL; + liChunkIter ci; + + if (0 == cq->length) break; + + ci = li_chunkqueue_iter(cq); + switch (li_chunkiter_read(ci, 0, blocksize, &block_data, &block_len, &err)) { + case LI_HANDLER_GO_ON: + break; + case LI_HANDLER_ERROR: + if (NULL != err) { + _ERROR(f->srv, f->wrk, f->log_context, "Couldn't read data from chunkqueue: %s", err->message); + g_error_free(err); + } + /* fall through */ + default: + f_abort_ssl(f); + goto out; + } + + /** + * SSL_write man-page + * + * WARNING + * When an SSL_write() operation has to be repeated because of + * SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE, it must be + * repeated with the same arguments. + * + */ + + ERR_clear_error(); + r = SSL_write(f->ssl, block_data, block_len); + if (f->client_initiated_renegotiation) { + _ERROR(f->srv, f->wrk, f->log_context, "%s", "SSL: client initiated renegotitation, closing connection"); + f_abort_ssl(f); + goto out; + } + if (r <= 0) { + do_handle_error(f, "SSL_write", r, TRUE); + goto out; + } + + li_chunkqueue_skip(cq, r); + write_max -= r; + } while (r == block_len && write_max > 0); + + if (cq->is_closed && 0 == cq->length) { + r = SSL_shutdown(f->ssl); + switch (r) { + case 0: /* don't care about bidirectional shutdown */ + case 1: /* bidirectional shutdown finished */ + f->plain_source.out->is_closed = TRUE; + f->crypt_source.out->is_closed = TRUE; + f->crypt_drain.out->is_closed = TRUE; + f_close_ssl(f); + break; + default: + do_handle_error(f, "SSL_shutdown", r, TRUE); + f_abort_ssl(f); + break; + } + } else if (0 < cq->length && 0 != li_chunkqueue_limit_available(f->crypt_source.out)) { + li_stream_again_later(&f->plain_drain); + } + +out: + f_release(f); +} + +/* ssl crypted out -> io */ +static void stream_crypt_source_cb(liStream *stream, liStreamEvent event) { + liOpenSSLFilter *f = LI_CONTAINER_OF(stream, liOpenSSLFilter, crypt_source); + switch (event) { + case LI_STREAM_NEW_DATA: + /* data comes through SSL */ + break; + case LI_STREAM_NEW_CQLIMIT: + break; + case LI_STREAM_CONNECTED_DEST: /* io out */ + break; + case LI_STREAM_CONNECTED_SOURCE: /* plain_drain */ + break; + case LI_STREAM_DISCONNECTED_DEST: /* io out disconnect */ + if (!stream->out->is_closed || 0 != stream->out->length) { + f_abort_ssl(f); /* didn't read everything */ + } + break; + case LI_STREAM_DISCONNECTED_SOURCE: /* plain_drain */ + if (!stream->out->is_closed) { /* f_close_ssl before we were ready */ + f_abort_ssl(f); + } + break; + case LI_STREAM_DESTROY: + f_release(f); + break; + } +} +/* io -> ssl crypted in */ +static void stream_crypt_drain_cb(liStream *stream, liStreamEvent event) { + liOpenSSLFilter *f = LI_CONTAINER_OF(stream, liOpenSSLFilter, crypt_drain); + switch (event) { + case LI_STREAM_NEW_DATA: + if (!stream->out->is_closed && NULL != stream->source) { + li_chunkqueue_steal_all(stream->out, stream->source->out); + stream->out->is_closed = stream->out->is_closed || stream->source->out->is_closed; + li_stream_notify(stream); /* tell plain_source to do SSL_read */ + } + if (stream->out->is_closed) { + li_stream_disconnect(stream); + } + break; + case LI_STREAM_NEW_CQLIMIT: + break; + case LI_STREAM_CONNECTED_DEST: /* plain_source */ + break; + case LI_STREAM_CONNECTED_SOURCE: /* io in */ + break; + case LI_STREAM_DISCONNECTED_DEST: /* plain_source */ + if (!stream->out->is_closed || 0 != stream->out->length) { + f_abort_ssl(f); /* didn't read everything */ + } + break; + case LI_STREAM_DISCONNECTED_SOURCE: /* io in disconnect */ + if (!stream->out->is_closed) { + f_abort_ssl(f); /* conn aborted */ + } + break; + case LI_STREAM_DESTROY: + f_release(f); + break; + } +} +/* ssl (plain) -> app */ +static void stream_plain_source_cb(liStream *stream, liStreamEvent event) { + liOpenSSLFilter *f = LI_CONTAINER_OF(stream, liOpenSSLFilter, plain_source); + switch (event) { + case LI_STREAM_NEW_DATA: + do_ssl_read(f); + if (f->write_wants_read) do_ssl_write(f); + li_stream_notify(stream); + break; + case LI_STREAM_NEW_CQLIMIT: + break; + case LI_STREAM_CONNECTED_DEST: /* app */ + break; + case LI_STREAM_CONNECTED_SOURCE: /* crypt_drain */ + break; + case LI_STREAM_DISCONNECTED_DEST: /* app */ + if (!stream->out->is_closed || 0 != stream->out->length) { + f_abort_ssl(f); /* didn't read everything */ + } + break; + case LI_STREAM_DISCONNECTED_SOURCE: /* crypt_drain */ + if (!stream->out->is_closed) { + f_abort_ssl(f); /* didn't get everything */ + } + break; + case LI_STREAM_DESTROY: + f_release(f); + break; + } +} +/* app -> ssl (plain) */ +static void stream_plain_drain_cb(liStream *stream, liStreamEvent event) { + liOpenSSLFilter *f = LI_CONTAINER_OF(stream, liOpenSSLFilter, plain_drain); + switch (event) { + case LI_STREAM_NEW_DATA: + if (!stream->out->is_closed && NULL != stream->source) { + li_chunkqueue_steal_all(stream->out, stream->source->out); + stream->out->is_closed = stream->out->is_closed || stream->source->out->is_closed; + } + do_ssl_write(f); + if (stream->out->is_closed) { + li_stream_disconnect(stream); + } + break; + case LI_STREAM_NEW_CQLIMIT: + break; + case LI_STREAM_CONNECTED_DEST: /* crypt_source */ + break; + case LI_STREAM_CONNECTED_SOURCE: /* app */ + break; + case LI_STREAM_DISCONNECTED_DEST: + if (!stream->out->is_closed || 0 != stream->out->length) { + f_abort_ssl(f); /* didn't read everything */ + } + break; + case LI_STREAM_DISCONNECTED_SOURCE: + if (!stream->out->is_closed) { + f_abort_ssl(f); /* didn't get everything */ + } + break; + case LI_STREAM_DESTROY: + f_release(f); + break; + } +} +static void stream_crypt_source_limit_notify_cb(gpointer context, gboolean locked) { + liOpenSSLFilter *f = context; + if (!locked && !f->closing) li_stream_again_later(&f->plain_drain); +} + +static void openssl_info_callback(const SSL *ssl, int where, int ret) { + UNUSED(ret); + + if (0 != (where & SSL_CB_HANDSHAKE_START)) { + liOpenSSLFilter *f = SSL_get_app_data(ssl); + if (f->initial_handshaked_finished) { + f->client_initiated_renegotiation = TRUE; + } + } +} + +liOpenSSLFilter* li_openssl_filter_new( + liServer *srv, liWorker *wrk, + const liOpenSSLFilterCallbacks *callbacks, gpointer data, + SSL_CTX *ssl_ctx, liStream *crypt_source, liStream *crypt_drain +) { + liEventLoop *loop = crypt_source->loop; + liOpenSSLFilter *f; + SSL *ssl; + liCQLimit *out_limit; + + ssl = SSL_new(ssl_ctx); + if (NULL == ssl) return NULL; + + f = g_slice_new0(liOpenSSLFilter); + f->refcount = 5; /* 1 + 4 streams */ + f->callbacks = callbacks; + f->callback_data = data; + f->srv = srv; + f->wrk = wrk; + + f->ssl = ssl; + SSL_set_app_data(f->ssl, f); + SSL_set_info_callback(f->ssl, openssl_info_callback); + f->bio = BIO_new(&chunkqueue_bio_method); + f->bio->ptr = f; + SSL_set_bio(f->ssl, f->bio, f->bio); + /* BIO_set_callback(f->bio, BIO_debug_callback); */ + + f->initial_handshaked_finished = 0; + f->client_initiated_renegotiation = 0; + f->closing = f->aborted = 0; + f->write_wants_read = 0; + + li_stream_init(&f->crypt_source, loop, stream_crypt_source_cb); + li_stream_init(&f->crypt_drain, loop, stream_crypt_drain_cb); + li_stream_init(&f->plain_source, loop, stream_plain_source_cb); + li_stream_init(&f->plain_drain, loop, stream_plain_drain_cb); + + /* "virtual" connections - the content goes through SSL. needed for liCQLimit. */ + li_stream_connect(&f->plain_drain, &f->crypt_source); + li_stream_connect(&f->crypt_drain, &f->plain_source); + + li_stream_connect(crypt_source, &f->crypt_drain); + li_stream_connect(&f->crypt_source, crypt_drain); + + /* separate limit for buffer of encrypted data + * + * f->plain_drain is already connected to f->crypt_source, + * so they won't share the same limit */ + out_limit = li_cqlimit_new(); + out_limit->notify = stream_crypt_source_limit_notify_cb; + out_limit->context = f; + li_cqlimit_set_limit(out_limit, 32*1024); + li_chunkqueue_set_limit(crypt_drain->out, out_limit); + li_chunkqueue_set_limit(f->crypt_source.out, out_limit); + li_cqlimit_release(out_limit); + + SSL_set_accept_state(f->ssl); + + return f; +} + +void li_openssl_filter_free(liOpenSSLFilter *f) { + assert(NULL != f->callbacks); + f->callbacks = NULL; + f->callback_data = NULL; + + f_close_ssl(f); + + li_stream_release(&f->crypt_source); + li_stream_release(&f->crypt_drain); + li_stream_release(&f->plain_source); + li_stream_release(&f->plain_drain); + f_release(f); +} + +SSL* li_openssl_filter_ssl(liOpenSSLFilter *f) { + return f->ssl; +} diff --git a/src/modules/openssl_filter.h b/src/modules/openssl_filter.h new file mode 100644 index 0000000..3878b20 --- /dev/null +++ b/src/modules/openssl_filter.h @@ -0,0 +1,29 @@ +#ifndef _LIGHTTPD_OPENSSL_FILTER_H_ +#define _LIGHTTPD_OPENSSL_FILTER_H_ + +#include + +#include + +typedef struct liOpenSSLFilter liOpenSSLFilter; + +typedef void (*liOpenSSLFilterHandshakeCB)(liOpenSSLFilter *f, gpointer data, liStream *plain_source, liStream *plain_drain); +typedef void (*liOpenSSLFilterClosedCB)(liOpenSSLFilter *f, gpointer data); + +typedef struct liOpenSSLFilterCallbacks liOpenSSLFilterCallbacks; +struct liOpenSSLFilterCallbacks { + liOpenSSLFilterHandshakeCB handshake_cb; /* called after initial handshake is done */ + liOpenSSLFilterClosedCB closed_cb; +}; + +LI_API liOpenSSLFilter* li_openssl_filter_new( + liServer *srv, liWorker *wrk, + const liOpenSSLFilterCallbacks *callbacks, gpointer data, + SSL_CTX *ssl_ctx, liStream *crypt_source, liStream *crypt_drain); + +/* doesn't call closed_cb; but you can call this from closed_cb */ +LI_API void li_openssl_filter_free(liOpenSSLFilter *f); + +LI_API SSL* li_openssl_filter_ssl(liOpenSSLFilter *f); + +#endif diff --git a/src/modules/wscript b/src/modules/wscript index 3980971..0d9bd5f 100644 --- a/src/modules/wscript +++ b/src/modules/wscript @@ -54,7 +54,7 @@ def build(bld): lighty_mod(bld, 'mod_memcached', 'mod_memcached.c', ['lua'] if bld.env['USE_LUA'] == 1 else []) if env['USE_OPENSSL'] == 1: uselib = ['ssl','crypto'] - lighty_mod(bld, 'mod_openssl', 'mod_openssl.c', uselib) + lighty_mod(bld, 'mod_openssl', 'mod_openssl.c openssl_filter.c', uselib) lighty_mod(bld, 'mod_progress', 'mod_progress.c') lighty_mod(bld, 'mod_proxy', 'mod_proxy.c') lighty_mod(bld, 'mod_redirect', 'mod_redirect.c')