From 89c97b8cd8a4faf169965ec6383335d0c0a661ee Mon Sep 17 00:00:00 2001 From: Glenn Strauss Date: Mon, 26 Jul 2021 04:47:51 -0400 Subject: [PATCH] [core] conns_pool separate from conns list (#3084) keep conns_pool of struct connection separate from conns list and allocate conns list to srv->srvconf.max_conns size at startup x-ref: "Memory fragmentation with HTTP/2 enabled" https://redmine.lighttpd.net/issues/3084 --- src/base.h | 5 +- src/connections.c | 128 ++++++++++++++++++++-------------------------- src/connections.h | 4 ++ src/server.c | 1 + 4 files changed, 64 insertions(+), 74 deletions(-) diff --git a/src/base.h b/src/base.h index 2162ebef..755c045e 100644 --- a/src/base.h +++ b/src/base.h @@ -21,7 +21,7 @@ struct connection { h2con *h2; int fd; /* the FD for this connection */ - int ndx; /* reverse mapping to server->connection[ndx] */ + uint32_t ndx; /* reverse mapping to server->connection[ndx] */ fdnode *fdn; /* fdevent (fdnode *) object */ /* fd states */ @@ -60,6 +60,8 @@ struct connection { uint32_t request_count; /* number of requests handled in this connection */ int keep_alive_idle; /* remember max_keep_alive_idle from config */ + + connection *next; }; typedef struct { @@ -169,6 +171,7 @@ struct server { int sockets_disabled; uint32_t max_conns; + connection *conns_pool; log_error_st *errh; diff --git a/src/connections.c b/src/connections.c index 6cce7d5c..215f77cf 100644 --- a/src/connections.c +++ b/src/connections.c @@ -47,60 +47,29 @@ static connection *connection_init(server *srv); static void connection_reset(connection *con); -__attribute_cold__ -__attribute_noinline__ -static void connections_extend(server * const srv, connections * const conns) { - const uint32_t n = (srv->srvconf.max_conns >= 128 && conns->size >= 16) - ? (0 != conns->size) ? 128 : 128 - 16 - : (srv->srvconf.max_conns > 16) ? 16 : srv->srvconf.max_conns; - if (conns->size < 128 && srv->srvconf.h2proto) - request_pool_extend(srv, n * 8); - - conns->size += n; - conns->ptr = realloc(conns->ptr, sizeof(*conns->ptr) * conns->size); - force_assert(NULL != conns->ptr); - - for (uint32_t i = conns->used; i < conns->size; ++i) { - conns->ptr[i] = connection_init(srv); - connection_reset(conns->ptr[i]); - } -} - static connection *connections_get_new_connection(server *srv) { - connections * const conns = &srv->conns; - - if (conns->size == conns->used) - connections_extend(srv, conns); - - conns->ptr[conns->used]->ndx = conns->used; - return conns->ptr[conns->used++]; + connections * const conns = &srv->conns; + connection *con; + if (srv->conns_pool) { + con = srv->conns_pool; + srv->conns_pool = con->next; + } + else { + con = connection_init(srv); + connection_reset(con); + if (srv->srvconf.h2proto) + request_pool_extend(srv, 8); + } + con->next = NULL; + return (conns->ptr[(con->ndx = conns->used++)] = con); } static void connection_del(server *srv, connection *con) { - connections * const conns = &srv->conns; - - if (-1 == con->ndx) return; - uint32_t i = (uint32_t)con->ndx; - - /* not last element */ - - if (i != --conns->used) { - connection * const temp = conns->ptr[i]; - conns->ptr[i] = conns->ptr[conns->used]; - conns->ptr[conns->used] = temp; - - conns->ptr[i]->ndx = i; - conns->ptr[conns->used]->ndx = -1; - } - - con->ndx = -1; -#if 0 - fprintf(stderr, "%s.%d: del: (%d)", __FILE__, __LINE__, conns->used); - for (i = 0; i < conns->used; i++) { - fprintf(stderr, "%d ", conns->ptr[i]->fd); - } - fprintf(stderr, "\n"); -#endif + con->next = srv->conns_pool; + srv->conns_pool = con; + connections * const conns = &srv->conns; + if (con->ndx != --conns->used) /* not last element */ + (conns->ptr[con->ndx] = conns->ptr[conns->used])->ndx = con->ndx; } static void connection_close(connection *con) { @@ -541,12 +510,7 @@ static connection *connection_init(server *srv) { connection * const con = calloc(1, sizeof(*con)); force_assert(NULL != con); - con->fd = 0; - con->ndx = -1; - con->bytes_written = 0; - con->bytes_read = 0; - - con->srv = srv; + con->srv = srv; con->plugin_slots = srv->plugin_slots; con->config_data_base = srv->config_data_base; @@ -563,26 +527,44 @@ static connection *connection_init(server *srv) { } -void connections_free(server *srv) { - connections * const conns = &srv->conns; - for (uint32_t i = 0; i < conns->size; ++i) { - connection *con = conns->ptr[i]; - request_st * const r = &con->request; +static void connection_free(connection * const con) { + request_st * const r = &con->request; - connection_reset(con); - if (con->write_queue != &r->write_queue) - chunkqueue_free(con->write_queue); - if (con->read_queue != &r->read_queue) - chunkqueue_free(con->read_queue); - request_free_data(r); + connection_reset(con); + if (con->write_queue != &r->write_queue) + chunkqueue_free(con->write_queue); + if (con->read_queue != &r->read_queue) + chunkqueue_free(con->read_queue); + request_free_data(r); - free(con->plugin_ctx); - free(con->dst_addr_buf.ptr); - free(con); - } + free(con->plugin_ctx); + free(con->dst_addr_buf.ptr); + free(con); +} - free(conns->ptr); - conns->ptr = NULL; +void connections_pool_clear(server * const srv) { + connection *con; + while ((con = srv->conns_pool)) { + srv->conns_pool = con->next; + connection_free(con); + } +} + +void connections_free(server *srv) { + connections_pool_clear(srv); + + connections * const conns = &srv->conns; + for (uint32_t i = 0; i < conns->used; ++i) + connection_free(conns->ptr[i]); + free(conns->ptr); + conns->ptr = NULL; +} + +void connections_init(server *srv) { + connections * const conns = &srv->conns; + conns->size = srv->srvconf.max_conns; + conns->ptr = calloc(conns->size, sizeof(*conns->ptr)); + force_assert(NULL != conns->ptr); } diff --git a/src/connections.h b/src/connections.h index c0190928..07519fa6 100644 --- a/src/connections.h +++ b/src/connections.h @@ -6,6 +6,10 @@ struct server_socket; /* declaration */ +void connections_pool_clear(server *srv); + +__attribute_cold__ +void connections_init(server *srv); __attribute_cold__ void connections_free(server *srv); diff --git a/src/server.c b/src/server.c index d0f5e29a..8d2077cf 100644 --- a/src/server.c +++ b/src/server.c @@ -1766,6 +1766,7 @@ static int server_main_setup (server * const srv, int argc, char **argv) { /* or use the default: we really don't want to hit max-fds */ srv->max_conns = srv->srvconf.max_conns = srv->max_fds/3; } + connections_init(srv); /* libev backend overwrites our SIGCHLD handler and calls waitpid on SIGCHLD; we want our own SIGCHLD handling. */ #ifdef HAVE_SIGACTION