diff --git a/src/Makefile.am b/src/Makefile.am index 5acf45c4..e1908693 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -30,7 +30,7 @@ mod_ssi_expr.c: mod_ssi_exprparser.h common_src=buffer.c log.c \ keyvalue.c chunk.c \ http_chunk.c stream.c fdevent.c \ - file_cache.c plugin.c joblist.c etag.c array.c \ + stat_cache.c plugin.c joblist.c etag.c array.c \ data_string.c data_count.c data_array.c \ data_integer.c md5.c data_fastcgi.c \ fdevent_select.c fdevent_linux_rtsig.c \ @@ -40,7 +40,8 @@ common_src=buffer.c log.c \ inet_ntop_cache.c crc32.c \ connections-glue.c \ configfile-glue.c \ - http-header-glue.c + http-header-glue.c \ + splaytree.c src = server.c response.c connections.c network.c \ network_write.c network_linux_sendfile.c \ @@ -59,9 +60,9 @@ if NO_RDYNAMIC # everything lib_LTLIBRARIES += liblightcomp.la liblightcomp_la_SOURCES=$(common_src) -liblightcomp_la_CFLAGS=$(AM_CFLAGS) +liblightcomp_la_CFLAGS=$(AM_CFLAGS) $(FAM_CFLAGS) liblightcomp_la_LDFLAGS = -avoid-version -no-undefined -liblightcomp_la_LIBADD = $(PCRE_LIB) $(SSL_LIB) +liblightcomp_la_LIBADD = $(PCRE_LIB) $(SSL_LIB) $(FAM_LIBS) common_libadd = liblightcomp.la else src += $(common_src) @@ -199,20 +200,21 @@ hdr = server.h buffer.h network.h log.h keyvalue.h \ response.h request.h fastcgi.h chunk.h \ settings.h http_chunk.h http_auth_digest.h \ md5.h md5_global.h http_auth.h stream.h \ - fdevent.h connections.h base.h file_cache.h \ + fdevent.h connections.h base.h stat_cache.h \ plugin.h mod_auth.h \ etag.h joblist.h array.h crc32.h \ network_backends.h configfile.h bitset.h \ mod_ssi.h mod_ssi_expr.h inet_ntop_cache.h \ configparser.h mod_ssi_exprparser.h \ - sys-mmap.h sys-socket.h mod_cml.h mod_cml_funcs.h + sys-mmap.h sys-socket.h mod_cml.h mod_cml_funcs.h \ + splaytree.h DEFS= @DEFS@ -DLIBRARY_DIR="\"$(libdir)\"" lighttpd_SOURCES = $(src) -lighttpd_LDADD = $(PCRE_LIB) $(DL_LIB) $(SENDFILE_LIB) $(ATTR_LIB) $(common_libadd) $(SSL_LIB) +lighttpd_LDADD = $(PCRE_LIB) $(DL_LIB) $(SENDFILE_LIB) $(ATTR_LIB) $(common_libadd) $(SSL_LIB) $(FAM_LIBS) lighttpd_LDFLAGS = -export-dynamic -lighttpd_CCPFLAGS = $(MYSQL_INCLUDES) +lighttpd_CCPFLAGS = $(MYSQL_INCLUDES) $(FAM_CFLAGS) array_SOURCES = array.c buffer.c data_string.c data_count.c array_CPPFLAGS= -DDEBUG_ARRAY diff --git a/src/base.h b/src/base.h index 60d34054..72aa6335 100644 --- a/src/base.h +++ b/src/base.h @@ -22,6 +22,7 @@ #include "settings.h" #include "fdevent.h" #include "sys-socket.h" +#include "splaytree.h" #if defined HAVE_LIBSSL && defined HAVE_OPENSSL_SSL_H @@ -29,6 +30,10 @@ # include #endif +#ifdef HAVE_FAM_H +# include +#endif + #ifndef O_LARGEFILE # define O_LARGEFILE 0 #endif @@ -167,26 +172,6 @@ typedef struct { } transfer_encoding; } response; -typedef struct { - buffer *name; - buffer *etag; - - struct stat st; - - int fd; - int fde_ndx; - - char *mmap_p; - size_t mmap_length; - off_t mmap_offset; - - size_t in_use; - size_t is_dirty; - - time_t stat_ts; - buffer *content_type; -} file_cache_entry; - typedef struct { buffer *scheme; buffer *authority; @@ -205,13 +190,32 @@ typedef struct { } physical; typedef struct { - file_cache_entry **ptr; + buffer *name; + buffer *etag; - size_t size; - size_t used; + struct stat st; + + time_t stat_ts; - buffer *dir_name; -} file_cache; +#ifdef HAVE_FAM_H + int dir_version; + int dir_ndx; +#endif + + buffer *content_type; +} stat_cache_entry; + +typedef struct { + splay_tree *files; /* the nodes of the tree are stat_cache_entry's */ + + buffer *dir_name; /* for building the dirname from the filename */ +#ifdef HAVE_FAM_H + splay_tree *dirs; /* the nodes of the tree are fam_dir_entry */ + + FAMConnection *fam; + int fam_fcce_ndx; +#endif +} stat_cache; typedef struct { array *indexfiles; @@ -348,8 +352,6 @@ typedef struct { connection_type mode; - file_cache_entry *fce; /* filecache entry for the selected file */ - void **plugin_ctx; /* plugin connection specific config */ specific_config conf; /* global connection specific config */ @@ -428,6 +430,12 @@ typedef struct { unsigned short log_request_header_on_error; unsigned short log_state_handling; + + enum { STAT_CACHE_ENGINE_UNSET, + STAT_CACHE_ENGINE_NONE, + STAT_CACHE_ENGINE_SIMPLE, + STAT_CACHE_ENGINE_FAM + } stat_cache_engine; } server_config; typedef struct { @@ -525,8 +533,7 @@ typedef struct { connections *joblist; connections *fdwaitqueue; - file_cache *file_cache; - buffer *file_cache_etag; + stat_cache *stat_cache; buffer_array *config_patches; diff --git a/src/config.c b/src/config.c index 5fadde52..5fcb6ab4 100644 --- a/src/config.c +++ b/src/config.c @@ -24,6 +24,7 @@ static int config_insert(server *srv) { size_t i; int ret = 0; + buffer *stat_cache_string; config_values_t cv[] = { { "server.bind", NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_SERVER }, /* 0 */ @@ -78,6 +79,7 @@ static int config_insert(server *srv) { { "dir-listing.encoding", NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION }, /* 41 */ { "server.errorlog-use-syslog", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_SERVER }, /* 42 */ { "server.range-requests", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, /* 43 */ + { "server.stat-cache-engine", NULL, T_CONFIG_STRING, T_CONFIG_SCOPE_CONNECTION }, /* 44 */ { "server.host", "use server.bind instead", T_CONFIG_DEPRECATED, T_CONFIG_SCOPE_UNSET }, { "server.docroot", "use server.document-root instead", T_CONFIG_DEPRECATED, T_CONFIG_SCOPE_UNSET }, @@ -111,6 +113,9 @@ static int config_insert(server *srv) { cv[42].destination = &(srv->srvconf.errorlog_use_syslog); + stat_cache_string = buffer_init(); + cv[44].destination = stat_cache_string; + srv->config_storage = calloc(1, srv->config_context->used * sizeof(specific_config *)); assert(srv->config_storage); @@ -193,6 +198,22 @@ static int config_insert(server *srv) { } } + if (buffer_is_empty(stat_cache_string)) { + srv->srvconf.stat_cache_engine = STAT_CACHE_ENGINE_NONE; + } else if (buffer_is_equal_string(stat_cache_string, CONST_STR_LEN("simple"))) { + srv->srvconf.stat_cache_engine = STAT_CACHE_ENGINE_SIMPLE; + } else if (buffer_is_equal_string(stat_cache_string, CONST_STR_LEN("fam"))) { + srv->srvconf.stat_cache_engine = STAT_CACHE_ENGINE_FAM; + } else if (buffer_is_equal_string(stat_cache_string, CONST_STR_LEN("disable"))) { + srv->srvconf.stat_cache_engine = STAT_CACHE_ENGINE_NONE; + } else { + log_error_write(srv, __FILE__, __LINE__, "sb", + "server.stat-cache-engine can be one of \"none\", \"simple\", \"fam\", but not:", stat_cache_string); + ret = HANDLER_ERROR; + } + + buffer_free(stat_cache_string); + return ret; } diff --git a/src/connections.c b/src/connections.c index af309933..26f71885 100644 --- a/src/connections.c +++ b/src/connections.c @@ -18,7 +18,7 @@ #include "response.h" #include "network.h" #include "http_chunk.h" -#include "file_cache.h" +#include "stat_cache.h" #include "joblist.h" #include "plugin.h" @@ -390,9 +390,10 @@ static int connection_handle_write_prepare(server *srv, connection *con) { case 200: /* class: header + body */ if (con->physical.path->used) { + stat_cache_entry *sce = NULL; con->file_finished = 1; - if (HANDLER_GO_ON != file_cache_get_entry(srv, con, con->physical.path, &(con->fce))) { + if (HANDLER_ERROR == stat_cache_get_entry(srv, con, con->physical.path, &sce)) { log_error_write(srv, __FILE__, __LINE__, "sb", strerror(errno), con->physical.path); @@ -401,13 +402,13 @@ static int connection_handle_write_prepare(server *srv, connection *con) { return -1; } - if (S_ISREG(con->fce->st.st_mode)) { + if (S_ISREG(sce->st.st_mode)) { if (con->request.http_method == HTTP_METHOD_GET || con->request.http_method == HTTP_METHOD_POST) { - http_chunk_append_file(srv, con, con->physical.path, 0, con->fce->st.st_size); + http_chunk_append_file(srv, con, con->physical.path, 0, sce->st.st_size); con->response.content_length = http_chunkqueue_length(srv, con); } else if (con->request.http_method == HTTP_METHOD_HEAD) { - con->response.content_length = con->fce->st.st_size; + con->response.content_length = sce->st.st_size; } else { connection_set_state(srv, con, CON_STATE_ERROR); return -1; @@ -415,7 +416,7 @@ static int connection_handle_write_prepare(server *srv, connection *con) { http_response_write_header(srv, con, con->response.content_length, - con->fce->st.st_mtime); + sce->st.st_mtime); } else { @@ -737,14 +738,14 @@ int connection_reset(server *srv, connection *con) { array_reset(con->environment); chunkqueue_reset(con->write_queue); - - if (con->fce) { - file_cache_entry_release(srv, con, con->fce); - con->fce = NULL; - } - + + /* the plugins should cleanup themself */ for (i = 0; i < srv->plugins.used; i++) { - con->plugin_ctx[0] = NULL; + if (con->plugin_ctx[i] != NULL) { + log_error_write(srv, __FILE__, __LINE__, "sb", "missing cleanup in", ((plugin **)(srv->plugins.ptr))[i]->name); + } + + con->plugin_ctx[i] = NULL; } con->header_len = 0; diff --git a/src/file_cache.c b/src/file_cache.c deleted file mode 100644 index dc1e1569..00000000 --- a/src/file_cache.c +++ /dev/null @@ -1,483 +0,0 @@ -#define _GNU_SOURCE - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "log.h" -#include "file_cache.h" -#include "fdevent.h" -#include "etag.h" - -#ifdef HAVE_ATTR_ATTRIBUTES_H -#include -#endif - -#include "sys-mmap.h" - -/* NetBSD 1.3.x needs it */ -#ifndef MAP_FAILED -# define MAP_FAILED -1 -#endif - -#ifndef O_LARGEFILE -# define O_LARGEFILE 0 -#endif - -#ifndef HAVE_LSTAT -#define lstat stat -#endif - -/* don't enable the dir-cache - * - * F_NOTIFY would be nice but only works with linux-rtsig - */ -#undef USE_LINUX_SIGIO - -file_cache *file_cache_init(void) { - file_cache *fc = NULL; - - fc = calloc(1, sizeof(*fc)); - - fc->dir_name = buffer_init(); - - return fc; -} - -static file_cache_entry * file_cache_entry_init(void) { - file_cache_entry *fce = NULL; - - fce = calloc(1, sizeof(*fce)); - - fce->fd = -1; - fce->fde_ndx = -1; - fce->name = buffer_init(); - fce->etag = buffer_init(); - fce->content_type = buffer_init(); - - return fce; -} - -static void file_cache_entry_free(server *srv, file_cache_entry *fce) { - if (!fce) return; - - if (fce->fd >= 0) { - close(fce->fd); - srv->cur_fds--; - } - - buffer_free(fce->etag); - buffer_free(fce->name); - buffer_free(fce->content_type); - - if (fce->mmap_p) munmap(fce->mmap_p, fce->mmap_length); - - free(fce); -} - -static int file_cache_entry_reset(server *srv, file_cache_entry *fce) { - if (fce->fd < 0) return 0; - - close(fce->fd); - srv->cur_fds--; - -#ifdef USE_LINUX_SIGIO - /* doesn't work anymore */ - if (fce->fde_ndx != -1) { - fdevent_event_del(srv->ev, &(fce->fde_ndx), fce->fd); - } -#else - UNUSED(srv); -#endif - - if (fce->mmap_p) { - munmap(fce->mmap_p, fce->mmap_length); - fce->mmap_p = NULL; - } - fce->fd = -1; - - buffer_reset(fce->etag); - buffer_reset(fce->name); - buffer_reset(fce->content_type); - - return 0; -} - -void file_cache_free(server *srv, file_cache *fc) { - size_t i; - for (i = 0; i < fc->used; i++) { - file_cache_entry_free(srv, fc->ptr[i]); - } - - free(fc->ptr); - - buffer_free(fc->dir_name); - - free(fc); - -} - -#ifdef HAVE_XATTR -int fce_attr_get(buffer *buf, char *name) { - int attrlen; - int ret; - - attrlen = 1024; - buffer_prepare_copy(buf, attrlen); - attrlen--; - if(0 == (ret = attr_get(name, "Content-Type", buf->ptr, &attrlen, 0))) { - buf->used = attrlen + 1; - buf->ptr[attrlen] = '\0'; - } - return ret; -} -#endif - -file_cache_entry * file_cache_get_unused_entry(server *srv) { - file_cache_entry *fce = NULL; - file_cache *fc = srv->file_cache; - size_t i; - - if (fc->size == 0) { - fc->size = 16; - fc->ptr = calloc(fc->size, sizeof(*fc->ptr)); - fc->used = 0; - } - for (i = 0; i < fc->used; i++) { - file_cache_entry *f = fc->ptr[i]; - - if (f->fd == -1) { - return f; - } - } - - if (fc->used < fc->size) { - fce = file_cache_entry_init(); - fc->ptr[fc->used++] = fce; - } else { - /* the cache is full, time to resize */ - - fc->size += 16; - fc->ptr = realloc(fc->ptr, sizeof(*fc->ptr) * fc->size); - - fce = file_cache_entry_init(); - fc->ptr[fc->used++] = fce; - } - - return fce; -} - -handler_t file_cache_handle_fdevent(void *_srv, void *_fce, int revent) { - size_t i; - server *srv = _srv; - file_cache_entry *fce = _fce; - file_cache *fc = srv->file_cache;; - - UNUSED(revent); - /* */ -#if 0 - log_error_write(srv, __FILE__, __LINE__, "sds", "dir has changed: ", fce->fd, fce->name->ptr); -#endif - /* touch all files below this directory */ - - for (i = 0; i < fc->used; i++) { - file_cache_entry *f = fc->ptr[i]; - - if (fce == f) continue; - - if (0 == strncmp(fce->name->ptr, f->name->ptr, fce->name->used - 1)) { -#if 0 - log_error_write(srv, __FILE__, __LINE__, "ss", "file hit: ", f->name->ptr); -#endif - f->is_dirty = 1; - } - } - - return HANDLER_GO_ON; -} - - -#if 0 -/* dead code, might be reused somewhere again */ - -int file_cache_check_cache() { - file_cache_entry *first_unused_fce = NULL; - file_cache *fc = srv->file_cache; - size_t i; - - /* check the cache */ - for (i = 0; i < fc->used; i++) { - fce = fc->ptr[i]; - - if (buffer_is_equal(name, fce->name)) { - log_error_write(srv, __FILE__, __LINE__, "sb", "cache hit:", name); - -#ifdef USE_LINUX_SIGIO - if (fce->is_dirty == 0) { - fce->in_use++; - return fce; - } -#endif - - /* get the etag information */ - if (-1 == (stat(name->ptr, &(fce->st)))) { - fce->in_use = 0; - - log_error_write(srv, __FILE__, __LINE__, "sbs", "stat failed: ", name, strerror(errno)); - - file_cache_entry_reset(srv, fce); - - return NULL; - } - fce->stat_ts = srv->cur_ts; - - /* create etag */ - etag_create(srv->file_cache_etag, &(fce->st)); - - if (!buffer_is_equal(srv->file_cache_etag, fce->etag)) { - size_t s_len = 0, k; - /* etag has changed: reopen file */ - - file_cache_entry_reset(srv, fce); - - if (-1 == (fce->fd = open(fce->name->ptr, O_RDONLY | O_LARGEFILE))) { - log_error_write(srv, __FILE__, __LINE__, "ss", "open failed: ", strerror(errno)); - - buffer_reset(fce->name); - return NULL; - } - - srv->cur_fds++; - - buffer_copy_string_buffer(fce->etag, srv->file_cache_etag); - - /* determine mimetype */ - buffer_reset(fce->content_type); - - s_len = name->used - 1; - - for (k = 0; k < con->conf.mimetypes->used; k++) { - data_string *ds = (data_string *)con->conf.mimetypes->data[k]; - size_t ct_len; - - ct_len = ds->key->used - 1; - - if (buffer_is_equal_right_len(name, ds->key, ct_len)) { - buffer_copy_string_buffer(fce->content_type, ds->value); - break; - } - } - -#ifdef HAVE_XATTR - if (buffer_is_empty(fce->content_type)) { - fce_attr_get(fce->content_type, name->ptr); - } -#endif - } - -#ifdef USE_LINUX_SIGIO - fce->is_dirty = 0; -#endif - - fce->in_use++; - - if (fce->fd == -1) { - log_error_write(srv, __FILE__, __LINE__, "sb", "fd is still -1 !", fce->name); - } - if (fce->st.st_size == 0) { - log_error_write(srv, __FILE__, __LINE__, "sb", "size is still 0 !", fce->name); - } - - - - - return fce; - } - - if (fce->in_use == 0) { - if (!first_unused_fce) first_unused_fce = fce; - - if (srv->cur_ts - fce->stat_ts > 10) { - file_cache_entry_reset(srv, fce); - } - } - } - - if (first_unused_fce) { - file_cache_entry_reset(srv, fce); - - fce = first_unused_fce; - } else { - /* not found, insert */ - fce = file_cache_get_unused_entry(srv); - } -} -#endif - - -handler_t file_cache_get_entry(server *srv, connection *con, buffer *name, file_cache_entry **o_fce_ptr) { - file_cache_entry *fce = NULL; - file_cache_entry *o_fce = *o_fce_ptr; - - - UNUSED(con); - - /* still valid ? */ - if (o_fce != NULL) { - if (buffer_is_equal(name, o_fce->name) && - (o_fce->fd != -1) && - (o_fce->stat_ts == srv->cur_ts) - ) { - return HANDLER_GO_ON; - } else { - o_fce->in_use--; - } - file_cache_entry_reset(srv, o_fce); - } - - - fce = file_cache_get_unused_entry(srv); - - buffer_copy_string_buffer(fce->name, name); - fce->in_use = 0; - fce->fd = -1; - - if (-1 == (con->conf.follow_symlink ? stat(name->ptr, &(fce->st)) : lstat(name->ptr, &(fce->st)))) { - int oerrno = errno; -#if 0 - log_error_write(srv, __FILE__, __LINE__, "sbs", "stat failed:", name, strerror(errno)); -#endif - file_cache_entry_reset(srv, fce); - - buffer_reset(fce->name); - - errno = oerrno; - return HANDLER_ERROR; - } - - fce->stat_ts = srv->cur_ts; - - if (S_ISREG(fce->st.st_mode)) { - size_t k, s_len; -#ifdef USE_LINUX_SIGIO - file_cache_entry *dir_fce; - char *slash; -#endif - - if (-1 == (fce->fd = open(name->ptr, O_RDONLY | O_LARGEFILE))) { - int oerrno = errno; - if (errno == EMFILE || errno == EINTR) { - return HANDLER_WAIT_FOR_FD; - } - - log_error_write(srv, __FILE__, __LINE__, "sbs", - "open failed for:", name, - strerror(errno)); - - buffer_reset(fce->name); - - errno = oerrno; - return HANDLER_ERROR; - } - - srv->cur_fds++; - - /* determine mimetype */ - buffer_reset(fce->content_type); - - s_len = name->used - 1; - - for (k = 0; k < con->conf.mimetypes->used; k++) { - data_string *ds = (data_string *)con->conf.mimetypes->data[k]; - size_t ct_len; - - if (ds->key->used == 0) continue; - - ct_len = ds->key->used - 1; - - if (s_len < ct_len) continue; - - if (0 == strncasecmp(name->ptr + s_len - ct_len, ds->key->ptr, ct_len)) { - buffer_copy_string_buffer(fce->content_type, ds->value); - break; - } - } - -#ifdef HAVE_XATTR - if (buffer_is_empty(fce->content_type)) { - fce_attr_get(fce->content_type, name->ptr); - } -#endif - - etag_create(fce->etag, &(fce->st)); - -#ifdef USE_LINUX_SIGIO - /* register sigio for the directory */ - dir_fce = file_cache_get_unused_entry(srv); - - buffer_copy_string_buffer(fc->dir_name, name); - - /* get dirname */ - if (0 == (slash = strrchr(fc->dir_name->ptr, '/'))) { - SEGFAULT(); - } - *(slash+1) = '\0'; - - if (-1 == (dir_fce->fd = open(fc->dir_name->ptr, O_RDONLY))) { - int oerrno = errno; - log_error_write(srv, __FILE__, __LINE__, "sbs", - "open failed:", fc->dir_name, strerror(errno)); - - errno = oerrno; - return HANDLER_ERROR; - } - - srv->cur_fds++; - - if (fcntl(dir_fce->fd, F_NOTIFY, DN_CREATE|DN_DELETE|DN_MODIFY|DN_MULTISHOT) < 0) { - int oerrno = errno; - log_error_write(srv, __FILE__, __LINE__, "ss", - "fcntl failed:", strerror(errno)); - - close(dir_fce->fd); - srv->cur_fds--; - - errno = oerrno; - return HANDLER_ERROR; - } - - /* ->used is not updated -> no _buffer copy */ - buffer_copy_string(dir_fce->name, fc->dir_name->ptr); - - /* register fd-handler */ - fdevent_register(srv->ev, dir_fce->fd, file_cache_handle_fdevent, dir_fce); - fdevent_event_add(srv->ev, &(dir_fce->fde_ndx), dir_fce->fd, FDEVENT_IN); -# if 1 - log_error_write(srv, __FILE__, __LINE__, "sddb", "fdevent_event_add:", fce->fde_ndx, fce->fd, fce->name); -# endif -#endif - } - - fce->in_use++; - - *o_fce_ptr = fce; - - return HANDLER_GO_ON; -} - -int file_cache_entry_release(server *srv, connection *con, file_cache_entry *fce) { - UNUSED(srv); - UNUSED(con); - - if (fce->in_use > 0) fce->in_use--; - file_cache_entry_reset(srv, fce); - - return 0; -} - diff --git a/src/file_cache.h b/src/file_cache.h deleted file mode 100644 index b45a7cea..00000000 --- a/src/file_cache.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _FILE_CACHE_H_ -#define _FILE_CACHE_H_ - -#include "base.h" - -file_cache *file_cache_init(void); -void file_cache_free(server *srv, file_cache *fc); - -handler_t file_cache_get_entry(server *srv, connection *con, buffer *name, file_cache_entry **o_fce); -int file_cache_entry_release(server *srv, connection *con, file_cache_entry *fce); - -#endif diff --git a/src/mod_compress.c b/src/mod_compress.c index c75b7fba..19d42134 100644 --- a/src/mod_compress.c +++ b/src/mod_compress.c @@ -13,6 +13,7 @@ #include "log.h" #include "buffer.h" #include "response.h" +#include "stat_cache.h" #include "plugin.h" @@ -318,7 +319,7 @@ static int deflate_file_to_buffer_bzip2(server *srv, connection *con, plugin_dat } #endif -static int deflate_file_to_file(server *srv, connection *con, plugin_data *p, buffer *fn, file_cache_entry *fce, int type) { +static int deflate_file_to_file(server *srv, connection *con, plugin_data *p, buffer *fn, stat_cache_entry *sce, int type) { int ifd, ofd; int ret = -1; void *start; @@ -326,14 +327,14 @@ static int deflate_file_to_file(server *srv, connection *con, plugin_data *p, bu ssize_t r; /* overflow */ - if ((off_t)(fce->st.st_size * 1.1) < fce->st.st_size) return -1; + if ((off_t)(sce->st.st_size * 1.1) < sce->st.st_size) return -1; /* don't mmap files > size_t * * we could use a sliding window, but currently there is no need for it */ - if (fce->st.st_size > SIZE_MAX) return -1; + if (sce->st.st_size > SIZE_MAX) return -1; buffer_reset(p->ofn); buffer_copy_string_buffer(p->ofn, p->conf.compress_cache_dir); @@ -380,7 +381,7 @@ static int deflate_file_to_file(server *srv, connection *con, plugin_data *p, bu return -1; } - buffer_append_string_buffer(p->ofn, fce->etag); + buffer_append_string_buffer(p->ofn, sce->etag); if (-1 == (ofd = open(p->ofn->ptr, O_WRONLY | O_CREAT | O_EXCL, 0600))) { if (errno == EEXIST) { @@ -409,7 +410,7 @@ static int deflate_file_to_file(server *srv, connection *con, plugin_data *p, bu } - if (MAP_FAILED == (start = mmap(NULL, fce->st.st_size, PROT_READ, MAP_SHARED, ifd, 0))) { + if (MAP_FAILED == (start = mmap(NULL, sce->st.st_size, PROT_READ, MAP_SHARED, ifd, 0))) { log_error_write(srv, __FILE__, __LINE__, "sbss", "mmaping", fn, "failed", strerror(errno)); close(ofd); @@ -420,15 +421,15 @@ static int deflate_file_to_file(server *srv, connection *con, plugin_data *p, bu switch(type) { #ifdef USE_ZLIB case HTTP_ACCEPT_ENCODING_GZIP: - ret = deflate_file_to_buffer_gzip(srv, con, p, start, fce->st.st_size, fce->st.st_mtime); + ret = deflate_file_to_buffer_gzip(srv, con, p, start, sce->st.st_size, sce->st.st_mtime); break; case HTTP_ACCEPT_ENCODING_DEFLATE: - ret = deflate_file_to_buffer_deflate(srv, con, p, start, fce->st.st_size); + ret = deflate_file_to_buffer_deflate(srv, con, p, start, sce->st.st_size); break; #endif #ifdef USE_BZ2LIB case HTTP_ACCEPT_ENCODING_BZIP2: - ret = deflate_file_to_buffer_bzip2(srv, con, p, start, fce->st.st_size); + ret = deflate_file_to_buffer_bzip2(srv, con, p, start, sce->st.st_size); break; #endif default: @@ -444,7 +445,7 @@ static int deflate_file_to_file(server *srv, connection *con, plugin_data *p, bu } - munmap(start, fce->st.st_size); + munmap(start, sce->st.st_size); close(ofd); close(ifd); @@ -455,21 +456,21 @@ static int deflate_file_to_file(server *srv, connection *con, plugin_data *p, bu return 0; } -static int deflate_file_to_buffer(server *srv, connection *con, plugin_data *p, buffer *fn, file_cache_entry *fce, int type) { +static int deflate_file_to_buffer(server *srv, connection *con, plugin_data *p, buffer *fn, stat_cache_entry *sce, int type) { int ifd; int ret = -1; void *start; buffer *b; /* overflow */ - if ((off_t)(fce->st.st_size * 1.1) < fce->st.st_size) return -1; + if ((off_t)(sce->st.st_size * 1.1) < sce->st.st_size) return -1; /* don't mmap files > size_t * * we could use a sliding window, but currently there is no need for it */ - if (fce->st.st_size > SIZE_MAX) return -1; + if (sce->st.st_size > SIZE_MAX) return -1; if (-1 == (ifd = open(fn->ptr, O_RDONLY))) { @@ -479,7 +480,7 @@ static int deflate_file_to_buffer(server *srv, connection *con, plugin_data *p, } - if (MAP_FAILED == (start = mmap(NULL, fce->st.st_size, PROT_READ, MAP_SHARED, ifd, 0))) { + if (MAP_FAILED == (start = mmap(NULL, sce->st.st_size, PROT_READ, MAP_SHARED, ifd, 0))) { log_error_write(srv, __FILE__, __LINE__, "sbss", "mmaping", fn, "failed", strerror(errno)); close(ifd); @@ -489,15 +490,15 @@ static int deflate_file_to_buffer(server *srv, connection *con, plugin_data *p, switch(type) { #ifdef USE_ZLIB case HTTP_ACCEPT_ENCODING_GZIP: - ret = deflate_file_to_buffer_gzip(srv, con, p, start, fce->st.st_size, fce->st.st_mtime); + ret = deflate_file_to_buffer_gzip(srv, con, p, start, sce->st.st_size, sce->st.st_mtime); break; case HTTP_ACCEPT_ENCODING_DEFLATE: - ret = deflate_file_to_buffer_deflate(srv, con, p, start, fce->st.st_size); + ret = deflate_file_to_buffer_deflate(srv, con, p, start, sce->st.st_size); break; #endif #ifdef USE_BZ2LIB case HTTP_ACCEPT_ENCODING_BZIP2: - ret = deflate_file_to_buffer_bzip2(srv, con, p, start, fce->st.st_size); + ret = deflate_file_to_buffer_bzip2(srv, con, p, start, sce->st.st_size); break; #endif default: @@ -505,7 +506,7 @@ static int deflate_file_to_buffer(server *srv, connection *con, plugin_data *p, break; } - munmap(start, fce->st.st_size); + munmap(start, sce->st.st_size); close(ifd); if (ret != 0) return -1; @@ -574,6 +575,7 @@ PHYSICALPATH_FUNC(mod_compress_physical) { data_string *content_ds; size_t m, i; off_t max_fsize; + stat_cache_entry *sce = NULL; /* only GET and POST can get compressed */ if (con->request.http_method != HTTP_METHOD_GET && @@ -590,9 +592,11 @@ PHYSICALPATH_FUNC(mod_compress_physical) { max_fsize = p->conf.compress_max_filesize; + + stat_cache_get_entry(srv, con, con->physical.path, &sce); /* don't compress files that are too large as we need to much time to handle them */ - if (max_fsize && (con->fce->st.st_size >> 10) > max_fsize) return HANDLER_GO_ON; + if (max_fsize && (sce->st.st_size >> 10) > max_fsize) return HANDLER_GO_ON; if (NULL == (content_ds = (data_string *)array_get_element(con->response.headers, "Content-Type"))) { log_error_write(srv, __FILE__, __LINE__, "sbb", "Content-Type is not set for", con->physical.path, con->uri.path); @@ -665,14 +669,14 @@ PHYSICALPATH_FUNC(mod_compress_physical) { /* deflate it */ if (p->conf.compress_cache_dir->used) { if (0 == deflate_file_to_file(srv, con, p, - con->physical.path, con->fce, compression_type)) { + con->physical.path, sce, compression_type)) { struct tm *tm; time_t last_mod; response_header_insert(srv, con, CONST_STR_LEN("Content-Encoding"), compression_name, strlen(compression_name)); /* Set Last-Modified of ORIGINAL file */ - last_mod = con->fce->st.st_mtime; + last_mod = sce->st.st_mtime; for (i = 0; i < FILE_CACHE_MAX; i++) { if (srv->mtime_cache[i].mtime == last_mod) break; @@ -709,7 +713,7 @@ PHYSICALPATH_FUNC(mod_compress_physical) { return HANDLER_FINISHED; } } else if (0 == deflate_file_to_buffer(srv, con, p, - con->physical.path, con->fce, compression_type)) { + con->physical.path, sce, compression_type)) { response_header_insert(srv, con, CONST_STR_LEN("Content-Encoding"), compression_name, strlen(compression_name)); diff --git a/src/mod_evhost.c b/src/mod_evhost.c index 4a07a923..3e92c3d6 100644 --- a/src/mod_evhost.c +++ b/src/mod_evhost.c @@ -5,7 +5,7 @@ #include "plugin.h" #include "log.h" #include "response.h" -#include "file_cache.h" +#include "stat_cache.h" typedef struct { /* unparsed pieces */ @@ -267,6 +267,7 @@ static handler_t mod_evhost_uri_handler(server *srv, connection *con, void *p_d) array *parsed_host; register char *ptr; int not_good = 0; + stat_cache_entry *sce = NULL; /* not authority set */ if (con->uri.authority->used == 0) return HANDLER_GO_ON; @@ -309,10 +310,10 @@ static handler_t mod_evhost_uri_handler(server *srv, connection *con, void *p_d) array_free(parsed_host); - if (HANDLER_GO_ON != file_cache_get_entry(srv, con, p->tmp_buf, &(con->fce))) { + if (HANDLER_ERROR != stat_cache_get_entry(srv, con, p->tmp_buf, &sce)) { log_error_write(srv, __FILE__, __LINE__, "sb", strerror(errno), p->tmp_buf); not_good = 1; - } else if(!S_ISDIR(con->fce->st.st_mode)) { + } else if(!S_ISDIR(sce->st.st_mode)) { log_error_write(srv, __FILE__, __LINE__, "sb", "not a directory:", p->tmp_buf); not_good = 1; } diff --git a/src/mod_expire.c b/src/mod_expire.c index d1ece297..623f4dde 100644 --- a/src/mod_expire.c +++ b/src/mod_expire.c @@ -9,10 +9,9 @@ #include "response.h" #include "plugin.h" +#include "stat_cache.h" -#ifdef HAVE_CONFIG_H #include "config.h" -#endif /** * this is a expire module for a lighttpd @@ -315,6 +314,9 @@ URIHANDLER_FUNC(mod_expire_path_handler) { int ts; time_t t; size_t len; + stat_cache_entry *sce = NULL; + + stat_cache_get_entry(srv, con, con->physical.path, &sce); switch(mod_expire_get_offset(srv, p, ds->value, &ts)) { case 0: @@ -324,7 +326,7 @@ URIHANDLER_FUNC(mod_expire_path_handler) { case 1: /* modification */ - t = (ts += con->fce->st.st_mtime); + t = (ts += sce->st.st_mtime); break; default: /* -1 is handled at parse-time */ diff --git a/src/mod_fastcgi.c b/src/mod_fastcgi.c index bd9c7324..57587dd2 100644 --- a/src/mod_fastcgi.c +++ b/src/mod_fastcgi.c @@ -1249,7 +1249,7 @@ static int fcgi_requestid_del(server *srv, plugin_data *p, size_t request_id) { return 0; } -void fcgi_connection_cleanup(server *srv, handler_ctx *hctx) { +void fcgi_connection_close(server *srv, handler_ctx *hctx) { plugin_data *p; connection *con; @@ -1348,7 +1348,7 @@ static int fcgi_reconnect(server *srv, handler_ctx *hctx) { static handler_t fcgi_connection_reset(server *srv, connection *con, void *p_d) { plugin_data *p = p_d; - fcgi_connection_cleanup(srv, con->plugin_ctx[p->id]); + fcgi_connection_close(srv, con->plugin_ctx[p->id]); return HANDLER_GO_ON; } @@ -2650,7 +2650,7 @@ SUBREQUEST_FUNC(mod_fastcgi_handle_subrequest) { } fcgi_restart_dead_procs(srv, p, host); - fcgi_connection_cleanup(srv, hctx); + fcgi_connection_close(srv, hctx); buffer_reset(con->physical.path); con->mode = DIRECT; @@ -2672,7 +2672,7 @@ SUBREQUEST_FUNC(mod_fastcgi_handle_subrequest) { return HANDLER_WAIT_FOR_FD; } else { - fcgi_connection_cleanup(srv, hctx); + fcgi_connection_close(srv, hctx); buffer_reset(con->physical.path); con->mode = DIRECT; @@ -2694,30 +2694,6 @@ SUBREQUEST_FUNC(mod_fastcgi_handle_subrequest) { } } -static handler_t fcgi_connection_close(server *srv, handler_ctx *hctx) { - plugin_data *p; - connection *con; - - if (NULL == hctx) return HANDLER_GO_ON; - - p = hctx->plugin_data; - con = hctx->remote_conn; - - if (con->mode != p->id) return HANDLER_GO_ON; - - log_error_write(srv, __FILE__, __LINE__, "ssdsd", - "emergency exit: fastcgi:", - "connection-fd:", con->fd, - "fcgi-fd:", hctx->fd); - - - - fcgi_connection_cleanup(srv, hctx); - - return HANDLER_FINISHED; -} - - static handler_t fcgi_handle_fdevent(void *s, void *ctx, int revents) { server *srv = (server *)s; handler_ctx *hctx = ctx; @@ -2747,13 +2723,13 @@ static handler_t fcgi_handle_fdevent(void *s, void *ctx, int revents) { buffer_copy_string_buffer(con->physical.path, host->docroot); buffer_append_string_buffer(con->physical.path, con->uri.path); - fcgi_connection_cleanup(srv, hctx); + fcgi_connection_close(srv, hctx); con->mode = DIRECT; con->file_started = 1; /* fcgi_extension won't touch the request afterwards */ } else { /* we are done */ - fcgi_connection_cleanup(srv, hctx); + fcgi_connection_close(srv, hctx); } joblist_append(srv, con); @@ -2825,7 +2801,7 @@ static handler_t fcgi_handle_fdevent(void *s, void *ctx, int revents) { "connection-fd:", con->fd, "fcgi-fd:", hctx->fd); - fcgi_connection_cleanup(srv, hctx); + fcgi_connection_close(srv, hctx); connection_set_state(srv, con, CON_STATE_HANDLE_REQUEST); buffer_reset(con->physical.path); @@ -2833,7 +2809,7 @@ static handler_t fcgi_handle_fdevent(void *s, void *ctx, int revents) { con->mode = DIRECT; } else { /* response might have been already started, kill the connection */ - fcgi_connection_cleanup(srv, hctx); + fcgi_connection_close(srv, hctx); log_error_write(srv, __FILE__, __LINE__, "ssdsd", "response already sent out, termination connection", @@ -3176,7 +3152,9 @@ JOBLIST_FUNC(mod_fastcgi_handle_joblist) { static handler_t fcgi_connection_close_callback(server *srv, connection *con, void *p_d) { plugin_data *p = p_d; - return fcgi_connection_close(srv, con->plugin_ctx[p->id]); + fcgi_connection_close(srv, con->plugin_ctx[p->id]); + + return HANDLER_GO_ON; } TRIGGER_FUNC(mod_fastcgi_handle_trigger) { diff --git a/src/mod_mysql_vhost.c b/src/mod_mysql_vhost.c index ac227781..523cc0e2 100644 --- a/src/mod_mysql_vhost.c +++ b/src/mod_mysql_vhost.c @@ -8,7 +8,7 @@ #include "config.h" #include "log.h" -#include "file_cache.h" +#include "stat_cache.h" #ifdef HAVE_MYSQL #include #endif @@ -322,6 +322,7 @@ static int mod_mysql_vhost_setup_connection(server *srv, connection *con, plugin CONNECTION_FUNC(mod_mysql_vhost_handle_docroot) { plugin_data *p = p_d; plugin_connection_data *c; + stat_cache_entry *sce; unsigned cols; MYSQL_ROW row; @@ -368,11 +369,12 @@ CONNECTION_FUNC(mod_mysql_vhost_handle_docroot) { /* sanity check that really is a directory */ buffer_copy_string(p->tmp_buf, row[0]); BUFFER_APPEND_SLASH(p->tmp_buf); - if (file_cache_get_entry(srv, con, p->tmp_buf, &(con->fce)) != HANDLER_GO_ON) { + + if (HANDLER_ERROR == stat_cache_get_entry(srv, con, p->tmp_buf, &sce)) { log_error_write(srv, __FILE__, __LINE__, "sb", strerror(errno), p->tmp_buf); goto ERR500; } - if (!S_ISDIR(con->fce->st.st_mode)) { + if (!S_ISDIR(sce->st.st_mode)) { log_error_write(srv, __FILE__, __LINE__, "sb", "Not a directory", p->tmp_buf); goto ERR500; } diff --git a/src/mod_proxy.c b/src/mod_proxy.c index a5ff3e0c..7775df61 100644 --- a/src/mod_proxy.c +++ b/src/mod_proxy.c @@ -335,21 +335,19 @@ SETDEFAULTS_FUNC(mod_proxy_set_defaults) { return HANDLER_GO_ON; } -void proxy_connection_cleanup(server *srv, handler_ctx *hctx) { +void proxy_connection_close(server *srv, handler_ctx *hctx) { plugin_data *p; - connection *con; + connection *con; if (NULL == hctx) return; p = hctx->plugin_data; con = hctx->remote_conn; - if (con->mode != p->id) return; - - fdevent_event_del(srv->ev, &(hctx->fde_ndx), hctx->fd); - fdevent_unregister(srv->ev, hctx->fd); - if (hctx->fd != -1) { + fdevent_event_del(srv->ev, &(hctx->fde_ndx), hctx->fd); + fdevent_unregister(srv->ev, hctx->fd); + close(hctx->fd); srv->cur_fds--; } @@ -358,14 +356,6 @@ void proxy_connection_cleanup(server *srv, handler_ctx *hctx) { con->plugin_ctx[p->id] = NULL; } -static handler_t mod_proxy_connection_reset(server *srv, connection *con, void *p_d) { - plugin_data *p = p_d; - - proxy_connection_cleanup(srv, con->plugin_ctx[p->id]); - - return HANDLER_GO_ON; -} - static int proxy_establish_connection(server *srv, handler_ctx *hctx) { struct sockaddr *proxy_addr; struct sockaddr_in proxy_addr_in; @@ -877,7 +867,7 @@ SUBREQUEST_FUNC(mod_proxy_handle_subrequest) { host->is_disabled = 1; host->disable_ts = srv->cur_ts; - proxy_connection_cleanup(srv, hctx); + proxy_connection_close(srv, hctx); /* reset the enviroment and restart the sub-request */ buffer_reset(con->physical.path); @@ -906,23 +896,6 @@ SUBREQUEST_FUNC(mod_proxy_handle_subrequest) { } } -static handler_t proxy_connection_close(server *srv, handler_ctx *hctx) { - plugin_data *p; - connection *con; - - if (NULL == hctx) return HANDLER_GO_ON; - - p = hctx->plugin_data; - con = hctx->remote_conn; - - if (con->mode != p->id) return HANDLER_GO_ON; - - proxy_connection_cleanup(srv, hctx); - - return HANDLER_FINISHED; -} - - static handler_t proxy_handle_fdevent(void *s, void *ctx, int revents) { server *srv = (server *)s; handler_ctx *hctx = ctx; @@ -945,7 +918,7 @@ static handler_t proxy_handle_fdevent(void *s, void *ctx, int revents) { hctx->host->usage--; /* we are done */ - proxy_connection_cleanup(srv, hctx); + proxy_connection_close(srv, hctx); joblist_append(srv, con); return HANDLER_FINISHED; @@ -1016,12 +989,10 @@ static handler_t proxy_handle_fdevent(void *s, void *ctx, int revents) { } else if (revents & FDEVENT_ERR) { /* kill all connections to the proxy process */ - log_error_write(srv, __FILE__, __LINE__, "s", "proxy-FDEVENT_ERR"); + log_error_write(srv, __FILE__, __LINE__, "sd", "proxy-FDEVENT_ERR, but no HUP", revents); joblist_append(srv, con); proxy_connection_close(srv, hctx); - - return HANDLER_ERROR; } return HANDLER_FINISHED; @@ -1238,7 +1209,9 @@ static handler_t mod_proxy_check_extension(server *srv, connection *con, void *p static handler_t mod_proxy_connection_close_callback(server *srv, connection *con, void *p_d) { plugin_data *p = p_d; - return proxy_connection_close(srv, con->plugin_ctx[p->id]); + proxy_connection_close(srv, con->plugin_ctx[p->id]); + + return HANDLER_GO_ON; } /** @@ -1284,17 +1257,17 @@ TRIGGER_FUNC(mod_proxy_trigger) { int mod_proxy_plugin_init(plugin *p) { - p->version = LIGHTTPD_VERSION_ID; + p->version = LIGHTTPD_VERSION_ID; p->name = buffer_init_string("proxy"); p->init = mod_proxy_init; p->cleanup = mod_proxy_free; p->set_defaults = mod_proxy_set_defaults; - p->connection_reset = mod_proxy_connection_reset; - p->handle_connection_close = mod_proxy_connection_close_callback; + p->connection_reset = mod_proxy_connection_close_callback; /* end of req-resp cycle */ + p->handle_connection_close = mod_proxy_connection_close_callback; /* end of client connection */ p->handle_uri_clean = mod_proxy_check_extension; p->handle_subrequest = mod_proxy_handle_subrequest; - p->handle_trigger = mod_proxy_trigger; + p->handle_trigger = mod_proxy_trigger; p->data = NULL; diff --git a/src/mod_simple_vhost.c b/src/mod_simple_vhost.c index 9fcef860..bd6d9049 100644 --- a/src/mod_simple_vhost.c +++ b/src/mod_simple_vhost.c @@ -6,7 +6,7 @@ #include "base.h" #include "log.h" #include "buffer.h" -#include "file_cache.h" +#include "stat_cache.h" #include "plugin.h" @@ -119,6 +119,8 @@ SETDEFAULTS_FUNC(mod_simple_vhost_set_defaults) { } static int build_doc_root(server *srv, connection *con, plugin_data *p, buffer *out, buffer *host) { + stat_cache_entry *sce = NULL; + buffer_prepare_copy(out, 128); if (p->conf.server_root->used) { @@ -151,13 +153,11 @@ static int build_doc_root(server *srv, connection *con, plugin_data *p, buffer * BUFFER_APPEND_SLASH(out); } - if (HANDLER_GO_ON != file_cache_get_entry(srv, con, out, &(con->fce))) { + if (HANDLER_ERROR != stat_cache_get_entry(srv, con, out, &sce)) { log_error_write(srv, __FILE__, __LINE__, "sb", strerror(errno), out); return -1; - } - - if (!S_ISDIR(con->fce->st.st_mode)) { + } else if (!S_ISDIR(sce->st.st_mode)) { return -1; } diff --git a/src/mod_ssi.c b/src/mod_ssi.c index a5938ddf..131d785b 100644 --- a/src/mod_ssi.c +++ b/src/mod_ssi.c @@ -10,6 +10,7 @@ #include "base.h" #include "log.h" #include "buffer.h" +#include "stat_cache.h" #include "plugin.h" #include "stream.h" @@ -313,6 +314,7 @@ static int process_ssi_stmt(server *srv, connection *con, plugin_data *p, /* echo */ int var = 0, enc = 0; const char *var_val = NULL; + stat_cache_entry *sce = NULL; struct { const char *var; @@ -376,6 +378,8 @@ static int process_ssi_stmt(server *srv, connection *con, plugin_data *p, l[1], "var is missing"); break; } + + stat_cache_get_entry(srv, con, con->physical.path, &sce); switch(var) { case SSI_ECHO_USER_NAME: { @@ -383,18 +387,18 @@ static int process_ssi_stmt(server *srv, connection *con, plugin_data *p, b = chunkqueue_get_append_buffer(con->write_queue); #ifdef HAVE_PWD_H - if (NULL == (pw = getpwuid(con->fce->st.st_uid))) { - buffer_copy_long(b, con->fce->st.st_uid); + if (NULL == (pw = getpwuid(sce->st.st_uid))) { + buffer_copy_long(b, sce->st.st_uid); } else { buffer_copy_string(b, pw->pw_name); } #else - buffer_copy_long(b, con->fce->st.st_uid); + buffer_copy_long(b, sce->st.st_uid); #endif break; } case SSI_ECHO_LAST_MODIFIED: { - time_t t = con->fce->st.st_mtime; + time_t t = sce->st.st_mtime; b = chunkqueue_get_append_buffer(con->write_queue); if (0 == strftime(buf, sizeof(buf), p->timefmt->ptr, localtime(&t))) { diff --git a/src/network.c b/src/network.c index 48df3e70..19139404 100644 --- a/src/network.c +++ b/src/network.c @@ -12,7 +12,6 @@ #include "network.h" #include "fdevent.h" #include "log.h" -#include "file_cache.h" #include "connections.h" #include "plugin.h" #include "joblist.h" diff --git a/src/network_freebsd_sendfile.c b/src/network_freebsd_sendfile.c index 71f9fa6a..5b7be511 100644 --- a/src/network_freebsd_sendfile.c +++ b/src/network_freebsd_sendfile.c @@ -136,26 +136,38 @@ int network_write_chunkqueue_freebsdsendfile(server *srv, connection *con, chunk case FILE_CHUNK: { off_t offset, r; size_t toSend; + stat_cache_entry *sce = NULL; + int ifd; - if (HANDLER_GO_ON != file_cache_get_entry(srv, con, c->data.file.name, &(con->fce))) { + if (HANDLER_ERROR == stat_cache_get_entry(srv, con, c->data.file.name, &sce)) { log_error_write(srv, __FILE__, __LINE__, "sb", strerror(errno), c->data.file.name); return -1; } offset = c->data.file.offset + c->offset; - toSend = c->data.file.length - c->offset; - - if (offset > con->fce->st.st_size) { + /* limit the toSend to 2^31-1 bytes in a chunk */ + toSend = c->data.file.length - c->offset > ((1 << 30) - 1) ? + ((1 << 30) - 1) : c->data.file.length - c->offset; + + if (offset > sce->st.st_size) { log_error_write(srv, __FILE__, __LINE__, "sb", "file was shrinked:", c->data.file.name); return -1; } - + + if (-1 == (ifd = open(c->data.file.name->ptr, O_RDONLY))) { + log_error_write(srv, __FILE__, __LINE__, "ss", "open failed: ", strerror(errno)); + + return -1; + } + r = 0; /* FreeBSD sendfile() */ - if (-1 == sendfile(con->fce->fd, fd, offset, toSend, NULL, &r, 0)) { + if (-1 == sendfile(ifd, fd, offset, toSend, NULL, &r, 0)) { + close(ifd); + switch(errno) { case EAGAIN: break; @@ -163,11 +175,11 @@ int network_write_chunkqueue_freebsdsendfile(server *srv, connection *con, chunk return -2; default: log_error_write(srv, __FILE__, __LINE__, "ssd", "sendfile: ", strerror(errno), errno); - return -1; } } - + close(ifd); + c->offset += r; con->bytes_written += r; diff --git a/src/network_linux_sendfile.c b/src/network_linux_sendfile.c index 829c73b9..4f2d4112 100644 --- a/src/network_linux_sendfile.c +++ b/src/network_linux_sendfile.c @@ -20,8 +20,7 @@ #include "network.h" #include "fdevent.h" #include "log.h" -#include "file_cache.h" - +#include "stat_cache.h" int network_write_chunkqueue_linuxsendfile(server *srv, connection *con, chunkqueue *cq) { const int fd = con->fd; @@ -129,48 +128,52 @@ int network_write_chunkqueue_linuxsendfile(server *srv, connection *con, chunkqu ssize_t r; off_t offset; size_t toSend; + stat_cache_entry *sce = NULL; + int ifd; - switch(file_cache_get_entry(srv, con, c->data.file.name, &(con->fce))) { - case HANDLER_GO_ON: - offset = c->data.file.offset + c->offset; - /* limit the toSend to 2^31-1 bytes in a chunk */ - toSend = c->data.file.length - c->offset > ((1 << 30) - 1) ? - ((1 << 30) - 1) : c->data.file.length - c->offset; - - if (offset > con->fce->st.st_size) { - log_error_write(srv, __FILE__, __LINE__, "sb", "file was shrinked:", c->data.file.name); - - return -1; - } - - /* Linux sendfile() */ - if (-1 == (r = sendfile(fd, con->fce->fd, &offset, toSend))) { - if (errno != EAGAIN && - errno != EINTR) { - log_error_write(srv, __FILE__, __LINE__, "ssd", "sendfile:", strerror(errno), errno); - - return -1; - } - - r = 0; - } - - break; - case HANDLER_WAIT_FOR_FD: - /* comeback later */ - - log_error_write(srv, __FILE__, __LINE__, "ssd", "sendfile (handled):", strerror(errno), errno); - - r = 0; - - break; - default: + if (HANDLER_ERROR == stat_cache_get_entry(srv, con, c->data.file.name, &sce)) { log_error_write(srv, __FILE__, __LINE__, "sb", strerror(errno), c->data.file.name); + return -1; + } + + offset = c->data.file.offset + c->offset; + /* limit the toSend to 2^31-1 bytes in a chunk */ + toSend = c->data.file.length - c->offset > ((1 << 30) - 1) ? + ((1 << 30) - 1) : c->data.file.length - c->offset; + + if (offset > sce->st.st_size) { + log_error_write(srv, __FILE__, __LINE__, "sb", "file was shrinked:", c->data.file.name); return -1; } + if (-1 == (ifd = open(c->data.file.name->ptr, O_RDONLY))) { + log_error_write(srv, __FILE__, __LINE__, "ss", "open failed: ", strerror(errno)); + + return -1; + } + + /* Linux sendfile() */ + if (-1 == (r = sendfile(fd, ifd, &offset, toSend))) { + switch (errno) { + case EAGAIN: + case EINTR: + r = 0; + break; + case EPIPE: + case ECONNRESET: + close(ifd); + return -2; + default: + log_error_write(srv, __FILE__, __LINE__, "ssd", + "sendfile failed:", strerror(errno), fd); + close(ifd); + return -1; + } + } + close(ifd); + c->offset += r; con->bytes_written += r; diff --git a/src/network_openssl.c b/src/network_openssl.c index cee42db8..8e520295 100644 --- a/src/network_openssl.c +++ b/src/network_openssl.c @@ -20,7 +20,7 @@ #include "network.h" #include "fdevent.h" #include "log.h" -#include "file_cache.h" +#include "stat_cache.h" # include # include @@ -102,11 +102,11 @@ int network_write_chunkqueue_openssl(server *srv, connection *con, chunkqueue *c ssize_t r; off_t offset; size_t toSend; -# if defined USE_MMAP char *p; -# endif + stat_cache_entry *sce = NULL; + int ifd; - if (HANDLER_GO_ON != file_cache_get_entry(srv, con, c->data.file.name, &(con->fce))) { + if (HANDLER_ERROR == stat_cache_get_entry(srv, con, c->data.file.name, &sce)) { log_error_write(srv, __FILE__, __LINE__, "sb", strerror(errno), c->data.file.name); return -1; @@ -115,27 +115,24 @@ int network_write_chunkqueue_openssl(server *srv, connection *con, chunkqueue *c offset = c->data.file.offset + c->offset; toSend = c->data.file.length - c->offset; - -#if defined USE_MMAP - if (MAP_FAILED == (p = mmap(0, con->fce->st.st_size, PROT_READ, MAP_SHARED, con->fce->fd, 0))) { - log_error_write(srv, __FILE__, __LINE__, "ss", "mmap failed: ", strerror(errno)); + if (-1 == (ifd = open(c->data.file.name->ptr, O_RDONLY))) { + log_error_write(srv, __FILE__, __LINE__, "ss", "open failed: ", strerror(errno)); return -1; } + - s = p + offset; -#else - buffer_prepare_copy(srv->tmp_buf, toSend); - - lseek(con->fce->fd, offset, SEEK_SET); - if (-1 == (toSend = read(con->fce->fd, srv->tmp_buf->ptr, toSend))) { - log_error_write(srv, __FILE__, __LINE__, "ss", "read failed: ", strerror(errno)); + if (MAP_FAILED == (p = mmap(0, sce->st.st_size, PROT_READ, MAP_SHARED, ifd, 0))) { + log_error_write(srv, __FILE__, __LINE__, "ss", "mmap failed: ", strerror(errno)); + + close(ifd); return -1; } - s = srv->tmp_buf->ptr; -#endif + close(ifd); + + s = p + offset; if ((r = SSL_write(con->ssl, s, toSend)) <= 0) { switch ((ssl_r = SSL_get_error(con->ssl, r))) { @@ -157,9 +154,7 @@ int network_write_chunkqueue_openssl(server *srv, connection *con, chunkqueue *c /* clean shutdown on the remote side */ if (r == 0) { -#if defined USE_MMAP munmap(p, c->data.file.length); -#endif return -2; } @@ -169,9 +164,7 @@ int network_write_chunkqueue_openssl(server *srv, connection *con, chunkqueue *c ssl_r, r, ERR_error_string(ERR_get_error(), NULL)); -#if defined USE_MMAP munmap(p, c->data.file.length); -#endif return -1; } } else { @@ -179,9 +172,7 @@ int network_write_chunkqueue_openssl(server *srv, connection *con, chunkqueue *c con->bytes_written += r; } -#if defined USE_MMAP munmap(p, c->data.file.length); -#endif if (c->offset == c->data.file.length) { chunk_finished = 1; diff --git a/src/network_write.c b/src/network_write.c index 542120b9..97701e97 100644 --- a/src/network_write.c +++ b/src/network_write.c @@ -14,7 +14,7 @@ #include "network.h" #include "fdevent.h" #include "log.h" -#include "file_cache.h" +#include "stat_cache.h" #include "sys-socket.h" @@ -71,8 +71,10 @@ int network_write_chunkqueue_write(server *srv, connection *con, chunkqueue *cq) ssize_t r; off_t offset; size_t toSend; + stat_cache_entry *sce = NULL; + int ifd; - if (HANDLER_GO_ON != file_cache_get_entry(srv, con, c->data.file.name, &(con->fce))) { + if (HANDLER_ERROR == stat_cache_get_entry(srv, con, c->data.file.name, &sce)) { log_error_write(srv, __FILE__, __LINE__, "sb", strerror(errno), c->data.file.name); return -1; @@ -81,77 +83,52 @@ int network_write_chunkqueue_write(server *srv, connection *con, chunkqueue *cq) offset = c->data.file.offset + c->offset; toSend = c->data.file.length - c->offset; - if (offset > con->fce->st.st_size) { + if (offset > sce->st.st_size) { log_error_write(srv, __FILE__, __LINE__, "sb", "file was shrinked:", c->data.file.name); return -1; } - - if (-1 == con->fce->fd) { - log_error_write(srv, __FILE__, __LINE__, "sb", "fd is invalid", c->data.file.name); + + if (-1 == (ifd = open(c->data.file.name->ptr, O_RDONLY))) { + log_error_write(srv, __FILE__, __LINE__, "ss", "open failed: ", strerror(errno)); return -1; } #if defined USE_MMAP - /* check if the mapping fits */ - if (con->fce->mmap_p && - con->fce->mmap_length != con->fce->st.st_size && - con->fce->mmap_offset != 0) { - munmap(con->fce->mmap_p, con->fce->mmap_length); + if (MAP_FAILED == (p = mmap(0, sce->st.st_size, PROT_READ, MAP_SHARED, ifd, 0))) { + log_error_write(srv, __FILE__, __LINE__, "ss", "mmap failed: ", strerror(errno)); + + close(ifd); - con->fce->mmap_p = NULL; - } - - /* build mapping if neccesary */ - if (con->fce->mmap_p == NULL) { - if (MAP_FAILED == (p = mmap(0, con->fce->st.st_size, PROT_READ, MAP_SHARED, con->fce->fd, 0))) { - log_error_write(srv, __FILE__, __LINE__, "ss", "mmap failed: ", strerror(errno)); - - return -1; - } - con->fce->mmap_p = p; - con->fce->mmap_offset = 0; - con->fce->mmap_length = con->fce->st.st_size; - } else { - p = con->fce->mmap_p; + return -1; } - + close(ifd); + if ((r = write(fd, p + offset, toSend)) <= 0) { log_error_write(srv, __FILE__, __LINE__, "ss", "write failed: ", strerror(errno)); return -1; } - /* don't cache mmap()ings for files large then 64k */ - if (con->fce->mmap_length > 64 * 1024) { - munmap(con->fce->mmap_p, con->fce->mmap_length); - - con->fce->mmap_p = NULL; - } - + munmap(p, sce->st.st_size); #else buffer_prepare_copy(srv->tmp_buf, toSend); - lseek(con->fce->fd, offset, SEEK_SET); - if (-1 == (toSend = read(con->fce->fd, srv->tmp_buf->ptr, toSend))) { + lseek(ifd, offset, SEEK_SET); + if (-1 == (toSend = read(ifd, srv->tmp_buf->ptr, toSend))) { log_error_write(srv, __FILE__, __LINE__, "ss", "read: ", strerror(errno)); + close(ifd); return -1; } -#ifdef __WIN32 + close(ifd); + if (-1 == (r = send(fd, srv->tmp_buf->ptr, toSend, 0))) { log_error_write(srv, __FILE__, __LINE__, "ss", "write: ", strerror(errno)); return -1; } -#else - if (-1 == (r = write(fd, srv->tmp_buf->ptr, toSend))) { - log_error_write(srv, __FILE__, __LINE__, "ss", "write: ", strerror(errno)); - - return -1; - } -#endif #endif c->offset += r; con->bytes_written += r; diff --git a/src/network_writev.c b/src/network_writev.c index 3421241c..bb3b5dd8 100644 --- a/src/network_writev.c +++ b/src/network_writev.c @@ -23,7 +23,7 @@ #include "network.h" #include "fdevent.h" #include "log.h" -#include "file_cache.h" +#include "stat_cache.h" #ifndef UIO_MAXIOV # if defined(__FreeBSD__) || defined(__APPLE__) || defined(__NetBSD__) @@ -156,50 +156,40 @@ int network_write_chunkqueue_writev(server *srv, connection *con, chunkqueue *cq ssize_t r; off_t offset; size_t toSend; + stat_cache_entry *sce = NULL; + int ifd; - switch (file_cache_get_entry(srv, con, c->data.file.name, &(con->fce))) { + switch (stat_cache_get_entry(srv, con, c->data.file.name, &sce)) { + case HANDLER_COMEBACK: case HANDLER_GO_ON: - if (con->fce->st.st_size == 0 || - con->fce->fd == -1) { - - log_error_write(srv, __FILE__, __LINE__, "sbdd", "foo", c->data.file.name, - con->fce->st.st_size, con->fce->fd); - } - offset = c->data.file.offset + c->offset; toSend = c->data.file.length - c->offset; - if (offset > con->fce->st.st_size) { - log_error_write(srv, __FILE__, __LINE__, "sb", "file was shrinked:", c->data.file.name); + if (offset > sce->st.st_size) { + log_error_write(srv, __FILE__, __LINE__, "sb", + "file was shrinked:", c->data.file.name); return -1; } + + if (-1 == (ifd = open(c->data.file.name->ptr, O_RDONLY))) { + log_error_write(srv, __FILE__, __LINE__, "ss", "open failed: ", strerror(errno)); + + return -1; + } + -#if defined USE_MMAP - /* check if the mapping fits */ - if (con->fce->mmap_p && - con->fce->mmap_length != con->fce->st.st_size && - con->fce->mmap_offset != 0) { - munmap(con->fce->mmap_p, con->fce->mmap_length); + if (MAP_FAILED == (p = mmap(0, sce->st.st_size, PROT_READ, MAP_SHARED, ifd, 0))) { + log_error_write(srv, __FILE__, __LINE__, "ssbd", "mmap failed: ", + strerror(errno), c->data.file.name, ifd); + + close(ifd); - con->fce->mmap_p = NULL; + return -1; } + + close(ifd); - /* build mapping if neccesary */ - if (con->fce->mmap_p == NULL) { - if (MAP_FAILED == (p = mmap(0, con->fce->st.st_size, PROT_READ, MAP_SHARED, con->fce->fd, 0))) { - log_error_write(srv, __FILE__, __LINE__, "ssbd", "mmap failed: ", - strerror(errno), c->data.file.name, con->fce->fd); - - return -1; - } - con->fce->mmap_p = p; - con->fce->mmap_offset = 0; - con->fce->mmap_length = con->fce->st.st_size; - } else { - p = con->fce->mmap_p; - } - if ((r = write(fd, p + offset, toSend)) <= 0) { switch (errno) { case EAGAIN: @@ -217,38 +207,7 @@ int network_write_chunkqueue_writev(server *srv, connection *con, chunkqueue *cq } } - /* don't cache mmap()ings for files large then 64k */ - if (con->fce->mmap_length > 64 * 1024) { - munmap(con->fce->mmap_p, con->fce->mmap_length); - - con->fce->mmap_p = NULL; - } - -#else - buffer_prepare_copy(srv->tmp_buf, toSend); - - lseek(con->fce->fd, offset, SEEK_SET); - if (-1 == (toSend = read(con->fce->fd, srv->tmp_buf->ptr, toSend))) { - log_error_write(srv, __FILE__, __LINE__, "ss", - "read:", strerror(errno)); - - return -1; - } - - if (-1 == (r = write(fd, srv->tmp_buf->ptr, toSend))) { - log_error_write(srv, __FILE__, __LINE__, "ss", "write: ", strerror(errno)); - - return -1; - } -#endif - - - break; - case HANDLER_WAIT_FOR_FD: - - log_error_write(srv, __FILE__, __LINE__, "ssd", "writev (handled):", strerror(errno), errno); - - r = 0; + munmap(p, sce->st.st_size); break; default: diff --git a/src/response.c b/src/response.c index ab24e520..15975205 100644 --- a/src/response.c +++ b/src/response.c @@ -17,7 +17,7 @@ #include "response.h" #include "keyvalue.h" #include "log.h" -#include "file_cache.h" +#include "stat_cache.h" #include "etag.h" #include "connections.h" @@ -1182,15 +1182,14 @@ handler_t http_response_prepare(server *srv, connection *con) { char *pathinfo = NULL; int found = 0; size_t k; + stat_cache_entry *sce = NULL; if (con->conf.log_request_handling) { log_error_write(srv, __FILE__, __LINE__, "s", "-- handling physical path"); log_error_write(srv, __FILE__, __LINE__, "sb", "Path :", con->physical.path); } - switch (file_cache_get_entry(srv, con, con->physical.path, &(con->fce))) { - case HANDLER_WAIT_FOR_FD: - return HANDLER_WAIT_FOR_FD; + switch (stat_cache_get_entry(srv, con, con->physical.path, &sce)) { case HANDLER_ERROR: if (errno == EACCES) { con->http_status = 403; @@ -1300,7 +1299,7 @@ handler_t http_response_prepare(server *srv, connection *con) { } /* setup the right file cache entry (FCE) */ - switch (file_cache_get_entry(srv, con, con->physical.path, &(con->fce))) { + switch (stat_cache_get_entry(srv, con, con->physical.path, &sce)) { case HANDLER_ERROR: con->http_status = 404; @@ -1311,10 +1310,7 @@ handler_t http_response_prepare(server *srv, connection *con) { } return HANDLER_FINISHED; - case HANDLER_WAIT_FOR_FD: - return HANDLER_WAIT_FOR_FD; case HANDLER_GO_ON: - break; default: break; } @@ -1326,7 +1322,7 @@ handler_t http_response_prepare(server *srv, connection *con) { log_error_write(srv, __FILE__, __LINE__, "sb", "Path :", con->physical.path); } - if (S_ISDIR(con->fce->st.st_mode)) { + if (S_ISDIR(sce->st.st_mode)) { if (con->physical.path->ptr[con->physical.path->used - 2] != '/') { /* redirect to .../ */ @@ -1343,7 +1339,8 @@ handler_t http_response_prepare(server *srv, connection *con) { buffer_copy_string_buffer(srv->tmp_buf, con->physical.path); buffer_append_string_buffer(srv->tmp_buf, ds->value); - switch (file_cache_get_entry(srv, con, srv->tmp_buf, &(con->fce))) { + switch (stat_cache_get_entry(srv, con, srv->tmp_buf, &sce)) { + case HANDLER_COMEBACK: case HANDLER_GO_ON: /* rewrite uri.path to the real path (/ -> /index.php) */ buffer_append_string_buffer(con->uri.path, ds->value); @@ -1407,13 +1404,13 @@ handler_t http_response_prepare(server *srv, connection *con) { break; } - if (!S_ISREG(con->fce->st.st_mode)) { + if (!S_ISREG(sce->st.st_mode)) { con->http_status = 404; if (con->conf.log_file_not_found) { log_error_write(srv, __FILE__, __LINE__, "sbsb", "not a regular file:", con->uri.path, - "->", con->fce->name); + "->", sce->name); } return HANDLER_FINISHED; @@ -1438,16 +1435,16 @@ handler_t http_response_prepare(server *srv, connection *con) { /* set response content-type */ - if (buffer_is_empty(con->fce->content_type)) { + if (buffer_is_empty(sce->content_type)) { response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_STR_LEN("application/octet-stream")); } else { - response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(con->fce->content_type)); + response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(sce->content_type)); } /* generate e-tag */ - etag_mutate(con->physical.etag, con->fce->etag); + etag_mutate(con->physical.etag, sce->etag); - http_response_handle_cachable(srv, con, con->fce->st.st_mtime); + http_response_handle_cachable(srv, con, sce->st.st_mtime); if (con->conf.range_requests && con->http_status == 0 && diff --git a/src/server.c b/src/server.c index fd525485..f761fe7e 100644 --- a/src/server.c +++ b/src/server.c @@ -25,7 +25,7 @@ #include "http_chunk.h" #include "fdevent.h" #include "connections.h" -#include "file_cache.h" +#include "stat_cache.h" #include "plugin.h" #include "joblist.h" @@ -125,7 +125,6 @@ static server *server_init(void) { CLEAN(errorlog_buf); CLEAN(response_range); CLEAN(tmp_buf); - CLEAN(file_cache_etag); CLEAN(range_buf); CLEAN(empty_string); @@ -165,8 +164,8 @@ static server *server_init(void) { srv->fdwaitqueue = calloc(1, sizeof(*srv->fdwaitqueue)); assert(srv->fdwaitqueue); - srv->file_cache = file_cache_init(); - assert(srv->file_cache); + srv->stat_cache = stat_cache_init(); + assert(srv->stat_cache); srv->srvconf.modules = array_init(); @@ -206,7 +205,6 @@ static void server_free(server *srv) { CLEAN(errorlog_buf); CLEAN(response_range); CLEAN(tmp_buf); - CLEAN(file_cache_etag); CLEAN(range_buf); CLEAN(empty_string); @@ -263,7 +261,7 @@ static void server_free(server *srv) { joblist_free(srv, srv->joblist); fdwaitqueue_free(srv, srv->fdwaitqueue); - file_cache_free(srv, srv->file_cache); + stat_cache_free(srv->stat_cache); array_free(srv->srvconf.modules); array_free(srv->split_vals); @@ -673,7 +671,6 @@ int main (int argc, char **argv) { srv->cur_fds = open("/dev/null", O_RDONLY); close(srv->cur_fds); - #ifdef HAVE_SIGACTION memset(&act, 0, sizeof(act)); act.sa_handler = SIG_IGN; @@ -715,6 +712,13 @@ int main (int argc, char **argv) { getitimer(ITIMER_REAL, &interval); #endif +#ifdef HAVE_FAM_H + /* setup FAM */ + srv->stat_cache->fam_fcce_ndx = -1; + fdevent_register(srv->ev, FAMCONNECTION_GETFD(srv->stat_cache->fam), stat_cache_handle_fdevent, NULL); + fdevent_event_add(srv->ev, &(srv->stat_cache->fam_fcce_ndx), FAMCONNECTION_GETFD(srv->stat_cache->fam), FDEVENT_IN); +#endif + #ifdef HAVE_FORK /* start watcher and workers */ num_childs = srv->srvconf.max_worker; @@ -801,7 +805,9 @@ int main (int argc, char **argv) { /* trigger waitpid */ srv->cur_ts = min_ts; - + + /* cleanup stat-cache */ + stat_cache_trigger_cleanup(srv); /** * check all connections for timeouts * diff --git a/src/splaytree.c b/src/splaytree.c new file mode 100644 index 00000000..cf09a325 --- /dev/null +++ b/src/splaytree.c @@ -0,0 +1,213 @@ +/* + An implementation of top-down splaying with sizes + D. Sleator , January 1994. + + This extends top-down-splay.c to maintain a size field in each node. + This is the number of nodes in the subtree rooted there. This makes + it possible to efficiently compute the rank of a key. (The rank is + the number of nodes to the left of the given key.) It it also + possible to quickly find the node of a given rank. Both of these + operations are illustrated in the code below. The remainder of this + introduction is taken from top-down-splay.c. + + "Splay trees", or "self-adjusting search trees" are a simple and + efficient data structure for storing an ordered set. The data + structure consists of a binary tree, with no additional fields. It + allows searching, insertion, deletion, deletemin, deletemax, + splitting, joining, and many other operations, all with amortized + logarithmic performance. Since the trees adapt to the sequence of + requests, their performance on real access patterns is typically even + better. Splay trees are described in a number of texts and papers + [1,2,3,4]. + + The code here is adapted from simple top-down splay, at the bottom of + page 669 of [2]. It can be obtained via anonymous ftp from + spade.pc.cs.cmu.edu in directory /usr/sleator/public. + + The chief modification here is that the splay operation works even if the + item being splayed is not in the tree, and even if the tree root of the + tree is NULL. So the line: + + t = splay(i, t); + + causes it to search for item with key i in the tree rooted at t. If it's + there, it is splayed to the root. If it isn't there, then the node put + at the root is the last one before NULL that would have been reached in a + normal binary search for i. (It's a neighbor of i in the tree.) This + allows many other operations to be easily implemented, as shown below. + + [1] "Data Structures and Their Algorithms", Lewis and Denenberg, + Harper Collins, 1991, pp 243-251. + [2] "Self-adjusting Binary Search Trees" Sleator and Tarjan, + JACM Volume 32, No 3, July 1985, pp 652-686. + [3] "Data Structure and Algorithm Analysis", Mark Weiss, + Benjamin Cummins, 1992, pp 119-130. + [4] "Data Structures, Algorithms, and Performance", Derick Wood, + Addison-Wesley, 1993, pp 367-375 +*/ + +#include +#include +#include "splaytree.h" + +#define compare(i,j) ((i)-(j)) +/* This is the comparison. */ +/* Returns <0 if i0 if i>j */ + +#define node_size(x) (((x)==NULL) ? 0 : ((x)->size)) +/* This macro returns the size of a node. Unlike "x->size", */ +/* it works even if x=NULL. The test could be avoided by using */ +/* a special version of NULL which was a real node with size 0. */ + +/* Splay using the key i (which may or may not be in the tree.) + * The starting root is t, and the tree used is defined by rat + * size fields are maintained */ +splay_tree * splaytree_splay (splay_tree *t, int i) { + splay_tree N, *l, *r, *y; + int comp, root_size, l_size, r_size; + + if (t == NULL) return t; + N.left = N.right = NULL; + l = r = &N; + root_size = node_size(t); + l_size = r_size = 0; + + for (;;) { + comp = compare(i, t->key); + if (comp < 0) { + if (t->left == NULL) break; + if (compare(i, t->left->key) < 0) { + y = t->left; /* rotate right */ + t->left = y->right; + y->right = t; + t->size = node_size(t->left) + node_size(t->right) + 1; + t = y; + if (t->left == NULL) break; + } + r->left = t; /* link right */ + r = t; + t = t->left; + r_size += 1+node_size(r->right); + } else if (comp > 0) { + if (t->right == NULL) break; + if (compare(i, t->right->key) > 0) { + y = t->right; /* rotate left */ + t->right = y->left; + y->left = t; + t->size = node_size(t->left) + node_size(t->right) + 1; + t = y; + if (t->right == NULL) break; + } + l->right = t; /* link left */ + l = t; + t = t->right; + l_size += 1+node_size(l->left); + } else { + break; + } + } + l_size += node_size(t->left); /* Now l_size and r_size are the sizes of */ + r_size += node_size(t->right); /* the left and right trees we just built.*/ + t->size = l_size + r_size + 1; + + l->right = r->left = NULL; + + /* The following two loops correct the size fields of the right path */ + /* from the left child of the root and the right path from the left */ + /* child of the root. */ + for (y = N.right; y != NULL; y = y->right) { + y->size = l_size; + l_size -= 1+node_size(y->left); + } + for (y = N.left; y != NULL; y = y->left) { + y->size = r_size; + r_size -= 1+node_size(y->right); + } + + l->right = t->left; /* assemble */ + r->left = t->right; + t->left = N.right; + t->right = N.left; + + return t; +} + +splay_tree * splaytree_insert(splay_tree * t, int i, void *data) { +/* Insert key i into the tree t, if it is not already there. */ +/* Return a pointer to the resulting tree. */ + splay_tree * new; + + if (t != NULL) { + t = splaytree_splay(t, i); + if (compare(i, t->key)==0) { + return t; /* it's already there */ + } + } + new = (splay_tree *) malloc (sizeof (splay_tree)); + assert(new); + if (t == NULL) { + new->left = new->right = NULL; + } else if (compare(i, t->key) < 0) { + new->left = t->left; + new->right = t; + t->left = NULL; + t->size = 1+node_size(t->right); + } else { + new->right = t->right; + new->left = t; + t->right = NULL; + t->size = 1+node_size(t->left); + } + new->key = i; + new->data = data; + new->size = 1 + node_size(new->left) + node_size(new->right); + return new; +} + +splay_tree * splaytree_delete(splay_tree *t, int i) { +/* Deletes i from the tree if it's there. */ +/* Return a pointer to the resulting tree. */ + splay_tree * x; + int tsize; + + if (t==NULL) return NULL; + tsize = t->size; + t = splaytree_splay(t, i); + if (compare(i, t->key) == 0) { /* found it */ + if (t->left == NULL) { + x = t->right; + } else { + x = splaytree_splay(t->left, i); + x->right = t->right; + } + free(t); + if (x != NULL) { + x->size = tsize-1; + } + return x; + } else { + return t; /* It wasn't there */ + } +} + +splay_tree *find_rank(int r, splay_tree *t) { +/* Returns a pointer to the node in the tree with the given rank. */ +/* Returns NULL if there is no such node. */ +/* Does not change the tree. To guarantee logarithmic behavior, */ +/* the node found here should be splayed to the root. */ + int lsize; + if ((r < 0) || (r >= node_size(t))) return NULL; + for (;;) { + lsize = node_size(t->left); + if (r < lsize) { + t = t->left; + } else if (r > lsize) { + r = r - lsize -1; + t = t->right; + } else { + return t; + } + } +} + + diff --git a/src/splaytree.h b/src/splaytree.h new file mode 100644 index 00000000..12255843 --- /dev/null +++ b/src/splaytree.h @@ -0,0 +1,17 @@ +#ifndef _SPLAY_TREE_H_ +#define _SPLAY_TREE_H_ + +typedef struct tree_node { + struct tree_node * left, * right; + int key; + int size; /* maintained to be the number of nodes rooted here */ + + void *data; +} splay_tree; + + +splay_tree * splaytree_splay (splay_tree *t, int key); +splay_tree * splaytree_insert(splay_tree *t, int key, void *data); +splay_tree * splaytree_delete(splay_tree *t, int key); + +#endif diff --git a/src/stat_cache.c b/src/stat_cache.c new file mode 100644 index 00000000..c9c216df --- /dev/null +++ b/src/stat_cache.c @@ -0,0 +1,498 @@ +#define _GNU_SOURCE + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "log.h" +#include "stat_cache.h" +#include "fdevent.h" +#include "etag.h" + +#ifdef HAVE_ATTR_ATTRIBUTES_H +#include +#endif + +#ifdef HAVE_FAM_H +# include +#endif + +#include "sys-mmap.h" + +/* NetBSD 1.3.x needs it */ +#ifndef MAP_FAILED +# define MAP_FAILED -1 +#endif + +#ifndef O_LARGEFILE +# define O_LARGEFILE 0 +#endif + +#ifndef HAVE_LSTAT +#define lstat stat +#endif + +/* + * stat-cache + * + * we cache the stat() calls in our own storage + * the directories are cached in FAM + * + * if we get a change-event from FAM, we increment the version in the FAM->dir mapping + * + * if the stat()-cache is queried we check if the version id for the directory is the + * same and return immediatly. + * + * + * What we need: + * + * - for each stat-cache entry we need a fast indirect lookup on the directory name + * - for each FAMRequest we have to find the version in the directory cache (index as userdata) + * + * stat <<-> directory <-> FAMRequest + * + * if file is deleted, directory is dirty, file is rechecked ... + * if directory is deleted, directory mapping is removed + * + * */ + +#ifdef HAVE_FAM_H +typedef struct { + FAMRequest *req; + FAMConnection *fc; + + buffer *name; + + int version; +} fam_dir_entry; +#endif + +/* the directory name is too long to always compare on it + * - we need a hash + * - the hash-key is used as sorting criteria for a tree + * - a splay-tree is used as we can use the caching effect of it + */ + +/* we want to cleanup the stat-cache every few seconds, let's say 10 + * + * - remove entries which are outdated since 30s + * - remove entries which are fresh but havn't been used since 60s + * - if we don't have a stat-cache entry for a directory, release it from the monitor + */ + +stat_cache *stat_cache_init(void) { + stat_cache *fc = NULL; + + fc = calloc(1, sizeof(*fc)); + + fc->dir_name = buffer_init(); +#ifdef HAVE_FAM_H + fc->fam = calloc(1, sizeof(*fc->fam)); + + if (0 != FAMOpen2(fc->fam, "lighttpd")) { + return NULL; + } + + FAMNoExists(fc->fam); +#endif + + return fc; +} + +static stat_cache_entry * stat_cache_entry_init(void) { + stat_cache_entry *sce = NULL; + + sce = calloc(1, sizeof(*sce)); + + sce->name = buffer_init(); + sce->etag = buffer_init(); + sce->content_type = buffer_init(); + + return sce; +} + +static void stat_cache_entry_free(void *data) { + stat_cache_entry *sce = data; + if (!sce) return; + + buffer_free(sce->etag); + buffer_free(sce->name); + buffer_free(sce->content_type); + + free(sce); +} + +static void splaytree_delete_tree(splay_tree *t, void (*data_free)(void *)) { + if (!t) return; + + splaytree_delete_tree(t->left, data_free); + splaytree_delete_tree(t->right, data_free); + + if (data_free) { + data_free(t->data); + } + + free(t); +} + +#ifdef HAVE_FAM_H +static fam_dir_entry * fam_dir_entry_init(void) { + fam_dir_entry *fam_dir = NULL; + + fam_dir = calloc(1, sizeof(*fam_dir)); + + fam_dir->name = buffer_init(); + + return fam_dir; +} + +static void fam_dir_entry_free(void *data) { + fam_dir_entry *fam_dir = data; + + if (!fam_dir) return; + + FAMCancelMonitor(fam_dir->fc, fam_dir->req); + + buffer_free(fam_dir->name); + free(fam_dir->req); + + free(fam_dir); +} +#endif + +void stat_cache_free(stat_cache *fc) { + splaytree_delete_tree(fc->files, stat_cache_entry_free); + + buffer_free(fc->dir_name); + +#ifdef HAVE_FAM_H + splaytree_delete_tree(fc->dirs, fam_dir_entry_free); + + FAMClose(fc->fam); + free(fc->fam); +#endif + free(fc); +} + +#ifdef HAVE_XATTR +static int stat_cache_entry_attr_get(buffer *buf, char *name) { + int attrlen; + int ret; + + attrlen = 1024; + buffer_prepare_copy(buf, attrlen); + attrlen--; + if(0 == (ret = attr_get(name, "Content-Type", buf->ptr, &attrlen, 0))) { + buf->used = attrlen + 1; + buf->ptr[attrlen] = '\0'; + } + return ret; +} +#endif + +static int hashme(buffer *str) { + int hash = 0; + const char *s; + for (s = str->ptr; *s; s++) { + hash = hash * 53 + *s; + } + + return hash; +} + +#ifdef HAVE_FAM_H +handler_t stat_cache_handle_fdevent(void *_srv, void *_fce, int revent) { + size_t i; + server *srv = _srv; + stat_cache *sc = srv->stat_cache; + size_t events; + + + UNUSED(revent); + /* */ + + events = FAMPending(sc->fam); + + for (i = 0; i < events; i++) { + FAMEvent fe; + fam_dir_entry *fam_dir; + splay_tree *node; + int ndx; + + FAMNextEvent(sc->fam, &fe); + + /* handle event */ + + switch(fe.code) { + case FAMChanged: + case FAMDeleted: + case FAMMoved: + /* if the filename is a directory remove the entry */ + + fam_dir = fe.userdata; + fam_dir->version++; + + /* file/dir is still here */ + if (fe.code == FAMChanged) break; + + buffer_copy_string(sc->dir_name, fe.filename); + + ndx = hashme(sc->dir_name); + + sc->dirs = splaytree_splay(sc->dirs, ndx); + node = sc->dirs; + + if (node && (node->key == ndx)) { + fam_dir_entry_free(node->data); + sc->dirs = splaytree_delete(sc->dirs, ndx); + } + break; + default: + break; + } + } + + return HANDLER_GO_ON; +} + +static int buffer_copy_dirname(buffer *dst, buffer *file) { + size_t i; + + if (buffer_is_empty(file)) return -1; + + for (i = file->used - 1; i+1 > 0; i--) { + if (file->ptr[i] == '/') { + buffer_copy_string_len(dst, file->ptr, i); + return 0; + } + } + + return -1; +} +#endif + +/*** + * + * + * + * returns: + * - HANDLER_FINISHED on cache-miss (don't forget to reopen the file) + * - HANDLER_ERROR on stat() failed -> see errno for problem + */ + +handler_t stat_cache_get_entry(server *srv, connection *con, buffer *name, stat_cache_entry **ret_sce) { +#ifdef HAVE_FAM_H + fam_dir_entry *fam_dir = NULL; + int dir_ndx = -1; + splay_tree *dir_node = NULL; +#endif + stat_cache_entry *sce = NULL; + stat_cache *sc; + struct stat st; + + int file_ndx; + splay_tree *file_node = NULL; + + *ret_sce = NULL; + + /* + * check if the directory for this file has changed + */ + + sc = srv->stat_cache; + + file_ndx = hashme(name); + sc->files = splaytree_splay(sc->files, file_ndx); + + if (sc->files && (sc->files->key == file_ndx)) { + /* we have seen this file already and + * don't stat() it again in the same second */ + + file_node = sc->files; + + sce = file_node->data; + + if (srv->srvconf.stat_cache_engine == STAT_CACHE_ENGINE_SIMPLE) { + if (sce->stat_ts == srv->cur_ts) { + *ret_sce = sce; + return HANDLER_GO_ON; + } + } + } + +#ifdef HAVE_FAM_H + /* dir-check */ + if (srv->srvconf.stat_cache_engine == STAT_CACHE_ENGINE_FAM) { + if (0 != buffer_copy_dirname(sc->dir_name, name)) { + SEGFAULT(); + } + + dir_ndx = hashme(sc->dir_name); + + sc->dirs = splaytree_splay(sc->dirs, dir_ndx); + + if (sc->dirs && (sc->dirs->key == dir_ndx)) { + dir_node = sc->dirs; + } + + if (dir_node && file_node) { + /* we found a file */ + + sce = file_node->data; + fam_dir = dir_node->data; + + if (fam_dir->version == sce->dir_version) { + /* the stat()-cache entry is still ok */ + + *ret_sce = sce; + return HANDLER_GO_ON; + } + } + } +#endif + if (-1 == (con->conf.follow_symlink ? stat(name->ptr, &st) : lstat(name->ptr, &st))) { + /* stat() failed, ENOENT, ... and so on */ + return HANDLER_ERROR; + } + + if (S_ISREG(st.st_mode) || S_ISDIR(st.st_mode)) { + size_t k; + + if (NULL == sce) { + sce = stat_cache_entry_init(); + buffer_copy_string_buffer(sce->name, name); + + sc->files = splaytree_insert(sc->files, file_ndx, sce); + } + + sce->st = st; + sce->stat_ts = srv->cur_ts; + + if (S_ISREG(st.st_mode)) { + /* determine mimetype */ + buffer_reset(sce->content_type); + + for (k = 0; k < con->conf.mimetypes->used; k++) { + data_string *ds = (data_string *)con->conf.mimetypes->data[k]; + + if (ds->key->used == 0) continue; + + if (buffer_is_equal_right_len(name, ds->key, ds->key->used - 1)) { + buffer_copy_string_buffer(sce->content_type, ds->value); + break; + } + } + etag_create(sce->etag, &(sce->st)); +#ifdef HAVE_XATTR + if (buffer_is_empty(sce->content_type)) { + stat_cache_attr_get(sce->content_type, name->ptr); + } +#endif + } + +#ifdef HAVE_FAM_H + if (srv->srvconf.stat_cache_engine == STAT_CACHE_ENGINE_FAM) { + /* is this directory already registered ? */ + if (!dir_node) { + fam_dir = fam_dir_entry_init(); + fam_dir->fc = sc->fam; + + buffer_copy_string_buffer(fam_dir->name, sc->dir_name); + + fam_dir->version = 1; + + fam_dir->req = calloc(1, sizeof(FAMRequest)); + + if (0 != FAMMonitorDirectory(sc->fam, fam_dir->name->ptr, + fam_dir->req, fam_dir)) { + + log_error_write(srv, __FILE__, __LINE__, "sbs", + "monitoring dir failed:", + fam_dir->name, + FamErrlist[FAMErrno]); + + fam_dir_entry_free(fam_dir); + } else { + sc->dirs = splaytree_insert(sc->dirs, dir_ndx, fam_dir); + } + } else { + fam_dir = dir_node->data; + } + + /* bind the fam_fc to the stat() cache entry */ + + if (fam_dir) { + sce->dir_version = fam_dir->version; + sce->dir_ndx = dir_ndx; + } + } +#endif + } + *ret_sce = sce; + + return HANDLER_GO_ON; +} + +/** + * remove stat() from cache which havn't been stat()ed for + * more than 10 seconds + * + * + * walk though the stat-cache, collect the ids which are too old + * and remove them in a second loop + */ + +static int stat_cache_tag_old_entries(server *srv, splay_tree *t, int *keys, size_t *ndx) { + stat_cache_entry *sce; + + if (!t) return 0; + + stat_cache_tag_old_entries(srv, t->left, keys, ndx); + stat_cache_tag_old_entries(srv, t->right, keys, ndx); + + sce = t->data; + + if (srv->cur_ts - sce->stat_ts > 10) { + keys[(*ndx)++] = t->key; + } + + return 0; +} + +int stat_cache_trigger_cleanup(server *srv) { + stat_cache *sc; + size_t max_ndx = 0, i; + int *keys; + + sc = srv->stat_cache; + + if (!sc->files) return 0; + + keys = calloc(1, sizeof(size_t) * sc->files->size); + + stat_cache_tag_old_entries(srv, sc->files, keys, &max_ndx); + + for (i = 0; i < max_ndx; i++) { + int ndx = keys[i]; + splay_tree *node; + + sc->files = splaytree_splay(sc->files, ndx); + + node = sc->files; + + if (node && (node->key == ndx)) { + stat_cache_entry_free(node->data); + sc->files = splaytree_delete(sc->files, ndx); + } + } + + free(keys); + + return 0; +} diff --git a/src/stat_cache.h b/src/stat_cache.h new file mode 100644 index 00000000..e6f77966 --- /dev/null +++ b/src/stat_cache.h @@ -0,0 +1,13 @@ +#ifndef _FILE_CACHE_H_ +#define _FILE_CACHE_H_ + +#include "base.h" + +stat_cache *stat_cache_init(void); +void stat_cache_free(stat_cache *fc); + +handler_t stat_cache_get_entry(server *srv, connection *con, buffer *name, stat_cache_entry **fce); +handler_t stat_cache_handle_fdevent(void *_srv, void *_fce, int revent); + +int stat_cache_trigger_cleanup(server *srv); +#endif