From 4d2cf580df969d964ff265ddf2358926e0432dd5 Mon Sep 17 00:00:00 2001 From: Thomas Porzelt Date: Wed, 4 Mar 2009 02:54:38 +0100 Subject: [PATCH] add stat_cach_get_dir() to get directory listing; add stat_cache.ttl setup --- include/lighttpd/server.h | 2 + include/lighttpd/stat_cache.h | 39 +++++--- include/lighttpd/typedefs.h | 2 + src/plugin_core.c | 27 +++-- src/stat_cache.c | 179 +++++++++++++++++++++++++++------- src/worker.c | 2 +- 6 files changed, 192 insertions(+), 59 deletions(-) diff --git a/include/lighttpd/server.h b/include/lighttpd/server.h index de560a8..6d38c8e 100644 --- a/include/lighttpd/server.h +++ b/include/lighttpd/server.h @@ -75,6 +75,8 @@ struct server { guint keep_alive_queue_timeout; gdouble io_timeout; + + gdouble stat_cache_ttl; }; diff --git a/include/lighttpd/stat_cache.h b/include/lighttpd/stat_cache.h index 0f42b71..85920c3 100644 --- a/include/lighttpd/stat_cache.h +++ b/include/lighttpd/stat_cache.h @@ -39,28 +39,39 @@ #error Please include instead of this file #endif -struct stat_cache_entry { +struct stat_cache_entry_data { GString *path; GString *etag; GString *content_type; + gboolean failed; struct stat st; - ev_tstamp ts; /* timestamp the entry was created (not when the stat() was done) */ gint err; - gboolean failed; +}; + +struct stat_cache_entry { enum { - STAT_CACHE_ENTRY_WAITING, /* waiting for stat thread to do the work, no info available */ - STAT_CACHE_ENTRY_FINISHED, /* stat() done, info available */ + STAT_CACHE_ENTRY_SINGLE, /* single file, this is the default or "normal" */ + STAT_CACHE_ENTRY_DIR /* get a directory listing (with stat info) */ + } type; + + enum { + STAT_CACHE_ENTRY_WAITING, /* waiting for stat thread to do the work, no info available */ + STAT_CACHE_ENTRY_FINISHED, /* stat() done, info available */ } state; - GPtrArray *vrequests; /* vrequests waiting for this info */ + + stat_cache_entry_data data; + GArray *dirlist; /* array of stat_cache_dirlist_entry, used together with STAT_CACHE_ENTRY_DIR */ + ev_tstamp ts; /* timestamp the entry was created (not when the stat() was done) */ + GPtrArray *vrequests; /* vrequests waiting for this info */ guint refcount; - waitqueue_elem queue_elem; /* queue element for the delete_queue */ + waitqueue_elem queue_elem; /* queue element for the delete_queue */ gboolean in_cache; }; struct stat_cache { GHashTable *entries; - GAsyncQueue *job_queue_out; /* elements waiting for stat */ - GAsyncQueue *job_queue_in; /* elements with finished stat */ + GAsyncQueue *job_queue_out; /* elements waiting for stat */ + GAsyncQueue *job_queue_in; /* elements with finished stat */ waitqueue delete_queue; GThread *thread; ev_async job_watcher; @@ -73,17 +84,15 @@ struct stat_cache { void stat_cache_new(worker *wrk, gdouble ttl); void stat_cache_free(stat_cache *sc); -void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents); -void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents); -gpointer stat_cache_thread(gpointer data); - -void stat_cache_entry_free(stat_cache_entry *sce); /* gets a stat_cache_entry for a specified path returns NULL in case of a cache MISS and you should return HANDLER_WAIT_FOR_EVENT */ -LI_API stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path); +LI_API stat_cache_entry *stat_cache_get(vrequest *vr, GString *path); + +/* like stat_cache_entry_get but gets stat info for a whole directory */ +LI_API stat_cache_entry *stat_cache_get_dir(vrequest *vr, GString *path); /* release a stat_cache_entry so it can be cleaned up */ LI_API void stat_cache_entry_release(vrequest *vr); diff --git a/include/lighttpd/typedefs.h b/include/lighttpd/typedefs.h index 5bc7bfd..d44a2b8 100644 --- a/include/lighttpd/typedefs.h +++ b/include/lighttpd/typedefs.h @@ -204,6 +204,8 @@ typedef struct filters filters; struct worker; typedef struct worker worker; +struct stat_cache_entry_data; +typedef struct stat_cache_entry_data stat_cache_entry_data; struct stat_cache_entry; typedef struct stat_cache_entry stat_cache_entry; struct stat_cache; diff --git a/src/plugin_core.c b/src/plugin_core.c index 2a71bfb..0aabd48 100644 --- a/src/plugin_core.c +++ b/src/plugin_core.c @@ -197,15 +197,15 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont if (!vrequest_handle_direct(vr)) return HANDLER_GO_ON; } - sce = stat_cache_entry_get(vr, vr->physical.path); + sce = stat_cache_get(vr, vr->physical.path); if (!sce) return HANDLER_WAIT_FOR_EVENT; VR_DEBUG(vr, "serving static file: %s", vr->physical.path->str); - if (sce->failed) { + if (sce->data.failed) { /* stat failed */ - VR_DEBUG(vr, "stat() failed: %s (%d)", g_strerror(sce->err), sce->err); + VR_DEBUG(vr, "stat() failed: %s (%d)", g_strerror(sce->data.err), sce->data.err); switch (errno) { case ENOENT: @@ -237,7 +237,7 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont /* redirect to scheme + host + path + / + querystring if directory without trailing slash */ /* TODO: local addr if HTTP 1.0 without host header */ - if (S_ISDIR(sce->st.st_mode) && vr->request.uri.orig_path->str[vr->request.uri.orig_path->len-1] != '/') { + if (S_ISDIR(sce->data.st.st_mode) && vr->request.uri.orig_path->str[vr->request.uri.orig_path->len-1] != '/') { GString *host = vr->request.uri.authority->len ? vr->request.uri.authority : vr->con->local_addr_str; GString *uri = g_string_sized_new( 8 /* https:// */ + host->len + @@ -259,7 +259,7 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont http_header_overwrite(vr->response.headers, CONST_STR_LEN("Location"), GSTR_LEN(uri)); g_string_free(uri, TRUE); close(fd); - } else if (!S_ISREG(sce->st.st_mode)) { + } else if (!S_ISREG(sce->data.st.st_mode)) { vr->response.http_status = 404; close(fd); } else { @@ -269,7 +269,7 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont http_header_overwrite(vr->response.headers, CONST_STR_LEN("Content-Type"), GSTR_LEN(mime_str)); else http_header_overwrite(vr->response.headers, CONST_STR_LEN("Content-Type"), CONST_STR_LEN("application/octet-stream")); - chunkqueue_append_file_fd(vr->out, NULL, 0, sce->st.st_size, fd); + chunkqueue_append_file_fd(vr->out, NULL, 0, sce->data.st.st_size, fd); } } @@ -579,6 +579,19 @@ static gboolean core_io_timeout(server *srv, plugin* p, value *val) { return TRUE; } +static gboolean core_stat_cache_ttl(server *srv, plugin* p, value *val) { + UNUSED(p); + + if (!val || val->type != VALUE_NUMBER || val->data.number < 1) { + ERROR(srv, "%s", "stat_cache.ttl expects a positive number as parameter"); + return FALSE; + } + + srv->stat_cache_ttl = (gdouble)value_extract(val).number; + + return TRUE; +} + /* * OPTIONS */ @@ -1081,6 +1094,8 @@ static const plugin_setup setups[] = { { "workers", core_workers }, { "module_load", core_module_load }, { "io_timeout", core_io_timeout }, + { "stat_cache.ttl", core_stat_cache_ttl }, + { NULL, NULL } }; diff --git a/src/stat_cache.c b/src/stat_cache.c index 69e26bb..c8ebe13 100644 --- a/src/stat_cache.c +++ b/src/stat_cache.c @@ -1,10 +1,20 @@ #include +static void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents); +static void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents); +static gpointer stat_cache_thread(gpointer data); +static void stat_cache_entry_free(stat_cache_entry *sce); +static stat_cache_entry *stat_cache_get_internal(vrequest *vr, GString *path, gboolean dir); + void stat_cache_new(worker *wrk, gdouble ttl) { stat_cache *sc; GError *err; + /* ttl default 10s */ + if (ttl < 1) + ttl = 10.0; + sc = g_slice_new0(stat_cache); sc->ttl = ttl; sc->entries = g_hash_table_new_full((GHashFunc)g_string_hash, (GEqualFunc)g_string_equal, NULL, NULL); @@ -51,7 +61,7 @@ void stat_cache_free(stat_cache *sc) { g_slice_free(stat_cache, sc); } -void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) { +static void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) { stat_cache *sc = (stat_cache*) w->data; stat_cache_entry *sce; waitqueue_elem *wqe; @@ -68,7 +78,7 @@ void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) { } else { /* no more vrequests using this entry, finally free it */ if (sce->in_cache) - g_hash_table_remove(sc->entries, sce->path); + g_hash_table_remove(sc->entries, sce->data.path); stat_cache_entry_free(sce); } } @@ -76,7 +86,7 @@ void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) { waitqueue_update(&sc->delete_queue); } -void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) { +static void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) { guint i; stat_cache_entry *sce; stat_cache *sc = ((worker*)w->data)->stat_cache; @@ -86,7 +96,7 @@ void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) { UNUSED(revents); while ((sce = g_async_queue_try_pop(sc->job_queue_in)) != NULL) { - if (sce->failed) + if (sce->data.failed) sc->errors++; for (i = 0; i < sce->vrequests->len; i++) { @@ -98,16 +108,95 @@ void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) { } } -void stat_cache_entry_free(stat_cache_entry *sce) { +static void stat_cache_entry_free(stat_cache_entry *sce) { + guint i; + assert(sce->vrequests->len == 0); assert(sce->refcount == 0); - g_string_free(sce->path, TRUE); + + g_string_free(sce->data.path, TRUE); g_ptr_array_free(sce->vrequests, TRUE); + + if (sce->type == STAT_CACHE_ENTRY_DIR) { + for (i = 0; i < sce->dirlist->len; i++) { + g_string_free(g_array_index(sce->dirlist, stat_cache_entry_data, i).path, TRUE); + } + + g_array_free(sce->dirlist, TRUE); + } + g_slice_free(stat_cache_entry, sce); } -stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) { +static gpointer stat_cache_thread(gpointer data) { + stat_cache *sc = data; + stat_cache_entry *sce; + + while (TRUE) { + sce = g_async_queue_pop(sc->job_queue_out); + + /* stat cache entry with path == NULL indicates server stop */ + if (!sce->data.path) + break; + + if (sce->type == STAT_CACHE_ENTRY_SINGLE) { + if (stat(sce->data.path->str, &sce->data.st) == -1) { + sce->data.failed = TRUE; + sce->data.err = errno; + } else { + sce->data.failed = FALSE; + } + } else { + /* dirlisting */ + DIR *dirp; + gsize size; + struct dirent *entry; + struct dirent *result; + gint error; + stat_cache_entry_data sced; + + dirp = opendir(sce->data.path->str); + if (dirp == NULL) { + sce->data.failed = TRUE; + sce->data.err = errno; + } else { + size = dirent_buf_size(dirp); + assert(size != (gsize)-1); + entry = g_slice_alloc(size); + + + while ((error = readdir_r(dirp, entry, &result)) != 0 && result != NULL) { + sced.path = g_string_sized_new(32); + g_string_assign(sced.path, result->d_name); + if (stat(result->d_name, &sced.st) == -1) { + sced.failed = TRUE; + sced.err = errno; + } else { + sced.failed = FALSE; + } + g_array_append_val(sce->dirlist, sced); + } + + if (error) { + sce->data.failed = TRUE; + sce->data.err = error; + } + + g_slice_free1(size, entry); + closedir(dirp); + } + } + + g_atomic_int_set(&sce->state, STAT_CACHE_ENTRY_FINISHED); + g_async_queue_push(sc->job_queue_in, sce); + ev_async_send(sc->delete_queue.loop, &sc->job_watcher); + } + + return NULL; +} + +static stat_cache_entry *stat_cache_get_internal(vrequest *vr, GString *path, gboolean dir) { stat_cache *sc; stat_cache_entry *sce; @@ -127,20 +216,46 @@ stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) { vr->stat_cache_entry = sce; sce->refcount++; } + VR_DEBUG(vr, "stat_cache: %"G_GUINT64_FORMAT" hits, %"G_GUINT64_FORMAT" misses, %u items in cache", sc->hits, sc->misses, g_hash_table_size(sc->entries)); return sce; } else { /* entry old */ if (sce->refcount == 0) { /* no vrequests working on the entry, reuse it */ + if (sce->type == STAT_CACHE_ENTRY_DIR) { + if (!dir) { + guint i; + for (i = 0; i < sce->dirlist->len; i++) { + g_string_free(g_array_index(sce->dirlist, stat_cache_entry_data, i).path, TRUE); + } + + g_array_free(sce->dirlist, TRUE); + sce->type = STAT_CACHE_ENTRY_SINGLE; + } else { + g_array_set_size(sce->dirlist, 0); + } + } else { + /* single file */ + if (dir) { + sce->dirlist = g_array_sized_new(FALSE, FALSE, sizeof(stat_cache_entry_data), 32); + sce->type = STAT_CACHE_ENTRY_DIR; + } + } } else { /* there are still vrequests using this entry, replace with a new one */ sce->in_cache = FALSE; sce = g_slice_new0(stat_cache_entry); - sce->path = g_string_new_len(GSTR_LEN(path)); + sce->data.path = g_string_new_len(GSTR_LEN(path)); sce->vrequests = g_ptr_array_sized_new(8); sce->in_cache = TRUE; sce->queue_elem.data = sce; - g_hash_table_replace(sc->entries, sce->path, sce); + g_hash_table_replace(sc->entries, sce->data.path, sce); + if (dir) { + sce->type = STAT_CACHE_ENTRY_DIR; + sce->dirlist = g_array_sized_new(FALSE, FALSE, sizeof(stat_cache_entry_data), 32); + } else { + sce->type = STAT_CACHE_ENTRY_SINGLE; + } } sce->ts = CUR_TS(vr->con->wrk); @@ -164,7 +279,7 @@ stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) { } else { /* cache miss, allocate new entry */ sce = g_slice_new0(stat_cache_entry); - sce->path = g_string_new_len(GSTR_LEN(path)); + sce->data.path = g_string_new_len(GSTR_LEN(path)); sce->vrequests = g_ptr_array_sized_new(8); sce->ts = CUR_TS(vr->con->wrk); sce->state = STAT_CACHE_ENTRY_WAITING; @@ -174,40 +289,30 @@ stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) { g_ptr_array_add(sce->vrequests, vr); sce->refcount = 1; waitqueue_push(&sc->delete_queue, &sce->queue_elem); - g_hash_table_insert(sc->entries, sce->path, sce); + g_hash_table_insert(sc->entries, sce->data.path, sce); g_async_queue_push(sc->job_queue_out, sce); sc->misses++; + + if (dir) { + sce->type = STAT_CACHE_ENTRY_DIR; + sce->dirlist = g_array_sized_new(FALSE, FALSE, sizeof(stat_cache_entry_data), 32); + } else { + sce->type = STAT_CACHE_ENTRY_SINGLE; + } + return NULL; } } -void stat_cache_entry_release(vrequest *vr) { - vr->stat_cache_entry->refcount--; - vr->stat_cache_entry = NULL; +stat_cache_entry *stat_cache_get(vrequest *vr, GString *path) { + return stat_cache_get_internal(vr, path, FALSE); } +stat_cache_entry *stat_cache_get_dir(vrequest *vr, GString *path) { + return stat_cache_get_internal(vr, path, TRUE); +} -gpointer stat_cache_thread(gpointer data) { - stat_cache *sc = data; - stat_cache_entry *sce; - - while (TRUE) { - sce = g_async_queue_pop(sc->job_queue_out); - - /* stat cache entry with path == NULL indicates server stop */ - if (!sce->path) - break; - - if (stat(sce->path->str, &sce->st) == -1) { - sce->failed = TRUE; - sce->err = errno; - } else - sce->failed = FALSE; - - g_atomic_int_set(&sce->state, STAT_CACHE_ENTRY_FINISHED); - g_async_queue_push(sc->job_queue_in, sce); - ev_async_send(sc->delete_queue.loop, &sc->job_watcher); - } - - return NULL; +void stat_cache_entry_release(vrequest *vr) { + vr->stat_cache_entry->refcount--; + vr->stat_cache_entry = NULL; } \ No newline at end of file diff --git a/src/worker.c b/src/worker.c index 8392b4c..7d121e5 100644 --- a/src/worker.c +++ b/src/worker.c @@ -335,7 +335,7 @@ worker* worker_new(struct server *srv, struct ev_loop *loop) { ev_timer_init(&wrk->job_queue_watcher, worker_job_queue_cb, 0, 0); wrk->job_queue_watcher.data = wrk; - stat_cache_new(wrk, 10.0); + stat_cache_new(wrk, srv->stat_cache_ttl); return wrk; }