add stat_cach_get_dir() to get directory listing; add stat_cache.ttl setup

include/lighttpd/server.h

@@ -75,6 +75,8 @@ struct server {
 	guint keep_alive_queue_timeout;
 
 	gdouble io_timeout;
+
+	gdouble stat_cache_ttl;
 };
 
 

include/lighttpd/stat_cache.h

@@ -39,28 +39,39 @@
 #error Please include <lighttpd/base.h> instead of this file
 #endif
 
-struct stat_cache_entry {
+struct stat_cache_entry_data {
 	GString *path;
 	GString *etag;
 	GString *content_type;
+	gboolean failed;
 	struct stat st;
-	ev_tstamp ts;                  /* timestamp the entry was created (not when the stat() was done) */
 	gint err;
-	gboolean failed;
+};
+
+struct stat_cache_entry {
 	enum {
-		STAT_CACHE_ENTRY_WAITING,  /* waiting for stat thread to do the work, no info available */
-		STAT_CACHE_ENTRY_FINISHED, /* stat() done, info available */
+		STAT_CACHE_ENTRY_SINGLE,      /* single file, this is the default or "normal" */
+		STAT_CACHE_ENTRY_DIR          /* get a directory listing (with stat info) */
+	} type;
+
+	enum {
+		STAT_CACHE_ENTRY_WAITING,     /* waiting for stat thread to do the work, no info available */
+		STAT_CACHE_ENTRY_FINISHED,    /* stat() done, info available */
 	} state;
-	GPtrArray *vrequests;          /* vrequests waiting for this info */
+
+	stat_cache_entry_data data;
+	GArray *dirlist;                  /* array of stat_cache_dirlist_entry, used together with STAT_CACHE_ENTRY_DIR */
+	ev_tstamp ts;                     /* timestamp the entry was created (not when the stat() was done) */
+	GPtrArray *vrequests;             /* vrequests waiting for this info */
 	guint refcount;
-	waitqueue_elem queue_elem;     /* queue element for the delete_queue */
+	waitqueue_elem queue_elem;        /* queue element for the delete_queue */
 	gboolean in_cache;
 };
 
 struct stat_cache {
 	GHashTable *entries;
-	GAsyncQueue *job_queue_out;    /* elements waiting for stat */
-	GAsyncQueue *job_queue_in;     /* elements with finished stat */
+	GAsyncQueue *job_queue_out;       /* elements waiting for stat */
+	GAsyncQueue *job_queue_in;        /* elements with finished stat */
 	waitqueue delete_queue;
 	GThread *thread;
 	ev_async job_watcher;
@@ -73,17 +84,15 @@ struct stat_cache {
 
 void stat_cache_new(worker *wrk, gdouble ttl);
 void stat_cache_free(stat_cache *sc);
-void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents);
-void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents);
-gpointer stat_cache_thread(gpointer data);
-
-void stat_cache_entry_free(stat_cache_entry *sce);
 
 /*
  gets a stat_cache_entry for a specified path
  returns NULL in case of a cache MISS and you should return HANDLER_WAIT_FOR_EVENT
 */
-LI_API stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path);
+LI_API stat_cache_entry *stat_cache_get(vrequest *vr, GString *path);
+
+/* like stat_cache_entry_get but gets stat info for a whole directory */
+LI_API stat_cache_entry *stat_cache_get_dir(vrequest *vr, GString *path);
 
 /* release a stat_cache_entry so it can be cleaned up */
 LI_API void stat_cache_entry_release(vrequest *vr);

include/lighttpd/typedefs.h

@@ -204,6 +204,8 @@ typedef struct filters filters;
 struct worker;
 typedef struct worker worker;
 
+struct stat_cache_entry_data;
+typedef struct stat_cache_entry_data stat_cache_entry_data;
 struct stat_cache_entry;
 typedef struct stat_cache_entry stat_cache_entry;
 struct stat_cache;

src/plugin_core.c

@@ -197,15 +197,15 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont
 		if (!vrequest_handle_direct(vr)) return HANDLER_GO_ON;
 	}
 
-	sce = stat_cache_entry_get(vr, vr->physical.path);
+	sce = stat_cache_get(vr, vr->physical.path);
 	if (!sce)
 		return HANDLER_WAIT_FOR_EVENT;
 
 	VR_DEBUG(vr, "serving static file: %s", vr->physical.path->str);
 
-	if (sce->failed) {
+	if (sce->data.failed) {
 		/* stat failed */
-		VR_DEBUG(vr, "stat() failed: %s (%d)", g_strerror(sce->err), sce->err);
+		VR_DEBUG(vr, "stat() failed: %s (%d)", g_strerror(sce->data.err), sce->data.err);
 
 		switch (errno) {
 		case ENOENT:
@@ -237,7 +237,7 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont
 
 		/* redirect to scheme + host + path + / + querystring if directory without trailing slash */
 		/* TODO: local addr if HTTP 1.0 without host header */
-		if (S_ISDIR(sce->st.st_mode) && vr->request.uri.orig_path->str[vr->request.uri.orig_path->len-1] != '/') {
+		if (S_ISDIR(sce->data.st.st_mode) && vr->request.uri.orig_path->str[vr->request.uri.orig_path->len-1] != '/') {
 			GString *host = vr->request.uri.authority->len ? vr->request.uri.authority : vr->con->local_addr_str;
 			GString *uri = g_string_sized_new(
 				 8 /* https:// */ + host->len +
@@ -259,7 +259,7 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont
 			http_header_overwrite(vr->response.headers, CONST_STR_LEN("Location"), GSTR_LEN(uri));
 			g_string_free(uri, TRUE);
 			close(fd);
-		} else if (!S_ISREG(sce->st.st_mode)) {
+		} else if (!S_ISREG(sce->data.st.st_mode)) {
 			vr->response.http_status = 404;
 			close(fd);
 		} else {
@@ -269,7 +269,7 @@ static handler_t core_handle_static(vrequest *vr, gpointer param, gpointer *cont
 				http_header_overwrite(vr->response.headers, CONST_STR_LEN("Content-Type"), GSTR_LEN(mime_str));
 			else
 				http_header_overwrite(vr->response.headers, CONST_STR_LEN("Content-Type"), CONST_STR_LEN("application/octet-stream"));
-			chunkqueue_append_file_fd(vr->out, NULL, 0, sce->st.st_size, fd);
+			chunkqueue_append_file_fd(vr->out, NULL, 0, sce->data.st.st_size, fd);
 		}
 	}
 
@@ -579,6 +579,19 @@ static gboolean core_io_timeout(server *srv, plugin* p, value *val) {
 	return TRUE;
 }
 
+static gboolean core_stat_cache_ttl(server *srv, plugin* p, value *val) {
+	UNUSED(p);
+
+	if (!val || val->type != VALUE_NUMBER || val->data.number < 1) {
+		ERROR(srv, "%s", "stat_cache.ttl expects a positive number as parameter");
+		return FALSE;
+	}
+
+	srv->stat_cache_ttl = (gdouble)value_extract(val).number;
+
+	return TRUE;
+}
+
 /*
  * OPTIONS
  */
@@ -1081,6 +1094,8 @@ static const plugin_setup setups[] = {
 	{ "workers", core_workers },
 	{ "module_load", core_module_load },
 	{ "io_timeout", core_io_timeout },
+	{ "stat_cache.ttl", core_stat_cache_ttl },
+
 	{ NULL, NULL }
 };
 

src/stat_cache.c

@@ -1,10 +1,20 @@
 #include <lighttpd/base.h>
 
 
+static void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents);
+static void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents);
+static gpointer stat_cache_thread(gpointer data);
+static void stat_cache_entry_free(stat_cache_entry *sce);
+static stat_cache_entry *stat_cache_get_internal(vrequest *vr, GString *path, gboolean dir);
+
 void stat_cache_new(worker *wrk, gdouble ttl) {
 	stat_cache *sc;
 	GError *err;
 
+	/* ttl default 10s */
+	if (ttl < 1)
+		ttl = 10.0;
+
 	sc = g_slice_new0(stat_cache);
 	sc->ttl = ttl;
 	sc->entries = g_hash_table_new_full((GHashFunc)g_string_hash, (GEqualFunc)g_string_equal, NULL, NULL);
@@ -51,7 +61,7 @@ void stat_cache_free(stat_cache *sc) {
 	g_slice_free(stat_cache, sc);
 }
 
-void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) {
+static void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 	stat_cache *sc = (stat_cache*) w->data;
 	stat_cache_entry *sce;
 	waitqueue_elem *wqe;
@@ -68,7 +78,7 @@ void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 		} else {
 			/* no more vrequests using this entry, finally free it */
 			if (sce->in_cache)
-				g_hash_table_remove(sc->entries, sce->path);
+				g_hash_table_remove(sc->entries, sce->data.path);
 			stat_cache_entry_free(sce);
 		}
 	}
@@ -76,7 +86,7 @@ void stat_cache_delete_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 	waitqueue_update(&sc->delete_queue);
 }
 
-void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) {
+static void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) {
 	guint i;
 	stat_cache_entry *sce;
 	stat_cache *sc = ((worker*)w->data)->stat_cache;
@@ -86,7 +96,7 @@ void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) {
 	UNUSED(revents);
 
 	while ((sce = g_async_queue_try_pop(sc->job_queue_in)) != NULL) {
-		if (sce->failed)
+		if (sce->data.failed)
 			sc->errors++;
 
 		for (i = 0; i < sce->vrequests->len; i++) {
@@ -98,16 +108,95 @@ void stat_cache_job_cb(struct ev_loop *loop, ev_async *w, int revents) {
 	}
 }
 
-void stat_cache_entry_free(stat_cache_entry *sce) {
+static void stat_cache_entry_free(stat_cache_entry *sce) {
+	guint i;
+
 	assert(sce->vrequests->len == 0);
 	assert(sce->refcount == 0);
-	g_string_free(sce->path, TRUE);
+
+	g_string_free(sce->data.path, TRUE);
 	g_ptr_array_free(sce->vrequests, TRUE);
+
+	if (sce->type == STAT_CACHE_ENTRY_DIR) {
+		for (i = 0; i < sce->dirlist->len; i++) {
+			g_string_free(g_array_index(sce->dirlist, stat_cache_entry_data, i).path, TRUE);
+		}
+
+		g_array_free(sce->dirlist, TRUE);
+	}
+
 	g_slice_free(stat_cache_entry, sce);
 }
 
 
-stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) {
+static gpointer stat_cache_thread(gpointer data) {
+	stat_cache *sc = data;
+	stat_cache_entry *sce;
+
+	while (TRUE) {
+		sce = g_async_queue_pop(sc->job_queue_out);
+
+		/* stat cache entry with path == NULL indicates server stop */
+		if (!sce->data.path)
+			break;
+
+		if (sce->type == STAT_CACHE_ENTRY_SINGLE) {
+			if (stat(sce->data.path->str, &sce->data.st) == -1) {
+				sce->data.failed = TRUE;
+				sce->data.err = errno;
+			} else {
+				sce->data.failed = FALSE;
+			}
+		} else {
+			/* dirlisting */
+			DIR *dirp;
+			gsize size;
+			struct dirent *entry;
+			struct dirent *result;
+			gint error;
+			stat_cache_entry_data sced;
+
+			dirp = opendir(sce->data.path->str);
+			if (dirp == NULL) {
+				sce->data.failed = TRUE;
+				sce->data.err = errno;
+			} else {
+				size = dirent_buf_size(dirp);
+				assert(size != (gsize)-1);
+				entry = g_slice_alloc(size);
+
+
+				while ((error = readdir_r(dirp, entry, &result)) != 0 && result != NULL) {
+					sced.path = g_string_sized_new(32);
+					g_string_assign(sced.path, result->d_name);
+					if (stat(result->d_name, &sced.st) == -1) {
+						sced.failed = TRUE;
+						sced.err = errno;
+					} else {
+						sced.failed = FALSE;
+					}
+					g_array_append_val(sce->dirlist, sced);
+				}
+
+				if (error) {
+					sce->data.failed = TRUE;
+					sce->data.err = error;
+				}
+
+				g_slice_free1(size, entry);
+				closedir(dirp);
+			}
+		}
+
+		g_atomic_int_set(&sce->state, STAT_CACHE_ENTRY_FINISHED);
+		g_async_queue_push(sc->job_queue_in, sce);
+		ev_async_send(sc->delete_queue.loop, &sc->job_watcher);
+	}
+
+	return NULL;
+}
+
+static stat_cache_entry *stat_cache_get_internal(vrequest *vr, GString *path, gboolean dir) {
 	stat_cache *sc;
 	stat_cache_entry *sce;
 
@@ -127,20 +216,46 @@ stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) {
 					vr->stat_cache_entry = sce;
 					sce->refcount++;
 				}
+				VR_DEBUG(vr, "stat_cache: %"G_GUINT64_FORMAT" hits, %"G_GUINT64_FORMAT" misses, %u items in cache", sc->hits, sc->misses, g_hash_table_size(sc->entries));
 				return sce;
 			} else {
 				/* entry old */
 				if (sce->refcount == 0) {
 					/* no vrequests working on the entry, reuse it */
+					if (sce->type == STAT_CACHE_ENTRY_DIR) {
+						if (!dir) {
+							guint i;
+							for (i = 0; i < sce->dirlist->len; i++) {
+								g_string_free(g_array_index(sce->dirlist, stat_cache_entry_data, i).path, TRUE);
+							}
+
+							g_array_free(sce->dirlist, TRUE);
+							sce->type = STAT_CACHE_ENTRY_SINGLE;
+						} else {
+							g_array_set_size(sce->dirlist, 0);
+						}
+					} else {
+						/* single file */
+						if (dir) {
+							sce->dirlist = g_array_sized_new(FALSE, FALSE, sizeof(stat_cache_entry_data), 32);
+							sce->type = STAT_CACHE_ENTRY_DIR;
+						}
+					}
 				} else {
 					/* there are still vrequests using this entry, replace with a new one */
 					sce->in_cache = FALSE;
 					sce = g_slice_new0(stat_cache_entry);
-					sce->path = g_string_new_len(GSTR_LEN(path));
+					sce->data.path = g_string_new_len(GSTR_LEN(path));
 					sce->vrequests = g_ptr_array_sized_new(8);
 					sce->in_cache = TRUE;
 					sce->queue_elem.data = sce;
-					g_hash_table_replace(sc->entries, sce->path, sce);
+					g_hash_table_replace(sc->entries, sce->data.path, sce);
+					if (dir) {
+						sce->type = STAT_CACHE_ENTRY_DIR;
+						sce->dirlist = g_array_sized_new(FALSE, FALSE, sizeof(stat_cache_entry_data), 32);
+					} else {
+						sce->type = STAT_CACHE_ENTRY_SINGLE;
+					}
 				}
 
 				sce->ts = CUR_TS(vr->con->wrk);
@@ -164,7 +279,7 @@ stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) {
 	} else {
 		/* cache miss, allocate new entry */
 		sce = g_slice_new0(stat_cache_entry);
-		sce->path = g_string_new_len(GSTR_LEN(path));
+		sce->data.path = g_string_new_len(GSTR_LEN(path));
 		sce->vrequests = g_ptr_array_sized_new(8);
 		sce->ts = CUR_TS(vr->con->wrk);
 		sce->state = STAT_CACHE_ENTRY_WAITING;
@@ -174,40 +289,30 @@ stat_cache_entry *stat_cache_entry_get(vrequest *vr, GString *path) {
 		g_ptr_array_add(sce->vrequests, vr);
 		sce->refcount = 1;
 		waitqueue_push(&sc->delete_queue, &sce->queue_elem);
-		g_hash_table_insert(sc->entries, sce->path, sce);
+		g_hash_table_insert(sc->entries, sce->data.path, sce);
 		g_async_queue_push(sc->job_queue_out, sce);
 		sc->misses++;
+
+		if (dir) {
+			sce->type = STAT_CACHE_ENTRY_DIR;
+			sce->dirlist = g_array_sized_new(FALSE, FALSE, sizeof(stat_cache_entry_data), 32);
+		} else {
+			sce->type = STAT_CACHE_ENTRY_SINGLE;
+		}
+
 		return NULL;
 	}
 }
 
-void stat_cache_entry_release(vrequest *vr) {
-	vr->stat_cache_entry->refcount--;
-	vr->stat_cache_entry = NULL;
+stat_cache_entry *stat_cache_get(vrequest *vr, GString *path) {
+	return stat_cache_get_internal(vr, path, FALSE);
 }
 
+stat_cache_entry *stat_cache_get_dir(vrequest *vr, GString *path) {
+	return stat_cache_get_internal(vr, path, TRUE);
+}
 
-gpointer stat_cache_thread(gpointer data) {
-	stat_cache *sc = data;
-	stat_cache_entry *sce;
-
-	while (TRUE) {
-		sce = g_async_queue_pop(sc->job_queue_out);
-
-		/* stat cache entry with path == NULL indicates server stop */
-		if (!sce->path)
-			break;
-
-		if (stat(sce->path->str, &sce->st) == -1) {
-			sce->failed = TRUE;
-			sce->err = errno;
-		} else
-			sce->failed = FALSE;
-
-		g_atomic_int_set(&sce->state, STAT_CACHE_ENTRY_FINISHED);
-		g_async_queue_push(sc->job_queue_in, sce);
-		ev_async_send(sc->delete_queue.loop, &sc->job_watcher);
-	}
-
-	return NULL;
+void stat_cache_entry_release(vrequest *vr) {
+	vr->stat_cache_entry->refcount--;
+	vr->stat_cache_entry = NULL;
 }

src/worker.c

@@ -335,7 +335,7 @@ worker* worker_new(struct server *srv, struct ev_loop *loop) {
 	ev_timer_init(&wrk->job_queue_watcher, worker_job_queue_cb, 0, 0);
 	wrk->job_queue_watcher.data = wrk;
 
-	stat_cache_new(wrk, 10.0);
+	stat_cache_new(wrk, srv->stat_cache_ttl);
 
 	return wrk;
 }