[core] SIGCHLD handle_waitpid hook for modules

centralize most waitpid() handling in core server, with hooks for
modules to be informed of pid and status when a process exits.

This enables faster discovery (and restart) of exited processes,
and also allows for lighttpd to manage backend processes in the
parent (master) process when server.max-worker > 0.

src/base.h

@@ -600,6 +600,7 @@ struct server {
 
 	uid_t uid;
 	gid_t gid;
+	pid_t pid;
 };
 
 

src/fdevent.c

@@ -581,16 +581,8 @@ static pid_t fdevent_open_logger_pipe_spawn(const char *logger, int rfd) {
 }
 
 
-static void fdevent_waitpid_logger_pipe(fdevent_cmd_pipe *fcp, time_t ts) {
-    pid_t pid = fcp->pid;
-    if (pid > 0) {
-        switch (waitpid(pid, NULL, WNOHANG)) {
-          case 0:  return;
-          case -1: if (errno == EINTR) return; /* fall through */
-          default: break;
-        }
-        fcp->pid = -1;
-    }
+static void fdevent_restart_logger_pipe(fdevent_cmd_pipe *fcp, time_t ts) {
+    if (fcp->pid > 0) return;  /* assert */
     if (fcp->start + 5 < ts) { /* limit restart to once every 5 sec */
         /* restart child process using existing pipe fds */
         fcp->start = ts;
@@ -599,9 +591,31 @@ static void fdevent_waitpid_logger_pipe(fdevent_cmd_pipe *fcp, time_t ts) {
 }
 
 
-void fdevent_waitpid_logger_pipes(time_t ts) {
+void fdevent_restart_logger_pipes(time_t ts) {
+    for (size_t i = 0; i < cmd_pipes.used; ++i) {
+        fdevent_cmd_pipe * const fcp = cmd_pipes.ptr+i;
+        if (fcp->pid > 0) continue;
+        fdevent_restart_logger_pipe(fcp, ts);
+    }
+}
+
+
+int fdevent_waitpid_logger_pipe_pid(pid_t pid, time_t ts) {
     for (size_t i = 0; i < cmd_pipes.used; ++i) {
-        fdevent_waitpid_logger_pipe(cmd_pipes.ptr+i, ts);
+        fdevent_cmd_pipe * const fcp = cmd_pipes.ptr+i;
+        if (pid != fcp->pid) continue;
+        fcp->pid = -1;
+        fdevent_restart_logger_pipe(fcp, ts);
+        return 1;
+    }
+    return 0;
+}
+
+
+void fdevent_clr_logger_pipe_pids(void) {
+    for (size_t i = 0; i < cmd_pipes.used; ++i) {
+        fdevent_cmd_pipe *fcp = cmd_pipes.ptr+i;
+        fcp->pid = -1;
     }
 }
 

src/fdevent.h

@@ -223,9 +223,11 @@ pid_t fdevent_fork_execve(const char *name, char *argv[], char *envp[], int fdin
 int fdevent_open_logger(const char *logger);
 int fdevent_cycle_logger(const char *logger, int *curfd);
 int fdevent_reaped_logger_pipe(pid_t pid);
-void fdevent_waitpid_logger_pipes(time_t ts);
+int fdevent_waitpid_logger_pipe_pid(pid_t pid, time_t ts);
+void fdevent_restart_logger_pipes(time_t ts);
 void fdevent_close_logger_pipes(void);
 void fdevent_breakagelog_logger_pipe(int fd);
+void fdevent_clr_logger_pipe_pids(void);
 
 int fdevent_select_init(fdevents *ev);
 int fdevent_poll_init(fdevents *ev);

src/gw_backend.c

@@ -284,7 +284,12 @@ static void gw_proc_connect_error(server *srv, gw_host *host, gw_proc *proc, pid
              * - ECONNREFUSED for tcp-ip sockets
              * - ENOENT for unix-domain-sockets
              */
+          #if 0
             gw_proc_set_state(host, proc, PROC_STATE_DIED_WAIT_FOR_PID);
+          #else  /* treat as overloaded (future: unless we send kill() signal)*/
+            proc->disabled_until = srv->cur_ts + host->disable_time;
+            gw_proc_set_state(host, proc, PROC_STATE_OVERLOADED);
+          #endif
         }
     }
 
@@ -319,6 +324,25 @@ static void gw_proc_check_enable(server *srv, gw_host *host, gw_proc *proc) {
                     host->host, host->port, host->unixsocket);
 }
 
+static void gw_proc_waitpid_log(server *srv, gw_host *host, gw_proc *proc, int status) {
+    UNUSED(host);
+    if (WIFEXITED(status)) {
+        if (proc->state != PROC_STATE_KILLED) {
+            log_error_write(srv, __FILE__, __LINE__, "sdb",
+                            "child exited:",
+                            WEXITSTATUS(status), proc->connection_name);
+        }
+    } else if (WIFSIGNALED(status)) {
+        if (WTERMSIG(status) != SIGTERM && WTERMSIG(status) != SIGINT) {
+            log_error_write(srv, __FILE__, __LINE__, "sd",
+                            "child signalled:", WTERMSIG(status));
+        }
+    } else {
+        log_error_write(srv, __FILE__, __LINE__, "sd",
+                        "child died somehow:", status);
+    }
+}
+
 static int gw_proc_waitpid(server *srv, gw_host *host, gw_proc *proc) {
     int rc, status;
 
@@ -337,23 +361,14 @@ static int gw_proc_waitpid(server *srv, gw_host *host, gw_proc *proc) {
         log_error_write(srv, __FILE__, __LINE__, "sddss",
                         "pid ", proc->pid, proc->state,
                         "not found:", strerror(errno));
-    } else if (WIFEXITED(status)) {
-        if (proc->state != PROC_STATE_KILLED) {
-            log_error_write(srv, __FILE__, __LINE__, "sdb",
-                            "child exited:",
-                            WEXITSTATUS(status), proc->connection_name);
-        }
-    } else if (WIFSIGNALED(status)) {
-        if (WTERMSIG(status) != SIGTERM && WTERMSIG(status) != SIGINT) {
-            log_error_write(srv, __FILE__, __LINE__, "sd",
-                            "child signalled:", WTERMSIG(status));
-        }
-    } else {
-        log_error_write(srv, __FILE__, __LINE__, "sd",
-                        "child died somehow:", status);
+    }
+    else {
+        gw_proc_waitpid_log(srv, host, proc, status);
     }
 
     proc->pid = 0;
+    if (proc->state != PROC_STATE_KILLED)
+        proc->disabled_until = srv->cur_ts;
     gw_proc_set_state(host, proc, PROC_STATE_DIED);
     return 1;
 }
@@ -580,6 +595,8 @@ static int gw_spawn_connection(server *srv, gw_host *host, gw_proc *proc, int de
         if (-1 == proc->pid) {
             log_error_write(srv, __FILE__, __LINE__, "sb",
                             "gw-backend failed to start:", host->bin_path);
+            proc->pid = 0;
+            proc->disabled_until = srv->cur_ts;
             return -1;
         }
 
@@ -617,7 +634,13 @@ static int gw_spawn_connection(server *srv, gw_host *host, gw_proc *proc, int de
 
 static void gw_proc_spawn(server *srv, gw_host *host, int debug) {
     gw_proc *proc;
-    for (proc=host->unused_procs; proc && proc->pid != 0; proc=proc->next);
+    for (proc = host->unused_procs; proc; proc = proc->next) {
+        /* (proc->pid <= 0 indicates PROC_STATE_DIED, not PROC_STATE_KILLED) */
+        if (proc->pid > 0) continue;
+        /* (do not attempt to spawn another proc if a proc just exited) */
+        if (proc->disabled_until >= srv->cur_ts) return;
+        break;
+    }
     if (proc) {
         if (proc == host->unused_procs)
             host->unused_procs = proc->next;
@@ -677,6 +700,7 @@ static void gw_proc_kill(server *srv, gw_host *host, gw_proc *proc) {
 
     proc->prev = NULL;
     proc->next = host->unused_procs;
+    proc->disabled_until = 0;
 
     if (host->unused_procs)
         host->unused_procs->prev = proc;
@@ -975,20 +999,20 @@ static void gw_restart_dead_procs(server *srv, gw_host *host, int debug) {
                             proc->is_local, proc->load, proc->pid);
         }
 
-        /*
-         * if the remote side is overloaded, we check back after <n> seconds
-         *
-         */
         switch (proc->state) {
-        case PROC_STATE_KILLED:
-            /* should never happen as long as adaptive spawing is disabled */
-            force_assert(0);
-
-            break;
         case PROC_STATE_RUNNING:
             break;
         case PROC_STATE_OVERLOADED:
+            gw_proc_check_enable(srv, host, proc);
+            break;
+        case PROC_STATE_KILLED:
+            break;
         case PROC_STATE_DIED_WAIT_FOR_PID:
+            /*(state should not happen in workers if server.max-worker > 0)*/
+            /*(if PROC_STATE_DIED_WAIT_FOR_PID is used in future, might want
+             * to save proc->disabled_until before gw_proc_waitpid() since
+             * gw_proc_waitpid will set proc->disabled_until to srv->cur_ts,
+             * and so process will not be restarted below until one sec later)*/
             if (0 == gw_proc_waitpid(srv, host, proc)) {
                 gw_proc_check_enable(srv, host, proc);
             }
@@ -1005,6 +1029,9 @@ static void gw_restart_dead_procs(server *srv, gw_host *host, int debug) {
                  * let them terminate first */
                 if (proc->load != 0) break;
 
+                /* avoid spinning if child exits too quickly */
+                if (proc->disabled_until >= srv->cur_ts) break;
+
                 /* restart the child */
 
                 if (debug) {
@@ -1186,6 +1213,8 @@ int gw_set_defaults_backend(server *srv, gw_plugin_data *p, data_unset *du, size
         return 0;
     }
 
+    p->srv_pid = srv->pid;
+
     gw_mode = buffer_init();
 
     s->exts      = gw_extensions_init();
@@ -1852,7 +1881,10 @@ static handler_t gw_write_error(server *srv, gw_handler_ctx *hctx) {
     if (hctx->state == GW_STATE_INIT ||
         hctx->state == GW_STATE_CONNECT_DELAYED) {
 
-        gw_restart_dead_procs(srv, hctx->host, hctx->conf.debug);
+        /* (optimization to detect backend process exit while processing a
+         *  large number of ready events; (this block could be removed)) */
+        if (0 == srv->srvconf.max_worker)
+            gw_restart_dead_procs(srv, hctx->host, hctx->conf.debug);
 
         /* cleanup this request and let request handler start request again */
         if (hctx->reconnects++ < 5) return gw_reconnect(srv, hctx);
@@ -2027,9 +2059,13 @@ static handler_t gw_recv_response(server *srv, gw_handler_ctx *hctx) {
     case HANDLER_COMEBACK: /*(not expected; treat as error)*/
     case HANDLER_ERROR:
         if (b != hctx->response) buffer_free(b);
+        /* (optimization to detect backend process exit while processing a
+         *  large number of ready events; (this block could be removed)) */
         if (proc->is_local && 1 == proc->load && proc->pid == hctx->pid
-            && proc->state != PROC_STATE_DIED) {
-            if (0 != gw_proc_waitpid(srv, host, proc)) {
+            && proc->state != PROC_STATE_DIED && 0 == srv->srvconf.max_worker) {
+            /* intentionally check proc->disabed_until before gw_proc_waitpid */
+            if (proc->disabled_until < srv->cur_ts
+                && 0 != gw_proc_waitpid(srv, host, proc)) {
                 if (hctx->conf.debug) {
                     log_error_write(srv, __FILE__, __LINE__, "ssbsdsd",
                                     "--- gw spawning",
@@ -2430,7 +2466,7 @@ static void gw_handle_trigger_host(server *srv, gw_host *host, int debug) {
     }
 }
 
-void gw_handle_trigger_exts(server *srv, gw_exts *exts, int debug) {
+static void gw_handle_trigger_exts(server *srv, gw_exts *exts, int debug) {
     for (size_t j = 0; j < exts->used; ++j) {
         gw_extension *ex = exts->exts[j];
         for (size_t n = 0; n < ex->used; ++n) {
@@ -2441,6 +2477,9 @@ void gw_handle_trigger_exts(server *srv, gw_exts *exts, int debug) {
 
 handler_t gw_handle_trigger(server *srv, void *p_d) {
     gw_plugin_data *p = p_d;
+    if (0 != srv->srvconf.max_worker && p->srv_pid != srv->pid)
+        return HANDLER_GO_ON;
+
     for (size_t i = 0; i < srv->config_context->used; i++) {
         gw_plugin_config *conf = p->config_storage[i];
         gw_exts *exts = conf->exts;
@@ -2451,3 +2490,54 @@ handler_t gw_handle_trigger(server *srv, void *p_d) {
 
     return HANDLER_GO_ON;
 }
+
+handler_t gw_handle_waitpid_cb(server *srv, void *p_d, pid_t pid, int status) {
+    gw_plugin_data *p = p_d;
+    if (0 != srv->srvconf.max_worker && p->srv_pid != srv->pid)
+        return HANDLER_GO_ON;
+
+    for (size_t i = 0; i < srv->config_context->used; ++i) {
+        gw_plugin_config *conf = p->config_storage[i];
+        gw_exts *exts = conf->exts;
+        int debug = conf->debug ? conf->debug : p->config_storage[0]->debug;
+        if (NULL == exts) continue;
+        for (size_t j = 0; j < exts->used; ++j) {
+            gw_extension *ex = exts->exts[j];
+            for (size_t n = 0; n < ex->used; ++n) {
+                gw_host *host = ex->hosts[n];
+                gw_proc *proc;
+                for (proc = host->first; proc; proc = proc->next) {
+                    if (!proc->is_local || proc->pid != pid) continue;
+
+                    gw_proc_waitpid_log(srv, host, proc, status);
+                    gw_proc_set_state(host, proc, PROC_STATE_DIED);
+                    proc->pid = 0;
+
+                    /* restart, but avoid spinning if child exits too quickly */
+                    if (proc->disabled_until < srv->cur_ts) {
+                        if (proc->state != PROC_STATE_KILLED)
+                            proc->disabled_until = srv->cur_ts;
+                        if (gw_spawn_connection(srv, host, proc, debug)) {
+                            log_error_write(srv, __FILE__, __LINE__, "s",
+                                            "ERROR: spawning gw failed.");
+                        }
+                    }
+
+                    return HANDLER_FINISHED;
+                }
+                for (proc = host->unused_procs; proc; proc = proc->next) {
+                    if (!proc->is_local || proc->pid != pid) continue;
+
+                    gw_proc_waitpid_log(srv, host, proc, status);
+                    if (proc->state != PROC_STATE_KILLED)
+                        proc->disabled_until = srv->cur_ts;
+                    gw_proc_set_state(host, proc, PROC_STATE_DIED);
+                    proc->pid = 0;
+                    return HANDLER_FINISHED;
+                }
+            }
+        }
+    }
+
+    return HANDLER_GO_ON;
+}

src/gw_backend.h

@@ -277,6 +277,7 @@ typedef struct gw_plugin_data {
     gw_plugin_config **config_storage;
 
     gw_plugin_config conf; /* used only as long as no gw_handler_ctx is setup */
+    pid_t srv_pid;
 } gw_plugin_data;
 
 /* connection specific data */
@@ -337,8 +338,8 @@ int gw_set_defaults_balance(server *srv, gw_plugin_config *s, data_unset *du);
 handler_t gw_check_extension(server *srv, connection *con, gw_plugin_data *p, int uri_path_handler, size_t hctx_sz);
 handler_t gw_connection_reset(server *srv, connection *con, void *p_d);
 handler_t gw_handle_subrequest(server *srv, connection *con, void *p_d);
-void gw_handle_trigger_exts(server *srv, gw_exts *exts, int debug);
 handler_t gw_handle_trigger(server *srv, void *p_d);
+handler_t gw_handle_waitpid_cb(server *srv, void *p_d, pid_t pid, int status);
 
 void gw_set_transparent(server *srv, gw_handler_ctx *hctx);
 

src/mod_cgi.c

@@ -46,7 +46,7 @@ typedef struct {
 } char_array;
 
 typedef struct {
-	pid_t *ptr;
+	struct { pid_t pid; void *ctx; } *ptr;
 	size_t used;
 	size_t size;
 } buffer_pid_t;
@@ -218,17 +218,9 @@ SETDEFAULTS_FUNC(mod_fastcgi_set_defaults) {
 }
 
 
-static int cgi_pid_add(server *srv, plugin_data *p, pid_t pid) {
-	int m = -1;
-	size_t i;
+static void cgi_pid_add(plugin_data *p, pid_t pid, void *ctx) {
 	buffer_pid_t *r = &(p->cgi_pid);
 
-	UNUSED(srv);
-
-	for (i = 0; i < r->used; i++) {
-		if (r->ptr[i] > m) m = r->ptr[i];
-	}
-
 	if (r->size == 0) {
 		r->size = 16;
 		r->ptr = malloc(sizeof(*r->ptr) * r->size);
@@ -239,31 +231,29 @@ static int cgi_pid_add(server *srv, plugin_data *p, pid_t pid) {
 		force_assert(r->ptr);
 	}
 
-	r->ptr[r->used++] = pid;
+	r->ptr[r->used].pid = pid;
+	r->ptr[r->used].ctx = ctx;
+	++r->used;
+}
 
-	return m;
+static void cgi_pid_kill(plugin_data *p, pid_t pid) {
+    buffer_pid_t *r = &(p->cgi_pid);
+    for (size_t i = 0; i < r->used; ++i) {
+        if (r->ptr[i].pid == pid) {
+            r->ptr[i].ctx = NULL;
+            kill(pid, SIGTERM);
+            return;
+        }
+    }
 }
 
-static int cgi_pid_del(server *srv, plugin_data *p, pid_t pid) {
-	size_t i;
+static void cgi_pid_del(plugin_data *p, size_t i) {
 	buffer_pid_t *r = &(p->cgi_pid);
 
-	UNUSED(srv);
-
-	for (i = 0; i < r->used; i++) {
-		if (r->ptr[i] == pid) break;
-	}
-
-	if (i != r->used) {
-		/* found */
-
 		if (i != r->used - 1) {
 			r->ptr[i] = r->ptr[r->used - 1];
 		}
 		r->used--;
-	}
-
-	return 0;
 }
 
 
@@ -276,8 +266,6 @@ static void cgi_connection_close_fdtocgi(server *srv, handler_ctx *hctx) {
 }
 
 static void cgi_connection_close(server *srv, handler_ctx *hctx) {
-	int status;
-	pid_t pid;
 	plugin_data *p = hctx->plugin_data;
 	connection *con = hctx->remote_conn;
 
@@ -298,63 +286,14 @@ static void cgi_connection_close(server *srv, handler_ctx *hctx) {
 		cgi_connection_close_fdtocgi(srv, hctx); /*(closes only hctx->fdtocgi)*/
 	}
 
-	pid = hctx->pid;
+	if (hctx->pid > 0) {
+		cgi_pid_kill(p, hctx->pid);
+	}
 
 	con->plugin_ctx[p->id] = NULL;
 
 	cgi_handler_ctx_free(hctx);
 
-	/* if waitpid hasn't been called by response.c yet, do it here */
-	if (pid > 0) {
-		/* check if the CGI-script is already gone */
-		switch(waitpid(pid, &status, WNOHANG)) {
-		case 0:
-			/* not finished yet */
-#if 0
-			log_error_write(srv, __FILE__, __LINE__, "sd", "(debug) child isn't done yet, pid:", pid);
-#endif
-			break;
-		case -1:
-			/* */
-			if (errno == EINTR) break;
-
-			/*
-			 * errno == ECHILD happens if _subrequest catches the process-status before
-			 * we have read the response of the cgi process
-			 *
-			 * -> catch status
-			 * -> WAIT_FOR_EVENT
-			 * -> read response
-			 * -> we get here with waitpid == ECHILD
-			 *
-			 */
-			if (errno != ECHILD) {
-				log_error_write(srv, __FILE__, __LINE__, "ss", "waitpid failed: ", strerror(errno));
-			}
-			/* anyway: don't wait for it anymore */
-			pid = 0;
-			break;
-		default:
-			if (WIFEXITED(status)) {
-#if 0
-				log_error_write(srv, __FILE__, __LINE__, "sd", "(debug) cgi exited fine, pid:", pid);
-#endif
-			} else {
-				log_error_write(srv, __FILE__, __LINE__, "sd", "cgi died, pid:", pid);
-			}
-
-			pid = 0;
-			break;
-		}
-
-		if (pid) {
-			kill(pid, SIGTERM);
-
-			/* cgi-script is still alive, queue the PID for removal */
-			cgi_pid_add(srv, p, pid);
-		}
-	}
-
 	/* finish response (if not already con->file_started, con->file_finished) */
 	if (con->mode == p->id) {
 		http_response_backend_done(srv, con);
@@ -498,18 +437,11 @@ static handler_t cgi_handle_fdevent(server *srv, void *ctx, int revents) {
 				return HANDLER_ERROR;
 			}
 			if (0 == con->http_status) con->http_status = 200; /* OK */
-		} else {
-# if 0
-			log_error_write(srv, __FILE__, __LINE__, "sddd", "got HUP from cgi", con->fd, hctx->fd, revents);
-# endif
 		}
 		cgi_connection_close(srv, hctx);
 	} else if (revents & FDEVENT_ERR) {
 		/* kill all connections to the cgi process */
 		cgi_connection_close(srv, hctx);
-#if 1
-		log_error_write(srv, __FILE__, __LINE__, "s", "cgi-FDEVENT_ERR");
-#endif
 		return HANDLER_ERROR;
 	}
 
@@ -858,13 +790,13 @@ static int cgi_create_env(server *srv, connection *con, plugin_data *p, handler_
 		close(from_cgi_fds[1]);
 		close(to_cgi_fds[0]);
 
-		/* register PID and wait for them asynchronously */
-
 		hctx->fd = from_cgi_fds[0];
 		hctx->fde_ndx = -1;
 
 		++srv->cur_fds;
 
+		cgi_pid_add(p, hctx->pid, hctx);
+
 		if (0 == con->request.content_length) {
 			close(to_cgi_fds[1]);
 		} else {
@@ -1010,62 +942,6 @@ URIHANDLER_FUNC(cgi_is_handled) {
 	return HANDLER_GO_ON;
 }
 
-TRIGGER_FUNC(cgi_trigger) {
-	plugin_data *p = p_d;
-	size_t ndx;
-	/* the trigger handle only cares about lonely PID which we have to wait for */
-
-	for (ndx = 0; ndx < p->cgi_pid.used; ndx++) {
-		int status;
-
-		switch(waitpid(p->cgi_pid.ptr[ndx], &status, WNOHANG)) {
-		case 0:
-			/* not finished yet */
-#if 0
-			log_error_write(srv, __FILE__, __LINE__, "sd", "(debug) child isn't done yet, pid:", p->cgi_pid.ptr[ndx]);
-#endif
-			break;
-		case -1:
-			if (errno == ECHILD) {
-				/* someone else called waitpid... remove the pid to stop looping the error each time */
-				log_error_write(srv, __FILE__, __LINE__, "s", "cgi child vanished, probably someone else called waitpid");
-
-				cgi_pid_del(srv, p, p->cgi_pid.ptr[ndx]);
-				ndx--;
-				continue;
-			}
-
-			log_error_write(srv, __FILE__, __LINE__, "ss", "waitpid failed: ", strerror(errno));
-
-			return HANDLER_ERROR;
-		default:
-
-			if (WIFEXITED(status)) {
-#if 0
-				log_error_write(srv, __FILE__, __LINE__, "sd", "(debug) cgi exited fine, pid:", p->cgi_pid.ptr[ndx]);
-#endif
-			} else if (WIFSIGNALED(status)) {
-				/* FIXME: what if we killed the CGI script with a kill(..., SIGTERM) ?
-				 */
-				if (WTERMSIG(status) != SIGTERM) {
-					log_error_write(srv, __FILE__, __LINE__, "sd", "cleaning up CGI: process died with signal", WTERMSIG(status));
-				}
-			} else {
-				log_error_write(srv, __FILE__, __LINE__, "s", "cleaning up CGI: ended unexpectedly");
-			}
-
-			cgi_pid_del(srv, p, p->cgi_pid.ptr[ndx]);
-			/* del modified the buffer structure
-			 * and copies the last entry to the current one
-			 * -> recheck the current index
-			 */
-			ndx--;
-		}
-	}
-
-	return HANDLER_GO_ON;
-}
-
 /*
  * - HANDLER_GO_ON : not our job
  * - HANDLER_FINISHED: got response
@@ -1124,9 +1000,6 @@ SUBREQUEST_FUNC(mod_cgi_handle_subrequest) {
 
 			return HANDLER_FINISHED;
 		}
-#if 0
-	log_error_write(srv, __FILE__, __LINE__, "sdd", "subrequest, pid =", hctx, hctx->pid);
-#endif
 	} else if (!chunkqueue_is_empty(con->request_content_queue)) {
 		if (0 != cgi_write_request(srv, hctx, hctx->fdtocgi)) {
 			cgi_connection_close(srv, hctx);
@@ -1139,6 +1012,42 @@ SUBREQUEST_FUNC(mod_cgi_handle_subrequest) {
 }
 
 
+static handler_t cgi_waitpid_cb(server *srv, void *p_d, pid_t pid, int status) {
+    plugin_data *p = (plugin_data *)p_d;
+    for (size_t i = 0; i < p->cgi_pid.used; ++i) {
+        handler_ctx *hctx;
+        if (pid != p->cgi_pid.ptr[i].pid) continue;
+
+        hctx = (handler_ctx *)p->cgi_pid.ptr[i].ctx;
+        cgi_pid_del(p, i);
+
+        if (WIFEXITED(status)) {
+            /* (skip logging (non-zero) CGI exit; might be very noisy) */
+        }
+        else if (WIFSIGNALED(status)) {
+            /* ignore SIGTERM if sent by cgi_connection_close() (NULL == hctx)*/
+            if (WTERMSIG(status) != SIGTERM || NULL == hctx) {
+                log_error_write(srv, __FILE__, __LINE__, "sdsd", "CGI pid", pid,
+                                "died with signal", WTERMSIG(status));
+            }
+        }
+        else {
+            log_error_write(srv, __FILE__, __LINE__, "sds",
+                            "CGI pid", pid, "ended unexpectedly");
+        }
+
+        if (hctx) {
+            hctx->pid = -1;
+            cgi_handle_fdevent(srv, hctx, FDEVENT_HUP);
+        }
+
+        return HANDLER_FINISHED;
+    }
+
+    return HANDLER_GO_ON;
+}
+
+
 int mod_cgi_plugin_init(plugin *p);
 int mod_cgi_plugin_init(plugin *p) {
 	p->version     = LIGHTTPD_VERSION_ID;
@@ -1147,7 +1056,7 @@ int mod_cgi_plugin_init(plugin *p) {
 	p->connection_reset = cgi_connection_close_callback;
 	p->handle_subrequest_start = cgi_is_handled;
 	p->handle_subrequest = mod_cgi_handle_subrequest;
-	p->handle_trigger = cgi_trigger;
+	p->handle_waitpid = cgi_waitpid_cb;
 	p->init           = mod_cgi_init;
 	p->cleanup        = mod_cgi_free;
 	p->set_defaults   = mod_fastcgi_set_defaults;

src/mod_fastcgi.c

@@ -538,6 +538,7 @@ int mod_fastcgi_plugin_init(plugin *p) {
 	p->handle_subrequest_start = fcgi_check_extension_2;
 	p->handle_subrequest       = gw_handle_subrequest;
 	p->handle_trigger          = gw_handle_trigger;
+	p->handle_waitpid          = gw_handle_waitpid_cb;
 
 	p->data         = NULL;
 

src/mod_proxy.c

@@ -1011,6 +1011,7 @@ int mod_proxy_plugin_init(plugin *p) {
 	p->handle_uri_clean        = mod_proxy_check_extension;
 	p->handle_subrequest       = gw_handle_subrequest;
 	p->handle_trigger          = gw_handle_trigger;
+	p->handle_waitpid          = gw_handle_waitpid_cb;
 
 	p->data         = NULL;
 

src/mod_rrdtool.c

@@ -32,8 +32,10 @@ typedef struct {
 
 	int read_fd, write_fd;
 	pid_t rrdtool_pid;
+	pid_t srv_pid;
 
 	int rrdtool_running;
+	time_t rrdtool_startup_ts;
 
 	plugin_config **config_storage;
 	plugin_config conf;
@@ -73,9 +75,10 @@ FREE_FUNC(mod_rrd_free) {
 
 	free(p->config_storage);
 
-	if (p->rrdtool_pid > 0) {
-		close(p->read_fd);
-		close(p->write_fd);
+	if (p->read_fd >= 0) close(p->read_fd);
+	if (p->write_fd >= 0) close(p->write_fd);
+
+	if (p->rrdtool_pid > 0 && p->srv_pid == srv->pid) {
 		/* collect status */
 		while (-1 == waitpid(p->rrdtool_pid, NULL, 0) && errno == EINTR) ;
 	}
@@ -89,6 +92,13 @@ static int mod_rrd_create_pipe(server *srv, plugin_data *p) {
 	char *args[3];
 	int to_rrdtool_fds[2];
 	int from_rrdtool_fds[2];
+	/* mod_rrdtool does not work with server.max-workers > 0
+	 * since the data between workers is not aggregated,
+	 * and it is not valid to send data to rrdtool more than once a sec
+	 * (which would happen with multiple workers writing to same pipe)
+	 * If pipes were to be shared, then existing pipes would need to be
+	 * reused here, if they already exist (not -1), and after flushing
+	 * existing contents (read and discard from read-end of pipes). */
 	if (pipe(to_rrdtool_fds)) {
 		log_error_write(srv, __FILE__, __LINE__, "ss",
 				"pipe failed: ", strerror(errno));
@@ -110,8 +120,11 @@ static int mod_rrd_create_pipe(server *srv, plugin_data *p) {
 	if (-1 != p->rrdtool_pid) {
 		close(from_rrdtool_fds[1]);
 		close(to_rrdtool_fds[0]);
+		if (p->read_fd >= 0) close(p->read_fd);
+		if (p->write_fd >= 0) close(p->write_fd);
 		p->write_fd = to_rrdtool_fds[1];
 		p->read_fd = from_rrdtool_fds[0];
+		p->srv_pid = srv->pid;
 		return 0;
 	} else {
 		log_error_write(srv, __FILE__, __LINE__, "SBss", "fork/exec(", p->conf.path_rrdtool_bin, "):", strerror(errno));
@@ -266,6 +279,16 @@ static int mod_rrd_patch_connection(server *srv, connection *con, plugin_data *p
 }
 #undef PATCH
 
+static int mod_rrd_exec(server *srv, plugin_data *p) {
+    if (mod_rrd_create_pipe(srv, p)) {
+        return -1;
+    }
+
+    p->rrdtool_running = 1;
+    p->rrdtool_startup_ts = srv->cur_ts;
+    return 0;
+}
+
 SETDEFAULTS_FUNC(mod_rrd_set_defaults) {
 	plugin_data *p = p_d;
 	size_t i;
@@ -316,6 +339,8 @@ SETDEFAULTS_FUNC(mod_rrd_set_defaults) {
 
 	p->conf.path_rrdtool_bin = p->config_storage[0]->path_rrdtool_bin;
 	p->rrdtool_running = 0;
+	p->read_fd  = -1;
+	p->write_fd = -1;
 
 	if (!activate) return HANDLER_GO_ON;
 
@@ -327,21 +352,30 @@ SETDEFAULTS_FUNC(mod_rrd_set_defaults) {
 		return HANDLER_ERROR;
 	}
 
-	/* open the pipe to rrdtool */
-	if (mod_rrd_create_pipe(srv, p)) {
-		return HANDLER_ERROR;
-	}
-
-	p->rrdtool_running = 1;
+	return 0 == mod_rrd_exec(srv, p) ? HANDLER_GO_ON : HANDLER_ERROR;
+}
 
-	return HANDLER_GO_ON;
+static void mod_rrd_fatal_error(server *srv, plugin_data *p) {
+    /* future: might send kill() signal to p->rrdtool_pid to trigger restart */
+    p->rrdtool_running = 0;
+    UNUSED(srv);
 }
 
 TRIGGER_FUNC(mod_rrd_trigger) {
 	plugin_data *p = p_d;
 	size_t i;
 
-	if (!p->rrdtool_running) return HANDLER_GO_ON;
+	if (!p->rrdtool_running) {
+		/* limit restart to once every 5 sec */
+		/*(0 == p->rrdtool_pid if never activated; not used)*/
+		if (-1 == p->rrdtool_pid
+		    && p->srv_pid == srv->pid
+		    && p->rrdtool_startup_ts + 5 < srv->cur_ts) {
+			mod_rrd_exec(srv, p);
+		}
+		return HANDLER_GO_ON;
+	}
+
 	if ((srv->cur_ts % 60) != 0) return HANDLER_GO_ON;
 
 	for (i = 0; i < srv->config_context->used; i++) {
@@ -364,20 +398,18 @@ TRIGGER_FUNC(mod_rrd_trigger) {
 		buffer_append_string_len(p->cmd, CONST_STR_LEN("\n"));
 
 		if (-1 == safe_write(p->write_fd, CONST_BUF_LEN(p->cmd))) {
-			p->rrdtool_running = 0;
-
 			log_error_write(srv, __FILE__, __LINE__, "ss",
 					"rrdtool-write: failed", strerror(errno));
 
+			mod_rrd_fatal_error(srv, p);
 			return HANDLER_ERROR;
 		}
 
 		if (-1 == safe_read(p->read_fd, p->resp)) {
-			p->rrdtool_running = 0;
-
 			log_error_write(srv, __FILE__, __LINE__, "ss",
 					"rrdtool-read: failed", strerror(errno));
 
+			mod_rrd_fatal_error(srv, p);
 			return HANDLER_ERROR;
 		}
 
@@ -385,11 +417,10 @@ TRIGGER_FUNC(mod_rrd_trigger) {
 		    p->resp->ptr[1] != 'K') {
 			/* don't fail on this error if we just started (graceful restart, the old one might have just updated too) */
 			if (!(strstr(p->resp->ptr, "(minimum one second step)") && (srv->cur_ts - srv->startup_ts < 3))) {
-				p->rrdtool_running = 0;
-
 				log_error_write(srv, __FILE__, __LINE__, "sbb",
 					"rrdtool-response:", p->cmd, p->resp);
 
+				mod_rrd_fatal_error(srv, p);
 				return HANDLER_ERROR;
 			}
 		}
@@ -401,11 +432,28 @@ TRIGGER_FUNC(mod_rrd_trigger) {
 	return HANDLER_GO_ON;
 }
 
+static handler_t mod_rrd_waitpid_cb(server *srv, void *p_d, pid_t pid, int status) {
+	plugin_data *p = p_d;
+	if (pid != p->rrdtool_pid) return HANDLER_GO_ON;
+	if (srv->pid != p->srv_pid) return HANDLER_GO_ON;
+
+	p->rrdtool_running = 0;
+	p->rrdtool_pid = -1;
+
+	/* limit restart to once every 5 sec */
+	if (p->rrdtool_startup_ts + 5 < srv->cur_ts)
+		mod_rrd_exec(srv, p);
+
+	UNUSED(status);
+	return HANDLER_FINISHED;
+}
+
 REQUESTDONE_FUNC(mod_rrd_account) {
 	plugin_data *p = p_d;
 
+	/*(0 == p->rrdtool_pid if never activated; not used)*/
+	if (0 == p->rrdtool_pid) return HANDLER_GO_ON;
 	mod_rrd_patch_connection(srv, con, p);
-	if (!p->rrdtool_running) return HANDLER_GO_ON;
 
 	*(p->conf.requests_ptr)      += 1;
 	*(p->conf.bytes_written_ptr) += con->bytes_written;
@@ -424,6 +472,7 @@ int mod_rrdtool_plugin_init(plugin *p) {
 	p->set_defaults= mod_rrd_set_defaults;
 
 	p->handle_trigger      = mod_rrd_trigger;
+	p->handle_waitpid      = mod_rrd_waitpid_cb;
 	p->handle_request_done = mod_rrd_account;
 
 	p->data        = NULL;

src/mod_scgi.c

@@ -297,6 +297,7 @@ int mod_scgi_plugin_init(plugin *p) {
 	p->handle_subrequest_start = scgi_check_extension_2;
 	p->handle_subrequest       = gw_handle_subrequest;
 	p->handle_trigger          = gw_handle_trigger;
+	p->handle_waitpid          = gw_handle_waitpid_cb;
 
 	p->data         = NULL;
 

src/mod_wstunnel.c

@@ -630,6 +630,7 @@ int mod_wstunnel_plugin_init(plugin *p) {
     p->handle_uri_clean  = mod_wstunnel_check_extension;
     p->handle_subrequest = gw_handle_subrequest;
     p->handle_trigger    = mod_wstunnel_handle_trigger;
+    p->handle_waitpid    = gw_handle_waitpid_cb;
     p->data              = NULL;
     return 0;
 }

src/plugin.c

@@ -39,6 +39,7 @@ typedef enum {
 	PLUGIN_FUNC_HANDLE_CONNECTION_CLOSE,
 	PLUGIN_FUNC_HANDLE_TRIGGER,
 	PLUGIN_FUNC_HANDLE_SIGHUP,
+	PLUGIN_FUNC_HANDLE_WAITPID,
 	PLUGIN_FUNC_HANDLE_SUBREQUEST,
 	PLUGIN_FUNC_HANDLE_SUBREQUEST_START,
 	PLUGIN_FUNC_HANDLE_RESPONSE_START,
@@ -297,7 +298,6 @@ int plugins_load(server *srv) {
 	handler_t plugins_call_##y(server *srv, connection *con) {\
 		plugin **slot;\
 		size_t j;\
-		if (!srv->plugin_slots) return HANDLER_GO_ON;\
 		slot = ((plugin ***)(srv->plugin_slots))[x];\
 		if (!slot) return HANDLER_GO_ON;\
 		for (j = 0; j < srv->plugins.used && slot[j]; j++) { \
@@ -385,6 +385,18 @@ PLUGIN_TO_SLOT(PLUGIN_FUNC_SET_DEFAULTS, set_defaults)
 
 #undef PLUGIN_TO_SLOT
 
+handler_t plugins_call_handle_waitpid(server *srv, pid_t pid, int status) {
+	plugin ** const slot =
+	  ((plugin ***)(srv->plugin_slots))[PLUGIN_FUNC_HANDLE_WAITPID];
+	if (!slot) return HANDLER_GO_ON;
+	for (size_t i = 0; i < srv->plugins.used && slot[i]; ++i) {
+		plugin *p = slot[i];
+		handler_t r = p->handle_waitpid(srv, p->data, pid, status);
+		if (r != HANDLER_GO_ON) return r;
+	}
+	return HANDLER_GO_ON;
+}
+
 #if 0
 /**
  *
@@ -469,6 +481,7 @@ handler_t plugins_call_init(server *srv) {
 		PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_CONNECTION_CLOSE, handle_connection_close);
 		PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_TRIGGER, handle_trigger);
 		PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_SIGHUP, handle_sighup);
+		PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_WAITPID, handle_waitpid);
 		PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_SUBREQUEST, handle_subrequest);
 		PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_SUBREQUEST_START, handle_subrequest_start);
 		PLUGIN_TO_SLOT(PLUGIN_FUNC_HANDLE_RESPONSE_START, handle_response_start);

src/plugin.h

@@ -37,7 +37,8 @@ typedef struct {
 	handler_t (* cleanup)                (server *srv, void *p_d);
 	                                                                                   /* is called ... */
 	handler_t (* handle_trigger)         (server *srv, void *p_d);                     /* once a second */
-	handler_t (* handle_sighup)          (server *srv, void *p_d);                     /* at a signup */
+	handler_t (* handle_sighup)          (server *srv, void *p_d);                     /* at a sighup */
+	handler_t (* handle_waitpid)         (server *srv, void *p_d, pid_t pid, int status); /* upon a child process exit */
 
 	handler_t (* handle_uri_raw)         (server *srv, connection *con, void *p_d);    /* after uri_raw is set */
 	handler_t (* handle_uri_clean)       (server *srv, connection *con, void *p_d);    /* after uri is set */
@@ -84,6 +85,7 @@ handler_t plugins_call_connection_reset(server *srv, connection *con);
 
 handler_t plugins_call_handle_trigger(server *srv);
 handler_t plugins_call_handle_sighup(server *srv);
+handler_t plugins_call_handle_waitpid(server *srv, pid_t pid, int status);
 
 handler_t plugins_call_init(server *srv);
 handler_t plugins_call_set_defaults(server *srv);

src/server.c

@@ -90,6 +90,7 @@ static server_socket_array graceful_sockets;
 static volatile sig_atomic_t graceful_restart = 0;
 static volatile sig_atomic_t graceful_shutdown = 0;
 static volatile sig_atomic_t srv_shutdown = 0;
+static volatile sig_atomic_t handle_sig_child = 0;
 static volatile sig_atomic_t handle_sig_alarm = 1;
 static volatile sig_atomic_t handle_sig_hup = 0;
 
@@ -135,6 +136,7 @@ static void sigaction_handler(int sig, siginfo_t *si, void *context) {
 		last_sighup_info = *si;
 		break;
 	case SIGCHLD:
+		handle_sig_child = 1;
 		break;
 	}
 }
@@ -160,7 +162,7 @@ static void signal_handler(int sig) {
 		break;
 	case SIGALRM: handle_sig_alarm = 1; break;
 	case SIGHUP:  handle_sig_hup = 1; break;
-	case SIGCHLD:  break;
+	case SIGCHLD: handle_sig_child = 1; break;
 	}
 }
 #endif
@@ -1482,10 +1484,11 @@ static int server_main (server * const srv, int argc, char **argv) {
 
 	srv->gid = getgid();
 	srv->uid = getuid();
+	srv->pid = getpid();
 
 	/* write pid file */
 	if (pid_fd > 2) {
-		buffer_copy_int(srv->tmp_buf, getpid());
+		buffer_copy_int(srv->tmp_buf, srv->pid);
 		buffer_append_string_len(srv->tmp_buf, CONST_STR_LEN("\n"));
 		if (-1 == write_all(pid_fd, CONST_BUF_LEN(srv->tmp_buf))) {
 			log_error_write(srv, __FILE__, __LINE__, "ss", "Couldn't write pid file:", strerror(errno));
@@ -1584,6 +1587,7 @@ static int server_main (server * const srv, int argc, char **argv) {
 		pid_t pid;
 		const int npids = num_childs;
 		int child = 0;
+		unsigned int timer = 0;
 		for (int n = 0; n < npids; ++n) pids[n] = -1;
 		while (!child && !srv_shutdown && !graceful_shutdown) {
 			if (num_childs > 0) {
@@ -1592,6 +1596,7 @@ static int server_main (server * const srv, int argc, char **argv) {
 					return -1;
 				case 0:
 					child = 1;
+					alarm(0);
 					break;
 				default:
 					num_childs--;
@@ -1607,9 +1612,15 @@ static int server_main (server * const srv, int argc, char **argv) {
 				int status;
 
 				if (-1 != (pid = wait(&status))) {
+					srv->cur_ts = time(NULL);
+					if (plugins_call_handle_waitpid(srv, pid, status) != HANDLER_GO_ON) {
+						if (!timer) alarm((timer = 5));
+						continue;
+					}
 					switch (fdevent_reaped_logger_pipe(pid)) {
 					  default: break;
-					  case -1: alarm(5); /* fall through */
+					  case -1: if (!timer) alarm((timer = 5));
+						   /* fall through */
 					  case  1: continue;
 					}
 					/** 
@@ -1625,6 +1636,7 @@ static int server_main (server * const srv, int argc, char **argv) {
 				} else {
 					switch (errno) {
 					case EINTR:
+						srv->cur_ts = time(NULL);
 						/**
 						 * if we receive a SIGHUP we have to close our logs ourself as we don't 
 						 * have the mainloop who can help us here
@@ -1641,7 +1653,9 @@ static int server_main (server * const srv, int argc, char **argv) {
 						}
 						if (handle_sig_alarm) {
 							handle_sig_alarm = 0;
-							fdevent_waitpid_logger_pipes(time(NULL));
+							timer = 0;
+							plugins_call_handle_trigger(srv);
+							fdevent_restart_logger_pipes(srv->cur_ts);
 						}
 						break;
 					default:
@@ -1691,6 +1705,8 @@ static int server_main (server * const srv, int argc, char **argv) {
 		}
 		buffer_reset(srv->srvconf.pid_file);
 
+		fdevent_clr_logger_pipe_pids();
+		srv->pid = getpid();
 		li_rand_reseed();
 	}
 #endif
@@ -1827,7 +1843,6 @@ static int server_main (server * const srv, int argc, char **argv) {
 					break;
 				}
 
-				/* trigger waitpid */
 				srv->cur_ts = min_ts;
 
 				/* check idle time limit, if enabled */
@@ -1857,8 +1872,6 @@ static int server_main (server * const srv, int argc, char **argv) {
 				for (i = 0; i < srv->config_context->used; ++i) {
 					srv->config_storage[i]->global_bytes_per_second_cnt = 0;
 				}
-				/* check piped-loggers and restart, unless shutting down */
-				if (!graceful_shutdown && !srv_shutdown && 0 == srv->srvconf.max_worker) fdevent_waitpid_logger_pipes(min_ts);
 				/* if graceful_shutdown, accelerate cleanup of recently completed request/responses */
 				if (graceful_shutdown && !srv_shutdown) server_graceful_shutdown_maint(srv);
 				/**
@@ -1971,6 +1984,26 @@ static int server_main (server * const srv, int argc, char **argv) {
 			}
 		}
 
+		if (handle_sig_child) {
+			pid_t pid;
+			handle_sig_child = 0;
+			do {
+				int status;
+				pid = waitpid(-1, &status, WNOHANG);
+				if (pid > 0) {
+					if (plugins_call_handle_waitpid(srv, pid, status) != HANDLER_GO_ON) {
+						continue;
+					}
+					if (0 == srv->srvconf.max_worker) {
+						/* check piped-loggers and restart, even if shutting down */
+						if (fdevent_waitpid_logger_pipe_pid(pid, srv->cur_ts)) {
+							continue;
+						}
+					}
+				}
+			} while (pid > 0 || (-1 == pid && errno == EINTR));
+		}
+
 		if (graceful_shutdown) {
 			server_graceful_state(srv);
 			srv->sockets_disabled = 1;