[core] buffer large responses to tempfiles (fixes #758, fixes #760, fixes #933, fixes #1387, #1283, fixes #2083)

This replaces buffering entire response in memory which might lead to
huge memory footprint and possibly to memory exhaustion.

use tempfiles of fixed size so disk space is freed as each file sent

update callers of http_chunk_append_mem() and http_chunk_append_buffer()
to handle failures when writing to tempfile.

x-ref:
  "memory fragmentation leads to high memory usage after peaks"
  https://redmine.lighttpd.net/issues/758
  "Random crashing on FreeBSD 6.1"
  https://redmine.lighttpd.net/issues/760
  "lighty should buffer responses (after it grows above certain size) on disk"
  https://redmine.lighttpd.net/issues/933
  "Memory usage increases when proxy+ssl+large file"
  https://redmine.lighttpd.net/issues/1283
  "lighttpd+fastcgi memory problem"
  https://redmine.lighttpd.net/issues/1387
  "Excessive Memory usage with streamed files from PHP"
  https://redmine.lighttpd.net/issues/2083

src/chunk.c

@@ -461,7 +461,7 @@ static chunk *chunkqueue_get_append_tempfile(chunkqueue *cq) {
 
 static void chunkqueue_remove_empty_chunks(chunkqueue *cq);
 
-static int chunkqueue_append_to_tempfile(server *srv, chunkqueue *dest, const char *mem, size_t len) {
+int chunkqueue_append_mem_to_tempfile(server *srv, chunkqueue *dest, const char *mem, size_t len) {
 	chunk *dst_c;
 	ssize_t written;
 
@@ -599,7 +599,7 @@ int chunkqueue_steal_with_tempfiles(server *srv, chunkqueue *dest, chunkqueue *s
 
 		case MEM_CHUNK:
 			/* store "use" bytes from memory chunk in tempfile */
-			if (0 != chunkqueue_append_to_tempfile(srv, dest, c->mem->ptr + c->offset, use)) {
+			if (0 != chunkqueue_append_mem_to_tempfile(srv, dest, c->mem->ptr + c->offset, use)) {
 				return -1;
 			}
 

src/chunk.h

@@ -57,6 +57,9 @@ void chunkqueue_append_mem(chunkqueue *cq, const char *mem, size_t len); /* copi
 void chunkqueue_append_buffer(chunkqueue *cq, buffer *mem); /* may reset "mem" */
 void chunkqueue_prepend_buffer(chunkqueue *cq, buffer *mem); /* may reset "mem" */
 
+struct server; /*(declaration)*/
+int chunkqueue_append_mem_to_tempfile(struct server *srv, chunkqueue *cq, const char *mem, size_t len);
+
 /* functions to handle buffers to read into: */
 /* return a pointer to a buffer in *mem with size *len;
  *  it should be at least min_size big, and use alloc_size if

src/http_chunk.c

@@ -94,45 +94,78 @@ int http_chunk_append_file(server *srv, connection *con, buffer *fn) {
 	return 0;
 }
 
-void http_chunk_append_buffer(server *srv, connection *con, buffer *mem) {
-	chunkqueue *cq;
-
-	force_assert(NULL != con);
-
-	if (buffer_string_is_empty(mem)) return;
-
-	cq = con->write_queue;
+static int http_chunk_append_to_tempfile(server *srv, connection *con, const char * mem, size_t len) {
+	chunkqueue * const cq = con->write_queue;
 
 	if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) {
-		http_chunk_append_len(srv, con, buffer_string_length(mem));
+		/*http_chunk_append_len(srv, con, len);*/
+		buffer *b = srv->tmp_chunk_len;
+
+		buffer_string_set_length(b, 0);
+		buffer_append_uint_hex(b, len);
+		buffer_append_string_len(b, CONST_STR_LEN("\r\n"));
+
+		if (0 != chunkqueue_append_mem_to_tempfile(srv, cq, CONST_BUF_LEN(b))) {
+			return -1;
+		}
 	}
 
-	chunkqueue_append_buffer(cq, mem);
+	if (0 != chunkqueue_append_mem_to_tempfile(srv, cq, mem, len)) {
+		return -1;
+	}
 
 	if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) {
-		chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n"));
+		if (0 != chunkqueue_append_mem_to_tempfile(srv, cq, CONST_STR_LEN("\r\n"))) {
+			return -1;
+		}
 	}
+
+	return 0;
 }
 
-void http_chunk_append_mem(server *srv, connection *con, const char * mem, size_t len) {
-	chunkqueue *cq;
+static int http_chunk_append_data(server *srv, connection *con, buffer *b, const char * mem, size_t len) {
 
-	force_assert(NULL != con);
-	force_assert(NULL != mem || 0 == len);
+	chunkqueue * const cq = con->write_queue;
+	chunk *c = cq->last;
+	if (0 == len) return 0;
 
-	if (NULL == mem || 0 == len) return;
+	/* current usage does not append_mem or append_buffer after appending
+	 * file, so not checking if users of this interface have appended large
+	 * (references to) files to chunkqueue, which would not be in memory */
 
-	cq = con->write_queue;
+	if ((c && c->type == FILE_CHUNK && c->file.is_temp)
+	    || cq->bytes_in - cq->bytes_out + len > 64 * 1024) {
+		return http_chunk_append_to_tempfile(srv, con, b ? b->ptr : mem, len);
+	}
+
+	/* not appending to prior mem chunk just in case using openssl
+	 * and need to resubmit same args as prior call to openssl (required?)*/
 
 	if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) {
 		http_chunk_append_len(srv, con, len);
 	}
 
-	chunkqueue_append_mem(cq, mem, len);
+	/*(chunkqueue_append_buffer() might steal buffer contents)*/
+	b ? chunkqueue_append_buffer(cq, b) : chunkqueue_append_mem(cq, mem, len);
 
 	if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) {
 		chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n"));
 	}
+
+	return 0;
+}
+
+int http_chunk_append_buffer(server *srv, connection *con, buffer *mem) {
+	force_assert(NULL != con);
+
+	return http_chunk_append_data(srv, con, mem, NULL, buffer_string_length(mem));
+}
+
+int http_chunk_append_mem(server *srv, connection *con, const char * mem, size_t len) {
+	force_assert(NULL != con);
+	force_assert(NULL != mem || 0 == len);
+
+	return http_chunk_append_data(srv, con, NULL, mem, len);
 }
 
 void http_chunk_close(server *srv, connection *con) {

src/http_chunk.h

@@ -5,8 +5,8 @@
 #include "server.h"
 #include <sys/types.h>
 
-void http_chunk_append_mem(server *srv, connection *con, const char * mem, size_t len); /* copies memory */
-void http_chunk_append_buffer(server *srv, connection *con, buffer *mem); /* may reset "mem" */
+int http_chunk_append_mem(server *srv, connection *con, const char * mem, size_t len); /* copies memory */
+int http_chunk_append_buffer(server *srv, connection *con, buffer *mem); /* may reset "mem" */
 int http_chunk_append_file(server *srv, connection *con, buffer *fn); /* copies "fn" */
 int http_chunk_append_file_range(server *srv, connection *con, buffer *fn, off_t offset, off_t len); /* copies "fn" */
 void http_chunk_close(server *srv, connection *con);

src/mod_cgi.c

@@ -500,7 +500,9 @@ static int cgi_demux_response(server *srv, handler_ctx *hctx) {
 						con->response.transfer_encoding = HTTP_TRANSFER_ENCODING_CHUNKED;
 					}
 
-					http_chunk_append_buffer(srv, con, hctx->response_header);
+					if (0 != http_chunk_append_buffer(srv, con, hctx->response_header)) {
+						return FDEVENT_HANDLED_ERROR;
+					}
 				} else {
 					const char *bstart;
 					size_t blen;
@@ -541,14 +543,18 @@ static int cgi_demux_response(server *srv, handler_ctx *hctx) {
 					}
 
 					if (blen > 0) {
-						http_chunk_append_mem(srv, con, bstart, blen);
+						if (0 != http_chunk_append_mem(srv, con, bstart, blen)) {
+							return FDEVENT_HANDLED_ERROR;
+						}
 					}
 				}
 
 				con->file_started = 1;
 			}
 		} else {
-			http_chunk_append_buffer(srv, con, hctx->response);
+			if (0 != http_chunk_append_buffer(srv, con, hctx->response)) {
+				return FDEVENT_HANDLED_ERROR;
+			}
 		}
 
 #if 0
@@ -756,7 +762,10 @@ static handler_t cgi_handle_fdevent(server *srv, void *ctx, int revents) {
 		/* check if we still have a unfinished header package which is a body in reality */
 		if (con->file_started == 0 && !buffer_string_is_empty(hctx->response_header)) {
 			con->file_started = 1;
-			http_chunk_append_buffer(srv, con, hctx->response_header);
+			if (0 != http_chunk_append_buffer(srv, con, hctx->response_header)) {
+				cgi_connection_close(srv, hctx);
+				return HANDLER_ERROR;
+			}
 		}
 
 # if 0

src/mod_fastcgi.c

@@ -2627,7 +2627,11 @@ static int fcgi_demux_response(server *srv, handler_ctx *hctx) {
 			}
 
 			if (hctx->send_content_body && !buffer_string_is_empty(packet.b)) {
-				http_chunk_append_buffer(srv, con, packet.b);
+				if (0 != http_chunk_append_buffer(srv, con, packet.b)) {
+					/* error writing to tempfile;
+					 * truncate response or send 500 if nothing sent yet */
+					fin = 1;
+				}
 			}
 			break;
 		case FCGI_STDERR:

src/mod_proxy.c

@@ -700,11 +700,22 @@ static int proxy_demux_response(server *srv, handler_ctx *hctx) {
 				}
 
 				con->file_started = 1;
-				if (blen > 0) http_chunk_append_mem(srv, con, c + 4, blen);
+				if (blen > 0) {
+					if (0 != http_chunk_append_mem(srv, con, c + 4, blen)) {
+						/* error writing to tempfile;
+						 * truncate response or send 500 if nothing sent yet */
+						fin = 1;
+						con->file_started = 0;
+					}
+				}
 				buffer_reset(hctx->response);
 			}
 		} else {
-			http_chunk_append_buffer(srv, con, hctx->response);
+			if (0 != http_chunk_append_buffer(srv, con, hctx->response)) {
+				/* error writing to tempfile;
+				 * truncate response or send 500 if nothing sent yet */
+				fin = 1;
+			}
 			buffer_reset(hctx->response);
 		}
 

src/mod_scgi.c

@@ -1968,7 +1968,11 @@ static int scgi_demux_response(server *srv, handler_ctx *hctx) {
 						con->response.transfer_encoding = HTTP_TRANSFER_ENCODING_CHUNKED;
 					}
 
-					http_chunk_append_buffer(srv, con, hctx->response_header);
+					if (0 != http_chunk_append_buffer(srv, con, hctx->response_header)) {
+						/* error writing to tempfile;
+						 * truncate response or send 500 if nothing sent yet */
+						return 1;
+					}
 				} else {
 					size_t blen = buffer_string_length(hctx->response_header) - hlen;
 
@@ -1993,14 +1997,22 @@ static int scgi_demux_response(server *srv, handler_ctx *hctx) {
 					}
 
 					if (blen > 0) {
-						http_chunk_append_mem(srv, con, hctx->response_header->ptr + hlen, blen);
+						if (0 != http_chunk_append_mem(srv, con, hctx->response_header->ptr + hlen, blen)) {
+							/* error writing to tempfile;
+							 * truncate response or send 500 if nothing sent yet */
+							return 1;
+						}
 					}
 				}
 
 				con->file_started = 1;
 			}
 		} else {
-			http_chunk_append_buffer(srv, con, hctx->response);
+			if (0 != http_chunk_append_buffer(srv, con, hctx->response)) {
+				/* error writing to tempfile;
+				 * truncate response or send 500 if nothing sent yet */
+				return 1;
+			}
 		}
 
 #if 0