Remove chunkqueue_get_{append,prepend}* API

Although those were "easy" to use, they violated the abstraction:
  content of the chunkqueue should only be modified via the API.
  Replace with chunkqueue_get_memory() and chunkqueue_use_memory() for
  functions that read data from network (reusing large buffers),
  chunkqueue_steal_with_tempfiles() to store request bodies on disk
  temporarily.
  Modules that were generating content and need a buffer maintain the
  buffer manually (have to be careful to free the buffer on errors, as
  it isn't part of the chunkqueue yet).

From: Stefan Bühler <stbuehler@web.de>

git-svn-id: svn://svn.lighttpd.net/lighttpd/branches/lighttpd-1.4.x@2976 152afb58-edef-0310-8abb-c4023f1b3aa9

src/buffer.c

@@ -139,6 +139,27 @@ char* buffer_prepare_append(buffer *b, size_t size) {
 	return b->ptr + b->used - 1;
 }
 
+char* buffer_string_prepare_copy(buffer *b, size_t size) {
+	force_assert(NULL != b);
+
+	buffer_prepare_copy(b, size);
+	b->used = 1;
+
+	return b->ptr;
+}
+
+char* buffer_string_prepare_append(buffer *b, size_t size) {
+	force_assert(NULL !=  b);
+
+	if (0 == b->used) {
+		return buffer_string_prepare_copy(b, size);
+	} else {
+		force_assert('\0' == b->ptr[b->used - 1]);
+		return buffer_prepare_append(b, size);
+	}
+}
+
+
 void buffer_commit(buffer *b, size_t size)
 {
 	force_assert(NULL != b);
@@ -231,7 +252,7 @@ void buffer_append_long_hex(buffer *b, unsigned long value) {
 		} while (0 != copy);
 	}
 
-	buf = buffer_prepare_append(b, shift);
+	buf = buffer_string_prepare_append(b, shift);
 	buffer_commit(b, shift); /* will fill below */
 
 	shift <<= 2; /* count bits now */

src/buffer.h

@@ -62,6 +62,11 @@ char* buffer_prepare_copy(buffer *b, size_t size);
  */
 char* buffer_prepare_append(buffer *b, size_t size);
 
+/* similar to buffer_prepare_copy(b, size), but sets b->used = 1 */
+char* buffer_string_prepare_copy(buffer *b, size_t size);
+/* similar to buffer_prepare_append(b, size), but sets b->used = 1 if used was b->0 before */
+char* buffer_string_prepare_append(buffer *b, size_t size);
+
 /* use after prepare_(copy,append) when you have written data to the buffer
  * to increase the buffer length by size. also sets the terminating zero.
  * requires enough space is present for the terminating zero (prepare with the
@@ -136,6 +141,7 @@ int light_isalpha(int c);
 int light_isalnum(int c);
 
 static inline size_t buffer_string_length(const buffer *b); /* buffer string length without terminating 0 */
+static inline size_t buffer_string_space(const buffer *b); /* maximum length of string that can be stored without reallocating */
 static inline void buffer_append_slash(buffer *b); /* append '/' no non-empty strings not ending in '/' */
 
 #define BUFFER_APPEND_STRING_CONST(x, y) \
@@ -161,6 +167,12 @@ static inline size_t buffer_string_length(const buffer *b) {
 	return NULL != b && 0 != b->used ? b->used - 1 : 0;
 }
 
+static inline size_t buffer_string_space(const buffer *b) {
+	if (NULL == b || b->size == 0) return 0;
+	if (0 == b->used) return b->size - 1;
+	return b->size - b->used;
+}
+
 static inline void buffer_append_slash(buffer *b) {
 	size_t len = buffer_string_length(b);
 	if (len > 0 && '/' != b->ptr[len-1]) BUFFER_APPEND_STRING_CONST(b, "/");

src/chunk.c

@@ -5,6 +5,8 @@
  */
 
 #include "chunk.h"
+#include "base.h"
+#include "log.h"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -233,28 +235,84 @@ void chunkqueue_append_mem(chunkqueue *cq, const char * mem, size_t len) {
 	chunkqueue_append_chunk(cq, c);
 }
 
-buffer * chunkqueue_get_prepend_buffer(chunkqueue *cq) {
+void chunkqueue_get_memory(chunkqueue *cq, char **mem, size_t *len, size_t min_size, size_t alloc_size) {
+	static const size_t REALLOC_MAX_SIZE = 256;
 	chunk *c;
+	buffer *b;
+	char *dummy_mem;
+	size_t dummy_len;
 
-	c = chunkqueue_get_unused_chunk(cq);
+	force_assert(NULL != cq);
+	if (NULL == mem) mem = &dummy_mem;
+	if (NULL == len) len = &dummy_len;
 
-	c->type = MEM_CHUNK;
+	/* default values: */
+	if (0 == min_size) min_size = 1024;
+	if (0 == alloc_size) alloc_size = 4096;
+	if (alloc_size < min_size) alloc_size = min_size;
 
-	chunkqueue_prepend_chunk(cq, c);
+	if (NULL != cq->last && MEM_CHUNK == cq->last->type) {
+		size_t have;
 
-	return c->mem;
-}
+		b = cq->last->mem;
+		have = buffer_string_space(b);
 
-buffer *chunkqueue_get_append_buffer(chunkqueue *cq) {
-	chunk *c;
+		/* unused buffer: allocate space */
+		if (buffer_string_is_empty(b)) {
+			buffer_string_prepare_copy(b, alloc_size);
+			have = buffer_string_space(b);
+		}
+		/* if buffer is really small just make it bigger */
+		else if (have < min_size && b->size <= REALLOC_MAX_SIZE) {
+			size_t new_size = b->used + min_size, append;
+			if (new_size < alloc_size) new_size = alloc_size;
+
+			append = new_size - b->used;
+			if (append >= min_size) {
+				buffer_string_prepare_append(b, append);
+				have = buffer_string_space(b);
+			}
+		}
 
-	c = chunkqueue_get_unused_chunk(cq);
+		/* return pointer into existing buffer if large enough */
+		if (have >= min_size) {
+			*mem = b->ptr + buffer_string_length(b);
+			*len = have;
+			return;
+		}
+	}
 
+	/* allocate new chunk */
+	c = chunkqueue_get_unused_chunk(cq);
 	c->type = MEM_CHUNK;
-
 	chunkqueue_append_chunk(cq, c);
 
-	return c->mem;
+	b = c->mem;
+	buffer_string_prepare_append(b, alloc_size);
+
+	*mem = b->ptr + buffer_string_length(b);
+	*len = buffer_string_space(b);
+}
+
+void chunkqueue_use_memory(chunkqueue *cq, size_t len) {
+	buffer *b;
+
+	force_assert(NULL != cq);
+	force_assert(NULL != cq->last && MEM_CHUNK == cq->last->type);
+	b = cq->last->mem;
+
+	force_assert(b->used > 0);
+	force_assert(len <= buffer_string_space(b));
+
+	if (len > 0) {
+		b->used += len;
+		b->ptr[b->used - 1] = '\0';
+	} else if (buffer_string_is_empty(b)) {
+		/* unused buffer: can't remove chunk easily from
+		 * end of list, so just reset the buffer
+		 */
+		buffer_reset(b);
+	}
 }
 
 void chunkqueue_set_tempdirs(chunkqueue *cq, array *tempdirs) {
@@ -262,13 +320,67 @@ void chunkqueue_set_tempdirs(chunkqueue *cq, array *tempdirs) {
 	cq->tempdirs = tempdirs;
 }
 
-chunk *chunkqueue_get_append_tempfile(chunkqueue *cq) {
-	chunk *c;
-	buffer *template = buffer_init_string("/var/tmp/lighttpd-upload-XXXXXX");
+void chunkqueue_steal(chunkqueue *dest, chunkqueue *src, off_t len) {
+	while (len > 0) {
+		chunk *c = src->first;
+		off_t clen = 0, use;
 
-	c = chunkqueue_get_unused_chunk(cq);
+		if (NULL == c) break;
 
-	c->type = FILE_CHUNK;
+		switch (c->type) {
+		case MEM_CHUNK:
+			clen = buffer_string_length(c->mem);
+			break;
+		case FILE_CHUNK:
+			clen = c->file.length;
+			break;
+		}
+		force_assert(clen >= c->offset);
+		clen -= c->offset;
+		use = len >= clen ? clen : len;
+
+		src->bytes_out += use;
+		dest->bytes_in += use;
+		len -= use;
+
+		if (0 == clen) {
+			/* drop empty chunk */
+			src->first = c->next;
+			if (c == src->last) src->last = NULL;
+			chunkqueue_push_unused_chunk(src, c);
+			continue;
+		}
+
+		if (use == clen) {
+			/* move complete chunk */
+			src->first = c->next;
+			if (c == src->last) src->last = NULL;
+
+			chunkqueue_append_chunk(dest, c);
+			continue;
+		}
+
+		/* partial chunk with length "use" */
+
+		switch (c->type) {
+		case MEM_CHUNK:
+			chunkqueue_append_mem(dest, c->mem->ptr + c->offset, use);
+			break;
+		case FILE_CHUNK:
+			/* tempfile flag is in "last" chunk after the split */
+			chunkqueue_append_file(dest, c->file.name, c->file.start + c->offset, use);
+			break;
+		}
+
+		c->offset += use;
+		force_assert(0 == len);
+	}
+}
+
+static chunk *chunkqueue_get_append_tempfile(chunkqueue *cq) {
+	chunk *c;
+	buffer *template = buffer_init_string("/var/tmp/lighttpd-upload-XXXXXX");
+	int fd;
 
 	if (cq->tempdirs && cq->tempdirs->used) {
 		size_t i;
@@ -282,19 +394,21 @@ chunk *chunkqueue_get_append_tempfile(chunkqueue *cq) {
 			buffer_append_slash(template);
 			buffer_append_string_len(template, CONST_STR_LEN("lighttpd-upload-XXXXXX"));
 
-			if (-1 != (c->file.fd = mkstemp(template->ptr))) {
-				/* only trigger the unlink if we created the temp-file successfully */
-				c->file.is_temp = 1;
-				break;
-			}
+			if (-1 != (fd = mkstemp(template->ptr))) break;
 		}
 	} else {
-		if (-1 != (c->file.fd = mkstemp(template->ptr))) {
-			/* only trigger the unlink if we created the temp-file successfully */
-			c->file.is_temp = 1;
-		}
+		fd = mkstemp(template->ptr);
 	}
 
+	if (-1 == fd) {
+		buffer_free(template);
+		return NULL;
+	}
+
+	c = chunkqueue_get_unused_chunk(cq);
+	c->type = FILE_CHUNK;
+	c->file.fd = fd;
+	c->file.is_temp = 1;
 	buffer_copy_buffer(c->file.name, template);
 	c->file.length = 0;
 
@@ -305,10 +419,76 @@ chunk *chunkqueue_get_append_tempfile(chunkqueue *cq) {
 	return c;
 }
 
-void chunkqueue_steal(chunkqueue *dest, chunkqueue *src, off_t len) {
+static int chunkqueue_append_to_tempfile(server *srv, chunkqueue *dest, const char *mem, size_t len) {
+	chunk *dst_c = NULL;
+	ssize_t written;
+	/* copy everything to max 1Mb sized tempfiles */
+
+	/*
+	 * if the last chunk is
+	 * - smaller than 1Mb (size < 1Mb)
+	 * - not read yet (offset == 0)
+	 * -> append to it
+	 * otherwise
+	 * -> create a new chunk
+	 *
+	 * */
+
+	if (NULL != dest->last
+		&& FILE_CHUNK != dest->last->type
+		&& dest->last->file.is_temp
+		&& -1 != dest->last->file.fd
+		&& 0 == dest->last->offset) {
+		/* ok, take the last chunk for our job */
+		dst_c = dest->last;
+
+		if (dest->last->file.length >= 1 * 1024 * 1024) {
+			/* the chunk is too large now, close it */
+			if (-1 != dst_c->file.fd) {
+				close(dst_c->file.fd);
+				dst_c->file.fd = -1;
+			}
+			dst_c = chunkqueue_get_append_tempfile(dest);
+		}
+	} else {
+		dst_c = chunkqueue_get_append_tempfile(dest);
+	}
+
+	if (NULL == dst_c) {
+		/* we don't have file to write to,
+		 * EACCES might be one reason.
+		 *
+		 * Instead of sending 500 we send 413 and say the request is too large
+		 */
+
+		log_error_write(srv, __FILE__, __LINE__, "sbs",
+				"denying upload as opening temp-file for upload failed:",
+				dst_c->file.name, strerror(errno));
+
+		return -1;
+	}
+
+	if (0 > (written = write(dst_c->file.fd, mem, len)) || (size_t) written != len) {
+		/* write failed for some reason ... disk full ? */
+		log_error_write(srv, __FILE__, __LINE__, "sbs",
+				"denying upload as writing to file failed:",
+				dst_c->file.name, strerror(errno));
+
+		close(dst_c->file.fd);
+		dst_c->file.fd = -1;
+
+		return -1;
+	}
+
+	dst_c->file.length += len;
+
+	return 0;
+}
+
+int chunkqueue_steal_with_tempfiles(server *srv, chunkqueue *dest, chunkqueue *src, off_t len) {
 	while (len > 0) {
 		chunk *c = src->first;
-		off_t clen = 0;
+		off_t clen = 0, use;
 
 		if (NULL == c) break;
 
@@ -322,36 +502,57 @@ void chunkqueue_steal(chunkqueue *dest, chunkqueue *src, off_t len) {
 		}
 		force_assert(clen >= c->offset);
 		clen -= c->offset;
+		use = len >= clen ? clen : len;
 
-		if (len >= clen) {
-			/* move complete chunk */
+		src->bytes_out += use;
+		dest->bytes_in += use;
+		len -= use;
+
+		if (0 == clen) {
+			/* drop empty chunk */
 			src->first = c->next;
 			if (c == src->last) src->last = NULL;
-
-			chunkqueue_append_chunk(dest, c);
-			src->bytes_out += clen;
-			dest->bytes_in += clen;
-			len -= clen;
+			chunkqueue_push_unused_chunk(src, c);
 			continue;
 		}
 
-		/* partial chunk with length "len" */
+		if (FILE_CHUNK == c->type) {
+			if (use == clen) {
+				/* move complete chunk */
+				src->first = c->next;
+				if (c == src->last) src->last = NULL;
 
-		switch (c->type) {
-		case MEM_CHUNK:
-			chunkqueue_append_mem(dest, c->mem->ptr + c->offset, len);
-			break;
-		case FILE_CHUNK:
-			/* tempfile flag is in "last" chunk after the split */
-			chunkqueue_append_file(dest, c->file.name, c->file.start + c->offset, len);
-			break;
+				chunkqueue_append_chunk(dest, c);
+			} else {
+				/* partial chunk with length "use" */
+				/* tempfile flag is in "last" chunk after the split */
+				chunkqueue_append_file(dest, c->file.name, c->file.start + c->offset, use);
+
+				c->offset += use;
+				force_assert(0 == len);
+			}
+			continue;
 		}
 
-		c->offset += len;
-		src->bytes_out += len;
-		dest->bytes_in += len;
-		len = 0;
+		/* store "use" bytes from memory chunk in tempfile */
+		if (0 != chunkqueue_append_to_tempfile(srv, dest, c->mem->ptr + c->offset, use)) {
+			/* undo counters */
+			src->bytes_out -= use;
+			dest->bytes_in -= use;
+			return -1;
+		}
+
+
+		c->offset += use;
+		if (use == clen) {
+			/* finished chunk */
+			src->first = c->next;
+			if (c == src->last) src->last = NULL;
+			chunkqueue_push_unused_chunk(src, c);
+		}
 	}
+
+	return 0;
 }
 
 off_t chunkqueue_length(chunkqueue *cq) {

src/chunk.h

@@ -48,24 +48,37 @@ typedef struct {
 } chunkqueue;
 
 chunkqueue *chunkqueue_init(void);
-void chunkqueue_set_tempdirs(chunkqueue *c, array *tempdirs);
-void chunkqueue_append_file(chunkqueue *c, buffer *fn, off_t offset, off_t len); /* copies "fn" */
-void chunkqueue_append_mem(chunkqueue *c, const char *mem, size_t len); /* copies memory */
-void chunkqueue_append_buffer(chunkqueue *c, buffer *mem); /* may reset "mem" */
-void chunkqueue_prepend_buffer(chunkqueue *c, buffer *mem); /* may reset "mem" */
-
-buffer * chunkqueue_get_append_buffer(chunkqueue *c);
-buffer * chunkqueue_get_prepend_buffer(chunkqueue *c);
-chunk * chunkqueue_get_append_tempfile(chunkqueue *cq);
+void chunkqueue_set_tempdirs(chunkqueue *cq, array *tempdirs);
+void chunkqueue_append_file(chunkqueue *cq, buffer *fn, off_t offset, off_t len); /* copies "fn" */
+void chunkqueue_append_mem(chunkqueue *cq, const char *mem, size_t len); /* copies memory */
+void chunkqueue_append_buffer(chunkqueue *cq, buffer *mem); /* may reset "mem" */
+void chunkqueue_prepend_buffer(chunkqueue *cq, buffer *mem); /* may reset "mem" */
+
+/* functions to handle buffers to read into: */
+/* return a pointer to a buffer in *mem with size *len;
+ *  it should be at least min_size big, and use alloc_size if
+ *  new memory is allocated.
+ * modifying the chunkqueue invalidates the memory area.
+ * should always be followed by chunkqueue_get_memory(),
+ *  even if nothing was read.
+ * pass 0 for min_size/alloc_size for default values
+ */
+void chunkqueue_get_memory(chunkqueue *cq, char **mem, size_t *len, size_t min_size, size_t alloc_size);
+/* append first len bytes of the memory queried with
+ * chunkqueue_get_memory to the chunkqueue
+ */
+void chunkqueue_use_memory(chunkqueue *cq, size_t len);
 
 void chunkqueue_remove_finished_chunks(chunkqueue *cq);
 
 void chunkqueue_steal(chunkqueue *dest, chunkqueue *src, off_t len);
+struct server;
+int chunkqueue_steal_with_tempfiles(struct server *srv, chunkqueue *dest, chunkqueue *src, off_t len);
 
-off_t chunkqueue_length(chunkqueue *c);
-void chunkqueue_free(chunkqueue *c);
-void chunkqueue_reset(chunkqueue *c);
+off_t chunkqueue_length(chunkqueue *cq);
+void chunkqueue_free(chunkqueue *cq);
+void chunkqueue_reset(chunkqueue *cq);
 
-int chunkqueue_is_empty(chunkqueue *c);
+int chunkqueue_is_empty(chunkqueue *cq);
 
 #endif

src/connections.c

@@ -197,31 +197,22 @@ static void dump_packet(const unsigned char *data, size_t len) {
 
 static int connection_handle_read_ssl(server *srv, connection *con) {
 #ifdef USE_OPENSSL
-	int r, ssl_err, len, count = 0, read_offset, toread;
-	buffer *b = NULL;
+	int r, ssl_err, len, count = 0;
+	char *mem = NULL;
+	size_t mem_len = 0;
 
 	if (!con->srv_socket->is_ssl) return -1;
 
 	ERR_clear_error();
 	do {
-		if (NULL != con->read_queue->last) {
-			b = con->read_queue->last->mem;
-		}
-
-		if (NULL == b || b->size - b->used < 1024) {
-			b = chunkqueue_get_append_buffer(con->read_queue);
-			len = SSL_pending(con->ssl);
-			if (len < 4*1024) len = 4*1024; /* always alloc >= 4k buffer */
-			buffer_prepare_copy(b, len);
-
-			/* overwrite everything with 0 */
-			memset(b->ptr, 0, b->size);
-		}
-
-		read_offset = (b->used > 0) ? b->used - 1 : 0;
-		toread = b->size - 1 - read_offset;
+		chunkqueue_get_memory(con->read_queue, &mem, &mem_len, 0, SSL_pending(con->ssl));
+#if 0
+		/* overwrite everything with 0 */
+		memset(mem, 0, mem_len);
+#endif
 
-		len = SSL_read(con->ssl, b->ptr + read_offset, toread);
+		len = SSL_read(con->ssl, mem, mem_len);
+		chunkqueue_use_memory(con->read_queue, len > 0 ? len : 0);
 
 		if (con->renegotiations > 1 && con->conf.ssl_disable_client_renegotiation) {
 			log_error_write(srv, __FILE__, __LINE__, "s", "SSL: renegotiation initiated by client, killing connection");
@@ -230,15 +221,10 @@ static int connection_handle_read_ssl(server *srv, connection *con) {
 		}
 
 		if (len > 0) {
-			if (b->used > 0) b->used--;
-			b->used += len;
-			b->ptr[b->used++] = '\0';
-
 			con->bytes_read += len;
-
 			count += len;
 		}
-	} while (len == toread && count < MAX_READ_LIMIT);
+	} while (len == (ssize_t) mem_len && count < MAX_READ_LIMIT);
 
 
 	if (len < 0) {
@@ -331,44 +317,36 @@ static int connection_handle_read_ssl(server *srv, connection *con) {
 /* 0: everything ok, -1: error, -2: con closed */
 static int connection_handle_read(server *srv, connection *con) {
 	int len;
-	buffer *b;
-	int toread, read_offset;
+	char *mem = NULL;
+	size_t mem_len = 0;
+	int toread;
 
 	if (con->srv_socket->is_ssl) {
 		return connection_handle_read_ssl(srv, con);
 	}
 
-	b = (NULL != con->read_queue->last) ? con->read_queue->last->mem : NULL;
-
 	/* default size for chunks is 4kb; only use bigger chunks if FIONREAD tells
 	 *  us more than 4kb is available
 	 * if FIONREAD doesn't signal a big chunk we fill the previous buffer
 	 *  if it has >= 1kb free
 	 */
 #if defined(__WIN32)
-	if (NULL == b || b->size - b->used < 1024) {
-		b = chunkqueue_get_append_buffer(con->read_queue);
-		buffer_prepare_copy(b, 4 * 1024);
-	}
+	chunkqueue_get_memory(con->read_queue, &mem, &mem_len, 0, 4096);
 
-	read_offset = (b->used == 0) ? 0 : b->used - 1;
-	len = recv(con->fd, b->ptr + read_offset, b->size - 1 - read_offset, 0);
+	len = recv(con->fd, mem, mem_len, 0);
 #else
 	if (ioctl(con->fd, FIONREAD, &toread) || toread == 0 || toread <= 4*1024) {
-		if (NULL == b || b->size - b->used < 1024) {
-			b = chunkqueue_get_append_buffer(con->read_queue);
-			buffer_prepare_copy(b, 4 * 1024);
-		}
-	} else {
 		if (toread > MAX_READ_LIMIT) toread = MAX_READ_LIMIT;
-		b = chunkqueue_get_append_buffer(con->read_queue);
-		buffer_prepare_copy(b, toread);
+	} else {
+		toread = 4096;
 	}
+	chunkqueue_get_memory(con->read_queue, &mem, &mem_len, 0, toread);
 
-	read_offset = (b->used == 0) ? 0 : b->used - 1;
-	len = read(con->fd, b->ptr + read_offset, b->size - 1 - read_offset);
+	len = read(con->fd, mem, mem_len);
 #endif
 
+	chunkqueue_use_memory(con->read_queue, len > 0 ? len : 0);
+
 	if (len < 0) {
 		con->is_readable = 0;
 
@@ -394,16 +372,12 @@ static int connection_handle_read(server *srv, connection *con) {
 		/* pipelining */
 
 		return -2;
-	} else if ((size_t)len < b->size - 1) {
+	} else if (len != (ssize_t) mem_len) {
 		/* we got less then expected, wait for the next fd-event */
 
 		con->is_readable = 0;
 	}
 
-	if (b->used > 0) b->used--;
-	b->used += len;
-	b->ptr[b->used++] = '\0';
-
 	con->bytes_read += len;
 #if 0
 	dump_packet(b->ptr, len);
@@ -494,7 +468,7 @@ static int connection_handle_write_prepare(server *srv, connection *con) {
 			buffer_reset(con->physical.path);
 
 			con->file_finished = 1;
-			b = chunkqueue_get_append_buffer(con->write_queue);
+			b = buffer_init();
 
 			/* build default error-page */
 			buffer_copy_string_len(b, CONST_STR_LEN(
@@ -522,6 +496,10 @@ static int connection_handle_write_prepare(server *srv, connection *con) {
 					     "</html>\n"
 					     ));
 
+			http_chunk_append_buffer(srv, con, b);
+			buffer_free(b);
+			http_chunk_close(srv, con);
+
 			response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_STR_LEN("text/html"));
 		}
 		break;
@@ -1029,119 +1007,10 @@ found_header_end:
 		}
 		break;
 	case CON_STATE_READ_POST:
-		for (c = cq->first; c && (dst_cq->bytes_in != (off_t)con->request.content_length); c = c->next) {
-			off_t weWant, weHave, toRead;
-
-			weWant = con->request.content_length - dst_cq->bytes_in;
-
-			force_assert(c->mem->used);
-
-			weHave = c->mem->used - c->offset - 1;
-
-			toRead = weHave > weWant ? weWant : weHave;
-
-			/* the new way, copy everything into a chunkqueue whcih might use tempfiles */
-			if (con->request.content_length > 64 * 1024) {
-				chunk *dst_c = NULL;
-				/* copy everything to max 1Mb sized tempfiles */
-
-				/*
-				 * if the last chunk is
-				 * - smaller than 1Mb (size < 1Mb)
-				 * - not read yet (offset == 0)
-				 * -> append to it
-				 * otherwise
-				 * -> create a new chunk
-				 *
-				 * */
-
-				if (dst_cq->last &&
-				    dst_cq->last->type == FILE_CHUNK &&
-				    dst_cq->last->file.is_temp &&
-				    dst_cq->last->offset == 0) {
-					/* ok, take the last chunk for our job */
-
-			 		if (dst_cq->last->file.length < 1 * 1024 * 1024) {
-						dst_c = dst_cq->last;
-
-						if (dst_c->file.fd == -1) {
-							/* this should not happen as we cache the fd, but you never know */
-							dst_c->file.fd = open(dst_c->file.name->ptr, O_WRONLY | O_APPEND);
-							fd_close_on_exec(dst_c->file.fd);
-						}
-					} else {
-						/* the chunk is too large now, close it */
-						dst_c = dst_cq->last;
-
-						if (dst_c->file.fd != -1) {
-							close(dst_c->file.fd);
-							dst_c->file.fd = -1;
-						}
-						dst_c = chunkqueue_get_append_tempfile(dst_cq);
-					}
-				} else {
-					dst_c = chunkqueue_get_append_tempfile(dst_cq);
-				}
-
-				/* we have a chunk, let's write to it */
-
-				if (dst_c->file.fd == -1) {
-					/* we don't have file to write to,
-					 * EACCES might be one reason.
-					 *
-					 * Instead of sending 500 we send 413 and say the request is too large
-					 *  */
-
-					log_error_write(srv, __FILE__, __LINE__, "sbs",
-							"denying upload as opening to temp-file for upload failed:",
-							dst_c->file.name, strerror(errno));
-
-					con->http_status = 413; /* Request-Entity too large */
-					con->keep_alive = 0;
-					connection_set_state(srv, con, CON_STATE_HANDLE_REQUEST);
-
-					break;
-				}
-
-				if (toRead != write(dst_c->file.fd, c->mem->ptr + c->offset, toRead)) {
-					/* write failed for some reason ... disk full ? */
-					log_error_write(srv, __FILE__, __LINE__, "sbs",
-							"denying upload as writing to file failed:",
-							dst_c->file.name, strerror(errno));
-
-					con->http_status = 413; /* Request-Entity too large */
-					con->keep_alive = 0;
-					connection_set_state(srv, con, CON_STATE_HANDLE_REQUEST);
-
-					close(dst_c->file.fd);
-					dst_c->file.fd = -1;
-
-					break;
-				}
-
-				dst_c->file.length += toRead;
-
-				if (dst_cq->bytes_in + toRead == (off_t)con->request.content_length) {
-					/* we read everything, close the chunk */
-					close(dst_c->file.fd);
-					dst_c->file.fd = -1;
-				}
-			} else {
-				buffer *b;
-
-				if (dst_cq->last &&
-				    dst_cq->last->type == MEM_CHUNK) {
-					b = dst_cq->last->mem;
-				} else {
-					b = chunkqueue_get_append_buffer(dst_cq);
-					/* prepare buffer size for remaining POST data; is < 64kb */
-					buffer_prepare_copy(b, con->request.content_length - dst_cq->bytes_in);
-				}
-				buffer_append_string_len(b, c->mem->ptr + c->offset, toRead);
-			}
-
-			c->offset += toRead;
-			dst_cq->bytes_in += toRead;
+		if (0 != chunkqueue_steal_with_tempfiles(srv, dst_cq, cq, con->request.content_length - dst_cq->bytes_in )) {
+			con->http_status = 413; /* Request-Entity too large */
+			con->keep_alive = 0;
+			connection_set_state(srv, con, CON_STATE_HANDLE_REQUEST);
 		}
 
 		/* Content is ready */

src/mod_compress.c

@@ -583,7 +583,6 @@ static int deflate_file_to_buffer(server *srv, connection *con, plugin_data *p,
 	int ifd;
 	int ret = -1;
 	void *start;
-	buffer *b;
 
 	/* overflow */
 	if ((off_t)(sce->st.st_size * 1.1) < sce->st.st_size) return -1;
@@ -651,8 +650,7 @@ static int deflate_file_to_buffer(server *srv, connection *con, plugin_data *p,
 	if (ret != 0) return -1;
 
 	chunkqueue_reset(con->write_queue);
-	b = chunkqueue_get_append_buffer(con->write_queue);
-	buffer_copy_string_len(b, p->b->ptr, p->b->used);
+	chunkqueue_append_mem(con->write_queue, p->b->ptr, p->b->used);
 
 	buffer_reset(con->physical.path);
 

src/mod_dirlisting.c

@@ -784,7 +784,7 @@ static int http_list_directory(server *srv, connection *con, plugin_data *p, buf
 
 	if (files.used) http_dirls_sort(files.ent, files.used);
 
-	out = chunkqueue_get_append_buffer(con->write_queue);
+	out = buffer_init();
 	buffer_copy_string_len(out, CONST_STR_LEN("<?xml version=\"1.0\" encoding=\""));
 	if (buffer_string_is_empty(p->conf.encoding)) {
 		buffer_append_string_len(out, CONST_STR_LEN("iso-8859-1"));
@@ -899,6 +899,8 @@ static int http_list_directory(server *srv, connection *con, plugin_data *p, buf
 	}
 
 	con->file_finished = 1;
+	chunkqueue_append_buffer(con->write_queue, out);
+	buffer_free(out);
 
 	return 0;
 }

src/mod_fastcgi.c

@@ -52,13 +52,6 @@
 
 #include "version.h"
 
-#define FCGI_ENV_ADD_CHECK(ret, con) \
-	if (ret == -1) { \
-		con->http_status = 400; \
-		con->file_finished = 1; \
-		return -1; \
-	};
-
 /*
  *
  * TODO:
@@ -1769,6 +1762,12 @@ static connection_result_t fcgi_establish_connection(server *srv, handler_ctx *h
 	return CONNECTION_OK;
 }
 
+#define FCGI_ENV_ADD_CHECK(ret, con) \
+	if (ret == -1) { \
+		con->http_status = 400; \
+		con->file_finished = 1; \
+		return -1; \
+	};
 static int fcgi_env_add_request_headers(server *srv, connection *con, plugin_data *p) {
 	size_t i;
 
@@ -1834,11 +1833,9 @@ static int fcgi_env_add_request_headers(server *srv, connection *con, plugin_dat
 	return 0;
 }
 
-
 static int fcgi_create_env(server *srv, handler_ctx *hctx, size_t request_id) {
 	FCGI_BeginRequestRecord beginRecord;
 	FCGI_Header header;
-	buffer *b;
 
 	char buf[LI_ITOSTRING_LENGTH];
 	const char *s;
@@ -1863,10 +1860,6 @@ static int fcgi_create_env(server *srv, handler_ctx *hctx, size_t request_id) {
 	beginRecord.body.flags = 0;
 	memset(beginRecord.body.reserved, 0, sizeof(beginRecord.body.reserved));
 
-	b = chunkqueue_get_append_buffer(hctx->wb);
-
-	buffer_copy_string_len(b, (const char *)&beginRecord, sizeof(beginRecord));
-
 	/* send FCGI_PARAMS */
 	buffer_prepare_copy(p->fcgi_env, 1024);
 
@@ -2054,14 +2047,22 @@ static int fcgi_create_env(server *srv, handler_ctx *hctx, size_t request_id) {
 
 	FCGI_ENV_ADD_CHECK(fcgi_env_add_request_headers(srv, con, p), con);
 
-	fcgi_header(&(header), FCGI_PARAMS, request_id, p->fcgi_env->used, 0);
-	buffer_append_string_len(b, (const char *)&header, sizeof(header));
-	buffer_append_string_len(b, (const char *)p->fcgi_env->ptr, p->fcgi_env->used);
+	{
+		buffer *b = buffer_init();
 
-	fcgi_header(&(header), FCGI_PARAMS, request_id, 0, 0);
-	buffer_append_string_len(b, (const char *)&header, sizeof(header));
+		buffer_copy_string_len(b, (const char *)&beginRecord, sizeof(beginRecord));
 
-	hctx->wb->bytes_in += b->used - 1;
+		fcgi_header(&(header), FCGI_PARAMS, request_id, p->fcgi_env->used, 0);
+		buffer_append_string_len(b, (const char *)&header, sizeof(header));
+		buffer_append_string_len(b, (const char *)p->fcgi_env->ptr, p->fcgi_env->used);
+
+		fcgi_header(&(header), FCGI_PARAMS, request_id, 0, 0);
+		buffer_append_string_len(b, (const char *)&header, sizeof(header));
+
+		hctx->wb->bytes_in += b->used - 1;
+		chunkqueue_append_buffer(hctx->wb, b);
+		buffer_free(b);
+	}
 
 	if (con->request.content_length) {
 		chunkqueue *req_cq = con->request_content_queue;
@@ -2433,38 +2434,21 @@ static int fcgi_demux_response(server *srv, handler_ctx *hctx) {
 		return -1;
 	}
 
-	/* init read-buffer */
-
 	if (toread > 0) {
-		buffer *b;
-		chunk *cq_first = hctx->rb->first;
-		chunk *cq_last = hctx->rb->last;
-
-		b = chunkqueue_get_append_buffer(hctx->rb);
-		buffer_prepare_copy(b, toread + 1);
-
-		/* append to read-buffer */
-		if (-1 == (r = read(hctx->fd, b->ptr, toread))) {
-			if (errno == EAGAIN) {
-				/* roll back the last chunk allocation,
-                                   and continue on next iteration        */
-				buffer_free(hctx->rb->last->mem);
-				free(hctx->rb->last);
-				hctx->rb->first = cq_first;
-				hctx->rb->last = cq_last;
-				return 0;
-			}
+		char *mem;
+		size_t mem_len;
+
+		chunkqueue_get_memory(hctx->rb, &mem, &mem_len, 0, toread);
+		r = read(hctx->fd, mem, mem_len);
+		chunkqueue_use_memory(hctx->rb, r > 0 ? r : 0);
+
+		if (-1 == r) {
+			if (errno == EAGAIN) return 0;
 			log_error_write(srv, __FILE__, __LINE__, "sds",
 					"unexpected end-of-file (perhaps the fastcgi process died):",
 					fcgi_fd, strerror(errno));
 			return -1;
 		}
-
-		/* this should be catched by the b > 0 above */
-		force_assert(r);
-
-		b->used = r + 1; /* one extra for the fake \0 */
-		b->ptr[b->used - 1] = '\0';
 	} else {
 		log_error_write(srv, __FILE__, __LINE__, "ssdsb",
 				"unexpected end-of-file (perhaps the fastcgi process died):",
@@ -2973,8 +2957,8 @@ static handler_t fcgi_write_request(server *srv, handler_ctx *hctx) {
 					"fcgi-request is already in use:", hctx->request_id);
 		}
 
-		/* fall through */
 		if (-1 == fcgi_create_env(srv, hctx, hctx->request_id)) return HANDLER_ERROR;
+
 		fcgi_set_state(srv, hctx, FCGI_STATE_WRITE);
 		/* fall through */
 	case FCGI_STATE_WRITE:

src/mod_flv_streaming.c

@@ -207,7 +207,6 @@ URIHANDLER_FUNC(mod_flv_streaming_path_handler) {
 		if (0 == strncmp(con->physical.path->ptr + s_len - ct_len, ds->value->ptr, ct_len)) {
 			data_string *get_param;
 			stat_cache_entry *sce = NULL;
-			buffer *b;
 			int start;
 			char *err = NULL;
 			/* if there is a start=[0-9]+ in the header use it as start,
@@ -242,10 +241,9 @@ URIHANDLER_FUNC(mod_flv_streaming_path_handler) {
 			}
 
 			/* we are safe now, let's build a flv header */
-			b = chunkqueue_get_append_buffer(con->write_queue);
-			buffer_copy_string_len(b, CONST_STR_LEN("FLV\x1\x1\0\0\0\x9\0\0\0\x9"));
-
+			http_chunk_append_mem(srv, con, CONST_STR_LEN("FLV\x1\x1\0\0\0\x9\0\0\0\x9"));
 			http_chunk_append_file(srv, con, con->physical.path, start, sce->st.st_size - start);
+			http_chunk_close(srv, con);
 
 			response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_STR_LEN("video/x-flv"));
 

src/mod_proxy.c

@@ -449,7 +449,7 @@ static int proxy_create_env(server *srv, handler_ctx *hctx) {
 
 	/* build header */
 
-	b = chunkqueue_get_append_buffer(hctx->wb);
+	b = buffer_init();
 
 	/* request line */
 	buffer_copy_string(b, get_http_method_name(con->request.http_method));
@@ -486,6 +486,9 @@ static int proxy_create_env(server *srv, handler_ctx *hctx) {
 	buffer_append_string_len(b, CONST_STR_LEN("\r\n"));
 
 	hctx->wb->bytes_in += b->used - 1;
+	chunkqueue_append_buffer(hctx->wb, b);
+	buffer_free(b);
+
 	/* body */
 
 	if (con->request.content_length) {

src/mod_scgi.c

@@ -1629,7 +1629,7 @@ static int scgi_create_env(server *srv, handler_ctx *hctx) {
 
 	scgi_env_add_request_headers(srv, con, p);
 
-	b = chunkqueue_get_append_buffer(hctx->wb);
+	b = buffer_init();
 
 	buffer_append_int(b, p->scgi_env->used);
 	buffer_append_string_len(b, CONST_STR_LEN(":"));
@@ -1637,6 +1637,8 @@ static int scgi_create_env(server *srv, handler_ctx *hctx) {
 	buffer_append_string_len(b, CONST_STR_LEN(","));
 
 	hctx->wb->bytes_in += b->used - 1;
+	chunkqueue_append_buffer(hctx->wb, b);
+	buffer_free(b);
 
 	if (con->request.content_length) {
 		chunkqueue *req_cq = con->request_content_queue;

src/mod_ssi.c

@@ -430,7 +430,7 @@ static int process_ssi_stmt(server *srv, connection *con, plugin_data *p, const
 		case SSI_ECHO_USER_NAME: {
 			struct passwd *pw;
 
-			b = chunkqueue_get_append_buffer(con->write_queue);
+			b = buffer_init();
 #ifdef HAVE_PWD_H
 			if (NULL == (pw = getpwuid(sce->st.st_uid))) {
 				buffer_copy_int(b, sce->st.st_uid);
@@ -440,67 +440,62 @@ static int process_ssi_stmt(server *srv, connection *con, plugin_data *p, const
 #else
 			buffer_copy_int(b, sce->st.st_uid);
 #endif
+			chunkqueue_append_buffer(con->write_queue, b);
+			buffer_free(b);
 			break;
 		}
 		case SSI_ECHO_LAST_MODIFIED:	{
 			time_t t = sce->st.st_mtime;
 
-			b = chunkqueue_get_append_buffer(con->write_queue);
 			if (0 == strftime(buf, sizeof(buf), p->timefmt->ptr, localtime(&t))) {
-				buffer_copy_string_len(b, CONST_STR_LEN("(none)"));
+				chunkqueue_append_mem(con->write_queue, CONST_STR_LEN("(none)"));
 			} else {
-				buffer_copy_string(b, buf);
+				chunkqueue_append_mem(con->write_queue, buf, strlen(buf));
 			}
 			break;
 		}
 		case SSI_ECHO_DATE_LOCAL: {
 			time_t t = time(NULL);
 
-			b = chunkqueue_get_append_buffer(con->write_queue);
 			if (0 == strftime(buf, sizeof(buf), p->timefmt->ptr, localtime(&t))) {
-				buffer_copy_string_len(b, CONST_STR_LEN("(none)"));
+				chunkqueue_append_mem(con->write_queue, CONST_STR_LEN("(none)"));
 			} else {
-				buffer_copy_string(b, buf);
+				chunkqueue_append_mem(con->write_queue, buf, strlen(buf));
 			}
 			break;
 		}
 		case SSI_ECHO_DATE_GMT: {
 			time_t t = time(NULL);
 
-			b = chunkqueue_get_append_buffer(con->write_queue);
 			if (0 == strftime(buf, sizeof(buf), p->timefmt->ptr, gmtime(&t))) {
-				buffer_copy_string_len(b, CONST_STR_LEN("(none)"));
+				chunkqueue_append_mem(con->write_queue, CONST_STR_LEN("(none)"));
 			} else {
-				buffer_copy_string(b, buf);
+				chunkqueue_append_mem(con->write_queue, buf, strlen(buf));
 			}
 			break;
 		}
 		case SSI_ECHO_DOCUMENT_NAME: {
 			char *sl;
 
-			b = chunkqueue_get_append_buffer(con->write_queue);
 			if (NULL == (sl = strrchr(con->physical.path->ptr, '/'))) {
-				buffer_copy_buffer(b, con->physical.path);
+				chunkqueue_append_mem(con->write_queue, CONST_BUF_LEN(con->physical.path));
 			} else {
-				buffer_copy_string(b, sl + 1);
+				chunkqueue_append_mem(con->write_queue, sl + 1, strlen(sl + 1));
 			}
 			break;
 		}
 		case SSI_ECHO_DOCUMENT_URI: {
-			b = chunkqueue_get_append_buffer(con->write_queue);
-			buffer_copy_buffer(b, con->uri.path);
+			chunkqueue_append_mem(con->write_queue, CONST_BUF_LEN(con->uri.path));
 			break;
 		}
 		default: {
 			data_string *ds;
 			/* check if it is a cgi-var */
 
-			b = chunkqueue_get_append_buffer(con->write_queue);
-
 			if (NULL != (ds = (data_string *)array_get_element(p->ssi_cgi_env, var_val))) {
-				buffer_copy_buffer(b, ds->value);
+				chunkqueue_append_mem(con->write_queue, CONST_BUF_LEN(ds->value));
 			} else {
-				buffer_copy_string_len(b, CONST_STR_LEN("(none)"));
+				chunkqueue_append_mem(con->write_queue, CONST_STR_LEN("(none)"));
 			}
 
 			break;
@@ -583,7 +578,7 @@ static int process_ssi_stmt(server *srv, connection *con, plugin_data *p, const
 
 			switch (ssicmd) {
 			case SSI_FSIZE:
-				b = chunkqueue_get_append_buffer(con->write_queue);
+				b = buffer_init();
 				if (p->sizefmt) {
 					int j = 0;
 					const char *abr[] = { " B", " kB", " MB", " GB", " TB", NULL };
@@ -597,13 +592,14 @@ static int process_ssi_stmt(server *srv, connection *con, plugin_data *p, const
 				} else {
 					buffer_copy_int(b, st.st_size);
 				}
+				chunkqueue_append_buffer(con->write_queue, b);
+				buffer_free(b);
 				break;
 			case SSI_FLASTMOD:
-				b = chunkqueue_get_append_buffer(con->write_queue);
 				if (0 == strftime(buf, sizeof(buf), p->timefmt->ptr, localtime(&t))) {
-					buffer_copy_string_len(b, CONST_STR_LEN("(none)"));
+					chunkqueue_append_mem(con->write_queue, CONST_STR_LEN("(none)"));
 				} else {
-					buffer_copy_string(b, buf);
+					chunkqueue_append_mem(con->write_queue, buf, strlen(buf));
 				}
 				break;
 			case SSI_INCLUDE:
@@ -611,7 +607,7 @@ static int process_ssi_stmt(server *srv, connection *con, plugin_data *p, const
 
 				/* Keep the newest mtime of included files */
 				if (st.st_mtime > include_file_last_mtime)
-				  include_file_last_mtime = st.st_mtime;
+					include_file_last_mtime = st.st_mtime;
 
 				break;
 			}
@@ -683,7 +679,7 @@ static int process_ssi_stmt(server *srv, connection *con, plugin_data *p, const
 	case SSI_PRINTENV:
 		if (p->if_is_false) break;
 
-		b = chunkqueue_get_append_buffer(con->write_queue);
+		b = buffer_init();
 		for (i = 0; i < p->ssi_vars->used; i++) {
 			data_string *ds = (data_string *)p->ssi_vars->data[p->ssi_vars->sorted[i]];
 
@@ -700,6 +696,8 @@ static int process_ssi_stmt(server *srv, connection *con, plugin_data *p, const
 			buffer_append_string_encoded(b, CONST_BUF_LEN(ds->value), ENCODING_MINIMAL_XML);
 			buffer_append_string_len(b, CONST_STR_LEN("\n"));
 		}
+		chunkqueue_append_buffer(con->write_queue, b);
+		buffer_free(b);
 
 		break;
 	case SSI_EXEC: {
@@ -791,17 +789,14 @@ static int process_ssi_stmt(server *srv, connection *con, plugin_data *p, const
 						}
 
 						if (toread > 0) {
-							b = chunkqueue_get_append_buffer(con->write_queue);
+							char *mem;
+							size_t mem_len;
 
-							buffer_prepare_copy(b, toread);
+							chunkqueue_get_memory(con->write_queue, &mem, &mem_len, 0, toread);
+							r = read(from_exec_fds[0], mem, mem_len);
+							chunkqueue_use_memory(con->write_queue, r > 0 ? r : 0);
 
-							if ((r = read(from_exec_fds[0], b->ptr, b->size - 1)) < 0) {
-								/* read failed */
-								break;
-							} else {
-								b->used = r;
-								b->ptr[b->used++] = '\0';
-							}
+							if (r < 0) break; /* read failed */
 						} else {
 							break;
 						}

src/mod_staticfile.c

@@ -285,9 +285,7 @@ static int http_response_parse_range(server *srv, connection *con, plugin_data *
 		if (!error) {
 			if (multipart) {
 				/* write boundary-header */
-				buffer *b;
-
-				b = chunkqueue_get_append_buffer(con->write_queue);
+				buffer *b = buffer_init();
 
 				buffer_copy_string_len(b, CONST_STR_LEN("\r\n--"));
 				buffer_append_string(b, boundary);
@@ -307,7 +305,8 @@ static int http_response_parse_range(server *srv, connection *con, plugin_data *
 				buffer_append_string_len(b, CONST_STR_LEN("\r\n\r\n"));
 
 				con->response.content_length += b->used - 1;
-
+				chunkqueue_append_buffer(con->write_queue, b);
+				buffer_free(b);
 			}
 
 			chunkqueue_append_file(con->write_queue, con->physical.path, start, end - start + 1);
@@ -320,15 +319,15 @@ static int http_response_parse_range(server *srv, connection *con, plugin_data *
 
 	if (multipart) {
 		/* add boundary end */
-		buffer *b;
-
-		b = chunkqueue_get_append_buffer(con->write_queue);
+		buffer *b = buffer_init();
 
 		buffer_copy_string_len(b, "\r\n--", 4);
 		buffer_append_string(b, boundary);
 		buffer_append_string_len(b, "--\r\n", 4);
 
 		con->response.content_length += b->used - 1;
+		chunkqueue_append_buffer(con->write_queue, b);
+		buffer_free(b);
 
 		/* set header-fields */
 

src/mod_status.c

@@ -199,7 +199,7 @@ static int mod_status_get_multiplier(double *avg, char *multiplier, int size) {
 
 static handler_t mod_status_handle_server_status_html(server *srv, connection *con, void *p_d) {
 	plugin_data *p = p_d;
-	buffer *b;
+	buffer *b = buffer_init();
 	size_t j;
 	double avg;
 	char multiplier = '\0';
@@ -208,8 +208,6 @@ static handler_t mod_status_handle_server_status_html(server *srv, connection *c
 
 	int days, hours, mins, seconds;
 
-	b = chunkqueue_get_append_buffer(con->write_queue);
-
 	buffer_copy_string_len(b, CONST_STR_LEN(
 				 "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n"
 				 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\"\n"
@@ -555,6 +553,9 @@ static handler_t mod_status_handle_server_status_html(server *srv, connection *c
 		      "</html>\n"
 		      ));
 
+	chunkqueue_append_buffer(con->write_queue, b);
+	buffer_free(b);
+
 	response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_STR_LEN("text/html"));
 
 	return 0;
@@ -563,15 +564,13 @@ static handler_t mod_status_handle_server_status_html(server *srv, connection *c
 
 static handler_t mod_status_handle_server_status_text(server *srv, connection *con, void *p_d) {
 	plugin_data *p = p_d;
-	buffer *b;
+	buffer *b = buffer_init();
 	double avg;
 	time_t ts;
 	char buf[32];
 	unsigned int k;
 	unsigned int l;
 
-	b = chunkqueue_get_append_buffer(con->write_queue);
-
 	/* output total number of requests */
 	buffer_append_string_len(b, CONST_STR_LEN("Total Accesses: "));
 	avg = p->abs_requests;
@@ -598,13 +597,13 @@ static handler_t mod_status_handle_server_status_text(server *srv, connection *c
 	buffer_append_string_len(b, CONST_STR_LEN("\n"));
 
 	buffer_append_string_len(b, CONST_STR_LEN("IdleServers: "));
-       buffer_append_int(b, srv->conns->size - srv->conns->used);
-       buffer_append_string_len(b, CONST_STR_LEN("\n"));
+	buffer_append_int(b, srv->conns->size - srv->conns->used);
+	buffer_append_string_len(b, CONST_STR_LEN("\n"));
 
-       /* output scoreboard */
-       buffer_append_string_len(b, CONST_STR_LEN("Scoreboard: "));
-       for (k = 0; k < srv->conns->used; k++) {
-        	connection *c = srv->conns->ptr[k];
+	/* output scoreboard */
+	buffer_append_string_len(b, CONST_STR_LEN("Scoreboard: "));
+	for (k = 0; k < srv->conns->used; k++) {
+		connection *c = srv->conns->ptr[k];
 		const char *state = connection_get_short_state(c->state);
 		buffer_append_string_len(b, state, 1);
 	}
@@ -613,15 +612,17 @@ static handler_t mod_status_handle_server_status_text(server *srv, connection *c
 	}
 	buffer_append_string_len(b, CONST_STR_LEN("\n"));
 
-	/* set text/plain output */
+	chunkqueue_append_buffer(con->write_queue, b);
+	buffer_free(b);
 
+	/* set text/plain output */
 	response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_STR_LEN("text/plain"));
 
 	return 0;
 }
 
 static handler_t mod_status_handle_server_statistics(server *srv, connection *con, void *p_d) {
-	buffer *b;
+	buffer *b = buffer_init();
 	size_t i;
 	array *st = srv->status;
 	UNUSED(p_d);
@@ -634,8 +635,6 @@ static handler_t mod_status_handle_server_statistics(server *srv, connection *co
 		return HANDLER_FINISHED;
 	}
 
-	b = chunkqueue_get_append_buffer(con->write_queue);
-
 	for (i = 0; i < st->used; i++) {
 		size_t ndx = st->sorted[i];
 
@@ -645,6 +644,9 @@ static handler_t mod_status_handle_server_statistics(server *srv, connection *co
 		buffer_append_string_len(b, CONST_STR_LEN("\n"));
 	}
 
+	chunkqueue_append_buffer(con->write_queue, b);
+	buffer_free(b);
+
 	response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_STR_LEN("text/plain"));
 
 	con->http_status = 200;
@@ -671,7 +673,8 @@ static handler_t mod_status_handle_server_status(server *srv, connection *con, v
 
 static handler_t mod_status_handle_server_config(server *srv, connection *con, void *p_d) {
 	plugin_data *p = p_d;
-	buffer *b, *m = p->module_list;
+	buffer *b = buffer_init();
+	buffer *m = p->module_list;
 	size_t i;
 
 	struct ev_map { fdevent_handler_t et; const char *name; } event_handlers[] =
@@ -703,8 +706,6 @@ static handler_t mod_status_handle_server_config(server *srv, connection *con, v
 		{ FDEVENT_HANDLER_UNSET,          NULL }
 	};
 
-	b = chunkqueue_get_append_buffer(con->write_queue);
-
 	buffer_copy_string_len(b, CONST_STR_LEN(
 			   "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n"
 			   "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\"\n"
@@ -756,6 +757,9 @@ static handler_t mod_status_handle_server_config(server *srv, connection *con, v
 		      "</html>\n"
 		      ));
 
+	chunkqueue_append_buffer(con->