[core] defer optimization to read small files

defer optimization to read small files into memory until after
response_start hooks have a chance to run, e.g. until after
mod_deflate chooses whether or not to serve file from compressed
cache, if deflate.cache-dir is configured

src/chunk.c

@@ -824,3 +824,49 @@ int chunkqueue_open_file_chunk(chunkqueue * const restrict cq, log_error_st * co
 
 	return 0;
 }
+
+
+void
+chunkqueue_small_resp_optim (chunkqueue * const restrict cq)
+{
+    /*(caller must verify response is small (and non-empty) before calling)*/
+
+    /*(optimization to use fewer syscalls to send a small response by reading
+     * small files into memory, thereby avoiding use of sendfile() and multiple
+     * calls to writev()  (benefit for cleartext (non-TLS) and <= HTTP/1.1))
+     *(If TLS, then will shortly need to be in memory for encryption anyway)*/
+
+    /*assert(cq->first);*/
+    chunk *c = cq->first;
+    chunk * const filec = c->next;
+    if (c->type != MEM_CHUNK || filec != cq->last || filec->type != FILE_CHUNK)
+        return;
+
+    const int fd = filec->file.fd;
+    if (fd < 0) return; /*(require that file already be open)*/
+    off_t offset = filec->file.start + filec->offset;
+    if (0 != offset && -1 == lseek(fd, offset, SEEK_SET)) return;
+
+    /* Note: there should be no size change in chunkqueue,
+     * so cq->bytes_in and cq->bytes_out should not be modified */
+
+    buffer *b = c->mem;
+    off_t len = filec->file.length - filec->offset;
+    if ((size_t)len > chunk_buffer_string_space(b)) {
+        chunk * const nc = chunk_acquire((size_t)len+1);
+        c->next = nc;
+        nc->next = filec;
+        b = nc->mem;
+    }
+
+    char * const ptr = b->ptr + chunk_buffer_string_length(b);
+    ssize_t rd;
+    offset = 0; /*(reuse offset var for offset into mem buffer)*/
+    do {
+        rd = read(fd, ptr+offset, len-offset);
+    } while (rd > 0 ? (offset += rd, len -= rd) : errno == EINTR);
+    /*(contents of chunkqueue kept valid even if error reading from file)*/
+    buffer_commit(b, offset);
+    filec->offset += offset;
+    chunkqueue_remove_empty_chunks(cq);
+}

src/chunk.h

@@ -120,6 +120,8 @@ int chunkqueue_open_file_chunk(chunkqueue * restrict cq, struct log_error_st * c
 
 void chunkqueue_compact_mem(chunkqueue *cq, size_t clen);
 
+void chunkqueue_small_resp_optim (chunkqueue * restrict cq);
+
 __attribute_pure__
 off_t chunkqueue_length(chunkqueue *cq);
 

src/http_chunk.c

@@ -134,7 +134,7 @@ int http_chunk_append_file(request_st * const r, const buffer * const fn) {
 }
 
 int http_chunk_append_file_fd(request_st * const r, const buffer * const fn, const int fd, const off_t sz) {
-    if (sz > 32768) {
+    if (sz > 32768 || !r->resp_send_chunked) {
         http_chunk_append_file_fd_range(r, fn, fd, 0, sz);
         return 0;
     }

src/mod_deflate.c

@@ -1517,8 +1517,9 @@ REQUEST_FUNC(mod_deflate_handle_response_start) {
 	 * must not be chunkqueue temporary file
 	 * must be whole file, not partial content
 	 * Note: small files (< 32k (see http_chunk.c)) will have been read into
-	 *       memory and will end up getting stream-compressed rather than
-	 *       cached on disk as compressed file
+	 *       memory (if streaming HTTP/1.1 chunked response) and will end up
+	 *       getting stream-compressed rather than cached on disk as compressed
+	 *       file
 	 */
 	buffer *tb = NULL;
 	if (!buffer_is_empty(p->conf.cache_dir)

src/response.c

@@ -135,6 +135,13 @@ http_response_write_header (request_st * const r)
 	}
 
 	chunkqueue_prepend_buffer_commit(cq);
+
+	/*(optimization to use fewer syscalls to send a small response)*/
+	off_t cqlen;
+	if (r->resp_body_finished && (r->resp_htags & HTTP_HEADER_CONTENT_LENGTH)
+	    && (cqlen = chunkqueue_length(cq) - r->resp_header_len) > 0
+	    && cqlen <= 32768)
+		chunkqueue_small_resp_optim(cq);
 }