diff --git a/src/base.h b/src/base.h index 5a29606c..f62490b2 100644 --- a/src/base.h +++ b/src/base.h @@ -327,7 +327,6 @@ struct server { /* buffers */ buffer *tmp_buf; - buffer *tmp_chunk_len; connections conns; connections joblist; diff --git a/src/chunk.c b/src/chunk.c index a1e853a2..60f39a50 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -268,7 +268,7 @@ static chunk * chunkqueue_append_mem_chunk(chunkqueue *cq, size_t sz) { } __attribute_returns_nonnull__ -static chunk * chunkqueue_append_file_chunk(chunkqueue *cq, buffer *fn, off_t offset, off_t len) { +static chunk * chunkqueue_append_file_chunk(chunkqueue *cq, const buffer *fn, off_t offset, off_t len) { chunk *c = chunk_acquire(buffer_string_length(fn)+1); chunkqueue_append_chunk(cq, c); c->type = FILE_CHUNK; @@ -286,7 +286,7 @@ void chunkqueue_reset(chunkqueue *cq) { cq->tempdir_idx = 0; } -void chunkqueue_append_file_fd(chunkqueue *cq, buffer *fn, int fd, off_t offset, off_t len) { +void chunkqueue_append_file_fd(chunkqueue *cq, const buffer *fn, int fd, off_t offset, off_t len) { if (len > 0) { (chunkqueue_append_file_chunk(cq, fn, offset, len))->file.fd = fd; } @@ -295,7 +295,7 @@ void chunkqueue_append_file_fd(chunkqueue *cq, buffer *fn, int fd, off_t offset, } } -void chunkqueue_append_file(chunkqueue *cq, buffer *fn, off_t offset, off_t len) { +void chunkqueue_append_file(chunkqueue *cq, const buffer *fn, off_t offset, off_t len) { if (len > 0) { chunkqueue_append_file_chunk(cq, fn, offset, len); } diff --git a/src/chunk.h b/src/chunk.h index f9d905b4..381f18a1 100644 --- a/src/chunk.h +++ b/src/chunk.h @@ -64,8 +64,8 @@ void chunkqueue_set_chunk_size (size_t sz); void chunkqueue_set_tempdirs_default_reset (void); void chunkqueue_set_tempdirs_default (const array *tempdirs, off_t upload_temp_file_size); void chunkqueue_set_tempdirs(chunkqueue *cq, const array *tempdirs, off_t upload_temp_file_size); -void chunkqueue_append_file(chunkqueue *cq, buffer *fn, off_t offset, off_t len); /* copies "fn" */ -void chunkqueue_append_file_fd(chunkqueue *cq, buffer *fn, int fd, off_t offset, off_t len); /* copies "fn" */ +void chunkqueue_append_file(chunkqueue *cq, const buffer *fn, off_t offset, off_t len); /* copies "fn" */ +void chunkqueue_append_file_fd(chunkqueue *cq, const buffer *fn, int fd, off_t offset, off_t len); /* copies "fn" */ void chunkqueue_append_mem(chunkqueue *cq, const char *mem, size_t len); /* copies memory */ void chunkqueue_append_mem_min(chunkqueue *cq, const char * mem, size_t len); /* copies memory */ void chunkqueue_append_buffer(chunkqueue *cq, buffer *mem); /* may reset "mem" */ diff --git a/src/http_chunk.c b/src/http_chunk.c index d939a77c..aa102063 100644 --- a/src/http_chunk.c +++ b/src/http_chunk.c @@ -6,9 +6,9 @@ * */ +#include "http_chunk.h" #include "base.h" #include "chunk.h" -#include "http_chunk.h" #include "stat_cache.h" #include "fdevent.h" #include "log.h" @@ -22,28 +22,46 @@ #include #include -static buffer * http_chunk_header(buffer *b, uintmax_t len) { - buffer_clear(b); - buffer_append_uint_hex(b, len); - buffer_append_string_len(b, CONST_STR_LEN("\r\n")); - return b; +static void http_chunk_len_append(chunkqueue * const cq, uintmax_t len) { + char buf[24]; /* 64-bit (8 bytes) is 16 hex chars (+2 \r\n, +1 \0 = 19) */ + #if 0 + buffer b = { buf, 0, sizeof(buf) }; + buffer_append_uint_hex(&b, len); + buffer_append_string_len(&b, CONST_STR_LEN("\r\n")); + chunkqueue_append_mem(cq, b.ptr, b.used-1); + #else + int i = (int)(sizeof(buf)); + buf[--i] = '\n'; + buf[--i] = '\r'; + do { buf[--i] = "0123456789abcdef"[len & 0x0F]; } while (len >>= 4); + chunkqueue_append_mem(cq, buf+i, sizeof(buf)-i); + #endif } -static void http_chunk_append_len(connection *con, uintmax_t len) { - buffer *b = http_chunk_header(con->srv->tmp_chunk_len, len); - chunkqueue_append_mem(con->write_queue, CONST_BUF_LEN(b)); +static int http_chunk_len_append_tempfile(chunkqueue * const cq, uintmax_t len, log_error_st * const errh) { + char buf[24]; /* 64-bit (8 bytes) is 16 hex chars (+2 \r\n, +1 \0 = 19) */ + #if 0 + buffer b = { buf, 0, sizeof(buf) }; + buffer_append_uint_hex(&b, len); + buffer_append_string_len(&b, CONST_STR_LEN("\r\n")); + return chunkqueue_append_mem_to_tempfile(cq, b.ptr, b.used-1, errh); + #else + int i = (int)(sizeof(buf)); + buf[--i] = '\n'; + buf[--i] = '\r'; + do { buf[--i] = "0123456789abcdef"[len & 0x0F]; } while (len >>= 4); + return chunkqueue_append_mem_to_tempfile(cq, buf+i, sizeof(buf)-i, errh); + #endif } -static int http_chunk_append_file_open_fstat(connection *con, buffer *fn, struct stat *st) { - if (!con->conf.follow_symlink - && 0 != stat_cache_path_contains_symlink(con, fn)) { - return -1; - } - - return stat_cache_open_rdonly_fstat(fn, st, con->conf.follow_symlink); +static int http_chunk_append_file_open_fstat(connection * const con, const buffer * const fn, struct stat * const st) { + return + (con->conf.follow_symlink || !stat_cache_path_contains_symlink(con, fn)) + ? stat_cache_open_rdonly_fstat(fn, st, con->conf.follow_symlink) + : -1; } -static int http_chunk_append_read_fd_range(connection *con, buffer *fn, int fd, off_t offset, off_t len) { +static int http_chunk_append_read_fd_range(connection * const con, const buffer * const fn, const int fd, off_t offset, off_t len) { /* note: this routine should not be used for range requests * unless the total size of ranges requested is small */ /* note: future: could read into existing MEM_CHUNK in cq->last if @@ -51,12 +69,12 @@ static int http_chunk_append_read_fd_range(connection *con, buffer *fn, int fd, * offset in for cq->bytes_in in chunkqueue_append_buffer_commit() */ UNUSED(fn); - if (con->response.send_chunked) { - http_chunk_append_len(con, (uintmax_t)len); - } + chunkqueue * const cq = con->write_queue; + + if (con->response.send_chunked) + http_chunk_len_append(cq, (uintmax_t)len); if (0 != offset && -1 == lseek(fd, offset, SEEK_SET)) return -1; - chunkqueue * const cq = con->write_queue; buffer * const b = chunkqueue_append_buffer_open_sz(cq, len+2); ssize_t rd; offset = 0; @@ -65,210 +83,186 @@ static int http_chunk_append_read_fd_range(connection *con, buffer *fn, int fd, } while (rd > 0 ? (offset += rd, len -= rd) : errno == EINTR); buffer_commit(b, offset); - if (con->response.send_chunked) { + if (con->response.send_chunked) buffer_append_string_len(b, CONST_STR_LEN("\r\n")); - } chunkqueue_append_buffer_commit(cq); return (rd >= 0) ? 0 : -1; } -static void http_chunk_append_file_fd_range(connection *con, buffer *fn, int fd, off_t offset, off_t len) { - chunkqueue *cq = con->write_queue; +static void http_chunk_append_file_fd_range(connection * const con, const buffer * const fn, const int fd, const off_t offset, const off_t len) { + chunkqueue * const cq = con->write_queue; - if (con->response.send_chunked) { - http_chunk_append_len(con, (uintmax_t)len); - } + if (con->response.send_chunked) + http_chunk_len_append(cq, (uintmax_t)len); - chunkqueue_append_file_fd(cq, fn, fd, offset, len); + chunkqueue_append_file_fd(cq, fn, fd, offset, len); - if (con->response.send_chunked) { - chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); - } + if (con->response.send_chunked) + chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); } -int http_chunk_append_file_range(connection *con, buffer *fn, off_t offset, off_t len) { - struct stat st; - const int fd = http_chunk_append_file_open_fstat(con, fn, &st); - if (fd < 0) return -1; - - if (-1 == len) { - if (offset >= st.st_size) { - close(fd); - return (offset == st.st_size) ? 0 : -1; - } - len = st.st_size - offset; - } else if (st.st_size - offset < len) { - close(fd); - return -1; - } - - http_chunk_append_file_fd_range(con, fn, fd, offset, len); - return 0; -} +int http_chunk_append_file_range(connection * const con, const buffer * const fn, const off_t offset, off_t len) { + struct stat st; + const int fd = http_chunk_append_file_open_fstat(con, fn, &st); + if (fd < 0) return -1; -int http_chunk_append_file(connection *con, buffer *fn) { - struct stat st; - const int fd = http_chunk_append_file_open_fstat(con, fn, &st); - if (fd < 0) return -1; - http_chunk_append_file_fd(con, fn, fd, st.st_size); - return 0; -} + if (-1 == len) { + if (offset >= st.st_size) { + close(fd); + return (offset == st.st_size) ? 0 : -1; + } + len = st.st_size - offset; + } + else if (st.st_size - offset < len) { + close(fd); + return -1; + } -int http_chunk_append_file_fd(connection *con, buffer *fn, int fd, off_t sz) { - if (sz > 32768) { - http_chunk_append_file_fd_range(con, fn, fd, 0, sz); - return 0; - } else { - int rc = (0 != sz) /*(read small files into memory)*/ - ? http_chunk_append_read_fd_range(con, fn, fd, 0, sz) - : 0; - close(fd); - return rc; - } + http_chunk_append_file_fd_range(con, fn, fd, offset, len); + return 0; } -static int http_chunk_append_to_tempfile(connection *con, const char * mem, size_t len) { - chunkqueue * const cq = con->write_queue; - log_error_st * const errh = con->conf.errh; - - if (con->response.send_chunked) { - buffer *b = http_chunk_header(con->srv->tmp_chunk_len, len); - if (0 != chunkqueue_append_mem_to_tempfile(cq, CONST_BUF_LEN(b), errh)) { - return -1; - } - } - - if (0 != chunkqueue_append_mem_to_tempfile(cq, mem, len, errh)) { - return -1; - } +int http_chunk_append_file(connection * const con, const buffer * const fn) { + struct stat st; + const int fd = http_chunk_append_file_open_fstat(con, fn, &st); + if (fd < 0) return -1; + http_chunk_append_file_fd(con, fn, fd, st.st_size); + return 0; +} - if (con->response.send_chunked) { - if (0 != chunkqueue_append_mem_to_tempfile(cq, CONST_STR_LEN("\r\n"), errh)) { - return -1; - } - } +int http_chunk_append_file_fd(connection * const con, const buffer * const fn, const int fd, const off_t sz) { + if (sz > 32768) { + http_chunk_append_file_fd_range(con, fn, fd, 0, sz); + return 0; + } - return 0; + /*(read small files into memory)*/ + int rc = (0 != sz) ? http_chunk_append_read_fd_range(con,fn,fd,0,sz) : 0; + close(fd); + return rc; } -static int http_chunk_append_cq_to_tempfile(connection *con, chunkqueue *src, size_t len) { +static int http_chunk_append_to_tempfile(connection * const con, const char * const mem, const size_t len) { chunkqueue * const cq = con->write_queue; log_error_st * const errh = con->conf.errh; - if (con->response.send_chunked) { - buffer *b = http_chunk_header(con->srv->tmp_chunk_len, len); - if (0 != chunkqueue_append_mem_to_tempfile(cq, CONST_BUF_LEN(b), errh)) { - return -1; - } - } + if (con->response.send_chunked + && 0 != http_chunk_len_append_tempfile(cq, len, errh)) + return -1; - if (0 != chunkqueue_steal_with_tempfiles(cq, src, len, errh)) { + if (0 != chunkqueue_append_mem_to_tempfile(cq, mem, len, errh)) return -1; - } - if (con->response.send_chunked) { - if (0 != - chunkqueue_append_mem_to_tempfile(cq,CONST_STR_LEN("\r\n"),errh)) { - return -1; - } - } + if (con->response.send_chunked + && 0 != + chunkqueue_append_mem_to_tempfile(cq, CONST_STR_LEN("\r\n"), errh)) + return -1; return 0; } -static int http_chunk_uses_tempfile(connection *con, size_t len) { - chunkqueue * const cq = con->write_queue; - chunk *c = cq->last; +static int http_chunk_append_cq_to_tempfile(connection * const con, chunkqueue * const src, const size_t len) { + chunkqueue * const cq = con->write_queue; + log_error_st * const errh = con->conf.errh; - /* current usage does not append_mem or append_buffer after appending - * file, so not checking if users of this interface have appended large - * (references to) files to chunkqueue, which would not be in memory - * (but included in calculation for whether or not to use temp file) */ + if (con->response.send_chunked + && 0 != http_chunk_len_append_tempfile(cq, len, errh)) + return -1; - /*(allow slightly larger mem use if FDEVENT_STREAM_RESPONSE_BUFMIN - * to reduce creation of temp files when backend producer will be - * blocked until more data is sent to network to client)*/ + if (0 != chunkqueue_steal_with_tempfiles(cq, src, len, errh)) + return -1; + + if (con->response.send_chunked + && 0 != + chunkqueue_append_mem_to_tempfile(cq, CONST_STR_LEN("\r\n"), errh)) + return -1; - if ((c && c->type == FILE_CHUNK && c->file.is_temp) - || cq->bytes_in - cq->bytes_out + len - > 1024 * ((con->conf.stream_response_body & FDEVENT_STREAM_RESPONSE_BUFMIN) ? 128 : 64)) { - return 1; - } + return 0; +} - return 0; +__attribute_pure__ +static int http_chunk_uses_tempfile(const connection * const con, const chunkqueue * const cq, const size_t len) { + + /* current usage does not append_mem or append_buffer after appending + * file, so not checking if users of this interface have appended large + * (references to) files to chunkqueue, which would not be in memory + * (but included in calculation for whether or not to use temp file) */ + + /*(allow slightly larger mem use if FDEVENT_STREAM_RESPONSE_BUFMIN + * to reduce creation of temp files when backend producer will be + * blocked until more data is sent to network to client)*/ + + const chunk * const c = cq->last; + return + ((c && c->type == FILE_CHUNK && c->file.is_temp) + || cq->bytes_in - cq->bytes_out + len + > ((con->conf.stream_response_body & FDEVENT_STREAM_RESPONSE_BUFMIN) + ? 128*1024 + : 64*1024)); } -int http_chunk_append_buffer(connection *con, buffer *mem) { - chunkqueue * const cq = con->write_queue; +int http_chunk_append_buffer(connection * const con, buffer * const mem) { size_t len = buffer_string_length(mem); if (0 == len) return 0; - if (http_chunk_uses_tempfile(con, len)) { + chunkqueue * const cq = con->write_queue; + + if (http_chunk_uses_tempfile(con, cq, len)) return http_chunk_append_to_tempfile(con, mem->ptr, len); - } - if (con->response.send_chunked) { - http_chunk_append_len(con, len); - } + if (con->response.send_chunked) + http_chunk_len_append(cq, len); /*(chunkqueue_append_buffer() might steal buffer contents)*/ chunkqueue_append_buffer(cq, mem); - if (con->response.send_chunked) { + if (con->response.send_chunked) chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); - } return 0; } -int http_chunk_append_mem(connection *con, const char * mem, size_t len) { - chunkqueue * const cq = con->write_queue; +int http_chunk_append_mem(connection * const con, const char * const mem, const size_t len) { if (0 == len) return 0; force_assert(NULL != mem); - if (http_chunk_uses_tempfile(con, len)) { + chunkqueue * const cq = con->write_queue; + + if (http_chunk_uses_tempfile(con, cq, len)) return http_chunk_append_to_tempfile(con, mem, len); - } - if (con->response.send_chunked) { - http_chunk_append_len(con, len); - } + if (con->response.send_chunked) + http_chunk_len_append(cq, len); chunkqueue_append_mem(cq, mem, len); - if (con->response.send_chunked) { + if (con->response.send_chunked) chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); - } return 0; } -int http_chunk_transfer_cqlen(connection *con, chunkqueue *src, size_t len) { - chunkqueue * const cq = con->write_queue; +int http_chunk_transfer_cqlen(connection * const con, chunkqueue * const src, const size_t len) { if (0 == len) return 0; - if (http_chunk_uses_tempfile(con, len)) { + chunkqueue * const cq = con->write_queue; + + if (http_chunk_uses_tempfile(con, cq, len)) return http_chunk_append_cq_to_tempfile(con, src, len); - } - if (con->response.send_chunked) { - http_chunk_append_len(con, len); - } + if (con->response.send_chunked) + http_chunk_len_append(cq, len); chunkqueue_steal(cq, src, len); - if (con->response.send_chunked) { + if (con->response.send_chunked) chunkqueue_append_mem(cq, CONST_STR_LEN("\r\n")); - } return 0; } -void http_chunk_close(connection *con) { - force_assert(NULL != con); - - if (con->response.send_chunked) { - chunkqueue_append_mem(con->write_queue, CONST_STR_LEN("0\r\n\r\n")); - } +void http_chunk_close(connection * const con) { + if (con->response.send_chunked) + chunkqueue_append_mem(con->write_queue, CONST_STR_LEN("0\r\n\r\n")); } diff --git a/src/http_chunk.h b/src/http_chunk.h index baf557d0..052e5dd7 100644 --- a/src/http_chunk.h +++ b/src/http_chunk.h @@ -3,13 +3,15 @@ #include "first.h" #include "base_decls.h" +#include "buffer.h" +#include "chunk.h" int http_chunk_append_mem(connection *con, const char * mem, size_t len); /* copies memory */ int http_chunk_append_buffer(connection *con, buffer *mem); /* may reset "mem" */ int http_chunk_transfer_cqlen(connection *con, chunkqueue *src, size_t len); -int http_chunk_append_file(connection *con, buffer *fn); /* copies "fn" */ -int http_chunk_append_file_fd(connection *con, buffer *fn, int fd, off_t sz); -int http_chunk_append_file_range(connection *con, buffer *fn, off_t offset, off_t len); /* copies "fn" */ +int http_chunk_append_file(connection *con, const buffer *fn); /* copies "fn" */ +int http_chunk_append_file_fd(connection *con, const buffer *fn, int fd, off_t sz); +int http_chunk_append_file_range(connection *con, const buffer *fn, off_t offset, off_t len); /* copies "fn" */ void http_chunk_close(connection *con); #endif diff --git a/src/server.c b/src/server.c index 8868530c..ac9bb792 100644 --- a/src/server.c +++ b/src/server.c @@ -239,7 +239,6 @@ static server *server_init(void) { CLEAN(ts_date_str); CLEAN(tmp_buf); - CLEAN(tmp_chunk_len); #undef CLEAN for (int i = 0; i < FILE_CACHE_MAX; ++i) { @@ -280,7 +279,6 @@ static void server_free(server *srv) { CLEAN(ts_date_str); CLEAN(tmp_buf); - CLEAN(tmp_chunk_len); #undef CLEAN fdevent_free(srv->ev); diff --git a/src/stat_cache.c b/src/stat_cache.c index 90e70c41..4d43b58a 100644 --- a/src/stat_cache.c +++ b/src/stat_cache.c @@ -998,7 +998,7 @@ handler_t stat_cache_get_entry(server *srv, connection *con, buffer *name, stat_ return HANDLER_GO_ON; } -int stat_cache_path_contains_symlink(connection *con, buffer *name) { +int stat_cache_path_contains_symlink(connection *con, const buffer *name) { /* caller should check for symlinks only if we should block symlinks. */ /* catch the obvious symlinks @@ -1044,7 +1044,7 @@ int stat_cache_path_contains_symlink(connection *con, buffer *name) { return 0; } -int stat_cache_open_rdonly_fstat (buffer *name, struct stat *st, int symlinks) { +int stat_cache_open_rdonly_fstat (const buffer *name, struct stat *st, int symlinks) { /*(Note: O_NOFOLLOW affects only the final path segment, the target file, * not any intermediate symlinks along the path)*/ const int fd = fdevent_open_cloexec(name->ptr, symlinks, O_RDONLY, 0); diff --git a/src/stat_cache.h b/src/stat_cache.h index 9edfcc69..0bc63540 100644 --- a/src/stat_cache.h +++ b/src/stat_cache.h @@ -44,8 +44,8 @@ void stat_cache_delete_entry(server *srv, const char *name, size_t len); void stat_cache_delete_dir(server *srv, const char *name, size_t len); void stat_cache_invalidate_entry(server *srv, const char *name, size_t len); handler_t stat_cache_get_entry(server *srv, connection *con, buffer *name, stat_cache_entry **sce); -int stat_cache_path_contains_symlink(connection *con, buffer *name); -int stat_cache_open_rdonly_fstat (buffer *name, struct stat *st, int symlinks); +int stat_cache_path_contains_symlink(connection *con, const buffer *name); +int stat_cache_open_rdonly_fstat (const buffer *name, struct stat *st, int symlinks); int stat_cache_trigger_cleanup(server *srv); #endif