lighttpd 1.4.x https://www.lighttpd.net/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1243 lines
35 KiB

#include "first.h"
/**
* the network chunk-API
*
*
*/
#include "chunk.h"
#include "fdevent.h"
#include "log.h"
#include <sys/types.h>
#include <sys/stat.h>
#include "sys-mmap.h"
#include <stdlib.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
/* default 1MB, upper limit 128MB */
#define DEFAULT_TEMPFILE_SIZE (1 * 1024 * 1024)
#define MAX_TEMPFILE_SIZE (128 * 1024 * 1024)
static size_t chunk_buf_sz = 8192;
static chunk *chunks, *chunks_oversized;
static chunk *chunk_buffers;
static int chunks_oversized_n;
static const array *chunkqueue_default_tempdirs = NULL;
static off_t chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE;
void chunkqueue_set_chunk_size (size_t sz)
{
size_t x = 1024;
while (x < sz && x < (1u << 30)) x <<= 1;
chunk_buf_sz = sz > 0 ? x : 8192;
}
void chunkqueue_set_tempdirs_default_reset (void)
{
chunk_buf_sz = 8192;
chunkqueue_default_tempdirs = NULL;
chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE;
}
/* chunk buffer (c->mem) is never NULL; specialize routines from buffer.h */
__attribute_pure__
static inline size_t chunk_buffer_string_length(const buffer *b) {
return 0 != b->used ? b->used - 1 : 0;
}
__attribute_pure__
static inline int chunk_buffer_string_is_empty(const buffer *b) {
return b->used < 2;
}
__attribute_pure__
static inline size_t chunk_buffer_string_space(const buffer *b) {
return b->size ? b->size - (b->used | (0 == b->used)) : 0;
}
chunkqueue *chunkqueue_init(chunkqueue *cq) {
/* (if caller passes non-NULL cq, it must be 0-init) */
if (NULL == cq) {
cq = calloc(1, sizeof(*cq));
force_assert(NULL != cq);
}
cq->first = NULL;
cq->last = NULL;
cq->tempdirs = chunkqueue_default_tempdirs;
cq->upload_temp_file_size = chunkqueue_default_tempfile_size;
return cq;
}
__attribute_returns_nonnull__
static chunk *chunk_init(size_t sz) {
chunk * const restrict c = calloc(1, sizeof(*c));
force_assert(NULL != c);
#if 0 /*(zeroed by calloc())*/
fix buffer, chunk and http_chunk API * remove unused structs and functions (buffer_array, read_buffer) * change return type from int to void for many functions, as the return value (indicating error/success) was never checked, and the function would only fail on programming errors and not on invalid input; changed functions to use force_assert instead of returning an error. * all "len" parameters now are the real size of the memory to be read. the length of strings is given always without the terminating 0. * the "buffer" struct still counts the terminating 0 in ->used, provide buffer_string_length() to get the length of a string in a buffer. unset config "strings" have used == 0, which is used in some places to distinguish unset values from "" (empty string) values. * most buffer usages should now use it as string container. * optimise some buffer copying by "moving" data to other buffers * use (u)intmax_t for generic int-to-string functions * remove unused enum values: UNUSED_CHUNK, ENCODING_UNSET * converted BUFFER_APPEND_SLASH to inline function (no macro feature needed) * refactor: create chunkqueue_steal: moving (partial) chunks into another queue * http_chunk: added separate function to terminate chunked body instead of magic handling in http_chunk_append_mem(). http_chunk_append_* now handle empty chunks, and never terminate the chunked body. From: Stefan Bühler <stbuehler@web.de> git-svn-id: svn://svn.lighttpd.net/lighttpd/branches/lighttpd-1.4.x@2975 152afb58-edef-0310-8abb-c4023f1b3aa9
7 years ago
c->type = MEM_CHUNK;
c->next = NULL;
c->offset = 0;
c->file.length = 0;
c->file.mmap.length = c->file.mmap.offset = 0;
c->file.is_temp = 0;
#endif
c->file.fd = -1;
c->file.mmap.start = MAP_FAILED;
c->mem = buffer_init();
buffer_string_prepare_copy(c->mem, sz-1);
return c;
}
static void chunk_reset_file_chunk(chunk *c) {
if (c->file.is_temp) {
c->file.is_temp = 0;
if (!chunk_buffer_string_is_empty(c->mem))
unlink(c->mem->ptr);
}
if (c->file.refchg) {
c->file.refchg(c->file.ref, -1);
c->file.refchg = 0; /* NULL fn ptr */
c->file.ref = NULL;
}
else if (c->file.fd != -1) {
close(c->file.fd);
}
if (MAP_FAILED != c->file.mmap.start) {
munmap(c->file.mmap.start, c->file.mmap.length);
c->file.mmap.start = MAP_FAILED;
c->file.mmap.length = c->file.mmap.offset = 0;
}
c->file.fd = -1;
c->file.length = 0;
c->type = MEM_CHUNK;
}
static void chunk_reset(chunk *c) {
if (c->type == FILE_CHUNK) chunk_reset_file_chunk(c);
buffer_clear(c->mem);
fix buffer, chunk and http_chunk API * remove unused structs and functions (buffer_array, read_buffer) * change return type from int to void for many functions, as the return value (indicating error/success) was never checked, and the function would only fail on programming errors and not on invalid input; changed functions to use force_assert instead of returning an error. * all "len" parameters now are the real size of the memory to be read. the length of strings is given always without the terminating 0. * the "buffer" struct still counts the terminating 0 in ->used, provide buffer_string_length() to get the length of a string in a buffer. unset config "strings" have used == 0, which is used in some places to distinguish unset values from "" (empty string) values. * most buffer usages should now use it as string container. * optimise some buffer copying by "moving" data to other buffers * use (u)intmax_t for generic int-to-string functions * remove unused enum values: UNUSED_CHUNK, ENCODING_UNSET * converted BUFFER_APPEND_SLASH to inline function (no macro feature needed) * refactor: create chunkqueue_steal: moving (partial) chunks into another queue * http_chunk: added separate function to terminate chunked body instead of magic handling in http_chunk_append_mem(). http_chunk_append_* now handle empty chunks, and never terminate the chunked body. From: Stefan Bühler <stbuehler@web.de> git-svn-id: svn://svn.lighttpd.net/lighttpd/branches/lighttpd-1.4.x@2975 152afb58-edef-0310-8abb-c4023f1b3aa9
7 years ago
c->offset = 0;
}
fix buffer, chunk and http_chunk API * remove unused structs and functions (buffer_array, read_buffer) * change return type from int to void for many functions, as the return value (indicating error/success) was never checked, and the function would only fail on programming errors and not on invalid input; changed functions to use force_assert instead of returning an error. * all "len" parameters now are the real size of the memory to be read. the length of strings is given always without the terminating 0. * the "buffer" struct still counts the terminating 0 in ->used, provide buffer_string_length() to get the length of a string in a buffer. unset config "strings" have used == 0, which is used in some places to distinguish unset values from "" (empty string) values. * most buffer usages should now use it as string container. * optimise some buffer copying by "moving" data to other buffers * use (u)intmax_t for generic int-to-string functions * remove unused enum values: UNUSED_CHUNK, ENCODING_UNSET * converted BUFFER_APPEND_SLASH to inline function (no macro feature needed) * refactor: create chunkqueue_steal: moving (partial) chunks into another queue * http_chunk: added separate function to terminate chunked body instead of magic handling in http_chunk_append_mem(). http_chunk_append_* now handle empty chunks, and never terminate the chunked body. From: Stefan Bühler <stbuehler@web.de> git-svn-id: svn://svn.lighttpd.net/lighttpd/branches/lighttpd-1.4.x@2975 152afb58-edef-0310-8abb-c4023f1b3aa9
7 years ago
static void chunk_free(chunk *c) {
if (c->type == FILE_CHUNK) chunk_reset_file_chunk(c);
fix buffer, chunk and http_chunk API * remove unused structs and functions (buffer_array, read_buffer) * change return type from int to void for many functions, as the return value (indicating error/success) was never checked, and the function would only fail on programming errors and not on invalid input; changed functions to use force_assert instead of returning an error. * all "len" parameters now are the real size of the memory to be read. the length of strings is given always without the terminating 0. * the "buffer" struct still counts the terminating 0 in ->used, provide buffer_string_length() to get the length of a string in a buffer. unset config "strings" have used == 0, which is used in some places to distinguish unset values from "" (empty string) values. * most buffer usages should now use it as string container. * optimise some buffer copying by "moving" data to other buffers * use (u)intmax_t for generic int-to-string functions * remove unused enum values: UNUSED_CHUNK, ENCODING_UNSET * converted BUFFER_APPEND_SLASH to inline function (no macro feature needed) * refactor: create chunkqueue_steal: moving (partial) chunks into another queue * http_chunk: added separate function to terminate chunked body instead of magic handling in http_chunk_append_mem(). http_chunk_append_* now handle empty chunks, and never terminate the chunked body. From: Stefan Bühler <stbuehler@web.de> git-svn-id: svn://svn.lighttpd.net/lighttpd/branches/lighttpd-1.4.x@2975 152afb58-edef-0310-8abb-c4023f1b3aa9
7 years ago
buffer_free(c->mem);
free(c);
}
static chunk * chunk_pop_oversized(size_t sz) {
/* future: might have buckets of certain sizes, up to socket buf sizes */
if (chunks_oversized && chunks_oversized->mem->size >= sz) {
--chunks_oversized_n;
chunk *c = chunks_oversized;
chunks_oversized = c->next;
return c;
}
return NULL;
}
static void chunk_push_oversized(chunk * const c, const size_t sz) {
if (chunks_oversized_n < 64 && chunk_buf_sz >= 4096) {
++chunks_oversized_n;
chunk **co = &chunks_oversized;
while (*co && sz < (*co)->mem->size) co = &(*co)->next;
c->next = *co;
*co = c;
}
else
chunk_free(c);
}
__attribute_returns_nonnull__
static buffer * chunk_buffer_acquire_sz(size_t sz) {
chunk *c;
buffer *b;
if (sz <= chunk_buf_sz) {
if (chunks) {
c = chunks;
chunks = c->next;
}
else
c = chunk_init(chunk_buf_sz);
/* future: might choose to pop from chunks_oversized, if available
* (even if larger than sz) rather than allocating new chunk
* (and if doing so, might replace chunks_oversized_n) */
}
else {
/*(round up to nearest chunk_buf_sz)*/
sz = (sz + (chunk_buf_sz-1)) & ~(chunk_buf_sz-1);
c = chunk_pop_oversized(sz);
if (NULL == c)
c = chunk_init(sz);
}
c->next = chunk_buffers;
chunk_buffers = c;
b = c->mem;
c->mem = NULL;
return b;
}
buffer * chunk_buffer_acquire(void) {
return chunk_buffer_acquire_sz(chunk_buf_sz);
}
void chunk_buffer_release(buffer *b) {
if (NULL == b) return;
if (chunk_buffers) {
chunk *c = chunk_buffers;
chunk_buffers = c->next;
c->mem = b;
buffer_clear(b);
if (b->size == chunk_buf_sz) {
c->next = chunks;
chunks = c;
}
else if (b->size > chunk_buf_sz)
chunk_push_oversized(c, b->size);
else
chunk_free(c);
}
else {
buffer_free(b);
}
}
size_t chunk_buffer_prepare_append(buffer * const b, size_t sz) {
if (sz > chunk_buffer_string_space(b)) {
sz += b->used ? b->used : 1;
buffer * const cb = chunk_buffer_acquire_sz(sz);
/* swap buffer contents and copy original b->ptr into larger b->ptr */
/*(this does more than buffer_move())*/
buffer tb = *b;
*b = *cb;
*cb = tb;
if ((b->used = tb.used))
memcpy(b->ptr, tb.ptr, tb.used);
chunk_buffer_release(cb);
}
return chunk_buffer_string_space(b);
}
__attribute_returns_nonnull__
static chunk * chunk_acquire(size_t sz) {
if (sz <= chunk_buf_sz) {
if (chunks) {
chunk *c = chunks;
chunks = c->next;
return c;
}
sz = chunk_buf_sz;
}
else {
/*(round up to nearest chunk_buf_sz)*/
sz = (sz + (chunk_buf_sz-1)) & ~(chunk_buf_sz-1);
chunk *c = chunk_pop_oversized(sz);
if (c) return c;
}
return chunk_init(sz);
}
static void chunk_release(chunk *c) {
const size_t sz = c->mem->size;
if (sz == chunk_buf_sz) {
chunk_reset(c);
c->next = chunks;
chunks = c;
}
else if (sz > chunk_buf_sz) {
chunk_reset(c);
chunk_push_oversized(c, sz);
}
else {
chunk_free(c);
}
}
void chunkqueue_chunk_pool_clear(void)
{
for (chunk *next, *c = chunks; c; c = next) {
next = c->next;
chunk_free(c);
}
chunks = NULL;
for (chunk *next, *c = chunks_oversized; c; c = next) {
next = c->next;
chunk_free(c);
}
chunks_oversized = NULL;
chunks_oversized_n = 0;
}
void chunkqueue_chunk_pool_free(void)
{
chunkqueue_chunk_pool_clear();
for (chunk *next, *c = chunk_buffers; c; c = next) {
next = c->next;
c->mem = buffer_init(); /*(chunk_reset() expects c->mem != NULL)*/
chunk_free(c);
}
chunk_buffers = NULL;
}
__attribute_pure__
static off_t chunk_remaining_length(const chunk *c) {
/* MEM_CHUNK or FILE_CHUNK */
return (c->type == MEM_CHUNK
? (off_t)chunk_buffer_string_length(c->mem)
: c->file.length)
- c->offset;
}
static void chunkqueue_release_chunks(chunkqueue *cq) {
cq->last = NULL;
for (chunk *c; (c = cq->first); ) {
cq->first = c->next;
chunk_release(c);
}
}
void chunkqueue_free(chunkqueue *cq) {
if (NULL == cq) return;
chunkqueue_release_chunks(cq);
free(cq);
}
static void chunkqueue_prepend_chunk(chunkqueue * const restrict cq, chunk * const restrict c) {
if (NULL == (c->next = cq->first)) cq->last = c;
cq->first = c;
}
static void chunkqueue_append_chunk(chunkqueue * const restrict cq, chunk * const restrict c) {
c->next = NULL;
*(cq->last ? &cq->last->next : &cq->first) = c;
cq->last = c;
}
__attribute_returns_nonnull__
static chunk * chunkqueue_prepend_mem_chunk(chunkqueue *cq, size_t sz) {
chunk *c = chunk_acquire(sz);
chunkqueue_prepend_chunk(cq, c);
return c;
}
__attribute_returns_nonnull__
static chunk * chunkqueue_append_mem_chunk(chunkqueue *cq, size_t sz) {
chunk *c = chunk_acquire(sz);
chunkqueue_append_chunk(cq, c);
return c;
}
__attribute_returns_nonnull__
static chunk * chunkqueue_append_file_chunk(chunkqueue * const restrict cq, const buffer * const restrict fn, off_t offset, off_t len) {
chunk *c = chunk_acquire(buffer_string_length(fn)+1);
chunkqueue_append_chunk(cq, c);
c->type = FILE_CHUNK;
c->offset = offset;
c->file.length = offset + len;
cq->bytes_in += len;
buffer_copy_buffer(c->mem, fn);
return c;
}
void chunkqueue_reset(chunkqueue *cq) {
chunkqueue_release_chunks(cq);
cq->bytes_in = 0;
cq->bytes_out = 0;
cq->tempdir_idx = 0;
}
void chunkqueue_append_file_fd(chunkqueue * const restrict cq, const buffer * const restrict fn, int fd, off_t offset, off_t len) {
if (len > 0) {
(chunkqueue_append_file_chunk(cq, fn, offset, len))->file.fd = fd;
}
else {
close(fd);
}
[core] open fd when appending file to cq (fixes #2655) http_chunk_append_file() opens fd when appending file to chunkqueue. Defers calculation of content length until response is finished. This reduces race conditions pertaining to stat() and then (later) open(), when the result of the stat() was used for Content-Length or to generate chunked headers. Note: this does not change how lighttpd handles files that are modified in-place by another process after having been opened by lighttpd -- don't do that. This *does* improve handling of files that are frequently modified via a temporary file and then atomically renamed into place. mod_fastcgi has been modified to use http_chunk_append_file_range() with X-Sendfile2 and will open the target file multiple times if there are multiple ranges. Note: (future todo) not implemented for chunk.[ch] interfaces used by range requests in mod_staticfile or by mod_ssi. Those uses could lead to too many open fds. For mod_staticfile, limits should be put in place for max number of ranges accepted by mod_staticfile. For mod_ssi, limits would need to be placed on the maximum number of includes, and the primary SSI file split across lots of SSI directives should either copy the pieces or perhaps chunk.h could be extended to allow for an open fd to be shared across multiple chunks. Doing either of these would improve the performance of SSI since they would replace many file opens on the pieces of the SSI file around the SSI directives. x-ref: "Serving a file that is getting updated can cause an empty response or incorrect content-length error" https://redmine.lighttpd.net/issues/2655 github: Closes #49
6 years ago
}
void chunkqueue_append_file(chunkqueue * const restrict cq, const buffer * const restrict fn, off_t offset, off_t len) {
if (len > 0) {
chunkqueue_append_file_chunk(cq, fn, offset, len);
}
}
static int chunkqueue_append_mem_extend_chunk(chunkqueue * const restrict cq, const char * const restrict mem, size_t len) {
chunk *c = cq->last;
if (0 == len) return 1;
if (c != NULL && c->type == MEM_CHUNK
&& chunk_buffer_string_space(c->mem) >= len) {
buffer_append_string_len(c->mem, mem, len);
cq->bytes_in += len;
return 1;
}
return 0;
}
void chunkqueue_append_buffer(chunkqueue * const restrict cq, buffer * const restrict mem) {
chunk *c;
size_t len = buffer_string_length(mem);
if (len < 256 && chunkqueue_append_mem_extend_chunk(cq, mem->ptr, len)) return;
c = chunkqueue_append_mem_chunk(cq, chunk_buf_sz);
cq->bytes_in += len;
fix buffer, chunk and http_chunk API * remove unused structs and functions (buffer_array, read_buffer) * change return type from int to void for many functions, as the return value (indicating error/success) was never checked, and the function would only fail on programming errors and not on invalid input; changed functions to use force_assert instead of returning an error. * all "len" parameters now are the real size of the memory to be read. the length of strings is given always without the terminating 0. * the "buffer" struct still counts the terminating 0 in ->used, provide buffer_string_length() to get the length of a string in a buffer. unset config "strings" have used == 0, which is used in some places to distinguish unset values from "" (empty string) values. * most buffer usages should now use it as string container. * optimise some buffer copying by "moving" data to other buffers * use (u)intmax_t for generic int-to-string functions * remove unused enum values: UNUSED_CHUNK, ENCODING_UNSET * converted BUFFER_APPEND_SLASH to inline function (no macro feature needed) * refactor: create chunkqueue_steal: moving (partial) chunks into another queue * http_chunk: added separate function to terminate chunked body instead of magic handling in http_chunk_append_mem(). http_chunk_append_* now handle empty chunks, and never terminate the chunked body. From: Stefan Bühler <stbuehler@web.de> git-svn-id: svn://svn.lighttpd.net/lighttpd/branches/lighttpd-1.4.x@2975 152afb58-edef-0310-8abb-c4023f1b3aa9
7 years ago
buffer_move(c->mem, mem);
}
void chunkqueue_append_mem(chunkqueue * const restrict cq, const char * const restrict mem, size_t len) {
chunk *c;
if (len < chunk_buf_sz && chunkqueue_append_mem_extend_chunk(cq, mem, len))
return;
c = chunkqueue_append_mem_chunk(cq, len+1);
cq->bytes_in += len;
fix buffer, chunk and http_chunk API * remove unused structs and functions (buffer_array, read_buffer) * change return type from int to void for many functions, as the return value (indicating error/success) was never checked, and the function would only fail on programming errors and not on invalid input; changed functions to use force_assert instead of returning an error. * all "len" parameters now are the real size of the memory to be read. the length of strings is given always without the terminating 0. * the "buffer" struct still counts the terminating 0 in ->used, provide buffer_string_length() to get the length of a string in a buffer. unset config "strings" have used == 0, which is used in some places to distinguish unset values from "" (empty string) values. * most buffer usages should now use it as string container. * optimise some buffer copying by "moving" data to other buffers * use (u)intmax_t for generic int-to-string functions * remove unused enum values: UNUSED_CHUNK, ENCODING_UNSET * converted BUFFER_APPEND_SLASH to inline function (no macro feature needed) * refactor: create chunkqueue_steal: moving (partial) chunks into another queue * http_chunk: added separate function to terminate chunked body instead of magic handling in http_chunk_append_mem(). http_chunk_append_* now handle empty chunks, and never terminate the chunked body. From: Stefan Bühler <stbuehler@web.de> git-svn-id: svn://svn.lighttpd.net/lighttpd/branches/lighttpd-1.4.x@2975 152afb58-edef-0310-8abb-c4023f1b3aa9
7 years ago
buffer_copy_string_len(c->mem, mem, len);
}
void chunkqueue_append_mem_min(chunkqueue * const restrict cq, const char * const restrict mem, size_t len) {
chunk *c;
if (len < chunk_buf_sz && chunkqueue_append_mem_extend_chunk(cq, mem, len))
return;
c = chunk_init(len+1);
chunkqueue_append_chunk(cq, c);
cq->bytes_in += len;
buffer_copy_string_len(c->mem, mem, len);
}
void chunkqueue_append_chunkqueue(chunkqueue * const restrict cq, chunkqueue * const restrict src) {
if (src == NULL || NULL == src->first) return;
if (NULL == cq->first) {
cq->first = src->first;
} else {
cq->last->next = src->first;
}
cq->last = src->last;
cq->bytes_in += chunkqueue_length(src);
src->first = NULL;
src->last = NULL;
src->bytes_out = src->bytes_in;
}
buffer * chunkqueue_prepend_buffer_open_sz(chunkqueue *cq, size_t sz) {
chunk * const c = chunkqueue_prepend_mem_chunk(cq, sz);
return c->mem;
}
buffer * chunkqueue_prepend_buffer_open(chunkqueue *cq) {
return chunkqueue_prepend_buffer_open_sz(cq, chunk_buf_sz);
}
void chunkqueue_prepend_buffer_commit(chunkqueue *cq) {
cq->bytes_in += chunk_buffer_string_length(cq->first->mem);
}
buffer * chunkqueue_append_buffer_open_sz(chunkqueue *cq, size_t sz) {
chunk * const c = chunkqueue_append_mem_chunk(cq, sz);
return c->mem;
}
buffer * chunkqueue_append_buffer_open(chunkqueue *cq) {
return chunkqueue_append_buffer_open_sz(cq, chunk_buf_sz);
}
void chunkqueue_append_buffer_commit(chunkqueue *cq) {
cq->bytes_in += chunk_buffer_string_length(cq->last->mem);
}
static void chunkqueue_remove_empty_chunks(chunkqueue *cq);
char * chunkqueue_get_memory(chunkqueue * const restrict cq, size_t * const restrict len) {
size_t sz = *len ? *len : (chunk_buf_sz >> 1);
buffer *b;
chunk *c = cq->last;
if (NULL != c && MEM_CHUNK == c->type) {
/* return pointer into existing buffer if large enough */
size_t avail = chunk_buffer_string_space(c->mem);
if (avail >= sz) {
*len = avail;
b = c->mem;
return b->ptr + chunk_buffer_string_length(b);
}
}
/* allocate new chunk */
b = chunkqueue_append_buffer_open_sz(cq, sz);
*len = chunk_buffer_string_space(b);
return b->ptr;
}
void chunkqueue_use_memory(chunkqueue * const restrict cq, chunk *ckpt, size_t len) {
buffer *b = cq->last->mem;
if (len > 0) {
buffer_commit(b, len);
cq->bytes_in += len;
if (cq->last == ckpt || NULL == ckpt || MEM_CHUNK != ckpt->type
|| len > chunk_buffer_string_space(ckpt->mem)) return;
buffer_append_string_buffer(ckpt->mem, b);
}
else if (!chunk_buffer_string_is_empty(b)) { /*(cq->last == ckpt)*/
return; /* last chunk is not empty */
}
/* remove empty last chunk */
chunk_release(cq->last);
cq->last = ckpt;
*(ckpt ? &ckpt->next : &cq->first) = NULL;
}
void chunkqueue_update_file(chunkqueue * const restrict cq, chunk *c, off_t len) {
/*assert(c->type == FILE_CHUNK);*/
c->file.length += len;
cq->bytes_in += len;
}
void chunkqueue_set_tempdirs_default (const array *tempdirs, off_t upload_temp_file_size) {
chunkqueue_default_tempdirs = tempdirs;
chunkqueue_default_tempfile_size
= (0 == upload_temp_file_size) ? DEFAULT_TEMPFILE_SIZE
: (upload_temp_file_size > MAX_TEMPFILE_SIZE) ? MAX_TEMPFILE_SIZE
: upload_temp_file_size;
}
void chunkqueue_set_tempdirs(chunkqueue * const restrict cq, const array * const restrict tempdirs, off_t upload_temp_file_size) {
fix buffer, chunk and http_chunk API * remove unused structs and functions (buffer_array, read_buffer) * change return type from int to void for many functions, as the return value (indicating error/success) was never checked, and the function would only fail on programming errors and not on invalid input; changed functions to use force_assert instead of returning an error. * all "len" parameters now are the real size of the memory to be read. the length of strings is given always without the terminating 0. * the "buffer" struct still counts the terminating 0 in ->used, provide buffer_string_length() to get the length of a string in a buffer. unset config "strings" have used == 0, which is used in some places to distinguish unset values from "" (empty string) values. * most buffer usages should now use it as string container. * optimise some buffer copying by "moving" data to other buffers * use (u)intmax_t for generic int-to-string functions * remove unused enum values: UNUSED_CHUNK, ENCODING_UNSET * converted BUFFER_APPEND_SLASH to inline function (no macro feature needed) * refactor: create chunkqueue_steal: moving (partial) chunks into another queue * http_chunk: added separate function to terminate chunked body instead of magic handling in http_chunk_append_mem(). http_chunk_append_* now handle empty chunks, and never terminate the chunked body. From: Stefan Bühler <stbuehler@web.de> git-svn-id: svn://svn.lighttpd.net/lighttpd/branches/lighttpd-1.4.x@2975 152afb58-edef-0310-8abb-c4023f1b3aa9
7 years ago
force_assert(NULL != cq);
cq->tempdirs = tempdirs;
cq->upload_temp_file_size
= (0 == upload_temp_file_size) ? DEFAULT_TEMPFILE_SIZE
: (upload_temp_file_size > MAX_TEMPFILE_SIZE) ? MAX_TEMPFILE_SIZE
: upload_temp_file_size;
cq->tempdir_idx = 0;
}
static void chunkqueue_steal_partial_file_chunk(chunkqueue * const restrict dest, const chunk * const restrict c, const off_t len) {
chunkqueue_append_file(dest, c->mem, c->offset, len);
if (c->file.fd >= 0) {
chunk * const d = dest->last;
if (c->file.refchg) {
d->file.fd = c->file.fd;
d->file.ref = c->file.ref;
d->file.refchg = c->file.refchg;
d->file.refchg(d->file.ref, 1);
}
else
d->file.fd = fdevent_dup_cloexec(c->file.fd);
}
}
void chunkqueue_steal(chunkqueue * const restrict dest, chunkqueue * const restrict src, off_t len) {
while (len > 0) {
chunk *c = src->first;
off_t clen = 0, use;
if (NULL == c) break;
clen = chunk_remaining_length(c);
if (0 == clen) {
/* drop empty chunk */
src->first = c->next;
if (c == src->last) src->last = NULL;
chunk_release(c);
continue;
}
use = len >= clen ? clen : len;
len -= use;
if (use == clen) {
/* move complete chunk */
src->first = c->next;
if (c == src->last) src->last = NULL;
chunkqueue_append_chunk(dest, c);
dest->bytes_in += use;
} else {
/* partial chunk with length "use" */
switch (c->type) {
case MEM_CHUNK:
chunkqueue_append_mem(dest, c->mem->ptr + c->offset, use);
break;
case FILE_CHUNK:
/* tempfile flag is in "last" chunk after the split */
chunkqueue_steal_partial_file_chunk(dest, c, use);
break;
}
c->offset += use;
force_assert(0 == len);
}
src->bytes_out += use;
}
}
static chunk *chunkqueue_get_append_tempfile(chunkqueue * const restrict cq, log_error_st * const restrict errh) {
chunk *c;
buffer *template = buffer_init_string("/var/tmp/lighttpd-upload-XXXXXX");
int fd = -1;
if (cq->tempdirs && cq->tempdirs->used) {
/* we have several tempdirs, only if all of them fail we jump out */
for (errno = EIO; cq->tempdir_idx < cq->tempdirs->used; ++cq->tempdir_idx) {
data_string *ds = (data_string *)cq->tempdirs->data[cq->tempdir_idx];
buffer_copy_buffer(template, &ds->value);
buffer_append_path_len(template, CONST_STR_LEN("lighttpd-upload-XXXXXX"));
if (-1 != (fd = fdevent_mkstemp_append(template->ptr))) break;
}
} else {
fd = fdevent_mkstemp_append(template->ptr);
}
if (fd < 0) {
/* (report only the last error to mkstemp()
* if multiple temp dirs attempted) */
log_perror(errh, __FILE__, __LINE__,
"opening temp-file failed: %s", template->ptr);
buffer_free(template);
return NULL;
}