[multiple] shared code for struct chunk and mmap

chunkqueue_chunk_file_view()

reduces size of struct chunk

use mmap with mod_deflate libdeflate, if mmap available,
  even when lighttpd not built with --enable-mmap

avoid using mmap on temp files in chunkqueue (c->file.is_temp)
  since pread() with a reasonable block size is typically as fast
  or faster than mmap on files read sequentially and used only once,
  especially when writing results to limited-size socket buffers
  (and lighttpd temp files are, in most cases, by default about 1 MB)
  (Exception: sometimes mmap is used for convenience or to fulfill
   a requirement, e.g. one-shot libdeflate in mod_deflate)
  (There are many factors which influence speed of mmap versus pread,
   so this should not be construed as generic usage advice.)
master
Glenn Strauss 10 months ago
parent 9b55ac6da5
commit e05ce80502
  1. 299
      src/chunk.c
  2. 66
      src/chunk.h
  3. 83
      src/mod_deflate.c
  4. 95
      src/mod_webdav.c
  5. 110
      src/network_write.c
  6. 2
      src/sys-mmap.h

@ -22,6 +22,44 @@
#include <errno.h>
#include <string.h>
#ifdef HAVE_MMAP
#define MMAP_CHUNK_SIZE (512*1024)
__attribute_cold__
/*__attribute_noinline__*/
static off_t
mmap_pagemask (void)
{
#ifndef _WIN32
long pagesize = sysconf(_SC_PAGESIZE);
#else
long pagesize = -1; /*(not implemented (yet))*/
#endif
if (-1 == pagesize) pagesize = 4096;
force_assert(pagesize < MMAP_CHUNK_SIZE);
return ~((off_t)pagesize - 1); /* pagesize always power-of-2 */
}
#if 0
static off_t
mmap_align_offset (off_t start)
{
static off_t pagemask = 0;
if (0 == pagemask)
pagemask = mmap_pagemask();
return (start & pagemask);
}
#endif
#define mmap_align_offset(offset) ((offset) & chunk_pagemask)
static off_t chunk_pagemask = 0;
static int chunk_mmap_flags = MAP_SHARED;
#endif /* HAVE_MMAP */
/* default 1 MB */
#define DEFAULT_TEMPFILE_SIZE (1 * 1024 * 1024)
@ -44,6 +82,12 @@ void chunkqueue_set_tempdirs_default_reset (void)
chunk_buf_sz = 8192;
chunkqueue_default_tempdirs = NULL;
chunkqueue_default_tempfile_size = DEFAULT_TEMPFILE_SIZE;
#ifdef HAVE_MMAP /*(abuse this func to initialize statics as startup)*/
if (0 == chunk_pagemask)
chunk_pagemask = mmap_pagemask();
chunk_mmap_flags = MAP_SHARED;
#endif
}
chunkqueue *chunkqueue_init(chunkqueue *cq) {
@ -72,16 +116,16 @@ static chunk *chunk_init(void) {
c->next = NULL;
c->offset = 0;
c->file.length = 0;
c->file.mmap.length = c->file.mmap.offset = 0;
c->file.is_temp = 0;
c->file.view = NULL;
#endif
c->file.fd = -1;
c->file.mmap.start = MAP_FAILED;
c->mem = buffer_init();
return c;
}
__attribute_noinline__
__attribute_returns_nonnull__
static chunk *chunk_init_sz(size_t sz) {
chunk * const restrict c = chunk_init();
@ -89,6 +133,41 @@ static chunk *chunk_init_sz(size_t sz) {
return c;
}
#ifdef HAVE_MMAP
__attribute_malloc__
__attribute_returns_nonnull__
static void * chunk_file_view_init (void) {
chunk_file_view * const restrict cfv = calloc(1, sizeof(*cfv));
force_assert(NULL != cfv);
cfv->mptr = MAP_FAILED;
#if 0 /*(zeroed by calloc())*/
cfv->mlen = 0;
cfv->foff = 0;
#endif
cfv->refcnt = 1;
return cfv;
}
__attribute_nonnull__()
static chunk_file_view * chunk_file_view_release (chunk_file_view *cfv) {
if (0 == --cfv->refcnt) {
if (MAP_FAILED != cfv->mptr)
munmap(cfv->mptr, (size_t)cfv->mlen);
free(cfv);
}
return NULL;
}
__attribute_cold__
__attribute_noinline__
__attribute_nonnull__()
static chunk_file_view * chunk_file_view_failed (chunk_file_view *cfv) {
return chunk_file_view_release(cfv);
}
#endif /* HAVE_MMAP */
static void chunk_reset_file_chunk(chunk *c) {
if (c->file.is_temp) {
c->file.is_temp = 0;
@ -103,11 +182,10 @@ static void chunk_reset_file_chunk(chunk *c) {
else if (c->file.fd != -1) {
close(c->file.fd);
}
if (MAP_FAILED != c->file.mmap.start) {
munmap(c->file.mmap.start, c->file.mmap.length);
c->file.mmap.start = MAP_FAILED;
c->file.mmap.length = c->file.mmap.offset = 0;
}
#ifdef HAVE_MMAP
if (c->file.view)
c->file.view = chunk_file_view_release(c->file.view);
#endif
c->file.fd = -1;
c->file.length = 0;
c->type = MEM_CHUNK;
@ -157,6 +235,7 @@ static void chunk_push_oversized(chunk * const c, const size_t sz) {
}
}
__attribute_noinline__
__attribute_returns_nonnull__
static buffer * chunk_buffer_acquire_sz(const size_t sz) {
chunk *c;
@ -238,6 +317,7 @@ size_t chunk_buffer_prepare_append(buffer * const b, size_t sz) {
return buffer_string_space(b);
}
__attribute_noinline__
__attribute_returns_nonnull__
static chunk * chunk_acquire(size_t sz) {
if (sz <= (chunk_buf_sz|1)) {
@ -585,6 +665,10 @@ static void chunkqueue_dup_file_chunk_fd (chunk * const restrict d, const chunk
}
else
d->file.fd = fdevent_dup_cloexec(c->file.fd);
#ifdef HAVE_MMAP
if ((d->file.view = c->file.view))
++d->file.view->refcnt;
#endif
}
}
@ -1290,7 +1374,7 @@ chunkqueue_write_data (const int fd, const void *buf, size_t len)
}
#if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/
#ifdef HAVE_MMAP
__attribute_cold__
#endif
__attribute_noinline__
@ -1308,67 +1392,6 @@ chunkqueue_write_chunk_file_intermed (const int fd, chunk * const restrict c, lo
}
#if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/
/*(improved from network_write_mmap.c)*/
static off_t
mmap_align_offset (off_t start)
{
static off_t pagemask = 0;
if (0 == pagemask) {
#ifndef _WIN32
long pagesize = sysconf(_SC_PAGESIZE);
#else
long pagesize = -1; /*(not implemented (yet))*/
#endif
if (-1 == pagesize) pagesize = 4096;
pagemask = ~((off_t)pagesize - 1); /* pagesize always power-of-2 */
}
return (start & pagemask);
}
__attribute_noinline__
static char *
chunkqueue_mmap_chunk_len (chunk * const c, off_t len)
{
/* (re)mmap the buffer to file length if range is not covered completely */
/*(caller is responsible for handling SIGBUS if chunkqueue might contain
* untrusted file, i.e. any file other than lighttpd-created tempfile)*/
/*(tempfiles are expected for input, MAP_PRIVATE used for portability)*/
/*(mmaps and writes complete chunk instead of only small parts; files
* are expected to be temp files with reasonable chunk sizes)*/
if (MAP_FAILED == c->file.mmap.start
|| c->offset < c->file.mmap.offset
|| c->offset+len > (off_t)(c->file.mmap.offset + c->file.mmap.length)) {
if (MAP_FAILED != c->file.mmap.start) {
munmap(c->file.mmap.start, c->file.mmap.length);
/*c->file.mmap.start = MAP_FAILED;*//*(assigned below)*/
}
c->file.mmap.offset = mmap_align_offset(c->offset);
c->file.mmap.length = c->file.length - c->file.mmap.offset;
c->file.mmap.start =
mmap(NULL, c->file.mmap.length, PROT_READ, MAP_PRIVATE,
c->file.fd, c->file.mmap.offset);
if (MAP_FAILED == c->file.mmap.start) return NULL;
#if 0 /*(review callers before changing; some expect open file)*/
/* close() fd as soon as fully mmap() rather than when done w/ chunk
* (possibly worthwhile to keep active fd count lower) */
if (c->file.is_temp && !c->file.refchg) {
close(c->file.fd);
c->file.fd = -1;
}
#endif
}
return c->file.mmap.start + c->offset - c->file.mmap.offset;
}
#endif
#if defined HAVE_SYS_SENDFILE_H && defined HAVE_SENDFILE \
&& (!defined _LARGEFILE_SOURCE || defined HAVE_SENDFILE64) \
&& defined(__linux__) && !defined HAVE_SENDFILE_BROKEN
@ -1396,13 +1419,16 @@ chunkqueue_write_chunk_file (const int fd, chunk * const restrict c, log_error_s
sendfile(fd, c->file.fd, &offset, len < INT32_MAX ? len : INT32_MAX);
if (__builtin_expect( (wr >= 0), 1) || (errno != EINVAL && errno != ENOSYS))
return wr;
#endif
#if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/
/*(could fallback to mmap, but if sendfile fails on linux, mmap may, too)*/
#elif defined(HAVE_MMAP)
/*(chunkqueue_write_chunk() caller must protect against SIGBUS, if needed)*/
const char * const data = chunkqueue_mmap_chunk_len(c, len);
if (NULL != data)
return chunkqueue_write_data(fd, data, len);
const chunk_file_view * const restrict cfv =
chunkqueue_chunk_file_view(c, len, errh);
if (NULL != cfv) {
const off_t mmap_avail = chunk_file_view_dlen(cfv, c->offset);
return chunkqueue_write_data(fd, chunk_file_view_dptr(cfv, c->offset),
len <= mmap_avail ? len : mmap_avail);
}
#endif
return chunkqueue_write_chunk_file_intermed(fd, c, errh);
@ -1517,8 +1543,7 @@ chunkqueue_small_resp_optim (chunkqueue * const restrict cq)
#if 0
#if defined(_LP64) || defined(__LP64__) || defined(_WIN64)
#if defined(HAVE_MMAP) || defined(_WIN32) /*see local sys-mmap.h*/
#ifdef HAVE_MMAP
__attribute_noinline__
static off_t
chunk_setjmp_memcpy_cb (void *dst, const void *src, off_t len)
@ -1529,7 +1554,6 @@ chunk_setjmp_memcpy_cb (void *dst, const void *src, off_t len)
}
#endif
#endif
#endif
int
@ -1561,7 +1585,6 @@ chunkqueue_peek_data (chunkqueue * const cq,
case FILE_CHUNK:
if (c->file.fd >= 0 || 0 == chunk_open_file_chunk(c, errh)) {
off_t offset = c->offset;
off_t len = c->file.length - c->offset;
if (len > (off_t)space)
len = (off_t)space;
@ -1569,27 +1592,26 @@ chunkqueue_peek_data (chunkqueue * const cq,
break;
#if 0 /* XXX: might improve performance on some system workloads */
#if defined(_LP64) || defined(__LP64__) || defined(_WIN64)
#if defined(HAVE_MMAP) || defined(_WIN32) /*see local sys-mmap.h*/
#ifdef HAVE_MMAP
/* mmap file to access data
* (For now, also limit to 64-bit to avoid address space issues)
* (chunkqueue_mmap_chunk_len() uses MAP_PRIVATE, even though we
* might use MAP_SHARED, if supported, and !c->file.is_temp)
* fd need not be kept open for the mmap once
* the mmap has been created, but is currently kept open for
* other pre-existing logic which checks fd and opens file,
* such as the condition for entering this code block above. */
/* Note: current use is with somewhat large buffers, e.g. 128k.
* If larger buffers are used, then upper limit, e.g. 512k,
* should be set for 32-bit to avoid address space issues) */
/* Note: under heavy load (or microbenchmark), system-reported
* memory use for RSS can be very, very large, due to presence
* of lots and lots of temp file read-only memory maps.
* pmap -X and exclude lighttpd temporary files to get a better
* pmap -X and exclude lighttpd mmap files to get a better
* view of memory use */
/* Data should be large enough that mmap is worthwhile.
* mmap if file chunk > 16k or if already mapped */
char *mdata;
if ((c->file.mmap.start != MAP_FAILED
|| c->file.length - c->offset > 16384)
&& (mdata = chunkqueue_mmap_chunk_len(c, len))) {
const chunk_file_view * const restrict cfv = (!c->file.is_temp)
? chunkqueue_chunk_file_view(c, len, errh)
: NULL;
if (cfv && chunk_file_view_dlen(cfv, c->offset) >= len) {
/*(check (above) that mapped chunk length >= requested len)*/
char * const mdata = chunk_file_view_dptr(cfv, c->offset);
if (!c->file.is_temp) {/*(might be changed to policy flag)*/
if (sys_setjmp_eval3(chunk_setjmp_memcpy_cb,
data_in+*dlen, mdata, len) < 0) {
@ -1606,11 +1628,10 @@ chunkqueue_peek_data (chunkqueue * const cq,
break;
}
#endif
#endif
#endif
#ifndef HAVE_PREAD
if (-1 == lseek(c->file.fd, offset, SEEK_SET)) {
if (-1 == lseek(c->file.fd, c->offset, SEEK_SET)) {
log_perror(errh, __FILE__, __LINE__, "lseek(\"%s\")",
c->mem->ptr);
return -1;
@ -1619,9 +1640,9 @@ chunkqueue_peek_data (chunkqueue * const cq,
ssize_t rd;
do {
#ifdef HAVE_PREAD
rd =pread(c->file.fd, data_in + *dlen, (size_t)len, offset);
rd =pread(c->file.fd,data_in+*dlen,(size_t)len,c->offset);
#else
rd = read(c->file.fd, data_in + *dlen, (size_t)len);
rd = read(c->file.fd,data_in+*dlen,(size_t)len);
#endif
} while (-1 == rd && errno == EINTR);
if (rd <= 0) { /* -1 error; 0 EOF (unexpected) */
@ -1697,3 +1718,91 @@ chunkqueue_read_squash (chunkqueue * const restrict cq, log_error_st * const res
chunkqueue_append_chunk(cq, c);
return c->mem;
}
#ifdef HAVE_MMAP
const chunk_file_view *
chunkqueue_chunk_file_viewadj (chunk * const c, off_t n, log_error_st * restrict errh)
{
/*assert(c->type == FILE_CHUNK);*/
chunk_file_view * restrict cfv = c->file.view;
if (NULL == cfv) {
/* XXX: might add global config check to enable/disable mmap use here */
cfv = c->file.view = chunk_file_view_init();
}
else if (MAP_FAILED != cfv->mptr)
munmap(cfv->mptr, (size_t)cfv->mlen);
/*cfv->mptr= MAP_FAILED;*//*(assigned below)*/
if (c->file.fd < 0 && 0 != chunk_open_file_chunk(c, errh)) {
c->file.view = chunk_file_view_failed(cfv);
return NULL;
}
cfv->foff = mmap_align_offset(c->offset);
if (0 != n) {
cfv->mlen = c->offset - cfv->foff + n;
#if !(defined(_LP64) || defined(__LP64__) || defined(_WIN64))
/*(consider 512k blocks if this func is used more generically)*/
const off_t mmap_chunk_size = 8 * 1024 * 1024;
if (cfv->mlen > mmap_chunk_size)
cfv->mlen = mmap_chunk_size;
#endif
}
else
cfv->mlen = MMAP_CHUNK_SIZE;
/* XXX: 64-bit might consider larger min block size, or even entire file */
if (cfv->mlen < MMAP_CHUNK_SIZE)
cfv->mlen = MMAP_CHUNK_SIZE;
if (cfv->mlen > c->file.length - cfv->foff)
cfv->mlen = c->file.length - cfv->foff;
cfv->mptr = mmap(NULL, (size_t)cfv->mlen, PROT_READ,
c->file.is_temp ? MAP_PRIVATE : chunk_mmap_flags,
c->file.fd, cfv->foff);
if (__builtin_expect( (MAP_FAILED == cfv->mptr), 0)) {
if (__builtin_expect( (errno == EINVAL), 0)) {
chunk_mmap_flags &= ~MAP_SHARED;
chunk_mmap_flags |= MAP_PRIVATE;
cfv->mptr = mmap(NULL, (size_t)cfv->mlen, PROT_READ,
MAP_PRIVATE, c->file.fd, cfv->foff);
}
if (__builtin_expect( (MAP_FAILED == cfv->mptr), 0)) {
c->file.view = chunk_file_view_failed(cfv);
return NULL;
}
}
#if 0 /*(review callers before changing; some expect open file)*/
/* close() fd as soon as fully mmap() rather than when done w/ chunk
* (possibly worthwhile to keep active fd count lower)
* (probably only reasonable if entire file is mapped) */
if (c->file.is_temp && !c->file.refchg) {
close(c->file.fd);
c->file.fd = -1;
}
#endif
#if 0
/* disable madvise unless we find common cases where there is a benefit
* (??? madvise for full mmap length or only for original requested n ???)
* (??? might additional flags param to func to indicate madvise pref ???)
* (??? might experiment with Linux mmap flags MAP_POPULATE|MAP_PRIVATE)
* (??? might experiment with madvise MADV_POPULATE_READ (since Linux 5.14))
* note: caller might be in better position to know if starting an mmap
* which will be flushed in entirety, and perform madvise at that point,
* perhaps with MADV_SEQUENTIAL */
#ifdef HAVE_MADVISE
if (cfv->mlen > 65536) /*(skip syscall if size <= 64KB)*/
(void)madvise(cfv->mptr, (size_t)cfv->mlen, MADV_WILLNEED);
#endif
#endif
return cfv;
}
#endif /* HAVE_MMAP */

@ -2,10 +2,6 @@
#define _CHUNK_H_
#include "first.h"
#ifdef _AIX /*(AIX might #define mmap mmap64)*/
#include "sys-mmap.h"
#endif
#include "buffer.h"
#include "array.h"
#include "fdlog.h"
@ -14,6 +10,13 @@
#define MAX_READ_LIMIT (256*1024)
#define MAX_WRITE_LIMIT (256*1024)
typedef struct chunk_file_view {
char *mptr; /* base pointer of mmap'ed area */
off_t mlen; /* length of mmap'ed area */
off_t foff; /* offset from the start of the file */
int refcnt;
} chunk_file_view;
typedef struct chunk {
struct chunk *next;
enum { MEM_CHUNK, FILE_CHUNK } type;
@ -32,11 +35,9 @@ typedef struct chunk {
int fd;
int is_temp; /* file is temporary and will be deleted if on cleanup */
struct {
char *start; /* the start pointer of the mmap'ed area */
size_t length; /* size of the mmap'ed area */
off_t offset; /* start is <n> octet away from the start of the file */
} mmap;
#if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/
chunk_file_view *view;
#endif
void *ref;
void(*refchg)(void *, int);
} file;
@ -162,6 +163,7 @@ int chunkqueue_read_data (chunkqueue *cq, char *data, uint32_t dlen, log_error_s
buffer * chunkqueue_read_squash (chunkqueue * restrict cq, log_error_st * restrict errh);
__attribute_pure__
__attribute_nonnull__()
static inline off_t chunkqueue_length(const chunkqueue *cq);
static inline off_t chunkqueue_length(const chunkqueue *cq) {
return cq->bytes_in - cq->bytes_out;
@ -173,9 +175,55 @@ void chunkqueue_free(chunkqueue *cq);
void chunkqueue_reset(chunkqueue *cq);
__attribute_pure__
__attribute_nonnull__()
static inline int chunkqueue_is_empty(const chunkqueue *cq);
static inline int chunkqueue_is_empty(const chunkqueue *cq) {
return NULL == cq->first;
}
const chunk_file_view * chunkqueue_chunk_file_viewadj (chunk *c, off_t n, log_error_st * restrict errh);
__attribute_pure__
__attribute_nonnull__()
static inline char *
chunk_file_view_dptr (const chunk_file_view * const cfv, off_t offset);
static inline char *
chunk_file_view_dptr (const chunk_file_view * const cfv, off_t offset)
{
return cfv->mptr - cfv->foff + offset;
}
__attribute_pure__
__attribute_nonnull__()
static inline off_t
chunk_file_view_dlen (const chunk_file_view * const cfv, off_t offset);
static inline off_t
chunk_file_view_dlen (const chunk_file_view * const cfv, off_t offset)
{
return cfv->mlen + cfv->foff - offset;
}
static inline const chunk_file_view *
chunkqueue_chunk_file_view (chunk * const c, const off_t n, log_error_st * const restrict errh);
static inline const chunk_file_view *
chunkqueue_chunk_file_view (chunk * const c, const off_t n, log_error_st * const restrict errh)
{
/*assert(c->type == FILE_CHUNK);*/
#if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/
/* mmap buffer if offset is outside old mmap area or not mapped at all */
const chunk_file_view * const restrict cfv = c->file.view;
if (NULL == cfv
? c->file.length - c->offset >= 131072 /* TBD: min chunk size to mmap */
: (c->offset - cfv->foff < 0
|| chunk_file_view_dlen(cfv, c->offset) < (n ? n : 1)))
return chunkqueue_chunk_file_viewadj(c, n, errh);
return cfv;
#else
UNUSED(c);
UNUSED(n);
UNUSED(errh);
return NULL;
#endif
}
#endif

@ -101,6 +101,9 @@
#include <sys/types.h>
#include <sys/stat.h>
#include "sys-mmap.h"
#ifdef HAVE_MMAP
#include "sys-setjmp.h"
#endif
#include "sys-time.h"
#include <fcntl.h>
@ -154,12 +157,6 @@
#undef HAVE_LIBDEFLATE
#endif
#if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP
#define USE_MMAP
#include "sys-mmap.h"
#include "sys-setjmp.h"
#endif
/* request: accept-encoding */
#define HTTP_ACCEPT_ENCODING_IDENTITY BV(0)
#define HTTP_ACCEPT_ENCODING_GZIP BV(1)
@ -1427,48 +1424,6 @@ static int deflate_compress_cleanup(request_st * const r, handler_ctx * const hc
}
#ifdef USE_MMAP
static int mod_deflate_file_chunk_remap(chunk * const c, const off_t n)
{
if (c->file.mmap.start == MAP_FAILED
|| c->offset < c->file.mmap.offset
|| c->offset >= c->file.mmap.offset + (off_t)c->file.mmap.length) {
/* use large chunks since server blocks while compressing
* (mod_deflate is not recommended for large files) */
if (c->file.mmap.start != MAP_FAILED) /*(munmap, remap at new offset)*/
munmap(c->file.mmap.start, c->file.mmap.length);
c->file.mmap.offset = c->offset & ~(16384-1); /* pagesize multiple */
c->file.mmap.length = (size_t)(c->offset - c->file.mmap.offset + n);
#if !(defined(_LP64) || defined(__LP64__) || defined(_WIN64))
/*(consider 512k blocks if this func is used more generically)*/
const off_t mmap_chunk_size = 8 * 1024 * 1024;
if (c->file.mmap.length > (size_t)mmap_chunk_size)
c->file.mmap.length = (size_t)mmap_chunk_size;
#endif
c->file.mmap.start =
mmap(0, c->file.mmap.length, PROT_READ,
c->file.is_temp ? MAP_PRIVATE : MAP_SHARED,
c->file.fd, c->file.mmap.offset);
if (MAP_FAILED == c->file.mmap.start) {
if (errno != EINVAL) return 0;
c->file.mmap.start =
mmap(0, c->file.mmap.length, PROT_READ, MAP_PRIVATE,
c->file.fd, c->file.mmap.offset);
if (MAP_FAILED == c->file.mmap.start) return 0;
}
#ifdef HAVE_MADVISE
/*(consider func param for madvise if func is used more generically)*/
if (c->file.mmap.length > 65536) /*(skip syscall if size <= 64KB)*/
(void)madvise(c->file.mmap.start,c->file.mmap.length,MADV_WILLNEED);
#endif
}
return 1;
}
#endif
#ifdef HAVE_LIBDEFLATE
#include <libdeflate.h>
@ -1533,7 +1488,7 @@ static int mod_deflate_using_libdeflate_sm (handler_ctx * const hctx, const plug
}
#ifdef USE_MMAP
#ifdef HAVE_MMAP
#if defined(_LP64) || defined(__LP64__) || defined(_WIN64)
struct mod_deflate_setjmp_params {
@ -1547,7 +1502,7 @@ static off_t mod_deflate_using_libdeflate_setjmp_cb (void *dst, const void *src,
const struct mod_deflate_setjmp_params * const params = dst;
const handler_ctx * const hctx = src;
const chunk * const c = hctx->r->write_queue.first;
const char *in = c->file.mmap.start + c->offset - c->file.mmap.offset;
const char *in = chunk_file_view_dptr(c->file.view, c->offset);
return (off_t)((hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP)
? libdeflate_gzip_compress(params->compressor, in, (size_t)len,
params->out, params->outsz)
@ -1641,7 +1596,7 @@ static int mod_deflate_using_libdeflate (handler_ctx * const hctx, const plugin_
}
#endif /* defined(_LP64) || defined(__LP64__) || defined(_WIN64) */
#endif /* USE_MMAP */
#endif /* HAVE_MMAP */
#endif /* HAVE_LIBDEFLATE */
@ -1666,7 +1621,7 @@ static off_t mod_deflate_file_chunk_no_mmap(request_st * const r, handler_ctx *
}
#ifdef USE_MMAP
#ifdef ENABLE_MMAP
static off_t mod_deflate_file_chunk_setjmp_cb (void *dst, const void *src, off_t len)
{
@ -1676,12 +1631,18 @@ static off_t mod_deflate_file_chunk_setjmp_cb (void *dst, const void *src, off_t
static off_t mod_deflate_file_chunk_mmap(request_st * const r, handler_ctx * const hctx, chunk * const c, off_t n)
{
if (!mod_deflate_file_chunk_remap(c, n))
/* n is length of entire file since server blocks while compressing
* (mod_deflate is not recommended for large files;
* mod_deflate default upper limit is 128MB; deflate.max-compress-size) */
const chunk_file_view * const restrict cfv = (!c->file.is_temp)
? chunkqueue_chunk_file_view(c, n, r->conf.errh)
: NULL;
if (NULL == cfv)
return mod_deflate_file_chunk_no_mmap(r, hctx, c, n);
const off_t moffset = c->offset - c->file.mmap.offset;
char * const p = c->file.mmap.start + moffset;
off_t len = c->file.mmap.length - moffset;
const char * const p = chunk_file_view_dptr(cfv, c->offset);
off_t len = chunk_file_view_dlen(cfv, c->offset);
if (len > n) len = n;
off_t rc = sys_setjmp_eval3(mod_deflate_file_chunk_setjmp_cb, hctx, p, len);
if (__builtin_expect( (rc < 0), 0)) {
@ -1705,7 +1666,7 @@ static off_t mod_deflate_file_chunk(request_st * const r, handler_ctx * const hc
return -1;
}
}
#ifdef USE_MMAP
#ifdef ENABLE_MMAP
return mod_deflate_file_chunk_mmap(r, hctx, c, n);
#else
return mod_deflate_file_chunk_no_mmap(r, hctx, c, n);
@ -2097,16 +2058,18 @@ REQUEST_FUNC(mod_deflate_handle_response_start) {
#ifdef HAVE_LIBDEFLATE
chunk * const c = r->write_queue.first; /*(invalid after compression)*/
#ifdef USE_MMAP
#ifdef HAVE_MMAP
#if defined(_LP64) || defined(__LP64__) || defined(_WIN64)
/* optimization to compress single file in one-shot to writeable mmap */
/*(future: might extend to other compression types)*/
if (len > 65536 /* XXX: TBD what should min size be for this optimization?*/
/*(chunkqueue_chunk_file_view() current min size for mmap is 128k)*/
if (len > 131072 /* XXX: TBD what should min size be for optimization?*/
&& (hctx->compression_type == HTTP_ACCEPT_ENCODING_GZIP
|| hctx->compression_type == HTTP_ACCEPT_ENCODING_DEFLATE)
&& c == r->write_queue.last
&& c->type == FILE_CHUNK
&& mod_deflate_file_chunk_remap(c, len)) {
&& chunkqueue_chunk_file_view(c, len, r->conf.errh)
&& chunk_file_view_dlen(c->file.view, c->offset) >= len) { /*(cfv)*/
rc = HANDLER_GO_ON;
hctx->bytes_in = len;
if (mod_deflate_using_libdeflate(hctx, p)) {

@ -739,9 +739,6 @@ typedef struct {
#define MOD_WEBDAV_XMLNS_NS0 "xmlns:ns0=\"urn:uuid:c2f41010-65b3-11d1-a29f-00aa00c14882/\""
static char *
webdav_mmap_file_chunk (chunk * const c);
__attribute_cold__
__attribute_noinline__
static void
@ -761,8 +758,19 @@ webdav_xml_log_response (request_st * const r)
break;
case FILE_CHUNK:
/*(safe to mmap tempfiles from response XML)*/
s = webdav_mmap_file_chunk(c);
len = (uint32_t)c->file.length;
/*(response body provided in temporary file, so ok to mmap().
* Otherwise, must access through sys_setjmp_eval3()) */
/*(tempfiles (and xml response) should easily fit in uint32_t
* and are not expected to already be mmap'd. Avoid >= 128k
* requirement of chunkqueue_chunk_file_view() by using viewadj)*/
len = (uint32_t)(c->file.length - c->offset);
{
const chunk_file_view * const restrict cfv =
chunkqueue_chunk_file_viewadj(c, (off_t)len, r->conf.errh);
s = (cfv && chunk_file_view_dlen(cfv, c->offset) >= (off_t)len)
? chunk_file_view_dptr(cfv, c->offset)
: NULL;
}
if (s == NULL) continue;
break;
default:
@ -3725,73 +3733,29 @@ webdav_propfind_dir (webdav_propfind_bufs * const restrict pb)
}
static int
webdav_open_chunk_file_rd (chunk * const c)
{
if (c->file.fd < 0) /* open file if not already open *//*permit symlink*/
c->file.fd = fdevent_open_cloexec(c->mem->ptr, 1, O_RDONLY, 0);
return c->file.fd;
}
static int
webdav_mmap_file_rd (void ** const addr, const size_t length,
const int fd, const off_t offset)
{
/*(caller must ensure offset is properly aligned to mmap requirements)*/
if (0 == length) {
*addr = NULL; /*(something other than MAP_FAILED)*/
return 0;
}
#if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/
*addr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, offset);
if (*addr == MAP_FAILED && errno == EINVAL)
*addr = mmap(NULL, length, PROT_READ, MAP_PRIVATE, fd, offset);
return (*addr != MAP_FAILED ? 0 : -1);
#else
UNUSED(fd);
UNUSED(offset);
return -1;
#endif
}
#if defined(USE_PROPPATCH) || defined(USE_LOCKS)
static char *
webdav_mmap_file_chunk (chunk * const c)
webdav_mmap_file_chunk (chunk * const c, log_error_st * const errh)
{
#ifdef HAVE_MMAP
/*(request body provided in temporary file, so ok to mmap().
* Otherwise, must check defined(ENABLE_MMAP)) */
/* chunk_reset() or chunk_free() will clean up mmap'd chunk */
/* close c->file.fd only after mmap() succeeds, since it will not
* be able to be re-opened if it was a tmpfile that was unlinked */
* Otherwise, must access through sys_setjmp_eval3()) */
/*assert(c->type == FILE_CHUNK);*/
if (MAP_FAILED != c->file.mmap.start)
return c->file.mmap.start + c->offset - c->file.mmap.offset;
if (webdav_open_chunk_file_rd(c) < 0)
return NULL;
webdav_mmap_file_rd((void **)&c->file.mmap.start, (size_t)c->file.length,
c->file.fd, 0);
if (MAP_FAILED == c->file.mmap.start)
return NULL;
close(c->file.fd);
c->file.fd = -1;
c->file.mmap.length = c->file.length;
return c->file.mmap.start + c->offset - c->file.mmap.offset;
const off_t len = c->file.length - c->offset;
const chunk_file_view * const restrict cfv =
chunkqueue_chunk_file_view(c, len, errh);
return (cfv && chunk_file_view_dlen(cfv, c->offset) >= len)
? chunk_file_view_dptr(cfv, c->offset)
: NULL;
#else
UNUSED(c);
UNUSED(errh);
return NULL;
#endif
}
#if defined(USE_PROPPATCH) || defined(USE_LOCKS)
__attribute_noinline__
static xmlDoc *
webdav_parse_chunkqueue (request_st * const r,
@ -3820,8 +3784,7 @@ webdav_parse_chunkqueue (request_st * const r,
weHave = buffer_clen(c->mem) - c->offset;
}
else if (c->type == FILE_CHUNK) {
xmlstr = webdav_mmap_file_chunk(c);
/*xmlstr = c->file.mmap.start + c->offset - c->file.mmap.offset;*/
xmlstr = webdav_mmap_file_chunk(c, r->conf.errh);
if (NULL != xmlstr) {
weHave = c->file.length - c->offset;
}

@ -63,9 +63,11 @@
# define NETWORK_WRITE_USE_WRITEV
#endif
#if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP
#if defined(HAVE_MMAP) || defined(_WIN32) /*(see local sys-mmap.h)*/
#ifdef ENABLE_MMAP
# define NETWORK_WRITE_USE_MMAP
#endif
#endif
__attribute_cold__
@ -182,85 +184,8 @@ static int network_write_file_chunk_no_mmap(const int fd, chunkqueue * const cq,
#if defined(NETWORK_WRITE_USE_MMAP)
#include "sys-mmap.h"
#include "sys-setjmp.h"
#define MMAP_CHUNK_SIZE (512*1024)
static off_t mmap_align_offset(off_t start) {
static long pagesize = 0;
if (0 == pagesize) {
pagesize = sysconf(_SC_PAGESIZE);
force_assert(pagesize < MMAP_CHUNK_SIZE);
}
force_assert(start >= (start % pagesize));
return start - (start % pagesize);
}
static int
network_write_file_chunk_remap(chunk * const c)
{
/* mmap buffer if offset is outside old mmap area or not mapped at all */
if (MAP_FAILED == c->file.mmap.start
|| c->offset < c->file.mmap.offset
|| c->offset >= (off_t)(c->file.mmap.offset + c->file.mmap.length)) {
if (MAP_FAILED != c->file.mmap.start) {
munmap(c->file.mmap.start, c->file.mmap.length);
c->file.mmap.start = MAP_FAILED;
}
/* Optimizations for the future:
*
* adaptive mem-mapping
* the problem:
* we mmap() the whole file. If someone has a lot of large files and
* 32-bit machine the virtual address area will be exhausted and we
* will have a failing mmap() call.
* solution:
* only mmap 16M in one chunk and move the window as soon as we have
* finished the first 8M
*
* read-ahead buffering
* the problem:
* sending out several large files in parallel trashes read-ahead
* of the kernel leading to long wait-for-seek times.
* solutions: (increasing complexity)
* 1. use madvise
* 2. use a internal read-ahead buffer in the chunk-structure
* 3. use non-blocking IO for file-transfers
* */
c->file.mmap.offset = mmap_align_offset(c->offset);
/* all mmap()ed areas are MMAP_CHUNK_SIZE
* except the last which might be smaller */
c->file.mmap.length = MMAP_CHUNK_SIZE;
if (c->file.mmap.offset > c->file.length - (off_t)c->file.mmap.length) {
c->file.mmap.length = c->file.length - c->file.mmap.offset;
}
c->file.mmap.start = mmap(NULL, c->file.mmap.length, PROT_READ,
MAP_SHARED, c->file.fd, c->file.mmap.offset);
if (MAP_FAILED == c->file.mmap.start) {
return 0;
}
#if defined(HAVE_MADVISE)
/* don't advise files < 64Kb */
if (c->file.mmap.length > (64*1024)) {
/* darwin 7 is returning EINVAL all the time and I don't know how to
* detect this at runtime.
*
* ignore the return value for now */
madvise(c->file.mmap.start, c->file.mmap.length, MADV_WILLNEED);
}
#endif
}
return 1;
}
static off_t
network_write_setjmp_write_cb (void *fd, const void *data, off_t len)
{
@ -269,33 +194,22 @@ network_write_setjmp_write_cb (void *fd, const void *data, off_t len)
/* next chunk must be FILE_CHUNK. send mmap()ed file with write() */
static int network_write_file_chunk_mmap(const int fd, chunkqueue * const cq, off_t * const p_max_bytes, log_error_st * const errh) {
chunk* const c = cq->first;
chunk * const restrict c = cq->first;
const chunk_file_view * const restrict cfv = (!c->file.is_temp)
? chunkqueue_chunk_file_view(cq->first, 0, errh)/*use default 512k block*/
: NULL;
if (NULL == cfv)
return network_write_file_chunk_no_mmap(fd, cq, p_max_bytes, errh);
off_t toSend = c->file.length - c->offset;
if (toSend > *p_max_bytes) toSend = *p_max_bytes;
if (toSend <= 0) return network_remove_finished_chunks(cq, toSend);
if (c->file.fd < 0 && 0 != chunkqueue_open_file_chunk(cq, errh)) return -1;
if (!network_write_file_chunk_remap(c))
return network_write_file_chunk_no_mmap(fd, cq, p_max_bytes, errh);
off_t mmap_offset = c->offset - c->file.mmap.offset;
off_t mmap_avail = (off_t)c->file.mmap.length - mmap_offset;
const char * const data = c->file.mmap.start + mmap_offset;
const off_t mmap_avail = chunk_file_view_dlen(cfv, c->offset);
const char * const data = chunk_file_view_dptr(cfv, c->offset);
if (toSend > mmap_avail) toSend = mmap_avail;
off_t wr = sys_setjmp_eval3(network_write_setjmp_write_cb,
(void *)(uintptr_t)fd, data, toSend);
#if 0
/*(future: might writev() with multiple maps,
* so will not know which chunk failed)*/
if (wr < 0 && errno == EFAULT) { /*(see sys-setjmp.c)*/
log_error(errh, __FILE__, __LINE__,
"SIGBUS in mmap: %s %d", c->mem->ptr, c->file.fd);
munmap(c->file.mmap.start, c->file.mmap.length);
c->file.mmap.start = MAP_FAILED;
return -1;
}
#endif
return network_write_accounting(fd,cq,p_max_bytes,errh,(ssize_t)wr,toSend);
}

@ -26,6 +26,8 @@
#define PROT_READ PAGE_READONLY
#endif
#define HAVE_MMAP 1
#define munmap(addr, length) UnmapViewOfFile((LPCVOID)(addr))
static inline void *

Loading…
Cancel
Save