|
|
|
@ -144,6 +144,14 @@
|
|
|
|
|
# include <brotli/encode.h>
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#if defined HAVE_ZSTD_H && defined HAVE_ZSTD
|
|
|
|
|
# define USE_ZSTD
|
|
|
|
|
/* FIXME: wrote initial implementation to support zstd and then realized
|
|
|
|
|
* bufferless streaming compression is an experimental (unstable) zstd API */
|
|
|
|
|
# define ZSTD_STATIC_LINKING_ONLY
|
|
|
|
|
# include <zstd.h>
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP
|
|
|
|
|
#define USE_MMAP
|
|
|
|
|
|
|
|
|
@ -170,6 +178,7 @@ static void sigbus_handler(int sig) {
|
|
|
|
|
#define HTTP_ACCEPT_ENCODING_X_GZIP BV(5)
|
|
|
|
|
#define HTTP_ACCEPT_ENCODING_X_BZIP2 BV(6)
|
|
|
|
|
#define HTTP_ACCEPT_ENCODING_BR BV(7)
|
|
|
|
|
#define HTTP_ACCEPT_ENCODING_ZSTD BV(8)
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
const array *mimetypes;
|
|
|
|
@ -202,6 +211,9 @@ typedef struct {
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef USE_BROTLI
|
|
|
|
|
BrotliEncoderState *br;
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef USE_ZSTD
|
|
|
|
|
ZSTD_CCtx *cctx;
|
|
|
|
|
#endif
|
|
|
|
|
int dummy;
|
|
|
|
|
} u;
|
|
|
|
@ -244,7 +256,12 @@ static void handler_ctx_free(handler_ctx *hctx) {
|
|
|
|
|
|
|
|
|
|
INIT_FUNC(mod_deflate_init) {
|
|
|
|
|
plugin_data * const p = calloc(1, sizeof(plugin_data));
|
|
|
|
|
#ifdef USE_ZSTD
|
|
|
|
|
buffer_string_prepare_copy(&p->tmp_buf, /* 131072 */
|
|
|
|
|
ZSTD_COMPRESSBOUND(ZSTD_BLOCKSIZE_MAX));
|
|
|
|
|
#else
|
|
|
|
|
buffer_string_prepare_copy(&p->tmp_buf, 65536);
|
|
|
|
|
#endif
|
|
|
|
|
return p;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@ -384,7 +401,8 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
|
|
|
|
|
short allowed_encodings = 0;
|
|
|
|
|
if (encodings->used) {
|
|
|
|
|
for (uint32_t j = 0; j < encodings->used; ++j) {
|
|
|
|
|
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI)
|
|
|
|
|
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \
|
|
|
|
|
|| defined(USE_ZSTD)
|
|
|
|
|
data_string *ds = (data_string *)encodings->data[j];
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef USE_ZLIB
|
|
|
|
@ -411,6 +429,10 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
|
|
|
|
|
if (NULL != strstr(ds->value.ptr, "br"))
|
|
|
|
|
allowed_encodings |= HTTP_ACCEPT_ENCODING_BR;
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef USE_ZSTD
|
|
|
|
|
if (NULL != strstr(ds->value.ptr, "zstd"))
|
|
|
|
|
allowed_encodings |= HTTP_ACCEPT_ENCODING_ZSTD;
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else {
|
|
|
|
@ -427,6 +449,9 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
|
|
|
|
|
#ifdef USE_BROTLI
|
|
|
|
|
allowed_encodings |= HTTP_ACCEPT_ENCODING_BR;
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef USE_ZSTD
|
|
|
|
|
allowed_encodings |= HTTP_ACCEPT_ENCODING_ZSTD;
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
return allowed_encodings;
|
|
|
|
|
}
|
|
|
|
@ -612,7 +637,8 @@ SETDEFAULTS_FUNC(mod_deflate_set_defaults) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI)
|
|
|
|
|
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \
|
|
|
|
|
|| defined(USE_ZSTD)
|
|
|
|
|
static int mod_deflate_cache_file_append (handler_ctx * const hctx, const char *out, size_t len) {
|
|
|
|
|
ssize_t wr;
|
|
|
|
|
do {
|
|
|
|
@ -933,6 +959,71 @@ static int stream_br_end(handler_ctx *hctx) {
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#ifdef USE_ZSTD
|
|
|
|
|
|
|
|
|
|
static int stream_zstd_init(handler_ctx *hctx) {
|
|
|
|
|
ZSTD_CCtx * const cctx = hctx->u.cctx = ZSTD_createCCtx();
|
|
|
|
|
if (NULL == cctx) return -1;
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int stream_zstd_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) {
|
|
|
|
|
ZSTD_CCtx * const cctx = hctx->u.cctx;
|
|
|
|
|
|
|
|
|
|
/* Note: each chunkqueue chunk is sent in its own frame, which may be
|
|
|
|
|
* suboptimal. lighttpd might read FILE_CHUNK into a reused buffer, so
|
|
|
|
|
* we can not meet ZSTD_compressContinue() requirement that prior input
|
|
|
|
|
* is still accessible and unmodified (up to maximum distance size).
|
|
|
|
|
* Also, chunkqueue_mark_written() on MEM_CHUNK might result in something
|
|
|
|
|
* else reusing those chunk buffers
|
|
|
|
|
*
|
|
|
|
|
* future: migrate to use Zstd streaming API */
|
|
|
|
|
|
|
|
|
|
/* future: consider allowing tunables by encoder algorithm,
|
|
|
|
|
* (i.e. not generic "compression_level" across all compression algorithms)
|
|
|
|
|
* (ZSTD_CCtx_setParameter()) */
|
|
|
|
|
/*(note: we ignore any errors while tuning parameters here)*/
|
|
|
|
|
const plugin_data * const p = hctx->plugin_data;
|
|
|
|
|
int level = ZSTD_CLEVEL_DEFAULT;
|
|
|
|
|
if (p->conf.compression_level >= 0) /* -1 is lighttpd default for "unset" */
|
|
|
|
|
level = p->conf.compression_level;
|
|
|
|
|
ZSTD_compressBegin(cctx, level);
|
|
|
|
|
|
|
|
|
|
char * const out = hctx->output->ptr;
|
|
|
|
|
const size_t outsz = hctx->output->size;
|
|
|
|
|
hctx->bytes_in += st_size;
|
|
|
|
|
for (off_t off = 0; st_size; ) {
|
|
|
|
|
/* XXX: size check must match mod_deflate_init ZSTD_COMPRESSBOUND arg */
|
|
|
|
|
size_t len = (size_t)st_size;
|
|
|
|
|
const size_t rv = (len > ZSTD_BLOCKSIZE_MAX)
|
|
|
|
|
? ZSTD_compressContinue(cctx, out, outsz, start+off,
|
|
|
|
|
(len = ZSTD_BLOCKSIZE_MAX))
|
|
|
|
|
: ZSTD_compressEnd(cctx, out, outsz, start+off, len);
|
|
|
|
|
off += (off_t)len;
|
|
|
|
|
st_size -= (off_t)len;
|
|
|
|
|
if (ZSTD_isError(rv)) return -1;
|
|
|
|
|
hctx->bytes_out += (off_t)rv;
|
|
|
|
|
if (0 != stream_http_chunk_append_mem(hctx, out, rv))
|
|
|
|
|
return -1;
|
|
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int stream_zstd_flush(handler_ctx * const hctx, int end) {
|
|
|
|
|
UNUSED(hctx);
|
|
|
|
|
UNUSED(end);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int stream_zstd_end(handler_ctx *hctx) {
|
|
|
|
|
ZSTD_CCtx * const cctx = hctx->u.cctx;
|
|
|
|
|
ZSTD_freeCCtx(cctx);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
static int mod_deflate_stream_init(handler_ctx *hctx) {
|
|
|
|
|
switch(hctx->compression_type) {
|
|
|
|
|
#ifdef USE_ZLIB
|
|
|
|
@ -947,6 +1038,10 @@ static int mod_deflate_stream_init(handler_ctx *hctx) {
|
|
|
|
|
#ifdef USE_BROTLI
|
|
|
|
|
case HTTP_ACCEPT_ENCODING_BR:
|
|
|
|
|
return stream_br_init(hctx);
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef USE_ZSTD
|
|
|
|
|
case HTTP_ACCEPT_ENCODING_ZSTD:
|
|
|
|
|
return stream_zstd_init(hctx);
|
|
|
|
|
#endif
|
|
|
|
|
default:
|
|
|
|
|
return -1;
|
|
|
|
@ -968,6 +1063,10 @@ static int mod_deflate_compress(handler_ctx * const hctx, unsigned char * const
|
|
|
|
|
#ifdef USE_BROTLI
|
|
|
|
|
case HTTP_ACCEPT_ENCODING_BR:
|
|
|
|
|
return stream_br_compress(hctx, start, st_size);
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef USE_ZSTD
|
|
|
|
|
case HTTP_ACCEPT_ENCODING_ZSTD:
|
|
|
|
|
return stream_zstd_compress(hctx, start, st_size);
|
|
|
|
|
#endif
|
|
|
|
|
default:
|
|
|
|
|
UNUSED(start);
|
|
|
|
@ -990,6 +1089,10 @@ static int mod_deflate_stream_flush(handler_ctx * const hctx, int end) {
|
|
|
|
|
#ifdef USE_BROTLI
|
|
|
|
|
case HTTP_ACCEPT_ENCODING_BR:
|
|
|
|
|
return stream_br_flush(hctx, end);
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef USE_ZSTD
|
|
|
|
|
case HTTP_ACCEPT_ENCODING_ZSTD:
|
|
|
|
|
return stream_zstd_flush(hctx, end);
|
|
|
|
|
#endif
|
|
|
|
|
default:
|
|
|
|
|
UNUSED(end);
|
|
|
|
@ -1023,6 +1126,10 @@ static int mod_deflate_stream_end(handler_ctx *hctx) {
|
|
|
|
|
#ifdef USE_BROTLI
|
|
|
|
|
case HTTP_ACCEPT_ENCODING_BR:
|
|
|
|
|
return stream_br_end(hctx);
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef USE_ZSTD
|
|
|
|
|
case HTTP_ACCEPT_ENCODING_ZSTD:
|
|
|
|
|
return stream_zstd_end(hctx);
|
|
|
|
|
#endif
|
|
|
|
|
default:
|
|
|
|
|
return -1;
|
|
|
|
@ -1263,7 +1370,8 @@ static handler_t deflate_compress_response(request_st * const r, handler_ctx * c
|
|
|
|
|
static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const char **label) {
|
|
|
|
|
/* get client side support encodings */
|
|
|
|
|
int accept_encoding = 0;
|
|
|
|
|
#if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI)
|
|
|
|
|
#if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI) \
|
|
|
|
|
&& !defined(USE_ZSTD)
|
|
|
|
|
UNUSED(value);
|
|
|
|
|
UNUSED(label);
|
|
|
|
|
#else
|
|
|
|
@ -1284,6 +1392,13 @@ static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const
|
|
|
|
|
#ifdef USE_ZLIB
|
|
|
|
|
if (0 == memcmp(v, "gzip", 4))
|
|
|
|
|
accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP;
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef USE_ZSTD
|
|
|
|
|
#ifdef USE_ZLIB
|
|
|
|
|
else
|
|
|
|
|
#endif
|
|
|
|
|
if (0 == memcmp(v, "zstd", 4))
|
|
|
|
|
accept_encoding |= HTTP_ACCEPT_ENCODING_ZSTD;
|
|
|
|
|
#endif
|
|
|
|
|
break;
|
|
|
|
|
case 5:
|
|
|
|
@ -1330,6 +1445,12 @@ static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const
|
|
|
|
|
accept_encoding &= p->conf.allowed_encodings;
|
|
|
|
|
|
|
|
|
|
/* select best matching encoding */
|
|
|
|
|
#ifdef USE_ZSTD
|
|
|
|
|
if (accept_encoding & HTTP_ACCEPT_ENCODING_ZSTD) {
|
|
|
|
|
*label = "zstd";
|
|
|
|
|
return HTTP_ACCEPT_ENCODING_ZSTD;
|
|
|
|
|
} else
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef USE_BROTLI
|
|
|
|
|
if (accept_encoding & HTTP_ACCEPT_ENCODING_BR) {
|
|
|
|
|
*label = "br";
|
|
|
|
|