From 9211fb3d8680beccb7d3463c093d3f80112b9cd0 Mon Sep 17 00:00:00 2001 From: Glenn Strauss Date: Tue, 12 Jan 2021 18:29:20 -0500 Subject: [PATCH] [mod_deflate] support Accept-Encoding: zstd --- SConstruct | 10 ++++ configure.ac | 31 +++++++++++ src/CMakeLists.txt | 14 ++++- src/Makefile.am | 4 +- src/SConscript | 2 +- src/meson.build | 17 +++++- src/mod_deflate.c | 127 +++++++++++++++++++++++++++++++++++++++++++-- src/server.c | 5 ++ 8 files changed, 202 insertions(+), 8 deletions(-) diff --git a/SConstruct b/SConstruct index 9b524f8d..1873a603 100644 --- a/SConstruct +++ b/SConstruct @@ -269,6 +269,7 @@ vars.AddVariables( PackageVariable('with_xml', 'enable xml support (required for webdav props)', 'no'), BoolVariable('with_xxhash', 'build with system-provided xxhash', 'no'), BoolVariable('with_zlib', 'enable deflate/gzip compression', 'no'), + BoolVariable('with_zstd', 'enable zstd compression', 'no'), BoolVariable('with_all', 'enable all with_* features', 'no'), ) @@ -357,6 +358,7 @@ if 1: LIBXML2 = '', LIBXXHASH = '', LIBZ = '', + LIBZSTD = '', ) autoconf.haveCHeaders([ @@ -731,6 +733,14 @@ if 1: LIBZ = 'z', ) + if env['with_zstd']: + if not autoconf.CheckLibWithHeader('zstd', 'zstd.h', 'C'): + fail("Couldn't find zstd") + autoconf.env.Append( + CPPFLAGS = [ '-DHAVE_ZSTD_H', '-DHAVE_ZSTD' ], + LIBZSTD = 'zstd', + ) + env = autoconf.Finish() if re.compile("cygwin|mingw|midipix").search(env['PLATFORM']): diff --git a/configure.ac b/configure.ac index c234af9c..325a1b4b 100644 --- a/configure.ac +++ b/configure.ac @@ -962,6 +962,37 @@ if test "$WITH_ZLIB" != no; then AC_SUBST([Z_LIB]) fi +dnl zstd +AC_MSG_NOTICE([----------------------------------------]) +AC_MSG_CHECKING([for zstd support]) +AC_ARG_WITH([zstd], + [AS_HELP_STRING([--with-zstd], + [Enable zstd support for mod_deflate] + )], + [WITH_ZSTD=$withval], + [WITH_ZSTD=no] +) +AC_MSG_RESULT([$WITH_ZSTD]) + +if test "$WITH_ZSTD" != no; then + if test "$WITH_ZSTD" != yes; then + ZSTD_LIB="-L$WITH_ZSTD -lzstd" + CPPFLAGS="$CPPFLAGS -I$WITH_ZSTD" + else + AC_CHECK_HEADERS([zstd.h], [], + [AC_MSG_ERROR([zstd headers not found, install them or build without --with-zstd])] + ) + AC_CHECK_LIB([zstd], [ZSTD_versionNumber], + [ZSTD_LIB=-lzstd], + [AC_MSG_ERROR([zstd library not found, install it or build without --with-zstd])] + ) + fi + + AC_DEFINE([HAVE_ZSTD], [1], [libzstd]) + AC_DEFINE([HAVE_ZSTD_H], [1]) + AC_SUBST([ZSTD_LIB]) +fi + dnl bzip2 AC_MSG_NOTICE([----------------------------------------]) AC_MSG_CHECKING([for bzip2 support]) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 983e1c7a..04e46e50 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -32,6 +32,7 @@ option(WITH_WEBDAV_LOCKS "locks in webdav [default: off]") option(WITH_BROTLI "with brotli-support for mod_deflate [default: off]") option(WITH_BZIP "with bzip2-support for mod_deflate [default: off]") option(WITH_ZLIB "with deflate-support for mod_deflate [default: on]" ON) +option(WITH_ZSTD "with zstd-support for mod_deflate [default: off]") option(WITH_KRB5 "with Kerberos5-support for mod_auth [default: off]") option(WITH_LDAP "with LDAP-support for mod_auth mod_vhostdb_ldap [default: off]") option(WITH_PAM "with PAM-support for mod_auth [default: off]") @@ -607,6 +608,14 @@ else() unset(ZLIB_LIBRARY) endif() +if(WITH_ZSTD) + check_include_files(zstd.h HAVE_ZSTD_H) + check_library_exists(zstd ZSTD_versionNumber "" HAVE_ZSTD) +else() + unset(HAVE_ZSTD_H) + unset(HAVE_ZSTD) +endif() + if(WITH_BZIP) check_include_files(bzlib.h HAVE_BZLIB_H) check_library_exists(bz2 BZ2_bzCompress "" HAVE_LIBBZ2) @@ -1063,10 +1072,13 @@ if(WITH_SASL) target_link_libraries(mod_authn_sasl ${L_MOD_AUTHN_SASL}) endif() -if(HAVE_ZLIB_H OR HAVE_BZLIB_H OR HAVE_BROTLI) +if(HAVE_ZLIB_H OR HAVE_ZSTD_H OR HAVE_BZLIB_H OR HAVE_BROTLI) if(HAVE_ZLIB_H) set(L_MOD_DEFLATE ${L_MOD_DEFLATE} ${ZLIB_LIBRARY}) endif() + if(HAVE_ZSTD_H) + set(L_MOD_DEFLATE ${L_MOD_DEFLATE} zstd) + endif() if(HAVE_BZLIB_H) set(L_MOD_DEFLATE ${L_MOD_DEFLATE} bz2) endif() diff --git a/src/Makefile.am b/src/Makefile.am index 724648fa..b9ca95be 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -326,7 +326,7 @@ mod_access_la_LIBADD = $(common_libadd) lib_LTLIBRARIES += mod_deflate.la mod_deflate_la_SOURCES = mod_deflate.c mod_deflate_la_LDFLAGS = $(BROTLI_CFLAGS) $(common_module_ldflags) -mod_deflate_la_LIBADD = $(Z_LIB) $(BZ_LIB) $(BROTLI_LIBS) $(common_libadd) +mod_deflate_la_LIBADD = $(Z_LIB) $(ZSTD_LIB) $(BZ_LIB) $(BROTLI_LIBS) $(common_libadd) lib_LTLIBRARIES += mod_auth.la mod_auth_la_SOURCES = mod_auth.c @@ -525,7 +525,7 @@ lighttpd_LDADD = \ $(common_libadd) \ $(CRYPT_LIB) $(CRYPTO_LIB) $(XXHASH_LIBS) \ $(XML_LIBS) $(SQLITE_LIBS) $(UUID_LIBS) $(ELFTC_LIB) \ - $(PCRE_LIB) $(Z_LIB) $(BZ_LIB) $(BROTLI_LIBS) \ + $(PCRE_LIB) $(Z_LIB) $(ZSTD_LIB) $(BZ_LIB) $(BROTLI_LIBS) \ $(DL_LIB) $(SENDFILE_LIB) $(ATTR_LIB) \ $(FAM_LIBS) $(LIBEV_LIBS) $(LIBUNWIND_LIBS) lighttpd_LDFLAGS = -export-dynamic diff --git a/src/SConscript b/src/SConscript index 6e60a3a3..defade2d 100644 --- a/src/SConscript +++ b/src/SConscript @@ -105,7 +105,7 @@ modules = { 'mod_auth' : { 'src' : [ 'mod_auth.c' ], 'lib' : [ env['LIBCRYPTO'] ] }, 'mod_authn_file' : { 'src' : [ 'mod_authn_file.c' ], 'lib' : [ env['LIBCRYPT'], env['LIBCRYPTO'] ] }, 'mod_cgi' : { 'src' : [ 'mod_cgi.c' ] }, - 'mod_deflate' : { 'src' : [ 'mod_deflate.c' ], 'lib' : [ env['LIBZ'], env['LIBBZ2'], env['LIBBROTLI'], 'm' ] }, + 'mod_deflate' : { 'src' : [ 'mod_deflate.c' ], 'lib' : [ env['LIBZ'], env['LIBZSTD'], env['LIBBZ2'], env['LIBBROTLI'], 'm' ] }, 'mod_dirlisting' : { 'src' : [ 'mod_dirlisting.c' ], 'lib' : [ env['LIBPCRE'] ] }, 'mod_evasive' : { 'src' : [ 'mod_evasive.c' ] }, 'mod_evhost' : { 'src' : [ 'mod_evhost.c' ] }, diff --git a/src/meson.build b/src/meson.build index 1ad451b7..b8470e85 100644 --- a/src/meson.build +++ b/src/meson.build @@ -683,6 +683,21 @@ if get_option('with_zlib') conf_data.set('HAVE_LIBZ', true) endif +libzstd = [] +if get_option('with_zstd') + libz = dependency('zstd', required: false) + if libzstd.found() + libzstd = [ libzstd ] + else + libzstd = [ compiler.find_library('zstd') ] + if not(compiler.has_function('ZSTD_versionNumber', args: defs, dependencies: libzstd, prefix: '#include ')) + error('Couldn\'t find zstd header / library') + endif + endif + conf_data.set('HAVE_ZSTD_H', true) + conf_data.set('HAVE_ZSTD', true) +endif + configure_file( output : 'config.h', configuration : conf_data, @@ -977,7 +992,7 @@ modules = [ [ 'mod_alias', [ 'mod_alias.c' ] ], [ 'mod_auth', [ 'mod_auth.c' ], [ libcrypto ] ], [ 'mod_authn_file', [ 'mod_authn_file.c' ], [ libcrypt, libcrypto ] ], - [ 'mod_deflate', [ 'mod_deflate.c' ], libbz2 + libz + libbrotli ], + [ 'mod_deflate', [ 'mod_deflate.c' ], libbz2 + libz + libzstd + libbrotli ], [ 'mod_dirlisting', [ 'mod_dirlisting.c' ], libpcre ], [ 'mod_evasive', [ 'mod_evasive.c' ] ], [ 'mod_evhost', [ 'mod_evhost.c' ] ], diff --git a/src/mod_deflate.c b/src/mod_deflate.c index d3e2f3b1..d88c1f8b 100644 --- a/src/mod_deflate.c +++ b/src/mod_deflate.c @@ -144,6 +144,14 @@ # include #endif +#if defined HAVE_ZSTD_H && defined HAVE_ZSTD +# define USE_ZSTD +/* FIXME: wrote initial implementation to support zstd and then realized + * bufferless streaming compression is an experimental (unstable) zstd API */ +# define ZSTD_STATIC_LINKING_ONLY +# include +#endif + #if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP #define USE_MMAP @@ -170,6 +178,7 @@ static void sigbus_handler(int sig) { #define HTTP_ACCEPT_ENCODING_X_GZIP BV(5) #define HTTP_ACCEPT_ENCODING_X_BZIP2 BV(6) #define HTTP_ACCEPT_ENCODING_BR BV(7) +#define HTTP_ACCEPT_ENCODING_ZSTD BV(8) typedef struct { const array *mimetypes; @@ -202,6 +211,9 @@ typedef struct { #endif #ifdef USE_BROTLI BrotliEncoderState *br; + #endif + #ifdef USE_ZSTD + ZSTD_CCtx *cctx; #endif int dummy; } u; @@ -244,7 +256,12 @@ static void handler_ctx_free(handler_ctx *hctx) { INIT_FUNC(mod_deflate_init) { plugin_data * const p = calloc(1, sizeof(plugin_data)); + #ifdef USE_ZSTD + buffer_string_prepare_copy(&p->tmp_buf, /* 131072 */ + ZSTD_COMPRESSBOUND(ZSTD_BLOCKSIZE_MAX)); + #else buffer_string_prepare_copy(&p->tmp_buf, 65536); + #endif return p; } @@ -384,7 +401,8 @@ static short mod_deflate_encodings_to_flags(const array *encodings) { short allowed_encodings = 0; if (encodings->used) { for (uint32_t j = 0; j < encodings->used; ++j) { - #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) + #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \ + || defined(USE_ZSTD) data_string *ds = (data_string *)encodings->data[j]; #endif #ifdef USE_ZLIB @@ -411,6 +429,10 @@ static short mod_deflate_encodings_to_flags(const array *encodings) { if (NULL != strstr(ds->value.ptr, "br")) allowed_encodings |= HTTP_ACCEPT_ENCODING_BR; #endif + #ifdef USE_ZSTD + if (NULL != strstr(ds->value.ptr, "zstd")) + allowed_encodings |= HTTP_ACCEPT_ENCODING_ZSTD; + #endif } } else { @@ -427,6 +449,9 @@ static short mod_deflate_encodings_to_flags(const array *encodings) { #ifdef USE_BROTLI allowed_encodings |= HTTP_ACCEPT_ENCODING_BR; #endif + #ifdef USE_ZSTD + allowed_encodings |= HTTP_ACCEPT_ENCODING_ZSTD; + #endif } return allowed_encodings; } @@ -612,7 +637,8 @@ SETDEFAULTS_FUNC(mod_deflate_set_defaults) { } -#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) +#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \ + || defined(USE_ZSTD) static int mod_deflate_cache_file_append (handler_ctx * const hctx, const char *out, size_t len) { ssize_t wr; do { @@ -933,6 +959,71 @@ static int stream_br_end(handler_ctx *hctx) { #endif +#ifdef USE_ZSTD + +static int stream_zstd_init(handler_ctx *hctx) { + ZSTD_CCtx * const cctx = hctx->u.cctx = ZSTD_createCCtx(); + if (NULL == cctx) return -1; + return 0; +} + +static int stream_zstd_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) { + ZSTD_CCtx * const cctx = hctx->u.cctx; + + /* Note: each chunkqueue chunk is sent in its own frame, which may be + * suboptimal. lighttpd might read FILE_CHUNK into a reused buffer, so + * we can not meet ZSTD_compressContinue() requirement that prior input + * is still accessible and unmodified (up to maximum distance size). + * Also, chunkqueue_mark_written() on MEM_CHUNK might result in something + * else reusing those chunk buffers + * + * future: migrate to use Zstd streaming API */ + + /* future: consider allowing tunables by encoder algorithm, + * (i.e. not generic "compression_level" across all compression algorithms) + * (ZSTD_CCtx_setParameter()) */ + /*(note: we ignore any errors while tuning parameters here)*/ + const plugin_data * const p = hctx->plugin_data; + int level = ZSTD_CLEVEL_DEFAULT; + if (p->conf.compression_level >= 0) /* -1 is lighttpd default for "unset" */ + level = p->conf.compression_level; + ZSTD_compressBegin(cctx, level); + + char * const out = hctx->output->ptr; + const size_t outsz = hctx->output->size; + hctx->bytes_in += st_size; + for (off_t off = 0; st_size; ) { + /* XXX: size check must match mod_deflate_init ZSTD_COMPRESSBOUND arg */ + size_t len = (size_t)st_size; + const size_t rv = (len > ZSTD_BLOCKSIZE_MAX) + ? ZSTD_compressContinue(cctx, out, outsz, start+off, + (len = ZSTD_BLOCKSIZE_MAX)) + : ZSTD_compressEnd(cctx, out, outsz, start+off, len); + off += (off_t)len; + st_size -= (off_t)len; + if (ZSTD_isError(rv)) return -1; + hctx->bytes_out += (off_t)rv; + if (0 != stream_http_chunk_append_mem(hctx, out, rv)) + return -1; + } + return 0; +} + +static int stream_zstd_flush(handler_ctx * const hctx, int end) { + UNUSED(hctx); + UNUSED(end); + return 0; +} + +static int stream_zstd_end(handler_ctx *hctx) { + ZSTD_CCtx * const cctx = hctx->u.cctx; + ZSTD_freeCCtx(cctx); + return 0; +} + +#endif + + static int mod_deflate_stream_init(handler_ctx *hctx) { switch(hctx->compression_type) { #ifdef USE_ZLIB @@ -947,6 +1038,10 @@ static int mod_deflate_stream_init(handler_ctx *hctx) { #ifdef USE_BROTLI case HTTP_ACCEPT_ENCODING_BR: return stream_br_init(hctx); +#endif +#ifdef USE_ZSTD + case HTTP_ACCEPT_ENCODING_ZSTD: + return stream_zstd_init(hctx); #endif default: return -1; @@ -968,6 +1063,10 @@ static int mod_deflate_compress(handler_ctx * const hctx, unsigned char * const #ifdef USE_BROTLI case HTTP_ACCEPT_ENCODING_BR: return stream_br_compress(hctx, start, st_size); +#endif +#ifdef USE_ZSTD + case HTTP_ACCEPT_ENCODING_ZSTD: + return stream_zstd_compress(hctx, start, st_size); #endif default: UNUSED(start); @@ -990,6 +1089,10 @@ static int mod_deflate_stream_flush(handler_ctx * const hctx, int end) { #ifdef USE_BROTLI case HTTP_ACCEPT_ENCODING_BR: return stream_br_flush(hctx, end); +#endif +#ifdef USE_ZSTD + case HTTP_ACCEPT_ENCODING_ZSTD: + return stream_zstd_flush(hctx, end); #endif default: UNUSED(end); @@ -1023,6 +1126,10 @@ static int mod_deflate_stream_end(handler_ctx *hctx) { #ifdef USE_BROTLI case HTTP_ACCEPT_ENCODING_BR: return stream_br_end(hctx); +#endif +#ifdef USE_ZSTD + case HTTP_ACCEPT_ENCODING_ZSTD: + return stream_zstd_end(hctx); #endif default: return -1; @@ -1263,7 +1370,8 @@ static handler_t deflate_compress_response(request_st * const r, handler_ctx * c static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const char **label) { /* get client side support encodings */ int accept_encoding = 0; - #if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI) + #if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI) \ + && !defined(USE_ZSTD) UNUSED(value); UNUSED(label); #else @@ -1284,6 +1392,13 @@ static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const #ifdef USE_ZLIB if (0 == memcmp(v, "gzip", 4)) accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP; + #endif + #ifdef USE_ZSTD + #ifdef USE_ZLIB + else + #endif + if (0 == memcmp(v, "zstd", 4)) + accept_encoding |= HTTP_ACCEPT_ENCODING_ZSTD; #endif break; case 5: @@ -1330,6 +1445,12 @@ static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const accept_encoding &= p->conf.allowed_encodings; /* select best matching encoding */ +#ifdef USE_ZSTD + if (accept_encoding & HTTP_ACCEPT_ENCODING_ZSTD) { + *label = "zstd"; + return HTTP_ACCEPT_ENCODING_ZSTD; + } else +#endif #ifdef USE_BROTLI if (accept_encoding & HTTP_ACCEPT_ENCODING_BR) { *label = "br"; diff --git a/src/server.c b/src/server.c index 3b301260..dcabb207 100644 --- a/src/server.c +++ b/src/server.c @@ -555,6 +555,11 @@ static void show_features (void) { #else "\t- zlib support\n" #endif +#if defined HAVE_ZSTD_H && defined HAVE_ZSTD + "\t+ zstd support\n" +#else + "\t- zstd support\n" +#endif #if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2 "\t+ bzip2 support\n" #else