[mod_deflate] support Accept-Encoding: zstd

SConstruct

@@ -269,6 +269,7 @@ vars.AddVariables(
 	PackageVariable('with_xml', 'enable xml support (required for webdav props)', 'no'),
 	BoolVariable('with_xxhash', 'build with system-provided xxhash', 'no'),
 	BoolVariable('with_zlib', 'enable deflate/gzip compression', 'no'),
+	BoolVariable('with_zstd', 'enable zstd compression', 'no'),
 
 	BoolVariable('with_all', 'enable all with_* features', 'no'),
 )
@@ -357,6 +358,7 @@ if 1:
 		LIBXML2 = '',
 		LIBXXHASH = '',
 		LIBZ = '',
+		LIBZSTD = '',
 	)
 
 	autoconf.haveCHeaders([
@@ -731,6 +733,14 @@ if 1:
 			LIBZ = 'z',
 		)
 
+	if env['with_zstd']:
+		if not autoconf.CheckLibWithHeader('zstd', 'zstd.h', 'C'):
+			fail("Couldn't find zstd")
+		autoconf.env.Append(
+			CPPFLAGS = [ '-DHAVE_ZSTD_H', '-DHAVE_ZSTD' ],
+			LIBZSTD = 'zstd',
+		)
+
 	env = autoconf.Finish()
 
 if re.compile("cygwin|mingw|midipix").search(env['PLATFORM']):

configure.ac

@@ -962,6 +962,37 @@ if test "$WITH_ZLIB" != no; then
   AC_SUBST([Z_LIB])
 fi
 
+dnl zstd
+AC_MSG_NOTICE([----------------------------------------])
+AC_MSG_CHECKING([for zstd support])
+AC_ARG_WITH([zstd],
+  [AS_HELP_STRING([--with-zstd],
+    [Enable zstd support for mod_deflate]
+  )],
+  [WITH_ZSTD=$withval],
+  [WITH_ZSTD=no]
+)
+AC_MSG_RESULT([$WITH_ZSTD])
+
+if test "$WITH_ZSTD" != no; then
+  if test "$WITH_ZSTD" != yes; then
+    ZSTD_LIB="-L$WITH_ZSTD -lzstd"
+    CPPFLAGS="$CPPFLAGS -I$WITH_ZSTD"
+  else
+    AC_CHECK_HEADERS([zstd.h], [],
+      [AC_MSG_ERROR([zstd headers not found, install them or build without --with-zstd])]
+    )
+    AC_CHECK_LIB([zstd], [ZSTD_versionNumber],
+      [ZSTD_LIB=-lzstd],
+      [AC_MSG_ERROR([zstd library not found, install it or build without --with-zstd])]
+    )
+  fi
+
+  AC_DEFINE([HAVE_ZSTD], [1], [libzstd])
+  AC_DEFINE([HAVE_ZSTD_H], [1])
+  AC_SUBST([ZSTD_LIB])
+fi
+
 dnl bzip2
 AC_MSG_NOTICE([----------------------------------------])
 AC_MSG_CHECKING([for bzip2 support])

src/CMakeLists.txt

@@ -32,6 +32,7 @@ option(WITH_WEBDAV_LOCKS "locks in webdav [default: off]")
 option(WITH_BROTLI "with brotli-support for mod_deflate [default: off]")
 option(WITH_BZIP "with bzip2-support for mod_deflate [default: off]")
 option(WITH_ZLIB "with deflate-support for mod_deflate [default: on]" ON)
+option(WITH_ZSTD "with zstd-support for mod_deflate [default: off]")
 option(WITH_KRB5 "with Kerberos5-support for mod_auth [default: off]")
 option(WITH_LDAP "with LDAP-support for mod_auth mod_vhostdb_ldap [default: off]")
 option(WITH_PAM "with PAM-support for mod_auth [default: off]")
@@ -607,6 +608,14 @@ else()
 	unset(ZLIB_LIBRARY)
 endif()
 
+if(WITH_ZSTD)
+	check_include_files(zstd.h HAVE_ZSTD_H)
+	check_library_exists(zstd ZSTD_versionNumber "" HAVE_ZSTD)
+else()
+	unset(HAVE_ZSTD_H)
+	unset(HAVE_ZSTD)
+endif()
+
 if(WITH_BZIP)
 	check_include_files(bzlib.h HAVE_BZLIB_H)
 	check_library_exists(bz2 BZ2_bzCompress "" HAVE_LIBBZ2)
@@ -1063,10 +1072,13 @@ if(WITH_SASL)
 	target_link_libraries(mod_authn_sasl ${L_MOD_AUTHN_SASL})
 endif()
 
-if(HAVE_ZLIB_H OR HAVE_BZLIB_H OR HAVE_BROTLI)
+if(HAVE_ZLIB_H OR HAVE_ZSTD_H OR HAVE_BZLIB_H OR HAVE_BROTLI)
 	if(HAVE_ZLIB_H)
 		set(L_MOD_DEFLATE ${L_MOD_DEFLATE} ${ZLIB_LIBRARY})
 	endif()
+	if(HAVE_ZSTD_H)
+		set(L_MOD_DEFLATE ${L_MOD_DEFLATE} zstd)
+	endif()
 	if(HAVE_BZLIB_H)
 		set(L_MOD_DEFLATE ${L_MOD_DEFLATE} bz2)
 	endif()

src/Makefile.am

@@ -326,7 +326,7 @@ mod_access_la_LIBADD = $(common_libadd)
 lib_LTLIBRARIES += mod_deflate.la
 mod_deflate_la_SOURCES = mod_deflate.c
 mod_deflate_la_LDFLAGS = $(BROTLI_CFLAGS) $(common_module_ldflags)
-mod_deflate_la_LIBADD = $(Z_LIB) $(BZ_LIB) $(BROTLI_LIBS) $(common_libadd)
+mod_deflate_la_LIBADD = $(Z_LIB) $(ZSTD_LIB) $(BZ_LIB) $(BROTLI_LIBS) $(common_libadd)
 
 lib_LTLIBRARIES += mod_auth.la
 mod_auth_la_SOURCES = mod_auth.c
@@ -525,7 +525,7 @@ lighttpd_LDADD = \
   $(common_libadd) \
   $(CRYPT_LIB) $(CRYPTO_LIB) $(XXHASH_LIBS) \
   $(XML_LIBS) $(SQLITE_LIBS) $(UUID_LIBS) $(ELFTC_LIB) \
-  $(PCRE_LIB) $(Z_LIB) $(BZ_LIB) $(BROTLI_LIBS) \
+  $(PCRE_LIB) $(Z_LIB) $(ZSTD_LIB) $(BZ_LIB) $(BROTLI_LIBS) \
   $(DL_LIB) $(SENDFILE_LIB) $(ATTR_LIB) \
   $(FAM_LIBS) $(LIBEV_LIBS) $(LIBUNWIND_LIBS)
 lighttpd_LDFLAGS = -export-dynamic

src/SConscript

@@ -105,7 +105,7 @@ modules = {
 	'mod_auth' : { 'src' : [ 'mod_auth.c' ], 'lib' : [ env['LIBCRYPTO'] ] },
 	'mod_authn_file' : { 'src' : [ 'mod_authn_file.c' ], 'lib' : [ env['LIBCRYPT'], env['LIBCRYPTO'] ] },
 	'mod_cgi' : { 'src' : [ 'mod_cgi.c' ] },
-	'mod_deflate' : { 'src' : [ 'mod_deflate.c' ], 'lib' : [ env['LIBZ'], env['LIBBZ2'], env['LIBBROTLI'], 'm' ] },
+	'mod_deflate' : { 'src' : [ 'mod_deflate.c' ], 'lib' : [ env['LIBZ'], env['LIBZSTD'], env['LIBBZ2'], env['LIBBROTLI'], 'm' ] },
 	'mod_dirlisting' : { 'src' : [ 'mod_dirlisting.c' ], 'lib' : [ env['LIBPCRE'] ] },
 	'mod_evasive' : { 'src' : [ 'mod_evasive.c' ] },
 	'mod_evhost' : { 'src' : [ 'mod_evhost.c' ] },

src/meson.build

@@ -683,6 +683,21 @@ if get_option('with_zlib')
 	conf_data.set('HAVE_LIBZ', true)
 endif
 
+libzstd = []
+if get_option('with_zstd')
+	libz = dependency('zstd', required: false)
+	if libzstd.found()
+		libzstd = [ libzstd ]
+	else
+		libzstd = [ compiler.find_library('zstd') ]
+		if not(compiler.has_function('ZSTD_versionNumber', args: defs, dependencies: libzstd, prefix: '#include <zstd.h>'))
+			error('Couldn\'t find zstd header / library')
+		endif
+	endif
+	conf_data.set('HAVE_ZSTD_H', true)
+	conf_data.set('HAVE_ZSTD', true)
+endif
+
 configure_file(
 	output : 'config.h',
 	configuration : conf_data,
@@ -977,7 +992,7 @@ modules = [
 	[ 'mod_alias', [ 'mod_alias.c' ] ],
 	[ 'mod_auth', [ 'mod_auth.c' ], [ libcrypto ] ],
 	[ 'mod_authn_file', [ 'mod_authn_file.c' ], [ libcrypt, libcrypto ] ],
-	[ 'mod_deflate', [ 'mod_deflate.c' ], libbz2 + libz + libbrotli ],
+	[ 'mod_deflate', [ 'mod_deflate.c' ], libbz2 + libz + libzstd + libbrotli ],
 	[ 'mod_dirlisting', [ 'mod_dirlisting.c' ], libpcre ],
 	[ 'mod_evasive', [ 'mod_evasive.c' ] ],
 	[ 'mod_evhost', [ 'mod_evhost.c' ] ],

src/mod_deflate.c

@@ -144,6 +144,14 @@
 # include <brotli/encode.h>
 #endif
 
+#if defined HAVE_ZSTD_H && defined HAVE_ZSTD
+# define USE_ZSTD
+/* FIXME: wrote initial implementation to support zstd and then realized
+ * bufferless streaming compression is an experimental (unstable) zstd API */
+# define ZSTD_STATIC_LINKING_ONLY
+# include <zstd.h>
+#endif
+
 #if defined HAVE_SYS_MMAN_H && defined HAVE_MMAP && defined ENABLE_MMAP
 #define USE_MMAP
 
@@ -170,6 +178,7 @@ static void sigbus_handler(int sig) {
 #define HTTP_ACCEPT_ENCODING_X_GZIP   BV(5)
 #define HTTP_ACCEPT_ENCODING_X_BZIP2  BV(6)
 #define HTTP_ACCEPT_ENCODING_BR       BV(7)
+#define HTTP_ACCEPT_ENCODING_ZSTD     BV(8)
 
 typedef struct {
 	const array	*mimetypes;
@@ -202,6 +211,9 @@ typedef struct {
 	      #endif
 	      #ifdef USE_BROTLI
 		BrotliEncoderState *br;
+	      #endif
+	      #ifdef USE_ZSTD
+		ZSTD_CCtx *cctx;
 	      #endif
 		int dummy;
 	} u;
@@ -244,7 +256,12 @@ static void handler_ctx_free(handler_ctx *hctx) {
 
 INIT_FUNC(mod_deflate_init) {
     plugin_data * const p = calloc(1, sizeof(plugin_data));
+  #ifdef USE_ZSTD
+    buffer_string_prepare_copy(&p->tmp_buf,       /* 131072 */
+                               ZSTD_COMPRESSBOUND(ZSTD_BLOCKSIZE_MAX));
+  #else
     buffer_string_prepare_copy(&p->tmp_buf, 65536);
+  #endif
     return p;
 }
 
@@ -384,7 +401,8 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
     short allowed_encodings = 0;
     if (encodings->used) {
         for (uint32_t j = 0; j < encodings->used; ++j) {
-          #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI)
+          #if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \
+           || defined(USE_ZSTD)
             data_string *ds = (data_string *)encodings->data[j];
           #endif
           #ifdef USE_ZLIB
@@ -411,6 +429,10 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
             if (NULL != strstr(ds->value.ptr, "br"))
                 allowed_encodings |= HTTP_ACCEPT_ENCODING_BR;
           #endif
+          #ifdef USE_ZSTD
+            if (NULL != strstr(ds->value.ptr, "zstd"))
+                allowed_encodings |= HTTP_ACCEPT_ENCODING_ZSTD;
+          #endif
         }
     }
     else {
@@ -427,6 +449,9 @@ static short mod_deflate_encodings_to_flags(const array *encodings) {
       #ifdef USE_BROTLI
         allowed_encodings |= HTTP_ACCEPT_ENCODING_BR;
       #endif
+      #ifdef USE_ZSTD
+        allowed_encodings |= HTTP_ACCEPT_ENCODING_ZSTD;
+      #endif
     }
     return allowed_encodings;
 }
@@ -612,7 +637,8 @@ SETDEFAULTS_FUNC(mod_deflate_set_defaults) {
 }
 
 
-#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI)
+#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI) \
+ || defined(USE_ZSTD)
 static int mod_deflate_cache_file_append (handler_ctx * const hctx, const char *out, size_t len) {
     ssize_t wr;
     do {
@@ -933,6 +959,71 @@ static int stream_br_end(handler_ctx *hctx) {
 #endif
 
 
+#ifdef USE_ZSTD
+
+static int stream_zstd_init(handler_ctx *hctx) {
+    ZSTD_CCtx * const cctx = hctx->u.cctx = ZSTD_createCCtx();
+    if (NULL == cctx) return -1;
+    return 0;
+}
+
+static int stream_zstd_compress(handler_ctx * const hctx, unsigned char * const start, off_t st_size) {
+    ZSTD_CCtx * const cctx = hctx->u.cctx;
+
+    /* Note: each chunkqueue chunk is sent in its own frame, which may be
+     * suboptimal.  lighttpd might read FILE_CHUNK into a reused buffer, so
+     * we can not meet ZSTD_compressContinue() requirement that prior input
+     * is still accessible and unmodified (up to maximum distance size).
+     * Also, chunkqueue_mark_written() on MEM_CHUNK might result in something
+     * else reusing those chunk buffers
+     *
+     * future: migrate to use Zstd streaming API */
+
+    /* future: consider allowing tunables by encoder algorithm,
+     * (i.e. not generic "compression_level" across all compression algorithms)
+     * (ZSTD_CCtx_setParameter()) */
+    /*(note: we ignore any errors while tuning parameters here)*/
+    const plugin_data * const p = hctx->plugin_data;
+    int level = ZSTD_CLEVEL_DEFAULT;
+    if (p->conf.compression_level >= 0) /* -1 is lighttpd default for "unset" */
+        level = p->conf.compression_level;
+    ZSTD_compressBegin(cctx, level);
+
+    char * const out = hctx->output->ptr;
+    const size_t outsz = hctx->output->size;
+    hctx->bytes_in += st_size;
+    for (off_t off = 0; st_size; ) {
+        /* XXX: size check must match mod_deflate_init ZSTD_COMPRESSBOUND arg */
+        size_t len = (size_t)st_size;
+        const size_t rv = (len > ZSTD_BLOCKSIZE_MAX)
+          ? ZSTD_compressContinue(cctx, out, outsz, start+off,
+                                  (len = ZSTD_BLOCKSIZE_MAX))
+          : ZSTD_compressEnd(cctx, out, outsz, start+off, len);
+        off += (off_t)len;
+        st_size -= (off_t)len;
+        if (ZSTD_isError(rv)) return -1;
+        hctx->bytes_out += (off_t)rv;
+        if (0 != stream_http_chunk_append_mem(hctx, out, rv))
+            return -1;
+    }
+    return 0;
+}
+
+static int stream_zstd_flush(handler_ctx * const hctx, int end) {
+    UNUSED(hctx);
+    UNUSED(end);
+    return 0;
+}
+
+static int stream_zstd_end(handler_ctx *hctx) {
+    ZSTD_CCtx * const cctx = hctx->u.cctx;
+    ZSTD_freeCCtx(cctx);
+    return 0;
+}
+
+#endif
+
+
 static int mod_deflate_stream_init(handler_ctx *hctx) {
 	switch(hctx->compression_type) {
 #ifdef USE_ZLIB
@@ -947,6 +1038,10 @@ static int mod_deflate_stream_init(handler_ctx *hctx) {
 #ifdef USE_BROTLI
 	case HTTP_ACCEPT_ENCODING_BR:
 		return stream_br_init(hctx);
+#endif
+#ifdef USE_ZSTD
+	case HTTP_ACCEPT_ENCODING_ZSTD:
+		return stream_zstd_init(hctx);
 #endif
 	default:
 		return -1;
@@ -968,6 +1063,10 @@ static int mod_deflate_compress(handler_ctx * const hctx, unsigned char * const
 #ifdef USE_BROTLI
 	case HTTP_ACCEPT_ENCODING_BR:
 		return stream_br_compress(hctx, start, st_size);
+#endif
+#ifdef USE_ZSTD
+	case HTTP_ACCEPT_ENCODING_ZSTD:
+		return stream_zstd_compress(hctx, start, st_size);
 #endif
 	default:
 		UNUSED(start);
@@ -990,6 +1089,10 @@ static int mod_deflate_stream_flush(handler_ctx * const hctx, int end) {
 #ifdef USE_BROTLI
 	case HTTP_ACCEPT_ENCODING_BR:
 		return stream_br_flush(hctx, end);
+#endif
+#ifdef USE_ZSTD
+	case HTTP_ACCEPT_ENCODING_ZSTD:
+		return stream_zstd_flush(hctx, end);
 #endif
 	default:
 		UNUSED(end);
@@ -1023,6 +1126,10 @@ static int mod_deflate_stream_end(handler_ctx *hctx) {
 #ifdef USE_BROTLI
 	case HTTP_ACCEPT_ENCODING_BR:
 		return stream_br_end(hctx);
+#endif
+#ifdef USE_ZSTD
+	case HTTP_ACCEPT_ENCODING_ZSTD:
+		return stream_zstd_end(hctx);
 #endif
 	default:
 		return -1;
@@ -1263,7 +1370,8 @@ static handler_t deflate_compress_response(request_st * const r, handler_ctx * c
 static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const char **label) {
 	/* get client side support encodings */
 	int accept_encoding = 0;
-      #if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI)
+      #if !defined(USE_ZLIB) && !defined(USE_BZ2LIB) && !defined(USE_BROTLI) \
+       && !defined(USE_ZSTD)
 	UNUSED(value);
 	UNUSED(label);
       #else
@@ -1284,6 +1392,13 @@ static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const
                #ifdef USE_ZLIB
                 if (0 == memcmp(v, "gzip", 4))
                     accept_encoding |= HTTP_ACCEPT_ENCODING_GZIP;
+               #endif
+               #ifdef USE_ZSTD
+                #ifdef USE_ZLIB
+                else
+                #endif
+                if (0 == memcmp(v, "zstd", 4))
+                    accept_encoding |= HTTP_ACCEPT_ENCODING_ZSTD;
                #endif
                 break;
               case 5:
@@ -1330,6 +1445,12 @@ static int mod_deflate_choose_encoding (const char *value, plugin_data *p, const
 	accept_encoding &= p->conf.allowed_encodings;
 
 	/* select best matching encoding */
+#ifdef USE_ZSTD
+	if (accept_encoding & HTTP_ACCEPT_ENCODING_ZSTD) {
+		*label = "zstd";
+		return HTTP_ACCEPT_ENCODING_ZSTD;
+	} else
+#endif
 #ifdef USE_BROTLI
 	if (accept_encoding & HTTP_ACCEPT_ENCODING_BR) {
 		*label = "br";

src/server.c

@@ -555,6 +555,11 @@ static void show_features (void) {
 #else
       "\t- zlib support\n"
 #endif
+#if defined HAVE_ZSTD_H && defined HAVE_ZSTD
+      "\t+ zstd support\n"
+#else
+      "\t- zstd support\n"
+#endif
 #if defined HAVE_BZLIB_H && defined HAVE_LIBBZ2
       "\t+ bzip2 support\n"
 #else