lighttpd
/
lighttpd1.4
2
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

[mod_deflate] deflate.cache-dir compressed cache 

(achieves feature parity with mod_compress)
master
Glenn Strauss 2 years ago
parent
5b15c1f449
commit
be6da785c5
3 changed files with 188 additions and 27 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      src/fdevent.c
  2. 1
      src/fdevent.h
  3. 206
      src/mod_deflate.c

8
src/fdevent.c
Unescape View File

@ -724,9 +724,15 @@ int fdevent_set_stdin_stdout_stderr(int fdin, int fdout, int fderr) {
}
#include <stdio.h> /* perror() */
#include <stdio.h> /* perror() rename() */
#include <signal.h> /* signal() */
int fdevent_rename(const char *oldpath, const char *newpath) {
return rename(oldpath, newpath);
}
pid_t fdevent_fork_execve(const char *name, char *argv[], char *envp[], int fdin, int fdout, int fderr, int dfd) {
#ifdef HAVE_FORK

1
src/fdevent.h
Unescape View File

@ -82,6 +82,7 @@ int fdevent_socket_cloexec(int domain, int type, int protocol);
int fdevent_socket_nb_cloexec(int domain, int type, int protocol);
int fdevent_open_cloexec(const char *pathname, int symlinks, int flags, mode_t mode);
int fdevent_mkstemp_append(char *path);
int fdevent_rename(const char *oldpath, const char *newpath);
struct sockaddr;
int fdevent_accept_listenfd(int listenfd, struct sockaddr *addr, size_t *addrlen);

206
src/mod_deflate.c
Unescape View File

@ -86,7 +86,6 @@
* to avoid compressing, even if a broader deflate.mimetypes matched,
* e.g. to compress all "text/" except "text/special".
* - mod_compress and mod_deflate might merge overlapping feature sets
* (mod_compress.cache-dir does not yet have an equivalent in mod_deflate)
*
* Implementation notes:
* - http_chunk_append_mem() used instead of http_chunk_append_buffer()
@ -109,7 +108,7 @@
#include <string.h>
#include <errno.h>
#include <time.h>
#include <unistd.h> /* read() */
#include <unistd.h> /* getpid() read() unlink() write() */
#include "base.h"
#include "fdevent.h"
@ -119,6 +118,7 @@
#include "http_chunk.h"
#include "http_header.h"
#include "response.h"
#include "stat_cache.h"
#include "plugin.h"
@ -178,6 +178,7 @@ static void sigbus_handler(int sig) {
typedef struct {
const array *mimetypes;
const buffer *cache_dir;
unsigned int max_compress_size;
unsigned short min_compress_size;
unsigned short output_buffer_size;
@ -216,6 +217,8 @@ typedef struct {
plugin_data *plugin_data;
request_st *r;
int compression_type;
int cache_fd;
char *cache_fn;
} handler_ctx;
static handler_ctx *handler_ctx_init() {
@ -223,11 +226,18 @@ static handler_ctx *handler_ctx_init() {
hctx = calloc(1, sizeof(*hctx));
hctx->in_queue = chunkqueue_init();
hctx->cache_fd = -1;
return hctx;
}
static void handler_ctx_free(handler_ctx *hctx) {
if (hctx->cache_fn) {
unlink(hctx->cache_fn);
free(hctx->cache_fn);
}
if (-1 != hctx->cache_fd)
close(hctx->cache_fd);
#if 0
if (hctx->output != &p->tmp_buf) {
buffer_free(hctx->output);
@ -248,6 +258,70 @@ FREE_FUNC(mod_deflate_free) {
free(p->tmp_buf.ptr);
}
#if defined(_WIN32) && !defined(__CYGWIN__)
#define mkdir(x,y) mkdir(x)
#endif
static int mkdir_for_file (char *fn) {
for (char *p = fn; (p = strchr(p + 1, '/')) != NULL; ) {
if (p[1] == '\0') return 0; /* ignore trailing slash */
*p = '\0';
int rc = mkdir(fn, 0700);
*p = '/';
if (0 != rc && errno != EEXIST) return -1;
}
return 0;
}
static int mkdir_recursive (char *dir) {
return 0 == mkdir_for_file(dir) && (0 == mkdir(dir,0700) || errno == EEXIST)
? 0
: -1;
}
static buffer * mod_deflate_cache_file_name(request_st * const r, const buffer *cache_dir, const buffer * const etag) {
/* XXX: future: for shorter paths into the cache, we could checksum path,
* and then shard it to avoid a huge single directory.
* Alternatively, could use &r->uri.path, minus any
* (matching) &r->pathinfo suffix, with result url-encoded */
buffer * const tb = r->tmp_buf;
buffer_copy_buffer(tb, cache_dir);
buffer_append_string_len(tb, CONST_BUF_LEN(&r->physical.path));
buffer_append_string_len(tb, CONST_STR_LEN("-"));
buffer_append_string_len(tb, etag->ptr+1, buffer_string_length(etag)-2);
return tb;
}
static void mod_deflate_cache_file_open (handler_ctx * const hctx, const buffer * const fn) {
/* race exists whereby up to # workers might attempt to compress same
* file at same time if requested at same time, but this is unlikely
* and resolves itself by atomic rename into place when done */
const uint32_t fnlen = buffer_string_length(fn);
hctx->cache_fn = malloc(fnlen+1+LI_ITOSTRING_LENGTH+1);
force_assert(hctx->cache_fn);
memcpy(hctx->cache_fn, fn->ptr, fnlen);
hctx->cache_fn[fnlen] = '.';
const size_t ilen =
li_itostrn(hctx->cache_fn+fnlen+1, LI_ITOSTRING_LENGTH, getpid());
hctx->cache_fn[fnlen+1+ilen] = '\0';
hctx->cache_fd = fdevent_open_cloexec(hctx->cache_fn, 1, O_RDWR|O_CREAT, 0600);
if (-1 == hctx->cache_fd) {
free(hctx->cache_fn);
hctx->cache_fn = NULL;
}
}
static int mod_deflate_cache_file_finish (request_st * const r, handler_ctx * const hctx, const buffer * const fn) {
if (0 != fdevent_rename(hctx->cache_fn, fn->ptr))
return -1;
free(hctx->cache_fn);
hctx->cache_fn = NULL;
chunkqueue_reset(r->write_queue);
int rc = http_chunk_append_file_fd(r, fn, hctx->cache_fd, hctx->bytes_out);
hctx->cache_fd = -1;
return rc;
}
static void mod_deflate_merge_config_cpv(plugin_config * const pconf, const config_plugin_value_t * const cpv) {
switch (cpv->k_id) { /* index into static config_plugin_keys_t cpk[] */
case 0: /* deflate.mimetypes */
@ -274,6 +348,9 @@ static void mod_deflate_merge_config_cpv(plugin_config * const pconf, const conf
case 7: /* deflate.max-loadavg */
pconf->max_loadavg = cpv->v.d;
break;
case 8: /* deflate.cache-dir */
pconf->cache_dir = cpv->v.b;
break;
default:/* should not happen */
return;
}
@ -370,6 +447,9 @@ SETDEFAULTS_FUNC(mod_deflate_set_defaults) {
,{ CONST_STR_LEN("deflate.max-loadavg"),
T_CONFIG_STRING,
T_CONFIG_SCOPE_CONNECTION }
,{ CONST_STR_LEN("deflate.cache-dir"),
T_CONFIG_STRING,
T_CONFIG_SCOPE_CONNECTION }
,{ NULL, 0,
T_CONFIG_UNSET,
T_CONFIG_SCOPE_UNSET }
@ -422,6 +502,21 @@ SETDEFAULTS_FUNC(mod_deflate_set_defaults) {
? strtod(cpv->v.b->ptr, NULL)
: 0.0;
break;
case 8: /* deflate.cache-dir */
if (!buffer_string_is_empty(cpv->v.b)) {
buffer *b;
*(const buffer **)&b = cpv->v.b;
const uint32_t len = buffer_string_length(b);
if (len > 0 && '/' == b->ptr[len-1])
buffer_string_set_length(b, len-1); /*remove end slash*/
struct stat st;
if (0 != stat(b->ptr,&st) && 0 != mkdir_recursive(b->ptr)) {
log_perror(srv->errh, __FILE__, __LINE__,
"can't stat %s %s", cpk[cpv->k_id].k, b->ptr);
return HANDLER_ERROR;
}
}
break;
default:/* should not happen */
break;
}
@ -449,9 +544,19 @@ SETDEFAULTS_FUNC(mod_deflate_set_defaults) {
#if defined(USE_ZLIB) || defined(USE_BZ2LIB) || defined(USE_BROTLI)
static int mod_deflate_cache_file_append (handler_ctx * const hctx, const char *out, size_t len) {
ssize_t wr;
do {
wr = write(hctx->cache_fd, out, len);
} while (wr > 0 ? ((out += wr), (len -= (size_t)wr)) : errno == EINTR);
return (0 == len) ? 0 : -1;
}
static int stream_http_chunk_append_mem(handler_ctx * const hctx, const char * const out, size_t len) {
/* future: might also write stream to hctx temporary file in compressed file cache */
return http_chunk_append_mem(hctx->r, out, len);
if (0 == len) return 0;
return (-1 == hctx->cache_fd)
? http_chunk_append_mem(hctx->r, out, len)
: mod_deflate_cache_file_append(hctx, out, len);
}
#endif
@ -823,15 +928,15 @@ static int mod_deflate_stream_flush(handler_ctx * const hctx, int end) {
}
}
static void mod_deflate_note_ratio(request_st * const r, handler_ctx * const hctx) {
static void mod_deflate_note_ratio(request_st * const r, const off_t bytes_out, const off_t bytes_in) {
/* store compression ratio in environment
* for possible logging by mod_accesslog
* (late in response handling, so not seen by most other modules) */
/*(should be called only at end of successful response compression)*/
char ratio[LI_ITOSTRING_LENGTH];
if (0 == hctx->bytes_in) return;
if (0 == bytes_in) return;
size_t len =
li_itostrn(ratio, sizeof(ratio), hctx->bytes_out * 100 / hctx->bytes_in);
li_itostrn(ratio, sizeof(ratio), bytes_out * 100 / bytes_in);
http_header_env_set(r, CONST_STR_LEN("ratio"), ratio, len);
}
@ -856,19 +961,13 @@ static int mod_deflate_stream_end(handler_ctx *hctx) {
}
static int deflate_compress_cleanup(request_st * const r, handler_ctx * const hctx) {
const plugin_data *p = hctx->plugin_data;
r->plugin_ctx[p->id] = NULL;
int rc = mod_deflate_stream_end(hctx);
if (0 != rc)
log_error(r->conf.errh, __FILE__, __LINE__, "error closing stream");
#if 1 /* unnecessary if deflate.min-compress-size is set to a reasonable value */
if (hctx->bytes_in < hctx->bytes_out) {
if (0 == rc && hctx->bytes_in < hctx->bytes_out)
log_error(r->conf.errh, __FILE__, __LINE__,
"uri %s in=%lld smaller than out=%lld", r->target.ptr,
(long long)hctx->bytes_in, (long long)hctx->bytes_out);
}
#endif
handler_ctx_free(hctx);
@ -1203,9 +1302,10 @@ REQUEST_FUNC(mod_deflate_handle_response_start) {
handler_ctx *hctx;
const char *label;
off_t len;
size_t etaglen = 0;
uint32_t etaglen;
int compression_type;
handler_t rc;
int had_vary = 0;
/*(current implementation requires response be complete)*/
if (!r->resp_body_finished) return HANDLER_GO_ON;
@ -1262,6 +1362,7 @@ REQUEST_FUNC(mod_deflate_handle_response_start) {
/* Vary: Accept-Encoding (response might change according to request Accept-Encoding) */
if (NULL != (vb = http_header_response_get(r, HTTP_HEADER_VARY, CONST_STR_LEN("Vary")))) {
had_vary = 1;
if (NULL == strstr(vb->ptr, "Accept-Encoding")) {
buffer_append_string_len(vb, CONST_STR_LEN(",Accept-Encoding"));
}
@ -1274,9 +1375,9 @@ REQUEST_FUNC(mod_deflate_handle_response_start) {
/* check ETag as is done in http_response_handle_cachable()
* (slightly imperfect (close enough?) match of ETag "000000" to "000000-gzip") */
vb = http_header_response_get(r, HTTP_HEADER_ETAG, CONST_STR_LEN("ETag"));
etaglen = (NULL != vb) ? buffer_string_length(vb) : 0;
if (NULL != vb && (r->rqst_htags & HTTP_HEADER_IF_NONE_MATCH)) {
const buffer *if_none_match = http_header_response_get(r, HTTP_HEADER_IF_NONE_MATCH, CONST_STR_LEN("If-None-Match"));
etaglen = buffer_string_length(vb);
if (etaglen
&& r->http_status < 300 /*(want 2xx only)*/
&& NULL != if_none_match
@ -1334,8 +1435,52 @@ REQUEST_FUNC(mod_deflate_handle_response_start) {
return HANDLER_GO_ON;
}
/* future: might use ETag to check if compressed content is in compressed file cache */
/*if (etaglen) { ... } *//* return if in file cache after updating r->write_queue */
/* restrict items eligible for cache of compressed responses
* (This module does not aim to be a full caching proxy)
* response must be complete (not streaming response)
* must not have prior Vary response header (before Accept-Encoding added)
* must not have Range response header
* must have ETag
* must be file
* must be single FILE_CHUNK in chunkqueue
* must not be chunkqueue temporary file
* must be whole file, not partial content
* Note: small files (< 32k (see http_chunk.c)) will have been read into
* memory and will end up getting stream-compressed rather than
* cached on disk as compressed file
*/
buffer *tb = NULL;
if (!buffer_is_empty(p->conf.cache_dir)
&& !had_vary
&& etaglen > 2
&& r->resp_body_finished
&& r->write_queue->first == r->write_queue->last
&& r->write_queue->first->type == FILE_CHUNK
&& r->write_queue->first->file.start == 0
&& !r->write_queue->first->file.is_temp
&& !http_header_response_get(r, HTTP_HEADER_RANGE,
CONST_STR_LEN("Range"))) {
tb = mod_deflate_cache_file_name(r, p->conf.cache_dir, vb);
/*(checked earlier and skipped if Transfer-Encoding had been set)*/
stat_cache_entry *sce = stat_cache_get_entry(tb);
if (NULL != sce) {
chunkqueue_reset(r->write_queue);
if (0 != http_chunk_append_file(r, tb))
return HANDLER_ERROR;
if (r->resp_htags & HTTP_HEADER_CONTENT_LENGTH)
http_header_response_unset(r, HTTP_HEADER_CONTENT_LENGTH,
CONST_STR_LEN("Content-Length"));
mod_deflate_note_ratio(r, sce->st.st_size, len);
return HANDLER_GO_ON;
}
/* sanity check that response was whole file;
* (racy since using stat_cache, but cache file only if match) */
sce = stat_cache_get_entry(r->write_queue->first->mem);
if (NULL == sce || sce->st.st_size != len)
tb = NULL;
if (0 != mkdir_for_file(tb->ptr))
tb = NULL;
}
/* enable compression */
p->conf.sync_flush =
@ -1347,6 +1492,8 @@ REQUEST_FUNC(mod_deflate_handle_response_start) {
/* setup output buffer */
buffer_clear(&p->tmp_buf);
hctx->output = &p->tmp_buf;
/* open cache file if caching compressed file */
if (tb) mod_deflate_cache_file_open(hctx, tb);
if (0 != mod_deflate_stream_init(hctx)) {
/*(should not happen unless ENOMEM)*/
handler_ctx_free(hctx);
@ -1374,22 +1521,29 @@ REQUEST_FUNC(mod_deflate_handle_response_start) {
r->plugin_ctx[p->id] = hctx;
rc = deflate_compress_response(r, hctx);
if (HANDLER_GO_ON != rc) {
if (HANDLER_FINISHED == rc) {
mod_deflate_note_ratio(r, hctx);
if (HANDLER_GO_ON == rc) return HANDLER_GO_ON;
if (HANDLER_FINISHED == rc) {
if (-1 == hctx->cache_fd
|| 0 == mod_deflate_cache_file_finish(r, hctx, tb)) {
mod_deflate_note_ratio(r, hctx->bytes_out, hctx->bytes_in);
rc = HANDLER_GO_ON;
}
if (deflate_compress_cleanup(r, hctx) < 0) return HANDLER_ERROR;
if (HANDLER_ERROR == rc) return HANDLER_ERROR;
else
rc = HANDLER_ERROR;
}
return HANDLER_GO_ON;
r->plugin_ctx[p->id] = NULL;
if (deflate_compress_cleanup(r, hctx) < 0) return HANDLER_ERROR;
return rc;
}
static handler_t mod_deflate_cleanup(request_st * const r, void *p_d) {
plugin_data *p = p_d;
handler_ctx *hctx = r->plugin_ctx[p->id];
if (NULL != hctx) deflate_compress_cleanup(r, hctx);
if (NULL != hctx) {
r->plugin_ctx[p->id] = NULL;
deflate_compress_cleanup(r, hctx);
}
return HANDLER_GO_ON;
}

Write Preview
Loading…
Cancel
Save