From 4a51d2ab8ccfa8f8da476da100963c1b26b5b67f Mon Sep 17 00:00:00 2001 From: Glenn Strauss Date: Thu, 21 Apr 2016 17:33:16 -0400 Subject: [PATCH 1/4] [core] http_response_send_file() shared code (#2017) move code from mod_staticfile.c to http-header-glue.c to allow reuse (includes ETag, Last-Modified headers, Range requests for static files) operate on path arg instead of con->physical.path skip Range requests if con->http_status already set >= 300 remove redundant calls to stat_cache_get_entry() handling Range requests x-ref: "X-Sendfile handoff to mod-static-file in 1.4.x" https://redmine.lighttpd.net/issues/2017 --- src/http-header-glue.c | 342 ++++++++++++++++++++++++++++++++++++++++ src/mod_staticfile.c | 346 +---------------------------------------- src/response.h | 1 + 3 files changed, 345 insertions(+), 344 deletions(-) diff --git a/src/http-header-glue.c b/src/http-header-glue.c index 5cbb8097..15ea8291 100644 --- a/src/http-header-glue.c +++ b/src/http-header-glue.c @@ -5,7 +5,9 @@ #include "buffer.h" #include "log.h" #include "etag.h" +#include "http_chunk.h" #include "response.h" +#include "stat_cache.h" #include #include @@ -323,3 +325,343 @@ int http_response_handle_cachable(server *srv, connection *con, buffer *mtime) { return HANDLER_GO_ON; } + + +static int http_response_parse_range(server *srv, connection *con, buffer *path, stat_cache_entry *sce) { + int multipart = 0; + int error; + off_t start, end; + const char *s, *minus; + char *boundary = "fkj49sn38dcn3"; + data_string *ds; + buffer *content_type = NULL; + + start = 0; + end = sce->st.st_size - 1; + + con->response.content_length = 0; + + if (NULL != (ds = (data_string *)array_get_element(con->response.headers, "Content-Type"))) { + content_type = ds->value; + } + + for (s = con->request.http_range, error = 0; + !error && *s && NULL != (minus = strchr(s, '-')); ) { + char *err; + off_t la, le; + + if (s == minus) { + /* - */ + + le = strtoll(s, &err, 10); + + if (le == 0) { + /* RFC 2616 - 14.35.1 */ + + con->http_status = 416; + error = 1; + } else if (*err == '\0') { + /* end */ + s = err; + + end = sce->st.st_size - 1; + start = sce->st.st_size + le; + } else if (*err == ',') { + multipart = 1; + s = err + 1; + + end = sce->st.st_size - 1; + start = sce->st.st_size + le; + } else { + error = 1; + } + + } else if (*(minus+1) == '\0' || *(minus+1) == ',') { + /* - */ + + la = strtoll(s, &err, 10); + + if (err == minus) { + /* ok */ + + if (*(err + 1) == '\0') { + s = err + 1; + + end = sce->st.st_size - 1; + start = la; + + } else if (*(err + 1) == ',') { + multipart = 1; + s = err + 2; + + end = sce->st.st_size - 1; + start = la; + } else { + error = 1; + } + } else { + /* error */ + error = 1; + } + } else { + /* - */ + + la = strtoll(s, &err, 10); + + if (err == minus) { + le = strtoll(minus+1, &err, 10); + + /* RFC 2616 - 14.35.1 */ + if (la > le) { + error = 1; + } + + if (*err == '\0') { + /* ok, end*/ + s = err; + + end = le; + start = la; + } else if (*err == ',') { + multipart = 1; + s = err + 1; + + end = le; + start = la; + } else { + /* error */ + + error = 1; + } + } else { + /* error */ + + error = 1; + } + } + + if (!error) { + if (start < 0) start = 0; + + /* RFC 2616 - 14.35.1 */ + if (end > sce->st.st_size - 1) end = sce->st.st_size - 1; + + if (start > sce->st.st_size - 1) { + error = 1; + + con->http_status = 416; + } + } + + if (!error) { + if (multipart) { + /* write boundary-header */ + buffer *b = buffer_init(); + + buffer_copy_string_len(b, CONST_STR_LEN("\r\n--")); + buffer_append_string(b, boundary); + + /* write Content-Range */ + buffer_append_string_len(b, CONST_STR_LEN("\r\nContent-Range: bytes ")); + buffer_append_int(b, start); + buffer_append_string_len(b, CONST_STR_LEN("-")); + buffer_append_int(b, end); + buffer_append_string_len(b, CONST_STR_LEN("/")); + buffer_append_int(b, sce->st.st_size); + + buffer_append_string_len(b, CONST_STR_LEN("\r\nContent-Type: ")); + buffer_append_string_buffer(b, content_type); + + /* write END-OF-HEADER */ + buffer_append_string_len(b, CONST_STR_LEN("\r\n\r\n")); + + con->response.content_length += buffer_string_length(b); + chunkqueue_append_buffer(con->write_queue, b); + buffer_free(b); + } + + chunkqueue_append_file(con->write_queue, path, start, end - start + 1); + con->response.content_length += end - start + 1; + } + } + + /* something went wrong */ + if (error) return -1; + + if (multipart) { + /* add boundary end */ + buffer *b = buffer_init(); + + buffer_copy_string_len(b, "\r\n--", 4); + buffer_append_string(b, boundary); + buffer_append_string_len(b, "--\r\n", 4); + + con->response.content_length += buffer_string_length(b); + chunkqueue_append_buffer(con->write_queue, b); + buffer_free(b); + + /* set header-fields */ + + buffer_copy_string_len(srv->tmp_buf, CONST_STR_LEN("multipart/byteranges; boundary=")); + buffer_append_string(srv->tmp_buf, boundary); + + /* overwrite content-type */ + response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(srv->tmp_buf)); + } else { + /* add Content-Range-header */ + + buffer_copy_string_len(srv->tmp_buf, CONST_STR_LEN("bytes ")); + buffer_append_int(srv->tmp_buf, start); + buffer_append_string_len(srv->tmp_buf, CONST_STR_LEN("-")); + buffer_append_int(srv->tmp_buf, end); + buffer_append_string_len(srv->tmp_buf, CONST_STR_LEN("/")); + buffer_append_int(srv->tmp_buf, sce->st.st_size); + + response_header_insert(srv, con, CONST_STR_LEN("Content-Range"), CONST_BUF_LEN(srv->tmp_buf)); + } + + /* ok, the file is set-up */ + return 0; +} + + +void http_response_send_file (server *srv, connection *con, buffer *path) { + stat_cache_entry *sce = NULL; + buffer *mtime = NULL; + data_string *ds; + int allow_caching = 1; + + if (HANDLER_ERROR == stat_cache_get_entry(srv, con, path, &sce)) { + con->http_status = (errno == ENOENT) ? 404 : 403; + + log_error_write(srv, __FILE__, __LINE__, "sbsb", + "not a regular file:", con->uri.path, + "->", path); + + return; + } + + /* we only handline regular files */ +#ifdef HAVE_LSTAT + if ((sce->is_symlink == 1) && !con->conf.follow_symlink) { + con->http_status = 403; + + if (con->conf.log_request_handling) { + log_error_write(srv, __FILE__, __LINE__, "s", "-- access denied due symlink restriction"); + log_error_write(srv, __FILE__, __LINE__, "sb", "Path :", path); + } + + return; + } +#endif + if (!S_ISREG(sce->st.st_mode)) { + con->http_status = 403; + + if (con->conf.log_file_not_found) { + log_error_write(srv, __FILE__, __LINE__, "sbsb", + "not a regular file:", con->uri.path, + "->", sce->name); + } + + return; + } + + /* mod_compress might set several data directly, don't overwrite them */ + + /* set response content-type, if not set already */ + + if (NULL == array_get_element(con->response.headers, "Content-Type")) { + if (buffer_string_is_empty(sce->content_type)) { + /* we are setting application/octet-stream, but also announce that + * this header field might change in the seconds few requests + * + * This should fix the aggressive caching of FF and the script download + * seen by the first installations + */ + response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_STR_LEN("application/octet-stream")); + + allow_caching = 0; + } else { + response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(sce->content_type)); + } + } + + if (con->conf.range_requests) { + response_header_overwrite(srv, con, CONST_STR_LEN("Accept-Ranges"), CONST_STR_LEN("bytes")); + } + + if (allow_caching) { + if (con->etag_flags != 0 && !buffer_string_is_empty(sce->etag)) { + if (NULL == array_get_element(con->response.headers, "ETag")) { + /* generate e-tag */ + etag_mutate(con->physical.etag, sce->etag); + + response_header_overwrite(srv, con, CONST_STR_LEN("ETag"), CONST_BUF_LEN(con->physical.etag)); + } + } + + /* prepare header */ + if (NULL == (ds = (data_string *)array_get_element(con->response.headers, "Last-Modified"))) { + mtime = strftime_cache_get(srv, sce->st.st_mtime); + response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"), CONST_BUF_LEN(mtime)); + } else { + mtime = ds->value; + } + + if (HANDLER_FINISHED == http_response_handle_cachable(srv, con, mtime)) { + return; + } + } + + if (con->request.http_range && con->conf.range_requests && con->http_status < 300) { + int do_range_request = 1; + /* check if we have a conditional GET */ + + if (NULL != (ds = (data_string *)array_get_element(con->request.headers, "If-Range"))) { + /* if the value is the same as our ETag, we do a Range-request, + * otherwise a full 200 */ + + if (ds->value->ptr[0] == '"') { + /** + * client wants a ETag + */ + if (!con->physical.etag) { + do_range_request = 0; + } else if (!buffer_is_equal(ds->value, con->physical.etag)) { + do_range_request = 0; + } + } else if (!mtime) { + /** + * we don't have a Last-Modified and can match the If-Range: + * + * sending all + */ + do_range_request = 0; + } else if (!buffer_is_equal(ds->value, mtime)) { + do_range_request = 0; + } + } + + if (do_range_request) { + /* content prepared, I'm done */ + con->file_finished = 1; + + if (0 == http_response_parse_range(srv, con, path, sce)) { + con->http_status = 206; + } + return; + } + } + + /* if we are still here, prepare body */ + + /* we add it here for all requests + * the HEAD request will drop it afterwards again + */ + if (0 == sce->st.st_size || 0 == http_chunk_append_file(srv, con, path)) { + con->http_status = 200; + con->file_finished = 1; + } else { + con->http_status = 403; + } +} diff --git a/src/mod_staticfile.c b/src/mod_staticfile.c index a0fa2933..66b75c26 100644 --- a/src/mod_staticfile.c +++ b/src/mod_staticfile.c @@ -34,8 +34,6 @@ typedef struct { typedef struct { PLUGIN_DATA; - buffer *range_buf; - plugin_config **config_storage; plugin_config conf; @@ -47,8 +45,6 @@ INIT_FUNC(mod_staticfile_init) { p = calloc(1, sizeof(*p)); - p->range_buf = buffer_init(); - return p; } @@ -73,7 +69,6 @@ FREE_FUNC(mod_staticfile_free) { } free(p->config_storage); } - buffer_free(p->range_buf); free(p); @@ -156,215 +151,10 @@ static int mod_staticfile_patch_connection(server *srv, connection *con, plugin_ } #undef PATCH -static int http_response_parse_range(server *srv, connection *con, plugin_data *p) { - int multipart = 0; - int error; - off_t start, end; - const char *s, *minus; - char *boundary = "fkj49sn38dcn3"; - data_string *ds; - stat_cache_entry *sce = NULL; - buffer *content_type = NULL; - - if (HANDLER_ERROR == stat_cache_get_entry(srv, con, con->physical.path, &sce)) { - SEGFAULT(); - } - - start = 0; - end = sce->st.st_size - 1; - - con->response.content_length = 0; - - if (NULL != (ds = (data_string *)array_get_element(con->response.headers, "Content-Type"))) { - content_type = ds->value; - } - - for (s = con->request.http_range, error = 0; - !error && *s && NULL != (minus = strchr(s, '-')); ) { - char *err; - off_t la, le; - - if (s == minus) { - /* - */ - - le = strtoll(s, &err, 10); - - if (le == 0) { - /* RFC 2616 - 14.35.1 */ - - con->http_status = 416; - error = 1; - } else if (*err == '\0') { - /* end */ - s = err; - - end = sce->st.st_size - 1; - start = sce->st.st_size + le; - } else if (*err == ',') { - multipart = 1; - s = err + 1; - - end = sce->st.st_size - 1; - start = sce->st.st_size + le; - } else { - error = 1; - } - - } else if (*(minus+1) == '\0' || *(minus+1) == ',') { - /* - */ - - la = strtoll(s, &err, 10); - - if (err == minus) { - /* ok */ - - if (*(err + 1) == '\0') { - s = err + 1; - - end = sce->st.st_size - 1; - start = la; - - } else if (*(err + 1) == ',') { - multipart = 1; - s = err + 2; - - end = sce->st.st_size - 1; - start = la; - } else { - error = 1; - } - } else { - /* error */ - error = 1; - } - } else { - /* - */ - - la = strtoll(s, &err, 10); - - if (err == minus) { - le = strtoll(minus+1, &err, 10); - - /* RFC 2616 - 14.35.1 */ - if (la > le) { - error = 1; - } - - if (*err == '\0') { - /* ok, end*/ - s = err; - - end = le; - start = la; - } else if (*err == ',') { - multipart = 1; - s = err + 1; - - end = le; - start = la; - } else { - /* error */ - - error = 1; - } - } else { - /* error */ - - error = 1; - } - } - - if (!error) { - if (start < 0) start = 0; - - /* RFC 2616 - 14.35.1 */ - if (end > sce->st.st_size - 1) end = sce->st.st_size - 1; - - if (start > sce->st.st_size - 1) { - error = 1; - - con->http_status = 416; - } - } - - if (!error) { - if (multipart) { - /* write boundary-header */ - buffer *b = buffer_init(); - - buffer_copy_string_len(b, CONST_STR_LEN("\r\n--")); - buffer_append_string(b, boundary); - - /* write Content-Range */ - buffer_append_string_len(b, CONST_STR_LEN("\r\nContent-Range: bytes ")); - buffer_append_int(b, start); - buffer_append_string_len(b, CONST_STR_LEN("-")); - buffer_append_int(b, end); - buffer_append_string_len(b, CONST_STR_LEN("/")); - buffer_append_int(b, sce->st.st_size); - - buffer_append_string_len(b, CONST_STR_LEN("\r\nContent-Type: ")); - buffer_append_string_buffer(b, content_type); - - /* write END-OF-HEADER */ - buffer_append_string_len(b, CONST_STR_LEN("\r\n\r\n")); - - con->response.content_length += buffer_string_length(b); - chunkqueue_append_buffer(con->write_queue, b); - buffer_free(b); - } - - chunkqueue_append_file(con->write_queue, con->physical.path, start, end - start + 1); - con->response.content_length += end - start + 1; - } - } - - /* something went wrong */ - if (error) return -1; - - if (multipart) { - /* add boundary end */ - buffer *b = buffer_init(); - - buffer_copy_string_len(b, "\r\n--", 4); - buffer_append_string(b, boundary); - buffer_append_string_len(b, "--\r\n", 4); - - con->response.content_length += buffer_string_length(b); - chunkqueue_append_buffer(con->write_queue, b); - buffer_free(b); - - /* set header-fields */ - - buffer_copy_string_len(p->range_buf, CONST_STR_LEN("multipart/byteranges; boundary=")); - buffer_append_string(p->range_buf, boundary); - - /* overwrite content-type */ - response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(p->range_buf)); - } else { - /* add Content-Range-header */ - - buffer_copy_string_len(p->range_buf, CONST_STR_LEN("bytes ")); - buffer_append_int(p->range_buf, start); - buffer_append_string_len(p->range_buf, CONST_STR_LEN("-")); - buffer_append_int(p->range_buf, end); - buffer_append_string_len(p->range_buf, CONST_STR_LEN("/")); - buffer_append_int(p->range_buf, sce->st.st_size); - - response_header_insert(srv, con, CONST_STR_LEN("Content-Range"), CONST_BUF_LEN(p->range_buf)); - } - - /* ok, the file is set-up */ - return 0; -} - URIHANDLER_FUNC(mod_staticfile_subrequest) { plugin_data *p = p_d; size_t k; - stat_cache_entry *sce = NULL; - buffer *mtime = NULL; data_string *ds; - int allow_caching = 1; /* someone else has done a decision for us */ if (con->http_status != 0) return HANDLER_GO_ON; @@ -412,140 +202,8 @@ URIHANDLER_FUNC(mod_staticfile_subrequest) { log_error_write(srv, __FILE__, __LINE__, "s", "-- handling file as static file"); } - if (HANDLER_ERROR == stat_cache_get_entry(srv, con, con->physical.path, &sce)) { - con->http_status = 403; - - log_error_write(srv, __FILE__, __LINE__, "sbsb", - "not a regular file:", con->uri.path, - "->", con->physical.path); - - return HANDLER_FINISHED; - } - - /* we only handline regular files */ -#ifdef HAVE_LSTAT - if ((sce->is_symlink == 1) && !con->conf.follow_symlink) { - con->http_status = 403; - - if (con->conf.log_request_handling) { - log_error_write(srv, __FILE__, __LINE__, "s", "-- access denied due symlink restriction"); - log_error_write(srv, __FILE__, __LINE__, "sb", "Path :", con->physical.path); - } - - buffer_reset(con->physical.path); - return HANDLER_FINISHED; - } -#endif - if (!S_ISREG(sce->st.st_mode)) { - con->http_status = 404; - - if (con->conf.log_file_not_found) { - log_error_write(srv, __FILE__, __LINE__, "sbsb", - "not a regular file:", con->uri.path, - "->", sce->name); - } - - return HANDLER_FINISHED; - } - - /* mod_compress might set several data directly, don't overwrite them */ - - /* set response content-type, if not set already */ - - if (NULL == array_get_element(con->response.headers, "Content-Type")) { - if (buffer_string_is_empty(sce->content_type)) { - /* we are setting application/octet-stream, but also announce that - * this header field might change in the seconds few requests - * - * This should fix the aggressive caching of FF and the script download - * seen by the first installations - */ - response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_STR_LEN("application/octet-stream")); - - allow_caching = 0; - } else { - response_header_overwrite(srv, con, CONST_STR_LEN("Content-Type"), CONST_BUF_LEN(sce->content_type)); - } - } - - if (con->conf.range_requests) { - response_header_overwrite(srv, con, CONST_STR_LEN("Accept-Ranges"), CONST_STR_LEN("bytes")); - } - - if (allow_caching) { - if (p->conf.etags_used && con->etag_flags != 0 && !buffer_string_is_empty(sce->etag)) { - if (NULL == array_get_element(con->response.headers, "ETag")) { - /* generate e-tag */ - etag_mutate(con->physical.etag, sce->etag); - - response_header_overwrite(srv, con, CONST_STR_LEN("ETag"), CONST_BUF_LEN(con->physical.etag)); - } - } - - /* prepare header */ - if (NULL == (ds = (data_string *)array_get_element(con->response.headers, "Last-Modified"))) { - mtime = strftime_cache_get(srv, sce->st.st_mtime); - response_header_overwrite(srv, con, CONST_STR_LEN("Last-Modified"), CONST_BUF_LEN(mtime)); - } else { - mtime = ds->value; - } - - if (HANDLER_FINISHED == http_response_handle_cachable(srv, con, mtime)) { - return HANDLER_FINISHED; - } - } - - if (con->request.http_range && con->conf.range_requests) { - int do_range_request = 1; - /* check if we have a conditional GET */ - - if (NULL != (ds = (data_string *)array_get_element(con->request.headers, "If-Range"))) { - /* if the value is the same as our ETag, we do a Range-request, - * otherwise a full 200 */ - - if (ds->value->ptr[0] == '"') { - /** - * client wants a ETag - */ - if (!con->physical.etag) { - do_range_request = 0; - } else if (!buffer_is_equal(ds->value, con->physical.etag)) { - do_range_request = 0; - } - } else if (!mtime) { - /** - * we don't have a Last-Modified and can match the If-Range: - * - * sending all - */ - do_range_request = 0; - } else if (!buffer_is_equal(ds->value, mtime)) { - do_range_request = 0; - } - } - - if (do_range_request) { - /* content prepared, I'm done */ - con->file_finished = 1; - - if (0 == http_response_parse_range(srv, con, p)) { - con->http_status = 206; - } - return HANDLER_FINISHED; - } - } - - /* if we are still here, prepare body */ - - /* we add it here for all requests - * the HEAD request will drop it afterwards again - */ - if (0 == sce->st.st_size || 0 == http_chunk_append_file(srv, con, con->physical.path)) { - con->http_status = 200; - con->file_finished = 1; - } else { - con->http_status = 403; - } + if (!p->conf.etags_used) con->etag_flags = 0; + http_response_send_file(srv, con, con->physical.path); return HANDLER_FINISHED; } diff --git a/src/response.h b/src/response.h index e493a1c9..f38b8413 100644 --- a/src/response.h +++ b/src/response.h @@ -16,6 +16,7 @@ int response_header_append(server *srv, connection *con, const char *key, size_t handler_t http_response_prepare(server *srv, connection *con); int http_response_redirect_to_directory(server *srv, connection *con); int http_response_handle_cachable(server *srv, connection *con, buffer * mtime); +void http_response_send_file (server *srv, connection *con, buffer *path); buffer * strftime_cache_get(server *srv, time_t last_mod); #endif From b9940f9856c166dc7368207d1869cb203774db87 Mon Sep 17 00:00:00 2001 From: Glenn Strauss Date: Thu, 21 Apr 2016 21:01:30 -0400 Subject: [PATCH 2/4] [mod_fastcgi] use http_response_xsendfile() (fixes #799, fixes #851, fixes #2017, fixes #2076) handle X-Sendfile and X-LIGHTTPD-send-file w/ http_response_xsendfile() if host is configured ( "x-sendfile" = "enable" ) Note: X-Sendfile path is url-decoded for consistency, like X-Sendfile2 (response headers should be url-encoded to avoid tripping over chars allowed in filesystem but which might change response header parsing semantics) Note: deprecated: "allow-x-send-file"; use "x-sendfile" Note: deprecated: X-LIGHTTPD-send-file header; use X-Sendfile header Note: deprecated: X-Sendfile2 header; use X-Sendfile header For now, X-Sendfile2 is still handled internally by mod_fastcgi. Since http_response_send_file() supports HTTP Range requests, X-Sendfile2 is effectively obsolete. However, any code, e.g. PHP, currently using X-Sendfile2 is probably manually generating 206 Partial Content status and Range response headers. A future version of lighttpd might *remove* X-Sendfile2. Existing code should be converted to use X-Sendfile, which is easily done by removing all the special logic around using X-Sendfile2, since the 206 Partial Content status and Range response headers are handled in http_response_send_file(). x-ref: "mod_fastcgi + X-Sendfile -> mod_staticfile" https://redmine.lighttpd.net/issues/799 "Feature Request: New option "x-send-file-docroot"" https://redmine.lighttpd.net/issues/851 "X-Sendfile handoff to mod-static-file in 1.4.x" https://redmine.lighttpd.net/issues/2017 "X-sendfile should be able to set content-type" https://redmine.lighttpd.net/issues/2076 --- doc/outdated/fastcgi.txt | 9 ++-- src/http-header-glue.c | 61 ++++++++++++++++++++++++- src/mod_fastcgi.c | 98 ++++++++++++++++++++++++++-------------- src/response.h | 1 + 4 files changed, 131 insertions(+), 38 deletions(-) diff --git a/doc/outdated/fastcgi.txt b/doc/outdated/fastcgi.txt index eee5f791..ee3c0b92 100644 --- a/doc/outdated/fastcgi.txt +++ b/doc/outdated/fastcgi.txt @@ -107,7 +107,8 @@ fastcgi.server "max-procs" => , # OPTIONAL "broken-scriptfilename" => , # OPTIONAL "disable-time" => , # optional - "allow-x-send-file" => , # optional + "x-sendfile" => , # optional (replaces "allow-x-send-file") + "x-sendfile-docroot" => , # optional "kill-signal" => , # OPTIONAL "fix-root-scriptname" => , # OPTIONAL @@ -143,8 +144,10 @@ fastcgi.server PHP can extract PATH_INFO from it (default: disabled) :"disable-time": time to wait before a disabled backend is checked again - :"allow-x-send-file": controls if X-LIGHTTPD-send-file headers - are allowed + :"x-sendfile": controls if X-Sendfile backend response header is allowed + (deprecated headers: X-Sendfile2 and X-LIGHTTPD-send-file) + ("x-sendfile" replaces "allow-x-sendfile") + :"x-sendfile-docroot": list of directory trees permitted with X-Sendfile :"fix-root-scriptname": fix broken path-info split for "/" extension ("prefix") If bin-path is set: diff --git a/src/http-header-glue.c b/src/http-header-glue.c index 15ea8291..898917ec 100644 --- a/src/http-header-glue.c +++ b/src/http-header-glue.c @@ -529,7 +529,7 @@ void http_response_send_file (server *srv, connection *con, buffer *path) { stat_cache_entry *sce = NULL; buffer *mtime = NULL; data_string *ds; - int allow_caching = 1; + int allow_caching = (0 == con->http_status || 200 == con->http_status); if (HANDLER_ERROR == stat_cache_get_entry(srv, con, path, &sce)) { con->http_status = (errno == ENOENT) ? 404 : 403; @@ -613,7 +613,9 @@ void http_response_send_file (server *srv, connection *con, buffer *path) { } } - if (con->request.http_range && con->conf.range_requests && con->http_status < 300) { + if (con->request.http_range && con->conf.range_requests + && (200 == con->http_status || 0 == con->http_status) + && NULL == array_get_element(con->response.headers, "Content-Encoding")) { int do_range_request = 1; /* check if we have a conditional GET */ @@ -665,3 +667,58 @@ void http_response_send_file (server *srv, connection *con, buffer *path) { con->http_status = 403; } } + +void http_response_xsendfile (server *srv, connection *con, buffer *path, const array *xdocroot) { + const int status = con->http_status; + int valid = 1; + + /* reset Content-Length, if set by backend + * Content-Length might later be set to size of X-Sendfile static file, + * determined by open(), fstat() to reduces race conditions if the file + * is modified between stat() (stat_cache_get_entry()) and open(). */ + if (con->parsed_response & HTTP_CONTENT_LENGTH) { + data_string *ds = (data_string *) array_get_element(con->response.headers, "Content-Length"); + if (ds) buffer_reset(ds->value); + con->parsed_response &= ~HTTP_CONTENT_LENGTH; + con->response.content_length = -1; + } + + buffer_urldecode_path(path); + buffer_path_simplify(path, path); + if (con->conf.force_lowercase_filenames) { + buffer_to_lower(path); + } + + /* check that path is under xdocroot(s) + * - xdocroot should have trailing slash appended at config time + * - con->conf.force_lowercase_filenames is not a server-wide setting, + * and so can not be definitively applied to xdocroot at config time*/ + if (xdocroot->used) { + size_t i, xlen = buffer_string_length(path); + for (i = 0; i < xdocroot->used; ++i) { + data_string *ds = (data_string *)xdocroot->data[i]; + size_t dlen = buffer_string_length(ds->value); + if (dlen <= xlen + && (!con->conf.force_lowercase_filenames + ? 0 == memcmp(path->ptr, ds->value->ptr, dlen) + : 0 == strncasecmp(path->ptr, ds->value->ptr, dlen))) { + break; + } + } + if (i == xdocroot->used) { + log_error_write(srv, __FILE__, __LINE__, "SBs", + "X-Sendfile (", path, + ") not under configured x-sendfile-docroot(s)"); + con->http_status = 403; + valid = 0; + } + } + + if (valid) http_response_send_file(srv, con, path); + + if (con->http_status >= 400 && status < 300) { + con->mode = DIRECT; + } else if (0 != status && 200 != status) { + con->http_status = status; + } +} diff --git a/src/mod_fastcgi.c b/src/mod_fastcgi.c index 73b9de71..36dcd687 100644 --- a/src/mod_fastcgi.c +++ b/src/mod_fastcgi.c @@ -226,11 +226,12 @@ typedef struct { unsigned short fix_root_path_name; /* - * If the backend includes X-LIGHTTPD-send-file in the response + * If the backend includes X-Sendfile in the response * we use the value as filename and ignore the content. * */ - unsigned short allow_xsendfile; + unsigned short xsendfile_allow; + array *xsendfile_docroot; ssize_t load; /* replace by host->load */ @@ -549,6 +550,7 @@ static fcgi_extension_host *fastcgi_host_init(void) { f->bin_env = array_init(); f->bin_env_copy = array_init(); f->strip_request_uri = buffer_init(); + f->xsendfile_docroot = array_init(); return f; } @@ -564,6 +566,7 @@ static void fastcgi_host_free(fcgi_extension_host *h) { buffer_free(h->strip_request_uri); array_free(h->bin_env); array_free(h->bin_env_copy); + array_free(h->xsendfile_docroot); fastcgi_process_free(h->first); fastcgi_process_free(h->unused_procs); @@ -1287,6 +1290,8 @@ SETDEFAULTS_FUNC(mod_fastcgi_set_defaults) { { "kill-signal", NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION }, /* 14 */ { "fix-root-scriptname", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, /* 15 */ { "listen-backlog", NULL, T_CONFIG_INT, T_CONFIG_SCOPE_CONNECTION }, /* 16 */ + { "x-sendfile", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, /* 17 */ + { "x-sendfile-docroot",NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION }, /* 18 */ { NULL, NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET } }; @@ -1310,7 +1315,7 @@ SETDEFAULTS_FUNC(mod_fastcgi_set_defaults) { host->mode = FCGI_RESPONDER; host->disable_time = 1; host->break_scriptfilename_for_php = 0; - host->allow_xsendfile = 0; /* handle X-LIGHTTPD-send-file */ + host->xsendfile_allow = 0; host->kill_signal = SIGTERM; host->fix_root_path_name = 0; host->listen_backlog = 1024; @@ -1329,11 +1334,13 @@ SETDEFAULTS_FUNC(mod_fastcgi_set_defaults) { fcv[9].destination = host->bin_env; fcv[10].destination = host->bin_env_copy; fcv[11].destination = &(host->break_scriptfilename_for_php); - fcv[12].destination = &(host->allow_xsendfile); + fcv[12].destination = &(host->xsendfile_allow); fcv[13].destination = host->strip_request_uri; fcv[14].destination = &(host->kill_signal); fcv[15].destination = &(host->fix_root_path_name); fcv[16].destination = &(host->listen_backlog); + fcv[17].destination = &(host->xsendfile_allow); + fcv[18].destination = host->xsendfile_docroot; if (0 != config_insert_values_internal(srv, da_host->value, fcv, T_CONFIG_SCOPE_CONNECTION)) { goto error; @@ -1484,6 +1491,25 @@ SETDEFAULTS_FUNC(mod_fastcgi_set_defaults) { } } + if (host->xsendfile_docroot->used) { + size_t k; + for (k = 0; k < host->xsendfile_docroot->used; ++k) { + data_string *ds = (data_string *)host->xsendfile_docroot->data[k]; + if (ds->type != TYPE_STRING) { + log_error_write(srv, __FILE__, __LINE__, "s", + "unexpected type for x-sendfile-docroot; expected: \"x-sendfile-docroot\" => ( \"/allowed/path\", ... )"); + goto error; + } + if (ds->value->ptr[0] != '/') { + log_error_write(srv, __FILE__, __LINE__, "SBs", + "x-sendfile-docroot paths must begin with '/'; invalid: \"", ds->value, "\""); + goto error; + } + buffer_path_simplify(ds->value, ds->value); + buffer_append_slash(ds->value); + } + } + /* if extension already exists, take it */ fastcgi_extension_insert(s->exts, da_ext->key, host); host = NULL; @@ -2132,7 +2158,6 @@ static int fcgi_response_parse(server *srv, connection *con, plugin_data *p, buf for (s = in->ptr; NULL != (ns = strchr(s, '\n')); s = ns + 1) { char *key, *value; int key_len; - data_string *ds = NULL; /* a good day. Someone has read the specs and is sending a \r\n to us */ @@ -2162,6 +2187,7 @@ static int fcgi_response_parse(server *srv, connection *con, plugin_data *p, buf /* don't forward Status: */ if (0 != strncasecmp(key, "Status", key_len)) { + data_string *ds; if (NULL == (ds = (data_string *)array_get_unused_element(con->response.headers, TYPE_STRING))) { ds = data_response_init(); } @@ -2201,7 +2227,7 @@ static int fcgi_response_parse(server *srv, connection *con, plugin_data *p, buf } break; case 11: - if (host->allow_xsendfile && 0 == strncasecmp(key, "X-Sendfile2", key_len)&& hctx->send_content_body) { + if (host->xsendfile_allow && 0 == strncasecmp(key, "X-Sendfile2", key_len) && hctx->send_content_body) { char *pos = value; have_sendfile2 = 1; @@ -2227,6 +2253,30 @@ static int fcgi_response_parse(server *srv, connection *con, plugin_data *p, buf for (pos = ++range; *pos && *pos != ' ' && *pos != ','; pos++) ; buffer_urldecode_path(srv->tmp_buf); + buffer_path_simplify(srv->tmp_buf, srv->tmp_buf); + if (con->conf.force_lowercase_filenames) { + buffer_to_lower(srv->tmp_buf); + } + if (host->xsendfile_docroot->used) { + size_t i, xlen = buffer_string_length(srv->tmp_buf); + for (i = 0; i < host->xsendfile_docroot->used; ++i) { + data_string *ds = (data_string *)host->xsendfile_docroot->data[i]; + size_t dlen = buffer_string_length(ds->value); + if (dlen <= xlen + && (!con->conf.force_lowercase_filenames + ? 0 == memcmp(srv->tmp_buf->ptr, ds->value->ptr, dlen) + : 0 == strncasecmp(srv->tmp_buf->ptr, ds->value->ptr, dlen))) { + break; + } + } + if (i == host->xsendfile_docroot->used) { + log_error_write(srv, __FILE__, __LINE__, "SBs", + "X-Sendfile2 (", srv->tmp_buf, + ") not under configured x-sendfile-docroot(s)"); + return 403; + } + } + if (HANDLER_ERROR == stat_cache_get_entry(srv, con, srv->tmp_buf, &sce)) { if (p->conf.debug) { log_error_write(srv, __FILE__, __LINE__, "sb", @@ -2526,44 +2576,26 @@ static int fcgi_demux_response(server *srv, handler_ctx *hctx) { hctx->send_content_body = 0; } - if (host->allow_xsendfile && hctx->send_content_body && + if (host->xsendfile_allow && hctx->send_content_body && (NULL != (ds = (data_string *) array_get_element(con->response.headers, "X-LIGHTTPD-send-file")) || NULL != (ds = (data_string *) array_get_element(con->response.headers, "X-Sendfile")))) { - if (0 == http_chunk_append_file(srv, con, ds->value)) { - /* found */ - data_string *dcls = (data_string *) array_get_element(con->response.headers, "Content-Length"); - if (dcls) buffer_reset(dcls->value); - con->parsed_response &= ~HTTP_CONTENT_LENGTH; - con->response.content_length = -1; - hctx->send_content_body = 0; /* ignore the content */ - } else { - log_error_write(srv, __FILE__, __LINE__, "sb", - "send-file error: couldn't get stat_cache entry for:", - ds->value); - con->http_status = 404; - hctx->send_content_body = 0; - con->file_started = 1; + http_response_xsendfile(srv, con, ds->value, host->xsendfile_docroot); + if (con->mode == DIRECT) { + fin = 1; break; } - } - - if (hctx->send_content_body && buffer_string_length(packet.b) > 0) { - /* enable chunked-transfer-encoding */ - if (con->request.http_version == HTTP_VERSION_1_1 && - !(con->parsed_response & HTTP_CONTENT_LENGTH)) { - con->response.transfer_encoding = HTTP_TRANSFER_ENCODING_CHUNKED; - } - - http_chunk_append_buffer(srv, con, packet.b); + hctx->send_content_body = 0; /* ignore the content */ } - } else if (hctx->send_content_body && !buffer_string_is_empty(packet.b)) { + + /* enable chunked-transfer-encoding */ if (con->request.http_version == HTTP_VERSION_1_1 && !(con->parsed_response & HTTP_CONTENT_LENGTH)) { - /* enable chunked-transfer-encoding */ con->response.transfer_encoding = HTTP_TRANSFER_ENCODING_CHUNKED; } + } + if (hctx->send_content_body && !buffer_string_is_empty(packet.b)) { http_chunk_append_buffer(srv, con, packet.b); } break; diff --git a/src/response.h b/src/response.h index f38b8413..6ccb87a8 100644 --- a/src/response.h +++ b/src/response.h @@ -17,6 +17,7 @@ handler_t http_response_prepare(server *srv, connection *con); int http_response_redirect_to_directory(server *srv, connection *con); int http_response_handle_cachable(server *srv, connection *con, buffer * mtime); void http_response_send_file (server *srv, connection *con, buffer *path); +void http_response_xsendfile (server *srv, connection *con, buffer *path, const array *xdocroot); buffer * strftime_cache_get(server *srv, time_t last_mod); #endif From 0a907c643bed54bf19cef1a2a296882e1886f809 Mon Sep 17 00:00:00 2001 From: Glenn Strauss Date: Thu, 21 Apr 2016 23:05:48 -0400 Subject: [PATCH 3/4] [mod_scgi] X-Sendfile feature (fixes #2253) handle X-Sendfile with http_response_xsendfile() if host configured ( "x-sendfile" = "enable" ) x-ref: "scgi x-sendfile" https://redmine.lighttpd.net/issues/2253 --- src/mod_scgi.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/mod_scgi.c b/src/mod_scgi.c index 5c2fcfe5..e1adc947 100644 --- a/src/mod_scgi.c +++ b/src/mod_scgi.c @@ -212,6 +212,15 @@ typedef struct { */ unsigned short fix_root_path_name; + + /* + * If the backend includes X-Sendfile in the response + * we use the value as filename and ignore the content. + * + */ + unsigned short xsendfile_allow; + array *xsendfile_docroot; + ssize_t load; /* replace by host->load */ size_t max_id; /* corresponds most of the time to @@ -416,6 +425,7 @@ static scgi_extension_host *scgi_host_init(void) { f->bin_path = buffer_init(); f->bin_env = array_init(); f->bin_env_copy = array_init(); + f->xsendfile_docroot = array_init(); return f; } @@ -429,6 +439,7 @@ static void scgi_host_free(scgi_extension_host *h) { buffer_free(h->bin_path); array_free(h->bin_env); array_free(h->bin_env_copy); + array_free(h->xsendfile_docroot); scgi_process_free(h->first); scgi_process_free(h->unused_procs); @@ -1053,6 +1064,8 @@ SETDEFAULTS_FUNC(mod_scgi_set_defaults) { { "bin-copy-environment", NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION }, /* 12 */ { "fix-root-scriptname", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, /* 13 */ { "listen-backlog", NULL, T_CONFIG_INT, T_CONFIG_SCOPE_CONNECTION }, /* 14 */ + { "x-sendfile", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, /* 15 */ + { "x-sendfile-docroot",NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION }, /* 16 */ { NULL, NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET } @@ -1077,6 +1090,7 @@ SETDEFAULTS_FUNC(mod_scgi_set_defaults) { df->disable_time = 60; df->fix_root_path_name = 0; df->listen_backlog = 1024; + df->xsendfile_allow = 0; fcv[0].destination = df->host; fcv[1].destination = df->docroot; @@ -1095,6 +1109,8 @@ SETDEFAULTS_FUNC(mod_scgi_set_defaults) { fcv[12].destination = df->bin_env_copy; fcv[13].destination = &(df->fix_root_path_name); fcv[14].destination = &(df->listen_backlog); + fcv[15].destination = &(df->xsendfile_allow); + fcv[16].destination = df->xsendfile_docroot; if (0 != config_insert_values_internal(srv, da_host->value, fcv, T_CONFIG_SCOPE_CONNECTION)) { @@ -1227,6 +1243,25 @@ SETDEFAULTS_FUNC(mod_scgi_set_defaults) { df->max_procs = 1; } + if (df->xsendfile_docroot->used) { + size_t k; + for (k = 0; k < df->xsendfile_docroot->used; ++k) { + data_string *ds = (data_string *)df->xsendfile_docroot->data[k]; + if (ds->type != TYPE_STRING) { + log_error_write(srv, __FILE__, __LINE__, "s", + "unexpected type for x-sendfile-docroot; expected: \"x-sendfile-docroot\" => ( \"/allowed/path\", ... )"); + goto error; + } + if (ds->value->ptr[0] != '/') { + log_error_write(srv, __FILE__, __LINE__, "SBs", + "x-sendfile-docroot paths must begin with '/'; invalid: \"", ds->value, "\""); + goto error; + } + buffer_path_simplify(ds->value, ds->value); + buffer_append_slash(ds->value); + } + } + /* if extension already exists, take it */ scgi_extension_insert(s->exts, da_ext->key, df); df = NULL; @@ -1916,6 +1951,14 @@ static int scgi_demux_response(server *srv, handler_ctx *hctx) { /* parse the response header */ scgi_response_parse(srv, con, p, hctx->response_header, eol); + if (hctx->host->xsendfile_allow) { + data_string *ds; + if (NULL != (ds = (data_string *) array_get_element(con->response.headers, "X-Sendfile"))) { + http_response_xsendfile(srv, con, ds->value, hctx->host->xsendfile_docroot); + return 1; + } + } + /* enable chunked-transfer-encoding */ if (con->request.http_version == HTTP_VERSION_1_1 && !(con->parsed_response & HTTP_CONTENT_LENGTH)) { From 1f23ba9adf1d89d7d776312805c894a41b3f9296 Mon Sep 17 00:00:00 2001 From: Glenn Strauss Date: Thu, 21 Apr 2016 23:57:14 -0400 Subject: [PATCH 4/4] [mod_cgi] X-Sendfile feature (fixes #2313) handle X-Sendfile with http_response_xsendfile() if cgi.x-sendfile = "enable" x-ref: "X-sendfile support for mod_cgi" https://redmine.lighttpd.net/issues/2313 --- src/mod_cgi.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/mod_cgi.c b/src/mod_cgi.c index c6f70b90..8427ae82 100644 --- a/src/mod_cgi.c +++ b/src/mod_cgi.c @@ -6,6 +6,7 @@ #include "log.h" #include "connections.h" #include "joblist.h" +#include "response.h" #include "http_chunk.h" #include "network_backends.h" @@ -55,6 +56,8 @@ typedef struct { typedef struct { array *cgi; unsigned short execute_x_only; + unsigned short xsendfile_allow; + array *xsendfile_docroot; } plugin_config; typedef struct { @@ -133,6 +136,7 @@ FREE_FUNC(mod_cgi_free) { if (NULL == s) continue; array_free(s->cgi); + array_free(s->xsendfile_docroot); free(s); } @@ -157,6 +161,8 @@ SETDEFAULTS_FUNC(mod_fastcgi_set_defaults) { config_values_t cv[] = { { "cgi.assign", NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION }, /* 0 */ { "cgi.execute-x-only", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, /* 1 */ + { "x-sendfile", NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION }, /* 2 */ + { "x-sendfile-docroot", NULL, T_CONFIG_ARRAY, T_CONFIG_SCOPE_CONNECTION }, /* 3 */ { NULL, NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET} }; @@ -174,15 +180,38 @@ SETDEFAULTS_FUNC(mod_fastcgi_set_defaults) { s->cgi = array_init(); s->execute_x_only = 0; + s->xsendfile_allow= 0; + s->xsendfile_docroot = array_init(); cv[0].destination = s->cgi; cv[1].destination = &(s->execute_x_only); + cv[2].destination = &(s->xsendfile_allow); + cv[3].destination = s->xsendfile_docroot; p->config_storage[i] = s; if (0 != config_insert_values_global(srv, config->value, cv, i == 0 ? T_CONFIG_SCOPE_SERVER : T_CONFIG_SCOPE_CONNECTION)) { return HANDLER_ERROR; } + + if (s->xsendfile_docroot->used) { + size_t j; + for (j = 0; j < s->xsendfile_docroot->used; ++j) { + data_string *ds = (data_string *)s->xsendfile_docroot->data[j]; + if (ds->type != TYPE_STRING) { + log_error_write(srv, __FILE__, __LINE__, "s", + "unexpected type for key cgi.x-sendfile-docroot; expected: cgi.x-sendfile-docroot = ( \"/allowed/path\", ... )"); + return HANDLER_ERROR; + } + if (ds->value->ptr[0] != '/') { + log_error_write(srv, __FILE__, __LINE__, "SBs", + "cgi.x-sendfile-docroot paths must begin with '/'; invalid: \"", ds->value, "\""); + return HANDLER_ERROR; + } + buffer_path_simplify(ds->value, ds->value); + buffer_append_slash(ds->value); + } + } } return HANDLER_GO_ON; @@ -497,6 +526,14 @@ static int cgi_demux_response(server *srv, handler_ctx *hctx) { /* parse the response header */ cgi_response_parse(srv, con, p, hctx->response_header); + if (p->conf.xsendfile_allow) { + data_string *ds; + if (NULL != (ds = (data_string *) array_get_element(con->response.headers, "X-Sendfile"))) { + http_response_xsendfile(srv, con, ds->value, p->conf.xsendfile_docroot); + return FDEVENT_HANDLED_FINISHED; + } + } + /* enable chunked-transfer-encoding */ if (con->request.http_version == HTTP_VERSION_1_1 && !(con->parsed_response & HTTP_CONTENT_LENGTH)) { @@ -1307,6 +1344,8 @@ static int mod_cgi_patch_connection(server *srv, connection *con, plugin_data *p PATCH(cgi); PATCH(execute_x_only); + PATCH(xsendfile_allow); + PATCH(xsendfile_docroot); /* skip the first, the global context */ for (i = 1; i < srv->config_context->used; i++) { @@ -1324,6 +1363,10 @@ static int mod_cgi_patch_connection(server *srv, connection *con, plugin_data *p PATCH(cgi); } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("cgi.execute-x-only"))) { PATCH(execute_x_only); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("cgi.x-sendfile"))) { + PATCH(xsendfile_allow); + } else if (buffer_is_equal_string(du->key, CONST_STR_LEN("cgi.x-sendfile-docroot"))) { + PATCH(xsendfile_docroot); } } }