[mod_fastcgi] use http_response_xsendfile() (fixes #799, fixes #851, fixes #2017, fixes #2076)

handle X-Sendfile and X-LIGHTTPD-send-file w/ http_response_xsendfile()
  if host is configured ( "x-sendfile" = "enable" )

Note: X-Sendfile path is url-decoded for consistency, like X-Sendfile2
      (response headers should be url-encoded to avoid tripping over
       chars allowed in filesystem but which might change response
       header parsing semantics)

Note: deprecated: "allow-x-send-file";         use "x-sendfile"
Note: deprecated: X-LIGHTTPD-send-file header; use X-Sendfile header
Note: deprecated: X-Sendfile2 header;          use X-Sendfile header
For now, X-Sendfile2 is still handled internally by mod_fastcgi.

Since http_response_send_file() supports HTTP Range requests,
X-Sendfile2 is effectively obsolete.  However, any code, e.g. PHP,
currently using X-Sendfile2 is probably manually generating 206 Partial
Content status and Range response headers.  A future version of lighttpd
might *remove* X-Sendfile2.  Existing code should be converted to use
X-Sendfile, which is easily done by removing all the special logic
around using X-Sendfile2, since the 206 Partial Content status and Range
response headers are handled in http_response_send_file().

x-ref:
  "mod_fastcgi + X-Sendfile -> mod_staticfile"
  https://redmine.lighttpd.net/issues/799
  "Feature Request: New option "x-send-file-docroot""
  https://redmine.lighttpd.net/issues/851
  "X-Sendfile handoff to mod-static-file in 1.4.x"
  https://redmine.lighttpd.net/issues/2017
  "X-sendfile should be able to set content-type"
  https://redmine.lighttpd.net/issues/2076

doc/outdated/fastcgi.txt

@@ -107,7 +107,8 @@ fastcgi.server
           "max-procs" => <integer>,   # OPTIONAL
           "broken-scriptfilename" => <boolean>, # OPTIONAL
           "disable-time" => <integer>, # optional
-          "allow-x-send-file" => <boolean>, # optional
+          "x-sendfile" => <boolean>, # optional (replaces "allow-x-send-file")
+          "x-sendfile-docroot" => <boolean>, # optional
           "kill-signal" => <integer>, # OPTIONAL
           "fix-root-scriptname" => <boolean>,
                                       # OPTIONAL
@@ -143,8 +144,10 @@ fastcgi.server
                 PHP can extract PATH_INFO from it (default: disabled)
   :"disable-time": time to wait before a disabled backend is checked
                 again
-  :"allow-x-send-file": controls if X-LIGHTTPD-send-file headers
-                are allowed
+  :"x-sendfile": controls if X-Sendfile backend response header is allowed
+                (deprecated headers: X-Sendfile2 and X-LIGHTTPD-send-file)
+                ("x-sendfile" replaces "allow-x-sendfile")
+  :"x-sendfile-docroot": list of directory trees permitted with X-Sendfile
   :"fix-root-scriptname": fix broken path-info split for "/" extension ("prefix")
 
   If bin-path is set:

src/http-header-glue.c

@@ -529,7 +529,7 @@ void http_response_send_file (server *srv, connection *con, buffer *path) {
 	stat_cache_entry *sce = NULL;
 	buffer *mtime = NULL;
 	data_string *ds;
-	int allow_caching = 1;
+	int allow_caching = (0 == con->http_status || 200 == con->http_status);
 
 	if (HANDLER_ERROR == stat_cache_get_entry(srv, con, path, &sce)) {
 		con->http_status = (errno == ENOENT) ? 404 : 403;
@@ -613,7 +613,9 @@ void http_response_send_file (server *srv, connection *con, buffer *path) {
 		}
 	}
 
-	if (con->request.http_range && con->conf.range_requests && con->http_status < 300) {
+	if (con->request.http_range && con->conf.range_requests
+	    && (200 == con->http_status || 0 == con->http_status)
+	    && NULL == array_get_element(con->response.headers, "Content-Encoding")) {
 		int do_range_request = 1;
 		/* check if we have a conditional GET */
 
@@ -665,3 +667,58 @@ void http_response_send_file (server *srv, connection *con, buffer *path) {
 		con->http_status = 403;
 	}
 }
+
+void http_response_xsendfile (server *srv, connection *con, buffer *path, const array *xdocroot) {
+	const int status = con->http_status;
+	int valid = 1;
+
+	/* reset Content-Length, if set by backend
+	 * Content-Length might later be set to size of X-Sendfile static file,
+	 * determined by open(), fstat() to reduces race conditions if the file
+	 * is modified between stat() (stat_cache_get_entry()) and open(). */
+	if (con->parsed_response & HTTP_CONTENT_LENGTH) {
+		data_string *ds = (data_string *) array_get_element(con->response.headers, "Content-Length");
+		if (ds) buffer_reset(ds->value);
+		con->parsed_response &= ~HTTP_CONTENT_LENGTH;
+		con->response.content_length = -1;
+	}
+
+	buffer_urldecode_path(path);
+	buffer_path_simplify(path, path);
+	if (con->conf.force_lowercase_filenames) {
+		buffer_to_lower(path);
+	}
+
+	/* check that path is under xdocroot(s)
+	 * - xdocroot should have trailing slash appended at config time
+	 * - con->conf.force_lowercase_filenames is not a server-wide setting,
+	 *   and so can not be definitively applied to xdocroot at config time*/
+	if (xdocroot->used) {
+		size_t i, xlen = buffer_string_length(path);
+		for (i = 0; i < xdocroot->used; ++i) {
+			data_string *ds = (data_string *)xdocroot->data[i];
+			size_t dlen = buffer_string_length(ds->value);
+			if (dlen <= xlen
+			    && (!con->conf.force_lowercase_filenames
+				? 0 == memcmp(path->ptr, ds->value->ptr, dlen)
+				: 0 == strncasecmp(path->ptr, ds->value->ptr, dlen))) {
+				break;
+			}
+		}
+		if (i == xdocroot->used) {
+			log_error_write(srv, __FILE__, __LINE__, "SBs",
+					"X-Sendfile (", path,
+					") not under configured x-sendfile-docroot(s)");
+			con->http_status = 403;
+			valid = 0;
+		}
+	}
+
+	if (valid) http_response_send_file(srv, con, path);
+
+	if (con->http_status >= 400 && status < 300) {
+		con->mode = DIRECT;
+	} else if (0 != status && 200 != status) {
+		con->http_status = status;
+	}
+}

src/mod_fastcgi.c

@@ -226,11 +226,12 @@ typedef struct {
 	unsigned short fix_root_path_name;
 
 	/*
-	 * If the backend includes X-LIGHTTPD-send-file in the response
+	 * If the backend includes X-Sendfile in the response
 	 * we use the value as filename and ignore the content.
 	 *
 	 */
-	unsigned short allow_xsendfile;
+	unsigned short xsendfile_allow;
+	array *xsendfile_docroot;
 
 	ssize_t load; /* replace by host->load */
 
@@ -549,6 +550,7 @@ static fcgi_extension_host *fastcgi_host_init(void) {
 	f->bin_env = array_init();
 	f->bin_env_copy = array_init();
 	f->strip_request_uri = buffer_init();
+	f->xsendfile_docroot = array_init();
 
 	return f;
 }
@@ -564,6 +566,7 @@ static void fastcgi_host_free(fcgi_extension_host *h) {
 	buffer_free(h->strip_request_uri);
 	array_free(h->bin_env);
 	array_free(h->bin_env_copy);
+	array_free(h->xsendfile_docroot);
 
 	fastcgi_process_free(h->first);
 	fastcgi_process_free(h->unused_procs);
@@ -1287,6 +1290,8 @@ SETDEFAULTS_FUNC(mod_fastcgi_set_defaults) {
 						{ "kill-signal",        NULL, T_CONFIG_SHORT, T_CONFIG_SCOPE_CONNECTION },       /* 14 */
 						{ "fix-root-scriptname",   NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION },  /* 15 */
 						{ "listen-backlog",    NULL, T_CONFIG_INT,   T_CONFIG_SCOPE_CONNECTION },        /* 16 */
+						{ "x-sendfile",        NULL, T_CONFIG_BOOLEAN, T_CONFIG_SCOPE_CONNECTION },      /* 17 */
+						{ "x-sendfile-docroot",NULL, T_CONFIG_ARRAY,  T_CONFIG_SCOPE_CONNECTION },      /* 18 */
 
 						{ NULL,                NULL, T_CONFIG_UNSET, T_CONFIG_SCOPE_UNSET }
 					};
@@ -1310,7 +1315,7 @@ SETDEFAULTS_FUNC(mod_fastcgi_set_defaults) {
 					host->mode = FCGI_RESPONDER;
 					host->disable_time = 1;
 					host->break_scriptfilename_for_php = 0;
-					host->allow_xsendfile = 0; /* handle X-LIGHTTPD-send-file */
+					host->xsendfile_allow = 0;
 					host->kill_signal = SIGTERM;
 					host->fix_root_path_name = 0;
 					host->listen_backlog = 1024;
@@ -1329,11 +1334,13 @@ SETDEFAULTS_FUNC(mod_fastcgi_set_defaults) {
 					fcv[9].destination = host->bin_env;
 					fcv[10].destination = host->bin_env_copy;
 					fcv[11].destination = &(host->break_scriptfilename_for_php);
-					fcv[12].destination = &(host->allow_xsendfile);
+					fcv[12].destination = &(host->xsendfile_allow);
 					fcv[13].destination = host->strip_request_uri;
 					fcv[14].destination = &(host->kill_signal);
 					fcv[15].destination = &(host->fix_root_path_name);
 					fcv[16].destination = &(host->listen_backlog);
+					fcv[17].destination = &(host->xsendfile_allow);
+					fcv[18].destination = host->xsendfile_docroot;
 
 					if (0 != config_insert_values_internal(srv, da_host->value, fcv, T_CONFIG_SCOPE_CONNECTION)) {
 						goto error;
@@ -1484,6 +1491,25 @@ SETDEFAULTS_FUNC(mod_fastcgi_set_defaults) {
 						}
 					}
 
+					if (host->xsendfile_docroot->used) {
+						size_t k;
+						for (k = 0; k < host->xsendfile_docroot->used; ++k) {
+							data_string *ds = (data_string *)host->xsendfile_docroot->data[k];
+							if (ds->type != TYPE_STRING) {
+								log_error_write(srv, __FILE__, __LINE__, "s",
+									"unexpected type for x-sendfile-docroot; expected: \"x-sendfile-docroot\" => ( \"/allowed/path\", ... )");
+								goto error;
+							}
+							if (ds->value->ptr[0] != '/') {
+								log_error_write(srv, __FILE__, __LINE__, "SBs",
+									"x-sendfile-docroot paths must begin with '/'; invalid: \"", ds->value, "\"");
+								goto error;
+							}
+							buffer_path_simplify(ds->value, ds->value);
+							buffer_append_slash(ds->value);
+						}
+					}
+
 					/* if extension already exists, take it */
 					fastcgi_extension_insert(s->exts, da_ext->key, host);
 					host = NULL;
@@ -2132,7 +2158,6 @@ static int fcgi_response_parse(server *srv, connection *con, plugin_data *p, buf
 	for (s = in->ptr; NULL != (ns = strchr(s, '\n')); s = ns + 1) {
 		char *key, *value;
 		int key_len;
-		data_string *ds = NULL;
 
 		/* a good day. Someone has read the specs and is sending a \r\n to us */
 
@@ -2162,6 +2187,7 @@ static int fcgi_response_parse(server *srv, connection *con, plugin_data *p, buf
 
 			/* don't forward Status: */
 			if (0 != strncasecmp(key, "Status", key_len)) {
+				data_string *ds;
 				if (NULL == (ds = (data_string *)array_get_unused_element(con->response.headers, TYPE_STRING))) {
 					ds = data_response_init();
 				}
@@ -2201,7 +2227,7 @@ static int fcgi_response_parse(server *srv, connection *con, plugin_data *p, buf
 			}
 			break;
 		case 11:
-			if (host->allow_xsendfile && 0 == strncasecmp(key, "X-Sendfile2", key_len)&& hctx->send_content_body) {
+			if (host->xsendfile_allow && 0 == strncasecmp(key, "X-Sendfile2", key_len) && hctx->send_content_body) {
 				char *pos = value;
 				have_sendfile2 = 1;
 
@@ -2227,6 +2253,30 @@ static int fcgi_response_parse(server *srv, connection *con, plugin_data *p, buf
 					for (pos = ++range; *pos && *pos != ' ' && *pos != ','; pos++) ;
 
 					buffer_urldecode_path(srv->tmp_buf);
+					buffer_path_simplify(srv->tmp_buf, srv->tmp_buf);
+					if (con->conf.force_lowercase_filenames) {
+						buffer_to_lower(srv->tmp_buf);
+					}
+					if (host->xsendfile_docroot->used) {
+						size_t i, xlen = buffer_string_length(srv->tmp_buf);
+						for (i = 0; i < host->xsendfile_docroot->used; ++i) {
+							data_string *ds = (data_string *)host->xsendfile_docroot->data[i];
+							size_t dlen = buffer_string_length(ds->value);
+							if (dlen <= xlen
+							    && (!con->conf.force_lowercase_filenames
+								? 0 == memcmp(srv->tmp_buf->ptr, ds->value->ptr, dlen)
+								: 0 == strncasecmp(srv->tmp_buf->ptr, ds->value->ptr, dlen))) {
+								break;
+							}
+						}
+						if (i == host->xsendfile_docroot->used) {
+							log_error_write(srv, __FILE__, __LINE__, "SBs",
+									"X-Sendfile2 (", srv->tmp_buf,
+									") not under configured x-sendfile-docroot(s)");
+							return 403;
+						}
+					}
+
 					if (HANDLER_ERROR == stat_cache_get_entry(srv, con, srv->tmp_buf, &sce)) {
 						if (p->conf.debug) {
 							log_error_write(srv, __FILE__, __LINE__, "sb",
@@ -2526,44 +2576,26 @@ static int fcgi_demux_response(server *srv, handler_ctx *hctx) {
 					hctx->send_content_body = 0;
 				}
 
-				if (host->allow_xsendfile && hctx->send_content_body &&
+				if (host->xsendfile_allow && hctx->send_content_body &&
 				    (NULL != (ds = (data_string *) array_get_element(con->response.headers, "X-LIGHTTPD-send-file"))
 					  || NULL != (ds = (data_string *) array_get_element(con->response.headers, "X-Sendfile")))) {
-					if (0 == http_chunk_append_file(srv, con, ds->value)) {
-						/* found */
-						data_string *dcls = (data_string *) array_get_element(con->response.headers, "Content-Length");
-						if (dcls) buffer_reset(dcls->value);
-						con->parsed_response &= ~HTTP_CONTENT_LENGTH;
-						con->response.content_length = -1;
-						hctx->send_content_body = 0; /* ignore the content */
-					} else {
-						log_error_write(srv, __FILE__, __LINE__, "sb",
-							"send-file error: couldn't get stat_cache entry for:",
-							ds->value);
-						con->http_status = 404;
-						hctx->send_content_body = 0;
-						con->file_started = 1;
+					http_response_xsendfile(srv, con, ds->value, host->xsendfile_docroot);
+					if (con->mode == DIRECT) {
+						fin = 1;
 						break;
 					}
-				}
 
-
-				if (hctx->send_content_body && buffer_string_length(packet.b) > 0) {
-					/* enable chunked-transfer-encoding */
-					if (con->request.http_version == HTTP_VERSION_1_1 &&
-					    !(con->parsed_response & HTTP_CONTENT_LENGTH)) {
-						con->response.transfer_encoding = HTTP_TRANSFER_ENCODING_CHUNKED;
-					}
-
-					http_chunk_append_buffer(srv, con, packet.b);
+					hctx->send_content_body = 0; /* ignore the content */
 				}
-			} else if (hctx->send_content_body && !buffer_string_is_empty(packet.b)) {
+
+				/* enable chunked-transfer-encoding */
 				if (con->request.http_version == HTTP_VERSION_1_1 &&
 				    !(con->parsed_response & HTTP_CONTENT_LENGTH)) {
-					/* enable chunked-transfer-encoding */
 					con->response.transfer_encoding = HTTP_TRANSFER_ENCODING_CHUNKED;
 				}
+			}
 
+			if (hctx->send_content_body && !buffer_string_is_empty(packet.b)) {
 				http_chunk_append_buffer(srv, con, packet.b);
 			}
 			break;

src/response.h

@@ -17,6 +17,7 @@ handler_t http_response_prepare(server *srv, connection *con);
 int http_response_redirect_to_directory(server *srv, connection *con);
 int http_response_handle_cachable(server *srv, connection *con, buffer * mtime);
 void http_response_send_file (server *srv, connection *con, buffer *path);
+void http_response_xsendfile (server *srv, connection *con, buffer *path, const array *xdocroot);
 
 buffer * strftime_cache_get(server *srv, time_t last_mod);
 #endif