[core] handle if backend sends Transfer-Encoding (#2786)

It is still not a good idea for backend to send Transfer-Encoding unless
backend is mod_proxy, and mod_proxy should not currently receive chunked
response since mod_proxy sends HTTP/1.0 request.

If mod_proxy is changed to sent HTTP/1.1 request, then lighttpd would
need to check if client is HTTP/1.0 and would need to de-chunk and
remove any other transfer-codings if not supported by next-hop.

x-ref:
  "error 500 (mod_cgi.c.601) cgi died"
  https://redmine.lighttpd.net/issues/2786

src/base.h

@@ -131,6 +131,7 @@ typedef union {
 #define HTTP_CONTENT_LENGTH BV(2)
 #define HTTP_DATE           BV(3)
 #define HTTP_LOCATION       BV(4)
+#define HTTP_TRANSFER_ENCODING BV(5)
 
 typedef struct {
 	/** HEADER */

src/connections.c

@@ -386,8 +386,7 @@ static int connection_handle_write_prepare(server *srv, connection *con) {
 	if (con->file_finished) {
 		/* we have all the content and chunked encoding is not used, set a content-length */
 
-		if ((!(con->parsed_response & HTTP_CONTENT_LENGTH)) &&
-		    (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) == 0) {
+		if (!(con->parsed_response & (HTTP_CONTENT_LENGTH|HTTP_TRANSFER_ENCODING))) {
 			off_t qlen = chunkqueue_length(con->write_queue);
 
 			/**
@@ -424,8 +423,7 @@ static int connection_handle_write_prepare(server *srv, connection *con) {
 		 * - Transfer-Encoding: chunked (HTTP/1.1)
 		 */
 
-		if (((con->parsed_response & HTTP_CONTENT_LENGTH) == 0) &&
-		    ((con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) == 0)) {
+		if (!(con->parsed_response & (HTTP_CONTENT_LENGTH|HTTP_TRANSFER_ENCODING))) {
 			if (con->request.http_version == HTTP_VERSION_1_1) {
 				off_t qlen = chunkqueue_length(con->write_queue);
 				con->response.transfer_encoding = HTTP_TRANSFER_ENCODING_CHUNKED;
@@ -438,6 +436,7 @@ static int connection_handle_write_prepare(server *srv, connection *con) {
 					chunkqueue_prepend_buffer(con->write_queue, b);
 					chunkqueue_append_mem(con->write_queue, CONST_STR_LEN("\r\n"));
 				}
+				response_header_append(srv, con, CONST_STR_LEN("Transfer-Encoding"), CONST_STR_LEN("chunked"));
 			} else {
 				con->keep_alive = 0;
 			}
@@ -469,6 +468,12 @@ static int connection_handle_write_prepare(server *srv, connection *con) {
 
 		chunkqueue_reset(con->write_queue);
 		con->response.transfer_encoding &= ~HTTP_TRANSFER_ENCODING_CHUNKED;
+		if (con->parsed_response & HTTP_TRANSFER_ENCODING) {
+			data_string *ds;
+			if (NULL != (ds = (data_string*) array_get_element(con->response.headers, "Transfer-Encoding"))) {
+				buffer_reset(ds->value); /* Headers with empty values are ignored for output */
+			}
+		}
 	}
 
 	http_response_write_header(srv, con);

src/mod_cgi.c

@@ -373,6 +373,11 @@ static int cgi_response_parse(server *srv, connection *con, plugin_data *p, buff
 					con->parsed_response |= HTTP_CONTENT_LENGTH;
 				}
 				break;
+			case 17:
+				if (0 == strncasecmp(key, "Transfer-Encoding", key_len)) {
+					con->parsed_response |= HTTP_TRANSFER_ENCODING;
+				}
+				break;
 			default:
 				break;
 			}

src/mod_deflate.c

@@ -1036,7 +1036,7 @@ CONNECTION_FUNC(mod_deflate_handle_response_start) {
 	/*(current implementation requires response be complete)*/
 	if (!con->file_finished) return HANDLER_GO_ON;
 	if (con->request.http_method == HTTP_METHOD_HEAD) return HANDLER_GO_ON;
-	if (con->parsed_response & HTTP_TRANSFER_ENCODING_CHUNKED) return HANDLER_GO_ON;
+	if (con->parsed_response & HTTP_TRANSFER_ENCODING) return HANDLER_GO_ON;
 
 	/* disable compression for some http status types. */
 	switch(con->http_status) {

src/mod_fastcgi.c

@@ -2250,6 +2250,11 @@ range_success: ;
 				if (con->response.content_length < 0) con->response.content_length = 0;
 			}
 			break;
+		case 17:
+			if (0 == strncasecmp(key, "Transfer-Encoding", key_len)) {
+				con->parsed_response |= HTTP_TRANSFER_ENCODING;
+			}
+			break;
 		default:
 			break;
 		}

src/mod_proxy.c

@@ -811,6 +811,11 @@ static int proxy_response_parse(server *srv, connection *con, plugin_data *p, bu
 				con->parsed_response |= HTTP_CONTENT_LENGTH;
 			}
 			break;
+		case 17:
+			if (0 == strncasecmp(key, "Transfer-Encoding", key_len)) {
+				con->parsed_response |= HTTP_TRANSFER_ENCODING;
+			}
+			break;
 		default:
 			break;
 		}

src/mod_scgi.c

@@ -1763,6 +1763,11 @@ static int scgi_response_parse(server *srv, connection *con, plugin_data *p, buf
 					con->parsed_response |= HTTP_CONTENT_LENGTH;
 				}
 				break;
+			case 17:
+				if (0 == strncasecmp(key, "Transfer-Encoding", key_len)) {
+					con->parsed_response |= HTTP_TRANSFER_ENCODING;
+				}
+				break;
 			default:
 				break;
 			}

src/response.c

@@ -59,11 +59,6 @@ int http_response_write_header(server *srv, connection *con) {
 		}
 	}
 
-	if (con->response.transfer_encoding & HTTP_TRANSFER_ENCODING_CHUNKED) {
-		response_header_overwrite(srv, con, CONST_STR_LEN("Transfer-Encoding"), CONST_STR_LEN("chunked"));
-	}
-
-
 	/* add all headers */
 	for (i = 0; i < con->response.headers->used; i++) {
 		data_string *ds;