[core] decode Transfer-Encoding: chunked from gw

decode Transfer-Encoding: chunked from gw (gateway backends)

Transfer-Encoding: chunked is a hop-by-hop header.

Handling chunked encoding remove a hurdle for mod_proxy to send HTTP/1.1
requests to backends and be able to handle HTTP/1.1 responses.

Other backends ought not to send Transfer-Encoding: chunked, but in
practice, some implementations do.
This commit is contained in:
Glenn Strauss 2020-07-28 07:32:29 -04:00
parent dbd438cca0
commit 7420526ddb
5 changed files with 274 additions and 5 deletions

View File

@ -330,6 +330,44 @@ static void connection_handle_errdoc(request_st * const r) {
CONST_STR_LEN("text/html"));
}
__attribute_cold__
static void
connection_merge_trailers (request_st * const r)
{
/* attempt to merge trailers into headers; header not yet sent by caller */
if (buffer_string_is_empty(&r->gw_dechunk->b)) return;
const int done = r->gw_dechunk->done;
if (!done) return; /* XXX: !done; could scan for '\n' and send only those */
/* do not include trailers if success status (when response was read from
* backend) subsequently changed to error status. http_chunk could add the
* trailers, but such actions are better on a different code layer than in
* http_chunk.c */
if (done < 400 && r->http_status >= 400) return;
/* XXX: trailers passed through; no sanity check currently done
* https://tools.ietf.org/html/rfc7230#section-4.1.2
*
* Not checking for disallowed fields
* Not handling (deprecated) line wrapping
* Not strictly checking fields
*/
const char *k = strchr(r->gw_dechunk->b.ptr, '\n'); /*(skip final chunk)*/
if (NULL == k) return; /*(should not happen)*/
++k;
for (const char *v, *e; (e = strchr(k, '\n')); k = e+1) {
v = memchr(k, ':', (size_t)(e - k));
if (NULL == v || v == k || *k == ' ' || *k == '\t') continue;
uint32_t klen = (uint32_t)(v - k);
do { ++v; } while (*v == ' ' || *v == '\t');
if (*v == '\r' || *v == '\n') continue;
enum http_header_e id = http_header_hkey_get(k, klen);
http_header_response_insert(r, id, k, klen, v, (size_t)(e - v));
}
http_header_response_unset(r, HTTP_HEADER_OTHER, CONST_STR_LEN("Trailer"));
buffer_clear(&r->gw_dechunk->b);
}
static int connection_handle_write_prepare(request_st * const r) {
if (NULL == r->handler_module) {
/* static files */
@ -381,6 +419,9 @@ static int connection_handle_write_prepare(request_st * const r) {
break;
}
if (r->gw_dechunk)
connection_merge_trailers(r);
/* Allow filter plugins to change response headers before they are written. */
switch(plugins_call_handle_response_start(r)) {
case HANDLER_GO_ON:

View File

@ -243,6 +243,13 @@ void http_response_body_clear (request_st * const r, int preserve_length) {
if (r->resp_htags & HTTP_HEADER_CONTENT_LENGTH) {
http_header_response_unset(r, HTTP_HEADER_CONTENT_LENGTH, CONST_STR_LEN("Content-Length"));
}
/*(if not preserving Content-Length, do not preserve trailers, if any)*/
r->resp_decode_chunked = 0;
if (r->gw_dechunk) {
free(r->gw_dechunk->b.ptr);
free(r->gw_dechunk);
r->gw_dechunk = NULL;
}
}
chunkqueue_reset(r->write_queue);
}
@ -1034,7 +1041,13 @@ static int http_response_process_headers(request_st * const r, http_response_opt
if (*value == '+') ++value;
break;
case HTTP_HEADER_TRANSFER_ENCODING:
break;
/*(assumes "Transfer-Encoding: chunked"; does not verify)*/
r->resp_decode_chunked = 1;
r->gw_dechunk = calloc(1, sizeof(response_dechunk));
/* XXX: future: might consider using chunk_buffer_acquire()
* and chunk_buffer_release() for r->gw_dechunk->b */
force_assert(r->gw_dechunk);
continue;
default:
break;
}
@ -1197,9 +1210,8 @@ handler_t http_response_parse_headers(request_st * const r, http_response_opts *
}
if (blen > 0) {
if (0 != http_chunk_append_mem(r, bstart, blen)) {
if (0 != http_chunk_decode_append_mem(r, bstart, blen))
return HANDLER_ERROR;
}
}
/* (callback for response headers complete) */
@ -1310,7 +1322,7 @@ handler_t http_response_read(request_st * const r, http_response_opts * const op
/* accumulate response in b until headers completed (or error) */
if (r->resp_body_started) buffer_clear(b);
} else {
if (0 != http_chunk_append_buffer(r, b)) {
if (0 != http_chunk_decode_append_buffer(r, b)) {
/* error writing to tempfile;
* truncate response or send 500 if nothing sent yet */
return HANDLER_ERROR;

View File

@ -265,6 +265,212 @@ int http_chunk_transfer_cqlen(request_st * const r, chunkqueue * const src, cons
}
void http_chunk_close(request_st * const r) {
if (r->resp_send_chunked)
if (!r->resp_send_chunked) return;
if (r->gw_dechunk && !buffer_string_is_empty(&r->gw_dechunk->b)) {
/* XXX: trailers passed through; no sanity check currently done */
chunkqueue_append_buffer(r->write_queue, &r->gw_dechunk->b);
if (!r->gw_dechunk->done)
r->keep_alive = 0;
}
else
chunkqueue_append_mem(r->write_queue, CONST_STR_LEN("0\r\n\r\n"));
}
static int
http_chunk_decode_append_data (request_st * const r, const char *mem, off_t len)
{
/*(silently discard data, if any, after final \r\n)*/
if (r->gw_dechunk->done) return 0;
buffer * const h = &r->gw_dechunk->b;
off_t te_chunked = r->gw_dechunk->gw_chunked;
while (len) {
if (0 == te_chunked) {
const char *p = strchr(mem, '\n');
/*(likely better ways to handle chunked header crossing chunkqueue
* chunks, but this situation is not expected to occur frequently)*/
if (NULL == p) { /* incomplete HTTP chunked header line */
uint32_t hlen = buffer_string_length(h);
if ((off_t)(1024 - hlen) < len) {
log_error(r->conf.errh, __FILE__, __LINE__,
"chunked header line too long");
return -1;
}
buffer_append_string_len(h, mem, len);
break;
}
off_t hsz = ++p - mem;
unsigned char *s = (unsigned char *)mem;
if (!buffer_string_is_empty(h)) {
uint32_t hlen = buffer_string_length(h);
if (NULL == memchr(h->ptr, '\n', hlen)) {
if ((off_t)(1024 - hlen) < hsz) {
log_error(r->conf.errh, __FILE__, __LINE__,
"chunked header line too long");
return -1;
}
buffer_append_string_len(h, mem, hsz);
}
s = (unsigned char *)h->ptr;
}
for (unsigned char u; (u=(unsigned char)hex2int(*s))!=0xFF; ++s) {
if (te_chunked > (off_t)(1uLL<<(8*sizeof(off_t)-5))-1) {
log_error(r->conf.errh, __FILE__, __LINE__,
"chunked data size too large");
return -1;
}
te_chunked <<= 4;
te_chunked |= u;
}
if ((char *)s == mem || (char *)s == h->ptr) return -1; /*(no hex)*/
while (*s == ' ' || *s == '\t') ++s;
if (*s != '\r' && *s != ';') { /*(not strictly checking \r\n)*/
log_error(r->conf.errh, __FILE__, __LINE__,
"chunked header invalid chars");
return -1;
}
if (0 == te_chunked) {
/* do not consume final chunked header until
* (optional) trailers received along with
* request-ending blank line "\r\n" */
if (len - hsz == 2 && p[0] == '\r' && p[1] == '\n') {
/* common case with no trailers; final \r\n received */
/*(silently discard data, if any, after final \r\n)*/
#if 0 /*(avoid allocation for common case; users must check)*/
if (buffer_is_empty(h))
buffer_copy_string_len(h, CONST_STR_LEN("0\r\n\r\n"));
#else
buffer_clear(h);
#endif
r->gw_dechunk->done = r->http_status;
break;
}
/* accumulate trailers and check for end of trailers */
/* XXX: reuse r->conf.max_request_field_size
* or have separate limit? */
uint32_t hlen = buffer_string_length(h);
if ((off_t)(r->conf.max_request_field_size - hlen) < hsz) {
/* truncate excessively long trailers */
r->gw_dechunk->done = r->http_status;
hsz = (off_t)(r->conf.max_request_field_size - hlen);
buffer_append_string_len(h, mem, hsz);
p = strrchr(h->ptr, '\n');
if (NULL != p)
buffer_string_set_length(h, p + 1 - h->ptr);
else { /*(should not happen)*/
buffer_clear(h);
buffer_append_string_len(h, CONST_STR_LEN("0\r\n"));
}
buffer_append_string_len(h, CONST_STR_LEN("\r\n"));
break;
}
buffer_append_string_len(h, mem, hsz);
hlen += (uint32_t)hsz; /* uint32_t fits in (buffer *) */
if (hlen < 4) break;
p = h->ptr + hlen - 4;
if (p[0]=='\r'&&p[1]=='\n'&&p[2]=='\r'&&p[3]=='\n')
r->gw_dechunk->done = r->http_status;
else if ((p = strstr(h->ptr, "\r\n\r\n"))) {
r->gw_dechunk->done = r->http_status;
/*(silently discard data, if any, after final \r\n)*/
buffer_string_set_length(h, (uint32_t)(p+4-h->ptr));
}
break;
}
mem += hsz;
len -= hsz;
if (te_chunked > (off_t)(1uLL<<(8*sizeof(off_t)-5))-1-2) {
log_error(r->conf.errh, __FILE__, __LINE__,
"chunked data size too large");
return -1;
}
te_chunked += 2; /*(for trailing "\r\n" after chunked data)*/
}
if (te_chunked >= 2) {
off_t clen = te_chunked - 2;
if (clen > len) clen = len;
if (0 != http_chunk_append_mem(r, mem, clen))
return -1;
mem += clen;
len -= clen;
te_chunked -= clen;
if (te_chunked == 2) {
if (len >= 2) {
if (mem[0] != '\r' || mem[1] != '\n') return -1;
mem += 2;
len -= 2;
te_chunked = 0;
}
else if (len == 1 && mem[0] != '\r') return -1;
}
}
else if (1 == te_chunked) {
/* finish reading chunk block "\r\n" */
if (mem[0] != '\n') return -1;
++mem;
--len;
te_chunked = 0;
}
}
r->gw_dechunk->gw_chunked = te_chunked;
return 0;
}
int http_chunk_decode_append_buffer(request_st * const r, buffer * const mem)
{
/*(called by funcs receiving data from backends, which might be chunked)*/
/*(separate from http_chunk_append_buffer() called by numerous others)*/
if (!r->resp_decode_chunked)
return http_chunk_append_buffer(r, mem);
/* no need to decode chunked to immediately re-encode chunked,
* though would be more robust to still validate chunk lengths sent
* (or else we might wait for keep-alive while client waits for final chunk)
* Before finishing response/stream, we *are not* checking if we got final
* chunk of chunked encoding from backend. If we were, we could consider
* closing HTTP/1.0 and HTTP/1.1 connections (no keep-alive), and in HTTP/2
* we could consider sending RST_STREAM error. http_chunk_close() would
* only handle case of streaming chunked to client */
if (r->resp_send_chunked) {
r->resp_send_chunked = 0;
int rc = http_chunk_append_buffer(r, mem); /* might append to tmpfile */
r->resp_send_chunked = 1;
return rc;
}
/* might avoid copy by transferring buffer if buffer is all data that is
* part of large chunked block, but choosing to *not* expand that out here*/
return http_chunk_decode_append_data(r, CONST_BUF_LEN(mem));
}
int http_chunk_decode_append_mem(request_st * const r, const char * const mem, const size_t len)
{
/*(called by funcs receiving data from backends, which might be chunked)*/
/*(separate from http_chunk_append_mem() called by numerous others)*/
if (!r->resp_decode_chunked)
return http_chunk_append_mem(r, mem, len);
/* no need to decode chunked to immediately re-encode chunked,
* though would be more robust to still validate chunk lengths sent
* (or else we might wait for keep-alive while client waits for final chunk)
* Before finishing response/stream, we *are not* checking if we got final
* chunk of chunked encoding from backend. If we were, we could consider
* closing HTTP/1.0 and HTTP/1.1 connections (no keep-alive), and in HTTP/2
* we could consider sending RST_STREAM error. http_chunk_close() would
* only handle case of streaming chunked to client */
if (r->resp_send_chunked) {
r->resp_send_chunked = 0;
int rc = http_chunk_append_mem(r, mem, len); /*might append to tmpfile*/
r->resp_send_chunked = 1;
return rc;
}
return http_chunk_decode_append_data(r, mem, (off_t)len);
}

View File

@ -8,6 +8,8 @@
int http_chunk_append_mem(request_st *r, const char * mem, size_t len); /* copies memory */
int http_chunk_append_buffer(request_st *r, buffer *mem); /* may reset "mem" */
int http_chunk_decode_append_mem(request_st * const r, const char * const mem, const size_t len);
int http_chunk_decode_append_buffer(request_st * const r, buffer * const mem); /* may reset "mem" */
int http_chunk_transfer_cqlen(request_st *r, chunkqueue *src, size_t len);
int http_chunk_append_file(request_st *r, const buffer *fn); /* copies "fn" */
int http_chunk_append_file_fd(request_st *r, const buffer *fn, int fd, off_t sz);

View File

@ -95,6 +95,12 @@ typedef struct {
buffer etag;
} physical;
typedef struct {
off_t gw_chunked;
buffer b;
int done;
} response_dechunk;
/* the order of the items should be the same as they are processed
* read before write as we use this later e.g. <= CON_STATE_REQUEST_END */
typedef enum {
@ -159,6 +165,7 @@ struct request_st {
char resp_body_finished;
char resp_body_started;
char resp_send_chunked;
char resp_decode_chunked;
char loops_per_request; /* catch endless loops in a single request */
char keep_alive; /* only request.c can enable it, all other just disable */
@ -167,6 +174,7 @@ struct request_st {
struct chunkqueue *write_queue; /* HTTP response queue [ file, mem ] */
struct chunkqueue *read_queue; /* HTTP request queue [ mem ] */
buffer *tmp_buf; /* shared; same as srv->tmp_buf */
response_dechunk *gw_dechunk;
struct timespec start_hp;
time_t start_ts;