lighttpd
/
lighttpd2
2
0
Fork 0
Code Issues Wiki

[core] support chunked uploads

personal/stbuehler/wip
Stefan Bühler 2013-05-25 16:20:25 +02:00
parent 4706cc5f60
commit 3deb7c9e79
9 changed files with 92 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/lighttpd/base.h
View File

@ -20,6 +20,7 @@
#include <lighttpd/waitqueue.h>
#include <lighttpd/stream.h>
#include <lighttpd/filter.h>
#include <lighttpd/filter_chunked.h>
#include <lighttpd/radix.h>
#include <lighttpd/base_lua.h>
@ -45,7 +46,6 @@
#include <lighttpd/connection.h>
#include <lighttpd/filter_chunked.h>
#include <lighttpd/collect.h>
#include <lighttpd/network.h>
#include <lighttpd/etag.h>

1
include/lighttpd/connection.h
View File

@ -59,6 +59,7 @@ struct liConnection {
gboolean response_headers_sent, expect_100_cont, out_has_all_data;
liStream in, out;
liFilterChunkedDecodeState in_chunked_decode_state;
liVRequest *mainvr;
liHttpRequestCtx req_parser_ctx;

8
include/lighttpd/filter_chunked.h
View File

@ -1,15 +1,17 @@
#ifndef _LIGHTTPD_FILTER_CHUNKED_H_
#define _LIGHTTPD_FILTER_CHUNKED_H_
#include <lighttpd/base.h>
#ifndef _LIGHTTPD_BASE_H_
#error Please include <lighttpd/base.h> instead of this file
#endif
/* initialize with zero */
typedef struct {
int parse_state;
goffset cur_chunklen;
} liFilterDecodeState;
} liFilterChunkedDecodeState;
LI_API liHandlerResult li_filter_chunked_encode(liVRequest *vr, liChunkQueue *out, liChunkQueue *in);
LI_API liHandlerResult li_filter_chunked_decode(liVRequest *vr, liChunkQueue *out, liChunkQueue *in, liFilterDecodeState *state);
LI_API gboolean li_filter_chunked_decode(liVRequest *vr, liChunkQueue *out, liChunkQueue *in, liFilterChunkedDecodeState *state);
#endif

2
include/lighttpd/request.h
View File

@ -32,7 +32,7 @@ struct liRequest {
liHttpHeaders *headers;
/* Parsed headers: */
goffset content_length; /* -1 if not specified */
goffset content_length; /* -1 if not specified; implies chunked transfer-encoding */
};
LI_API void li_request_init(liRequest *req);

18
src/main/connection.c
View File

@ -334,11 +334,17 @@ static void _connection_http_in_cb(liStream *stream, liStreamEvent event) {
if (con->state != LI_CON_STATE_READ_REQUEST_HEADER && !in->is_closed) {
goffset newbytes = 0;
if (vr->request.content_length == -1) {
/* TODO: parse chunked encoded request body, filters */
/* li_chunkqueue_steal_all(con->in, con->raw_in); */
con->info.keep_alive = FALSE;
in->is_closed = TRUE;
if (-1 == vr->request.content_length) {
if (!in->is_closed) {
if (!li_filter_chunked_decode(vr, in, raw_in, &con->in_chunked_decode_state)) {
if (CORE_OPTION(LI_CORE_OPTION_DEBUG_REQUEST_HANDLING).boolean) {
VR_DEBUG(vr, "%s", "failed decoding chunked request body");
}
li_connection_error(con);
return;
}
newbytes = 1; /* always notify */
}
} else {
if (in->bytes_in < vr->request.content_length) {
newbytes = li_chunkqueue_steal_len(in, raw_in, vr->request.content_length - in->bytes_in);
@ -741,6 +747,8 @@ static void li_connection_reset_keep_alive(liConnection *con) {
li_stream_disconnect_dest(&con->in);
con->out.out->is_closed = FALSE;
memset(&con->in_chunked_decode_state, 0, sizeof(con->in_chunked_decode_state));
/* restore chunkqueue limits */
li_chunkqueue_use_limit(con->con_sock.raw_in->out, 512*1024);
li_chunkqueue_use_limit(con->con_sock.raw_out->out, 512*1024);

31
src/main/filter_chunked.c
View File

@ -1,5 +1,5 @@
#include <lighttpd/filter_chunked.h>
#include <lighttpd/base.h>
/* len != 0 */
static void http_chunk_append_len(liChunkQueue *cq, size_t len) {
@ -46,13 +46,13 @@ liHandlerResult li_filter_chunked_encode(liVRequest *vr, liChunkQueue *out, liCh
#define read_char(c) do { \
while (!p || p >= pe) { \
GError *err = NULL; \
res = li_chunk_parser_next(&ctx, &p, &pe, &err); \
liHandlerResult res = li_chunk_parser_next(&ctx, &p, &pe, &err); \
if (NULL != err) { \
VR_ERROR(vr, "%s", err->message); \
g_error_free(err); \
} \
if (res == LI_HANDLER_WAIT_FOR_EVENT && in->is_closed) { \
res = LI_HANDLER_ERROR; \
if (LI_HANDLER_ERROR == res || (LI_HANDLER_WAIT_FOR_EVENT == res && in->is_closed)) { \
goto error; \
} \
if (res != LI_HANDLER_GO_ON) goto leave; \
} \
@ -60,8 +60,7 @@ liHandlerResult li_filter_chunked_encode(liVRequest *vr, liChunkQueue *out, liCh
} while(0);
liHandlerResult li_filter_chunked_decode(liVRequest *vr, liChunkQueue *out, liChunkQueue *in, liFilterDecodeState *state) {
liHandlerResult res = LI_HANDLER_GO_ON;
gboolean li_filter_chunked_decode(liVRequest *vr, liChunkQueue *out, liChunkQueue *in, liFilterChunkedDecodeState *state) {
liChunkParserCtx ctx;
gchar *p = NULL, *pe = NULL;
gchar c;
@ -142,6 +141,9 @@ liHandlerResult li_filter_chunked_decode(liVRequest *vr, liChunkQueue *out, liCh
} else {
state->parse_state = 20;
}
} else {
/* wait for more data for the current chunk */
return TRUE;
}
break;
case 4:
@ -177,20 +179,19 @@ liHandlerResult li_filter_chunked_decode(liVRequest *vr, liChunkQueue *out, liCh
break;
case 14:
out->is_closed = TRUE;
res = LI_HANDLER_GO_ON;
goto leave;
case 20:
res = LI_HANDLER_ERROR;
goto leave;
goto error;
}
}
leave:
if (res == LI_HANDLER_ERROR) {
out->is_closed = TRUE;
li_chunkqueue_skip_all(in);
state->parse_state = 20;
}
li_chunkqueue_skip(in, ctx.bytes_in);
return res;
return TRUE;
error:
out->is_closed = TRUE;
li_chunkqueue_skip_all(in);
state->parse_state = 20;
return FALSE;
}

33
src/main/request.c
View File

@ -119,6 +119,7 @@ gboolean li_request_validate_header(liConnection *con) {
liRequest *req = &con->mainvr->request;
liHttpHeader *hh;
GList *l;
gboolean transfer_encoding_chunked = FALSE;
if (con->info.is_ssl) {
g_string_append_len(req->uri.scheme, CONST_STR_LEN("https"));
@ -221,6 +222,30 @@ gboolean li_request_validate_header(liConnection *con) {
con->mainvr->request.content_length = r;
}
/* Transfer-Encoding: chunked */
l = li_http_header_find_first(req->headers, CONST_STR_LEN("transfer-encoding"));
if (l) {
for ( ; l ; l = li_http_header_find_next(l, CONST_STR_LEN("transfer-encoding")) ) {
hh = (liHttpHeader*) l->data;
if (0 == g_ascii_strcasecmp( LI_HEADER_VALUE(hh), "identity" )) {
/* ignore */
continue;
} if (0 == g_ascii_strcasecmp( LI_HEADER_VALUE(hh), "chunked" )) {
if (transfer_encoding_chunked) {
/* The "chunked" transfer-coding MUST NOT be applied more than once to a message-body */
bad_request(con, 400); /* bad request */
return FALSE;
}
transfer_encoding_chunked = TRUE;
con->mainvr->request.content_length = -1;
} else {
/* we only support chunked transfer-encoding */
bad_request(con, 501); /* Unimplemented */
return FALSE;
}
}
}
/* Expect: 100-continue */
l = li_http_header_find_first(req->headers, CONST_STR_LEN("expect"));
if (l) {
@ -264,8 +289,8 @@ gboolean li_request_validate_header(liConnection *con) {
con->mainvr->request.content_length = 0;
break;
case LI_HTTP_METHOD_POST:
/* content-length is required for them */
if (con->mainvr->request.content_length == -1) {
/* content-length or chunked encoding is required for them */
if (con->mainvr->request.content_length == -1 && !transfer_encoding_chunked) {
/* content-length is missing */
VR_ERROR(con->mainvr, "%s", "POST-request, but content-length missing -> 411");
@ -274,9 +299,9 @@ gboolean li_request_validate_header(liConnection *con) {
}
break;
default:
if (con->mainvr->request.content_length == -1)
/* they may have a content-length or use chunked encoding */
if (con->mainvr->request.content_length == -1 && !transfer_encoding_chunked)
con->mainvr->request.content_length = 0;
/* the may have a content-length */
break;
}

4
src/unittests/test-chunk.c
View File

@ -26,7 +26,7 @@ static void cq_assert_eq(liChunkQueue *cq, const gchar *s, size_t len) {
static void test_filter_chunked_decode(void) {
liChunkQueue *cq = li_chunkqueue_new(), *cq2 = li_chunkqueue_new();
liFilterDecodeState decode_state;
liFilterChunkedDecodeState decode_state;
cq_load_str(cq, CONST_STR_LEN(
"14\r\n"
@ -37,7 +37,7 @@ static void test_filter_chunked_decode(void) {
cq->is_closed = TRUE;
memset(&decode_state, 0, sizeof(decode_state));
li_chunkqueue_reset(cq2);
g_assert(LI_HANDLER_GO_ON == li_filter_chunked_decode(NULL, cq2, cq, &decode_state));
g_assert(li_filter_chunked_decode(NULL, cq2, cq, &decode_state));
cq_assert_eq(cq2, CONST_STR_LEN(
"01234567890123456789"
));

24
tests/t-cgi.py
View File

@ -57,12 +57,36 @@ class TestUploadLarge1(CurlRequest):
EXPECT_RESPONSE_BODY = BODY_SHA1
EXPECT_RESPONSE_CODE = 200
class ChunkedBodyReader:
def __init__(self, body, chunksize = 32*1024):
self.body = body
self.chunksize = chunksize
self.pos = 0
def read(self, size):
current = self.pos
rem = len(self.body) - current
size = min(rem, self.chunksize, size)
self.pos += size
return self.body[current:current+size]
class TestUploadLargeChunked1(CurlRequest):
URL = "/uploadcheck.cgi"
EXPECT_RESPONSE_BODY = BODY_SHA1
EXPECT_RESPONSE_CODE = 200
REQUEST_HEADERS = ["Transfer-Encoding: chunked"]
def PrepareRequest(self, reqheaders):
c = self.curl
c.setopt(c.UPLOAD, 1)
c.setopt(pycurl.READFUNCTION, ChunkedBodyReader(BODY).read)
class Test(GroupTest):
group = [
TestPathInfo1,
TestRequestUri1,
TestUploadLarge1,
TestUploadLargeChunked1
]
config = """