lighttpd
/
lighttpd1.4
2
0
Fork 0
Code Issues Releases Wiki Activity

removed con->request.content (mem-buffer) by a chunk-queue

Browse Source 
which can buffer to tempfiles of the content is to large


git-svn-id: svn://svn.lighttpd.net/lighttpd/branches/lighttpd-merge-1.4.x@741 152afb58-edef-0310-8abb-c4023f1b3aa9
svn/tags/lighttpd-1.4.6
Jan Kneschke 18 years ago
parent 5bd52eea71
commit 1c09f28447
9 changed files with 596 additions and 188 deletions
Show Stats Download Patch File Download Diff File

2
src/Makefile.am
Unescape Escape View File

@ -78,7 +78,7 @@ lib_LTLIBRARIES += mod_cml.la
mod_cml_la_SOURCES = mod_cml.c mod_cml_lua.c mod_cml_funcs.c
mod_cml_la_CFLAGS = $(AM_CFLAGS) $(LUA_CFLAGS)
mod_cml_la_LDFLAGS = -module -export-dynamic -avoid-version -no-undefined
mod_cml_la_LIBADD = $(MEMCACHE_LIB) $(common_libadd) $(LUA_LIBS)
mod_cml_la_LIBADD = $(MEMCACHE_LIB) $(common_libadd) $(LUA_LIBS) -lm
lib_LTLIBRARIES += mod_trigger_b4_dl.la
mod_trigger_b4_dl_la_SOURCES = mod_trigger_b4_dl.c

6
src/base.h
Unescape Escape View File

@ -157,7 +157,6 @@ typedef struct {
array *headers;
/* CONTENT */
buffer *content;
size_t content_length; /* returned by strtoul() */
/* internal representation */
@ -330,8 +329,9 @@ typedef struct {
int file_started;
int file_finished;
chunkqueue *write_queue;
chunkqueue *read_queue;
chunkqueue *write_queue; /* a large queue for low-level write ( HTTP response ) [ file, mem ] */
chunkqueue *read_queue; /* a small queue for low-level read ( HTTP request ) [ mem ] */
chunkqueue *request_content_queue; /* takes request-content into tempfile if necessary [ tempfile, mem ]*/
int traffic_limit_reached;

117
src/connections.c
Unescape Escape View File

@ -193,10 +193,11 @@ static void dump_packet(const unsigned char *data, size_t len) {
static int connection_handle_read(server *srv, connection *con) {
int len;
buffer *b;
int toread;
#ifdef USE_OPENSSL
server_socket *srv_sock = con->srv_socket;
#endif
b = chunkqueue_get_append_buffer(con->read_queue);
buffer_prepare_copy(b, 4096);
@ -204,11 +205,27 @@ static int connection_handle_read(server *srv, connection *con) {
if (srv_sock->is_ssl) {
len = SSL_read(con->ssl, b->ptr, b->size - 1);
} else {
if (ioctl(con->fd, FIONREAD, &toread)) {
log_error_write(srv, __FILE__, __LINE__, "sd",
"unexpected end-of-file:",
con->fd);
return -1;
}
buffer_prepare_copy(b, toread);
len = read(con->fd, b->ptr, b->size - 1);
}
#elif defined(__WIN32)
len = recv(con->fd, b->ptr, b->size - 1, 0);
#else
if (ioctl(con->fd, FIONREAD, &toread)) {
log_error_write(srv, __FILE__, __LINE__, "sd",
"unexpected end-of-file:",
con->fd);
return -1;
}
buffer_prepare_copy(b, toread);
len = read(con->fd, b->ptr, b->size - 1);
#endif
@ -538,7 +555,6 @@ connection *connection_init(server *srv) {
CLEAN(request.request_line);
CLEAN(request.request);
CLEAN(request.pathinfo);
CLEAN(request.content);
CLEAN(request.orig_uri);
@ -563,6 +579,7 @@ connection *connection_init(server *srv) {
#undef CLEAN
con->write_queue = chunkqueue_init();
con->read_queue = chunkqueue_init();
con->request_content_queue = chunkqueue_init();
con->request.headers = array_init();
con->response.headers = array_init();
con->environment = array_init();
@ -588,6 +605,7 @@ void connections_free(server *srv) {
chunkqueue_free(con->write_queue);
chunkqueue_free(con->read_queue);
chunkqueue_free(con->request_content_queue);
array_free(con->request.headers);
array_free(con->response.headers);
array_free(con->environment);
@ -599,7 +617,6 @@ void connections_free(server *srv) {
CLEAN(request.request_line);
CLEAN(request.request);
CLEAN(request.pathinfo);
CLEAN(request.content);
CLEAN(request.orig_uri);
@ -669,7 +686,6 @@ int connection_reset(server *srv, connection *con) {
CLEAN(request.uri);
CLEAN(request.request_line);
CLEAN(request.pathinfo);
CLEAN(request.content);
CLEAN(request.request);
CLEAN(request.orig_uri);
@ -712,6 +728,7 @@ int connection_reset(server *srv, connection *con) {
array_reset(con->environment);
chunkqueue_reset(con->write_queue);
chunkqueue_reset(con->request_content_queue);
/* the plugins should cleanup themself */
for (i = 0; i < srv->plugins.used; i++) {
@ -791,6 +808,8 @@ int connection_handle_read_state(server *srv, connection *con) {
char *h_term = NULL;
chunk *c;
chunkqueue *cq = con->read_queue;
chunkqueue *dst_cq = con->request_content_queue;
size_t memusage;
if (con->is_readable) {
con->read_idle_ts = srv->cur_ts;
@ -905,66 +924,86 @@ int connection_handle_read_state(server *srv, connection *con) {
c->offset = c->mem->used - 1;
}
}
if (c->offset + 1 == c->mem->used) {
/* chunk is empty, move it to unused */
cq->first = c->next;
c->next = cq->unused;
cq->unused = c;
if (cq->first == NULL) cq->last = NULL;
assert(c != c->next);
}
/* con->request.request is setup up */
if (h_term) {
connection_set_state(srv, con, CON_STATE_REQUEST_END);
} else if (chunkqueue_length(cq) > 64 * 1024) {
} else if (con->request.request->used > 64 * 1024) {
log_error_write(srv, __FILE__, __LINE__, "sd", "http-header larger then 64k -> disconnected", chunkqueue_length(cq));
connection_set_state(srv, con, CON_STATE_ERROR);
}
break;
case CON_STATE_READ_POST:
for (c = cq->first; c && (con->request.content->used != con->request.content_length + 1); c = cq->first) {
for (c = cq->first; c && (dst_cq->bytes_in != con->request.content_length); c = c->next) {
off_t weWant, weHave, toRead;
int buffer_to_file = 0;
weWant = con->request.content_length - (con->request.content->used ? con->request.content->used - 1 : 0);
/* without the terminating \0 */
weWant = con->request.content_length - dst_cq->bytes_in;
assert(c->mem->used);
weHave = c->mem->used - c->offset - 1;
toRead = weHave > weWant ? weWant : weHave;
buffer_append_string_len(con->request.content, c->mem->ptr + c->offset, toRead);
c->offset += toRead;
if (c->offset + 1 >= c->mem->used) {
/* chunk is empty, move it to unused */
cq->first = c->next;
c->next = cq->unused;
cq->unused = c;
if (cq->first == NULL) cq->last = NULL;
assert(c != c->next);
/* the new way, copy everything into a chunkqueue whcih might use tempfiles */
if (con->request.content_length > 64 * 1024) {
chunk *dst_c = NULL;
/* copy everything to max 1Mb sized tempfiles */
/*
* if the last chunk is
* - smaller than 1Mb (size < 1Mb)
* - not read yet (offset == 0)
* -> append to it
* otherwise
* -> create a new chunk
*
* */
if (dst_cq->last &&
dst_cq->last->type == FILE_CHUNK &&
dst_cq->last->file.is_temp &&
dst_cq->last->offset == 0 &&
dst_cq->last->file.length < 1 * 1024 * 1024) {
/* ok, take the last chunk for our job */
dst_c = dst_cq->last;
dst_c->file.fd = open(dst_c->file.name->ptr, O_WRONLY | O_APPEND);
} else {
dst_c = chunkqueue_get_append_tempfile(dst_cq);
}
/* we have a chunk, let's write to it */
assert(dst_c->file.fd != -1);
assert(toRead == write(dst_c->file.fd, c->mem->ptr + c->offset, toRead));
dst_c->file.length += toRead;
close(dst_c->file.fd);
dst_c->file.fd = -1;
} else {
assert(toRead);
buffer *b;
b = chunkqueue_get_append_buffer(dst_cq);
buffer_copy_string_len(b, c->mem->ptr + c->offset, toRead);
}
c->offset += toRead;
dst_cq->bytes_in += toRead;
}
/* Content is ready */
if (con->request.content->used == con->request.content_length + 1) {
if (dst_cq->bytes_in == con->request.content_length) {
connection_set_state(srv, con, CON_STATE_HANDLE_REQUEST);
}
break;
}
chunkqueue_remove_finished_chunks(cq);
return 0;
}

76
src/mod_cgi.c
Unescape Escape View File

@ -4,6 +4,8 @@
#else
#include <sys/socket.h>
#include <sys/wait.h>
#include <sys/mman.h>
#include <sys/fcntl.h>
#include <netinet/in.h>
@ -956,8 +958,78 @@ static int cgi_create_env(server *srv, connection *con, plugin_data *p, buffer *
handler_ctx *hctx;
/* father */
if (con->request.content->used) {
write(to_cgi_fds[1], con->request.content->ptr, con->request.content_length);
if (con->request.content_length) {
chunkqueue *cq = con->request_content_queue;
chunk *c;
assert(chunkqueue_length(cq) == con->request.content_length);
/* there is content to send */
for (c = cq->first; c; c = cq->first) {
int r = 0;
/* copy all chunks */
switch(c->type) {
case FILE_CHUNK:
if (c->file.mmap.start == MAP_FAILED) {
if (-1 == c->file.fd && /* open the file if not already open */
-1 == (c->file.fd = open(c->file.name->ptr, O_RDONLY))) {
log_error_write(srv, __FILE__, __LINE__, "ss", "open failed: ", strerror(errno));
return -1;
}
c->file.mmap.length = c->file.length;
if (MAP_FAILED == (c->file.mmap.start = mmap(0, c->file.mmap.length, PROT_READ, MAP_SHARED, c->file.fd, 0))) {
log_error_write(srv, __FILE__, __LINE__, "ssbd", "mmap failed: ",
strerror(errno), c->file.name, c->file.fd);
return -1;
}
close(c->file.fd);
c->file.fd = -1;
/* chunk_reset() or chunk_free() will cleanup for us */
}
if ((r = write(to_cgi_fds[1], c->file.mmap.start + c->offset, c->file.length - c->offset)) < 0) {
switch(errno) {
case ENOSPC:
con->http_status = 507;
break;
default:
con->http_status = 403;
break;
}
}
break;
case MEM_CHUNK:
if ((r = write(to_cgi_fds[1], c->mem->ptr + c->offset, c->mem->used - c->offset - 1)) < 0) {
switch(errno) {
case ENOSPC:
con->http_status = 507;
break;
default:
con->http_status = 403;
break;
}
}
break;
}
if (r > 0) {
c->offset += r;
cq->bytes_out += r;
} else {
break;
}
chunkqueue_remove_finished_chunks(cq);
}
}
close(from_cgi_fds[1]);

232
src/mod_fastcgi.c
Unescape Escape View File

@ -23,6 +23,7 @@
#include "inet_ntop_cache.h"
#include "stat_cache.h"
#include "network_backends.h"
#include <fastcgi.h>
#include <stdio.h>
@ -318,10 +319,8 @@ typedef struct {
int reconnects; /* number of reconnect attempts */
buffer *write_buffer;
size_t write_offset;
chunkqueue *rb;
chunkqueue *rb; /* read queue */
chunkqueue *wb; /* write queue */
buffer *response_header;
@ -359,7 +358,6 @@ static handler_ctx * handler_ctx_init() {
hctx->fde_ndx = -1;
hctx->response_header = buffer_init();
hctx->write_buffer = buffer_init();
hctx->request_id = 0;
hctx->state = FCGI_STATE_INIT;
@ -371,15 +369,16 @@ static handler_ctx * handler_ctx_init() {
hctx->send_content_body = 1;
hctx->rb = chunkqueue_init();
hctx->wb = chunkqueue_init();
return hctx;
}
static void handler_ctx_free(handler_ctx *hctx) {
buffer_free(hctx->response_header);
buffer_free(hctx->write_buffer);
chunkqueue_free(hctx->rb);
chunkqueue_free(hctx->wb);
free(hctx);
}
@ -1618,9 +1617,9 @@ static int fcgi_env_add_request_headers(server *srv, connection *con, plugin_dat
static int fcgi_create_env(server *srv, handler_ctx *hctx, size_t request_id) {
FCGI_BeginRequestRecord beginRecord;
FCGI_Header header;
buffer *b;
char buf[32];
size_t offset;
const char *s;
#ifdef HAVE_IPV6
char b2[INET6_ADDRSTRLEN + 1];
@ -1642,8 +1641,10 @@ static int fcgi_create_env(server *srv, handler_ctx *hctx, size_t request_id) {
beginRecord.body.roleB1 = 0;
beginRecord.body.flags = 0;
memset(beginRecord.body.reserved, 0, sizeof(beginRecord.body.reserved));
b = chunkqueue_get_append_buffer(hctx->wb);
buffer_copy_memory(hctx->write_buffer, (const char *)&beginRecord, sizeof(beginRecord));
buffer_copy_memory(b, (const char *)&beginRecord, sizeof(beginRecord));
/* send FCGI_PARAMS */
buffer_prepare_copy(p->fcgi_env, 1024);
@ -1800,29 +1801,139 @@ static int fcgi_create_env(server *srv, handler_ctx *hctx, size_t request_id) {
fcgi_env_add_request_headers(srv, con, p);
fcgi_header(&(header), FCGI_PARAMS, request_id, p->fcgi_env->used, 0);
buffer_append_memory(hctx->write_buffer, (const char *)&header, sizeof(header));
buffer_append_memory(hctx->write_buffer, (const char *)p->fcgi_env->ptr, p->fcgi_env->used);
buffer_append_memory(b, (const char *)&header, sizeof(header));
buffer_append_memory(b, (const char *)p->fcgi_env->ptr, p->fcgi_env->used);
fcgi_header(&(header), FCGI_PARAMS, request_id, 0, 0);
buffer_append_memory(hctx->write_buffer, (const char *)&header, sizeof(header));
/* send FCGI_STDIN */
/* something to send ? */
for (offset = 0; offset != con->request.content_length; ) {
/* send chunks of 1024 bytes */
size_t toWrite = con->request.content_length - offset > 4096 ? 4096 : con->request.content_length - offset;
fcgi_header(&(header), FCGI_STDIN, request_id, toWrite, 0);
buffer_append_memory(hctx->write_buffer, (const char *)&header, sizeof(header));
buffer_append_memory(hctx->write_buffer, (const char *)(con->request.content->ptr + offset), toWrite);
offset += toWrite;
buffer_append_memory(b, (const char *)&header, sizeof(header));
b->used++; /* add virtual \0 */
hctx->wb->bytes_in += b->used - 1;
if (con->request.content_length) {
chunkqueue *req_cq = con->request_content_queue;
chunk *req_c;
size_t offset;
/* something to send ? */
for (offset = 0, req_c = req_cq->first; offset != req_cq->bytes_in; ) {
size_t weWant = req_cq->bytes_in - offset > FCGI_MAX_LENGTH ? FCGI_MAX_LENGTH : req_cq->bytes_in - offset;
size_t written = 0;
size_t weHave = 0;
/* we announce toWrite octects
* now take all the request_content chunk that we need to fill this request
* */
b = chunkqueue_get_append_buffer(hctx->wb);
fcgi_header(&(header), FCGI_STDIN, request_id, weWant, 0);
buffer_copy_memory(b, (const char *)&header, sizeof(header));
hctx->wb->bytes_in += sizeof(header);
if (p->conf.debug > 10) {
fprintf(stderr, "%s.%d: tosend: %d / %Ld\n", __FILE__, __LINE__, offset, req_cq->bytes_in);
}
for (written = 0; written != weWant; ) {
if (p->conf.debug > 10) {
fprintf(stderr, "%s.%d: chunk: %d / %d\n", __FILE__, __LINE__, written, weWant);
}
switch (req_c->type) {
case FILE_CHUNK:
weHave = req_c->file.length - req_c->offset;
if (weHave > weWant - written) weHave = weWant - written;
if (p->conf.debug > 10) {
fprintf(stderr, "%s.%d: sending %d bytes from (%Ld / %Ld) %s\n",
__FILE__, __LINE__,
weHave,
req_c->offset,
req_c->file.length,
req_c->file.name->ptr);
}
assert(weHave != 0);
chunkqueue_append_file(hctx->wb, req_c->file.name, req_c->offset, weHave);
req_c->offset += weHave;
req_cq->bytes_out += weHave;
written += weHave;
hctx->wb->bytes_in += weHave;
/* steal the tempfile
*
* This is tricky:
* - we reference the tempfile from the request-content-queue several times
* if the req_c is larger than FCGI_MAX_LENGTH
* - we can't simply cleanup the request-content-queue as soon as possible
* as it would remove the tempfiles
* - the idea is to 'steal' the tempfiles and attach the is_temp flag to the last
* referencing chunk of the fastcgi-write-queue
*
* */
if (req_c->offset == req_c->file.length) {
chunk *c;
if (p->conf.debug > 10) {
fprintf(stderr, "%s.%d: next chunk\n", __FILE__, __LINE__);
}
c = hctx->wb->last;
assert(c->type == FILE_CHUNK);
assert(req_c->file.is_temp == 1);
c->file.is_temp = 1;
req_c->file.is_temp = 0;
chunkqueue_remove_finished_chunks(req_cq);
req_c = req_cq->first;
}
break;
case MEM_CHUNK:
/* append to the buffer */
weHave = req_c->mem->used - 1 - req_c->offset;
if (weHave > weWant - written) weHave = weWant - written;
buffer_append_memory(b, req_c->mem->ptr + req_c->offset, weHave);
req_c->offset += weHave;
req_cq->bytes_out += weHave;
written += weHave;
hctx->wb->bytes_in += weHave;
if (req_c->offset == req_c->mem->used - 1) {
chunkqueue_remove_finished_chunks(req_cq);
req_c = req_cq->first;
}
break;
default:
break;
}
}
b->used++; /* add virtual \0 */
offset += weWant;
}
}
b = chunkqueue_get_append_buffer(hctx->wb);
/* terminate STDIN */
fcgi_header(&(header), FCGI_STDIN, request_id, 0, 0);
buffer_append_memory(hctx->write_buffer, (const char *)&header, sizeof(header));
buffer_copy_memory(b, (const char *)&header, sizeof(header));
b->used++; /* add virtual \0 */
hctx->wb->bytes_in += sizeof(header);
#if 0
for (i = 0; i < hctx->write_buffer->used; i++) {
@ -2449,7 +2560,7 @@ static handler_t fcgi_write_request(server *srv, handler_ctx *hctx) {
fcgi_extension_host *host= hctx->host;
connection *con = hctx->remote_conn;
int r;
int ret;
/* sanity check */
if (!host ||
@ -2466,9 +2577,9 @@ static handler_t fcgi_write_request(server *srv, handler_ctx *hctx) {
switch(hctx->state) {
case FCGI_STATE_INIT:
r = host->unixsocket->used ? AF_UNIX : AF_INET;
ret = host->unixsocket->used ? AF_UNIX : AF_INET;
if (-1 == (hctx->fd = socket(r, SOCK_STREAM, 0))) {
if (-1 == (hctx->fd = socket(ret, SOCK_STREAM, 0))) {
if (errno == EMFILE ||
errno == EINTR) {
log_error_write(srv, __FILE__, __LINE__, "sd",
@ -2588,25 +2699,34 @@ static handler_t fcgi_write_request(server *srv, handler_ctx *hctx) {
fcgi_create_env(srv, hctx, hctx->request_id);
fcgi_set_state(srv, hctx, FCGI_STATE_WRITE);
hctx->write_offset = 0;
/* fall through */
case FCGI_STATE_WRITE:
/* why aren't we using the network_ interface here ? */
r = write(hctx->fd,
hctx->write_buffer->ptr + hctx->write_offset,
hctx->write_buffer->used - hctx->write_offset);
#if defined USE_LINUX_SENDFILE
ret = network_write_chunkqueue_linuxsendfile(srv, con, hctx->fd, hctx->wb);
#elif defined USE_FREEBSD_SENDFILE
ret = network_write_chunkqueue_freebsdsendfile(srv, con, hctx->fd, hctx->wb);
#elif defined USE_SOLARIS_SENDFILEV
ret = network_write_chunkqueue_solarissendfilev(srv, con, hctx->fd, hctx->wb);
#elif defined USE_WRITEV
ret = network_write_chunkqueue_writev(srv, con, hctx->fd, hctx->wb);
#else
ret = network_write_chunkqueue_write(srv, con, hctx->fd, hctx->wb);
#endif
chunkqueue_remove_finished_chunks(hctx->wb);
if (-1 == r) {
if (errno == ENOTCONN) {
if (ret < 0) {
switch(errno) {
case ENOTCONN:
/* the connection got dropped after accept()
*
* this is most of the time a PHP which dies
* after PHP_FCGI_MAX_REQUESTS
*
*/
if (hctx->write_offset == 0 &&
if (hctx->wb->bytes_out == 0 &&
hctx->reconnects < 5) {
usleep(10000); /* take away the load of the webserver
* to let the php a chance to restart
@ -2625,35 +2745,34 @@ static handler_t fcgi_write_request(server *srv, handler_ctx *hctx) {
log_error_write(srv, __FILE__, __LINE__, "ssdsd",
"[REPORT ME] connection was dropped after accept(). reconnect() denied:",
"write-offset:", hctx->write_offset,
"write-offset:", hctx->wb->bytes_out,
"reconnect attempts:", hctx->reconnects);
return HANDLER_ERROR;
}
if ((errno != EAGAIN) &&
(errno != EINTR)) {
case EAGAIN:
case EINTR:
fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT);
return HANDLER_WAIT_FOR_EVENT;
default:
log_error_write(srv, __FILE__, __LINE__, "ssd",
"write failed:", strerror(errno), errno);
return HANDLER_ERROR;
} else {
fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT);
return HANDLER_WAIT_FOR_EVENT;
}
}
hctx->write_offset += r;
if (hctx->write_offset == hctx->write_buffer->used) {
if (hctx->wb->bytes_out == hctx->wb->bytes_in) {
/* we don't need the out event anymore */
fdevent_event_del(srv->ev, &(hctx->fde_ndx), hctx->fd);
fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_IN);
fcgi_set_state(srv, hctx, FCGI_STATE_READ);
} else {
fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT);
return HANDLER_WAIT_FOR_EVENT;
}
break;
case FCGI_STATE_READ:
/* waiting for a response */
@ -2735,7 +2854,7 @@ SUBREQUEST_FUNC(mod_fastcgi_handle_subrequest) {
buffer_reset(con->physical.path);
con->mode = DIRECT;
joblist_append(srv, con);
joblist_append(srv, con); /* really ? */
/* mis-using HANDLER_WAIT_FOR_FD to break out of the loop
* and hope that the childs will be restarted
@ -2758,6 +2877,7 @@ SUBREQUEST_FUNC(mod_fastcgi_handle_subrequest) {
buffer_reset(con->physical.path);
con->mode = DIRECT;
con->http_status = 503;
joblist_append(srv, con); /* really ? */
return HANDLER_FINISHED;
}
@ -2865,11 +2985,11 @@ static handler_t fcgi_handle_fdevent(void *s, void *ctx, int revents) {
if (con->file_started == 0) {
/* nothing has been send out yet, try to use another child */
if (hctx->write_offset == 0 &&
if (hctx->wb->bytes_out == 0 &&
hctx->reconnects < 5) {
fcgi_reconnect(srv, hctx);
log_error_write(srv, __FILE__, __LINE__, "sdsdsd",
log_error_write(srv, __FILE__, __LINE__, "ssdsd",
"response not sent, request not sent, reconnection.",
"connection-fd:", con->fd,
"fcgi-fd:", hctx->fd);
@ -2877,8 +2997,8 @@ static handler_t fcgi_handle_fdevent(void *s, void *ctx, int revents) {
return HANDLER_WAIT_FOR_FD;
}
log_error_write(srv, __FILE__, __LINE__, "sdsdsd",
"response not sent, request sent:", hctx->write_offset,
log_error_write(srv, __FILE__, __LINE__, "sosdsd",
"response not sent, request sent:", hctx->wb->bytes_out,
"connection-fd:", con->fd,
"fcgi-fd:", hctx->fd);

139
src/mod_proxy.c
Unescape Escape View File

@ -23,6 +23,7 @@
#include "inet_ntop_cache.h"
#include "crc32.h"
#include "network_backends.h"
#include <stdio.h>
@ -98,11 +99,9 @@ typedef struct {
buffer *response;
buffer *response_header;
buffer *write_buffer;
size_t write_offset;
chunkqueue *wb;
int fd; /* fd to the proxy process */
int fde_ndx; /* index into the fd-event buffer */
@ -128,7 +127,7 @@ static handler_ctx * handler_ctx_init() {
hctx->response = buffer_init();
hctx->response_header = buffer_init();
hctx->write_buffer = buffer_init();
hctx->wb = chunkqueue_init();
hctx->fd = -1;
hctx->fde_ndx = -1;
@ -139,7 +138,7 @@ static handler_ctx * handler_ctx_init() {
static void handler_ctx_free(handler_ctx *hctx) {
buffer_free(hctx->response);
buffer_free(hctx->response_header);
buffer_free(hctx->write_buffer);
chunkqueue_free(hctx->wb);
free(hctx);
}
@ -402,18 +401,19 @@ static int proxy_create_env(server *srv, handler_ctx *hctx) {
size_t i;
connection *con = hctx->remote_conn;
buffer *b;
UNUSED(srv);
/* build header */
buffer_reset(hctx->write_buffer);
b = chunkqueue_get_append_buffer(hctx->wb);
/* request line */
buffer_copy_string(hctx->write_buffer, get_http_method_name(con->request.http_method));
BUFFER_APPEND_STRING_CONST(hctx->write_buffer, " ");
buffer_copy_string(b, get_http_method_name(con->request.http_method));
BUFFER_APPEND_STRING_CONST(b, " ");
buffer_append_string_buffer(hctx->write_buffer, con->request.uri);
BUFFER_APPEND_STRING_CONST(hctx->write_buffer, " HTTP/1.0\r\n");
buffer_append_string_buffer(b, con->request.uri);
BUFFER_APPEND_STRING_CONST(b, " HTTP/1.0\r\n");
/* request header */
for (i = 0; i < con->request.headers->used; i++) {
@ -424,25 +424,73 @@ static int proxy_create_env(server *srv, handler_ctx *hctx) {
if (ds->value->used && ds->key->used) {
if (0 == strcmp(ds->key->ptr, "Connection")) continue;
buffer_append_string_buffer(hctx->write_buffer, ds->key);
BUFFER_APPEND_STRING_CONST(hctx->write_buffer, ": ");
buffer_append_string_buffer(hctx->write_buffer, ds->value);
BUFFER_APPEND_STRING_CONST(hctx->write_buffer, "\r\n");
buffer_append_string_buffer(b, ds->key);
BUFFER_APPEND_STRING_CONST(b, ": ");
buffer_append_string_buffer(b, ds->value);
BUFFER_APPEND_STRING_CONST(b, "\r\n");
}
}
BUFFER_APPEND_STRING_CONST(hctx->write_buffer, "X-Forwarded-For: ");
buffer_append_string(hctx->write_buffer, inet_ntop_cache_get_ip(srv, &(con->dst_addr)));
BUFFER_APPEND_STRING_CONST(hctx->write_buffer, "\r\n");
BUFFER_APPEND_STRING_CONST(b, "X-Forwarded-For: ");
buffer_append_string(b, inet_ntop_cache_get_ip(srv, &(con->dst_addr)));
BUFFER_APPEND_STRING_CONST(b, "\r\n");
BUFFER_APPEND_STRING_CONST(hctx->write_buffer, "\r\n");
BUFFER_APPEND_STRING_CONST(b, "\r\n");
hctx->wb->bytes_in += b->used - 1;
/* body */
if (con->request.content_length) {
/* the buffer-string functions add an extra \0 at the end the memory-function don't */
hctx->write_buffer->used--;
buffer_append_memory(hctx->write_buffer, con->request.content->ptr, con->request.content_length);
chunkqueue *req_cq = con->request_content_queue;
chunk *req_c;
size_t offset;
/* something to send ? */
for (offset = 0, req_c = req_cq->first; offset != req_cq->bytes_in; req_c = req_c->next) {
size_t weWant = req_cq->bytes_in - offset;
size_t weHave = 0;
/* we announce toWrite octects
* now take all the request_content chunk that we need to fill this request
* */
switch (req_c->type) {
case FILE_CHUNK:
weHave = req_c->file.length - req_c->offset;
if (weHave > weWant) weHave = weWant;
chunkqueue_append_file(hctx->wb, req_c->file.name, req_c->offset, weHave);
req_c->offset += weHave;
req_cq->bytes_out += weHave;
hctx->wb->bytes_in += weHave;
break;
case MEM_CHUNK:
/* append to the buffer */
weHave = req_c->mem->used - 1 - req_c->offset;
if (weHave > weWant) weHave = weWant;
b = chunkqueue_get_append_buffer(hctx->wb);
buffer_append_memory(b, req_c->mem->ptr + req_c->offset, weHave);
b->used++; /* add virtual \0 */
req_c->offset += weHave;
req_cq->bytes_out += weHave;
hctx->wb->bytes_in += weHave;
break;
default:
break;
}
offset += weHave;
}
}
return 0;
@ -652,17 +700,16 @@ static int proxy_demux_response(server *srv, handler_ctx *hctx) {
static handler_t proxy_write_request(server *srv, handler_ctx *hctx) {
data_proxy *host= hctx->host;
plugin_data *p = hctx->plugin_data;
connection *con = hctx->remote_conn;
int r;
int ret;
if (!host ||
(!host->host->used || !host->port)) return -1;
switch(hctx->state) {
case PROXY_STATE_INIT:
r = AF_INET;
if (-1 == (hctx->fd = socket(r, SOCK_STREAM, 0))) {
if (-1 == (hctx->fd = socket(AF_INET, SOCK_STREAM, 0))) {
log_error_write(srv, __FILE__, __LINE__, "ss", "socket failed: ", strerror(errno));
return HANDLER_ERROR;
}
@ -734,33 +781,45 @@ static handler_t proxy_write_request(server *srv, handler_ctx *hctx) {
proxy_create_env(srv, hctx);
proxy_set_state(srv, hctx, PROXY_STATE_WRITE);
hctx->write_offset = 0;
fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT);
/* fall through */
case PROXY_STATE_WRITE:
/* continue with the code after the switch */
if (-1 == (r = write(hctx->fd,
hctx->write_buffer->ptr + hctx->write_offset,
hctx->write_buffer->used - hctx->write_offset))) {
case PROXY_STATE_WRITE:;
#if defined USE_LINUX_SENDFILE
ret = network_write_chunkqueue_linuxsendfile(srv, con, hctx->fd, hctx->wb);
#elif defined USE_FREEBSD_SENDFILE
ret = network_write_chunkqueue_freebsdsendfile(srv, con, hctx->fd, hctx->wb);
#elif defined USE_SOLARIS_SENDFILEV
ret = network_write_chunkqueue_solarissendfilev(srv, con, hctx->fd, hctx->wb);
#elif defined USE_WRITEV
ret = network_write_chunkqueue_writev(srv, con, hctx->fd, hctx->wb);
#else
ret = network_write_chunkqueue_write(srv, con, hctx->fd, hctx->wb);
#endif
chunkqueue_remove_finished_chunks(hctx->wb);
if (-1 == ret) {
if (errno != EAGAIN &&
errno != EINTR) {
log_error_write(srv, __FILE__, __LINE__, "ssd", "write failed:", strerror(errno), r);
log_error_write(srv, __FILE__, __LINE__, "ssd", "write failed:", strerror(errno), errno);
return HANDLER_ERROR;
} else {
fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT);
return HANDLER_WAIT_FOR_EVENT;
}
}
hctx->write_offset += r;
if (hctx->write_offset == hctx->write_buffer->used) {
if (hctx->wb->bytes_out == hctx->wb->bytes_in) {
proxy_set_state(srv, hctx, PROXY_STATE_READ);
fdevent_event_del(srv->ev, &(hctx->fde_ndx), hctx->fd);
fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_IN);
} else {
fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT);
return HANDLER_WAIT_FOR_EVENT;
}
return HANDLER_WAIT_FOR_EVENT;

130
src/mod_scgi.c
Unescape Escape View File

@ -22,6 +22,7 @@
#include "plugin.h"
#include "inet_ntop_cache.h"
#include "network_backends.h"
#include <stdio.h>
@ -298,10 +299,8 @@ typedef struct {
int reconnects; /* number of reconnect attempts */
buffer *write_buffer;
size_t write_offset;
read_buffer *rb;
chunkqueue *wb;
buffer *response_header;
@ -338,7 +337,6 @@ static handler_ctx * handler_ctx_init() {
hctx->response = buffer_init();
hctx->response_header = buffer_init();
hctx->write_buffer = buffer_init();
hctx->request_id = 0;
hctx->state = FCGI_STATE_INIT;
@ -350,6 +348,8 @@ static handler_ctx * handler_ctx_init() {
hctx->fd = -1;
hctx->reconnects = 0;
hctx->wb = chunkqueue_init();
return hctx;
}
@ -357,7 +357,8 @@ static handler_ctx * handler_ctx_init() {
static void handler_ctx_free(handler_ctx *hctx) {
buffer_free(hctx->response);
buffer_free(hctx->response_header);
buffer_free(hctx->write_buffer);
chunkqueue_free(hctx->wb);
if (hctx->rb) {
if (hctx->rb->ptr) free(hctx->rb->ptr);
@ -1409,11 +1410,11 @@ static int scgi_env_add_request_headers(server *srv, connection *con, plugin_dat
static int scgi_create_env(server *srv, handler_ctx *hctx) {
char buf[32];
size_t offset;
const char *s;
#ifdef HAVE_IPV6
char b2[INET6_ADDRSTRLEN + 1];
#endif
buffer *b;
plugin_data *p = hctx->plugin_data;
scgi_extension_host *host= hctx->host;
@ -1564,23 +1565,66 @@ static int scgi_create_env(server *srv, handler_ctx *hctx) {
#endif
scgi_env_add_request_headers(srv, con, p);
b = chunkqueue_get_append_buffer(hctx->wb);
buffer_append_long(hctx->write_buffer, p->scgi_env->used);
buffer_append_string_len(hctx->write_buffer, CONST_STR_LEN(":"));
buffer_append_string_len(hctx->write_buffer, (const char *)p->scgi_env->ptr, p->scgi_env->used);
buffer_append_string_len(hctx->write_buffer, CONST_STR_LEN(","));
hctx->write_buffer->used--;
/* send FCGI_STDIN */
buffer_append_long(b, p->scgi_env->used);
buffer_append_string_len(b, CONST_STR_LEN(":"));
buffer_append_string_len(b, (const char *)p->scgi_env->ptr, p->scgi_env->used);
buffer_append_string_len(b, CONST_STR_LEN(","));
hctx->wb->bytes_in += b->used - 1;
/* something to send ? */
for (offset = 0; offset != con->request.content_length; ) {
/* send chunks of 1024 bytes */
size_t toWrite = con->request.content_length - offset > 4096 ? 4096 : con->request.content_length - offset;
buffer_append_memory(hctx->write_buffer, (const char *)(con->request.content->ptr + offset), toWrite);
offset += toWrite;
if (con->request.content_length) {
chunkqueue *req_cq = con->request_content_queue;
chunk *req_c;
size_t offset;
/* something to send ? */
for (offset = 0, req_c = req_cq->first; offset != req_cq->bytes_in; req_c = req_c->next) {
size_t weWant = req_cq->bytes_in - offset;
size_t weHave = 0;
/* we announce toWrite octects
* now take all the request_content chunk that we need to fill this request
* */
switch (req_c->type) {
case FILE_CHUNK:
weHave = req_c->file.length - req_c->offset;
if (weHave > weWant) weHave = weWant;
chunkqueue_append_file(hctx->wb, req_c->file.name, req_c->offset, weHave);
req_c->offset += weHave;
req_cq->bytes_out += weHave;
hctx->wb->bytes_in += weHave;
break;
case MEM_CHUNK:
/* append to the buffer */
weHave = req_c->mem->used - 1 - req_c->offset;
if (weHave > weWant) weHave = weWant;
b = chunkqueue_get_append_buffer(hctx->wb);
buffer_append_memory(b, req_c->mem->ptr + req_c->offset, weHave);
b->used++; /* add virtual \0 */
req_c->offset += weHave;
req_cq->bytes_out += weHave;
hctx->wb->bytes_in += weHave;
break;
default:
break;
}
offset += weHave;
}
}
#if 0
@ -2078,7 +2122,7 @@ static handler_t scgi_write_request(server *srv, handler_ctx *hctx) {
scgi_extension_host *host= hctx->host;
connection *con = hctx->remote_conn;
int r;
int ret;
/* sanity check */
if (!host ||
@ -2095,9 +2139,9 @@ static handler_t scgi_write_request(server *srv, handler_ctx *hctx) {
switch(hctx->state) {
case FCGI_STATE_INIT:
r = host->unixsocket->used ? AF_UNIX : AF_INET;
ret = host->unixsocket->used ? AF_UNIX : AF_INET;
if (-1 == (hctx->fd = socket(r, SOCK_STREAM, 0))) {
if (-1 == (hctx->fd = socket(ret, SOCK_STREAM, 0))) {
if (errno == EMFILE ||
errno == EINTR) {
log_error_write(srv, __FILE__, __LINE__, "sd",
@ -2210,17 +2254,25 @@ static handler_t scgi_write_request(server *srv, handler_ctx *hctx) {
scgi_create_env(srv, hctx);
scgi_set_state(srv, hctx, FCGI_STATE_WRITE);
hctx->write_offset = 0;
/* fall through */
case FCGI_STATE_WRITE:
/* why aren't we using the network_ interface here ? */
r = write(hctx->fd,
hctx->write_buffer->ptr + hctx->write_offset,
hctx->write_buffer->used - hctx->write_offset);
if (-1 == r) {
#if defined USE_LINUX_SENDFILE
ret = network_write_chunkqueue_linuxsendfile(srv, con, hctx->fd, hctx->wb);
#elif defined USE_FREEBSD_SENDFILE
ret = network_write_chunkqueue_freebsdsendfile(srv, con, hctx->fd, hctx->wb);
#elif defined USE_SOLARIS_SENDFILEV
ret = network_write_chunkqueue_solarissendfilev(srv, con, hctx->fd, hctx->wb);
#elif defined USE_WRITEV
ret = network_write_chunkqueue_writev(srv, con, hctx->fd, hctx->wb);
#else
ret = network_write_chunkqueue_write(srv, con, hctx->fd, hctx->wb);
#endif
chunkqueue_remove_finished_chunks(hctx->wb);
if (-1 == ret) {
if (errno == ENOTCONN) {
/* the connection got dropped after accept()
*
@ -2228,7 +2280,7 @@ static handler_t scgi_write_request(server *srv, handler_ctx *hctx) {
* after PHP_FCGI_MAX_REQUESTS
*
*/
if (hctx->write_offset == 0 &&
if (hctx->wb->bytes_out == 0 &&
hctx->reconnects < 5) {
usleep(10000); /* take away the load of the webserver
* to let the php a chance to restart
@ -2247,7 +2299,7 @@ static handler_t scgi_write_request(server *srv, handler_ctx *hctx) {
log_error_write(srv, __FILE__, __LINE__, "ssdsd",
"[REPORT ME] connection was dropped after accept(). reconnect() denied:",
"write-offset:", hctx->write_offset,
"write-offset:", hctx->wb->bytes_out,
"reconnect attempts:", hctx->reconnects);
return HANDLER_ERROR;
@ -2267,13 +2319,15 @@ static handler_t scgi_write_request(server *srv, handler_ctx *hctx) {
}
}
hctx->write_offset += r;
if (hctx->write_offset == hctx->write_buffer->used) {
if (hctx->wb->bytes_out == hctx->wb->bytes_in) {
/* we don't need the out event anymore */
fdevent_event_del(srv->ev, &(hctx->fde_ndx), hctx->fd);
fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_IN);
scgi_set_state(srv, hctx, FCGI_STATE_READ);
} else {
fdevent_event_add(srv->ev, &(hctx->fde_ndx), hctx->fd, FDEVENT_OUT);
return HANDLER_WAIT_FOR_EVENT;
}
break;
@ -2481,7 +2535,7 @@ static handler_t scgi_handle_fdevent(void *s, void *ctx, int revents) {
if (con->file_started == 0) {
/* nothing has been send out yet, try to use another child */
if (hctx->write_offset == 0 &&
if (hctx->wb->bytes_out == 0 &&
hctx->reconnects < 5) {
scgi_reconnect(srv, hctx);
@ -2494,7 +2548,7 @@ static handler_t scgi_handle_fdevent(void *s, void *ctx, int revents) {
}
log_error_write(srv, __FILE__, __LINE__, "sdsdsd",
"response not sent, request sent:", hctx->write_offset,
"response not sent, request sent:", hctx->wb->bytes_out,
"connection-fd:", con->fd,
"fcgi-fd:", hctx->fd);

2
src/mod_status.c
Unescape Escape View File

@ -478,7 +478,7 @@ static handler_t mod_status_handle_server_status_html(server *srv, connection *c
BUFFER_APPEND_STRING_CONST(b, "</td><td class=\"int\">");
if (con->request.content_length) {
buffer_append_long(b, c->request.content->used);
buffer_append_long(b, c->request_content_queue->bytes_in);
BUFFER_APPEND_STRING_CONST(b, "/");
buffer_append_long(b, c->request.content_length);
} else {

80
src/mod_webdav.c
Unescape Escape View File

@ -8,8 +8,13 @@
#include <unistd.h>
#include <fcntl.h>
#include <stdio.h>
#include <assert.h>
#include <sys/mman.h>
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#if defined(HAVE_LIBXML_H) && defined(HAVE_SQLITE3_H)
#define USE_PROPPATCH
#include <libxml/tree.h>
@ -18,7 +23,6 @@
#include <sqlite3.h>
#endif
#include "base.h"
#include "log.h"
#include "buffer.h"
@ -1224,31 +1228,91 @@ URIHANDLER_FUNC(mod_webdav_subrequest_handler) {
return HANDLER_FINISHED;
case HTTP_METHOD_PUT: {
int fd;
chunkqueue *cq = con->request_content_queue;
if (p->conf.is_readonly) {
con->http_status = 403;
return HANDLER_FINISHED;
}
assert(chunkqueue_length(cq) == con->request.content_length);
/* taken what we have in the request-body and write it to a file */
if (-1 == (fd = open(con->physical.path->ptr, O_WRONLY|O_CREAT|O_TRUNC, 0600))) {
/* we can't open the file */
con->http_status = 403;
} else {
chunk *c;
con->http_status = 201; /* created */
if (-1 == (write(fd, con->request.content->ptr, con->request.content->used - 1))) {
switch(errno) {
case ENOSPC:
con->http_status = 507;
for (c = cq->first; c; c = cq->first) {
int r = 0;
/* copy all chunks */
switch(c->type) {
case FILE_CHUNK:
if (c->file.mmap.start == MAP_FAILED) {
if (-1 == c->file.fd && /* open the file if not already open */
-1 == (c->file.fd = open(c->file.name->ptr, O_RDONLY))) {
log_error_write(srv, __FILE__, __LINE__, "ss", "open failed: ", strerror(errno));
return -1;
}
if (MAP_FAILED == (c->file.mmap.start = mmap(0, c->file.length, PROT_READ, MAP_SHARED, c->file.fd, 0))) {
log_error_write(srv, __FILE__, __LINE__, "ssbd", "mmap failed: ",
strerror(errno), c->file.name, c->file.fd);
return -1;
}
c->file.mmap.length = c->file.length;
close(c->file.fd);
c->file.fd = -1;
/* chunk_reset() or chunk_free() will cleanup for us */
}
if ((r = write(fd, c->file.mmap.start + c->offset, c->file.length - c->offset)) < 0) {
switch(errno) {
case ENOSPC:
con->http_status = 507;
break;
default:
con->http_status = 403;
break;
}
}
break;
default: