kore

Kore is a web application platform for writing scalable, concurrent web based processes in C or Python.
Commits | Files | Refs | README | LICENSE | git clone https://git.kore.io/kore.git

commit ce874f09eaed8b0265cf62c370633ee82c676b1d
parent 0647901ef5ccccdd970cd3d63c4d09dde27a7175
Author: Joris Vink <joris@coders.se>
Date:   Fri,  8 Jan 2016 20:56:21 +0100

Cut memory usage for incoming HTTP bodies in half.

Kore pre-allocates a kore_buf for the full size of the
incoming HTTP body ... but also was passing the full
size to the net_recv_reset() function.

Instead of this, properly read smaller chunks from the
network and append them to the body buffer as they roll in.

Diffstat:
includes/http.h | 1+
src/http.c | 45++++++++++++++++++++++++++-------------------
2 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/includes/http.h b/includes/http.h @@ -186,6 +186,7 @@ struct http_request { char *agent; struct connection *owner; struct kore_buf *http_body; + u_int64_t content_length; void *hdlr_extra; char *query_string; u_int8_t *multipart_body; diff --git a/src/http.c b/src/http.c @@ -494,11 +494,11 @@ int http_header_recv(struct netbuf *nb) { size_t len; - u_int64_t clen; struct http_header *hdr; struct http_request *req; + u_int64_t bytes_left; u_int8_t *end_headers; - int h, i, v, skip, bytes_left; + int h, i, v, skip; char *request[4], *host[3], *hbuf; char *p, *headers[HTTP_REQ_HEADER_MAX]; struct connection *c = (struct connection *)nb->owner; @@ -604,7 +604,7 @@ http_header_recv(struct netbuf *nb) return (KORE_RESULT_OK); } - clen = kore_strtonum(p, 10, 0, LONG_MAX, &v); + req->content_length = kore_strtonum(p, 10, 0, LONG_MAX, &v); if (v == KORE_RESULT_ERROR) { kore_debug("content-length invalid: %s", p); req->flags |= HTTP_REQUEST_DELETE; @@ -612,36 +612,37 @@ http_header_recv(struct netbuf *nb) return (KORE_RESULT_OK); } - if (clen == 0) { + if (req->content_length == 0) { req->flags |= HTTP_REQUEST_COMPLETE; req->flags &= ~HTTP_REQUEST_EXPECT_BODY; return (KORE_RESULT_OK); } - if (clen > http_body_max) { + if (req->content_length > http_body_max) { kore_log(LOG_NOTICE, "body too large (%ld > %ld)", - clen, http_body_max); + req->content_length, http_body_max); req->flags |= HTTP_REQUEST_DELETE; http_error_response(req->owner, 413); return (KORE_RESULT_OK); } - req->http_body = kore_buf_create(clen); + req->http_body = kore_buf_create(req->content_length); kore_buf_append(req->http_body, end_headers, (nb->s_off - len)); - bytes_left = clen - (nb->s_off - len); + bytes_left = req->content_length - (nb->s_off - len); if (bytes_left > 0) { kore_debug("%ld/%ld (%ld - %ld) more bytes for body", - bytes_left, clen, nb->s_off, len); - net_recv_reset(c, bytes_left, http_body_recv); + bytes_left, req->content_length, nb->s_off, len); + net_recv_reset(c, + MIN(bytes_left, NETBUF_SEND_PAYLOAD_MAX), + http_body_recv); c->rnb->extra = req; - c->rnb->flags &= ~NETBUF_CALL_CB_ALWAYS; + http_request_sleep(req); } else if (bytes_left == 0) { req->flags |= HTTP_REQUEST_COMPLETE; req->flags &= ~HTTP_REQUEST_EXPECT_BODY; } else { - kore_debug("bytes_left would become zero (%ld)", clen); http_error_response(req->owner, 500); } } @@ -1090,17 +1091,23 @@ http_file_add(struct http_request *req, const char *name, const char *filename, static int http_body_recv(struct netbuf *nb) { + u_int64_t bytes_left; struct http_request *req = (struct http_request *)nb->extra; kore_buf_append(req->http_body, nb->buf, nb->s_off); - req->flags |= HTTP_REQUEST_COMPLETE; - req->flags &= ~HTTP_REQUEST_EXPECT_BODY; - - nb->extra = NULL; - nb->flags |= NETBUF_CALL_CB_ALWAYS; - - kore_debug("received all body data for request %p", req); + if (req->http_body->offset == req->content_length) { + nb->extra = NULL; + req->flags |= HTTP_REQUEST_COMPLETE; + req->flags &= ~HTTP_REQUEST_EXPECT_BODY; + http_request_wakeup(req); + net_recv_reset(nb->owner, http_header_max, http_header_recv); + } else { + bytes_left = req->content_length - req->http_body->offset; + net_recv_reset(nb->owner, + MIN(bytes_left, NETBUF_SEND_PAYLOAD_MAX), + http_body_recv); + } return (KORE_RESULT_OK); }