commit ce874f09eaed8b0265cf62c370633ee82c676b1d
parent 0647901ef5ccccdd970cd3d63c4d09dde27a7175
Author: Joris Vink <joris@coders.se>
Date: Fri, 8 Jan 2016 20:56:21 +0100
Cut memory usage for incoming HTTP bodies in half.
Kore pre-allocates a kore_buf for the full size of the
incoming HTTP body ... but also was passing the full
size to the net_recv_reset() function.
Instead of this, properly read smaller chunks from the
network and append them to the body buffer as they roll in.
Diffstat:
2 files changed, 27 insertions(+), 19 deletions(-)
diff --git a/includes/http.h b/includes/http.h
@@ -186,6 +186,7 @@ struct http_request {
char *agent;
struct connection *owner;
struct kore_buf *http_body;
+ u_int64_t content_length;
void *hdlr_extra;
char *query_string;
u_int8_t *multipart_body;
diff --git a/src/http.c b/src/http.c
@@ -494,11 +494,11 @@ int
http_header_recv(struct netbuf *nb)
{
size_t len;
- u_int64_t clen;
struct http_header *hdr;
struct http_request *req;
+ u_int64_t bytes_left;
u_int8_t *end_headers;
- int h, i, v, skip, bytes_left;
+ int h, i, v, skip;
char *request[4], *host[3], *hbuf;
char *p, *headers[HTTP_REQ_HEADER_MAX];
struct connection *c = (struct connection *)nb->owner;
@@ -604,7 +604,7 @@ http_header_recv(struct netbuf *nb)
return (KORE_RESULT_OK);
}
- clen = kore_strtonum(p, 10, 0, LONG_MAX, &v);
+ req->content_length = kore_strtonum(p, 10, 0, LONG_MAX, &v);
if (v == KORE_RESULT_ERROR) {
kore_debug("content-length invalid: %s", p);
req->flags |= HTTP_REQUEST_DELETE;
@@ -612,36 +612,37 @@ http_header_recv(struct netbuf *nb)
return (KORE_RESULT_OK);
}
- if (clen == 0) {
+ if (req->content_length == 0) {
req->flags |= HTTP_REQUEST_COMPLETE;
req->flags &= ~HTTP_REQUEST_EXPECT_BODY;
return (KORE_RESULT_OK);
}
- if (clen > http_body_max) {
+ if (req->content_length > http_body_max) {
kore_log(LOG_NOTICE, "body too large (%ld > %ld)",
- clen, http_body_max);
+ req->content_length, http_body_max);
req->flags |= HTTP_REQUEST_DELETE;
http_error_response(req->owner, 413);
return (KORE_RESULT_OK);
}
- req->http_body = kore_buf_create(clen);
+ req->http_body = kore_buf_create(req->content_length);
kore_buf_append(req->http_body, end_headers,
(nb->s_off - len));
- bytes_left = clen - (nb->s_off - len);
+ bytes_left = req->content_length - (nb->s_off - len);
if (bytes_left > 0) {
kore_debug("%ld/%ld (%ld - %ld) more bytes for body",
- bytes_left, clen, nb->s_off, len);
- net_recv_reset(c, bytes_left, http_body_recv);
+ bytes_left, req->content_length, nb->s_off, len);
+ net_recv_reset(c,
+ MIN(bytes_left, NETBUF_SEND_PAYLOAD_MAX),
+ http_body_recv);
c->rnb->extra = req;
- c->rnb->flags &= ~NETBUF_CALL_CB_ALWAYS;
+ http_request_sleep(req);
} else if (bytes_left == 0) {
req->flags |= HTTP_REQUEST_COMPLETE;
req->flags &= ~HTTP_REQUEST_EXPECT_BODY;
} else {
- kore_debug("bytes_left would become zero (%ld)", clen);
http_error_response(req->owner, 500);
}
}
@@ -1090,17 +1091,23 @@ http_file_add(struct http_request *req, const char *name, const char *filename,
static int
http_body_recv(struct netbuf *nb)
{
+ u_int64_t bytes_left;
struct http_request *req = (struct http_request *)nb->extra;
kore_buf_append(req->http_body, nb->buf, nb->s_off);
- req->flags |= HTTP_REQUEST_COMPLETE;
- req->flags &= ~HTTP_REQUEST_EXPECT_BODY;
-
- nb->extra = NULL;
- nb->flags |= NETBUF_CALL_CB_ALWAYS;
-
- kore_debug("received all body data for request %p", req);
+ if (req->http_body->offset == req->content_length) {
+ nb->extra = NULL;
+ req->flags |= HTTP_REQUEST_COMPLETE;
+ req->flags &= ~HTTP_REQUEST_EXPECT_BODY;
+ http_request_wakeup(req);
+ net_recv_reset(nb->owner, http_header_max, http_header_recv);
+ } else {
+ bytes_left = req->content_length - req->http_body->offset;
+ net_recv_reset(nb->owner,
+ MIN(bytes_left, NETBUF_SEND_PAYLOAD_MAX),
+ http_body_recv);
+ }
return (KORE_RESULT_OK);
}