commit fcb86ddb8bb96d1961d0970e311ad5aaeb4bbe4c
parent 96641d3caa05f350286dd9b9ea343382a0ed01a2
Author: Joris Vink <joris@coders.se>
Date: Mon, 18 Jan 2016 11:30:22 +0100
Massive rework of HTTP layer.
This commit is a flag day, your old modules will almost certainly
need to be updated in order to build properly with these changes.
Summary of changes:
- Offload HTTP bodies to disk if they are large (inspired by #100).
(disabled by default)
- The http_argument_get* macros now takes an explicit http_request parameter.
- Kore will now throw 404 errors almost immediately after an HTTP request
has come in instead of waiting until all data has arrived.
API changes:
- http_argument_get* macros now require an explicit http_request parameter.
(no more magic invokations).
- http_generic_404() is gone
- http_populate_arguments() is gone
- http_body_bytes() is gone
- http_body_text() is gone
- http_body_read() has been added
- http_populate_post() has been added
- http_populate_get() has been added
- http_file_read() has been added
- http_file_rewind() has been added
- http_file_lookup() no longer takes name, fname, data and len parameters.
- http_file_lookup() now returns a struct http_file pointer.
- http_populate_multipart_form() no longer takes an secondary parameter.
New configuration options:
- http_body_disk_offload:
Number of bytes after which Kore will offload the HTTP body to
disk instead of retaining it in memory. If 0 this feature is
disabled. (Default: 0)
- http_body_disk_path:
The path where Kore will store temporary HTTP body files.
(this directory does not get created if http_body_disk_offload is 0).
New example:
The upload example has been added, demonstrating how to deal with file
uploads from a multipart form.
Diffstat:
17 files changed, 933 insertions(+), 458 deletions(-)
diff --git a/conf/kore.conf.example b/conf/kore.conf.example
@@ -61,6 +61,16 @@ workers 4
# http_body_max Maximum size of an HTTP body (in bytes).
# If set to 0 disallows requests with a body
# all together.
+#
+# http_body_disk_offload Number of bytes after which Kore will use
+# a temporary file to hold the HTTP body
+# instead of holding it in memory. If set to
+# 0 no disk offloading will be done. This is
+# turned off by default.
+#
+# http_body_disk_path Path where Kore will store any temporary
+# HTTP body files.
+#
# http_keepalive_time Maximum seconds an HTTP connection can be
# kept alive by the browser.
# (Set to 0 to disable keepalive completely).
@@ -76,6 +86,8 @@ workers 4
#http_keepalive_time 0
#http_hsts_enable 31536000
#http_request_limit 1000
+#http_body_disk_offload 0
+#http_body_disk_path tmp_files
# Websocket specific settings.
# websocket_maxframe Specifies the maximum frame size we can receive
diff --git a/examples/generic/conf/generic.conf b/examples/generic/conf/generic.conf
@@ -5,6 +5,9 @@ load ./generic.so example_load
tls_dhparam dh2048.pem
+http_body_max 1024000000
+http_body_disk_offload 1024000
+
validator v_example function v_example_func
validator v_regex regex ^/test/[a-z]*$
validator v_number regex ^[0-9]*$
diff --git a/examples/generic/src/example.c b/examples/generic/src/example.c
@@ -17,6 +17,12 @@
#include <kore/kore.h>
#include <kore/http.h>
+#include <openssl/sha.h>
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+
#include "assets.h"
int example_load(int);
@@ -132,9 +138,9 @@ serve_b64test(struct http_request *req)
int
serve_file_upload(struct http_request *req)
{
- int r;
u_int8_t *d;
struct kore_buf *b;
+ struct http_file *f;
u_int32_t len;
char *name, buf[BUFSIZ];
@@ -142,16 +148,20 @@ serve_file_upload(struct http_request *req)
kore_buf_append(b, asset_upload_html, asset_len_upload_html);
if (req->method == HTTP_METHOD_POST) {
- http_populate_multipart_form(req, &r);
- if (http_argument_get_string("firstname", &name, &len)) {
- kore_buf_replace_string(b, "$firstname$", name, len);
+ if (req->http_body_fd != -1)
+ kore_log(LOG_NOTICE, "file is on disk");
+
+ http_populate_multipart_form(req);
+ if (http_argument_get_string(req, "firstname", &name)) {
+ kore_buf_replace_string(b, "$firstname$",
+ name, strlen(name));
} else {
kore_buf_replace_string(b, "$firstname$", NULL, 0);
}
- if (http_file_lookup(req, "file", &name, &d, &len)) {
+ if ((f = http_file_lookup(req, "file")) != NULL) {
(void)snprintf(buf, sizeof(buf),
- "%s is %d bytes", name, len);
+ "%s is %ld bytes", f->filename, f->length);
kore_buf_replace_string(b,
"$upload$", buf, strlen(buf));
} else {
@@ -232,7 +242,10 @@ serve_params_test(struct http_request *req)
int r, i;
char *test, name[10];
- http_populate_arguments(req);
+ if (req->method == HTTP_METHOD_GET)
+ http_populate_get(req);
+ else if (req->method == HTTP_METHOD_POST)
+ http_populate_post(req);
b = kore_buf_create(asset_len_params_html);
kore_buf_append(b, asset_params_html, asset_len_params_html);
@@ -240,14 +253,14 @@ serve_params_test(struct http_request *req)
/*
* The GET parameters will be filtered out on POST.
*/
- if (http_argument_get_string("arg1", &test, &len)) {
- kore_buf_replace_string(b, "$arg1$", test, len);
+ if (http_argument_get_string(req, "arg1", &test)) {
+ kore_buf_replace_string(b, "$arg1$", test, strlen(test));
} else {
kore_buf_replace_string(b, "$arg1$", NULL, 0);
}
- if (http_argument_get_string("arg2", &test, &len)) {
- kore_buf_replace_string(b, "$arg2$", test, len);
+ if (http_argument_get_string(req, "arg2", &test)) {
+ kore_buf_replace_string(b, "$arg2$", test, strlen(test));
} else {
kore_buf_replace_string(b, "$arg2$", NULL, 0);
}
@@ -257,7 +270,7 @@ serve_params_test(struct http_request *req)
kore_buf_replace_string(b, "$test2$", NULL, 0);
kore_buf_replace_string(b, "$test3$", NULL, 0);
- if (http_argument_get_uint16("id", &r))
+ if (http_argument_get_uint16(req, "id", &r))
kore_log(LOG_NOTICE, "id: %d", r);
else
kore_log(LOG_NOTICE, "No id set");
@@ -272,9 +285,9 @@ serve_params_test(struct http_request *req)
for (i = 1; i < 4; i++) {
(void)snprintf(name, sizeof(name), "test%d", i);
- if (http_argument_get_string(name, &test, &len)) {
+ if (http_argument_get_string(req, name, &test)) {
(void)snprintf(name, sizeof(name), "$test%d$", i);
- kore_buf_replace_string(b, name, test, len);
+ kore_buf_replace_string(b, name, test, strlen(test));
} else {
(void)snprintf(name, sizeof(name), "$test%d$", i);
kore_buf_replace_string(b, name, NULL, 0);
diff --git a/examples/integers/src/check_integers.c b/examples/integers/src/check_integers.c
@@ -15,28 +15,28 @@ page(struct http_request *req)
u_int32_t u32, len;
u_int8_t c, *data;
- http_populate_arguments(req);
+ http_populate_get(req);
buf = kore_buf_create(128);
- if (http_argument_get_byte("id", &c))
+ if (http_argument_get_byte(req, "id", &c))
kore_buf_appendf(buf, "byte\t%c\n", c);
- if (http_argument_get_int16("id", &s16))
+ if (http_argument_get_int16(req, "id", &s16))
kore_buf_appendf(buf, "int16\t%d\n", s16);
- if (http_argument_get_uint16("id", &u16))
+ if (http_argument_get_uint16(req, "id", &u16))
kore_buf_appendf(buf, "uint16\t%d\n", u16);
- if (http_argument_get_int32("id", &s32))
+ if (http_argument_get_int32(req, "id", &s32))
kore_buf_appendf(buf, "int32\t%d\n", s32);
- if (http_argument_get_uint32("id", &u32))
+ if (http_argument_get_uint32(req, "id", &u32))
kore_buf_appendf(buf, "uint32\t%d\n", u32);
- if (http_argument_get_int64("id", &s64))
+ if (http_argument_get_int64(req, "id", &s64))
kore_buf_appendf(buf, "int64\t%ld\n", s64);
- if (http_argument_get_uint64("id", &u64))
+ if (http_argument_get_uint64(req, "id", &u64))
kore_buf_appendf(buf, "uint64\t%lu\n", u64);
data = kore_buf_release(buf, &len);
diff --git a/examples/json_yajl/src/json_yajl.c b/examples/json_yajl/src/json_yajl.c
@@ -1,3 +1,19 @@
+/*
+ * Copyright (c) 2013-2016 Joris Vink <joris@coders.se>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
#include <kore/kore.h>
#include <kore/http.h>
@@ -8,10 +24,12 @@ int page(struct http_request *);
int
page(struct http_request *req)
{
+ ssize_t ret;
struct kore_buf *buf;
char *body;
yajl_val node, v;
char eb[1024];
+ u_int8_t data[BUFSIZ];
const char *path[] = { "foo", "bar", NULL };
/* We only allow POST/PUT methods. */
@@ -23,14 +41,27 @@ page(struct http_request *req)
}
/*
- * Grab the entire body we received as text (NUL-terminated).
- * Note: this can return NULL and the result MUST be freed.
+ * Read the entire received body into a memory buffer.
*/
- if ((body = http_body_text(req)) == NULL) {
- http_response(req, 400, NULL, 0);
- return (KORE_RESULT_OK);
+ buf = kore_buf_create(128);
+ for (;;) {
+ ret = http_body_read(req, data, sizeof(data));
+ if (ret == -1) {
+ kore_buf_free(buf);
+ kore_log(LOG_NOTICE, "error reading body");
+ http_response(req, 500, NULL, 0);
+ return (KORE_RESULT_OK);
+ }
+
+ if (ret == 0)
+ break;
+
+ kore_buf_append(buf, data, ret);
}
+ /* Grab our body data as a NUL-terminated string. */
+ body = kore_buf_stringify(buf);
+
/* Parse the body via yajl now. */
node = yajl_tree_parse(body, eb, sizeof(eb));
if (node == NULL) {
@@ -40,12 +71,13 @@ page(struct http_request *req)
kore_log(LOG_NOTICE, "parse error: unknown");
}
- kore_mem_free(body);
+ kore_buf_free(buf);
http_response(req, 400, NULL, 0);
return (KORE_RESULT_OK);
}
- buf = kore_buf_create(128);
+ /* Reuse old buffer, don't need it anymore for body. */
+ kore_buf_reset(buf);
/* Attempt to grab foo.bar from the JSON tree. */
v = yajl_tree_get(node, path, yajl_t_string);
diff --git a/examples/ktunnel/src/ktunnel.c b/examples/ktunnel/src/ktunnel.c
@@ -49,9 +49,9 @@ open_connection(struct http_request *req)
}
/* Parse the query string and grab our arguments. */
- http_populate_arguments(req);
- if (!http_argument_get_string("host", &host, NULL) ||
- !http_argument_get_string("port", &port, NULL)) {
+ http_populate_get(req);
+ if (!http_argument_get_string(req, "host", &host) ||
+ !http_argument_get_string(req, "port", &port)) {
http_response(req, HTTP_STATUS_BAD_REQUEST, NULL, 0);
return (KORE_RESULT_OK);
}
diff --git a/examples/parameters/src/parameters.c b/examples/parameters/src/parameters.c
@@ -22,9 +22,7 @@ int page(struct http_request *);
int
page(struct http_request *req)
{
- int p;
u_int16_t id;
- u_int32_t len;
char *sid;
struct kore_buf *buf;
@@ -40,17 +38,8 @@ page(struct http_request *req)
* See conf/parameters.conf on how that is done, this is an
* important step as without the params block you will never
* get any parameters returned from Kore.
- *
- * http_populate_arguments() returns the number of arguments
- * that were successfully processed and are available.
*/
- p = http_populate_arguments(req);
-
- /* If we had no arguments available what so ever, return 400. */
- if (p == 0) {
- http_response(req, 400, NULL, 0);
- return (KORE_RESULT_OK);
- }
+ http_populate_get(req);
/*
* Lets grab the "id" parameter if available. Kore can obtain
@@ -73,11 +62,11 @@ page(struct http_request *req)
buf = kore_buf_create(128);
/* Grab it as a string, we shouldn't free the result in sid. */
- if (http_argument_get_string("id", &sid, &len))
- kore_buf_appendf(buf, "id as a string: '%s' (%d)\n", sid, len);
+ if (http_argument_get_string(req, "id", &sid))
+ kore_buf_appendf(buf, "id as a string: '%s'\n", sid);
/* Grab it as an actual u_int16_t. */
- if (http_argument_get_uint16("id", &id))
+ if (http_argument_get_uint16(req, "id", &id))
kore_buf_appendf(buf, "id as an u_int16_t: %d\n", id);
/* Now return the result to the client with a 200 status code. */
diff --git a/examples/tasks/src/tasks.c b/examples/tasks/src/tasks.c
@@ -58,8 +58,8 @@ page_handler(struct http_request *req)
*/
if (req->hdlr_extra == NULL) {
/* Grab the user argument */
- http_populate_arguments(req);
- if (!http_argument_get_string("user", &user, &len)) {
+ http_populate_get(req);
+ if (!http_argument_get_string(req, "user", &user)) {
http_response(req, 500, "ERROR\n", 6);
return (KORE_RESULT_OK);
}
@@ -87,7 +87,7 @@ page_handler(struct http_request *req)
* GET request to its channel.
*/
kore_task_run(&state->task);
- kore_task_channel_write(&state->task, user, len);
+ kore_task_channel_write(&state->task, user, strlen(user));
/*
* Tell Kore to retry us later.
@@ -142,7 +142,6 @@ page_handler(struct http_request *req)
int
post_back(struct http_request *req)
{
- u_int32_t len;
char *user;
if (req->method != HTTP_METHOD_POST) {
@@ -150,14 +149,14 @@ post_back(struct http_request *req)
return (KORE_RESULT_OK);
}
- http_populate_arguments(req);
- if (!http_argument_get_string("user", &user, &len)) {
+ http_populate_post(req);
+ if (!http_argument_get_string(req, "user", &user)) {
http_response(req, 500, NULL, 0);
return (KORE_RESULT_OK);
}
/* Simply echo the supplied user argument back. */
- http_response(req, 200, user, len);
+ http_response(req, 200, user, strlen(user));
return (KORE_RESULT_OK);
}
diff --git a/examples/upload/.gitignore b/examples/upload/.gitignore
@@ -0,0 +1,5 @@
+*.o
+.objs
+upload.so
+assets.h
+cert
diff --git a/examples/upload/conf/upload.conf b/examples/upload/conf/upload.conf
@@ -0,0 +1,19 @@
+# Placeholder configuration
+
+bind 127.0.0.1 8888
+load ./upload.so
+
+http_body_max 1024000000
+http_body_disk_offload 4096
+
+validator v_name regex ^[a-zA-Z]*$
+validator v_number regex ^[0-9]*$
+
+domain 127.0.0.1 {
+ static / page
+
+ params post / {
+ validate field1 v_name
+ validate field2 v_number
+ }
+}
diff --git a/examples/upload/src/upload.c b/examples/upload/src/upload.c
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2016 Joris Vink <joris@coders.se>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/*
+ * This example demonstrates how to properly deal with file uploads
+ * coming from a multipart form.
+ *
+ * The basics are quite trivial:
+ * 1) call http_populate_multipart_form()
+ * 2) find the file using http_file_lookup().
+ * 3) read the file data using http_file_read().
+ *
+ * In this example the contents is written to a newly created file
+ * on the server that matches the naming given by the uploader.
+ *
+ * Note that the above is probably not what you want to do in real life.
+ */
+
+#include <kore/kore.h>
+#include <kore/http.h>
+
+#include <fcntl.h>
+#include <unistd.h>
+
+int page(struct http_request *);
+
+int
+page(struct http_request *req)
+{
+ int fd;
+ struct http_file *file;
+ u_int8_t buf[BUFSIZ];
+ ssize_t ret, written;
+
+ /* Only deal with POSTs. */
+ if (req->method != HTTP_METHOD_POST) {
+ http_response(req, 405, NULL, 0);
+ return (KORE_RESULT_OK);
+ }
+
+ /* Parse the multipart data that was present. */
+ http_populate_multipart_form(req);
+
+ /* Find our file. */
+ if ((file = http_file_lookup(req, "file")) == NULL) {
+ http_response(req, 400, NULL, 0);
+ return (KORE_RESULT_OK);
+ }
+
+ /* Open dump file where we will write file contents. */
+ fd = open(file->filename, O_CREAT | O_TRUNC | O_WRONLY, 0700);
+ if (fd == -1) {
+ http_response(req, 500, NULL, 0);
+ return (KORE_RESULT_OK);
+ }
+
+ /* While we have data from http_file_read(), write it. */
+ ret = KORE_RESULT_ERROR;
+ for (;;) {
+ ret = http_file_read(file, buf, sizeof(buf));
+ if (ret == -1) {
+ kore_log(LOG_ERR, "failed to read from file");
+ http_response(req, 500, NULL, 0);
+ goto cleanup;
+ }
+
+ if (ret == 0)
+ break;
+
+ written = write(fd, buf, ret);
+ if (written == -1) {
+ kore_log(LOG_ERR,"write(%s): %s",
+ file->filename, errno_s);
+ http_response(req, 500, NULL, 0);
+ goto cleanup;
+ }
+
+ if (written != ret) {
+ kore_log(LOG_ERR, "partial write on %s",
+ file->filename);
+ http_response(req, 500, NULL, 0);
+ goto cleanup;
+ }
+ }
+
+ ret = KORE_RESULT_OK;
+ http_response(req, 200, NULL, 0);
+ kore_log(LOG_INFO, "file '%s' successfully received",
+ file->filename);
+
+cleanup:
+ if (close(fd) == -1)
+ kore_log(LOG_WARNING, "close(%s): %s", file->filename, errno_s);
+
+ if (ret == KORE_RESULT_ERROR) {
+ if (unlink("dump") == -1) {
+ kore_log(LOG_WARNING, "unlink(%s): %s",
+ file->filename, errno_s);
+ }
+ ret = KORE_RESULT_OK;
+ }
+
+ return (KORE_RESULT_OK);
+}
diff --git a/includes/http.h b/includes/http.h
@@ -33,6 +33,10 @@ extern "C" {
#define HTTP_MAX_QUERY_ARGS 20
#define HTTP_MAX_COOKIES 10
#define HTTP_REQUEST_LIMIT 1000
+#define HTTP_BODY_DISK_PATH "tmp_files"
+#define HTTP_BODY_DISK_OFFLOAD 0
+#define HTTP_BODY_PATH_MAX 256
+#define HTTP_BOUNDARY_MAX 80
#define HTTP_ARG_TYPE_RAW 0
#define HTTP_ARG_TYPE_BYTE 1
@@ -58,19 +62,13 @@ struct http_header {
struct http_arg {
char *name;
- void *value;
- u_int32_t len;
-
char *s_value;
- u_int32_t s_len;
TAILQ_ENTRY(http_arg) list;
};
-#define COPY_ARG_TYPE(v, l, t) \
+#define COPY_ARG_TYPE(v, t) \
do { \
- if (l != NULL) \
- *l = sizeof(t); \
*(t *)nout = v; \
} while (0);
@@ -81,7 +79,7 @@ struct http_arg {
nval = (type)kore_strtonum64(q->s_value, sign, &err); \
if (err != KORE_RESULT_OK) \
return (KORE_RESULT_ERROR); \
- COPY_ARG_TYPE(nval, len, type); \
+ COPY_ARG_TYPE(nval, type); \
} while (0);
#define COPY_ARG_INT(min, max, type) \
@@ -91,23 +89,13 @@ struct http_arg {
nval = kore_strtonum(q->s_value, 10, min, max, &err); \
if (err != KORE_RESULT_OK) \
return (KORE_RESULT_ERROR); \
- COPY_ARG_TYPE(nval, len, type); \
- } while (0);
-
-#define CACHE_STRING() \
- do { \
- if (q->s_value == NULL) { \
- q->s_len = q->len + 1; \
- q->s_value = kore_malloc(q->s_len); \
- kore_strlcpy(q->s_value, q->value, q->s_len); \
- } \
+ COPY_ARG_TYPE(nval, type); \
} while (0);
#define COPY_AS_INTTYPE_64(type, sign) \
do { \
if (nout == NULL) \
return (KORE_RESULT_ERROR); \
- CACHE_STRING(); \
COPY_ARG_INT64(type, sign); \
} while (0);
@@ -115,45 +103,44 @@ struct http_arg {
do { \
if (nout == NULL) \
return (KORE_RESULT_ERROR); \
- CACHE_STRING(); \
COPY_ARG_INT(min, max, type); \
} while (0);
-#define http_argument_type(r, n, so, no, l, t) \
- http_argument_get(r, n, so, no, l, t)
+#define http_argument_type(r, n, so, no, t) \
+ http_argument_get(r, n, so, no, t)
-#define http_argument_get_string(n, o, l) \
- http_argument_type(req, n, (void **)o, NULL, l, HTTP_ARG_TYPE_STRING)
+#define http_argument_get_string(r, n, o) \
+ http_argument_type(r, n, (void **)o, NULL, HTTP_ARG_TYPE_STRING)
-#define http_argument_get_byte(n, o) \
- http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_BYTE)
+#define http_argument_get_byte(r, n, o) \
+ http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_BYTE)
-#define http_argument_get_uint16(n, o) \
- http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_UINT16)
+#define http_argument_get_uint16(r, n, o) \
+ http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT16)
-#define http_argument_get_int16(n, o) \
- http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_INT16)
+#define http_argument_get_int16(r, n, o) \
+ http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT16)
-#define http_argument_get_uint32(n, o) \
- http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_UINT32)
+#define http_argument_get_uint32(r, n, o) \
+ http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT32)
-#define http_argument_get_int32(n, o) \
- http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_INT32)
+#define http_argument_get_int32(r, n, o) \
+ http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT32)
-#define http_argument_get_uint64(n, o) \
- http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_UINT64)
+#define http_argument_get_uint64(r, n, o) \
+ http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT64)
-#define http_argument_get_int64(n, o) \
- http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_INT64)
+#define http_argument_get_int64(r, n, o) \
+ http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT64)
struct http_file {
char *name;
char *filename;
-
- u_int8_t *data;
- u_int32_t len;
-
+ size_t position;
+ size_t offset;
+ size_t length;
+ struct http_request *req;
TAILQ_ENTRY(http_file) list;
};
@@ -163,20 +150,21 @@ struct http_file {
#define HTTP_METHOD_DELETE 3
#define HTTP_METHOD_HEAD 4
-#define HTTP_REQUEST_COMPLETE 0x01
-#define HTTP_REQUEST_DELETE 0x02
-#define HTTP_REQUEST_SLEEPING 0x04
-#define HTTP_REQUEST_PGSQL_QUEUE 0x10
-#define HTTP_REQUEST_EXPECT_BODY 0x20
-#define HTTP_REQUEST_RETAIN_EXTRA 0x40
-#define HTTP_REQUEST_NO_CONTENT_LENGTH 0x80
+#define HTTP_REQUEST_COMPLETE 0x0001
+#define HTTP_REQUEST_DELETE 0x0002
+#define HTTP_REQUEST_SLEEPING 0x0004
+#define HTTP_REQUEST_PGSQL_QUEUE 0x0010
+#define HTTP_REQUEST_EXPECT_BODY 0x0020
+#define HTTP_REQUEST_RETAIN_EXTRA 0x0040
+#define HTTP_REQUEST_NO_CONTENT_LENGTH 0x0080
+#define HTTP_REQUEST_AUTHED 0x0100
struct kore_task;
struct http_request {
u_int8_t method;
- u_int8_t flags;
u_int8_t fsm_state;
+ u_int16_t flags;
u_int16_t status;
u_int64_t start;
u_int64_t end;
@@ -186,10 +174,13 @@ struct http_request {
char *agent;
struct connection *owner;
struct kore_buf *http_body;
- u_int64_t content_length;
+ int http_body_fd;
+ char *http_body_path;
+ size_t http_body_length;
+ size_t http_body_offset;
+ size_t content_length;
void *hdlr_extra;
char *query_string;
- u_int8_t *multipart_body;
struct kore_module_handle *hdlr;
LIST_HEAD(, kore_task) tasks;
@@ -214,6 +205,10 @@ extern u_int64_t http_body_max;
extern u_int64_t http_hsts_enable;
extern u_int16_t http_keepalive_time;
extern u_int32_t http_request_limit;
+extern u_int64_t http_body_disk_offload;
+extern char *http_body_disk_path;
+
+void kore_accesslog(struct http_request *);
void http_init(void);
void http_process(void);
@@ -222,9 +217,8 @@ time_t http_date_to_time(char *);
void http_request_free(struct http_request *);
void http_request_sleep(struct http_request *);
void http_request_wakeup(struct http_request *);
-char *http_body_text(struct http_request *);
-void http_process_request(struct http_request *, int);
-u_int8_t *http_body_bytes(struct http_request *, u_int32_t *);
+void http_process_request(struct http_request *);
+ssize_t http_body_read(struct http_request *, void *, size_t);
void http_response(struct http_request *, int, void *, u_int32_t);
void http_response_stream(struct http_request *, int, void *,
u_int64_t, int (*cb)(struct netbuf *), void *);
@@ -240,15 +234,15 @@ int http_state_run(struct http_state *, u_int8_t,
int http_argument_urldecode(char *);
int http_header_recv(struct netbuf *);
-int http_generic_404(struct http_request *);
-int http_populate_arguments(struct http_request *);
-int http_populate_multipart_form(struct http_request *, int *);
+void http_populate_get(struct http_request *);
+void http_populate_post(struct http_request *);
+void http_populate_multipart_form(struct http_request *);
int http_argument_get(struct http_request *,
- const char *, void **, void *, u_int32_t *, int);
-int http_file_lookup(struct http_request *, const char *, char **,
- u_int8_t **, u_int32_t *);
+ const char *, void **, void *, int);
-void kore_accesslog(struct http_request *);
+void http_file_rewind(struct http_file *);
+ssize_t http_file_read(struct http_file *, void *, size_t);
+struct http_file *http_file_lookup(struct http_request *, const char *);
enum http_status_code {
HTTP_STATUS_CONTINUE = 100,
diff --git a/includes/kore.h b/includes/kore.h
@@ -322,8 +322,7 @@ struct kore_validator {
};
#endif
-#define KORE_BUF_INITIAL 128
-#define KORE_BUF_INCREMENT KORE_BUF_INITIAL
+#define KORE_BUF_INCREMENT 4096
struct kore_buf {
u_int8_t *data;
@@ -592,8 +591,9 @@ void kore_buf_free(struct kore_buf *);
struct kore_buf *kore_buf_create(u_int32_t);
void kore_buf_append(struct kore_buf *, const void *, u_int32_t);
u_int8_t *kore_buf_release(struct kore_buf *, u_int32_t *);
-void kore_buf_reset(struct kore_buf *);
+void kore_buf_reset(struct kore_buf *);
+char *kore_buf_stringify(struct kore_buf *);
void kore_buf_appendf(struct kore_buf *, const char *, ...);
void kore_buf_appendv(struct kore_buf *, const char *, va_list);
void kore_buf_appendb(struct kore_buf *, struct kore_buf *);
diff --git a/src/auth.c b/src/auth.c
@@ -77,6 +77,7 @@ kore_auth_run(struct http_request *req, struct kore_auth *auth)
switch (r) {
case KORE_RESULT_OK:
+ req->flags |= HTTP_REQUEST_AUTHED;
kore_debug("kore_auth_run() for %s successful", req->path);
/* FALLTHROUGH */
case KORE_RESULT_RETRY:
diff --git a/src/config.c b/src/config.c
@@ -65,6 +65,8 @@ static int configure_http_body_max(char **);
static int configure_http_hsts_enable(char **);
static int configure_http_keepalive_time(char **);
static int configure_http_request_limit(char **);
+static int configure_http_body_disk_offload(char **);
+static int configure_http_body_disk_path(char **);
static int configure_validator(char **);
static int configure_params(char **);
static int configure_validate(char **);
@@ -122,6 +124,8 @@ static struct {
{ "http_hsts_enable", configure_http_hsts_enable },
{ "http_keepalive_time", configure_http_keepalive_time },
{ "http_request_limit", configure_http_request_limit },
+ { "http_body_disk_offload", configure_http_body_disk_offload },
+ { "http_body_disk_path", configure_http_body_disk_path },
{ "validator", configure_validator },
{ "params", configure_params },
{ "validate", configure_validate },
@@ -546,6 +550,36 @@ configure_http_body_max(char **argv)
}
static int
+configure_http_body_disk_offload(char **argv)
+{
+ int err;
+
+ if (argv[1] == NULL)
+ return (KORE_RESULT_ERROR);
+
+ http_body_disk_offload = kore_strtonum(argv[1], 10, 0, LONG_MAX, &err);
+ if (err != KORE_RESULT_OK) {
+ printf("bad http_body_disk_offload value: %s\n", argv[1]);
+ return (KORE_RESULT_ERROR);
+ }
+
+ return (KORE_RESULT_OK);
+}
+
+static int
+configure_http_body_disk_path(char **argv)
+{
+ if (argv[1] == NULL)
+ return (KORE_RESULT_ERROR);
+
+ if (http_body_disk_path != HTTP_BODY_DISK_PATH)
+ kore_mem_free(http_body_disk_path);
+
+ http_body_disk_path = kore_strdup(argv[1]);
+ return (KORE_RESULT_OK);
+}
+
+static int
configure_http_hsts_enable(char **argv)
{
int err;
diff --git a/src/http.c b/src/http.c
@@ -19,6 +19,9 @@
#include <ctype.h>
#include <inttypes.h>
+#include <fcntl.h>
+#include <unistd.h>
+
#include "kore.h"
#include "http.h"
@@ -30,14 +33,21 @@
#include "tasks.h"
#endif
-static int http_body_recv(struct netbuf *);
-static void http_error_response(struct connection *, int);
-static void http_argument_add(struct http_request *, const char *,
- void *, u_int32_t, int);
-static void http_file_add(struct http_request *, const char *,
- const char *, u_int8_t *, u_int32_t);
-static void http_response_normal(struct http_request *,
- struct connection *, int, void *, u_int32_t);
+static int http_body_recv(struct netbuf *);
+static int http_body_rewind(struct http_request *);
+static void http_error_response(struct connection *, int);
+static void http_argument_add(struct http_request *, const char *, char *);
+static void http_response_normal(struct http_request *,
+ struct connection *, int, void *, u_int32_t);
+static void multipart_add_field(struct http_request *, struct kore_buf *,
+ const char *, const char *, const int);
+static void multipart_file_add(struct http_request *, struct kore_buf *,
+ const char *, const char *, const char *, const int);
+static int multipart_find_data(struct kore_buf *, struct kore_buf *,
+ size_t *, struct http_request *, const void *, size_t);
+static int multipart_parse_headers(struct http_request *,
+ struct kore_buf *, struct kore_buf *,
+ const char *, const int);
static struct kore_buf *header_buf;
static char http_version[32];
@@ -48,6 +58,7 @@ static struct kore_pool http_request_pool;
static struct kore_pool http_header_pool;
static struct kore_pool http_host_pool;
static struct kore_pool http_path_pool;
+static struct kore_pool http_body_path;
int http_request_count = 0;
u_int32_t http_request_limit = HTTP_REQUEST_LIMIT;
@@ -55,6 +66,8 @@ u_int64_t http_hsts_enable = HTTP_HSTS_ENABLE;
u_int16_t http_header_max = HTTP_HEADER_MAX_LEN;
u_int16_t http_keepalive_time = HTTP_KEEPALIVE_TIME;
u_int64_t http_body_max = HTTP_BODY_MAX_LEN;
+u_int64_t http_body_disk_offload = HTTP_BODY_DISK_OFFLOAD;
+char *http_body_disk_path = HTTP_BODY_DISK_PATH;
void
http_init(void)
@@ -84,6 +97,8 @@ http_init(void)
"http_host_pool", KORE_DOMAINNAME_LEN, prealloc);
kore_pool_init(&http_path_pool,
"http_path_pool", HTTP_URI_LEN, prealloc);
+ kore_pool_init(&http_body_path,
+ "http_body_path", HTTP_BODY_PATH_MAX, prealloc);
}
int
@@ -93,8 +108,9 @@ http_request_new(struct connection *c, const char *host,
{
char *p;
struct http_request *req;
+ struct kore_module_handle *hdlr;
int m, flags;
- size_t hostlen, pathlen;
+ size_t hostlen, pathlen, qsoff;
kore_debug("http_request_new(%p, %s, %s, %s, %s)", c, host,
method, path, version);
@@ -114,6 +130,21 @@ http_request_new(struct connection *c, const char *host,
return (KORE_RESULT_ERROR);
}
+ if ((p = strchr(path, '?')) != NULL) {
+ *p = '\0';
+ qsoff = p - path;
+ } else {
+ qsoff = 0;
+ }
+
+ if ((hdlr = kore_module_handler_find(host, path)) == NULL) {
+ http_error_response(c, 404);
+ return (KORE_RESULT_ERROR);
+ }
+
+ if (p != NULL)
+ *p = '?';
+
if (!strcasecmp(method, "get")) {
m = HTTP_METHOD_GET;
flags = HTTP_REQUEST_COMPLETE;
@@ -141,28 +172,35 @@ http_request_new(struct connection *c, const char *host,
req->owner = c;
req->status = 0;
req->method = m;
- req->hdlr = NULL;
+ req->hdlr = hdlr;
req->agent = NULL;
req->flags = flags;
req->fsm_state = 0;
req->http_body = NULL;
+ req->http_body_fd = -1;
req->hdlr_extra = NULL;
req->query_string = NULL;
- req->multipart_body = NULL;
+ req->http_body_length = 0;
+ req->http_body_offset = 0;
+ req->http_body_path = NULL;
if ((p = strrchr(host, ':')) != NULL)
*p = '\0';
req->host = kore_pool_get(&http_host_pool);
- (void)memcpy(req->host, host, hostlen);
+ memcpy(req->host, host, hostlen);
req->host[hostlen] = '\0';
req->path = kore_pool_get(&http_path_pool);
- (void)memcpy(req->path, path, pathlen);
+ memcpy(req->path, path, pathlen);
req->path[pathlen] = '\0';
- if ((req->query_string = strchr(req->path, '?')) != NULL)
+ if (qsoff > 0) {
+ req->query_string = req->path + qsoff;
*(req->query_string)++ = '\0';
+ } else {
+ req->query_string = NULL;
+ }
TAILQ_INIT(&(req->resp_headers));
TAILQ_INIT(&(req->req_headers));
@@ -236,63 +274,49 @@ http_process(void)
continue;
count++;
- http_process_request(req, 0);
+ http_process_request(req);
}
}
void
-http_process_request(struct http_request *req, int retry_only)
+http_process_request(struct http_request *req)
{
- struct kore_module_handle *hdlr;
- int r, (*cb)(struct http_request *);
+ int r, (*cb)(struct http_request *);
kore_debug("http_process_request: %p->%p (%s)",
req->owner, req, req->path);
- if (req->flags & HTTP_REQUEST_DELETE)
+ if (req->flags & HTTP_REQUEST_DELETE || req->hdlr == NULL)
return;
- if (req->hdlr != NULL)
- hdlr = req->hdlr;
- else
- hdlr = kore_module_handler_find(req->host, req->path);
-
req->start = kore_time_ms();
- if (hdlr == NULL) {
- r = http_generic_404(req);
- } else {
- if (req->hdlr != hdlr && hdlr->auth != NULL)
- r = kore_auth_run(req, hdlr->auth);
- else
- r = KORE_RESULT_OK;
+ if (req->hdlr->auth != NULL && !(req->flags & HTTP_REQUEST_AUTHED))
+ r = kore_auth_run(req, req->hdlr->auth);
+ else
+ r = KORE_RESULT_OK;
- switch (r) {
- case KORE_RESULT_OK:
- req->hdlr = hdlr;
- cb = hdlr->addr;
- worker->active_hdlr = hdlr;
- r = cb(req);
- worker->active_hdlr = NULL;
- break;
- case KORE_RESULT_RETRY:
- break;
- case KORE_RESULT_ERROR:
- /*
- * Set r to KORE_RESULT_OK so we can properly
- * flush the result from kore_auth_run().
- */
- r = KORE_RESULT_OK;
- break;
- default:
- fatal("kore_auth() returned unknown %d", r);
- }
+ switch (r) {
+ case KORE_RESULT_OK:
+ cb = req->hdlr->addr;
+ worker->active_hdlr = req->hdlr;
+ r = cb(req);
+ worker->active_hdlr = NULL;
+ break;
+ case KORE_RESULT_RETRY:
+ break;
+ case KORE_RESULT_ERROR:
+ /*
+ * Set r to KORE_RESULT_OK so we can properly
+ * flush the result from kore_auth_run().
+ */
+ r = KORE_RESULT_OK;
+ break;
+ default:
+ fatal("kore_auth() returned unknown %d", r);
}
req->end = kore_time_ms();
req->total += req->end - req->start;
- if (retry_only == 1 && r != KORE_RESULT_RETRY)
- fatal("http_process_request: expected RETRY but got %d", r);
-
switch (r) {
case KORE_RESULT_OK:
r = net_send_flush(req->owner);
@@ -308,7 +332,7 @@ http_process_request(struct http_request *req, int retry_only)
fatal("A page handler returned an unknown result: %d", r);
}
- if (hdlr != NULL && hdlr->dom->accesslog != -1)
+ if (req->hdlr->dom->accesslog != -1)
kore_accesslog(req);
req->flags |= HTTP_REQUEST_DELETE;
@@ -403,12 +427,8 @@ http_request_free(struct http_request *req)
TAILQ_REMOVE(&(req->arguments), q, list);
kore_mem_free(q->name);
-
- if (q->value != NULL)
- kore_mem_free(q->value);
if (q->s_value != NULL)
kore_mem_free(q->s_value);
-
kore_mem_free(q);
}
@@ -423,8 +443,17 @@ http_request_free(struct http_request *req)
if (req->http_body != NULL)
kore_buf_free(req->http_body);
- if (req->multipart_body != NULL)
- kore_mem_free(req->multipart_body);
+
+ if (req->http_body_fd != -1)
+ (void)close(req->http_body_fd);
+
+ if (req->http_body_path != NULL) {
+ if (unlink(req->http_body_path) == -1) {
+ kore_log(LOG_NOTICE, "failed to unlink %s: %s",
+ req->http_body_path, errno_s);
+ }
+ kore_pool_put(&http_body_path, req->http_body_path);
+ }
if (req->hdlr_extra != NULL &&
!(req->flags & HTTP_REQUEST_RETAIN_EXTRA))
@@ -494,11 +523,12 @@ int
http_header_recv(struct netbuf *nb)
{
size_t len;
+ ssize_t ret;
struct http_header *hdr;
struct http_request *req;
u_int64_t bytes_left;
u_int8_t *end_headers;
- int h, i, v, skip;
+ int h, i, v, skip, l;
char *request[4], *host[3], *hbuf;
char *p, *headers[HTTP_REQ_HEADER_MAX];
struct connection *c = (struct connection *)nb->owner;
@@ -519,7 +549,6 @@ http_header_recv(struct netbuf *nb)
*end_headers = '\0';
end_headers += skip;
- nb->flags |= NETBUF_FORCE_REMOVE;
len = end_headers - nb->buf;
hbuf = (char *)nb->buf;
@@ -626,9 +655,40 @@ http_header_recv(struct netbuf *nb)
return (KORE_RESULT_OK);
}
- req->http_body = kore_buf_create(req->content_length);
- kore_buf_append(req->http_body, end_headers,
- (nb->s_off - len));
+ req->http_body_length = req->content_length;
+
+ if (http_body_disk_offload > 0 &&
+ req->content_length > http_body_disk_offload) {
+ req->http_body_path = kore_pool_get(&http_body_path);
+ l = snprintf(req->http_body_path, HTTP_BODY_PATH_MAX,
+ "%s/http_body.XXXXXX", http_body_disk_path);
+ if (l == -1 || (size_t)l >= HTTP_BODY_PATH_MAX) {
+ req->flags |= HTTP_REQUEST_DELETE;
+ http_error_response(req->owner, 500);
+ return (KORE_RESULT_ERROR);
+ }
+
+ req->http_body = NULL;
+ req->http_body_fd = mkstemp(req->http_body_path);
+ if (req->http_body_fd == -1) {
+ req->flags |= HTTP_REQUEST_DELETE;
+ http_error_response(req->owner, 500);
+ return (KORE_RESULT_OK);
+ }
+
+ ret = write(req->http_body_fd,
+ end_headers, (nb->s_off - len));
+ if (ret == -1 || (size_t)ret != (nb->s_off - len)) {
+ req->flags |= HTTP_REQUEST_DELETE;
+ http_error_response(req->owner, 500);
+ return (KORE_RESULT_OK);
+ }
+ } else {
+ req->http_body_fd = -1;
+ req->http_body = kore_buf_create(req->content_length);
+ kore_buf_append(req->http_body, end_headers,
+ (nb->s_off - len));
+ }
bytes_left = req->content_length - (nb->s_off - len);
if (bytes_left > 0) {
@@ -642,6 +702,11 @@ http_header_recv(struct netbuf *nb)
} else if (bytes_left == 0) {
req->flags |= HTTP_REQUEST_COMPLETE;
req->flags &= ~HTTP_REQUEST_EXPECT_BODY;
+ if (!http_body_rewind(req)) {
+ req->flags |= HTTP_REQUEST_DELETE;
+ http_error_response(req->owner, 500);
+ return (KORE_RESULT_OK);
+ }
} else {
http_error_response(req->owner, 500);
}
@@ -651,92 +716,48 @@ http_header_recv(struct netbuf *nb)
}
int
-http_populate_arguments(struct http_request *req)
-{
- u_int32_t len;
- int i, v, c, count;
- char *query, *args[HTTP_MAX_QUERY_ARGS], *val[3];
-
- if (req->method == HTTP_METHOD_POST) {
- if (req->http_body == NULL)
- return (0);
- query = http_body_text(req);
- } else {
- if (req->query_string == NULL)
- return (0);
- query = kore_strdup(req->query_string);
- }
-
- count = 0;
- v = kore_split_string(query, "&", args, HTTP_MAX_QUERY_ARGS);
- for (i = 0; i < v; i++) {
- c = kore_split_string(args[i], "=", val, 3);
- if (c != 1 && c != 2) {
- kore_debug("malformed query argument");
- continue;
- }
-
- if (val[1] != NULL) {
- len = strlen(val[1]);
- http_argument_add(req, val[0], val[1],
- len, HTTP_ARG_TYPE_STRING);
- count++;
- }
- }
-
- kore_mem_free(query);
- return (count);
-}
-
-int
http_argument_get(struct http_request *req, const char *name,
- void **out, void *nout, u_int32_t *len, int type)
+ void **out, void *nout, int type)
{
struct http_arg *q;
- if (len != NULL)
- *len = 0;
-
TAILQ_FOREACH(q, &(req->arguments), list) {
- if (!strcmp(q->name, name)) {
- switch (type) {
- case HTTP_ARG_TYPE_RAW:
- if (len != NULL)
- *len = q->len;
- *out = q->value;
- return (KORE_RESULT_OK);
- case HTTP_ARG_TYPE_BYTE:
- COPY_ARG_TYPE(*(u_int8_t *)q->value,
- len, u_int8_t);
- return (KORE_RESULT_OK);
- case HTTP_ARG_TYPE_INT16:
- COPY_AS_INTTYPE(SHRT_MIN, SHRT_MAX, int16_t);
- return (KORE_RESULT_OK);
- case HTTP_ARG_TYPE_UINT16:
- COPY_AS_INTTYPE(0, USHRT_MAX, u_int16_t);
- return (KORE_RESULT_OK);
- case HTTP_ARG_TYPE_INT32:
- COPY_AS_INTTYPE(INT_MIN, INT_MAX, int32_t);
- return (KORE_RESULT_OK);
- case HTTP_ARG_TYPE_UINT32:
- COPY_AS_INTTYPE(0, UINT_MAX, u_int32_t);
- return (KORE_RESULT_OK);
- case HTTP_ARG_TYPE_INT64:
- COPY_AS_INTTYPE_64(int64_t, 1);
- return (KORE_RESULT_OK);
- case HTTP_ARG_TYPE_UINT64:
- COPY_AS_INTTYPE_64(u_int64_t, 0);
- return (KORE_RESULT_OK);
- case HTTP_ARG_TYPE_STRING:
- CACHE_STRING();
- *out = q->s_value;
- if (len != NULL)
- *len = q->s_len - 1;
- return (KORE_RESULT_OK);
- default:
- return (KORE_RESULT_ERROR);
- }
+ if (strcmp(q->name, name))
+ continue;
+
+ switch (type) {
+ case HTTP_ARG_TYPE_RAW:
+ *out = q->s_value;
+ return (KORE_RESULT_OK);
+ case HTTP_ARG_TYPE_BYTE:
+ COPY_ARG_TYPE(*(u_int8_t *)q->s_value, u_int8_t);
+ return (KORE_RESULT_OK);
+ case HTTP_ARG_TYPE_INT16:
+ COPY_AS_INTTYPE(SHRT_MIN, SHRT_MAX, int16_t);
+ return (KORE_RESULT_OK);
+ case HTTP_ARG_TYPE_UINT16:
+ COPY_AS_INTTYPE(0, USHRT_MAX, u_int16_t);
+ return (KORE_RESULT_OK);
+ case HTTP_ARG_TYPE_INT32:
+ COPY_AS_INTTYPE(INT_MIN, INT_MAX, int32_t);
+ return (KORE_RESULT_OK);
+ case HTTP_ARG_TYPE_UINT32:
+ COPY_AS_INTTYPE(0, UINT_MAX, u_int32_t);
+ return (KORE_RESULT_OK);
+ case HTTP_ARG_TYPE_INT64:
+ COPY_AS_INTTYPE_64(int64_t, 1);
+ return (KORE_RESULT_OK);
+ case HTTP_ARG_TYPE_UINT64:
+ COPY_AS_INTTYPE_64(u_int64_t, 0);
+ return (KORE_RESULT_OK);
+ case HTTP_ARG_TYPE_STRING:
+ *out = q->s_value;
+ return (KORE_RESULT_OK);
+ default:
+ break;
}
+
+ return (KORE_RESULT_ERROR);
}
return (KORE_RESULT_ERROR);
@@ -790,210 +811,237 @@ http_argument_urldecode(char *arg)
return (KORE_RESULT_OK);
}
-int
-http_file_lookup(struct http_request *req, const char *name, char **fname,
- u_int8_t **data, u_int32_t *len)
+struct http_file *
+http_file_lookup(struct http_request *req, const char *name)
{
struct http_file *f;
TAILQ_FOREACH(f, &(req->files), list) {
- if (!strcmp(f->name, name)) {
- *len = f->len;
- *data = f->data;
- *fname = f->filename;
- return (KORE_RESULT_OK);
- }
+ if (!strcmp(f->name, name))
+ return (f);
}
- return (KORE_RESULT_ERROR);
+ return (NULL);
}
-int
-http_populate_multipart_form(struct http_request *req, int *v)
+ssize_t
+http_file_read(struct http_file *file, void *buf, size_t len)
{
- int h, i, c, l;
- u_int32_t blen, slen;
- u_int8_t *s, *end, *e, *end_headers, *data;
- char *d, *val, *type, *boundary, *fname;
- char *headers[5], *args[5], *opt[5], *name;
-
- *v = 0;
-
- if (req->method != HTTP_METHOD_POST)
- return (KORE_RESULT_ERROR);
-
- if (!http_request_header(req, "content-type", &type))
- return (KORE_RESULT_ERROR);
-
- h = kore_split_string(type, ";", args, 3);
- if (h != 2)
- return (KORE_RESULT_ERROR);
-
- if (strcasecmp(args[0], "multipart/form-data"))
- return (KORE_RESULT_ERROR);
-
- if ((val = strchr(args[1], '=')) == NULL)
- return (KORE_RESULT_ERROR);
+ ssize_t ret;
+ size_t toread, off;
+
+ if (file->length < file->offset)
+ return (-1);
+ if ((file->offset + len) < file->offset)
+ return (-1);
+ if ((file->position + file->offset) < file->position)
+ return (-1);
+
+ off = file->position + file->offset;
+ toread = MIN(len, (file->length - file->offset));
+ if (toread <= 0)
+ return (0);
+
+ if (file->req->http_body_fd != -1) {
+ if (lseek(file->req->http_body_fd, off, SEEK_SET) == -1) {
+ kore_log(LOG_ERR, "http_file_read: lseek(%s): %s",
+ file->req->http_body_path, errno_s);
+ return (-1);
+ }
- val++;
- slen = strlen(val);
- boundary = kore_malloc(slen + 3);
- if (!kore_snprintf(boundary, slen + 3, &l, "--%s", val)) {
- kore_mem_free(boundary);
- return (KORE_RESULT_ERROR);
+ for (;;) {
+ ret = read(file->req->http_body_fd, buf, toread);
+ if (ret == -1) {
+ if (errno == EINTR)
+ continue;
+ kore_log(LOG_ERR, "failed to read %s: %s",
+ file->req->http_body_path, errno_s);
+ return (-1);
+ }
+ if (ret == 0)
+ return (0);
+ break;
+ }
+ } else if (file->req->http_body != NULL) {
+ if (off > file->req->http_body->length)
+ return (0);
+ memcpy(buf, file->req->http_body->data + off, toread);
+ ret = toread;
+ } else {
+ kore_log(LOG_ERR, "http_file_read: called without body");
+ return (-1);
}
- slen = l;
+ file->offset += (size_t)ret;
+ return (ret);
+}
- req->multipart_body = http_body_bytes(req, &blen);
- if (slen < 3 || blen < (slen * 2)) {
- kore_mem_free(boundary);
- return (KORE_RESULT_ERROR);
- }
+void
+http_file_rewind(struct http_file *file)
+{
+ file->offset = 0;
+}
- end = req->multipart_body + blen - 2;
- if (end < req->multipart_body || (end - 2) < req->multipart_body) {
- kore_mem_free(boundary);
- return (KORE_RESULT_ERROR);
- }
+void
+http_populate_post(struct http_request *req)
+{
+ ssize_t ret;
+ struct kore_buf *buf;
+ char data[BUFSIZ], *val[3], *string, *p;
- if (memcmp((end - slen - 2), boundary, slen) ||
- memcmp((end - 2), "--", 2)) {
- kore_mem_free(boundary);
- return (KORE_RESULT_ERROR);
- }
+ if (req->method != HTTP_METHOD_POST)
+ return;
- s = req->multipart_body + slen + 2;
- while (s < end) {
- e = kore_mem_find(s, end - s, boundary, slen);
- if (e == NULL) {
- kore_mem_free(boundary);
- return (KORE_RESULT_ERROR);
- }
+ buf = kore_buf_create(128);
+ for (;;) {
+ ret = http_body_read(req, data, sizeof(data));
+ if (ret == -1)
+ goto out;
+ if (ret == 0)
+ break;
- *(e - 2) = '\0';
- end_headers = kore_mem_find(s, (e - 2) - s, "\r\n\r\n", 4);
- if (end_headers == NULL) {
- kore_mem_free(boundary);
- return (KORE_RESULT_ERROR);
+ if ((p = kore_mem_find(data, ret, "&", 1)) == NULL) {
+ kore_buf_append(buf, data, ret);
+ continue;
+ } else {
+ kore_buf_append(buf, data, p - data);
+ ret -= (p - data);
}
- *end_headers = '\0';
- data = end_headers + 4;
-
- h = kore_split_string((char *)s, "\r\n", headers, 5);
- for (i = 0; i < h; i++) {
- c = kore_split_string(headers[i], ":", args, 5);
- if (c != 2)
- continue;
-
- /* Ignore other headers for now. */
- if (strcasecmp(args[0], "content-disposition"))
- continue;
+ string = kore_buf_stringify(buf);
+ kore_split_string(string, "=", val, 3);
+ if (val[0] != NULL && val[1] != NULL)
+ http_argument_add(req, val[0], val[1]);
- for (d = args[1]; isspace(*d); d++)
- ;
+ kore_buf_reset(buf);
+ if (ret > 1)
+ kore_buf_append(buf, p + 1, ret - 1);
+ }
- c = kore_split_string(d, ";", opt, 5);
- if (c < 2)
- continue;
+ if (buf->offset != 0) {
+ string = kore_buf_stringify(buf);
+ kore_split_string(string, "=", val, 3);
+ if (val[0] != NULL && val[1] != NULL)
+ http_argument_add(req, val[0], val[1]);
+ }
- if (strcasecmp(opt[0], "form-data"))
- continue;
+out:
+ kore_buf_free(buf);
+}
- if ((val = strchr(opt[1], '=')) == NULL)
- continue;
- if (strlen(val) < 3)
- continue;
+void
+http_populate_get(struct http_request *req)
+{
+ int i, v;
+ char *query, *args[HTTP_MAX_QUERY_ARGS], *val[3];
- val++;
- kore_strip_chars(val, '"', &name);
+ if (req->method != HTTP_METHOD_GET || req->query_string == NULL)
+ return;
- if (opt[2] == NULL) {
- *v = *v + 1;
- http_argument_add(req, name,
- data, (e - 2) - data, HTTP_ARG_TYPE_STRING);
- kore_mem_free(name);
- continue;
- }
+ query = kore_strdup(req->query_string);
+ v = kore_split_string(query, "&", args, HTTP_MAX_QUERY_ARGS);
+ for (i = 0; i < v; i++) {
+ kore_split_string(args[i], "=", val, 3);
+ if (val[0] != NULL && val[1] != NULL)
+ http_argument_add(req, val[0], val[1]);
+ }
- for (d = opt[2]; isspace(*d); d++)
- ;
+ kore_mem_free(query);
+}
- if (!strncasecmp(d, "filename=", 9)) {
- if ((val = strchr(d, '=')) == NULL) {
- kore_mem_free(name);
- continue;
- }
-
- val++;
- kore_strip_chars(val, '"', &fname);
- if (strlen(fname) > 0) {
- *v = *v + 1;
- http_file_add(req, name, fname,
- data, (e - 2) - data);
- }
-
- kore_mem_free(fname);
- } else {
- kore_debug("got unknown: %s", opt[2]);
- }
+void
+http_populate_multipart_form(struct http_request *req)
+{
+ int h, blen, count;
+ struct kore_buf *in, *out;
+ char *type, *val, *args[3];
+ char boundary[HTTP_BOUNDARY_MAX];
- kore_mem_free(name);
- }
+ if (req->method != HTTP_METHOD_POST)
+ return;
- s = e + slen + 2;
- }
+ if (!http_request_header(req, "content-type", &type))
+ return;
- kore_mem_free(boundary);
+ h = kore_split_string(type, ";", args, 3);
+ if (h != 2)
+ return;
- return (KORE_RESULT_OK);
-}
+ if (strcasecmp(args[0], "multipart/form-data"))
+ return;
-int
-http_generic_404(struct http_request *req)
-{
- kore_debug("http_generic_404(%s, %d, %s)",
- req->host, req->method, req->path);
+ if ((val = strchr(args[1], '=')) == NULL)
+ return;
- http_response(req, 404, NULL, 0);
+ val++;
+ blen = snprintf(boundary, sizeof(boundary), "--%s", val);
+ if (blen == -1 || (size_t)blen >= sizeof(boundary))
+ return;
- return (KORE_RESULT_OK);
-}
+ in = kore_buf_create(128);
+ out = kore_buf_create(128);
-char *
-http_body_text(struct http_request *req)
-{
- u_int32_t len;
- u_int8_t *data;
- char *text;
+ if (!multipart_find_data(in, NULL, NULL, req, boundary, blen))
+ goto cleanup;
- if (req->http_body == NULL)
- return (NULL);
+ count = 0;
- data = kore_buf_release(req->http_body, &len);
- req->http_body = NULL;
- len++;
+ for (;;) {
+ if (!multipart_find_data(in, NULL, NULL, req, "\r\n", 2))
+ break;
+ if (in->offset < 4 && req->http_body_length == 0)
+ break;
+ if (!multipart_find_data(in, out, NULL, req, "\r\n\r\n", 4))
+ break;
+ if (!multipart_parse_headers(req, in, out, boundary, blen))
+ break;
- text = kore_malloc(len);
- kore_strlcpy(text, (char *)data, len);
- kore_mem_free(data);
+ kore_buf_reset(out);
+ }
- return (text);
+cleanup:
+ kore_buf_free(in);
+ kore_buf_free(out);
}
-u_int8_t *
-http_body_bytes(struct http_request *req, u_int32_t *len)
+ssize_t
+http_body_read(struct http_request *req, void *out, size_t len)
{
- u_int8_t *data;
-
- if (req->http_body == NULL)
- return (NULL);
+ ssize_t ret;
+ size_t toread;
+
+ toread = MIN(req->http_body_length, len);
+ if (toread <= 0)
+ return (0);
+
+ if (req->http_body_fd != -1) {
+ for (;;) {
+ ret = read(req->http_body_fd, out, toread);
+ if (ret == -1) {
+ if (errno == EINTR)
+ continue;
+ kore_log(LOG_ERR, "failed to read %s: %s",
+ req->http_body_path, errno_s);
+ return (-1);
+ }
+ if (ret == 0)
+ return (0);
+ break;
+ }
+ } else if (req->http_body != NULL) {
+ memcpy(out,
+ (req->http_body->data + req->http_body->offset), toread);
+ req->http_body->offset += toread;
+ ret = toread;
+ } else {
+ kore_log(LOG_ERR, "http_body_read: called without body");
+ return (-1);
+ }
- data = kore_buf_release(req->http_body, len);
- req->http_body = NULL;
+ req->http_body_length -= (size_t)ret;
+ req->http_body_offset += (size_t)ret;
- return (data);
+ return (ret);
}
int
@@ -1035,74 +1083,254 @@ http_state_run(struct http_state *states, u_int8_t elm,
return (KORE_RESULT_OK);
}
-static void
-http_argument_add(struct http_request *req, const char *name,
- void *value, u_int32_t len, int type)
+static int
+multipart_find_data(struct kore_buf *in, struct kore_buf *out,
+ size_t *olen, struct http_request *req, const void *needle, size_t len)
{
- struct http_arg *q;
- struct kore_handler_params *p;
+ ssize_t ret;
+ size_t left;
+ u_int8_t *p, first, data[4096];
+
+ if (olen != NULL)
+ *olen = 0;
+
+ first = *(const u_int8_t *)needle;
+ for (;;) {
+ if (in->offset < len) {
+ ret = http_body_read(req, data, sizeof(data));
+ if (ret == -1)
+ return (KORE_RESULT_ERROR);
+ if (ret == 0)
+ return (KORE_RESULT_ERROR);
- if (len == 0 || value == NULL) {
- kore_debug("http_argument_add: with NULL value");
- return;
+ kore_buf_append(in, data, ret);
+ continue;
+ }
+
+ p = kore_mem_find(in->data, in->offset, &first, 1);
+ if (p == NULL) {
+ if (out != NULL)
+ kore_buf_append(out, in->data, in->offset);
+ if (olen != NULL)
+ *olen += in->offset;
+ kore_buf_reset(in);
+ continue;
+ }
+
+ left = in->offset - (p - in->data);
+ if (left < len) {
+ if (out != NULL)
+ kore_buf_append(out, in->data, (p - in->data));
+ if (olen != NULL)
+ *olen += (p - in->data);
+ memmove(in->data, p, left);
+ in->offset = left;
+ continue;
+ }
+
+ if (!memcmp(p, needle, len)) {
+ if (out != NULL)
+ kore_buf_append(out, in->data, p - in->data);
+ if (olen != NULL)
+ *olen += (p - in->data);
+
+ in->offset = left - len;
+ if (in->offset > 0)
+ memmove(in->data, p + len, in->offset);
+ return (KORE_RESULT_OK);
+ }
+
+ if (out != NULL)
+ kore_buf_append(out, in->data, (p - in->data) + 1);
+ if (olen != NULL)
+ *olen += (p - in->data) + 1;
+
+ in->offset = left - 1;
+ if (in->offset > 0)
+ memmove(in->data, p + 1, in->offset);
}
- TAILQ_FOREACH(p, &(req->hdlr->params), list) {
- if (p->method != req->method)
+ return (KORE_RESULT_ERROR);
+}
+
+static int
+multipart_parse_headers(struct http_request *req, struct kore_buf *in,
+ struct kore_buf *hbuf, const char *boundary, const int blen)
+{
+ int h, c, i;
+ char *headers[5], *args[5], *opt[5];
+ char *d, *val, *name, *fname, *string;
+
+ string = kore_buf_stringify(hbuf);
+ h = kore_split_string(string, "\r\n", headers, 5);
+ for (i = 0; i < h; i++) {
+ c = kore_split_string(headers[i], ":", args, 5);
+ if (c != 2)
continue;
- if (!strcmp(p->name, name)) {
- if (type == HTTP_ARG_TYPE_STRING) {
- http_argument_urldecode(value);
- len = strlen(value);
- }
+ /* Ignore other headers for now. */
+ if (strcasecmp(args[0], "content-disposition"))
+ continue;
+
+ for (d = args[1]; isspace(*d); d++)
+ ;
+
+ c = kore_split_string(d, ";", opt, 5);
+ if (c < 2)
+ continue;
+
+ if (strcasecmp(opt[0], "form-data"))
+ continue;
+
+ if ((val = strchr(opt[1], '=')) == NULL)
+ continue;
+ if (strlen(val) < 3)
+ continue;
+
+ val++;
+ kore_strip_chars(val, '"', &name);
+
+ if (opt[2] == NULL) {
+ multipart_add_field(req, in, name, boundary, blen);
+ kore_mem_free(name);
+ continue;
+ }
+
+ for (d = opt[2]; isspace(*d); d++)
+ ;
- if (kore_validator_check(req, p->validator, value)) {
- q = kore_malloc(sizeof(struct http_arg));
- q->len = len;
- q->s_value = NULL;
- q->name = kore_strdup(name);
- q->value = kore_malloc(len);
- memcpy(q->value, value, len);
- TAILQ_INSERT_TAIL(&(req->arguments), q, list);
+ if (!strncasecmp(d, "filename=", 9)) {
+ if ((val = strchr(d, '=')) == NULL) {
+ kore_mem_free(name);
+ continue;
}
- return;
+ val++;
+ kore_strip_chars(val, '"', &fname);
+ if (strlen(fname) > 0) {
+ multipart_file_add(req,
+ in, name, fname, boundary, blen);
+ }
+ kore_mem_free(fname);
+ } else {
+ kore_debug("got unknown: %s", opt[2]);
}
+
+ kore_mem_free(name);
+ }
+
+ return (KORE_RESULT_OK);
+}
+
+static void
+multipart_add_field(struct http_request *req, struct kore_buf *in,
+ const char *name, const char *boundary, const int blen)
+{
+ struct kore_buf *data;
+ char *string;
+
+ data = kore_buf_create(128);
+
+ if (!multipart_find_data(in, data, NULL, req, boundary, blen)) {
+ kore_buf_free(data);
+ return;
}
+
+ if (data->offset < 3) {
+ kore_buf_free(data);
+ return;
+ }
+
+ data->offset -= 2;
+ string = kore_buf_stringify(data);
+ http_argument_add(req, name, string);
+ kore_buf_free(data);
}
static void
-http_file_add(struct http_request *req, const char *name, const char *filename,
- u_int8_t *data, u_int32_t len)
+multipart_file_add(struct http_request *req, struct kore_buf *in,
+ const char *name, const char *fname, const char *boundary, const int blen)
{
struct http_file *f;
+ size_t position, len;
+
+ position= req->http_body_offset - in->offset;
+ if (!multipart_find_data(in, NULL, &len, req, boundary, blen))
+ return;
+
+ if (len < 3)
+ return;
+ len -= 2;
f = kore_malloc(sizeof(struct http_file));
- f->len = len;
- f->data = data;
+ f->req = req;
+ f->length = len;
+ f->position = position;
f->name = kore_strdup(name);
- f->filename = kore_strdup(filename);
+ f->filename = kore_strdup(fname);
TAILQ_INSERT_TAIL(&(req->files), f, list);
}
+static void
+http_argument_add(struct http_request *req, const char *name, char *value)
+{
+ struct http_arg *q;
+ struct kore_handler_params *p;
+
+ TAILQ_FOREACH(p, &(req->hdlr->params), list) {
+ if (p->method != req->method)
+ continue;
+ if (strcmp(p->name, name))
+ continue;
+
+ http_argument_urldecode(value);
+ if (!kore_validator_check(req, p->validator, value))
+ break;
+
+ q = kore_malloc(sizeof(struct http_arg));
+ q->name = kore_strdup(name);
+ q->s_value = kore_strdup(value);
+ TAILQ_INSERT_TAIL(&(req->arguments), q, list);
+ break;
+ }
+}
+
static int
http_body_recv(struct netbuf *nb)
{
+ ssize_t ret;
u_int64_t bytes_left;
struct http_request *req = (struct http_request *)nb->extra;
- kore_buf_append(req->http_body, nb->buf, nb->s_off);
+ if (req->http_body_fd != -1) {
+ ret = write(req->http_body_fd, nb->buf, nb->s_off);
+ if (ret == -1 || (size_t)ret != nb->s_off)
+ fatal("failed to write");
+ } else if (req->http_body != NULL) {
+ kore_buf_append(req->http_body, nb->buf, nb->s_off);
+ } else {
+ req->flags |= HTTP_REQUEST_DELETE;
+ http_error_response(req->owner, 500);
+ return (KORE_RESULT_OK);
+ }
- if (req->http_body->offset == req->content_length) {
+ req->content_length -= nb->s_off;
+
+ if (req->content_length == 0) {
nb->extra = NULL;
+ http_request_wakeup(req);
req->flags |= HTTP_REQUEST_COMPLETE;
req->flags &= ~HTTP_REQUEST_EXPECT_BODY;
- http_request_wakeup(req);
+ req->content_length = req->http_body_length;
+ if (!http_body_rewind(req)) {
+ req->flags |= HTTP_REQUEST_DELETE;
+ http_error_response(req->owner, 500);
+ return (KORE_RESULT_OK);
+ }
net_recv_reset(nb->owner, http_header_max, http_header_recv);
} else {
- bytes_left = req->content_length - req->http_body->offset;
+ bytes_left = req->content_length;
net_recv_reset(nb->owner,
MIN(bytes_left, NETBUF_SEND_PAYLOAD_MAX),
http_body_recv);
@@ -1111,6 +1339,22 @@ http_body_recv(struct netbuf *nb)
return (KORE_RESULT_OK);
}
+static int
+http_body_rewind(struct http_request *req)
+{
+ if (req->http_body_fd != -1) {
+ if (lseek(req->http_body_fd, 0, SEEK_SET) == -1) {
+ kore_log(LOG_ERR, "lseek(%s) failed: %s",
+ req->http_body_path, errno_s);
+ return (KORE_RESULT_ERROR);
+ }
+ } else {
+ kore_buf_reset(req->http_body);
+ }
+
+ return (KORE_RESULT_OK);
+}
+
static void
http_error_response(struct connection *c, int status)
{
diff --git a/src/kore.c b/src/kore.c
@@ -15,6 +15,8 @@
*/
#include <sys/types.h>
+#include <sys/stat.h>
+
#include <sys/socket.h>
#include <sys/resource.h>
@@ -23,6 +25,10 @@
#include "kore.h"
+#if !defined(KORE_NO_HTTP)
+#include "http.h"
+#endif
+
volatile sig_atomic_t sig_recv;
struct listener_head listeners;
@@ -161,6 +167,13 @@ main(int argc, char *argv[])
#if !defined(KORE_NO_HTTP)
kore_accesslog_init();
+ if (http_body_disk_offload > 0) {
+ if (mkdir(http_body_disk_path, 0700) == -1 && errno != EEXIST) {
+ printf("can't create http_body_disk_path '%s': %s\n",
+ http_body_disk_path, errno_s);
+ return (KORE_RESULT_ERROR);
+ }
+ }
#endif
sig_recv = 0;
