kore

An easy to use, scalable and secure web application framework for writing web APIs in C.
Commits | Files | Refs | README | LICENSE | git clone https://git.kore.io/kore.git

commit fcb86ddb8bb96d1961d0970e311ad5aaeb4bbe4c
parent 96641d3caa05f350286dd9b9ea343382a0ed01a2
Author: Joris Vink <joris@coders.se>
Date:   Mon, 18 Jan 2016 11:30:22 +0100

Massive rework of HTTP layer.

This commit is a flag day, your old modules will almost certainly
need to be updated in order to build properly with these changes.

Summary of changes:

- Offload HTTP bodies to disk if they are large (inspired by #100).
  (disabled by default)
- The http_argument_get* macros now takes an explicit http_request parameter.
- Kore will now throw 404 errors almost immediately after an HTTP request
  has come in instead of waiting until all data has arrived.

API changes:

- http_argument_get* macros now require an explicit http_request parameter.
  (no more magic invokations).
- http_generic_404() is gone
- http_populate_arguments() is gone
- http_body_bytes() is gone
- http_body_text() is gone
- http_body_read() has been added
- http_populate_post() has been added
- http_populate_get() has been added
- http_file_read() has been added
- http_file_rewind() has been added
- http_file_lookup() no longer takes name, fname, data and len parameters.
- http_file_lookup() now returns a struct http_file pointer.
- http_populate_multipart_form() no longer takes an secondary parameter.

New configuration options:

- http_body_disk_offload:
	Number of bytes after which Kore will offload the HTTP body to
	disk instead of retaining it in memory. If 0 this feature is
	disabled. (Default: 0)

- http_body_disk_path:
	The path where Kore will store temporary HTTP body files.
	(this directory does not get created if http_body_disk_offload is 0).

New example:

The upload example has been added, demonstrating how to deal with file
uploads from a multipart form.

Diffstat:
conf/kore.conf.example | 12++++++++++++
examples/generic/conf/generic.conf | 3+++
examples/generic/src/example.c | 41+++++++++++++++++++++++++++--------------
examples/integers/src/check_integers.c | 16++++++++--------
examples/json_yajl/src/json_yajl.c | 46+++++++++++++++++++++++++++++++++++++++-------
examples/ktunnel/src/ktunnel.c | 6+++---
examples/parameters/src/parameters.c | 19++++---------------
examples/tasks/src/tasks.c | 13++++++-------
examples/upload/.gitignore | 5+++++
examples/upload/conf/upload.conf | 19+++++++++++++++++++
examples/upload/src/upload.c | 117+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
includes/http.h | 118++++++++++++++++++++++++++++++++++++++-----------------------------------------
includes/kore.h | 6+++---
src/auth.c | 1+
src/config.c | 34++++++++++++++++++++++++++++++++++
src/http.c | 922++++++++++++++++++++++++++++++++++++++++++++++++++-----------------------------
src/kore.c | 13+++++++++++++
17 files changed, 933 insertions(+), 458 deletions(-)

diff --git a/conf/kore.conf.example b/conf/kore.conf.example @@ -61,6 +61,16 @@ workers 4 # http_body_max Maximum size of an HTTP body (in bytes). # If set to 0 disallows requests with a body # all together. +# +# http_body_disk_offload Number of bytes after which Kore will use +# a temporary file to hold the HTTP body +# instead of holding it in memory. If set to +# 0 no disk offloading will be done. This is +# turned off by default. +# +# http_body_disk_path Path where Kore will store any temporary +# HTTP body files. +# # http_keepalive_time Maximum seconds an HTTP connection can be # kept alive by the browser. # (Set to 0 to disable keepalive completely). @@ -76,6 +86,8 @@ workers 4 #http_keepalive_time 0 #http_hsts_enable 31536000 #http_request_limit 1000 +#http_body_disk_offload 0 +#http_body_disk_path tmp_files # Websocket specific settings. # websocket_maxframe Specifies the maximum frame size we can receive diff --git a/examples/generic/conf/generic.conf b/examples/generic/conf/generic.conf @@ -5,6 +5,9 @@ load ./generic.so example_load tls_dhparam dh2048.pem +http_body_max 1024000000 +http_body_disk_offload 1024000 + validator v_example function v_example_func validator v_regex regex ^/test/[a-z]*$ validator v_number regex ^[0-9]*$ diff --git a/examples/generic/src/example.c b/examples/generic/src/example.c @@ -17,6 +17,12 @@ #include <kore/kore.h> #include <kore/http.h> +#include <openssl/sha.h> + +#include <fcntl.h> +#include <stdlib.h> +#include <unistd.h> + #include "assets.h" int example_load(int); @@ -132,9 +138,9 @@ serve_b64test(struct http_request *req) int serve_file_upload(struct http_request *req) { - int r; u_int8_t *d; struct kore_buf *b; + struct http_file *f; u_int32_t len; char *name, buf[BUFSIZ]; @@ -142,16 +148,20 @@ serve_file_upload(struct http_request *req) kore_buf_append(b, asset_upload_html, asset_len_upload_html); if (req->method == HTTP_METHOD_POST) { - http_populate_multipart_form(req, &r); - if (http_argument_get_string("firstname", &name, &len)) { - kore_buf_replace_string(b, "$firstname$", name, len); + if (req->http_body_fd != -1) + kore_log(LOG_NOTICE, "file is on disk"); + + http_populate_multipart_form(req); + if (http_argument_get_string(req, "firstname", &name)) { + kore_buf_replace_string(b, "$firstname$", + name, strlen(name)); } else { kore_buf_replace_string(b, "$firstname$", NULL, 0); } - if (http_file_lookup(req, "file", &name, &d, &len)) { + if ((f = http_file_lookup(req, "file")) != NULL) { (void)snprintf(buf, sizeof(buf), - "%s is %d bytes", name, len); + "%s is %ld bytes", f->filename, f->length); kore_buf_replace_string(b, "$upload$", buf, strlen(buf)); } else { @@ -232,7 +242,10 @@ serve_params_test(struct http_request *req) int r, i; char *test, name[10]; - http_populate_arguments(req); + if (req->method == HTTP_METHOD_GET) + http_populate_get(req); + else if (req->method == HTTP_METHOD_POST) + http_populate_post(req); b = kore_buf_create(asset_len_params_html); kore_buf_append(b, asset_params_html, asset_len_params_html); @@ -240,14 +253,14 @@ serve_params_test(struct http_request *req) /* * The GET parameters will be filtered out on POST. */ - if (http_argument_get_string("arg1", &test, &len)) { - kore_buf_replace_string(b, "$arg1$", test, len); + if (http_argument_get_string(req, "arg1", &test)) { + kore_buf_replace_string(b, "$arg1$", test, strlen(test)); } else { kore_buf_replace_string(b, "$arg1$", NULL, 0); } - if (http_argument_get_string("arg2", &test, &len)) { - kore_buf_replace_string(b, "$arg2$", test, len); + if (http_argument_get_string(req, "arg2", &test)) { + kore_buf_replace_string(b, "$arg2$", test, strlen(test)); } else { kore_buf_replace_string(b, "$arg2$", NULL, 0); } @@ -257,7 +270,7 @@ serve_params_test(struct http_request *req) kore_buf_replace_string(b, "$test2$", NULL, 0); kore_buf_replace_string(b, "$test3$", NULL, 0); - if (http_argument_get_uint16("id", &r)) + if (http_argument_get_uint16(req, "id", &r)) kore_log(LOG_NOTICE, "id: %d", r); else kore_log(LOG_NOTICE, "No id set"); @@ -272,9 +285,9 @@ serve_params_test(struct http_request *req) for (i = 1; i < 4; i++) { (void)snprintf(name, sizeof(name), "test%d", i); - if (http_argument_get_string(name, &test, &len)) { + if (http_argument_get_string(req, name, &test)) { (void)snprintf(name, sizeof(name), "$test%d$", i); - kore_buf_replace_string(b, name, test, len); + kore_buf_replace_string(b, name, test, strlen(test)); } else { (void)snprintf(name, sizeof(name), "$test%d$", i); kore_buf_replace_string(b, name, NULL, 0); diff --git a/examples/integers/src/check_integers.c b/examples/integers/src/check_integers.c @@ -15,28 +15,28 @@ page(struct http_request *req) u_int32_t u32, len; u_int8_t c, *data; - http_populate_arguments(req); + http_populate_get(req); buf = kore_buf_create(128); - if (http_argument_get_byte("id", &c)) + if (http_argument_get_byte(req, "id", &c)) kore_buf_appendf(buf, "byte\t%c\n", c); - if (http_argument_get_int16("id", &s16)) + if (http_argument_get_int16(req, "id", &s16)) kore_buf_appendf(buf, "int16\t%d\n", s16); - if (http_argument_get_uint16("id", &u16)) + if (http_argument_get_uint16(req, "id", &u16)) kore_buf_appendf(buf, "uint16\t%d\n", u16); - if (http_argument_get_int32("id", &s32)) + if (http_argument_get_int32(req, "id", &s32)) kore_buf_appendf(buf, "int32\t%d\n", s32); - if (http_argument_get_uint32("id", &u32)) + if (http_argument_get_uint32(req, "id", &u32)) kore_buf_appendf(buf, "uint32\t%d\n", u32); - if (http_argument_get_int64("id", &s64)) + if (http_argument_get_int64(req, "id", &s64)) kore_buf_appendf(buf, "int64\t%ld\n", s64); - if (http_argument_get_uint64("id", &u64)) + if (http_argument_get_uint64(req, "id", &u64)) kore_buf_appendf(buf, "uint64\t%lu\n", u64); data = kore_buf_release(buf, &len); diff --git a/examples/json_yajl/src/json_yajl.c b/examples/json_yajl/src/json_yajl.c @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2013-2016 Joris Vink <joris@coders.se> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + #include <kore/kore.h> #include <kore/http.h> @@ -8,10 +24,12 @@ int page(struct http_request *); int page(struct http_request *req) { + ssize_t ret; struct kore_buf *buf; char *body; yajl_val node, v; char eb[1024]; + u_int8_t data[BUFSIZ]; const char *path[] = { "foo", "bar", NULL }; /* We only allow POST/PUT methods. */ @@ -23,14 +41,27 @@ page(struct http_request *req) } /* - * Grab the entire body we received as text (NUL-terminated). - * Note: this can return NULL and the result MUST be freed. + * Read the entire received body into a memory buffer. */ - if ((body = http_body_text(req)) == NULL) { - http_response(req, 400, NULL, 0); - return (KORE_RESULT_OK); + buf = kore_buf_create(128); + for (;;) { + ret = http_body_read(req, data, sizeof(data)); + if (ret == -1) { + kore_buf_free(buf); + kore_log(LOG_NOTICE, "error reading body"); + http_response(req, 500, NULL, 0); + return (KORE_RESULT_OK); + } + + if (ret == 0) + break; + + kore_buf_append(buf, data, ret); } + /* Grab our body data as a NUL-terminated string. */ + body = kore_buf_stringify(buf); + /* Parse the body via yajl now. */ node = yajl_tree_parse(body, eb, sizeof(eb)); if (node == NULL) { @@ -40,12 +71,13 @@ page(struct http_request *req) kore_log(LOG_NOTICE, "parse error: unknown"); } - kore_mem_free(body); + kore_buf_free(buf); http_response(req, 400, NULL, 0); return (KORE_RESULT_OK); } - buf = kore_buf_create(128); + /* Reuse old buffer, don't need it anymore for body. */ + kore_buf_reset(buf); /* Attempt to grab foo.bar from the JSON tree. */ v = yajl_tree_get(node, path, yajl_t_string); diff --git a/examples/ktunnel/src/ktunnel.c b/examples/ktunnel/src/ktunnel.c @@ -49,9 +49,9 @@ open_connection(struct http_request *req) } /* Parse the query string and grab our arguments. */ - http_populate_arguments(req); - if (!http_argument_get_string("host", &host, NULL) || - !http_argument_get_string("port", &port, NULL)) { + http_populate_get(req); + if (!http_argument_get_string(req, "host", &host) || + !http_argument_get_string(req, "port", &port)) { http_response(req, HTTP_STATUS_BAD_REQUEST, NULL, 0); return (KORE_RESULT_OK); } diff --git a/examples/parameters/src/parameters.c b/examples/parameters/src/parameters.c @@ -22,9 +22,7 @@ int page(struct http_request *); int page(struct http_request *req) { - int p; u_int16_t id; - u_int32_t len; char *sid; struct kore_buf *buf; @@ -40,17 +38,8 @@ page(struct http_request *req) * See conf/parameters.conf on how that is done, this is an * important step as without the params block you will never * get any parameters returned from Kore. - * - * http_populate_arguments() returns the number of arguments - * that were successfully processed and are available. */ - p = http_populate_arguments(req); - - /* If we had no arguments available what so ever, return 400. */ - if (p == 0) { - http_response(req, 400, NULL, 0); - return (KORE_RESULT_OK); - } + http_populate_get(req); /* * Lets grab the "id" parameter if available. Kore can obtain @@ -73,11 +62,11 @@ page(struct http_request *req) buf = kore_buf_create(128); /* Grab it as a string, we shouldn't free the result in sid. */ - if (http_argument_get_string("id", &sid, &len)) - kore_buf_appendf(buf, "id as a string: '%s' (%d)\n", sid, len); + if (http_argument_get_string(req, "id", &sid)) + kore_buf_appendf(buf, "id as a string: '%s'\n", sid); /* Grab it as an actual u_int16_t. */ - if (http_argument_get_uint16("id", &id)) + if (http_argument_get_uint16(req, "id", &id)) kore_buf_appendf(buf, "id as an u_int16_t: %d\n", id); /* Now return the result to the client with a 200 status code. */ diff --git a/examples/tasks/src/tasks.c b/examples/tasks/src/tasks.c @@ -58,8 +58,8 @@ page_handler(struct http_request *req) */ if (req->hdlr_extra == NULL) { /* Grab the user argument */ - http_populate_arguments(req); - if (!http_argument_get_string("user", &user, &len)) { + http_populate_get(req); + if (!http_argument_get_string(req, "user", &user)) { http_response(req, 500, "ERROR\n", 6); return (KORE_RESULT_OK); } @@ -87,7 +87,7 @@ page_handler(struct http_request *req) * GET request to its channel. */ kore_task_run(&state->task); - kore_task_channel_write(&state->task, user, len); + kore_task_channel_write(&state->task, user, strlen(user)); /* * Tell Kore to retry us later. @@ -142,7 +142,6 @@ page_handler(struct http_request *req) int post_back(struct http_request *req) { - u_int32_t len; char *user; if (req->method != HTTP_METHOD_POST) { @@ -150,14 +149,14 @@ post_back(struct http_request *req) return (KORE_RESULT_OK); } - http_populate_arguments(req); - if (!http_argument_get_string("user", &user, &len)) { + http_populate_post(req); + if (!http_argument_get_string(req, "user", &user)) { http_response(req, 500, NULL, 0); return (KORE_RESULT_OK); } /* Simply echo the supplied user argument back. */ - http_response(req, 200, user, len); + http_response(req, 200, user, strlen(user)); return (KORE_RESULT_OK); } diff --git a/examples/upload/.gitignore b/examples/upload/.gitignore @@ -0,0 +1,5 @@ +*.o +.objs +upload.so +assets.h +cert diff --git a/examples/upload/conf/upload.conf b/examples/upload/conf/upload.conf @@ -0,0 +1,19 @@ +# Placeholder configuration + +bind 127.0.0.1 8888 +load ./upload.so + +http_body_max 1024000000 +http_body_disk_offload 4096 + +validator v_name regex ^[a-zA-Z]*$ +validator v_number regex ^[0-9]*$ + +domain 127.0.0.1 { + static / page + + params post / { + validate field1 v_name + validate field2 v_number + } +} diff --git a/examples/upload/src/upload.c b/examples/upload/src/upload.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2016 Joris Vink <joris@coders.se> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* + * This example demonstrates how to properly deal with file uploads + * coming from a multipart form. + * + * The basics are quite trivial: + * 1) call http_populate_multipart_form() + * 2) find the file using http_file_lookup(). + * 3) read the file data using http_file_read(). + * + * In this example the contents is written to a newly created file + * on the server that matches the naming given by the uploader. + * + * Note that the above is probably not what you want to do in real life. + */ + +#include <kore/kore.h> +#include <kore/http.h> + +#include <fcntl.h> +#include <unistd.h> + +int page(struct http_request *); + +int +page(struct http_request *req) +{ + int fd; + struct http_file *file; + u_int8_t buf[BUFSIZ]; + ssize_t ret, written; + + /* Only deal with POSTs. */ + if (req->method != HTTP_METHOD_POST) { + http_response(req, 405, NULL, 0); + return (KORE_RESULT_OK); + } + + /* Parse the multipart data that was present. */ + http_populate_multipart_form(req); + + /* Find our file. */ + if ((file = http_file_lookup(req, "file")) == NULL) { + http_response(req, 400, NULL, 0); + return (KORE_RESULT_OK); + } + + /* Open dump file where we will write file contents. */ + fd = open(file->filename, O_CREAT | O_TRUNC | O_WRONLY, 0700); + if (fd == -1) { + http_response(req, 500, NULL, 0); + return (KORE_RESULT_OK); + } + + /* While we have data from http_file_read(), write it. */ + ret = KORE_RESULT_ERROR; + for (;;) { + ret = http_file_read(file, buf, sizeof(buf)); + if (ret == -1) { + kore_log(LOG_ERR, "failed to read from file"); + http_response(req, 500, NULL, 0); + goto cleanup; + } + + if (ret == 0) + break; + + written = write(fd, buf, ret); + if (written == -1) { + kore_log(LOG_ERR,"write(%s): %s", + file->filename, errno_s); + http_response(req, 500, NULL, 0); + goto cleanup; + } + + if (written != ret) { + kore_log(LOG_ERR, "partial write on %s", + file->filename); + http_response(req, 500, NULL, 0); + goto cleanup; + } + } + + ret = KORE_RESULT_OK; + http_response(req, 200, NULL, 0); + kore_log(LOG_INFO, "file '%s' successfully received", + file->filename); + +cleanup: + if (close(fd) == -1) + kore_log(LOG_WARNING, "close(%s): %s", file->filename, errno_s); + + if (ret == KORE_RESULT_ERROR) { + if (unlink("dump") == -1) { + kore_log(LOG_WARNING, "unlink(%s): %s", + file->filename, errno_s); + } + ret = KORE_RESULT_OK; + } + + return (KORE_RESULT_OK); +} diff --git a/includes/http.h b/includes/http.h @@ -33,6 +33,10 @@ extern "C" { #define HTTP_MAX_QUERY_ARGS 20 #define HTTP_MAX_COOKIES 10 #define HTTP_REQUEST_LIMIT 1000 +#define HTTP_BODY_DISK_PATH "tmp_files" +#define HTTP_BODY_DISK_OFFLOAD 0 +#define HTTP_BODY_PATH_MAX 256 +#define HTTP_BOUNDARY_MAX 80 #define HTTP_ARG_TYPE_RAW 0 #define HTTP_ARG_TYPE_BYTE 1 @@ -58,19 +62,13 @@ struct http_header { struct http_arg { char *name; - void *value; - u_int32_t len; - char *s_value; - u_int32_t s_len; TAILQ_ENTRY(http_arg) list; }; -#define COPY_ARG_TYPE(v, l, t) \ +#define COPY_ARG_TYPE(v, t) \ do { \ - if (l != NULL) \ - *l = sizeof(t); \ *(t *)nout = v; \ } while (0); @@ -81,7 +79,7 @@ struct http_arg { nval = (type)kore_strtonum64(q->s_value, sign, &err); \ if (err != KORE_RESULT_OK) \ return (KORE_RESULT_ERROR); \ - COPY_ARG_TYPE(nval, len, type); \ + COPY_ARG_TYPE(nval, type); \ } while (0); #define COPY_ARG_INT(min, max, type) \ @@ -91,23 +89,13 @@ struct http_arg { nval = kore_strtonum(q->s_value, 10, min, max, &err); \ if (err != KORE_RESULT_OK) \ return (KORE_RESULT_ERROR); \ - COPY_ARG_TYPE(nval, len, type); \ - } while (0); - -#define CACHE_STRING() \ - do { \ - if (q->s_value == NULL) { \ - q->s_len = q->len + 1; \ - q->s_value = kore_malloc(q->s_len); \ - kore_strlcpy(q->s_value, q->value, q->s_len); \ - } \ + COPY_ARG_TYPE(nval, type); \ } while (0); #define COPY_AS_INTTYPE_64(type, sign) \ do { \ if (nout == NULL) \ return (KORE_RESULT_ERROR); \ - CACHE_STRING(); \ COPY_ARG_INT64(type, sign); \ } while (0); @@ -115,45 +103,44 @@ struct http_arg { do { \ if (nout == NULL) \ return (KORE_RESULT_ERROR); \ - CACHE_STRING(); \ COPY_ARG_INT(min, max, type); \ } while (0); -#define http_argument_type(r, n, so, no, l, t) \ - http_argument_get(r, n, so, no, l, t) +#define http_argument_type(r, n, so, no, t) \ + http_argument_get(r, n, so, no, t) -#define http_argument_get_string(n, o, l) \ - http_argument_type(req, n, (void **)o, NULL, l, HTTP_ARG_TYPE_STRING) +#define http_argument_get_string(r, n, o) \ + http_argument_type(r, n, (void **)o, NULL, HTTP_ARG_TYPE_STRING) -#define http_argument_get_byte(n, o) \ - http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_BYTE) +#define http_argument_get_byte(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_BYTE) -#define http_argument_get_uint16(n, o) \ - http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_UINT16) +#define http_argument_get_uint16(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT16) -#define http_argument_get_int16(n, o) \ - http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_INT16) +#define http_argument_get_int16(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT16) -#define http_argument_get_uint32(n, o) \ - http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_UINT32) +#define http_argument_get_uint32(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT32) -#define http_argument_get_int32(n, o) \ - http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_INT32) +#define http_argument_get_int32(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT32) -#define http_argument_get_uint64(n, o) \ - http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_UINT64) +#define http_argument_get_uint64(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_UINT64) -#define http_argument_get_int64(n, o) \ - http_argument_type(req, n, NULL, o, NULL, HTTP_ARG_TYPE_INT64) +#define http_argument_get_int64(r, n, o) \ + http_argument_type(r, n, NULL, o, HTTP_ARG_TYPE_INT64) struct http_file { char *name; char *filename; - - u_int8_t *data; - u_int32_t len; - + size_t position; + size_t offset; + size_t length; + struct http_request *req; TAILQ_ENTRY(http_file) list; }; @@ -163,20 +150,21 @@ struct http_file { #define HTTP_METHOD_DELETE 3 #define HTTP_METHOD_HEAD 4 -#define HTTP_REQUEST_COMPLETE 0x01 -#define HTTP_REQUEST_DELETE 0x02 -#define HTTP_REQUEST_SLEEPING 0x04 -#define HTTP_REQUEST_PGSQL_QUEUE 0x10 -#define HTTP_REQUEST_EXPECT_BODY 0x20 -#define HTTP_REQUEST_RETAIN_EXTRA 0x40 -#define HTTP_REQUEST_NO_CONTENT_LENGTH 0x80 +#define HTTP_REQUEST_COMPLETE 0x0001 +#define HTTP_REQUEST_DELETE 0x0002 +#define HTTP_REQUEST_SLEEPING 0x0004 +#define HTTP_REQUEST_PGSQL_QUEUE 0x0010 +#define HTTP_REQUEST_EXPECT_BODY 0x0020 +#define HTTP_REQUEST_RETAIN_EXTRA 0x0040 +#define HTTP_REQUEST_NO_CONTENT_LENGTH 0x0080 +#define HTTP_REQUEST_AUTHED 0x0100 struct kore_task; struct http_request { u_int8_t method; - u_int8_t flags; u_int8_t fsm_state; + u_int16_t flags; u_int16_t status; u_int64_t start; u_int64_t end; @@ -186,10 +174,13 @@ struct http_request { char *agent; struct connection *owner; struct kore_buf *http_body; - u_int64_t content_length; + int http_body_fd; + char *http_body_path; + size_t http_body_length; + size_t http_body_offset; + size_t content_length; void *hdlr_extra; char *query_string; - u_int8_t *multipart_body; struct kore_module_handle *hdlr; LIST_HEAD(, kore_task) tasks; @@ -214,6 +205,10 @@ extern u_int64_t http_body_max; extern u_int64_t http_hsts_enable; extern u_int16_t http_keepalive_time; extern u_int32_t http_request_limit; +extern u_int64_t http_body_disk_offload; +extern char *http_body_disk_path; + +void kore_accesslog(struct http_request *); void http_init(void); void http_process(void); @@ -222,9 +217,8 @@ time_t http_date_to_time(char *); void http_request_free(struct http_request *); void http_request_sleep(struct http_request *); void http_request_wakeup(struct http_request *); -char *http_body_text(struct http_request *); -void http_process_request(struct http_request *, int); -u_int8_t *http_body_bytes(struct http_request *, u_int32_t *); +void http_process_request(struct http_request *); +ssize_t http_body_read(struct http_request *, void *, size_t); void http_response(struct http_request *, int, void *, u_int32_t); void http_response_stream(struct http_request *, int, void *, u_int64_t, int (*cb)(struct netbuf *), void *); @@ -240,15 +234,15 @@ int http_state_run(struct http_state *, u_int8_t, int http_argument_urldecode(char *); int http_header_recv(struct netbuf *); -int http_generic_404(struct http_request *); -int http_populate_arguments(struct http_request *); -int http_populate_multipart_form(struct http_request *, int *); +void http_populate_get(struct http_request *); +void http_populate_post(struct http_request *); +void http_populate_multipart_form(struct http_request *); int http_argument_get(struct http_request *, - const char *, void **, void *, u_int32_t *, int); -int http_file_lookup(struct http_request *, const char *, char **, - u_int8_t **, u_int32_t *); + const char *, void **, void *, int); -void kore_accesslog(struct http_request *); +void http_file_rewind(struct http_file *); +ssize_t http_file_read(struct http_file *, void *, size_t); +struct http_file *http_file_lookup(struct http_request *, const char *); enum http_status_code { HTTP_STATUS_CONTINUE = 100, diff --git a/includes/kore.h b/includes/kore.h @@ -322,8 +322,7 @@ struct kore_validator { }; #endif -#define KORE_BUF_INITIAL 128 -#define KORE_BUF_INCREMENT KORE_BUF_INITIAL +#define KORE_BUF_INCREMENT 4096 struct kore_buf { u_int8_t *data; @@ -592,8 +591,9 @@ void kore_buf_free(struct kore_buf *); struct kore_buf *kore_buf_create(u_int32_t); void kore_buf_append(struct kore_buf *, const void *, u_int32_t); u_int8_t *kore_buf_release(struct kore_buf *, u_int32_t *); -void kore_buf_reset(struct kore_buf *); +void kore_buf_reset(struct kore_buf *); +char *kore_buf_stringify(struct kore_buf *); void kore_buf_appendf(struct kore_buf *, const char *, ...); void kore_buf_appendv(struct kore_buf *, const char *, va_list); void kore_buf_appendb(struct kore_buf *, struct kore_buf *); diff --git a/src/auth.c b/src/auth.c @@ -77,6 +77,7 @@ kore_auth_run(struct http_request *req, struct kore_auth *auth) switch (r) { case KORE_RESULT_OK: + req->flags |= HTTP_REQUEST_AUTHED; kore_debug("kore_auth_run() for %s successful", req->path); /* FALLTHROUGH */ case KORE_RESULT_RETRY: diff --git a/src/config.c b/src/config.c @@ -65,6 +65,8 @@ static int configure_http_body_max(char **); static int configure_http_hsts_enable(char **); static int configure_http_keepalive_time(char **); static int configure_http_request_limit(char **); +static int configure_http_body_disk_offload(char **); +static int configure_http_body_disk_path(char **); static int configure_validator(char **); static int configure_params(char **); static int configure_validate(char **); @@ -122,6 +124,8 @@ static struct { { "http_hsts_enable", configure_http_hsts_enable }, { "http_keepalive_time", configure_http_keepalive_time }, { "http_request_limit", configure_http_request_limit }, + { "http_body_disk_offload", configure_http_body_disk_offload }, + { "http_body_disk_path", configure_http_body_disk_path }, { "validator", configure_validator }, { "params", configure_params }, { "validate", configure_validate }, @@ -546,6 +550,36 @@ configure_http_body_max(char **argv) } static int +configure_http_body_disk_offload(char **argv) +{ + int err; + + if (argv[1] == NULL) + return (KORE_RESULT_ERROR); + + http_body_disk_offload = kore_strtonum(argv[1], 10, 0, LONG_MAX, &err); + if (err != KORE_RESULT_OK) { + printf("bad http_body_disk_offload value: %s\n", argv[1]); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + +static int +configure_http_body_disk_path(char **argv) +{ + if (argv[1] == NULL) + return (KORE_RESULT_ERROR); + + if (http_body_disk_path != HTTP_BODY_DISK_PATH) + kore_mem_free(http_body_disk_path); + + http_body_disk_path = kore_strdup(argv[1]); + return (KORE_RESULT_OK); +} + +static int configure_http_hsts_enable(char **argv) { int err; diff --git a/src/http.c b/src/http.c @@ -19,6 +19,9 @@ #include <ctype.h> #include <inttypes.h> +#include <fcntl.h> +#include <unistd.h> + #include "kore.h" #include "http.h" @@ -30,14 +33,21 @@ #include "tasks.h" #endif -static int http_body_recv(struct netbuf *); -static void http_error_response(struct connection *, int); -static void http_argument_add(struct http_request *, const char *, - void *, u_int32_t, int); -static void http_file_add(struct http_request *, const char *, - const char *, u_int8_t *, u_int32_t); -static void http_response_normal(struct http_request *, - struct connection *, int, void *, u_int32_t); +static int http_body_recv(struct netbuf *); +static int http_body_rewind(struct http_request *); +static void http_error_response(struct connection *, int); +static void http_argument_add(struct http_request *, const char *, char *); +static void http_response_normal(struct http_request *, + struct connection *, int, void *, u_int32_t); +static void multipart_add_field(struct http_request *, struct kore_buf *, + const char *, const char *, const int); +static void multipart_file_add(struct http_request *, struct kore_buf *, + const char *, const char *, const char *, const int); +static int multipart_find_data(struct kore_buf *, struct kore_buf *, + size_t *, struct http_request *, const void *, size_t); +static int multipart_parse_headers(struct http_request *, + struct kore_buf *, struct kore_buf *, + const char *, const int); static struct kore_buf *header_buf; static char http_version[32]; @@ -48,6 +58,7 @@ static struct kore_pool http_request_pool; static struct kore_pool http_header_pool; static struct kore_pool http_host_pool; static struct kore_pool http_path_pool; +static struct kore_pool http_body_path; int http_request_count = 0; u_int32_t http_request_limit = HTTP_REQUEST_LIMIT; @@ -55,6 +66,8 @@ u_int64_t http_hsts_enable = HTTP_HSTS_ENABLE; u_int16_t http_header_max = HTTP_HEADER_MAX_LEN; u_int16_t http_keepalive_time = HTTP_KEEPALIVE_TIME; u_int64_t http_body_max = HTTP_BODY_MAX_LEN; +u_int64_t http_body_disk_offload = HTTP_BODY_DISK_OFFLOAD; +char *http_body_disk_path = HTTP_BODY_DISK_PATH; void http_init(void) @@ -84,6 +97,8 @@ http_init(void) "http_host_pool", KORE_DOMAINNAME_LEN, prealloc); kore_pool_init(&http_path_pool, "http_path_pool", HTTP_URI_LEN, prealloc); + kore_pool_init(&http_body_path, + "http_body_path", HTTP_BODY_PATH_MAX, prealloc); } int @@ -93,8 +108,9 @@ http_request_new(struct connection *c, const char *host, { char *p; struct http_request *req; + struct kore_module_handle *hdlr; int m, flags; - size_t hostlen, pathlen; + size_t hostlen, pathlen, qsoff; kore_debug("http_request_new(%p, %s, %s, %s, %s)", c, host, method, path, version); @@ -114,6 +130,21 @@ http_request_new(struct connection *c, const char *host, return (KORE_RESULT_ERROR); } + if ((p = strchr(path, '?')) != NULL) { + *p = '\0'; + qsoff = p - path; + } else { + qsoff = 0; + } + + if ((hdlr = kore_module_handler_find(host, path)) == NULL) { + http_error_response(c, 404); + return (KORE_RESULT_ERROR); + } + + if (p != NULL) + *p = '?'; + if (!strcasecmp(method, "get")) { m = HTTP_METHOD_GET; flags = HTTP_REQUEST_COMPLETE; @@ -141,28 +172,35 @@ http_request_new(struct connection *c, const char *host, req->owner = c; req->status = 0; req->method = m; - req->hdlr = NULL; + req->hdlr = hdlr; req->agent = NULL; req->flags = flags; req->fsm_state = 0; req->http_body = NULL; + req->http_body_fd = -1; req->hdlr_extra = NULL; req->query_string = NULL; - req->multipart_body = NULL; + req->http_body_length = 0; + req->http_body_offset = 0; + req->http_body_path = NULL; if ((p = strrchr(host, ':')) != NULL) *p = '\0'; req->host = kore_pool_get(&http_host_pool); - (void)memcpy(req->host, host, hostlen); + memcpy(req->host, host, hostlen); req->host[hostlen] = '\0'; req->path = kore_pool_get(&http_path_pool); - (void)memcpy(req->path, path, pathlen); + memcpy(req->path, path, pathlen); req->path[pathlen] = '\0'; - if ((req->query_string = strchr(req->path, '?')) != NULL) + if (qsoff > 0) { + req->query_string = req->path + qsoff; *(req->query_string)++ = '\0'; + } else { + req->query_string = NULL; + } TAILQ_INIT(&(req->resp_headers)); TAILQ_INIT(&(req->req_headers)); @@ -236,63 +274,49 @@ http_process(void) continue; count++; - http_process_request(req, 0); + http_process_request(req); } } void -http_process_request(struct http_request *req, int retry_only) +http_process_request(struct http_request *req) { - struct kore_module_handle *hdlr; - int r, (*cb)(struct http_request *); + int r, (*cb)(struct http_request *); kore_debug("http_process_request: %p->%p (%s)", req->owner, req, req->path); - if (req->flags & HTTP_REQUEST_DELETE) + if (req->flags & HTTP_REQUEST_DELETE || req->hdlr == NULL) return; - if (req->hdlr != NULL) - hdlr = req->hdlr; - else - hdlr = kore_module_handler_find(req->host, req->path); - req->start = kore_time_ms(); - if (hdlr == NULL) { - r = http_generic_404(req); - } else { - if (req->hdlr != hdlr && hdlr->auth != NULL) - r = kore_auth_run(req, hdlr->auth); - else - r = KORE_RESULT_OK; + if (req->hdlr->auth != NULL && !(req->flags & HTTP_REQUEST_AUTHED)) + r = kore_auth_run(req, req->hdlr->auth); + else + r = KORE_RESULT_OK; - switch (r) { - case KORE_RESULT_OK: - req->hdlr = hdlr; - cb = hdlr->addr; - worker->active_hdlr = hdlr; - r = cb(req); - worker->active_hdlr = NULL; - break; - case KORE_RESULT_RETRY: - break; - case KORE_RESULT_ERROR: - /* - * Set r to KORE_RESULT_OK so we can properly - * flush the result from kore_auth_run(). - */ - r = KORE_RESULT_OK; - break; - default: - fatal("kore_auth() returned unknown %d", r); - } + switch (r) { + case KORE_RESULT_OK: + cb = req->hdlr->addr; + worker->active_hdlr = req->hdlr; + r = cb(req); + worker->active_hdlr = NULL; + break; + case KORE_RESULT_RETRY: + break; + case KORE_RESULT_ERROR: + /* + * Set r to KORE_RESULT_OK so we can properly + * flush the result from kore_auth_run(). + */ + r = KORE_RESULT_OK; + break; + default: + fatal("kore_auth() returned unknown %d", r); } req->end = kore_time_ms(); req->total += req->end - req->start; - if (retry_only == 1 && r != KORE_RESULT_RETRY) - fatal("http_process_request: expected RETRY but got %d", r); - switch (r) { case KORE_RESULT_OK: r = net_send_flush(req->owner); @@ -308,7 +332,7 @@ http_process_request(struct http_request *req, int retry_only) fatal("A page handler returned an unknown result: %d", r); } - if (hdlr != NULL && hdlr->dom->accesslog != -1) + if (req->hdlr->dom->accesslog != -1) kore_accesslog(req); req->flags |= HTTP_REQUEST_DELETE; @@ -403,12 +427,8 @@ http_request_free(struct http_request *req) TAILQ_REMOVE(&(req->arguments), q, list); kore_mem_free(q->name); - - if (q->value != NULL) - kore_mem_free(q->value); if (q->s_value != NULL) kore_mem_free(q->s_value); - kore_mem_free(q); } @@ -423,8 +443,17 @@ http_request_free(struct http_request *req) if (req->http_body != NULL) kore_buf_free(req->http_body); - if (req->multipart_body != NULL) - kore_mem_free(req->multipart_body); + + if (req->http_body_fd != -1) + (void)close(req->http_body_fd); + + if (req->http_body_path != NULL) { + if (unlink(req->http_body_path) == -1) { + kore_log(LOG_NOTICE, "failed to unlink %s: %s", + req->http_body_path, errno_s); + } + kore_pool_put(&http_body_path, req->http_body_path); + } if (req->hdlr_extra != NULL && !(req->flags & HTTP_REQUEST_RETAIN_EXTRA)) @@ -494,11 +523,12 @@ int http_header_recv(struct netbuf *nb) { size_t len; + ssize_t ret; struct http_header *hdr; struct http_request *req; u_int64_t bytes_left; u_int8_t *end_headers; - int h, i, v, skip; + int h, i, v, skip, l; char *request[4], *host[3], *hbuf; char *p, *headers[HTTP_REQ_HEADER_MAX]; struct connection *c = (struct connection *)nb->owner; @@ -519,7 +549,6 @@ http_header_recv(struct netbuf *nb) *end_headers = '\0'; end_headers += skip; - nb->flags |= NETBUF_FORCE_REMOVE; len = end_headers - nb->buf; hbuf = (char *)nb->buf; @@ -626,9 +655,40 @@ http_header_recv(struct netbuf *nb) return (KORE_RESULT_OK); } - req->http_body = kore_buf_create(req->content_length); - kore_buf_append(req->http_body, end_headers, - (nb->s_off - len)); + req->http_body_length = req->content_length; + + if (http_body_disk_offload > 0 && + req->content_length > http_body_disk_offload) { + req->http_body_path = kore_pool_get(&http_body_path); + l = snprintf(req->http_body_path, HTTP_BODY_PATH_MAX, + "%s/http_body.XXXXXX", http_body_disk_path); + if (l == -1 || (size_t)l >= HTTP_BODY_PATH_MAX) { + req->flags |= HTTP_REQUEST_DELETE; + http_error_response(req->owner, 500); + return (KORE_RESULT_ERROR); + } + + req->http_body = NULL; + req->http_body_fd = mkstemp(req->http_body_path); + if (req->http_body_fd == -1) { + req->flags |= HTTP_REQUEST_DELETE; + http_error_response(req->owner, 500); + return (KORE_RESULT_OK); + } + + ret = write(req->http_body_fd, + end_headers, (nb->s_off - len)); + if (ret == -1 || (size_t)ret != (nb->s_off - len)) { + req->flags |= HTTP_REQUEST_DELETE; + http_error_response(req->owner, 500); + return (KORE_RESULT_OK); + } + } else { + req->http_body_fd = -1; + req->http_body = kore_buf_create(req->content_length); + kore_buf_append(req->http_body, end_headers, + (nb->s_off - len)); + } bytes_left = req->content_length - (nb->s_off - len); if (bytes_left > 0) { @@ -642,6 +702,11 @@ http_header_recv(struct netbuf *nb) } else if (bytes_left == 0) { req->flags |= HTTP_REQUEST_COMPLETE; req->flags &= ~HTTP_REQUEST_EXPECT_BODY; + if (!http_body_rewind(req)) { + req->flags |= HTTP_REQUEST_DELETE; + http_error_response(req->owner, 500); + return (KORE_RESULT_OK); + } } else { http_error_response(req->owner, 500); } @@ -651,92 +716,48 @@ http_header_recv(struct netbuf *nb) } int -http_populate_arguments(struct http_request *req) -{ - u_int32_t len; - int i, v, c, count; - char *query, *args[HTTP_MAX_QUERY_ARGS], *val[3]; - - if (req->method == HTTP_METHOD_POST) { - if (req->http_body == NULL) - return (0); - query = http_body_text(req); - } else { - if (req->query_string == NULL) - return (0); - query = kore_strdup(req->query_string); - } - - count = 0; - v = kore_split_string(query, "&", args, HTTP_MAX_QUERY_ARGS); - for (i = 0; i < v; i++) { - c = kore_split_string(args[i], "=", val, 3); - if (c != 1 && c != 2) { - kore_debug("malformed query argument"); - continue; - } - - if (val[1] != NULL) { - len = strlen(val[1]); - http_argument_add(req, val[0], val[1], - len, HTTP_ARG_TYPE_STRING); - count++; - } - } - - kore_mem_free(query); - return (count); -} - -int http_argument_get(struct http_request *req, const char *name, - void **out, void *nout, u_int32_t *len, int type) + void **out, void *nout, int type) { struct http_arg *q; - if (len != NULL) - *len = 0; - TAILQ_FOREACH(q, &(req->arguments), list) { - if (!strcmp(q->name, name)) { - switch (type) { - case HTTP_ARG_TYPE_RAW: - if (len != NULL) - *len = q->len; - *out = q->value; - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_BYTE: - COPY_ARG_TYPE(*(u_int8_t *)q->value, - len, u_int8_t); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_INT16: - COPY_AS_INTTYPE(SHRT_MIN, SHRT_MAX, int16_t); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_UINT16: - COPY_AS_INTTYPE(0, USHRT_MAX, u_int16_t); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_INT32: - COPY_AS_INTTYPE(INT_MIN, INT_MAX, int32_t); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_UINT32: - COPY_AS_INTTYPE(0, UINT_MAX, u_int32_t); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_INT64: - COPY_AS_INTTYPE_64(int64_t, 1); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_UINT64: - COPY_AS_INTTYPE_64(u_int64_t, 0); - return (KORE_RESULT_OK); - case HTTP_ARG_TYPE_STRING: - CACHE_STRING(); - *out = q->s_value; - if (len != NULL) - *len = q->s_len - 1; - return (KORE_RESULT_OK); - default: - return (KORE_RESULT_ERROR); - } + if (strcmp(q->name, name)) + continue; + + switch (type) { + case HTTP_ARG_TYPE_RAW: + *out = q->s_value; + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_BYTE: + COPY_ARG_TYPE(*(u_int8_t *)q->s_value, u_int8_t); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_INT16: + COPY_AS_INTTYPE(SHRT_MIN, SHRT_MAX, int16_t); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_UINT16: + COPY_AS_INTTYPE(0, USHRT_MAX, u_int16_t); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_INT32: + COPY_AS_INTTYPE(INT_MIN, INT_MAX, int32_t); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_UINT32: + COPY_AS_INTTYPE(0, UINT_MAX, u_int32_t); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_INT64: + COPY_AS_INTTYPE_64(int64_t, 1); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_UINT64: + COPY_AS_INTTYPE_64(u_int64_t, 0); + return (KORE_RESULT_OK); + case HTTP_ARG_TYPE_STRING: + *out = q->s_value; + return (KORE_RESULT_OK); + default: + break; } + + return (KORE_RESULT_ERROR); } return (KORE_RESULT_ERROR); @@ -790,210 +811,237 @@ http_argument_urldecode(char *arg) return (KORE_RESULT_OK); } -int -http_file_lookup(struct http_request *req, const char *name, char **fname, - u_int8_t **data, u_int32_t *len) +struct http_file * +http_file_lookup(struct http_request *req, const char *name) { struct http_file *f; TAILQ_FOREACH(f, &(req->files), list) { - if (!strcmp(f->name, name)) { - *len = f->len; - *data = f->data; - *fname = f->filename; - return (KORE_RESULT_OK); - } + if (!strcmp(f->name, name)) + return (f); } - return (KORE_RESULT_ERROR); + return (NULL); } -int -http_populate_multipart_form(struct http_request *req, int *v) +ssize_t +http_file_read(struct http_file *file, void *buf, size_t len) { - int h, i, c, l; - u_int32_t blen, slen; - u_int8_t *s, *end, *e, *end_headers, *data; - char *d, *val, *type, *boundary, *fname; - char *headers[5], *args[5], *opt[5], *name; - - *v = 0; - - if (req->method != HTTP_METHOD_POST) - return (KORE_RESULT_ERROR); - - if (!http_request_header(req, "content-type", &type)) - return (KORE_RESULT_ERROR); - - h = kore_split_string(type, ";", args, 3); - if (h != 2) - return (KORE_RESULT_ERROR); - - if (strcasecmp(args[0], "multipart/form-data")) - return (KORE_RESULT_ERROR); - - if ((val = strchr(args[1], '=')) == NULL) - return (KORE_RESULT_ERROR); + ssize_t ret; + size_t toread, off; + + if (file->length < file->offset) + return (-1); + if ((file->offset + len) < file->offset) + return (-1); + if ((file->position + file->offset) < file->position) + return (-1); + + off = file->position + file->offset; + toread = MIN(len, (file->length - file->offset)); + if (toread <= 0) + return (0); + + if (file->req->http_body_fd != -1) { + if (lseek(file->req->http_body_fd, off, SEEK_SET) == -1) { + kore_log(LOG_ERR, "http_file_read: lseek(%s): %s", + file->req->http_body_path, errno_s); + return (-1); + } - val++; - slen = strlen(val); - boundary = kore_malloc(slen + 3); - if (!kore_snprintf(boundary, slen + 3, &l, "--%s", val)) { - kore_mem_free(boundary); - return (KORE_RESULT_ERROR); + for (;;) { + ret = read(file->req->http_body_fd, buf, toread); + if (ret == -1) { + if (errno == EINTR) + continue; + kore_log(LOG_ERR, "failed to read %s: %s", + file->req->http_body_path, errno_s); + return (-1); + } + if (ret == 0) + return (0); + break; + } + } else if (file->req->http_body != NULL) { + if (off > file->req->http_body->length) + return (0); + memcpy(buf, file->req->http_body->data + off, toread); + ret = toread; + } else { + kore_log(LOG_ERR, "http_file_read: called without body"); + return (-1); } - slen = l; + file->offset += (size_t)ret; + return (ret); +} - req->multipart_body = http_body_bytes(req, &blen); - if (slen < 3 || blen < (slen * 2)) { - kore_mem_free(boundary); - return (KORE_RESULT_ERROR); - } +void +http_file_rewind(struct http_file *file) +{ + file->offset = 0; +} - end = req->multipart_body + blen - 2; - if (end < req->multipart_body || (end - 2) < req->multipart_body) { - kore_mem_free(boundary); - return (KORE_RESULT_ERROR); - } +void +http_populate_post(struct http_request *req) +{ + ssize_t ret; + struct kore_buf *buf; + char data[BUFSIZ], *val[3], *string, *p; - if (memcmp((end - slen - 2), boundary, slen) || - memcmp((end - 2), "--", 2)) { - kore_mem_free(boundary); - return (KORE_RESULT_ERROR); - } + if (req->method != HTTP_METHOD_POST) + return; - s = req->multipart_body + slen + 2; - while (s < end) { - e = kore_mem_find(s, end - s, boundary, slen); - if (e == NULL) { - kore_mem_free(boundary); - return (KORE_RESULT_ERROR); - } + buf = kore_buf_create(128); + for (;;) { + ret = http_body_read(req, data, sizeof(data)); + if (ret == -1) + goto out; + if (ret == 0) + break; - *(e - 2) = '\0'; - end_headers = kore_mem_find(s, (e - 2) - s, "\r\n\r\n", 4); - if (end_headers == NULL) { - kore_mem_free(boundary); - return (KORE_RESULT_ERROR); + if ((p = kore_mem_find(data, ret, "&", 1)) == NULL) { + kore_buf_append(buf, data, ret); + continue; + } else { + kore_buf_append(buf, data, p - data); + ret -= (p - data); } - *end_headers = '\0'; - data = end_headers + 4; - - h = kore_split_string((char *)s, "\r\n", headers, 5); - for (i = 0; i < h; i++) { - c = kore_split_string(headers[i], ":", args, 5); - if (c != 2) - continue; - - /* Ignore other headers for now. */ - if (strcasecmp(args[0], "content-disposition")) - continue; + string = kore_buf_stringify(buf); + kore_split_string(string, "=", val, 3); + if (val[0] != NULL && val[1] != NULL) + http_argument_add(req, val[0], val[1]); - for (d = args[1]; isspace(*d); d++) - ; + kore_buf_reset(buf); + if (ret > 1) + kore_buf_append(buf, p + 1, ret - 1); + } - c = kore_split_string(d, ";", opt, 5); - if (c < 2) - continue; + if (buf->offset != 0) { + string = kore_buf_stringify(buf); + kore_split_string(string, "=", val, 3); + if (val[0] != NULL && val[1] != NULL) + http_argument_add(req, val[0], val[1]); + } - if (strcasecmp(opt[0], "form-data")) - continue; +out: + kore_buf_free(buf); +} - if ((val = strchr(opt[1], '=')) == NULL) - continue; - if (strlen(val) < 3) - continue; +void +http_populate_get(struct http_request *req) +{ + int i, v; + char *query, *args[HTTP_MAX_QUERY_ARGS], *val[3]; - val++; - kore_strip_chars(val, '"', &name); + if (req->method != HTTP_METHOD_GET || req->query_string == NULL) + return; - if (opt[2] == NULL) { - *v = *v + 1; - http_argument_add(req, name, - data, (e - 2) - data, HTTP_ARG_TYPE_STRING); - kore_mem_free(name); - continue; - } + query = kore_strdup(req->query_string); + v = kore_split_string(query, "&", args, HTTP_MAX_QUERY_ARGS); + for (i = 0; i < v; i++) { + kore_split_string(args[i], "=", val, 3); + if (val[0] != NULL && val[1] != NULL) + http_argument_add(req, val[0], val[1]); + } - for (d = opt[2]; isspace(*d); d++) - ; + kore_mem_free(query); +} - if (!strncasecmp(d, "filename=", 9)) { - if ((val = strchr(d, '=')) == NULL) { - kore_mem_free(name); - continue; - } - - val++; - kore_strip_chars(val, '"', &fname); - if (strlen(fname) > 0) { - *v = *v + 1; - http_file_add(req, name, fname, - data, (e - 2) - data); - } - - kore_mem_free(fname); - } else { - kore_debug("got unknown: %s", opt[2]); - } +void +http_populate_multipart_form(struct http_request *req) +{ + int h, blen, count; + struct kore_buf *in, *out; + char *type, *val, *args[3]; + char boundary[HTTP_BOUNDARY_MAX]; - kore_mem_free(name); - } + if (req->method != HTTP_METHOD_POST) + return; - s = e + slen + 2; - } + if (!http_request_header(req, "content-type", &type)) + return; - kore_mem_free(boundary); + h = kore_split_string(type, ";", args, 3); + if (h != 2) + return; - return (KORE_RESULT_OK); -} + if (strcasecmp(args[0], "multipart/form-data")) + return; -int -http_generic_404(struct http_request *req) -{ - kore_debug("http_generic_404(%s, %d, %s)", - req->host, req->method, req->path); + if ((val = strchr(args[1], '=')) == NULL) + return; - http_response(req, 404, NULL, 0); + val++; + blen = snprintf(boundary, sizeof(boundary), "--%s", val); + if (blen == -1 || (size_t)blen >= sizeof(boundary)) + return; - return (KORE_RESULT_OK); -} + in = kore_buf_create(128); + out = kore_buf_create(128); -char * -http_body_text(struct http_request *req) -{ - u_int32_t len; - u_int8_t *data; - char *text; + if (!multipart_find_data(in, NULL, NULL, req, boundary, blen)) + goto cleanup; - if (req->http_body == NULL) - return (NULL); + count = 0; - data = kore_buf_release(req->http_body, &len); - req->http_body = NULL; - len++; + for (;;) { + if (!multipart_find_data(in, NULL, NULL, req, "\r\n", 2)) + break; + if (in->offset < 4 && req->http_body_length == 0) + break; + if (!multipart_find_data(in, out, NULL, req, "\r\n\r\n", 4)) + break; + if (!multipart_parse_headers(req, in, out, boundary, blen)) + break; - text = kore_malloc(len); - kore_strlcpy(text, (char *)data, len); - kore_mem_free(data); + kore_buf_reset(out); + } - return (text); +cleanup: + kore_buf_free(in); + kore_buf_free(out); } -u_int8_t * -http_body_bytes(struct http_request *req, u_int32_t *len) +ssize_t +http_body_read(struct http_request *req, void *out, size_t len) { - u_int8_t *data; - - if (req->http_body == NULL) - return (NULL); + ssize_t ret; + size_t toread; + + toread = MIN(req->http_body_length, len); + if (toread <= 0) + return (0); + + if (req->http_body_fd != -1) { + for (;;) { + ret = read(req->http_body_fd, out, toread); + if (ret == -1) { + if (errno == EINTR) + continue; + kore_log(LOG_ERR, "failed to read %s: %s", + req->http_body_path, errno_s); + return (-1); + } + if (ret == 0) + return (0); + break; + } + } else if (req->http_body != NULL) { + memcpy(out, + (req->http_body->data + req->http_body->offset), toread); + req->http_body->offset += toread; + ret = toread; + } else { + kore_log(LOG_ERR, "http_body_read: called without body"); + return (-1); + } - data = kore_buf_release(req->http_body, len); - req->http_body = NULL; + req->http_body_length -= (size_t)ret; + req->http_body_offset += (size_t)ret; - return (data); + return (ret); } int @@ -1035,74 +1083,254 @@ http_state_run(struct http_state *states, u_int8_t elm, return (KORE_RESULT_OK); } -static void -http_argument_add(struct http_request *req, const char *name, - void *value, u_int32_t len, int type) +static int +multipart_find_data(struct kore_buf *in, struct kore_buf *out, + size_t *olen, struct http_request *req, const void *needle, size_t len) { - struct http_arg *q; - struct kore_handler_params *p; + ssize_t ret; + size_t left; + u_int8_t *p, first, data[4096]; + + if (olen != NULL) + *olen = 0; + + first = *(const u_int8_t *)needle; + for (;;) { + if (in->offset < len) { + ret = http_body_read(req, data, sizeof(data)); + if (ret == -1) + return (KORE_RESULT_ERROR); + if (ret == 0) + return (KORE_RESULT_ERROR); - if (len == 0 || value == NULL) { - kore_debug("http_argument_add: with NULL value"); - return; + kore_buf_append(in, data, ret); + continue; + } + + p = kore_mem_find(in->data, in->offset, &first, 1); + if (p == NULL) { + if (out != NULL) + kore_buf_append(out, in->data, in->offset); + if (olen != NULL) + *olen += in->offset; + kore_buf_reset(in); + continue; + } + + left = in->offset - (p - in->data); + if (left < len) { + if (out != NULL) + kore_buf_append(out, in->data, (p - in->data)); + if (olen != NULL) + *olen += (p - in->data); + memmove(in->data, p, left); + in->offset = left; + continue; + } + + if (!memcmp(p, needle, len)) { + if (out != NULL) + kore_buf_append(out, in->data, p - in->data); + if (olen != NULL) + *olen += (p - in->data); + + in->offset = left - len; + if (in->offset > 0) + memmove(in->data, p + len, in->offset); + return (KORE_RESULT_OK); + } + + if (out != NULL) + kore_buf_append(out, in->data, (p - in->data) + 1); + if (olen != NULL) + *olen += (p - in->data) + 1; + + in->offset = left - 1; + if (in->offset > 0) + memmove(in->data, p + 1, in->offset); } - TAILQ_FOREACH(p, &(req->hdlr->params), list) { - if (p->method != req->method) + return (KORE_RESULT_ERROR); +} + +static int +multipart_parse_headers(struct http_request *req, struct kore_buf *in, + struct kore_buf *hbuf, const char *boundary, const int blen) +{ + int h, c, i; + char *headers[5], *args[5], *opt[5]; + char *d, *val, *name, *fname, *string; + + string = kore_buf_stringify(hbuf); + h = kore_split_string(string, "\r\n", headers, 5); + for (i = 0; i < h; i++) { + c = kore_split_string(headers[i], ":", args, 5); + if (c != 2) continue; - if (!strcmp(p->name, name)) { - if (type == HTTP_ARG_TYPE_STRING) { - http_argument_urldecode(value); - len = strlen(value); - } + /* Ignore other headers for now. */ + if (strcasecmp(args[0], "content-disposition")) + continue; + + for (d = args[1]; isspace(*d); d++) + ; + + c = kore_split_string(d, ";", opt, 5); + if (c < 2) + continue; + + if (strcasecmp(opt[0], "form-data")) + continue; + + if ((val = strchr(opt[1], '=')) == NULL) + continue; + if (strlen(val) < 3) + continue; + + val++; + kore_strip_chars(val, '"', &name); + + if (opt[2] == NULL) { + multipart_add_field(req, in, name, boundary, blen); + kore_mem_free(name); + continue; + } + + for (d = opt[2]; isspace(*d); d++) + ; - if (kore_validator_check(req, p->validator, value)) { - q = kore_malloc(sizeof(struct http_arg)); - q->len = len; - q->s_value = NULL; - q->name = kore_strdup(name); - q->value = kore_malloc(len); - memcpy(q->value, value, len); - TAILQ_INSERT_TAIL(&(req->arguments), q, list); + if (!strncasecmp(d, "filename=", 9)) { + if ((val = strchr(d, '=')) == NULL) { + kore_mem_free(name); + continue; } - return; + val++; + kore_strip_chars(val, '"', &fname); + if (strlen(fname) > 0) { + multipart_file_add(req, + in, name, fname, boundary, blen); + } + kore_mem_free(fname); + } else { + kore_debug("got unknown: %s", opt[2]); } + + kore_mem_free(name); + } + + return (KORE_RESULT_OK); +} + +static void +multipart_add_field(struct http_request *req, struct kore_buf *in, + const char *name, const char *boundary, const int blen) +{ + struct kore_buf *data; + char *string; + + data = kore_buf_create(128); + + if (!multipart_find_data(in, data, NULL, req, boundary, blen)) { + kore_buf_free(data); + return; } + + if (data->offset < 3) { + kore_buf_free(data); + return; + } + + data->offset -= 2; + string = kore_buf_stringify(data); + http_argument_add(req, name, string); + kore_buf_free(data); } static void -http_file_add(struct http_request *req, const char *name, const char *filename, - u_int8_t *data, u_int32_t len) +multipart_file_add(struct http_request *req, struct kore_buf *in, + const char *name, const char *fname, const char *boundary, const int blen) { struct http_file *f; + size_t position, len; + + position= req->http_body_offset - in->offset; + if (!multipart_find_data(in, NULL, &len, req, boundary, blen)) + return; + + if (len < 3) + return; + len -= 2; f = kore_malloc(sizeof(struct http_file)); - f->len = len; - f->data = data; + f->req = req; + f->length = len; + f->position = position; f->name = kore_strdup(name); - f->filename = kore_strdup(filename); + f->filename = kore_strdup(fname); TAILQ_INSERT_TAIL(&(req->files), f, list); } +static void +http_argument_add(struct http_request *req, const char *name, char *value) +{ + struct http_arg *q; + struct kore_handler_params *p; + + TAILQ_FOREACH(p, &(req->hdlr->params), list) { + if (p->method != req->method) + continue; + if (strcmp(p->name, name)) + continue; + + http_argument_urldecode(value); + if (!kore_validator_check(req, p->validator, value)) + break; + + q = kore_malloc(sizeof(struct http_arg)); + q->name = kore_strdup(name); + q->s_value = kore_strdup(value); + TAILQ_INSERT_TAIL(&(req->arguments), q, list); + break; + } +} + static int http_body_recv(struct netbuf *nb) { + ssize_t ret; u_int64_t bytes_left; struct http_request *req = (struct http_request *)nb->extra; - kore_buf_append(req->http_body, nb->buf, nb->s_off); + if (req->http_body_fd != -1) { + ret = write(req->http_body_fd, nb->buf, nb->s_off); + if (ret == -1 || (size_t)ret != nb->s_off) + fatal("failed to write"); + } else if (req->http_body != NULL) { + kore_buf_append(req->http_body, nb->buf, nb->s_off); + } else { + req->flags |= HTTP_REQUEST_DELETE; + http_error_response(req->owner, 500); + return (KORE_RESULT_OK); + } - if (req->http_body->offset == req->content_length) { + req->content_length -= nb->s_off; + + if (req->content_length == 0) { nb->extra = NULL; + http_request_wakeup(req); req->flags |= HTTP_REQUEST_COMPLETE; req->flags &= ~HTTP_REQUEST_EXPECT_BODY; - http_request_wakeup(req); + req->content_length = req->http_body_length; + if (!http_body_rewind(req)) { + req->flags |= HTTP_REQUEST_DELETE; + http_error_response(req->owner, 500); + return (KORE_RESULT_OK); + } net_recv_reset(nb->owner, http_header_max, http_header_recv); } else { - bytes_left = req->content_length - req->http_body->offset; + bytes_left = req->content_length; net_recv_reset(nb->owner, MIN(bytes_left, NETBUF_SEND_PAYLOAD_MAX), http_body_recv); @@ -1111,6 +1339,22 @@ http_body_recv(struct netbuf *nb) return (KORE_RESULT_OK); } +static int +http_body_rewind(struct http_request *req) +{ + if (req->http_body_fd != -1) { + if (lseek(req->http_body_fd, 0, SEEK_SET) == -1) { + kore_log(LOG_ERR, "lseek(%s) failed: %s", + req->http_body_path, errno_s); + return (KORE_RESULT_ERROR); + } + } else { + kore_buf_reset(req->http_body); + } + + return (KORE_RESULT_OK); +} + static void http_error_response(struct connection *c, int status) { diff --git a/src/kore.c b/src/kore.c @@ -15,6 +15,8 @@ */ #include <sys/types.h> +#include <sys/stat.h> + #include <sys/socket.h> #include <sys/resource.h> @@ -23,6 +25,10 @@ #include "kore.h" +#if !defined(KORE_NO_HTTP) +#include "http.h" +#endif + volatile sig_atomic_t sig_recv; struct listener_head listeners; @@ -161,6 +167,13 @@ main(int argc, char *argv[]) #if !defined(KORE_NO_HTTP) kore_accesslog_init(); + if (http_body_disk_offload > 0) { + if (mkdir(http_body_disk_path, 0700) == -1 && errno != EEXIST) { + printf("can't create http_body_disk_path '%s': %s\n", + http_body_disk_path, errno_s); + return (KORE_RESULT_ERROR); + } + } #endif sig_recv = 0;