kore

An easy to use, scalable and secure web application framework for writing web APIs in C.
Commits | Files | Refs | README | LICENSE | git clone https://git.kore.io/kore.git

commit 80f5425698d42ef6b12b7a37f13fac49333557a2
parent 9be72aff575e38f650525122a3d0918b6587849c
Author: Joris Vink <joris@coders.se>
Date:   Thu, 28 Jun 2018 13:27:44 +0200

Add filemaps.

A filemap is a way of telling Kore to serve files from a directory
much like a traditional webserver can do.

Kore filemaps only handles files. Kore does not generate directory
indexes or deal with non-regular files.

The way files are sent to a client differs a bit per platform and
build options:

default:
  - mmap() backed file transfer due to TLS.

NOTLS=1
  - sendfile() under FreeBSD, macOS and Linux.
  - mmap() backed file for OpenBSD.

The opened file descriptors/mmap'd regions are cached and reused when
appropriate. If a file is no longer in use it will be closed and evicted
from the cache after 30 seconds.

New API's are available allowing developers to use these facilities via:
  void net_send_fileref(struct connection *, struct kore_fileref *);
  void http_response_fileref(struct http_request *, struct kore_fileref *);

Kore will attempt to match media types based on file extensions. A few
default types are built-in. Others can be added via the new "http_media_type"
configuration directive.

Diffstat:
Makefile | 12++++++++----
conf/kore.conf.example | 4++++
include/kore/http.h | 11+++++++++++
include/kore/kore.h | 43++++++++++++++++++++++++++++++++++++++++++-
src/bsd.c | 44++++++++++++++++++++++++++++++++++++++++++++
src/config.c | 59+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/connection.c | 9++-------
src/filemap.c | 184+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/fileref.c | 152+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/http.c | 105++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
src/kore.c | 10+++++-----
src/linux.c | 36++++++++++++++++++++++++++++++++++++
src/net.c | 76+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------
src/worker.c | 1+
14 files changed, 719 insertions(+), 27 deletions(-)

diff --git a/Makefile b/Makefile @@ -13,9 +13,9 @@ INCLUDE_DIR=$(PREFIX)/include/kore VERSION=src/version.c S_SRC= src/kore.c src/buf.c src/config.c src/connection.c \ - src/domain.c src/mem.c src/msg.c src/module.c src/net.c \ - src/pool.c src/runtime.c src/timer.c src/utils.c src/worker.c \ - src/keymgr.c $(VERSION) + src/domain.c src/filemap.c src/fileref.c src/mem.c src/msg.c \ + src/module.c src/net.c src/pool.c src/runtime.c src/timer.c \ + src/utils.c src/worker.c src/keymgr.c $(VERSION) FEATURES= FEATURES_INC= @@ -43,6 +43,10 @@ else CFLAGS+=-O3 endif +ifneq ("$(NOSENDFILE)", "") + CFLAGS+=-DKORE_NO_SENDFILE +endif + ifneq ("$(NOHTTP)", "") CFLAGS+=-DKORE_NO_HTTP FEATURES+=-DKORE_NO_HTTP @@ -147,7 +151,7 @@ objects: $(OBJDIR) $(S_OBJS) $(OBJDIR): @mkdir -p $(OBJDIR) -install: $(KORE) $(KODEV) +install: mkdir -p $(SHARE_DIR) mkdir -p $(INCLUDE_DIR) mkdir -p $(INSTALL_DIR) diff --git a/conf/kore.conf.example b/conf/kore.conf.example @@ -237,6 +237,10 @@ domain localhost { # Page handlers with authentication. static /private/test serve_private_test auth_example + # Allow access to files from the directory static_files via + # the /files/ URI. + filemap static_files /files/ + # Configure /params-test POST to only accept the following parameters. # They are automatically tested against the validator listed. # If the validator would fail Kore will automatically remove the diff --git a/include/kore/http.h b/include/kore/http.h @@ -254,6 +254,12 @@ struct http_state { int (*cb)(struct http_request *); }; +struct http_media_type { + char *ext; + char *type; + LIST_ENTRY(http_media_type) list; +}; + extern size_t http_body_max; extern u_int16_t http_header_max; extern u_int32_t http_request_ms; @@ -267,6 +273,7 @@ extern char *http_body_disk_path; void kore_accesslog(struct http_request *); void http_init(void); +void http_parent_init(void); void http_cleanup(void); void http_server_version(const char *); void http_process(void); @@ -278,8 +285,11 @@ void http_request_sleep(struct http_request *); void http_request_wakeup(struct http_request *); void http_process_request(struct http_request *); int http_body_rewind(struct http_request *); +int http_media_register(const char *, const char *); ssize_t http_body_read(struct http_request *, void *, size_t); void http_response(struct http_request *, int, const void *, size_t); +void http_response_fileref(struct http_request *, int, + struct kore_fileref *); void http_serveable(struct http_request *, const void *, size_t, const char *, const char *); void http_response_stream(struct http_request *, int, void *, @@ -296,6 +306,7 @@ void http_response_cookie(struct http_request *, const char *, const char *, const char *, time_t, u_int32_t, struct http_cookie **); +const char *http_media_type(const char *); void *http_state_get(struct http_request *); int http_state_exists(struct http_request *); void http_state_cleanup(struct http_request *); diff --git a/include/kore/kore.h b/include/kore/kore.h @@ -53,6 +53,12 @@ extern "C" { extern int daemon(int, int); #endif +#if !defined(KORE_NO_SENDFILE) && defined(KORE_NO_TLS) +#if defined(__MACH__) || defined(__FreeBSD_version) || defined(__linux__) +#define KORE_USE_PLATFORM_SENDFILE 1 +#endif +#endif + #define KORE_RESULT_ERROR 0 #define KORE_RESULT_OK 1 #define KORE_RESULT_RETRY 2 @@ -81,6 +87,7 @@ extern int daemon(int, int); #define NETBUF_RECV 0 #define NETBUF_SEND 1 #define NETBUF_SEND_PAYLOAD_MAX 8192 +#define SENDFILE_PAYLOAD_MAX (1024 * 1024 * 10) #define NETBUF_LAST_CHAIN 0 #define NETBUF_BEFORE_CHAIN 1 @@ -89,6 +96,7 @@ extern int daemon(int, int); #define NETBUF_FORCE_REMOVE 0x02 #define NETBUF_MUST_RESEND 0x04 #define NETBUF_IS_STREAM 0x10 +#define NETBUF_IS_FILEREF 0x20 #define X509_GET_CN(c, o, l) \ X509_NAME_get_text_by_NID(X509_get_subject_name(c), \ @@ -101,6 +109,19 @@ extern int daemon(int, int); struct http_request; #endif +struct kore_fileref { + int cnt; + off_t size; + char *path; + u_int64_t expiration; +#if !defined(KORE_USE_PLATFORM_SENDFILE) + void *base; +#else + int fd; +#endif + TAILQ_ENTRY(kore_fileref) list; +}; + struct netbuf { u_int8_t *buf; size_t s_off; @@ -109,8 +130,13 @@ struct netbuf { u_int8_t type; u_int8_t flags; - void *owner; + struct kore_fileref *file_ref; +#if defined(KORE_USE_PLATFORM_SENDFILE) + off_t fd_off; + off_t fd_len; +#endif + void *owner; void *extra; int (*cb)(struct netbuf *); @@ -510,6 +536,10 @@ void kore_platform_schedule_write(int, void *); void kore_platform_event_schedule(int, int, int, void *); void kore_platform_worker_setcpu(struct kore_worker *); +#if defined(KORE_USE_PLATFORM_SENDFILE) +int kore_platform_sendfile(struct connection *, struct netbuf *); +#endif + void kore_accesslog_init(void); void kore_accesslog_worker_init(void); int kore_accesslog_write(const void *, u_int32_t); @@ -611,6 +641,15 @@ void kore_msg_send(u_int16_t, u_int8_t, const void *, u_int32_t); int kore_msg_register(u_int8_t, void (*cb)(struct kore_msg *, const void *)); +void kore_filemap_init(void); +int kore_filemap_create(struct kore_domain *, const char *, + const char *); + +void kore_fileref_init(void); +struct kore_fileref *kore_fileref_get(const char *); +struct kore_fileref *kore_fileref_create(const char *, int, off_t); +void kore_fileref_release(struct kore_fileref *); + void kore_domain_init(void); void kore_domain_cleanup(void); int kore_domain_new(char *); @@ -675,6 +714,7 @@ void net_write64(u_int8_t *, u_int64_t); void net_init(void); void net_cleanup(void); +struct netbuf *net_netbuf_get(void); int net_send(struct connection *); int net_send_flush(struct connection *); int net_recv_flush(struct connection *); @@ -692,6 +732,7 @@ void net_recv_expand(struct connection *c, size_t, void net_send_queue(struct connection *, const void *, size_t); void net_send_stream(struct connection *, void *, size_t, int (*cb)(struct netbuf *), struct netbuf **); +void net_send_fileref(struct connection *, struct kore_fileref *); void kore_buf_free(struct kore_buf *); struct kore_buf *kore_buf_alloc(size_t); diff --git a/src/bsd.c b/src/bsd.c @@ -17,6 +17,8 @@ #include <sys/param.h> #include <sys/event.h> #include <sys/sysctl.h> +#include <sys/socket.h> +#include <sys/uio.h> #if defined(__FreeBSD_version) #include <sys/cpuset.h> @@ -276,3 +278,45 @@ kore_platform_proctitle(char *title) setproctitle("%s", title); #endif } + +#if defined(KORE_USE_PLATFORM_SENDFILE) +int +kore_platform_sendfile(struct connection *c, struct netbuf *nb) +{ + int ret; + off_t len, smin; + + smin = nb->fd_len - nb->fd_off; + len = MIN(SENDFILE_PAYLOAD_MAX, smin); + +#if defined(__MACH__) + ret = sendfile(nb->file_ref->fd, c->fd, nb->fd_off, &len, NULL, 0); +#else + ret = sendfile(nb->file_ref->fd, c->fd, nb->fd_off, len, NULL, &len, 0); +#endif + + if (ret == -1) { + if (errno == EAGAIN) { + nb->fd_off += len; + c->flags &= ~CONN_WRITE_POSSIBLE; + return (KORE_RESULT_OK); + } + + if (errno == EINTR) { + nb->fd_off += len; + return (KORE_RESULT_OK); + } + + return (KORE_RESULT_ERROR); + } + + nb->fd_off += len; + + if (len == 0 || nb->fd_off == nb->fd_len) { + net_remove_netbuf(&(c->send_queue), nb); + c->snb = NULL; + } + + return (KORE_RESULT_OK); +} +#endif diff --git a/src/config.c b/src/config.c @@ -75,12 +75,14 @@ static int configure_client_certificates(char *); #endif #if !defined(KORE_NO_HTTP) +static int configure_filemap(char *); static int configure_handler(int, char *); static int configure_static_handler(char *); static int configure_dynamic_handler(char *); static int configure_accesslog(char *); static int configure_http_header_max(char *); static int configure_http_body_max(char *); +static int configure_http_media_type(char *); static int configure_http_hsts_enable(char *); static int configure_http_keepalive_time(char *); static int configure_http_request_ms(char *); @@ -147,9 +149,11 @@ static struct { { "client_verify_depth", configure_client_verify_depth }, #endif #if !defined(KORE_NO_HTTP) + { "filemap", configure_filemap }, { "static", configure_static_handler }, { "dynamic", configure_dynamic_handler }, { "accesslog", configure_accesslog }, + { "http_media_type", configure_http_media_type }, { "http_header_max", configure_http_header_max }, { "http_body_max", configure_http_body_max }, { "http_hsts_enable", configure_http_hsts_enable }, @@ -606,6 +610,31 @@ configure_handler(int type, char *options) } static int +configure_filemap(char *options) +{ + char *argv[3]; + + if (current_domain == NULL) { + printf("filemap outside of domain context\n"); + return (KORE_RESULT_ERROR); + } + + kore_split_string(options, " ", argv, 3); + + if (argv[0] == NULL || argv[1] == NULL) { + printf("missing parameters for filemap\n"); + return (KORE_RESULT_ERROR); + } + + if (!kore_filemap_create(current_domain, argv[1], argv[0])) { + printf("cannot create filemap for %s\n", argv[1]); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + +static int configure_accesslog(char *path) { if (current_domain == NULL) { @@ -631,6 +660,36 @@ configure_accesslog(char *path) } static int +configure_http_media_type(char *type) +{ + int i; + char *extensions, *ext[10]; + + extensions = strchr(type, ' '); + if (extensions == NULL) { + printf("bad http_media_type value: %s\n", type); + return (KORE_RESULT_ERROR); + } + + *(extensions)++ = '\0'; + + kore_split_string(extensions, " \t", ext, 10); + for (i = 0; ext[i] != NULL; i++) { + if (!http_media_register(ext[i], type)) { + printf("duplicate extension found: %s\n", ext[i]); + return (KORE_RESULT_ERROR); + } + } + + if (i == 0) { + printf("missing extensions in: %s\n", type); + return (KORE_RESULT_ERROR); + } + + return (KORE_RESULT_OK); +} + +static int configure_http_header_max(char *option) { int err; diff --git a/src/connection.c b/src/connection.c @@ -360,13 +360,8 @@ kore_connection_remove(struct connection *c) for (nb = TAILQ_FIRST(&(c->send_queue)); nb != NULL; nb = next) { next = TAILQ_NEXT(nb, list); - TAILQ_REMOVE(&(c->send_queue), nb, list); - if (!(nb->flags & NETBUF_IS_STREAM)) { - kore_free(nb->buf); - } else if (nb->cb != NULL) { - (void)nb->cb(nb); - } - kore_pool_put(&nb_pool, nb); + nb->flags &= ~NETBUF_MUST_RESEND; + net_remove_netbuf(&(c->send_queue), nb); } if (c->rnb != NULL) { diff --git a/src/filemap.c b/src/filemap.c @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2018 Joris Vink <joris@coders.se> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include <sys/param.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include <fcntl.h> +#include <dirent.h> +#include <unistd.h> + +#include "kore.h" + +#if !defined(KORE_NO_HTTP) + +#include "http.h" + +struct filemap_entry { + char *root; + size_t root_len; + struct kore_domain *domain; + char *ondisk; + TAILQ_ENTRY(filemap_entry) list; +}; + +int filemap_resolve(struct http_request *); + +static void filemap_serve(struct http_request *, struct filemap_entry *); + +static TAILQ_HEAD(, filemap_entry) maps; + +void +kore_filemap_init(void) +{ + TAILQ_INIT(&maps); +} + +int +kore_filemap_create(struct kore_domain *dom, const char *path, const char *root) +{ + size_t sz; + int len; + struct filemap_entry *entry; + char regex[1024]; + + sz = strlen(root); + if (sz == 0) + return (KORE_RESULT_ERROR); + + if (root[0] != '/' || root[sz - 1] != '/') + return (KORE_RESULT_ERROR); + + len = snprintf(regex, sizeof(regex), "^%s.*$", root); + if (len == -1 || (size_t)len >= sizeof(regex)) + fatal("kore_filemap_create: buffer too small"); + + if (!kore_module_handler_new(regex, dom->domain, + "filemap_resolve", NULL, HANDLER_TYPE_DYNAMIC)) + return (KORE_RESULT_ERROR); + + entry = kore_calloc(1, sizeof(*entry)); + + entry->domain = dom; + entry->root_len = sz; + entry->root = kore_strdup(root); + entry->ondisk = kore_strdup(path); + + TAILQ_INSERT_TAIL(&maps, entry, list); + + return (KORE_RESULT_OK); +} + +int +filemap_resolve(struct http_request *req) +{ + size_t best_len; + struct filemap_entry *entry, *best; + + best = NULL; + best_len = 0; + + TAILQ_FOREACH(entry, &maps, list) { + if (!strncmp(entry->root, req->path, entry->root_len)) { + if (best == NULL || entry->root_len > best_len) { + best = entry; + best_len = entry->root_len; + continue; + } + } + } + + if (best == NULL) { + http_response(req, HTTP_STATUS_NOT_FOUND, NULL, 0); + return (KORE_RESULT_OK); + } + + filemap_serve(req, best); + + return (KORE_RESULT_OK); +} + +static void +filemap_serve(struct http_request *req, struct filemap_entry *map) +{ + struct stat st; + struct kore_fileref *ref; + int len, fd; + char fpath[MAXPATHLEN]; + + len = snprintf(fpath, sizeof(fpath), "%s/%s", map->ondisk, + req->path + map->root_len); + if (len == -1 || (size_t)len >= sizeof(fpath)) { + http_response(req, HTTP_STATUS_INTERNAL_ERROR, NULL, 0); + return; + } + + if ((ref = kore_fileref_get(fpath)) == NULL) { + if ((fd = open(fpath, O_RDONLY | O_NOFOLLOW)) == -1) { + switch (errno) { + case ENOENT: + req->status = HTTP_STATUS_NOT_FOUND; + break; + case EPERM: + case EACCES: + req->status = HTTP_STATUS_FORBIDDEN; + break; + default: + req->status = HTTP_STATUS_INTERNAL_ERROR; + break; + } + + http_response(req, req->status, NULL, 0); + return; + } + + if (fstat(fd, &st) == -1) { + http_response(req, HTTP_STATUS_INTERNAL_ERROR, NULL, 0); + goto cleanup; + } + + if (S_ISREG(st.st_mode)) { + if (st.st_size <= 0) { + http_response(req, + HTTP_STATUS_NOT_FOUND, NULL, 0); + goto cleanup; + } + + /* kore_fileref_create() takes ownership of the fd. */ + ref = kore_fileref_create(fpath, fd, st.st_size); + if (ref == NULL) { + http_response(req, + HTTP_STATUS_INTERNAL_ERROR, NULL, 0); + } else { + fd = -1; + } + } else { + http_response(req, HTTP_STATUS_NOT_FOUND, NULL, 0); + } + } + + if (ref != NULL) { + http_response_fileref(req, HTTP_STATUS_OK, ref); + fd = -1; + } + +cleanup: + if (fd != -1) + close(fd); +} + +#endif diff --git a/src/fileref.c b/src/fileref.c @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2018 Joris Vink <joris@coders.se> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include <sys/param.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/mman.h> + +#include "kore.h" + +/* cached filerefs expire after 30 seconds of inactivity. */ +#define FILEREF_EXPIRATION (1000 * 30) + +static void fileref_expiration_check(void *, u_int64_t); + +static TAILQ_HEAD(, kore_fileref) refs; +static struct kore_pool ref_pool; + +void +kore_fileref_init(void) +{ + TAILQ_INIT(&refs); + kore_pool_init(&ref_pool, "ref_pool", sizeof(struct kore_fileref), 100); + kore_timer_add(fileref_expiration_check, 10000, NULL, 0); +} + +struct kore_fileref * +kore_fileref_create(const char *path, int fd, off_t size) +{ + struct kore_fileref *ref; + + if ((ref = kore_fileref_get(path)) != NULL) + return (ref); + + ref = kore_pool_get(&ref_pool); + + ref->cnt = 1; + ref->size = size; + ref->path = kore_strdup(path); + +#if !defined(KORE_USE_PLATFORM_SENDFILE) + if ((uintmax_t)size> SIZE_MAX) + return (NULL); + + ref->base = mmap(NULL, (size_t)size, PROT_READ, MAP_PRIVATE, fd, 0); + if (ref->base == MAP_FAILED) + fatal("net_send_file: mmap failed: %s", errno_s); + if (madvise(ref->base, (size_t)size, MADV_SEQUENTIAL) == -1) + fatal("net_send_file: madvise: %s", errno_s); + close(fd); +#else + ref->fd = fd; +#endif + +#if defined(FILEREF_DEBUG) + kore_log(LOG_DEBUG, "ref:%p created", (void *)ref); +#endif + + TAILQ_INSERT_TAIL(&refs, ref, list); + + return (ref); +} + +/* + * Caller must call kore_fileref_release() after kore_fileref_get() even + * if they don't end up using the ref. + */ +struct kore_fileref * +kore_fileref_get(const char *path) +{ + struct kore_fileref *ref; + + TAILQ_FOREACH(ref, &refs, list) { + if (!strcmp(ref->path, path)) { + ref->cnt++; +#if defined(FILEREF_DEBUG) + kore_log(LOG_DEBUG, "ref:%p cnt:%d", + (void *)ref, ref->cnt); +#endif + TAILQ_REMOVE(&refs, ref, list); + TAILQ_INSERT_HEAD(&refs, ref, list); + return (ref); + } + } + + return (NULL); +} + +void +kore_fileref_release(struct kore_fileref *ref) +{ + ref->cnt--; + +#if defined(FILEREF_DEBUG) + kore_log(LOG_DEBUG, "ref:%p released cnt:%d", (void *)ref, ref->cnt); +#endif + + if (ref->cnt < 0) { + fatal("kore_fileref_release: cnt < 0 (%p:%d)", + (void *)ref, ref->cnt); + } + + if (ref->cnt == 0) { +#if defined(FILEREF_DEBUG) + kore_log(LOG_DEBUG, "ref:%p scheduling for removal", + (void *)ref); +#endif + ref->expiration = kore_time_ms() + FILEREF_EXPIRATION; + } +} + +static void +fileref_expiration_check(void *arg, u_int64_t now) +{ + struct kore_fileref *ref, *next; + + for (ref = TAILQ_FIRST(&refs); ref != NULL; ref = next) { + next = TAILQ_NEXT(ref, list); + + if (ref->cnt != 0) + continue; + + if (ref->expiration > now) + continue; + +#if defined(FILEREF_DEBUG) + kore_log(LOG_DEBUG, "ref:%p expired, removing", (void *)ref); +#endif + TAILQ_REMOVE(&refs, ref, list); + kore_free(ref->path); + +#if defined(KORE_USE_PLATFORM_SENDFILE) + close(ref->fd); +#else + (void)munmap(ref->base, ref->size); +#endif + kore_pool_put(&ref_pool, ref); + } +} diff --git a/src/http.c b/src/http.c @@ -42,6 +42,25 @@ #include "tasks.h" #endif +static struct { + const char *ext; + const char *type; +} builtin_media[] = { + { "gif", "image/gif" }, + { "png", "image/png" }, + { "jpeg", "image/jpeg" }, + { "jpg", "image/jpeg" }, + { "zip", "application/zip" }, + { "pdf", "application/pdf" }, + { "json", "application/json" }, + { "js", "application/javascript" }, + { "htm", "text/html" }, + { "txt", "text/plain" }, + { "css", "text/css" }, + { "html", "text/html" }, + { NULL, NULL }, +}; + static int http_body_recv(struct netbuf *); static void http_error_response(struct connection *, int); static void http_write_response_cookie(struct http_cookie *); @@ -68,6 +87,7 @@ static char http_version[64]; static u_int16_t http_version_len; static TAILQ_HEAD(, http_request) http_requests; static TAILQ_HEAD(, http_request) http_requests_sleeping; +static LIST_HEAD(, http_media_type) http_media_types; static struct kore_pool http_request_pool; static struct kore_pool http_header_pool; static struct kore_pool http_cookie_pool; @@ -84,9 +104,15 @@ u_int64_t http_body_disk_offload = HTTP_BODY_DISK_OFFLOAD; char *http_body_disk_path = HTTP_BODY_DISK_PATH; void +http_parent_init(void) +{ + LIST_INIT(&http_media_types); +} + +void http_init(void) { - int prealloc, l; + int prealloc, l, i; TAILQ_INIT(&http_requests); TAILQ_INIT(&http_requests_sleeping); @@ -111,6 +137,14 @@ http_init(void) kore_pool_init(&http_body_path, "http_body_path", HTTP_BODY_PATH_MAX, prealloc); + + for (i = 0; builtin_media[i].ext != NULL; i++) { + if (!http_media_register(builtin_media[i].ext, + builtin_media[i].type)) { + fatal("duplicate media type for %s", + builtin_media[i].ext); + } + } } void @@ -463,6 +497,35 @@ http_response_stream(struct http_request *req, int status, void *base, } } +void +http_response_fileref(struct http_request *req, int status, + struct kore_fileref *ref) +{ + const char *media_type; + + if (req->owner == NULL) + return; + + media_type = http_media_type(ref->path); + if (media_type != NULL) + http_response_header(req, "content-type", media_type); + + req->status = status; + switch (req->owner->proto) { + case CONN_PROTO_HTTP: + http_response_normal(req, req->owner, status, NULL, ref->size); + break; + default: + fatal("http_response_fd() bad proto %d", req->owner->proto); + /* NOTREACHED. */ + } + + if (req->method != HTTP_METHOD_HEAD) + net_send_fileref(req->owner, ref); + else + kore_fileref_release(ref); +} + int http_request_header(struct http_request *req, const char *header, const char **out) @@ -1914,3 +1977,43 @@ http_method_text(int method) return (r); } + +int +http_media_register(const char *ext, const char *type) +{ + struct http_media_type *media; + + LIST_FOREACH(media, &http_media_types, list) { + if (!strcasecmp(media->ext, ext)) + return (KORE_RESULT_ERROR); + } + + media = kore_calloc(1, sizeof(*media)); + media->ext = kore_strdup(ext); + media->type = kore_strdup(type); + + LIST_INSERT_HEAD(&http_media_types, media, list); + + return (KORE_RESULT_OK); +} + +const char * +http_media_type(const char *path) +{ + const char *p; + struct http_media_type *media; + + if ((p = strrchr(path, '.')) == NULL) + return (NULL); + + p++; + if (*p == '\0') + return (NULL); + + LIST_FOREACH(media, &http_media_types, list) { + if (!strcasecmp(media->ext, p)) + return (media->type); + } + + return (NULL); +} diff --git a/src/kore.c b/src/kore.c @@ -118,9 +118,7 @@ version(void) int main(int argc, char *argv[]) { -#if defined(KORE_SINGLE_BINARY) struct kore_runtime_call *rcall; -#endif int ch, flags; flags = 0; @@ -181,8 +179,10 @@ main(int argc, char *argv[]) kore_python_init(); #endif #if !defined(KORE_NO_HTTP) + http_parent_init(); kore_auth_init(); kore_validator_init(); + kore_filemap_init(); #endif kore_domain_init(); kore_module_init(); @@ -191,8 +191,7 @@ main(int argc, char *argv[]) #if !defined(KORE_SINGLE_BINARY) if (config_file == NULL) usage(); - kore_parse_config(); -#else +#endif kore_module_load(NULL, NULL, KORE_MODULE_NATIVE); kore_parse_config(); @@ -201,7 +200,6 @@ main(int argc, char *argv[]) kore_runtime_configure(rcall, argc, argv); kore_free(rcall); } -#endif kore_platform_init(); @@ -403,6 +401,8 @@ kore_signal_setup(void) } else { (void)signal(SIGINT, SIG_IGN); } + + (void)signal(SIGPIPE, SIG_IGN); } void diff --git a/src/linux.c b/src/linux.c @@ -14,8 +14,10 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include <sys/param.h> #include <sys/epoll.h> #include <sys/prctl.h> +#include <sys/sendfile.h> #include <sched.h> @@ -269,3 +271,37 @@ kore_platform_proctitle(char *title) kore_debug("prctl(): %s", errno_s); } } + +#if defined(KORE_USE_PLATFORM_SENDFILE) +int +kore_platform_sendfile(struct connection *c, struct netbuf *nb) +{ + size_t len; + ssize_t sent; + off_t smin; + + smin = nb->fd_len - nb->fd_off; + len = MIN(SENDFILE_PAYLOAD_MAX, smin); + + if ((sent = sendfile(c->fd, nb->file_ref->fd, NULL, len)) == -1) { + if (errno == EAGAIN) { + c->flags &= ~CONN_WRITE_POSSIBLE; + return (KORE_RESULT_OK); + } + + if (errno == EINTR) + return (KORE_RESULT_OK); + + return (KORE_RESULT_ERROR); + } + + nb->fd_off += (size_t)sent; + + if (sent == 0 || nb->fd_off == nb->fd_len) { + net_remove_netbuf(&(c->send_queue), nb); + c->snb = NULL; + } + + return (KORE_RESULT_OK); +} +#endif diff --git a/src/net.c b/src/net.c @@ -24,6 +24,7 @@ #define be64toh(x) OSSwapBigToHostInt64(x) #else #include <sys/endian.h> +#include <sys/stdint.h> #endif #include "kore.h" @@ -47,6 +48,33 @@ net_cleanup(void) kore_pool_cleanup(&nb_pool); } +struct netbuf * +net_netbuf_get(void) +{ + struct netbuf *nb; + + nb = kore_pool_get(&nb_pool); + + nb->cb = NULL; + nb->buf = NULL; + nb->owner = NULL; + nb->extra = NULL; + nb->file_ref = NULL; + + nb->type = 0; + nb->s_off = 0; + nb->b_len = 0; + nb->m_len = 0; + nb->flags = 0; + +#if defined(KORE_USE_PLATFORM_SENDFILE) + nb->fd_off = -1; + nb->fd_len = -1; +#endif + + return (nb); +} + void net_send_queue(struct connection *c, const void *data, size_t len) { @@ -76,11 +104,9 @@ net_send_queue(struct connection *c, const void *data, size_t len) } } - nb = kore_pool_get(&nb_pool); - nb->flags = 0; - nb->cb = NULL; + nb = net_netbuf_get(); + nb->owner = c; - nb->s_off = 0; nb->b_len = len; nb->type = NETBUF_SEND; @@ -103,10 +129,9 @@ net_send_stream(struct connection *c, void *data, size_t len, kore_debug("net_send_stream(%p, %p, %zu)", c, data, len); - nb = kore_pool_get(&nb_pool); + nb = net_netbuf_get(); nb->cb = cb; nb->owner = c; - nb->s_off = 0; nb->buf = data; nb->b_len = len; nb->m_len = nb->b_len; @@ -119,6 +144,30 @@ net_send_stream(struct connection *c, void *data, size_t len, } void +net_send_fileref(struct connection *c, struct kore_fileref *ref) +{ + struct netbuf *nb; + + nb = net_netbuf_get(); + nb->owner = c; + nb->file_ref = ref; + nb->type = NETBUF_SEND; + nb->flags = NETBUF_IS_FILEREF; + +#if defined(KORE_USE_PLATFORM_SENDFILE) + nb->fd_off = 0; + nb->fd_len = ref->size; +#else + nb->buf = ref->base; + nb->b_len = ref->size; + nb->m_len = nb->b_len; + nb->flags |= NETBUF_IS_STREAM; +#endif + + TAILQ_INSERT_TAIL(&(c->send_queue), nb, list); +} + +void net_recv_reset(struct connection *c, size_t len, int (*cb)(struct netbuf *)) { kore_debug("net_recv_reset(): %p %zu", c, len); @@ -145,13 +194,11 @@ net_recv_queue(struct connection *c, size_t len, int flags, if (c->rnb != NULL) fatal("net_recv_queue(): called incorrectly"); - c->rnb = kore_pool_get(&nb_pool); + c->rnb = net_netbuf_get(); c->rnb->cb = cb; c->rnb->owner = c; - c->rnb->s_off = 0; c->rnb->b_len = len; c->rnb->m_len = len; - c->rnb->extra = NULL; c->rnb->flags = flags; c->rnb->type = NETBUF_RECV; c->rnb->buf = kore_malloc(c->rnb->b_len); @@ -174,6 +221,14 @@ net_send(struct connection *c) size_t r, len, smin; c->snb = TAILQ_FIRST(&(c->send_queue)); + +#if defined(KORE_USE_PLATFORM_SENDFILE) + if ((c->snb->flags & NETBUF_IS_FILEREF) && + !(c->snb->flags & NETBUF_IS_STREAM)) { + return (kore_platform_sendfile(c, c->snb)); + } +#endif + if (c->snb->b_len != 0) { smin = c->snb->b_len - c->snb->s_off; len = MIN(NETBUF_SEND_PAYLOAD_MAX, smin); @@ -262,6 +317,9 @@ net_remove_netbuf(struct netbuf_head *list, struct netbuf *nb) (void)nb->cb(nb); } + if (nb->flags & NETBUF_IS_FILEREF) + kore_fileref_release(nb->file_ref); + TAILQ_REMOVE(list, nb, list); kore_pool_put(&nb_pool, nb); } diff --git a/src/worker.c b/src/worker.c @@ -317,6 +317,7 @@ kore_worker_entry(struct kore_worker *kw) kore_accesslog_worker_init(); #endif kore_timer_init(); + kore_fileref_init(); kore_connection_init(); kore_domain_load_crl(); kore_domain_keymgr_init();