kore

An easy to use, scalable and secure web application framework for writing web APIs in C.
Commits | Files | Refs | README | LICENSE | git clone https://git.kore.io/kore.git

commit 937c39f0416f4f50164b5ec6ac3a760c4c27984c
parent 296fe7a6d43959aadefefa630ef1c476feee939a
Author: Joris Vink <joris@coders.se>
Date:   Thu, 26 Sep 2019 15:49:00 +0200

Many many Python improvements.

- Kore can now fully be configured via Python code if one wants nothing to
  do with configuration files.

- Kore can now start single python files and no longer requires them to be
  inside a module directory.

- Pass all regex capture groups to the handler methods, allowing you to
  get access to them immediately.

- Change python websocket_handshake to take callable objects directly.

- Added a new deployment configuration option. If set to "dev" or
  "development" Kore will automatically foreground, no chroot / etc.
  If set to "production" Kore *will* chroot, drop privs, etc.

- Many more..

These are all backported from a project that I was working on a while
ago. I decided these should go back into mainline Kore.

Diffstat:
include/kore/http.h | 5+++++
include/kore/kore.h | 10+++++++---
include/kore/python_methods.h | 53++++++++++++++++++++++++++++++++++++++++++++++++++++-
src/config.c | 157+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------
src/domain.c | 2+-
src/http.c | 41+++++++++++++++++++++++++++++++++++++++--
src/kore.c | 49++++++++++++++++++++++++++++++++++---------------
src/module.c | 6++++--
src/python.c | 555+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
9 files changed, 800 insertions(+), 78 deletions(-)

diff --git a/include/kore/http.h b/include/kore/http.h @@ -230,6 +230,8 @@ struct http_file { #define HTTP_BODY_DIGEST_LEN 32 #define HTTP_BODY_DIGEST_STRLEN ((HTTP_BODY_DIGEST_LEN * 2) + 1) +#define HTTP_CAPTURE_GROUPS 17 + struct reqcall; struct kore_task; struct http_client; @@ -266,9 +268,11 @@ struct http_request { #if defined(KORE_USE_PYTHON) void *py_req; void *py_coro; + void *py_validator; struct reqcall *py_rqnext; #endif + regmatch_t cgroups[HTTP_CAPTURE_GROUPS]; u_int8_t http_body_digest[HTTP_BODY_DIGEST_LEN]; #if defined(KORE_USE_CURL) @@ -330,6 +334,7 @@ const char *http_status_text(int); const char *http_method_text(int); time_t http_date_to_time(char *); char *http_validate_header(char *); +int http_method_value(const char *); void http_start_recv(struct connection *); void http_request_free(struct http_request *); void http_request_sleep(struct http_request *); diff --git a/include/kore/kore.h b/include/kore/kore.h @@ -674,6 +674,10 @@ int kore_connection_accept(struct listener *, u_int64_t kore_time_ms(void); void kore_log_init(void); +#if defined(KORE_USE_PYTHON) +int kore_configure_setting(const char *, char *); +#endif + void *kore_malloc(size_t); void kore_parse_config(void); void kore_parse_config_file(FILE *); @@ -748,7 +752,7 @@ void kore_fileref_release(struct kore_fileref *); void kore_domain_init(void); void kore_domain_cleanup(void); -int kore_domain_new(char *); +int kore_domain_new(const char *); void kore_domain_free(struct kore_domain *); void kore_module_init(void); void kore_module_cleanup(void); @@ -766,8 +770,8 @@ void kore_domain_crl_add(struct kore_domain *, const void *, size_t); int kore_module_handler_new(const char *, const char *, const char *, const char *, int); void kore_module_handler_free(struct kore_module_handle *); -struct kore_module_handle *kore_module_handler_find(const char *, - const char *); +struct kore_module_handle *kore_module_handler_find(struct http_request *, + const char *, const char *); #endif struct kore_runtime_call *kore_runtime_getcall(const char *); diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h @@ -44,6 +44,7 @@ static PyObject *python_kore_fatal(PyObject *, PyObject *); static PyObject *python_kore_queue(PyObject *, PyObject *); static PyObject *python_kore_worker(PyObject *, PyObject *); static PyObject *python_kore_tracer(PyObject *, PyObject *); +static PyObject *python_kore_domain(PyObject *, PyObject *); static PyObject *python_kore_fatalx(PyObject *, PyObject *); static PyObject *python_kore_setname(PyObject *, PyObject *); static PyObject *python_kore_suspend(PyObject *, PyObject *); @@ -72,7 +73,7 @@ static PyObject *python_websocket_broadcast(PyObject *, PyObject *); #define METHOD(n, c, a) { n, (PyCFunction)c, a, NULL } #define GETTER(n, g) { n, (getter)g, NULL, NULL, NULL } -#define SETTER(n, s) { n, NULL, (setter)g, NULL, NULL } +#define SETTER(n, s) { n, NULL, (setter)s, NULL, NULL } #define GETSET(n, g, s) { n, (getter)g, (setter)s, NULL, NULL } static struct PyMethodDef pykore_methods[] = { @@ -85,6 +86,7 @@ static struct PyMethodDef pykore_methods[] = { METHOD("queue", python_kore_queue, METH_VARARGS), METHOD("worker", python_kore_worker, METH_VARARGS), METHOD("tracer", python_kore_tracer, METH_VARARGS), + METHOD("domain", python_kore_domain, METH_VARARGS), METHOD("gather", python_kore_gather, METH_VARARGS | METH_KEYWORDS), METHOD("fatal", python_kore_fatal, METH_VARARGS), METHOD("fatalx", python_kore_fatalx, METH_VARARGS), @@ -115,6 +117,55 @@ static struct PyModuleDef pykore_module = { PyModuleDef_HEAD_INIT, "kore", NULL, -1, pykore_methods }; +struct pyconfig { + PyObject_HEAD +}; + +static int pyconfig_setattr(PyObject *, PyObject *, PyObject *); + +static PyTypeObject pyconfig_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.config", + .tp_doc = "kore configuration", + .tp_setattro = pyconfig_setattr, + .tp_basicsize = sizeof(struct pyconfig), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +struct pydomain { + PyObject_HEAD + struct kore_domain *config; +}; + +static PyObject *pydomain_filemaps(struct pydomain *, PyObject *); +static PyObject *pydomain_route(struct pydomain *, PyObject *, PyObject *); + +static PyMethodDef pydomain_methods[] = { + METHOD("filemaps", pydomain_filemaps, METH_VARARGS), + METHOD("route", pydomain_route, METH_VARARGS | METH_KEYWORDS), + METHOD(NULL, NULL, -1) +}; + +static int pydomain_set_accesslog(struct pydomain *, PyObject *, void *); + +static PyGetSetDef pydomain_getset[] = { + SETTER("accesslog", pydomain_set_accesslog), + SETTER(NULL, NULL), +}; + +static void pydomain_dealloc(struct pydomain *); + +static PyTypeObject pydomain_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.domain", + .tp_doc = "kore domain configuration", + .tp_getset = pydomain_getset, + .tp_methods = pydomain_methods, + .tp_basicsize = sizeof(struct pydomain), + .tp_dealloc = (destructor)pydomain_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + #define PYSUSPEND_OP_INIT 1 #define PYSUSPEND_OP_WAIT 2 #define PYSUSPEND_OP_CONTINUE 3 diff --git a/src/config.c b/src/config.c @@ -53,6 +53,8 @@ static int configure_load(char *); static FILE *config_file_write(void); extern u_int8_t asset_builtin_kore_conf[]; extern u_int32_t asset_len_builtin_kore_conf; +#elif defined(KORE_USE_PYTHON) +static int configure_file(char *); #endif static int configure_include(char *); @@ -129,6 +131,7 @@ static int configure_task_threads(char *); #endif #if defined(KORE_USE_PYTHON) +static int configure_deployment(char *); static int configure_python_path(char *); static int configure_python_import(char *); #endif @@ -141,18 +144,46 @@ static int configure_curl_recv_max(char *); static struct { const char *name; int (*configure)(char *); -} config_names[] = { +} config_directives[] = { { "include", configure_include }, { "bind", configure_bind }, { "bind_unix", configure_bind_unix }, { "load", configure_load }, + { "domain", configure_domain }, #if defined(KORE_USE_PYTHON) { "python_path", configure_python_path }, { "python_import", configure_python_import }, #endif +#if !defined(KORE_NO_TLS) + { "certfile", configure_certfile }, + { "certkey", configure_certkey }, + { "client_verify", configure_client_verify }, + { "client_verify_depth", configure_client_verify_depth }, +#endif +#if !defined(KORE_NO_HTTP) + { "filemap", configure_filemap }, + { "static", configure_static_handler }, + { "dynamic", configure_dynamic_handler }, + { "accesslog", configure_accesslog }, + { "restrict", configure_restrict }, + { "validator", configure_validator }, + { "params", configure_params }, + { "validate", configure_validate }, + { "authentication", configure_authentication }, + { "authentication_uri", configure_authentication_uri }, + { "authentication_type", configure_authentication_type }, + { "authentication_value", configure_authentication_value }, + { "authentication_validator", configure_authentication_validator }, +#endif + { NULL, NULL }, +}; + +static struct { + const char *name; + int (*configure)(char *); +} config_settings[] = { { "root", configure_root }, { "chroot", configure_root }, - { "domain", configure_domain }, { "runas", configure_runas }, { "workers", configure_workers }, { "worker_max_connections", configure_max_connections }, @@ -172,19 +203,10 @@ static struct { { "rand_file", configure_rand_file }, { "keymgr_runas", configure_keymgr_runas }, { "keymgr_root", configure_keymgr_root }, - { "certfile", configure_certfile }, - { "certkey", configure_certkey }, - { "client_verify", configure_client_verify }, - { "client_verify_depth", configure_client_verify_depth }, #endif #if !defined(KORE_NO_HTTP) - { "filemap", configure_filemap }, { "filemap_ext", configure_filemap_ext }, { "filemap_index", configure_filemap_index }, - { "static", configure_static_handler }, - { "dynamic", configure_dynamic_handler }, - { "accesslog", configure_accesslog }, - { "restrict", configure_restrict }, { "http_media_type", configure_http_media_type }, { "http_header_max", configure_http_header_max }, { "http_header_timeout", configure_http_header_timeout }, @@ -196,17 +218,12 @@ static struct { { "http_request_limit", configure_http_request_limit }, { "http_body_disk_offload", configure_http_body_disk_offload }, { "http_body_disk_path", configure_http_body_disk_path }, - { "validator", configure_validator }, - { "params", configure_params }, - { "validate", configure_validate }, - { "authentication", configure_authentication }, - { "authentication_uri", configure_authentication_uri }, - { "authentication_type", configure_authentication_type }, - { "authentication_value", configure_authentication_value }, - { "authentication_validator", configure_authentication_validator }, { "websocket_maxframe", configure_websocket_maxframe }, { "websocket_timeout", configure_websocket_timeout }, #endif +#if defined(KORE_USE_PYTHON) + { "deployment", configure_deployment }, +#endif #if defined(KORE_USE_PGSQL) { "pgsql_conn_max", configure_pgsql_conn_max }, { "pgsql_queue_limit", configure_pgsql_queue_limit }, @@ -218,9 +235,14 @@ static struct { { "curl_timeout", configure_curl_timeout }, { "curl_recv_max", configure_curl_recv_max }, #endif +#if !defined(KORE_SINGLE_BINARY) && defined(KORE_USE_PYTHON) + { "file", configure_file }, +#endif { NULL, NULL }, }; +static int finalized = 0; + #if !defined(KORE_SINGLE_BINARY) char *config_file = NULL; #endif @@ -241,15 +263,26 @@ kore_parse_config(void) FILE *fp; char path[PATH_MAX]; + if (finalized) + return; + + fp = NULL; + #if !defined(KORE_SINGLE_BINARY) - if ((fp = fopen(config_file, "r")) == NULL) - fatal("configuration given cannot be opened: %s", config_file); + if (config_file != NULL) { + if ((fp = fopen(config_file, "r")) == NULL) { + fatal("configuration given cannot be opened: %s", + config_file); + } + } #else fp = config_file_write(); #endif - kore_parse_config_file(fp); - (void)fclose(fp); + if (fp != NULL) { + kore_parse_config_file(fp); + (void)fclose(fp); + } if (!kore_module_loaded()) fatal("no application module was loaded"); @@ -304,6 +337,8 @@ kore_parse_config(void) if (skip_chroot && !kore_quiet) kore_log(LOG_WARNING, "privsep: will not chroot"); + + finalized = 1; } void @@ -371,21 +406,58 @@ kore_parse_config_file(FILE *fp) continue; } - for (i = 0; config_names[i].name != NULL; i++) { - if (!strcmp(config_names[i].name, p)) { - if (config_names[i].configure(t)) + for (i = 0; config_directives[i].name != NULL; i++) { + if (!strcmp(config_directives[i].name, p)) { + if (config_directives[i].configure(t)) + break; + fatal("configuration error on line %d", lineno); + /* NOTREACHED */ + } + } + + if (config_directives[i].name != NULL) { + lineno++; + continue; + } + + for (i = 0; config_settings[i].name != NULL; i++) { + if (!strcmp(config_settings[i].name, p)) { + if (config_settings[i].configure(t)) break; fatal("configuration error on line %d", lineno); /* NOTREACHED */ } } - if (config_names[i].name == NULL) + if (config_settings[i].name == NULL) printf("ignoring \"%s\" on line %d\n", p, lineno); + lineno++; } } +#if defined(KORE_USE_PYTHON) +int +kore_configure_setting(const char *name, char *value) +{ + int i; + + if (finalized) + return (KORE_RESULT_ERROR); + + for (i = 0; config_settings[i].name != NULL; i++) { + if (!strcmp(config_settings[i].name, name)) { + if (config_settings[i].configure(value)) + return (KORE_RESULT_OK); + fatal("bad value '%s' for '%s'", value, name); + } + } + + kore_log(LOG_NOTICE, "ignoring unknown kore.config.%s setting", name); + return (KORE_RESULT_OK); +} +#endif + static int configure_include(char *path) { @@ -476,6 +548,15 @@ config_file_write(void) return (fp); } +#elif defined(KORE_USE_PYTHON) +static int +configure_file(char *file) +{ + kore_free(config_file); + config_file = kore_strdup(file); + + return (KORE_RESULT_OK); +} #endif #if !defined(KORE_NO_TLS) @@ -1458,6 +1539,28 @@ configure_task_threads(char *option) #if defined(KORE_USE_PYTHON) static int +configure_deployment(char *value) +{ + if (!strcmp(value, "dev") || !strcmp(value, "development")) { + foreground = 1; + skip_runas = 1; + skip_chroot = 1; + } else if (!strcmp(value, "production")) { + foreground = 0; + skip_runas = 0; + skip_chroot = 0; + } else { + kore_log(LOG_NOTICE, + "pyko.config.deployment: bad value '%s'", value); + return (KORE_RESULT_ERROR); + } + + kore_log(LOG_NOTICE, "deployment set to %s", value); + + return (KORE_RESULT_OK); +} + +static int configure_python_path(char *path) { kore_python_path(path); diff --git a/src/domain.c b/src/domain.c @@ -191,7 +191,7 @@ kore_domain_cleanup(void) } int -kore_domain_new(char *domain) +kore_domain_new(const char *domain) { struct kore_domain *dom; diff --git a/src/http.c b/src/http.c @@ -442,6 +442,10 @@ http_request_free(struct http_request *req) kore_python_coro_delete(req->py_coro); req->py_coro = NULL; } + if (req->py_validator != NULL) { + kore_python_coro_delete(req->py_validator); + req->py_validator = NULL; + } Py_XDECREF(req->py_req); #endif #if defined(KORE_USE_PGSQL) @@ -1546,7 +1550,10 @@ http_request_new(struct connection *c, const char *host, break; } - if ((hdlr = kore_module_handler_find(host, path)) == NULL) { + req = kore_pool_get(&http_request_pool); + + if ((hdlr = kore_module_handler_find(req, host, path)) == NULL) { + kore_pool_put(&http_request_pool, req); http_error_response(c, 404); return (NULL); } @@ -1579,6 +1586,7 @@ http_request_new(struct connection *c, const char *host, m = HTTP_METHOD_PATCH; flags |= HTTP_REQUEST_EXPECT_BODY; } else { + kore_pool_put(&http_request_pool, req); http_error_response(c, 400); return (NULL); } @@ -1586,17 +1594,18 @@ http_request_new(struct connection *c, const char *host, if (flags & HTTP_VERSION_1_0) { if (m != HTTP_METHOD_GET && m != HTTP_METHOD_POST && m != HTTP_METHOD_HEAD) { + kore_pool_put(&http_request_pool, req); http_error_response(c, HTTP_STATUS_METHOD_NOT_ALLOWED); return (NULL); } } if (!(hdlr->methods & m)) { + kore_pool_put(&http_request_pool, req); http_error_response(c, HTTP_STATUS_METHOD_NOT_ALLOWED); return (NULL); } - req = kore_pool_get(&http_request_pool); req->end = 0; req->total = 0; req->start = 0; @@ -1625,6 +1634,7 @@ http_request_new(struct connection *c, const char *host, req->py_req = NULL; req->py_coro = NULL; req->py_rqnext = NULL; + req->py_validator = NULL; #endif if (qsoff > 0) { @@ -2260,6 +2270,33 @@ http_method_text(int method) } int +http_method_value(const char *method) +{ + if (!strcasecmp(method, "GET")) + return (HTTP_METHOD_GET); + + if (!strcasecmp(method, "POST")) + return (HTTP_METHOD_POST); + + if (!strcasecmp(method, "PUT")) + return (HTTP_METHOD_PUT); + + if (!strcasecmp(method, "DELETE")) + return (HTTP_METHOD_DELETE); + + if (!strcasecmp(method, "HEAD")) + return (HTTP_METHOD_HEAD); + + if (!strcasecmp(method, "OPTIONS")) + return (HTTP_METHOD_OPTIONS); + + if (!strcasecmp(method, "PATCH")) + return (HTTP_METHOD_PATCH); + + return (0); +} + +int http_media_register(const char *ext, const char *type) { struct http_media_type *media; diff --git a/src/kore.c b/src/kore.c @@ -76,6 +76,24 @@ static void kore_proctitle_setup(void); static void kore_server_sslstart(void); static void kore_server_start(int, char *[]); +#if !defined(KORE_SINGLE_BINARY) && defined(KORE_USE_PYTHON) +#define KORE_PARENT_CONFIG_METHOD "koreapp.configure" +#define KORE_PARENT_TEARDOWN_METHOD "koreapp.cleanup" +#define KORE_PARENT_DAEMONIZED_METHOD "koreapp.daemonized" +#endif + +#if !defined(KORE_PARENT_CONFIG_METHOD) +#define KORE_PARENT_CONFIG_METHOD "kore_parent_configure" +#endif + +#if !defined(KORE_PARENT_TEARDOWN_METHOD) +#define KORE_PARENT_TEARDOWN_METHOD "kore_parent_teardown" +#endif + +#if !defined(KORE_PARENT_DAEMONIZED_METHOD) +#define KORE_PARENT_DAEMONIZED_METHOD "kore_parent_daemonized" +#endif + static void usage(void) { @@ -210,13 +228,13 @@ main(int argc, char *argv[]) kore_pymodule = pwd; } + config_file = NULL; + if (lstat(kore_pymodule, &st) == -1) fatal("failed to stat '%s': %s", kore_pymodule, errno_s); - if (!S_ISDIR(st.st_mode)) - fatal("%s: not a directory", kore_pymodule); - - config_file = "kore.conf"; + if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) + fatal("%s: not a directory or file", kore_pymodule); #elif !defined(KORE_SINGLE_BINARY) if (argc > 0) fatal("did you mean to run `kodev' instead?"); @@ -244,7 +262,7 @@ main(int argc, char *argv[]) kore_module_init(); kore_server_sslstart(); -#if !defined(KORE_SINGLE_BINARY) +#if !defined(KORE_SINGLE_BINARY) && !defined(KORE_USE_PYTHON) if (config_file == NULL) usage(); #endif @@ -254,21 +272,20 @@ main(int argc, char *argv[]) kore_python_init(); #if !defined(KORE_SINGLE_BINARY) kore_module_load(kore_pymodule, NULL, KORE_MODULE_PYTHON); - if (chdir(kore_pymodule) == -1) + if (S_ISDIR(st.st_mode) && chdir(kore_pymodule) == -1) fatal("chdir(%s): %s", kore_pymodule, errno_s); #endif #endif - - kore_parse_config(); - -#if defined(KORE_SINGLE_BINARY) - rcall = kore_runtime_getcall("kore_parent_configure"); +#if defined(KORE_SINGLE_BINARY) || defined(KORE_USE_PYTHON) + rcall = kore_runtime_getcall(KORE_PARENT_CONFIG_METHOD); if (rcall != NULL) { kore_runtime_configure(rcall, argc, argv); kore_free(rcall); } #endif + kore_parse_config(); + #if !defined(KORE_NO_HTTP) if (http_body_disk_offload > 0) { if (mkdir(http_body_disk_path, 0700) == -1 && errno != EEXIST) { @@ -287,7 +304,7 @@ main(int argc, char *argv[]) kore_worker_shutdown(); - rcall = kore_runtime_getcall("kore_parent_teardown"); + rcall = kore_runtime_getcall(KORE_PARENT_TEARDOWN_METHOD); if (rcall != NULL) { kore_runtime_execute(rcall); kore_free(rcall); @@ -690,11 +707,13 @@ kore_server_start(int argc, char *argv[]) u_int64_t netwait; int quit, last_sig; + rcall = NULL; + if (foreground == 0) { if (daemon(1, 0) == -1) fatal("cannot daemon(): %s", errno_s); #if defined(KORE_SINGLE_BINARY) - rcall = kore_runtime_getcall("kore_parent_daemonized"); + rcall = kore_runtime_getcall(KORE_PARENT_DAEMONIZED_METHOD); if (rcall != NULL) { kore_runtime_execute(rcall); kore_free(rcall); @@ -721,8 +740,8 @@ kore_server_start(int argc, char *argv[]) #endif } -#if !defined(KORE_SINGLE_BINARY) - rcall = kore_runtime_getcall("kore_parent_configure"); +#if !defined(KORE_SINGLE_BINARY) && !defined(KORE_USE_PYTHON) + rcall = kore_runtime_getcall(KORE_PARENT_CONFIG_METHOD); if (rcall != NULL) { kore_runtime_configure(rcall, argc, argv); kore_free(rcall); diff --git a/src/module.c b/src/module.c @@ -286,7 +286,8 @@ kore_module_handler_free(struct kore_module_handle *hdlr) } struct kore_module_handle * -kore_module_handler_find(const char *domain, const char *path) +kore_module_handler_find(struct http_request *req, const char *domain, + const char *path) { struct kore_domain *dom; struct kore_module_handle *hdlr; @@ -299,7 +300,8 @@ kore_module_handler_find(const char *domain, const char *path) if (!strcmp(hdlr->path, path)) return (hdlr); } else { - if (!regexec(&(hdlr->rctx), path, 0, NULL, 0)) + if (!regexec(&(hdlr->rctx), path, + HTTP_CAPTURE_GROUPS, req->cgroups, 0)) return (hdlr); } } diff --git a/src/python.c b/src/python.c @@ -25,6 +25,8 @@ #include <ctype.h> #include <libgen.h> #include <signal.h> +#include <fcntl.h> +#include <unistd.h> #include "kore.h" #include "http.h" @@ -92,6 +94,10 @@ static int pyhttp_iterobj_chunk_sent(struct netbuf *); static int pyhttp_iterobj_next(struct pyhttp_iterobj *); static void pyhttp_iterobj_disconnect(struct connection *); +static int pydomain_params(PyObject *, + struct kore_module_handle *, const char *, int); +static int pydomain_auth(PyObject *, struct kore_module_handle *); + #if defined(KORE_USE_PGSQL) static int pykore_pgsql_result(struct pykore_pgsql *); static void pykore_pgsql_callback(struct kore_pgsql *, void *); @@ -734,8 +740,9 @@ pyhttp_file_dealloc(struct pyhttp_file *pyfile) static int python_runtime_http_request(void *addr, struct http_request *req) { - int ret; + int ret, idx, cnt; PyObject *pyret, *args, *callable; + PyObject *cargs[HTTP_CAPTURE_GROUPS + 1]; if (req->py_coro != NULL) { python_coro_wakeup(req->py_coro); @@ -776,15 +783,47 @@ python_runtime_http_request(void *addr, struct http_request *req) break; } + cnt = 0; callable = (PyObject *)addr; - if ((args = PyTuple_New(1)) == NULL) - fatal("python_runtime_http_request: PyTuple_New failed"); + /* starts at 1 to skip the full path. */ + if (req->hdlr->type == HANDLER_TYPE_DYNAMIC) { + for (idx = 1; idx < HTTP_CAPTURE_GROUPS - 1; idx++) { + if (req->cgroups[idx].rm_so == -1 || + req->cgroups[idx].rm_eo == -1) + break; + + cargs[cnt] = PyUnicode_FromStringAndSize(req->path + + req->cgroups[idx].rm_so, + req->cgroups[idx].rm_eo - req->cgroups[idx].rm_so); + + if (cargs[cnt] == NULL) { + while (cnt-- >= 0) + Py_XDECREF(cargs[cnt]); + kore_python_log_error("http request"); + http_response(req, + HTTP_STATUS_INTERNAL_ERROR, NULL, 0); + return (KORE_RESULT_OK); + } + + cnt++; + } + } + + cargs[cnt] = NULL; + + if ((args = PyTuple_New(cnt + 1)) == NULL) + fatal("%s: PyTuple_New failed", __func__); Py_INCREF(req->py_req); if (PyTuple_SetItem(args, 0, req->py_req) != 0) fatal("python_runtime_http_request: PyTuple_SetItem failed"); + for (idx = 0; cargs[idx] != NULL; idx++) { + if (PyTuple_SetItem(args, 1 + idx, cargs[idx]) != 0) + fatal("%s: PyTuple_SetItem failed (%d)", __func__, idx); + } + PyErr_Clear(); pyret = PyObject_Call(callable, args, NULL); Py_DECREF(args); @@ -826,13 +865,13 @@ python_runtime_validator(void *addr, struct http_request *req, const void *data) fatal("%s: pyreq alloc failed", __func__); } - if (req->py_coro != NULL) { - coro = req->py_coro; + if (req->py_validator != NULL) { + coro = req->py_validator; python_coro_wakeup(coro); if (python_coro_run(coro) == KORE_RESULT_OK) { ret = python_validator_check(coro->result); kore_python_coro_delete(coro); - req->py_coro = NULL; + req->py_validator = NULL; return (ret); } @@ -872,12 +911,12 @@ python_runtime_validator(void *addr, struct http_request *req, const void *data) if (PyCoro_CheckExact(pyret)) { coro = python_coro_create(pyret, req); - req->py_coro = coro; + req->py_validator = coro; if (python_coro_run(coro) == KORE_RESULT_OK) { http_request_wakeup(req); ret = python_validator_check(coro->result); kore_python_coro_delete(coro); - req->py_coro = NULL; + req->py_validator = NULL; return (ret); } return (KORE_RESULT_RETRY); @@ -897,24 +936,16 @@ python_validator_check(PyObject *obj) if (obj == NULL) return (KORE_RESULT_ERROR); - if (!PyLong_Check(obj)) { + if (!PyBool_Check(obj)) { kore_log(LOG_WARNING, - "invalid return value from authenticator (not an int)"); + "validator did not return True/False"); ret = KORE_RESULT_ERROR; - } else { - ret = (int)PyLong_AsLong(obj); } - switch (ret) { - case KORE_RESULT_OK: - case KORE_RESULT_ERROR: - break; - default: - kore_log(LOG_WARNING, - "unsupported authenticator return value '%d'", ret); + if (obj == Py_True) + ret = KORE_RESULT_OK; + else ret = KORE_RESULT_ERROR; - break; - } return (ret); } @@ -1026,7 +1057,7 @@ python_runtime_configure(void *addr, int argc, char **argv) if (pyret == NULL) { kore_python_log_error("python_runtime_configure"); - fatal("failed to call configure method: wrong args?"); + fatal("failed to configure your application"); } Py_DECREF(pyret); @@ -1098,8 +1129,9 @@ python_runtime_connect(void *addr, struct connection *c) static PyMODINIT_FUNC python_module_init(void) { - int i; - PyObject *pykore; + int i; + struct pyconfig *config; + PyObject *pykore; if ((pykore = PyModule_Create(&pykore_module)) == NULL) fatal("python_module_init: failed to setup pykore module"); @@ -1109,6 +1141,7 @@ python_module_init(void) python_push_type("pytimer", pykore, &pytimer_type); python_push_type("pyqueue", pykore, &pyqueue_type); python_push_type("pysocket", pykore, &pysocket_type); + python_push_type("pydomain", pykore, &pydomain_type); python_push_type("pyconnection", pykore, &pyconnection_type); #if defined(KORE_USE_CURL) @@ -1123,9 +1156,62 @@ python_module_init(void) python_integers[i].value); } + if ((config = PyObject_New(struct pyconfig, &pyconfig_type)) == NULL) + fatal("failed to create config object"); + + if (PyObject_SetAttrString(pykore, "config", (PyObject *)config) == -1) + fatal("failed to add config object"); + return (pykore); } +static int +pyconfig_setattr(PyObject *self, PyObject *attr, PyObject *val) +{ + char *v; + int ret; + PyObject *repr; + const char *name, *value; + + ret = -1; + repr = NULL; + + if (!PyUnicode_Check(attr)) + fatal("setattr: attribute name not a unicode string"); + + if (PyLong_CheckExact(val)) { + if ((repr = PyObject_Repr(val)) == NULL) + return (-1); + value = PyUnicode_AsUTF8(repr); + } else if (PyUnicode_CheckExact(val)) { + value = PyUnicode_AsUTF8(val); + } else if (PyBool_Check(val)) { + if (val == Py_False) + value = "False"; + else + value = "True"; + } else { + fatal("invalid object, config expects integer, bool or string"); + } + + name = PyUnicode_AsUTF8(attr); + v = kore_strdup(value); + + if (!kore_configure_setting(name, v)) { + ret = -1; + PyErr_SetString(PyExc_RuntimeError, + "configured cannot be changed at runtime"); + } else { + ret = 0; + } + + kore_free(v); + + Py_XDECREF(repr); + + return (ret); +} + static void python_append_path(const char *path) { @@ -1406,6 +1492,57 @@ python_kore_tracer(PyObject *self, PyObject *args) } static PyObject * +python_kore_domain(PyObject *self, PyObject *args) +{ + const char *name; + struct pydomain *domain; +#if !defined(KORE_NO_TLS) + int depth; + const char *x509, *key, *ca; + + ca = NULL; + depth = -1; + + if (!PyArg_ParseTuple(args, "sss|si", &name, &x509, &key, &ca, &depth)) + return (NULL); + + if (ca != NULL && depth < 0) { + PyErr_Format(PyExc_RuntimeError, "invalid depth '%d'", depth); + return (NULL); + } +#else + if (!PyArg_ParseTuple(args, "s", &name)) + return (NULL); +#endif + + if (kore_domain_lookup(name) != NULL) { + PyErr_SetString(PyExc_RuntimeError, "domain exists"); + return (NULL); + } + + if ((domain = PyObject_New(struct pydomain, &pydomain_type)) == NULL) + return (NULL); + + if (!kore_domain_new(name)) + fatal("failed to create new domain configuration"); + + if ((domain->config = kore_domain_lookup(name)) == NULL) + fatal("failed to find new domain configuration"); + +#if !defined(KORE_NO_TLS) + domain->config->certkey = kore_strdup(key); + domain->config->certfile = kore_strdup(x509); + + if (ca != NULL) { + domain->config->cafile = kore_strdup(ca); + domain->config->x509_verify_depth = depth; + } +#endif + + return ((PyObject *)domain); +} + +static PyObject * python_kore_gather(PyObject *self, PyObject *args, PyObject *kwargs) { struct pygather_op *op; @@ -3795,12 +3932,33 @@ pyhttp_file_read(struct pyhttp_file *pyfile, PyObject *args) static PyObject * pyhttp_websocket_handshake(struct pyhttp_request *pyreq, PyObject *args) { - const char *onconnect, *onmsg, *ondisconnect; + struct connection *c; + PyObject *onconnect, *onmsg, *ondisconnect; - if (!PyArg_ParseTuple(args, "sss", &onconnect, &onmsg, &ondisconnect)) + if (!PyArg_ParseTuple(args, "OOO", &onconnect, &onmsg, &ondisconnect)) return (NULL); - kore_websocket_handshake(pyreq->req, onconnect, onmsg, ondisconnect); + kore_websocket_handshake(pyreq->req, NULL, NULL, NULL); + + c = pyreq->req->owner; + + Py_INCREF(onconnect); + Py_INCREF(onmsg); + Py_INCREF(ondisconnect); + + c->ws_connect = kore_calloc(1, sizeof(struct kore_runtime_call)); + c->ws_connect->addr = onconnect; + c->ws_connect->runtime = &kore_python_runtime; + + c->ws_message = kore_calloc(1, sizeof(struct kore_runtime_call)); + c->ws_message->addr = onmsg; + c->ws_message->runtime = &kore_python_runtime; + + c->ws_disconnect = kore_calloc(1, sizeof(struct kore_runtime_call)); + c->ws_disconnect->addr = ondisconnect; + c->ws_disconnect->runtime = &kore_python_runtime; + + python_runtime_connect(onconnect, c); Py_RETURN_TRUE; } @@ -4022,6 +4180,349 @@ pyhttp_file_get_filename(struct pyhttp_file *pyfile, void *closure) return (name); } +void +pydomain_dealloc(struct pydomain *domain) +{ + PyObject_Del((PyObject *)domain); +} + +static int +pydomain_set_accesslog(struct pydomain *domain, PyObject *arg, void *closure) +{ + const char *path; + + if (!PyUnicode_CheckExact(arg)) + return (-1); + + if (domain->config->accesslog != -1) { + PyErr_Format(PyExc_RuntimeError, + "domain %s accesslog already set", domain->config->domain); + return (-1); + } + + path = PyUnicode_AsUTF8(arg); + + domain->config->accesslog = open(path, + O_CREAT | O_APPEND | O_WRONLY, + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + + if (domain->config->accesslog == -1) { + PyErr_Format(PyExc_RuntimeError, + "failed to open accesslog for %s (%s:%s)", + domain->config->domain, path, errno_s); + return (-1); + } + + return (0); +} + +static PyObject * +pydomain_filemaps(struct pydomain *domain, PyObject *args) +{ + Py_ssize_t idx; + const char *url, *path; + PyObject *dict, *key, *value; + + if (!PyArg_ParseTuple(args, "O", &dict)) + return (NULL); + + if (!PyDict_CheckExact(dict)) + return (NULL); + + idx = 0; + while (PyDict_Next(dict, &idx, &key, &value)) { + if (!PyUnicode_CheckExact(key) || + !PyUnicode_CheckExact(value)) { + return (NULL); + } + + url = PyUnicode_AsUTF8(key); + path = PyUnicode_AsUTF8(value); + + if (!kore_filemap_create(domain->config, path, url)) { + PyErr_Format(PyExc_RuntimeError, + "failed to create filemap %s->%s for %s", + url, path, domain->config->domain); + return (NULL); + } + } + + Py_RETURN_NONE; +} + +static PyObject * +pydomain_route(struct pydomain *domain, PyObject *args, PyObject *kwargs) +{ + struct kore_module_handle *hdlr; + int method; + const char *path, *val; + Py_ssize_t list_len, idx; + PyObject *callable, *repr, *obj, *item; + + if (!PyArg_ParseTuple(args, "sO", &path, &callable)) + return (NULL); + + if (!PyCallable_Check(callable)) + return (NULL); + + TAILQ_FOREACH(hdlr, &domain->config->handlers, list) { + if (!strcmp(hdlr->path, path)) { + PyErr_Format(PyExc_RuntimeError, + "route '%s' exists", path); + return (NULL); + } + } + + if ((repr = PyObject_Repr(callable)) == NULL) + return (NULL); + + val = PyUnicode_AsUTF8(repr); + + hdlr = kore_calloc(1, sizeof(*hdlr)); + hdlr->dom = domain->config; + hdlr->func = kore_strdup(val); + hdlr->path = kore_strdup(path); + hdlr->methods = HTTP_METHOD_ALL; + TAILQ_INIT(&hdlr->params); + + Py_DECREF(repr); + + hdlr->rcall = kore_calloc(1, sizeof(struct kore_runtime_call)); + hdlr->rcall->addr = callable; + hdlr->rcall->runtime = &kore_python_runtime; + + if (kwargs != NULL) { + if ((obj = PyDict_GetItemString(kwargs, "methods")) != NULL) { + if (!PyList_CheckExact(obj)) { + kore_module_handler_free(hdlr); + return (NULL); + } + + hdlr->methods = 0; + list_len = PyList_Size(obj); + + for (idx = 0; idx < list_len; idx++) { + if ((item = PyList_GetItem(obj, idx)) == NULL) { + kore_module_handler_free(hdlr); + return (NULL); + } + + if ((val = PyUnicode_AsUTF8(item)) == NULL) { + kore_module_handler_free(hdlr); + return (NULL); + } + + method = http_method_value(val); + if (method == 0) { + PyErr_Format(PyExc_RuntimeError, + "unknown method '%s'", val); + kore_module_handler_free(hdlr); + return (NULL); + } + + hdlr->methods |= method; + if (method == HTTP_METHOD_GET) + hdlr->methods |= HTTP_METHOD_HEAD; + + if (!pydomain_params(kwargs, + hdlr, val, method)) { + kore_module_handler_free(hdlr); + return (NULL); + } + } + } + + if ((obj = PyDict_GetItemString(kwargs, "auth")) != NULL) { + if (!pydomain_auth(obj, hdlr)) { + kore_module_handler_free(hdlr); + return (NULL); + } + } + } + + if (path[0] == '/') { + hdlr->type = HANDLER_TYPE_STATIC; + } else { + hdlr->type = HANDLER_TYPE_DYNAMIC; + + if (regcomp(&hdlr->rctx, hdlr->path, REG_EXTENDED)) { + PyErr_SetString(PyExc_RuntimeError, + "failed to compile regex for path"); + kore_module_handler_free(hdlr); + return (NULL); + } + } + + Py_INCREF(callable); + TAILQ_INSERT_TAIL(&domain->config->handlers, hdlr, list); + + Py_RETURN_NONE; +} + +static int +pydomain_params(PyObject *kwargs, struct kore_module_handle *hdlr, + const char *method, int type) +{ + Py_ssize_t idx; + const char *val; + int vtype; + struct kore_validator *vldr; + struct kore_handler_params *param; + PyObject *obj, *key, *item; + + if ((obj = PyDict_GetItemString(kwargs, method)) == NULL) + return (KORE_RESULT_OK); + + if (!PyDict_CheckExact(obj)) + return (KORE_RESULT_ERROR); + + idx = 0; + while (PyDict_Next(obj, &idx, &key, &item)) { + if (!PyUnicode_CheckExact(key)) + return (KORE_RESULT_ERROR); + + val = PyUnicode_AsUTF8(key); + + if (PyUnicode_CheckExact(item)) { + vtype = KORE_VALIDATOR_TYPE_REGEX; + } else if (PyCallable_Check(item)) { + vtype = KORE_VALIDATOR_TYPE_FUNCTION; + } else { + PyErr_Format(PyExc_RuntimeError, + "validator '%s' must be regex or function", val); + return (KORE_RESULT_ERROR); + } + + vldr = kore_calloc(1, sizeof(*vldr)); + vldr->type = vtype; + + if (vtype == KORE_VALIDATOR_TYPE_REGEX) { + val = PyUnicode_AsUTF8(item); + if (regcomp(&(vldr->rctx), + val, REG_EXTENDED | REG_NOSUB)) { + PyErr_Format(PyExc_RuntimeError, + "Invalid regex (%s)", val); + kore_free(vldr); + return (KORE_RESULT_ERROR); + } + } else { + vldr->rcall = kore_calloc(1, sizeof(*vldr->rcall)); + vldr->rcall->addr = item; + vldr->rcall->runtime = &kore_python_runtime; + Py_INCREF(item); + } + + val = PyUnicode_AsUTF8(key); + vldr->name = kore_strdup(val); + + param = kore_calloc(1, sizeof(*param)); + param->flags = 0; + param->method = type; + param->validator = vldr; + param->name = kore_strdup(val); + + if (type == HTTP_METHOD_GET) + param->flags = KORE_PARAMS_QUERY_STRING; + + TAILQ_INSERT_TAIL(&hdlr->params, param, list); + } + + return (KORE_RESULT_OK); +} + +static int +pydomain_auth(PyObject *dict, struct kore_module_handle *hdlr) +{ + int type; + struct kore_auth *auth; + struct kore_validator *vldr; + PyObject *obj, *repr; + const char *value, *redir; + + if (!PyDict_CheckExact(dict)) + return (KORE_RESULT_ERROR); + + if ((obj = PyDict_GetItemString(dict, "type")) == NULL || + !PyUnicode_CheckExact(obj)) { + PyErr_Format(PyExc_RuntimeError, + "missing 'type' in auth dictionary for '%s'", hdlr->path); + return (KORE_RESULT_ERROR); + } + + value = PyUnicode_AsUTF8(obj); + + if (!strcmp(value, "cookie")) { + type = KORE_AUTH_TYPE_COOKIE; + } else if (!strcmp(value, "header")) { + type = KORE_AUTH_TYPE_HEADER; + } else { + PyErr_Format(PyExc_RuntimeError, + "invalid 'type' (%s) in auth dictionary for '%s'", + value, hdlr->path); + return (KORE_RESULT_ERROR); + } + + if ((obj = PyDict_GetItemString(dict, "value")) == NULL || + !PyUnicode_CheckExact(obj)) { + PyErr_Format(PyExc_RuntimeError, + "missing 'value' in auth dictionary for '%s'", hdlr->path); + return (KORE_RESULT_ERROR); + } + + value = PyUnicode_AsUTF8(obj); + + if ((obj = PyDict_GetItemString(dict, "redirect")) != NULL) { + if (!PyUnicode_CheckExact(obj)) { + PyErr_Format(PyExc_RuntimeError, + "redirect for auth on '%s' is not a string", + hdlr->path); + return (KORE_RESULT_ERROR); + } + + redir = PyUnicode_AsUTF8(obj); + } else { + redir = NULL; + } + + if ((obj = PyDict_GetItemString(dict, "verify")) == NULL || + !PyCallable_Check(obj)) { + PyErr_Format(PyExc_RuntimeError, + "missing 'verify' in auth dictionary for '%s'", hdlr->path); + return (KORE_RESULT_ERROR); + } + + auth = kore_calloc(1, sizeof(*auth)); + auth->type = type; + auth->value = kore_strdup(value); + + if (redir != NULL) + auth->redirect = kore_strdup(redir); + + vldr = kore_calloc(1, sizeof(*vldr)); + vldr->type = KORE_VALIDATOR_TYPE_FUNCTION; + + vldr->rcall = kore_calloc(1, sizeof(*vldr->rcall)); + vldr->rcall->addr = obj; + vldr->rcall->runtime = &kore_python_runtime; + Py_INCREF(obj); + + if ((repr = PyObject_Repr(obj)) == NULL) { + kore_free(vldr->rcall); + kore_free(vldr); + kore_free(auth); + return (KORE_RESULT_ERROR); + } + + value = PyUnicode_AsUTF8(repr); + vldr->name = kore_strdup(value); + Py_DECREF(repr); + + auth->validator = vldr; + hdlr->auth = auth; + + return (KORE_RESULT_OK); +} + #if defined(KORE_USE_PGSQL) static PyObject * python_kore_pgsql_query(PyObject *self, PyObject *args, PyObject *kwargs)