commit 2c88bc61200e6ecabc0aea2dcfa7474c5bdb0182
parent c89ba3daa3da19927e99515f982963d336080042
Author: Joris Vink <joris@coders.se>
Date: Wed, 24 Apr 2019 00:10:47 +0200
Add asynchronous libcurl support.
This commit adds the CURL=1 build option. When enabled allows
you to schedule CURL easy handles onto the Kore event loop.
It also adds an easy to use HTTP client API that abstracts away the
settings required from libcurl to make HTTP requests.
Tied together with HTTP request state machines this means you can
write fully asynchronous HTTP client requests in an easy way.
Additionally this exposes that API to the Python code as well
allowing you do to things like:
client = kore.httpclient("https://kore.io")
status, body = await client.get()
Introduces 2 configuration options:
- curl_recv_max
Max incoming bytes for a response.
- curl_timeout
Timeout in seconds before a transfer is cancelled.
This API also allows you to take the CURL easy handle and send emails
with it, run FTP, etc. All asynchronously.
Diffstat:
21 files changed, 1589 insertions(+), 7 deletions(-)
diff --git a/Makefile b/Makefile
@@ -105,6 +105,16 @@ ifneq ("$(PYTHON)", "")
FEATURES_INC+=$(KORE_PYTHON_INC)
endif
+ifneq ("$(CURL)", "")
+ S_SRC+=src/curl.c
+ KORE_CURL_LIB?=$(shell pkg-config --libs libcurl)
+ KORE_CURL_INC?=$(shell pkg-config --cflags libcurl)
+ LDFLAGS+=$(KORE_CURL_LIB)
+ CFLAGS+=$(KORE_CURL_INC) -DKORE_USE_CURL
+ FEATURES+=-DKORE_USE_CURL
+ FEATURES_INC+=$(KORE_CURL_INC)
+endif
+
ifneq ("$(SANITIZE)", "")
CFLAGS+=-fsanitize=$(SANITIZE)
LDFLAGS+=-fsanitize=$(SANITIZE)
diff --git a/examples/async-curl/.gitignore b/examples/async-curl/.gitignore
@@ -0,0 +1,6 @@
+*.o
+.flavor
+.objs
+ht.so
+assets.h
+cert
diff --git a/examples/async-curl/README.md b/examples/async-curl/README.md
@@ -0,0 +1,12 @@
+Kore asynchronous libcurl integration example.
+
+This example demonstrates how you can use the asynchronous libcurl
+api from Kore to perform HTTP client requests, or FTP requests, or send
+emails all in an asynchronous fashion.
+
+Run:
+```
+ $ kodev run
+ $ curl https://127.0.0.1:8888
+ $ curl https://127.0.0.1:8888/ftp
+```
diff --git a/examples/async-curl/conf/async-curl.conf b/examples/async-curl/conf/async-curl.conf
@@ -0,0 +1,14 @@
+# ht configuration
+
+bind 127.0.0.1 8888
+
+workers 1
+tls_dhparam dh2048.pem
+
+domain * {
+ certfile cert/server.pem
+ certkey cert/key.pem
+
+ static / http
+ static /ftp ftp
+}
diff --git a/examples/async-curl/conf/build.conf b/examples/async-curl/conf/build.conf
@@ -0,0 +1,34 @@
+# ht build config
+# You can switch flavors using: kodev flavor [newflavor]
+
+# Set to yes if you wish to produce a single binary instead
+# of a dynamic library. If you set this to yes you must also
+# set kore_source together with kore_flavor.
+single_binary=yes
+kore_source=../../
+kore_flavor=CURL=1
+
+# The flags below are shared between flavors
+cflags=-Wall -Wmissing-declarations -Wshadow
+cflags=-Wstrict-prototypes -Wmissing-prototypes
+cflags=-Wpointer-arith -Wcast-qual -Wsign-compare
+
+cxxflags=-Wall -Wmissing-declarations -Wshadow
+cxxflags=-Wpointer-arith -Wcast-qual -Wsign-compare
+
+# Mime types for assets served via the builtin asset_serve_*
+#mime_add=txt:text/plain; charset=utf-8
+#mime_add=png:image/png
+#mime_add=html:text/html; charset=utf-8
+
+dev {
+ # These flags are added to the shared ones when
+ # you build the "dev" flavor.
+ cflags=-g
+ cxxflags=-g
+}
+
+#prod {
+# You can specify additional flags here which are only
+# included if you build with the "prod" flavor.
+#}
diff --git a/examples/async-curl/src/ftp.c b/examples/async-curl/src/ftp.c
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2019 Joris Vink <joris@coders.se>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/*
+ * This example is the same as the HTTP one (see src/http.c) except
+ * we fetch an FTP URL.
+ */
+
+#include <kore/kore.h>
+#include <kore/http.h>
+#include <kore/curl.h>
+
+int ftp(struct http_request *);
+
+static int state_setup(struct http_request *);
+static int state_result(struct http_request *);
+
+static struct http_state states[] = {
+ KORE_HTTP_STATE(state_setup),
+ KORE_HTTP_STATE(state_result)
+};
+
+int
+ftp(struct http_request *req)
+{
+ return (http_state_run(states, 2, req));
+}
+
+static int
+state_setup(struct http_request *req)
+{
+ struct kore_curl *client;
+
+ client = http_state_create(req, sizeof(*client));
+
+ if (!kore_curl_init(client,
+ "http://ftp.eu.openbsd.org/pub/OpenBSD/README")) {
+ http_response(req, 500, NULL, 0);
+ return (HTTP_STATE_COMPLETE);
+ }
+
+ kore_curl_bind_request(client, req);
+ kore_curl_run(client);
+
+ req->fsm_state = 1;
+ return (HTTP_STATE_RETRY);
+}
+
+static int
+state_result(struct http_request *req)
+{
+ size_t len;
+ const u_int8_t *body;
+ struct kore_curl *client;
+
+ client = http_state_get(req);
+
+ if (!kore_curl_success(client)) {
+ kore_curl_logerror(client);
+ http_response(req, 500, NULL, 0);
+ } else {
+ kore_curl_response_as_bytes(client, &body, &len);
+ http_response(req, HTTP_STATUS_OK, body, len);
+ }
+
+ kore_curl_cleanup(client);
+
+ return (HTTP_STATE_COMPLETE);
+}
diff --git a/examples/async-curl/src/http.c b/examples/async-curl/src/http.c
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2019 Joris Vink <joris@coders.se>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/*
+ * This example demonstrates how easy it is to perform asynchronous
+ * HTTP client requests using the integrated libcurl support.
+ *
+ * In this example we setup 2 states for an HTTP request:
+ * 1) setup
+ * We initialize the HTTP request and fire it off.
+ * This will put our HTTP request to sleep and it be woken up
+ * by Kore when a response is available or something went wrong.
+ *
+ * 2) result
+ * After we have woken up we have access to the result.
+ */
+
+#include <kore/kore.h>
+#include <kore/http.h>
+#include <kore/curl.h>
+
+int http(struct http_request *);
+
+static int state_setup(struct http_request *);
+static int state_result(struct http_request *);
+
+/* Our states. */
+static struct http_state states[] = {
+ KORE_HTTP_STATE(state_setup),
+ KORE_HTTP_STATE(state_result)
+};
+
+/* Transcend into the HTTP state machine for a request. */
+int
+http(struct http_request *req)
+{
+ return (http_state_run(states, 2, req));
+}
+
+/*
+ * Setup the HTTP client request using the integrated curl API and the easy
+ * to use HTTP client api.
+ */
+static int
+state_setup(struct http_request *req)
+{
+ struct kore_curl *client;
+
+ client = http_state_create(req, sizeof(*client));
+
+ /* Initialize curl. */
+ if (!kore_curl_init(client, "https://kore.io")) {
+ http_response(req, 500, NULL, 0);
+ return (HTTP_STATE_COMPLETE);
+ }
+
+ /* Setup our HTTP client request. */
+ kore_curl_http_setup(client, HTTP_METHOD_GET, NULL, 0);
+
+ /* Add some headers. */
+ kore_curl_http_set_header(client, "x-source", "from-example");
+
+ /* We could opt to override some settings ourselves if we wanted. */
+ /* curl_easy_setopt(client->handle, CURLOPT_SSL_VERIFYHOST, 0); */
+ /* curl_easy_setopt(client->handle, CURLOPT_SSL_VERIFYPEER, 0); */
+
+ /*
+ * Bind the HTTP client request to our HTTP request so we get woken
+ * up once a response is available.
+ */
+ kore_curl_bind_request(client, req);
+
+ /*
+ * Now fire off the request onto the event loop.
+ * This will put us to sleep.
+ */
+ kore_curl_run(client);
+
+ /* Make sure we go onto the next state once woken up. */
+ req->fsm_state = 1;
+
+ /* Tell Kore we can't complete this immediately. */
+ return (HTTP_STATE_RETRY);
+}
+
+/*
+ * This state is called when a result for the HTTP request call is
+ * available to us.
+ */
+static int
+state_result(struct http_request *req)
+{
+ size_t len;
+ const u_int8_t *body;
+ const char *header;
+ struct kore_curl *client;
+
+ /* Get the state attached to the HTTP request. */
+ client = http_state_get(req);
+
+ /* Check if we were succesfull, if not log an error. */
+ if (!kore_curl_success(client)) {
+ kore_curl_logerror(client);
+ http_response(req, 500, NULL, 0);
+ } else {
+ /*
+ * Success! We now have the body available to us.
+ */
+ kore_curl_response_as_bytes(client, &body, &len);
+
+ /* We could check the existance of a header: */
+ if (kore_curl_http_get_header(client, "server", &header))
+ printf("got server header: '%s'\n", header);
+
+ /*
+ * Respond to our client with the status and body from
+ * the HTTP client request we did.
+ */
+ http_response(req, client->http.status, body, len);
+ }
+
+ /* Cleanup. */
+ kore_curl_cleanup(client);
+
+ /* State is now finished. */
+ return (HTTP_STATE_COMPLETE);
+}
diff --git a/examples/python-async/README.md b/examples/python-async/README.md
@@ -1,5 +1,8 @@
Kore python async/await examples.
+This example also shows off the asynchronous HTTP client support
+and requires libcurl on your machine.
+
Run:
```
$ kodev run
@@ -11,4 +14,5 @@ Test:
$ curl -k http://127.0.0.1:8888/lock
$ curl -k http://127.0.0.1:8888/proc
$ curl -k http://127.0.0.1:8888/socket
+ $ curl -k http://127.0.0.1:8888/httpclient
```
diff --git a/examples/python-async/conf/build.conf b/examples/python-async/conf/build.conf
@@ -6,7 +6,7 @@
# set kore_source together with kore_flavor.
single_binary=yes
kore_source=../../
-kore_flavor=PYTHON=1 NOTLS=1
+kore_flavor=PYTHON=1 CURL=1 NOTLS=1
# The flags below are shared between flavors
cflags=-Wall -Wmissing-declarations -Wshadow
diff --git a/examples/python-async/conf/python-async.conf b/examples/python-async/conf/python-async.conf
@@ -9,6 +9,7 @@ python_import ./src/async_lock.py
python_import ./src/async_queue.py
python_import ./src/async_process.py
python_import ./src/async_socket.py
+python_import ./src/async_http.py
domain * {
static /queue async_queue
@@ -17,4 +18,6 @@ domain * {
static /socket async_socket
static /socket-test socket_test
+
+ static /httpclient httpclient
}
diff --git a/examples/python-async/src/async_http.py b/examples/python-async/src/async_http.py
@@ -0,0 +1,47 @@
+#
+# Copyright (c) 2019 Joris Vink <joris@coders.se>
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+#
+
+#
+# Asynchronous HTTP client example.
+#
+
+import kore
+
+# Handler called for /httpclient
+async def httpclient(req):
+ # Create an httpclient.
+ client = kore.httpclient("https://kore.io")
+
+ # Do a simple GET request.
+ status, body = await client.get()
+ print("status: %d, body: '%s'" % (status, body))
+
+ # Reuse and perform another GET request, returning headers too this time.
+ status, headers, body = await client.get(return_headers=True)
+ print("status: %d, headers: '%s'" % (status, headers))
+
+ # What happens if we post something?
+ status, body = await client.post(body=b"hello world")
+ print("status: %d, body: '%s'" % (status, body))
+
+ # Add some custom headers to our requests.
+ status, body = await client.get(
+ headers={
+ "x-my-header": "async-http"
+ }
+ )
+
+ req.response(200, b'async done')
diff --git a/include/kore/curl.h b/include/kore/curl.h
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2019 Joris Vink <joris@coders.se>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __H_CURL_H
+#define __H_CURL_H
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#include <curl/curl.h>
+
+#include "http.h"
+
+#define KORE_CURL_TIMEOUT 60
+#define KORE_CURL_RECV_MAX (1024 * 1024 * 2)
+
+#define KORE_CURL_STATUS_UNKNOWN 0
+#define KORE_CURL_FLAG_HTTP_PARSED_HEADERS 0x0001
+#define KORE_CURL_FLAG_BOUND 0x0002
+
+#define KORE_CURL_TYPE_CUSTOM 1
+#define KORE_CURL_TYPE_HTTP_CLIENT 2
+
+struct kore_curl {
+ int type;
+ int flags;
+ CURLcode result;
+
+ char *url;
+ CURL *handle;
+ struct kore_buf *response;
+
+ struct http_request *req;
+ void *arg;
+ void (*cb)(struct kore_curl *, void *);
+
+ char errbuf[CURL_ERROR_SIZE];
+
+ /* For the simplified HTTP api. */
+ struct {
+ int status;
+ struct curl_slist *hdrlist;
+
+ struct kore_buf *tosend;
+ struct kore_buf *headers;
+
+ TAILQ_HEAD(, http_header) resp_hdrs;
+ } http;
+
+ LIST_ENTRY(kore_curl) list;
+};
+
+extern u_int16_t kore_curl_timeout;
+extern u_int64_t kore_curl_recv_max;
+
+void kore_curl_sysinit(void);
+void kore_curl_run(struct kore_curl *);
+void kore_curl_cleanup(struct kore_curl *);
+int kore_curl_success(struct kore_curl *);
+void kore_curl_logerror(struct kore_curl *);
+int kore_curl_init(struct kore_curl *, const char *);
+
+size_t kore_curl_tobuf(char *, size_t, size_t, void *);
+size_t kore_curl_frombuf(char *, size_t, size_t, void *);
+
+void kore_curl_http_parse_headers(struct kore_curl *);
+void kore_curl_http_set_header(struct kore_curl *, const char *,
+ const char *);
+int kore_curl_http_get_header(struct kore_curl *, const char *,
+ const char **);
+void kore_curl_http_setup(struct kore_curl *, int, const void *, size_t);
+
+char *kore_curl_response_as_string(struct kore_curl *);
+void kore_curl_response_as_bytes(struct kore_curl *,
+ const u_int8_t **, size_t *);
+
+void kore_curl_bind_request(struct kore_curl *, struct http_request *);
+void kore_curl_bind_callback(struct kore_curl *,
+ void (*cb)(struct kore_curl *, void *), void *);
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif
diff --git a/include/kore/http.h b/include/kore/http.h
@@ -221,6 +221,7 @@ struct http_file {
#define HTTP_BODY_DIGEST_STRLEN ((HTTP_BODY_DIGEST_LEN * 2) + 1)
struct kore_task;
+struct http_client;
struct http_request {
u_int8_t method;
@@ -254,8 +255,17 @@ struct http_request {
u_int8_t http_body_digest[HTTP_BODY_DIGEST_LEN];
+#if defined(KORE_USE_CURL)
+ LIST_HEAD(, kore_curl) chandles;
+#endif
+
+#if defined(KORE_USE_TASKS)
LIST_HEAD(, kore_task) tasks;
+#endif
+
+#if defined(KORE_USE_PGSQL)
LIST_HEAD(, kore_pgsql) pgsqls;
+#endif
TAILQ_HEAD(, http_cookie) req_cookies;
TAILQ_HEAD(, http_cookie) resp_cookies;
@@ -291,6 +301,7 @@ extern u_int32_t http_request_limit;
extern u_int32_t http_request_count;
extern u_int64_t http_body_disk_offload;
extern char *http_body_disk_path;
+extern struct kore_pool http_header_pool;
void kore_accesslog(struct http_request *);
@@ -302,6 +313,7 @@ void http_process(void);
const char *http_status_text(int);
const char *http_method_text(int);
time_t http_date_to_time(char *);
+char *http_validate_header(char *);
void http_request_free(struct http_request *);
void http_request_sleep(struct http_request *);
void http_request_wakeup(struct http_request *);
diff --git a/include/kore/kore.h b/include/kore/kore.h
@@ -164,6 +164,7 @@ TAILQ_HEAD(netbuf_head, netbuf);
#define KORE_TYPE_PGSQL_CONN 3
#define KORE_TYPE_TASK 4
#define KORE_TYPE_PYSOCKET 5
+#define KORE_TYPE_CURL_HANDLE 6
#define CONN_STATE_UNKNOWN 0
#define CONN_STATE_TLS_SHAKE 1
diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h
@@ -53,6 +53,11 @@ static PyObject *python_kore_gather(PyObject *, PyObject *, PyObject *);
static PyObject *python_kore_pgsql_register(PyObject *, PyObject *);
#endif
+#if defined(KORE_USE_CURL)
+static PyObject *python_kore_httpclient(PyObject *,
+ PyObject *, PyObject *);
+#endif
+
static PyObject *python_websocket_broadcast(PyObject *, PyObject *);
#define METHOD(n, c, a) { n, (PyCFunction)c, a, NULL }
@@ -81,6 +86,10 @@ static struct PyMethodDef pykore_methods[] = {
#if defined(KORE_USE_PGSQL)
METHOD("register_database", python_kore_pgsql_register, METH_VARARGS),
#endif
+#if defined(KORE_USE_CURL)
+ METHOD("httpclient", python_kore_httpclient,
+ METH_VARARGS | METH_KEYWORDS),
+#endif
{ NULL, NULL, 0, NULL }
};
@@ -639,6 +648,85 @@ static PyTypeObject pyhttp_file_type = {
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
};
+#if defined(KORE_USE_CURL)
+struct pyhttp_client {
+ PyObject_HEAD
+ char *url;
+ char *tlscert;
+ char *tlskey;
+};
+
+#define PYHTTP_CLIENT_OP_RUN 1
+#define PYHTTP_CLIENT_OP_RESULT 2
+
+struct pyhttp_client_op {
+ PyObject_HEAD
+ int state;
+ int headers;
+ struct kore_curl curl;
+ struct python_coro *coro;
+};
+
+static PyObject *pyhttp_client_op_await(PyObject *);
+static PyObject *pyhttp_client_op_iternext(struct pyhttp_client_op *);
+
+static void pyhttp_client_dealloc(struct pyhttp_client *);
+static void pyhttp_client_op_dealloc(struct pyhttp_client_op *);
+
+static PyObject *pyhttp_client_get(struct pyhttp_client *,
+ PyObject *, PyObject *);
+static PyObject *pyhttp_client_put(struct pyhttp_client *,
+ PyObject *, PyObject *);
+static PyObject *pyhttp_client_post(struct pyhttp_client *,
+ PyObject *, PyObject *);
+static PyObject *pyhttp_client_head(struct pyhttp_client *,
+ PyObject *, PyObject *);
+static PyObject *pyhttp_client_patch(struct pyhttp_client *,
+ PyObject *, PyObject *);
+static PyObject *pyhttp_client_delete(struct pyhttp_client *,
+ PyObject *, PyObject *);
+static PyObject *pyhttp_client_options(struct pyhttp_client *,
+ PyObject *, PyObject *);
+
+static PyMethodDef pyhttp_client_methods[] = {
+ METHOD("get", pyhttp_client_get, METH_VARARGS | METH_KEYWORDS),
+ METHOD("put", pyhttp_client_put, METH_VARARGS | METH_KEYWORDS),
+ METHOD("post", pyhttp_client_post, METH_VARARGS | METH_KEYWORDS),
+ METHOD("head", pyhttp_client_head, METH_VARARGS | METH_KEYWORDS),
+ METHOD("patch", pyhttp_client_patch, METH_VARARGS | METH_KEYWORDS),
+ METHOD("delete", pyhttp_client_delete, METH_VARARGS | METH_KEYWORDS),
+ METHOD("options", pyhttp_client_options, METH_VARARGS | METH_KEYWORDS),
+ METHOD(NULL, NULL, -1)
+};
+
+static PyTypeObject pyhttp_client_type = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "kore.httpclient",
+ .tp_doc = "An asynchronous HTTP client",
+ .tp_methods = pyhttp_client_methods,
+ .tp_basicsize = sizeof(struct pyhttp_client),
+ .tp_dealloc = (destructor)pyhttp_client_dealloc,
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+};
+
+static PyAsyncMethods pyhttp_client_op_async = {
+ (unaryfunc)pyhttp_client_op_await,
+ NULL,
+ NULL
+};
+
+static PyTypeObject pyhttp_client_op_type = {
+ PyVarObject_HEAD_INIT(NULL, 0)
+ .tp_name = "kore.httpclientop",
+ .tp_doc = "Asynchronous HTTP client operation",
+ .tp_as_async = &pyhttp_client_op_async,
+ .tp_iternext = (iternextfunc)pyhttp_client_op_iternext,
+ .tp_basicsize = sizeof(struct pyhttp_client_op),
+ .tp_dealloc = (destructor)pyhttp_client_op_dealloc,
+ .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+};
+#endif
+
#if defined(KORE_USE_PGSQL)
#define PYKORE_PGSQL_PREINIT 1
diff --git a/src/config.c b/src/config.c
@@ -41,6 +41,10 @@
#include "python_api.h"
#endif
+#if defined(KORE_USE_CURL)
+#include "curl.h"
+#endif
+
/* XXX - This is becoming a clusterfuck. Fix it. */
static int configure_load(char *);
@@ -129,6 +133,11 @@ static int configure_python_path(char *);
static int configure_python_import(char *);
#endif
+#if defined(KORE_USE_CURL)
+static int configure_curl_timeout(char *);
+static int configure_curl_recv_max(char *);
+#endif
+
static struct {
const char *name;
int (*configure)(char *);
@@ -205,6 +214,10 @@ static struct {
#if defined(KORE_USE_TASKS)
{ "task_threads", configure_task_threads },
#endif
+#if defined(KORE_USE_CURL)
+ { "curl_timeout", configure_curl_timeout },
+ { "curl_recv_max", configure_curl_recv_max },
+#endif
{ NULL, NULL },
};
@@ -1475,3 +1488,33 @@ configure_add_pledge(char *pledge)
return (KORE_RESULT_OK);
}
#endif
+
+#if defined(KORE_USE_CURL)
+static int
+configure_curl_recv_max(char *option)
+{
+ int err;
+
+ kore_curl_recv_max = kore_strtonum64(option, 1, &err);
+ if (err != KORE_RESULT_OK) {
+ printf("bad curl_recv_max value: %s\n", option);
+ return (KORE_RESULT_ERROR);
+ }
+
+ return (KORE_RESULT_OK);
+}
+
+static int
+configure_curl_timeout(char *option)
+{
+ int err;
+
+ kore_curl_timeout = kore_strtonum(option, 10, 0, USHRT_MAX, &err);
+ if (err != KORE_RESULT_OK) {
+ printf("bad kore_curl_timeout value: %s\n", option);
+ return (KORE_RESULT_ERROR);
+ }
+
+ return (KORE_RESULT_OK);
+}
+#endif
diff --git a/src/curl.c b/src/curl.c
@@ -0,0 +1,630 @@
+/*
+ * Copyright (c) 2019 Joris Vink <joris@coders.se>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/types.h>
+
+#include "kore.h"
+#include "http.h"
+#include "curl.h"
+
+#define FD_CACHE_BUCKETS 2048
+
+struct fd_cache {
+ struct kore_event evt;
+ int fd;
+ int scheduled;
+ LIST_ENTRY(fd_cache) list;
+};
+
+static void curl_process(void);
+static void curl_event_handle(void *, int);
+static int curl_timer(CURLM *, long, void *);
+static int curl_socket(CURL *, curl_socket_t, int, void *, void *);
+
+static struct fd_cache *fd_cache_get(int);
+
+static int running = 0;
+static CURLM *multi = NULL;
+static struct kore_timer *timer = NULL;
+static struct kore_pool fd_cache_pool;
+static LIST_HEAD(, fd_cache) cache[FD_CACHE_BUCKETS];
+
+u_int16_t kore_curl_timeout = KORE_CURL_TIMEOUT;
+u_int64_t kore_curl_recv_max = KORE_CURL_RECV_MAX;
+
+void
+kore_curl_sysinit(void)
+{
+ int i;
+ CURLMcode res;
+
+ if (curl_global_init(CURL_GLOBAL_ALL))
+ fatal("failed to initialize libcurl");
+
+ if ((multi = curl_multi_init()) == NULL)
+ fatal("curl_multi_init(): failed");
+
+ /* XXX - make configurable? */
+ curl_multi_setopt(multi, CURLMOPT_MAXCONNECTS, 500);
+
+ if ((res = curl_multi_setopt(multi,
+ CURLMOPT_SOCKETFUNCTION, curl_socket)) != CURLM_OK)
+ fatal("curl_multi_setopt: %s", curl_multi_strerror(res));
+
+ if ((res = curl_multi_setopt(multi,
+ CURLMOPT_TIMERFUNCTION, curl_timer)) != CURLM_OK)
+ fatal("curl_multi_setopt: %s", curl_multi_strerror(res));
+
+ for (i = 0; i < FD_CACHE_BUCKETS; i++)
+ LIST_INIT(&cache[i]);
+
+ kore_pool_init(&fd_cache_pool, "fd_cache_pool", 100,
+ sizeof(struct fd_cache));
+}
+
+int
+kore_curl_init(struct kore_curl *client, const char *url)
+{
+ CURL *handle;
+
+ memset(client, 0, sizeof(*client));
+
+ TAILQ_INIT(&client->http.resp_hdrs);
+
+ if ((handle = curl_easy_init()) == NULL)
+ return (KORE_RESULT_ERROR);
+
+ curl_easy_setopt(handle, CURLOPT_WRITEDATA, &client->response);
+ curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, kore_curl_tobuf);
+
+ curl_easy_setopt(handle, CURLOPT_URL, url);
+ curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
+ curl_easy_setopt(handle, CURLOPT_PRIVATE, client);
+ curl_easy_setopt(handle, CURLOPT_TIMEOUT, kore_curl_timeout);
+ curl_easy_setopt(handle, CURLOPT_ERRORBUFFER, client->errbuf);
+
+ client->handle = handle;
+ client->url = kore_strdup(url);
+ client->type = KORE_CURL_TYPE_CUSTOM;
+
+ return (KORE_RESULT_OK);
+}
+
+void
+kore_curl_cleanup(struct kore_curl *client)
+{
+ struct http_header *hdr, *next;
+
+ kore_free(client->url);
+
+ if (client->flags & KORE_CURL_FLAG_BOUND)
+ LIST_REMOVE(client, list);
+
+ if (client->handle != NULL) {
+ curl_multi_remove_handle(multi, client->handle);
+ curl_easy_cleanup(client->handle);
+ }
+
+ if (client->http.hdrlist != NULL)
+ curl_slist_free_all(client->http.hdrlist);
+
+ if (client->response != NULL)
+ kore_buf_free(client->response);
+
+ if (client->http.headers != NULL)
+ kore_buf_free(client->http.headers);
+
+ if (client->http.tosend != NULL)
+ kore_buf_free(client->http.tosend);
+
+ for (hdr = TAILQ_FIRST(&client->http.resp_hdrs);
+ hdr != NULL; hdr = next) {
+ next = TAILQ_NEXT(hdr, list);
+ TAILQ_REMOVE(&client->http.resp_hdrs, hdr, list);
+ kore_pool_put(&http_header_pool, hdr);
+ }
+}
+
+size_t
+kore_curl_tobuf(char *ptr, size_t size, size_t nmemb, void *udata)
+{
+ size_t len;
+ struct kore_buf **buf, *b;
+
+ if (SIZE_MAX / nmemb < size)
+ fatal("%s: %zu * %zu overflow", __func__, nmemb, size);
+
+ buf = udata;
+ len = size * nmemb;
+
+ if (*buf == NULL)
+ *buf = kore_buf_alloc(len);
+
+ b = *buf;
+
+ if (b->offset + len < b->offset)
+ fatal("%s: %zu+%zu overflows", __func__, b->offset, len);
+
+ if ((b->offset + len) > kore_curl_recv_max) {
+ kore_log(LOG_ERR,
+ "received too large transfer (%zu > %" PRIu64 ")",
+ b->offset + len, kore_curl_recv_max);
+ return (0);
+ }
+
+ kore_buf_append(b, ptr, len);
+
+ return (len);
+}
+
+size_t
+kore_curl_frombuf(char *ptr, size_t size, size_t nmemb, void *udata)
+{
+ size_t len;
+ struct kore_buf *buf;
+
+ if (SIZE_MAX / nmemb < size)
+ fatal("%s: %zu * %zu overflow", __func__, nmemb, size);
+
+ buf = udata;
+ len = size * nmemb;
+
+ if (buf->offset == buf->length)
+ return (0);
+
+ if (buf->offset + len < buf->offset)
+ fatal("%s: %zu+%zu overflows", __func__, buf->offset, len);
+
+ if ((buf->offset + len) < buf->length) {
+ memcpy(ptr, buf->data + buf->offset, len);
+ } else {
+ len = buf->length - buf->offset;
+ memcpy(ptr, buf->data + buf->offset, len);
+ }
+
+ buf->offset += len;
+
+ return (len);
+}
+
+void
+kore_curl_bind_request(struct kore_curl *client, struct http_request *req)
+{
+ if (client->cb != NULL)
+ fatal("%s: already bound to callback", __func__);
+
+ client->req = req;
+ http_request_sleep(req);
+
+ client->flags |= KORE_CURL_FLAG_BOUND;
+ LIST_INSERT_HEAD(&req->chandles, client, list);
+}
+
+void
+kore_curl_bind_callback(struct kore_curl *client,
+ void (*cb)(struct kore_curl *, void *), void *arg)
+{
+ if (client->req != NULL)
+ fatal("%s: already bound to request", __func__);
+
+ client->cb = cb;
+ client->arg = arg;
+}
+
+void
+kore_curl_run(struct kore_curl *client)
+{
+ curl_multi_add_handle(multi, client->handle);
+}
+
+int
+kore_curl_success(struct kore_curl *client)
+{
+ return (client->result == CURLE_OK);
+}
+
+void
+kore_curl_logerror(struct kore_curl *client)
+{
+ kore_log(LOG_NOTICE, "curl error: %s -> %s",
+ client->url, client->errbuf);
+}
+
+void
+kore_curl_response_as_bytes(struct kore_curl *client, const u_int8_t **body,
+ size_t *len)
+{
+ if (client->response == NULL) {
+ *len = 0;
+ *body = NULL;
+ } else {
+ *len = client->response->offset;
+ *body = client->response->data;
+ }
+}
+
+char *
+kore_curl_response_as_string(struct kore_curl *client)
+{
+ kore_buf_stringify(client->response, NULL);
+
+ return ((char *)client->response->data);
+}
+
+void
+kore_curl_http_setup(struct kore_curl *client, int method, const void *data,
+ size_t len)
+{
+ const char *mname;
+ int has_body;
+
+ if (client->handle == NULL)
+ fatal("%s: called without setup", __func__);
+
+ mname = NULL;
+ has_body = 1;
+
+ client->type = KORE_CURL_TYPE_HTTP_CLIENT;
+
+ curl_easy_setopt(client->handle, CURLOPT_HEADERDATA,
+ &client->http.headers);
+ curl_easy_setopt(client->handle, CURLOPT_HEADERFUNCTION,
+ kore_curl_tobuf);
+
+ kore_curl_http_set_header(client, "expect", "");
+
+ switch (method) {
+ case HTTP_METHOD_GET:
+ break;
+ case HTTP_METHOD_HEAD:
+ curl_easy_setopt(client->handle, CURLOPT_NOBODY, 1);
+ break;
+ case HTTP_METHOD_DELETE:
+ case HTTP_METHOD_OPTIONS:
+ break;
+ case HTTP_METHOD_PUT:
+ has_body = 1;
+ curl_easy_setopt(client->handle, CURLOPT_UPLOAD, 1);
+ break;
+ case HTTP_METHOD_PATCH:
+ mname = http_method_text(method);
+ /* fallthrough */
+ case HTTP_METHOD_POST:
+ has_body = 1;
+ curl_easy_setopt(client->handle, CURLOPT_POST, 1);
+ break;
+ default:
+ fatal("%s: unknown method %d", __func__, method);
+ }
+
+ if (has_body && data != NULL && len > 0) {
+ client->http.tosend = kore_buf_alloc(len);
+ kore_buf_append(client->http.tosend, data, len);
+ kore_buf_reset(client->http.tosend);
+
+ curl_easy_setopt(client->handle, CURLOPT_READDATA,
+ client->http.tosend);
+ curl_easy_setopt(client->handle, CURLOPT_READFUNCTION,
+ kore_curl_frombuf);
+ }
+
+ if (has_body) {
+ if (method == HTTP_METHOD_PUT) {
+ curl_easy_setopt(client->handle,
+ CURLOPT_INFILESIZE_LARGE, len);
+ } else {
+ curl_easy_setopt(client->handle,
+ CURLOPT_POSTFIELDSIZE_LARGE, len);
+ }
+ } else {
+ if (data != NULL || len != 0) {
+ fatal("%s: %d should not have a body",
+ __func__, method);
+ }
+ }
+
+ if (mname != NULL)
+ curl_easy_setopt(client->handle, CURLOPT_CUSTOMREQUEST, mname);
+}
+
+void
+kore_curl_http_set_header(struct kore_curl *client, const char *header,
+ const char *value)
+{
+ struct kore_buf buf;
+ const char *hdr;
+
+ kore_buf_init(&buf, 512);
+
+ if (value != NULL || *value != '\0') {
+ kore_buf_appendf(&buf, "%s: %s", header, value);
+ } else {
+ kore_buf_appendf(&buf, "%s:", header);
+ }
+
+ hdr = kore_buf_stringify(&buf, NULL);
+
+ client->http.hdrlist = curl_slist_append(client->http.hdrlist, hdr);
+ kore_buf_cleanup(&buf);
+
+ curl_easy_setopt(client->handle,
+ CURLOPT_HTTPHEADER, client->http.hdrlist);
+}
+
+int
+kore_curl_http_get_header(struct kore_curl *client, const char *header,
+ const char **out)
+{
+ struct http_header *hdr;
+
+ if (!(client->flags & KORE_CURL_FLAG_HTTP_PARSED_HEADERS))
+ kore_curl_http_parse_headers(client);
+
+ TAILQ_FOREACH(hdr, &(client->http.resp_hdrs), list) {
+ if (!strcasecmp(hdr->header, header)) {
+ *out = hdr->value;
+ return (KORE_RESULT_OK);
+ }
+ }
+
+ return (KORE_RESULT_ERROR);
+}
+
+void
+kore_curl_http_parse_headers(struct kore_curl *client)
+{
+ struct http_header *hdr;
+ int i, cnt;
+ char *value, *hbuf, *headers[HTTP_REQ_HEADER_MAX];
+
+ if (client->flags & KORE_CURL_FLAG_HTTP_PARSED_HEADERS)
+ fatal("%s: headers already parsed", __func__);
+
+ client->flags |= KORE_CURL_FLAG_HTTP_PARSED_HEADERS;
+
+ if (client->http.headers == NULL)
+ return;
+
+ hbuf = kore_buf_stringify(client->http.headers, NULL);
+ cnt = kore_split_string(hbuf, "\r\n", headers, HTTP_REQ_HEADER_MAX);
+
+ for (i = 0; i < cnt; i++) {
+ if ((value = http_validate_header(headers[i])) == NULL)
+ continue;
+
+ if (*value == '\0')
+ continue;
+
+ hdr = kore_pool_get(&http_header_pool);
+ hdr->header = headers[i];
+ hdr->value = value;
+ TAILQ_INSERT_TAIL(&(client->http.resp_hdrs), hdr, list);
+ }
+}
+
+static int
+curl_socket(CURL *easy, curl_socket_t fd, int action, void *arg, void *sock)
+{
+ CURLcode res;
+ struct fd_cache *fdc;
+ struct kore_curl *client;
+
+ client = NULL;
+
+ res = curl_easy_getinfo(easy, CURLINFO_PRIVATE, &client);
+ if (res != CURLE_OK)
+ fatal("curl_easy_getinfo: %s", curl_easy_strerror(res));
+
+ if (client == NULL)
+ fatal("%s: failed to get client context", __func__);
+
+ fdc = fd_cache_get(fd);
+
+ switch (action) {
+ case CURL_POLL_NONE:
+ break;
+ case CURL_POLL_IN:
+ case CURL_POLL_OUT:
+ case CURL_POLL_INOUT:
+ if (fdc->scheduled == 0) {
+ kore_platform_event_all(fd, fdc);
+ fdc->scheduled = 1;
+ }
+ break;
+ case CURL_POLL_REMOVE:
+ if (fdc->scheduled) {
+ fdc->evt.flags = 0;
+ fdc->scheduled = 0;
+ kore_platform_disable_read(fd);
+#if !defined(__linux__)
+ kore_platform_disable_write(fd);
+#endif
+ }
+ break;
+ default:
+ fatal("unknown action value: %d", action);
+ }
+
+ /*
+ * XXX - libcurl hates edge triggered io.
+ */
+ if (action == CURL_POLL_OUT || action == CURL_POLL_INOUT) {
+ if (fdc->evt.flags & KORE_EVENT_WRITE) {
+ if (fdc->scheduled) {
+ kore_platform_disable_read(fdc->fd);
+#if !defined(__linux__)
+ kore_platform_disable_write(fdc->fd);
+#endif
+ }
+
+ fdc->evt.flags = 0;
+ kore_platform_event_all(fdc->fd, fdc);
+ }
+ }
+
+ return (CURLM_OK);
+}
+
+static void
+curl_process(void)
+{
+ CURLcode res;
+ CURLMsg *msg;
+ CURL *handle;
+ struct kore_curl *client;
+ int pending;
+
+ pending = 0;
+
+ while ((msg = curl_multi_info_read(multi, &pending)) != NULL) {
+ if (msg->msg != CURLMSG_DONE)
+ continue;
+
+ handle = msg->easy_handle;
+
+ res = curl_easy_getinfo(handle, CURLINFO_PRIVATE, &client);
+ if (res != CURLE_OK)
+ fatal("curl_easy_getinfo: %s", curl_easy_strerror(res));
+
+ if (client == NULL)
+ fatal("%s: failed to get client context", __func__);
+
+ client->result = msg->data.result;
+
+ if (client->type == KORE_CURL_TYPE_HTTP_CLIENT) {
+ curl_easy_getinfo(client->handle,
+ CURLINFO_RESPONSE_CODE, &client->http.status);
+ }
+
+ curl_multi_remove_handle(multi, client->handle);
+ curl_easy_cleanup(client->handle);
+
+ client->handle = NULL;
+
+ if (client->req != NULL)
+ http_request_wakeup(client->req);
+ else if (client->cb != NULL)
+ client->cb(client, client->arg);
+ }
+}
+
+static void
+curl_timeout(void *uarg, u_int64_t now)
+{
+ CURLMcode res;
+
+ timer = NULL;
+
+ res = curl_multi_socket_action(multi, CURL_SOCKET_TIMEOUT, 0, &running);
+ if (res != CURLM_OK)
+ fatal("curl_multi_socket_action: %s", curl_multi_strerror(res));
+
+ curl_process();
+}
+
+static int
+curl_timer(CURLM *mctx, long timeout, void *arg)
+{
+ if (timeout < 0) {
+ if (timer != NULL) {
+ kore_timer_remove(timer);
+ timer = NULL;
+ }
+ return (CURLM_OK);
+ }
+
+ if (timer != NULL) {
+ kore_timer_remove(timer);
+ timer = NULL;
+ }
+
+ if (timeout == 0)
+ timeout = 10;
+
+ timer = kore_timer_add(curl_timeout, timeout, mctx, KORE_TIMER_ONESHOT);
+
+ return (CURLM_OK);
+}
+
+static void
+curl_event_handle(void *arg, int eof)
+{
+ int flags;
+ ssize_t bytes;
+ char buf[32];
+ struct fd_cache *fdc = arg;
+
+ flags = 0;
+
+ if (fdc->evt.flags & KORE_EVENT_READ)
+ flags |= CURL_CSELECT_IN;
+
+ if (fdc->evt.flags & KORE_EVENT_WRITE)
+ flags |= CURL_CSELECT_OUT;
+
+ if (eof)
+ flags = CURL_CSELECT_ERR;
+
+ curl_multi_socket_action(multi, fdc->fd, flags, &running);
+
+ /*
+ * XXX - libcurl doesn't work with edge triggered i/o so check
+ * if we need to reprime the event. Not optimal.
+ */
+ if (fdc->evt.flags & KORE_EVENT_READ) {
+ bytes = recv(fdc->fd, buf, sizeof(buf), MSG_PEEK);
+ if (bytes > 0) {
+ if (fdc->scheduled) {
+ kore_platform_disable_read(fdc->fd);
+#if !defined(__linux__)
+ kore_platform_disable_write(fdc->fd);
+#endif
+ }
+
+ fdc->evt.flags = 0;
+ kore_platform_event_all(fdc->fd, fdc);
+ }
+ }
+
+ curl_process();
+}
+
+static struct fd_cache *
+fd_cache_get(int fd)
+{
+ struct fd_cache *fdc;
+ int bucket;
+
+ bucket = fd % FD_CACHE_BUCKETS;
+
+ LIST_FOREACH(fdc, &cache[bucket], list) {
+ if (fdc->fd == fd)
+ return (fdc);
+ }
+
+ fdc = kore_pool_get(&fd_cache_pool);
+
+ fdc->fd = fd;
+ fdc->scheduled = 0;
+
+ fdc->evt.flags = 0;
+ fdc->evt.handle = curl_event_handle;
+ fdc->evt.type = KORE_TYPE_CURL_HANDLE;
+
+ LIST_INSERT_HEAD(&cache[bucket], fdc, list);
+
+ return (fdc);
+}
diff --git a/src/http.c b/src/http.c
@@ -43,6 +43,10 @@
#include "tasks.h"
#endif
+#if defined(KORE_USE_CURL)
+#include "curl.h"
+#endif
+
static struct {
const char *ext;
const char *type;
@@ -131,7 +135,6 @@ static int multipart_parse_headers(struct http_request *,
struct kore_buf *, struct kore_buf *,
const char *, const int);
-static char *http_validate_header(char *);
static struct http_request *http_request_new(struct connection *,
const char *, const char *, char *,
const char *);
@@ -144,10 +147,11 @@ static TAILQ_HEAD(, http_request) http_requests;
static TAILQ_HEAD(, http_request) http_requests_sleeping;
static LIST_HEAD(, http_media_type) http_media_types;
static struct kore_pool http_request_pool;
-static struct kore_pool http_header_pool;
static struct kore_pool http_cookie_pool;
static struct kore_pool http_body_path;
+struct kore_pool http_header_pool;
+
u_int32_t http_request_count = 0;
u_int32_t http_request_ms = HTTP_REQUEST_MS;
u_int16_t http_body_timeout = HTTP_BODY_TIMEOUT;
@@ -394,6 +398,9 @@ http_request_free(struct http_request *req)
#if defined(KORE_USE_PGSQL)
struct kore_pgsql *pgsql;
#endif
+#if defined(KORE_USE_CURL)
+ struct kore_curl *client;
+#endif
struct http_file *f, *fnext;
struct http_arg *q, *qnext;
struct http_header *hdr, *next;
@@ -428,7 +435,12 @@ http_request_free(struct http_request *req)
kore_pgsql_cleanup(pgsql);
}
#endif
-
+#if defined(KORE_USE_CURL)
+ while (!LIST_EMPTY(&req->chandles)) {
+ client = LIST_FIRST(&req->chandles);
+ kore_curl_cleanup(client);
+ }
+#endif
kore_debug("http_request_free: %p->%p", req->owner, req);
kore_free(req->headers);
@@ -2208,7 +2220,7 @@ http_media_type(const char *path)
return (NULL);
}
-static char *
+char *
http_validate_header(char *header)
{
u_int8_t idx;
diff --git a/src/kore.c b/src/kore.c
@@ -33,6 +33,10 @@
#include "http.h"
#endif
+#if defined(KORE_USE_CURL)
+#include "curl.h"
+#endif
+
#if defined(KORE_USE_PYTHON)
#include "python_api.h"
#endif
@@ -105,6 +109,9 @@ version(void)
#if defined(KORE_NO_HTTP)
printf("no-http ");
#endif
+#if defined(KORE_USE_CURL)
+ printf("curl ");
+#endif
#if defined(KORE_USE_PGSQL)
printf("pgsql ");
#endif
@@ -192,6 +199,9 @@ main(int argc, char *argv[])
kore_log_init();
#if !defined(KORE_NO_HTTP)
http_parent_init();
+#if defined(KORE_USE_CURL)
+ kore_curl_sysinit();
+#endif
kore_auth_init();
kore_validator_init();
kore_filemap_init();
diff --git a/src/python.c b/src/python.c
@@ -33,6 +33,10 @@
#include "pgsql.h"
#endif
+#if defined(KORE_USE_CURL)
+#include "curl.h"
+#endif
+
#include "python_api.h"
#include "python_methods.h"
@@ -77,6 +81,12 @@ static PyObject *pykore_pgsql_alloc(struct http_request *,
const char *, const char *);
#endif
+#if defined(KORE_USE_CURL)
+static void python_curl_callback(struct kore_curl *, void *);
+static PyObject *pyhttp_client_request(struct pyhttp_client *, int,
+ PyObject *);
+#endif
+
static void python_append_path(const char *);
static void python_push_integer(PyObject *, const char *, long);
static void python_push_type(const char *, PyObject *, PyTypeObject *);
@@ -963,6 +973,10 @@ python_module_init(void)
python_push_type("pysocket", pykore, &pysocket_type);
python_push_type("pyconnection", pykore, &pyconnection_type);
+#if defined(KORE_USE_CURL)
+ python_push_type("pyhttpclient", pykore, &pyhttp_client_type);
+#endif
+
python_push_type("pyhttp_file", pykore, &pyhttp_file_type);
python_push_type("pyhttp_request", pykore, &pyhttp_request_type);
@@ -2337,6 +2351,7 @@ pysocket_async_recv(struct pysocket_op *op)
return (NULL);
}
+ Py_DECREF(tuple);
PyErr_SetObject(PyExc_StopIteration, result);
Py_DECREF(result);
@@ -3743,3 +3758,323 @@ pyhttp_pgsql(struct pyhttp_request *pyreq, PyObject *args)
return ((PyObject *)obj);
}
#endif
+
+#if defined(KORE_USE_CURL)
+static PyObject *
+python_kore_httpclient(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+ PyObject *obj;
+ struct pyhttp_client *client;
+ const char *url, *v;
+
+ if (!PyArg_ParseTuple(args, "s", &url))
+ return (NULL);
+
+ client = PyObject_New(struct pyhttp_client, &pyhttp_client_type);
+ if (client == NULL)
+ return (NULL);
+
+ client->tlskey = NULL;
+ client->tlscert = NULL;
+ client->url = kore_strdup(url);
+
+ if (kwargs != NULL) {
+ if ((obj = PyDict_GetItemString(kwargs, "tlscert")) != NULL) {
+ if ((v = PyUnicode_AsUTF8(obj)) == NULL) {
+ Py_DECREF((PyObject *)client);
+ return (NULL);
+ }
+
+ client->tlscert = kore_strdup(v);
+ }
+
+ if ((obj = PyDict_GetItemString(kwargs, "tlskey")) != NULL) {
+ if ((v = PyUnicode_AsUTF8(obj)) == NULL) {
+ Py_DECREF((PyObject *)client);
+ return (NULL);
+ }
+
+ client->tlskey = kore_strdup(v);
+ }
+ }
+
+ if ((client->tlscert != NULL && client->tlskey == NULL) ||
+ (client->tlskey != NULL && client->tlscert == NULL)) {
+ Py_DECREF((PyObject *)client);
+ PyErr_SetString(PyExc_RuntimeError,
+ "invalid TLS client configuration");
+ return (NULL);
+ }
+
+ return ((PyObject *)client);
+}
+
+static void
+pyhttp_client_dealloc(struct pyhttp_client *client)
+{
+ kore_free(client->url);
+ kore_free(client->tlskey);
+ kore_free(client->tlscert);
+
+ PyObject_Del((PyObject *)client);
+}
+
+static PyObject *
+pyhttp_client_get(struct pyhttp_client *client, PyObject *args,
+ PyObject *kwargs)
+{
+ return (pyhttp_client_request(client, HTTP_METHOD_GET, kwargs));
+}
+
+static PyObject *
+pyhttp_client_put(struct pyhttp_client *client, PyObject *args,
+ PyObject *kwargs)
+{
+ return (pyhttp_client_request(client, HTTP_METHOD_PUT, kwargs));
+}
+
+static PyObject *
+pyhttp_client_post(struct pyhttp_client *client, PyObject *args,
+ PyObject *kwargs)
+{
+ return (pyhttp_client_request(client, HTTP_METHOD_POST, kwargs));
+}
+
+static PyObject *
+pyhttp_client_head(struct pyhttp_client *client, PyObject *args,
+ PyObject *kwargs)
+{
+ return (pyhttp_client_request(client, HTTP_METHOD_HEAD, kwargs));
+}
+
+static PyObject *
+pyhttp_client_patch(struct pyhttp_client *client, PyObject *args,
+ PyObject *kwargs)
+{
+ return (pyhttp_client_request(client, HTTP_METHOD_PATCH, kwargs));
+}
+
+static PyObject *
+pyhttp_client_delete(struct pyhttp_client *client, PyObject *args,
+ PyObject *kwargs)
+{
+ return (pyhttp_client_request(client, HTTP_METHOD_DELETE, kwargs));
+}
+
+static PyObject *
+pyhttp_client_options(struct pyhttp_client *client, PyObject *args,
+ PyObject *kwargs)
+{
+ return (pyhttp_client_request(client, HTTP_METHOD_OPTIONS, kwargs));
+}
+
+static PyObject *
+pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs)
+{
+ struct pyhttp_client_op *op;
+ char *ptr;
+ const char *k, *v;
+ Py_ssize_t length, idx;
+ PyObject *data, *headers, *key, *item;
+
+ ptr = NULL;
+ length = 0;
+ headers = NULL;
+
+ if (kwargs != NULL &&
+ ((headers = PyDict_GetItemString(kwargs, "headers")) != NULL)) {
+ if (!PyDict_CheckExact(headers)) {
+ PyErr_SetString(PyExc_RuntimeError,
+ "header keyword must be a dict");
+ return (NULL);
+ }
+ }
+
+ switch (m) {
+ case HTTP_METHOD_GET:
+ case HTTP_METHOD_HEAD:
+ case HTTP_METHOD_DELETE:
+ case HTTP_METHOD_OPTIONS:
+ break;
+ case HTTP_METHOD_PUT:
+ case HTTP_METHOD_POST:
+ case HTTP_METHOD_PATCH:
+ length = -1;
+
+ if (kwargs == NULL) {
+ PyErr_Format(PyExc_RuntimeError,
+ "no keyword arguments given, but body expected ",
+ http_method_text(m));
+ return (NULL);
+ }
+
+ if ((data = PyDict_GetItemString(kwargs, "body")) == NULL)
+ return (NULL);
+
+ if (PyBytes_AsStringAndSize(data, &ptr, &length) == -1)
+ return (NULL);
+
+ if (length < 0) {
+ PyErr_SetString(PyExc_TypeError, "invalid length");
+ return (NULL);
+ }
+ break;
+ default:
+ fatal("%s: unknown method %d", __func__, m);
+ }
+
+ op = PyObject_New(struct pyhttp_client_op, &pyhttp_client_op_type);
+ if (op == NULL)
+ return (NULL);
+
+ if (!kore_curl_init(&op->curl, client->url)) {
+ Py_DECREF((PyObject *)op);
+ PyErr_SetString(PyExc_RuntimeError, "failed to setup call");
+ return (NULL);
+ }
+
+ op->headers = 0;
+ op->coro = coro_running;
+ op->state = PYHTTP_CLIENT_OP_RUN;
+
+ Py_INCREF(client);
+
+ kore_curl_http_setup(&op->curl, m, ptr, length);
+ kore_curl_bind_callback(&op->curl, python_curl_callback, op);
+
+ /* Go in with our own bare hands. */
+ if (client->tlskey != NULL && client->tlscert != NULL) {
+ curl_easy_setopt(op->curl.handle, CURLOPT_SSLCERT,
+ client->tlscert);
+ curl_easy_setopt(op->curl.handle, CURLOPT_SSLKEY,
+ client->tlskey);
+ }
+
+ if (headers != NULL) {
+ idx = 0;
+ while (PyDict_Next(headers, &idx, &key, &item)) {
+ if ((k = PyUnicode_AsUTF8(key)) == NULL) {
+ Py_DECREF((PyObject *)op);
+ return (NULL);
+ }
+
+ if ((v = PyUnicode_AsUTF8(item)) == NULL) {
+ Py_DECREF((PyObject *)op);
+ return (NULL);
+ }
+
+ kore_curl_http_set_header(&op->curl, k, v);
+ }
+ }
+
+ if (kwargs != NULL &&
+ ((item = PyDict_GetItemString(kwargs, "return_headers")) != NULL)) {
+ if (item == Py_True) {
+ op->headers = 1;
+ } else if (item == Py_False) {
+ op->headers = 0;
+ } else {
+ Py_DECREF((PyObject *)op);
+ PyErr_SetString(PyExc_RuntimeError,
+ "return_headers not True or False");
+ return (NULL);
+ }
+ }
+
+ return ((PyObject *)op);
+}
+
+static void
+pyhttp_client_op_dealloc(struct pyhttp_client_op *op)
+{
+ kore_curl_cleanup(&op->curl);
+ PyObject_Del((PyObject *)op);
+}
+
+static PyObject *
+pyhttp_client_op_await(PyObject *op)
+{
+ Py_INCREF(op);
+ return (op);
+}
+
+static PyObject *
+pyhttp_client_op_iternext(struct pyhttp_client_op *op)
+{
+ size_t len;
+ struct http_header *hdr;
+ const u_int8_t *response;
+ PyObject *result, *tuple, *dict, *value;
+
+ if (op->state == PYHTTP_CLIENT_OP_RUN) {
+ kore_curl_run(&op->curl);
+ op->state = PYHTTP_CLIENT_OP_RESULT;
+ Py_RETURN_NONE;
+ }
+
+ if (!kore_curl_success(&op->curl)) {
+ PyErr_Format(PyExc_RuntimeError, "request to '%s' failed: %s",
+ op->curl.url, op->curl.errbuf);
+ return (NULL);
+ }
+
+ kore_curl_response_as_bytes(&op->curl, &response, &len);
+
+ if (op->headers) {
+ kore_curl_http_parse_headers(&op->curl);
+
+ if ((dict = PyDict_New()) == NULL)
+ return (NULL);
+
+ TAILQ_FOREACH(hdr, &op->curl.http.resp_hdrs, list) {
+ value = PyUnicode_FromString(hdr->value);
+ if (value == NULL) {
+ Py_DECREF(dict);
+ return (NULL);
+ }
+
+ if (PyDict_SetItemString(dict,
+ hdr->header, value) == -1) {
+ Py_DECREF(dict);
+ Py_DECREF(value);
+ return (NULL);
+ }
+
+ Py_DECREF(value);
+ }
+
+ if ((tuple = Py_BuildValue("(iOy#)", op->curl.http.status,
+ dict, (const char *)response, len)) == NULL)
+ return (NULL);
+
+ Py_DECREF(dict);
+ } else {
+ if ((tuple = Py_BuildValue("(iy#)", op->curl.http.status,
+ (const char *)response, len)) == NULL)
+ return (NULL);
+ }
+
+ result = PyObject_CallFunctionObjArgs(PyExc_StopIteration, tuple, NULL);
+ if (result == NULL) {
+ Py_DECREF(tuple);
+ return (NULL);
+ }
+
+ Py_DECREF(tuple);
+ PyErr_SetObject(PyExc_StopIteration, result);
+ Py_DECREF(result);
+
+ return (NULL);
+}
+
+static void
+python_curl_callback(struct kore_curl *curl, void *arg)
+{
+ struct pyhttp_client_op *op = arg;
+
+ if (op->coro->request != NULL)
+ http_request_wakeup(op->coro->request);
+ else
+ python_coro_wakeup(op->coro);
+}
+#endif
diff --git a/src/utils.c b/src/utils.c
@@ -639,7 +639,7 @@ static void
fatal_log(const char *fmt, va_list args)
{
char buf[2048];
- extern const char *__progname;
+ extern const char *kore_progname;
(void)vsnprintf(buf, sizeof(buf), fmt, args);
@@ -651,5 +651,5 @@ fatal_log(const char *fmt, va_list args)
kore_keymgr_cleanup(1);
#endif
- printf("%s: %s\n", __progname, buf);
+ printf("%s: %s\n", kore_progname, buf);
}
