commit 55aaef875d489e1617c8d27433c6d4bc6197bcf9
parent 89085246e532c95fd59ac886218a5ec2d57b2071
Author: Joris Vink <joris@coders.se>
Date: Fri, 27 Aug 2021 10:00:50 +0200
Add support for setting curlopts in kore.httpclient.
Much of the work done by Matthew Norström with minor cleanup by me.
Diffstat:
3 files changed, 140 insertions(+), 71 deletions(-)
diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h
@@ -838,14 +838,18 @@ struct pycurl_slist {
LIST_ENTRY(pycurl_slist) list;
};
-struct pycurl_handle {
- PyObject_HEAD
+struct pycurl_data {
struct kore_curl curl;
- char *url;
- struct kore_buf *body;
LIST_HEAD(, pycurl_slist) slists;
};
+struct pycurl_handle {
+ PyObject_HEAD
+ char *url;
+ struct kore_buf *body;
+ struct pycurl_data data;
+};
+
struct pycurl_handle_op {
PyObject_HEAD
int state;
@@ -863,11 +867,11 @@ static PyObject *pycurl_handle_run(struct pycurl_handle *, PyObject *);
static PyObject *pycurl_handle_setopt(struct pycurl_handle *, PyObject *);
static PyObject *pycurl_handle_setbody(struct pycurl_handle *, PyObject *);
-static PyObject *pycurl_handle_setopt_string(struct pycurl_handle *,
+static PyObject *pycurl_handle_setopt_string(struct pycurl_data *,
int, PyObject *);
-static PyObject *pycurl_handle_setopt_long(struct pycurl_handle *,
+static PyObject *pycurl_handle_setopt_long(struct pycurl_data *,
int, PyObject *);
-static PyObject *pycurl_handle_setopt_slist(struct pycurl_handle *,
+static PyObject *pycurl_handle_setopt_slist(struct pycurl_data *,
int, PyObject *);
static PyMethodDef pycurl_handle_methods[] = {
@@ -911,6 +915,7 @@ struct pyhttp_client {
char *tlskey;
char *tlscert;
char *cabundle;
+ PyObject *curlopt;
int tlsverify;
};
@@ -918,11 +923,12 @@ struct pyhttp_client_op {
PyObject_HEAD
int state;
int headers;
- struct kore_curl curl;
struct python_coro *coro;
struct pyhttp_client *client;
+ struct pycurl_data data;
};
+
static PyObject *pyhttp_client_op_await(PyObject *);
static PyObject *pyhttp_client_op_iternext(struct pyhttp_client_op *);
diff --git a/misc/curl/python_curlopt.h b/misc/curl/python_curlopt.h
@@ -3,7 +3,7 @@
struct {
const char *name;
int value;
- PyObject *(*cb)(struct pycurl_handle *, int, PyObject *);
+ PyObject *(*cb)(struct pycurl_data *, int, PyObject *);
} py_curlopt[] = {
{ "CURLOPT_WRITEDATA", 1, NULL },
{ "CURLOPT_URL", 2, pycurl_handle_setopt_string },
diff --git a/src/python.c b/src/python.c
@@ -27,6 +27,7 @@
#include <signal.h>
#include <fcntl.h>
#include <unistd.h>
+#include <stdarg.h>
#include "kore.h"
#include "http.h"
@@ -131,6 +132,8 @@ static void python_curl_http_callback(struct kore_curl *, void *);
static void python_curl_handle_callback(struct kore_curl *, void *);
static PyObject *pyhttp_client_request(struct pyhttp_client *, int,
PyObject *);
+static PyObject *python_kore_curl_setopt(struct pycurl_data *,
+ long, PyObject *);
#endif
static void python_append_path(const char *);
@@ -5731,6 +5734,30 @@ pykore_pgsql_result(struct pykore_pgsql *pysql)
#if defined(KORE_USE_CURL)
static PyObject *
+python_kore_curl_setopt(struct pycurl_data *data, long opt, PyObject *value)
+{
+ int i;
+
+ for (i = 0; py_curlopt[i].name != NULL; i++) {
+ if (py_curlopt[i].value == opt)
+ break;
+ }
+
+ if (py_curlopt[i].name == NULL) {
+ PyErr_Format(PyExc_RuntimeError, "invalid option '%d'", opt);
+ return (NULL);
+ }
+
+ if (py_curlopt[i].cb == NULL) {
+ PyErr_Format(PyExc_RuntimeError, "option '%s' not implemented",
+ py_curlopt[i].name);
+ return (NULL);
+ }
+
+ return (py_curlopt[i].cb(data, i, value));
+}
+
+static PyObject *
python_kore_curl_handle(PyObject *self, PyObject *args)
{
const char *url;
@@ -5744,12 +5771,12 @@ python_kore_curl_handle(PyObject *self, PyObject *args)
return (NULL);
handle->url = kore_strdup(url);
- memset(&handle->curl, 0, sizeof(handle->curl));
+ memset(&handle->data.curl, 0, sizeof(handle->data.curl));
handle->body = NULL;
- LIST_INIT(&handle->slists);
+ LIST_INIT(&handle->data.slists);
- if (!kore_curl_init(&handle->curl, handle->url, KORE_CURL_ASYNC)) {
+ if (!kore_curl_init(&handle->data.curl, handle->url, KORE_CURL_ASYNC)) {
Py_DECREF((PyObject *)handle);
PyErr_SetString(PyExc_RuntimeError, "failed to setup call");
return (NULL);
@@ -5763,7 +5790,7 @@ pycurl_handle_dealloc(struct pycurl_handle *handle)
{
struct pycurl_slist *psl;
- while ((psl = LIST_FIRST(&handle->slists))) {
+ while ((psl = LIST_FIRST(&handle->data.slists))) {
LIST_REMOVE(psl, list);
curl_slist_free_all(psl->slist);
kore_free(psl);
@@ -5773,7 +5800,7 @@ pycurl_handle_dealloc(struct pycurl_handle *handle)
kore_buf_free(handle->body);
kore_free(handle->url);
- kore_curl_cleanup(&handle->curl);
+ kore_curl_cleanup(&handle->data.curl);
PyObject_Del((PyObject *)handle);
}
@@ -5812,10 +5839,12 @@ pycurl_handle_setbody(struct pycurl_handle *handle, PyObject *args)
kore_buf_append(handle->body, ptr, length);
kore_buf_reset(handle->body);
- curl_easy_setopt(handle->curl.handle,
+ curl_easy_setopt(handle->data.curl.handle,
CURLOPT_READFUNCTION, kore_curl_frombuf);
- curl_easy_setopt(handle->curl.handle, CURLOPT_READDATA, handle->body);
- curl_easy_setopt(handle->curl.handle, CURLOPT_UPLOAD, 1);
+ curl_easy_setopt(handle->data.curl.handle,
+ CURLOPT_READDATA, handle->body);
+
+ curl_easy_setopt(handle->data.curl.handle, CURLOPT_UPLOAD, 1);
Py_RETURN_TRUE;
}
@@ -5823,34 +5852,17 @@ pycurl_handle_setbody(struct pycurl_handle *handle, PyObject *args)
static PyObject *
pycurl_handle_setopt(struct pycurl_handle *handle, PyObject *args)
{
- int i, opt;
+ int opt;
PyObject *value;
if (!PyArg_ParseTuple(args, "iO", &opt, &value))
return (NULL);
- for (i = 0; py_curlopt[i].name != NULL; i++) {
- if (py_curlopt[i].value == opt)
- break;
- }
-
- if (py_curlopt[i].name == NULL) {
- PyErr_Format(PyExc_RuntimeError, "invalid option '%d'", opt);
- return (NULL);
- }
-
- if (py_curlopt[i].cb == NULL) {
- PyErr_Format(PyExc_RuntimeError, "option '%s' not implemented",
- py_curlopt[i].name);
- return (NULL);
- }
-
- return (py_curlopt[i].cb(handle, i, value));
+ return (python_kore_curl_setopt(&handle->data, opt, value));
}
static PyObject *
-pycurl_handle_setopt_string(struct pycurl_handle *handle, int idx,
- PyObject *obj)
+pycurl_handle_setopt_string(struct pycurl_data *data, int idx, PyObject *obj)
{
const char *str;
@@ -5864,14 +5876,14 @@ pycurl_handle_setopt_string(struct pycurl_handle *handle, int idx,
if ((str = PyUnicode_AsUTF8(obj)) == NULL)
return (NULL);
- curl_easy_setopt(handle->curl.handle,
+ curl_easy_setopt(data->curl.handle,
CURLOPTTYPE_OBJECTPOINT + py_curlopt[idx].value, str);
Py_RETURN_TRUE;
}
static PyObject *
-pycurl_handle_setopt_long(struct pycurl_handle *handle, int idx, PyObject *obj)
+pycurl_handle_setopt_long(struct pycurl_data *data, int idx, PyObject *obj)
{
long val;
@@ -5887,14 +5899,14 @@ pycurl_handle_setopt_long(struct pycurl_handle *handle, int idx, PyObject *obj)
if (val == -1 && PyErr_Occurred())
return (NULL);
- curl_easy_setopt(handle->curl.handle,
+ curl_easy_setopt(data->curl.handle,
CURLOPTTYPE_LONG + py_curlopt[idx].value, val);
Py_RETURN_TRUE;
}
static PyObject *
-pycurl_handle_setopt_slist(struct pycurl_handle *handle, int idx, PyObject *obj)
+pycurl_handle_setopt_slist(struct pycurl_data *data, int idx, PyObject *obj)
{
struct pycurl_slist *psl;
PyObject *item;
@@ -5928,9 +5940,9 @@ pycurl_handle_setopt_slist(struct pycurl_handle *handle, int idx, PyObject *obj)
psl = kore_calloc(1, sizeof(*psl));
psl->slist = slist;
- LIST_INSERT_HEAD(&handle->slists, psl, list);
+ LIST_INSERT_HEAD(&data->slists, psl, list);
- curl_easy_setopt(handle->curl.handle,
+ curl_easy_setopt(data->curl.handle,
CURLOPTTYPE_OBJECTPOINT + py_curlopt[idx].value, slist);
Py_RETURN_TRUE;
@@ -5951,7 +5963,8 @@ pycurl_handle_run(struct pycurl_handle *handle, PyObject *args)
op->coro = coro_running;
op->state = CURL_CLIENT_OP_RUN;
- kore_curl_bind_callback(&handle->curl, python_curl_handle_callback, op);
+ kore_curl_bind_callback(&handle->data.curl,
+ python_curl_handle_callback, op);
return ((PyObject *)op);
}
@@ -5978,7 +5991,7 @@ pycurl_handle_op_iternext(struct pycurl_handle_op *op)
const u_int8_t *response;
if (op->state == CURL_CLIENT_OP_RUN) {
- kore_curl_run(&op->handle->curl);
+ kore_curl_run(&op->handle->data.curl);
op->state = CURL_CLIENT_OP_RESULT;
Py_RETURN_NONE;
}
@@ -5988,14 +6001,14 @@ pycurl_handle_op_iternext(struct pycurl_handle_op *op)
op->handle->body = NULL;
}
- if (!kore_curl_success(&op->handle->curl)) {
+ if (!kore_curl_success(&op->handle->data.curl)) {
/* Do not log the url here, may contain some sensitive data. */
PyErr_Format(PyExc_RuntimeError, "request failed: %s",
- kore_curl_strerror(&op->handle->curl));
+ kore_curl_strerror(&op->handle->data.curl));
return (NULL);
}
- kore_curl_response_as_bytes(&op->handle->curl, &response, &len);
+ kore_curl_response_as_bytes(&op->handle->data.curl, &response, &len);
if ((result = PyBytes_FromStringAndSize((const char *)response,
len)) == NULL)
@@ -6010,6 +6023,7 @@ pycurl_handle_op_iternext(struct pycurl_handle_op *op)
static PyObject *
python_kore_httpclient(PyObject *self, PyObject *args, PyObject *kwargs)
{
+ PyObject *vdict;
struct pyhttp_client *client;
const char *url, *v;
@@ -6022,6 +6036,7 @@ python_kore_httpclient(PyObject *self, PyObject *args, PyObject *kwargs)
client->unix = NULL;
client->tlskey = NULL;
+ client->curlopt = NULL;
client->tlscert = NULL;
client->cabundle = NULL;
@@ -6041,6 +6056,18 @@ python_kore_httpclient(PyObject *self, PyObject *args, PyObject *kwargs)
if ((v = python_string_from_dict(kwargs, "unix")) != NULL)
client->unix = kore_strdup(v);
+ if ((vdict = PyDict_GetItemString(kwargs, "curlopt")) != NULL) {
+ if (!PyDict_CheckExact(vdict)) {
+ Py_DECREF((PyObject *)client);
+ PyErr_SetString(PyExc_RuntimeError,
+ "curlopt must be a dict");
+ return (NULL);
+ }
+
+ client->curlopt = vdict;
+ Py_INCREF(client->curlopt);
+ }
+
python_bool_from_dict(kwargs, "tlsverify", &client->tlsverify);
}
@@ -6064,6 +6091,8 @@ pyhttp_client_dealloc(struct pyhttp_client *client)
kore_free(client->tlscert);
kore_free(client->cabundle);
+ Py_XDECREF(client->curlopt);
+
PyObject_Del((PyObject *)client);
}
@@ -6119,11 +6148,12 @@ pyhttp_client_options(struct pyhttp_client *client, PyObject *args,
static PyObject *
pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs)
{
+ long opt;
struct pyhttp_client_op *op;
char *ptr;
const char *k, *v;
Py_ssize_t length, idx;
- PyObject *data, *headers, *key, *item;
+ PyObject *data, *headers, *key, *item, *value;
ptr = NULL;
length = 0;
@@ -6170,12 +6200,12 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs)
default:
fatal("%s: unknown method %d", __func__, m);
}
-
+
op = PyObject_New(struct pyhttp_client_op, &pyhttp_client_op_type);
if (op == NULL)
return (NULL);
- if (!kore_curl_init(&op->curl, client->url, KORE_CURL_ASYNC)) {
+ if (!kore_curl_init(&op->data.curl, client->url, KORE_CURL_ASYNC)) {
Py_DECREF((PyObject *)op);
PyErr_SetString(PyExc_RuntimeError, "failed to setup call");
return (NULL);
@@ -6184,43 +6214,68 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs)
op->headers = 0;
op->coro = coro_running;
op->state = CURL_CLIENT_OP_RUN;
+ LIST_INIT(&op->data.slists);
Py_INCREF(client);
op->client = client;
- kore_curl_http_setup(&op->curl, m, ptr, length);
- kore_curl_bind_callback(&op->curl, python_curl_http_callback, op);
+ kore_curl_http_setup(&op->data.curl, m, ptr, length);
+ kore_curl_bind_callback(&op->data.curl, python_curl_http_callback, op);
/* Go in with our own bare hands. */
if (client->unix != NULL) {
#if defined(__linux__)
if (client->unix[0] == '@') {
- curl_easy_setopt(op->curl.handle,
+ curl_easy_setopt(op->data.curl.handle,
CURLOPT_ABSTRACT_UNIX_SOCKET, client->unix + 1);
} else {
- curl_easy_setopt(op->curl.handle,
+ curl_easy_setopt(op->data.curl.handle,
CURLOPT_UNIX_SOCKET_PATH, client->unix);
}
#else
- curl_easy_setopt(op->curl.handle, CURLOPT_UNIX_SOCKET_PATH,
+ curl_easy_setopt(op->data.curl.handle, CURLOPT_UNIX_SOCKET_PATH,
client->unix);
#endif
}
if (client->tlskey != NULL && client->tlscert != NULL) {
- curl_easy_setopt(op->curl.handle, CURLOPT_SSLCERT,
+ curl_easy_setopt(op->data.curl.handle, CURLOPT_SSLCERT,
client->tlscert);
- curl_easy_setopt(op->curl.handle, CURLOPT_SSLKEY,
+ curl_easy_setopt(op->data.curl.handle, CURLOPT_SSLKEY,
client->tlskey);
}
if (client->tlsverify == 0) {
- curl_easy_setopt(op->curl.handle, CURLOPT_SSL_VERIFYHOST, 0);
- curl_easy_setopt(op->curl.handle, CURLOPT_SSL_VERIFYPEER, 0);
+ curl_easy_setopt(op->data.curl.handle,
+ CURLOPT_SSL_VERIFYHOST, 0);
+ curl_easy_setopt(op->data.curl.handle,
+ CURLOPT_SSL_VERIFYPEER, 0);
+ }
+
+ if (client->curlopt != NULL) {
+ idx = 0;
+ while (PyDict_Next(client->curlopt, &idx, &key, &value)) {
+ if (!PyLong_CheckExact(key)) {
+ Py_DECREF((PyObject *)op);
+ PyErr_Format(PyExc_RuntimeError,
+ "invalid key in curlopt keyword");
+ return (NULL);
+ }
+
+ opt = PyLong_AsLong(key);
+
+ item = python_kore_curl_setopt(&op->data, opt, value);
+ if (item == NULL) {
+ Py_DECREF((PyObject *)op);
+ return (NULL);
+ }
+
+ Py_DECREF(item);
+ }
}
if (client->cabundle != NULL) {
- curl_easy_setopt(op->curl.handle, CURLOPT_CAINFO,
+ curl_easy_setopt(op->data.curl.handle, CURLOPT_CAINFO,
client->cabundle);
}
@@ -6237,7 +6292,7 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs)
return (NULL);
}
- kore_curl_http_set_header(&op->curl, k, v);
+ kore_curl_http_set_header(&op->data.curl, k, v);
}
}
@@ -6250,8 +6305,16 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs)
static void
pyhttp_client_op_dealloc(struct pyhttp_client_op *op)
{
+ struct pycurl_slist *psl;
+
+ while ((psl = LIST_FIRST(&op->data.slists))) {
+ LIST_REMOVE(psl, list);
+ curl_slist_free_all(psl->slist);
+ kore_free(psl);
+ }
+
Py_DECREF(op->client);
- kore_curl_cleanup(&op->curl);
+ kore_curl_cleanup(&op->data.curl);
PyObject_Del((PyObject *)op);
}
@@ -6271,26 +6334,26 @@ pyhttp_client_op_iternext(struct pyhttp_client_op *op)
PyObject *result, *tuple, *dict, *value;
if (op->state == CURL_CLIENT_OP_RUN) {
- kore_curl_run(&op->curl);
+ kore_curl_run(&op->data.curl);
op->state = CURL_CLIENT_OP_RESULT;
Py_RETURN_NONE;
}
- if (!kore_curl_success(&op->curl)) {
+ if (!kore_curl_success(&op->data.curl)) {
PyErr_Format(PyExc_RuntimeError, "request to '%s' failed: %s",
- op->curl.url, kore_curl_strerror(&op->curl));
+ op->data.curl.url, kore_curl_strerror(&op->data.curl));
return (NULL);
}
- kore_curl_response_as_bytes(&op->curl, &response, &len);
+ kore_curl_response_as_bytes(&op->data.curl, &response, &len);
if (op->headers) {
- kore_curl_http_parse_headers(&op->curl);
+ kore_curl_http_parse_headers(&op->data.curl);
if ((dict = PyDict_New()) == NULL)
return (NULL);
- TAILQ_FOREACH(hdr, &op->curl.http.resp_hdrs, list) {
+ TAILQ_FOREACH(hdr, &op->data.curl.http.resp_hdrs, list) {
value = PyUnicode_FromString(hdr->value);
if (value == NULL) {
Py_DECREF(dict);
@@ -6307,13 +6370,13 @@ pyhttp_client_op_iternext(struct pyhttp_client_op *op)
Py_DECREF(value);
}
- if ((tuple = Py_BuildValue("(iOy#)", op->curl.http.status,
+ if ((tuple = Py_BuildValue("(iOy#)", op->data.curl.http.status,
dict, (const char *)response, len)) == NULL)
return (NULL);
Py_DECREF(dict);
} else {
- if ((tuple = Py_BuildValue("(iy#)", op->curl.http.status,
+ if ((tuple = Py_BuildValue("(iy#)", op->data.curl.http.status,
(const char *)response, len)) == NULL)
return (NULL);
}
