kore

An easy to use, scalable and secure web application framework for writing web APIs in C.
Commits | Files | Refs | README | LICENSE | git clone https://git.kore.io/kore.git

commit e38c6e5d30ace6ace03dba70ea0428aca5eb8d82
parent 122a86013b363bf94dbdfe7cb4c397519ccc9b10
Author: Joris Vink <joris@coders.se>
Date:   Thu,  2 Jul 2020 08:41:17 +0200

Python: Several fixes for our async curl support.

- Fix the curl-extract-opt.sh generation script to work on newer
  curl releases as the header changed slightly.
- Use the correct handles when calling curl_easy_setopt() inside
  of our setopt functions exported via Python.
- Add a curl.setbody() method, allowing a body to be sent to be set.
  (eg when sending mail via SMTP).
- Regen of our python_curlopt.h from 7.71.1

Diffstat:
README.md | 2+-
include/kore/python_methods.h | 15+++++++++++++--
misc/curl-extract-opt.sh | 17+++++++++--------
misc/curl/python_curlopt.h | 48+++++++++++++++++++++++++++++++++++++-----------
src/python.c | 117+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
5 files changed, 172 insertions(+), 27 deletions(-)

diff --git a/README.md b/README.md @@ -57,7 +57,7 @@ Requirements (note: libressl 3.0.0+ works as a replacement) Requirement for asynchronous curl (optional) -* libcurl +* libcurl (7.64.0 or higher) Requirements for background tasks (optional) * pthreads diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h @@ -795,10 +795,17 @@ static PyTypeObject pyhttp_file_type = { #define CURL_CLIENT_OP_RUN 1 #define CURL_CLIENT_OP_RESULT 2 +struct pycurl_slist { + struct curl_slist *slist; + LIST_ENTRY(pycurl_slist) list; +}; + struct pycurl_handle { PyObject_HEAD - char *url; - struct kore_curl curl; + struct kore_curl curl; + char *url; + struct kore_buf *body; + LIST_HEAD(, pycurl_slist) slists; }; struct pycurl_handle_op { @@ -816,15 +823,19 @@ static void pycurl_handle_op_dealloc(struct pycurl_handle_op *); static PyObject *pycurl_handle_run(struct pycurl_handle *, PyObject *); static PyObject *pycurl_handle_setopt(struct pycurl_handle *, PyObject *); +static PyObject *pycurl_handle_setbody(struct pycurl_handle *, PyObject *); static PyObject *pycurl_handle_setopt_string(struct pycurl_handle *, int, PyObject *); static PyObject *pycurl_handle_setopt_long(struct pycurl_handle *, int, PyObject *); +static PyObject *pycurl_handle_setopt_slist(struct pycurl_handle *, + int, PyObject *); static PyMethodDef pycurl_handle_methods[] = { METHOD("run", pycurl_handle_run, METH_VARARGS), METHOD("setopt", pycurl_handle_setopt, METH_VARARGS), + METHOD("setbody", pycurl_handle_setbody, METH_VARARGS), METHOD(NULL, NULL, -1) }; diff --git a/misc/curl-extract-opt.sh b/misc/curl-extract-opt.sh @@ -32,15 +32,16 @@ struct { } py_curlopt[] = { __EOF -egrep "^.*CINIT\(.*\),$" "$1/curl.h" | \ +egrep "^.*CURLOPT\(.*\),$" "$1/curl.h" | \ cut -d'(' -f 2 | cut -d ')' -f 1 | sed 's/,/ /g' | \ - sed 's/OBJECTPOINT/NULL/g' | \ - sed 's/STRINGPOINT/pycurl_handle_setopt_string/g' | \ - sed 's/LONG/pycurl_handle_setopt_long/g' | \ - sed 's/SLISTPOINT/NULL/g' | \ - sed 's/FUNCTIONPOINT/NULL/g' | \ - sed 's/OFF_T/NULL/g' | \ - awk '{ printf "\t{ \"CURLOPT_%s\", %s, %s },\n", $1, $3, $2 } ' + sed 's/CURLOPTTYPE_OBJECTPOINT/NULL/g' | \ + sed 's/CURLOPTTYPE_STRINGPOINT/pycurl_handle_setopt_string/g' | \ + sed 's/CURLOPTTYPE_LONG/pycurl_handle_setopt_long/g' | \ + sed 's/CURLOPTTYPE_SLISTPOINT/pycurl_handle_setopt_slist/g' | \ + sed 's/CURLOPTTYPE_FUNCTIONPOINT/NULL/g' | \ + sed 's/CURLOPTTYPE_OFF_T/NULL/g' | \ + sed 's/CURLOPTTYPE_BLOB/NULL/g' | \ + awk '{ printf "\t{ \"%s\", %s, %s },\n", $1, $3, $2 } ' echo "\t{ NULL, 0, 0 }" echo "};" diff --git a/misc/curl/python_curlopt.h b/misc/curl/python_curlopt.h @@ -1,4 +1,4 @@ -/* Auto generated on Sat Jan 18 19:42:48 CET 2020 from 7.64.1 */ +/* Auto generated on Wed Jul 1 14:38:36 CEST 2020 from 7.71.1 */ struct { const char *name; @@ -26,12 +26,12 @@ struct { { "CURLOPT_LOW_SPEED_TIME", 20, pycurl_handle_setopt_long }, { "CURLOPT_RESUME_FROM", 21, pycurl_handle_setopt_long }, { "CURLOPT_COOKIE", 22, pycurl_handle_setopt_string }, - { "CURLOPT_HTTPHEADER", 23, NULL }, + { "CURLOPT_HTTPHEADER", 23, pycurl_handle_setopt_slist }, { "CURLOPT_HTTPPOST", 24, NULL }, { "CURLOPT_SSLCERT", 25, pycurl_handle_setopt_string }, { "CURLOPT_KEYPASSWD", 26, pycurl_handle_setopt_string }, { "CURLOPT_CRLF", 27, pycurl_handle_setopt_long }, - { "CURLOPT_QUOTE", 28, NULL }, + { "CURLOPT_QUOTE", 28, pycurl_handle_setopt_slist }, { "CURLOPT_HEADERDATA", 29, NULL }, { "CURLOPT_COOKIEFILE", 31, pycurl_handle_setopt_string }, { "CURLOPT_SSLVERSION", 32, pycurl_handle_setopt_long }, @@ -39,8 +39,21 @@ struct { { "CURLOPT_TIMEVALUE", 34, pycurl_handle_setopt_long }, { "CURLOPT_CUSTOMREQUEST", 36, pycurl_handle_setopt_string }, { "CURLOPT_STDERR", 37, NULL }, - { "CURLOPT_POSTQUOTE", 39, NULL }, + { "CURLOPT_POSTQUOTE", 39, pycurl_handle_setopt_slist }, + { "CURLOPT_OBSOLETE40", 40, NULL }, + { "CURLOPT_VERBOSE", 41, pycurl_handle_setopt_long }, + { "CURLOPT_HEADER", 42, pycurl_handle_setopt_long }, + { "CURLOPT_NOPROGRESS", 43, pycurl_handle_setopt_long }, + { "CURLOPT_NOBODY", 44, pycurl_handle_setopt_long }, + { "CURLOPT_FAILONERROR", 45, pycurl_handle_setopt_long }, + { "CURLOPT_UPLOAD", 46, pycurl_handle_setopt_long }, + { "CURLOPT_POST", 47, pycurl_handle_setopt_long }, + { "CURLOPT_DIRLISTONLY", 48, pycurl_handle_setopt_long }, + { "CURLOPT_APPEND", 50, pycurl_handle_setopt_long }, { "CURLOPT_NETRC", 51, pycurl_handle_setopt_long }, + { "CURLOPT_FOLLOWLOCATION", 52, pycurl_handle_setopt_long }, + { "CURLOPT_TRANSFERTEXT", 53, pycurl_handle_setopt_long }, + { "CURLOPT_PUT", 54, pycurl_handle_setopt_long }, { "CURLOPT_PROGRESSFUNCTION", 56, NULL }, { "CURLOPT_PROGRESSDATA", 57, NULL }, { "CURLOPT_AUTOREFERER", 58, pycurl_handle_setopt_long }, @@ -53,8 +66,9 @@ struct { { "CURLOPT_CAINFO", 65, pycurl_handle_setopt_string }, { "CURLOPT_MAXREDIRS", 68, pycurl_handle_setopt_long }, { "CURLOPT_FILETIME", 69, pycurl_handle_setopt_long }, - { "CURLOPT_TELNETOPTIONS", 70, NULL }, + { "CURLOPT_TELNETOPTIONS", 70, pycurl_handle_setopt_slist }, { "CURLOPT_MAXCONNECTS", 71, pycurl_handle_setopt_long }, + { "CURLOPT_OBSOLETE72", 72, pycurl_handle_setopt_long }, { "CURLOPT_FRESH_CONNECT", 74, pycurl_handle_setopt_long }, { "CURLOPT_FORBID_REUSE", 75, pycurl_handle_setopt_long }, { "CURLOPT_RANDOM_FILE", 76, pycurl_handle_setopt_string }, @@ -72,8 +86,9 @@ struct { { "CURLOPT_SSLKEYTYPE", 88, pycurl_handle_setopt_string }, { "CURLOPT_SSLENGINE", 89, pycurl_handle_setopt_string }, { "CURLOPT_SSLENGINE_DEFAULT", 90, pycurl_handle_setopt_long }, + { "CURLOPT_DNS_USE_GLOBAL_CACHE", 91, pycurl_handle_setopt_long }, { "CURLOPT_DNS_CACHE_TIMEOUT", 92, pycurl_handle_setopt_long }, - { "CURLOPT_PREQUOTE", 93, NULL }, + { "CURLOPT_PREQUOTE", 93, pycurl_handle_setopt_slist }, { "CURLOPT_DEBUGFUNCTION", 94, NULL }, { "CURLOPT_DEBUGDATA", 95, NULL }, { "CURLOPT_COOKIESESSION", 96, pycurl_handle_setopt_long }, @@ -84,7 +99,7 @@ struct { { "CURLOPT_PROXYTYPE", 101, pycurl_handle_setopt_long }, { "CURLOPT_ACCEPT_ENCODING", 102, pycurl_handle_setopt_string }, { "CURLOPT_PRIVATE", 103, NULL }, - { "CURLOPT_HTTP200ALIASES", 104, NULL }, + { "CURLOPT_HTTP200ALIASES", 104, pycurl_handle_setopt_slist }, { "CURLOPT_UNRESTRICTED_AUTH", 105, pycurl_handle_setopt_long }, { "CURLOPT_FTP_USE_EPRT", 106, pycurl_handle_setopt_long }, { "CURLOPT_HTTPAUTH", 107, pycurl_handle_setopt_long }, @@ -150,6 +165,7 @@ struct { { "CURLOPT_PROXYPASSWORD", 176, pycurl_handle_setopt_string }, { "CURLOPT_NOPROXY", 177, pycurl_handle_setopt_string }, { "CURLOPT_TFTP_BLKSIZE", 178, pycurl_handle_setopt_long }, + { "CURLOPT_SOCKS5_GSSAPI_SERVICE", 179, pycurl_handle_setopt_string }, { "CURLOPT_SOCKS5_GSSAPI_NEC", 180, pycurl_handle_setopt_long }, { "CURLOPT_PROTOCOLS", 181, pycurl_handle_setopt_long }, { "CURLOPT_REDIR_PROTOCOLS", 182, pycurl_handle_setopt_long }, @@ -157,7 +173,7 @@ struct { { "CURLOPT_SSH_KEYFUNCTION", 184, NULL }, { "CURLOPT_SSH_KEYDATA", 185, NULL }, { "CURLOPT_MAIL_FROM", 186, pycurl_handle_setopt_string }, - { "CURLOPT_MAIL_RCPT", 187, NULL }, + { "CURLOPT_MAIL_RCPT", 187, pycurl_handle_setopt_slist }, { "CURLOPT_FTP_USE_PRET", 188, pycurl_handle_setopt_long }, { "CURLOPT_RTSP_REQUEST", 189, pycurl_handle_setopt_long }, { "CURLOPT_RTSP_SESSION_ID", 190, pycurl_handle_setopt_string }, @@ -173,7 +189,7 @@ struct { { "CURLOPT_FNMATCH_FUNCTION", 200, NULL }, { "CURLOPT_CHUNK_DATA", 201, NULL }, { "CURLOPT_FNMATCH_DATA", 202, NULL }, - { "CURLOPT_RESOLVE", 203, NULL }, + { "CURLOPT_RESOLVE", 203, pycurl_handle_setopt_slist }, { "CURLOPT_TLSAUTH_USERNAME", 204, pycurl_handle_setopt_string }, { "CURLOPT_TLSAUTH_PASSWORD", 205, pycurl_handle_setopt_string }, { "CURLOPT_TLSAUTH_TYPE", 206, pycurl_handle_setopt_string }, @@ -198,7 +214,7 @@ struct { { "CURLOPT_SSL_ENABLE_NPN", 225, pycurl_handle_setopt_long }, { "CURLOPT_SSL_ENABLE_ALPN", 226, pycurl_handle_setopt_long }, { "CURLOPT_EXPECT_100_TIMEOUT_MS", 227, pycurl_handle_setopt_long }, - { "CURLOPT_PROXYHEADER", 228, NULL }, + { "CURLOPT_PROXYHEADER", 228, pycurl_handle_setopt_slist }, { "CURLOPT_HEADEROPT", 229, pycurl_handle_setopt_long }, { "CURLOPT_PINNEDPUBLICKEY", 230, pycurl_handle_setopt_string }, { "CURLOPT_UNIX_SOCKET_PATH", 231, pycurl_handle_setopt_string }, @@ -213,7 +229,7 @@ struct { { "CURLOPT_STREAM_DEPENDS", 240, NULL }, { "CURLOPT_STREAM_DEPENDS_E", 241, NULL }, { "CURLOPT_TFTP_NO_OPTIONS", 242, pycurl_handle_setopt_long }, - { "CURLOPT_CONNECT_TO", 243, NULL }, + { "CURLOPT_CONNECT_TO", 243, pycurl_handle_setopt_slist }, { "CURLOPT_TCP_FASTOPEN", 244, pycurl_handle_setopt_long }, { "CURLOPT_KEEP_SENDING_ON_ERROR", 245, pycurl_handle_setopt_long }, { "CURLOPT_PROXY_CAINFO", 246, pycurl_handle_setopt_string }, @@ -258,5 +274,15 @@ struct { { "CURLOPT_HTTP09_ALLOWED", 285, pycurl_handle_setopt_long }, { "CURLOPT_ALTSVC_CTRL", 286, pycurl_handle_setopt_long }, { "CURLOPT_ALTSVC", 287, pycurl_handle_setopt_string }, + { "CURLOPT_MAXAGE_CONN", 288, pycurl_handle_setopt_long }, + { "CURLOPT_SASL_AUTHZID", 289, pycurl_handle_setopt_string }, + { "CURLOPT_MAIL_RCPT_ALLLOWFAILS", 290, pycurl_handle_setopt_long }, + { "CURLOPT_SSLCERT_BLOB", 291, NULL }, + { "CURLOPT_SSLKEY_BLOB", 292, NULL }, + { "CURLOPT_PROXY_SSLCERT_BLOB", 293, NULL }, + { "CURLOPT_PROXY_SSLKEY_BLOB", 294, NULL }, + { "CURLOPT_ISSUERCERT_BLOB", 295, NULL }, + { "CURLOPT_PROXY_ISSUERCERT", 296, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_ISSUERCERT_BLOB", 297, NULL }, { NULL, 0, 0 } }; diff --git a/src/python.c b/src/python.c @@ -5404,6 +5404,9 @@ python_kore_curl_handle(PyObject *self, PyObject *args) handle->url = kore_strdup(url); memset(&handle->curl, 0, sizeof(handle->curl)); + handle->body = NULL; + LIST_INIT(&handle->slists); + if (!kore_curl_init(&handle->curl, handle->url, KORE_CURL_ASYNC)) { Py_DECREF((PyObject *)handle); PyErr_SetString(PyExc_RuntimeError, "failed to setup call"); @@ -5416,6 +5419,17 @@ python_kore_curl_handle(PyObject *self, PyObject *args) static void pycurl_handle_dealloc(struct pycurl_handle *handle) { + struct pycurl_slist *psl; + + while ((psl = LIST_FIRST(&handle->slists))) { + LIST_REMOVE(psl, list); + curl_slist_free_all(psl->slist); + kore_free(psl); + } + + if (handle->body != NULL) + kore_buf_free(handle->body); + kore_free(handle->url); kore_curl_cleanup(&handle->curl); @@ -5423,6 +5437,48 @@ pycurl_handle_dealloc(struct pycurl_handle *handle) } static PyObject * +pycurl_handle_setbody(struct pycurl_handle *handle, PyObject *args) +{ + PyObject *obj; + char *ptr; + Py_ssize_t length; + + if (!PyArg_ParseTuple(args, "O", &obj)) + return (NULL); + + if (handle->body != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "curl handle already has body attached"); + return (NULL); + } + + if (!PyBytes_CheckExact(obj)) { + PyErr_SetString(PyExc_RuntimeError, + "curl.setbody expects bytes"); + return (NULL); + } + + if (PyBytes_AsStringAndSize(obj, &ptr, &length) == -1) + return (NULL); + + if (length < 0) { + PyErr_SetString(PyExc_TypeError, "invalid length"); + return (NULL); + } + + handle->body = kore_buf_alloc(length); + kore_buf_append(handle->body, ptr, length); + kore_buf_reset(handle->body); + + curl_easy_setopt(handle->curl.handle, + CURLOPT_READFUNCTION, kore_curl_frombuf); + curl_easy_setopt(handle->curl.handle, CURLOPT_READDATA, handle->body); + curl_easy_setopt(handle->curl.handle, CURLOPT_UPLOAD, 1); + + Py_RETURN_TRUE; +} + +static PyObject * pycurl_handle_setopt(struct pycurl_handle *handle, PyObject *args) { int i, opt; @@ -5459,14 +5515,15 @@ pycurl_handle_setopt_string(struct pycurl_handle *handle, int idx, if (!PyUnicode_Check(obj)) { PyErr_Format(PyExc_RuntimeError, "option '%s' requires a string as argument", - py_curlopt[idx]); + py_curlopt[idx].name); return (NULL); } if ((str = PyUnicode_AsUTF8(obj)) == NULL) return (NULL); - curl_easy_setopt(&handle->curl, py_curlopt[idx].value, str); + curl_easy_setopt(handle->curl.handle, + CURLOPTTYPE_OBJECTPOINT + py_curlopt[idx].value, str); Py_RETURN_TRUE; } @@ -5478,7 +5535,8 @@ pycurl_handle_setopt_long(struct pycurl_handle *handle, int idx, PyObject *obj) if (!PyLong_CheckExact(obj)) { PyErr_Format(PyExc_RuntimeError, - "option '%s' requires a long as argument", py_curlopt[idx]); + "option '%s' requires a long as argument", + py_curlopt[idx].name); return (NULL); } @@ -5487,7 +5545,51 @@ pycurl_handle_setopt_long(struct pycurl_handle *handle, int idx, PyObject *obj) if (val == -1 && PyErr_Occurred()) return (NULL); - curl_easy_setopt(&handle->curl, py_curlopt[idx].value, val); + curl_easy_setopt(handle->curl.handle, + CURLOPTTYPE_LONG + py_curlopt[idx].value, val); + + Py_RETURN_TRUE; +} + +static PyObject * +pycurl_handle_setopt_slist(struct pycurl_handle *handle, int idx, PyObject *obj) +{ + struct pycurl_slist *psl; + PyObject *item; + const char *sval; + struct curl_slist *slist; + Py_ssize_t list_len, i; + + if (!PyList_CheckExact(obj)) { + PyErr_Format(PyExc_RuntimeError, + "option '%s' requires a list as argument", + py_curlopt[idx].name); + return (NULL); + } + + slist = NULL; + list_len = PyList_Size(obj); + + for (i = 0; i < list_len; i++) { + if ((item = PyList_GetItem(obj, i)) == NULL) + return (NULL); + + if (!PyUnicode_Check(item)) + return (NULL); + + if ((sval = PyUnicode_AsUTF8AndSize(item, NULL)) == NULL) + return (NULL); + + if ((slist = curl_slist_append(slist, sval)) == NULL) + fatal("%s: curl_slist_append failed", __func__); + } + + psl = kore_calloc(1, sizeof(*psl)); + psl->slist = slist; + LIST_INSERT_HEAD(&handle->slists, psl, list); + + curl_easy_setopt(handle->curl.handle, + CURLOPTTYPE_OBJECTPOINT + py_curlopt[idx].value, slist); Py_RETURN_TRUE; } @@ -5539,6 +5641,11 @@ pycurl_handle_op_iternext(struct pycurl_handle_op *op) Py_RETURN_NONE; } + if (op->handle->body != NULL) { + kore_buf_free(op->handle->body); + op->handle->body = NULL; + } + if (!kore_curl_success(&op->handle->curl)) { /* Do not log the url here, may contain some sensitive data. */ PyErr_Format(PyExc_RuntimeError, "request failed: %s", @@ -5684,7 +5791,7 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) ((headers = PyDict_GetItemString(kwargs, "headers")) != NULL)) { if (!PyDict_CheckExact(headers)) { PyErr_SetString(PyExc_RuntimeError, - "header keyword must be a dict"); + "headers keyword must be a dict"); return (NULL); } }