commit e38c6e5d30ace6ace03dba70ea0428aca5eb8d82
parent 122a86013b363bf94dbdfe7cb4c397519ccc9b10
Author: Joris Vink <joris@coders.se>
Date: Thu, 2 Jul 2020 08:41:17 +0200
Python: Several fixes for our async curl support.
- Fix the curl-extract-opt.sh generation script to work on newer
curl releases as the header changed slightly.
- Use the correct handles when calling curl_easy_setopt() inside
of our setopt functions exported via Python.
- Add a curl.setbody() method, allowing a body to be sent to be set.
(eg when sending mail via SMTP).
- Regen of our python_curlopt.h from 7.71.1
Diffstat:
5 files changed, 172 insertions(+), 27 deletions(-)
diff --git a/README.md b/README.md
@@ -57,7 +57,7 @@ Requirements
(note: libressl 3.0.0+ works as a replacement)
Requirement for asynchronous curl (optional)
-* libcurl
+* libcurl (7.64.0 or higher)
Requirements for background tasks (optional)
* pthreads
diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h
@@ -795,10 +795,17 @@ static PyTypeObject pyhttp_file_type = {
#define CURL_CLIENT_OP_RUN 1
#define CURL_CLIENT_OP_RESULT 2
+struct pycurl_slist {
+ struct curl_slist *slist;
+ LIST_ENTRY(pycurl_slist) list;
+};
+
struct pycurl_handle {
PyObject_HEAD
- char *url;
- struct kore_curl curl;
+ struct kore_curl curl;
+ char *url;
+ struct kore_buf *body;
+ LIST_HEAD(, pycurl_slist) slists;
};
struct pycurl_handle_op {
@@ -816,15 +823,19 @@ static void pycurl_handle_op_dealloc(struct pycurl_handle_op *);
static PyObject *pycurl_handle_run(struct pycurl_handle *, PyObject *);
static PyObject *pycurl_handle_setopt(struct pycurl_handle *, PyObject *);
+static PyObject *pycurl_handle_setbody(struct pycurl_handle *, PyObject *);
static PyObject *pycurl_handle_setopt_string(struct pycurl_handle *,
int, PyObject *);
static PyObject *pycurl_handle_setopt_long(struct pycurl_handle *,
int, PyObject *);
+static PyObject *pycurl_handle_setopt_slist(struct pycurl_handle *,
+ int, PyObject *);
static PyMethodDef pycurl_handle_methods[] = {
METHOD("run", pycurl_handle_run, METH_VARARGS),
METHOD("setopt", pycurl_handle_setopt, METH_VARARGS),
+ METHOD("setbody", pycurl_handle_setbody, METH_VARARGS),
METHOD(NULL, NULL, -1)
};
diff --git a/misc/curl-extract-opt.sh b/misc/curl-extract-opt.sh
@@ -32,15 +32,16 @@ struct {
} py_curlopt[] = {
__EOF
-egrep "^.*CINIT\(.*\),$" "$1/curl.h" | \
+egrep "^.*CURLOPT\(.*\),$" "$1/curl.h" | \
cut -d'(' -f 2 | cut -d ')' -f 1 | sed 's/,/ /g' | \
- sed 's/OBJECTPOINT/NULL/g' | \
- sed 's/STRINGPOINT/pycurl_handle_setopt_string/g' | \
- sed 's/LONG/pycurl_handle_setopt_long/g' | \
- sed 's/SLISTPOINT/NULL/g' | \
- sed 's/FUNCTIONPOINT/NULL/g' | \
- sed 's/OFF_T/NULL/g' | \
- awk '{ printf "\t{ \"CURLOPT_%s\", %s, %s },\n", $1, $3, $2 } '
+ sed 's/CURLOPTTYPE_OBJECTPOINT/NULL/g' | \
+ sed 's/CURLOPTTYPE_STRINGPOINT/pycurl_handle_setopt_string/g' | \
+ sed 's/CURLOPTTYPE_LONG/pycurl_handle_setopt_long/g' | \
+ sed 's/CURLOPTTYPE_SLISTPOINT/pycurl_handle_setopt_slist/g' | \
+ sed 's/CURLOPTTYPE_FUNCTIONPOINT/NULL/g' | \
+ sed 's/CURLOPTTYPE_OFF_T/NULL/g' | \
+ sed 's/CURLOPTTYPE_BLOB/NULL/g' | \
+ awk '{ printf "\t{ \"%s\", %s, %s },\n", $1, $3, $2 } '
echo "\t{ NULL, 0, 0 }"
echo "};"
diff --git a/misc/curl/python_curlopt.h b/misc/curl/python_curlopt.h
@@ -1,4 +1,4 @@
-/* Auto generated on Sat Jan 18 19:42:48 CET 2020 from 7.64.1 */
+/* Auto generated on Wed Jul 1 14:38:36 CEST 2020 from 7.71.1 */
struct {
const char *name;
@@ -26,12 +26,12 @@ struct {
{ "CURLOPT_LOW_SPEED_TIME", 20, pycurl_handle_setopt_long },
{ "CURLOPT_RESUME_FROM", 21, pycurl_handle_setopt_long },
{ "CURLOPT_COOKIE", 22, pycurl_handle_setopt_string },
- { "CURLOPT_HTTPHEADER", 23, NULL },
+ { "CURLOPT_HTTPHEADER", 23, pycurl_handle_setopt_slist },
{ "CURLOPT_HTTPPOST", 24, NULL },
{ "CURLOPT_SSLCERT", 25, pycurl_handle_setopt_string },
{ "CURLOPT_KEYPASSWD", 26, pycurl_handle_setopt_string },
{ "CURLOPT_CRLF", 27, pycurl_handle_setopt_long },
- { "CURLOPT_QUOTE", 28, NULL },
+ { "CURLOPT_QUOTE", 28, pycurl_handle_setopt_slist },
{ "CURLOPT_HEADERDATA", 29, NULL },
{ "CURLOPT_COOKIEFILE", 31, pycurl_handle_setopt_string },
{ "CURLOPT_SSLVERSION", 32, pycurl_handle_setopt_long },
@@ -39,8 +39,21 @@ struct {
{ "CURLOPT_TIMEVALUE", 34, pycurl_handle_setopt_long },
{ "CURLOPT_CUSTOMREQUEST", 36, pycurl_handle_setopt_string },
{ "CURLOPT_STDERR", 37, NULL },
- { "CURLOPT_POSTQUOTE", 39, NULL },
+ { "CURLOPT_POSTQUOTE", 39, pycurl_handle_setopt_slist },
+ { "CURLOPT_OBSOLETE40", 40, NULL },
+ { "CURLOPT_VERBOSE", 41, pycurl_handle_setopt_long },
+ { "CURLOPT_HEADER", 42, pycurl_handle_setopt_long },
+ { "CURLOPT_NOPROGRESS", 43, pycurl_handle_setopt_long },
+ { "CURLOPT_NOBODY", 44, pycurl_handle_setopt_long },
+ { "CURLOPT_FAILONERROR", 45, pycurl_handle_setopt_long },
+ { "CURLOPT_UPLOAD", 46, pycurl_handle_setopt_long },
+ { "CURLOPT_POST", 47, pycurl_handle_setopt_long },
+ { "CURLOPT_DIRLISTONLY", 48, pycurl_handle_setopt_long },
+ { "CURLOPT_APPEND", 50, pycurl_handle_setopt_long },
{ "CURLOPT_NETRC", 51, pycurl_handle_setopt_long },
+ { "CURLOPT_FOLLOWLOCATION", 52, pycurl_handle_setopt_long },
+ { "CURLOPT_TRANSFERTEXT", 53, pycurl_handle_setopt_long },
+ { "CURLOPT_PUT", 54, pycurl_handle_setopt_long },
{ "CURLOPT_PROGRESSFUNCTION", 56, NULL },
{ "CURLOPT_PROGRESSDATA", 57, NULL },
{ "CURLOPT_AUTOREFERER", 58, pycurl_handle_setopt_long },
@@ -53,8 +66,9 @@ struct {
{ "CURLOPT_CAINFO", 65, pycurl_handle_setopt_string },
{ "CURLOPT_MAXREDIRS", 68, pycurl_handle_setopt_long },
{ "CURLOPT_FILETIME", 69, pycurl_handle_setopt_long },
- { "CURLOPT_TELNETOPTIONS", 70, NULL },
+ { "CURLOPT_TELNETOPTIONS", 70, pycurl_handle_setopt_slist },
{ "CURLOPT_MAXCONNECTS", 71, pycurl_handle_setopt_long },
+ { "CURLOPT_OBSOLETE72", 72, pycurl_handle_setopt_long },
{ "CURLOPT_FRESH_CONNECT", 74, pycurl_handle_setopt_long },
{ "CURLOPT_FORBID_REUSE", 75, pycurl_handle_setopt_long },
{ "CURLOPT_RANDOM_FILE", 76, pycurl_handle_setopt_string },
@@ -72,8 +86,9 @@ struct {
{ "CURLOPT_SSLKEYTYPE", 88, pycurl_handle_setopt_string },
{ "CURLOPT_SSLENGINE", 89, pycurl_handle_setopt_string },
{ "CURLOPT_SSLENGINE_DEFAULT", 90, pycurl_handle_setopt_long },
+ { "CURLOPT_DNS_USE_GLOBAL_CACHE", 91, pycurl_handle_setopt_long },
{ "CURLOPT_DNS_CACHE_TIMEOUT", 92, pycurl_handle_setopt_long },
- { "CURLOPT_PREQUOTE", 93, NULL },
+ { "CURLOPT_PREQUOTE", 93, pycurl_handle_setopt_slist },
{ "CURLOPT_DEBUGFUNCTION", 94, NULL },
{ "CURLOPT_DEBUGDATA", 95, NULL },
{ "CURLOPT_COOKIESESSION", 96, pycurl_handle_setopt_long },
@@ -84,7 +99,7 @@ struct {
{ "CURLOPT_PROXYTYPE", 101, pycurl_handle_setopt_long },
{ "CURLOPT_ACCEPT_ENCODING", 102, pycurl_handle_setopt_string },
{ "CURLOPT_PRIVATE", 103, NULL },
- { "CURLOPT_HTTP200ALIASES", 104, NULL },
+ { "CURLOPT_HTTP200ALIASES", 104, pycurl_handle_setopt_slist },
{ "CURLOPT_UNRESTRICTED_AUTH", 105, pycurl_handle_setopt_long },
{ "CURLOPT_FTP_USE_EPRT", 106, pycurl_handle_setopt_long },
{ "CURLOPT_HTTPAUTH", 107, pycurl_handle_setopt_long },
@@ -150,6 +165,7 @@ struct {
{ "CURLOPT_PROXYPASSWORD", 176, pycurl_handle_setopt_string },
{ "CURLOPT_NOPROXY", 177, pycurl_handle_setopt_string },
{ "CURLOPT_TFTP_BLKSIZE", 178, pycurl_handle_setopt_long },
+ { "CURLOPT_SOCKS5_GSSAPI_SERVICE", 179, pycurl_handle_setopt_string },
{ "CURLOPT_SOCKS5_GSSAPI_NEC", 180, pycurl_handle_setopt_long },
{ "CURLOPT_PROTOCOLS", 181, pycurl_handle_setopt_long },
{ "CURLOPT_REDIR_PROTOCOLS", 182, pycurl_handle_setopt_long },
@@ -157,7 +173,7 @@ struct {
{ "CURLOPT_SSH_KEYFUNCTION", 184, NULL },
{ "CURLOPT_SSH_KEYDATA", 185, NULL },
{ "CURLOPT_MAIL_FROM", 186, pycurl_handle_setopt_string },
- { "CURLOPT_MAIL_RCPT", 187, NULL },
+ { "CURLOPT_MAIL_RCPT", 187, pycurl_handle_setopt_slist },
{ "CURLOPT_FTP_USE_PRET", 188, pycurl_handle_setopt_long },
{ "CURLOPT_RTSP_REQUEST", 189, pycurl_handle_setopt_long },
{ "CURLOPT_RTSP_SESSION_ID", 190, pycurl_handle_setopt_string },
@@ -173,7 +189,7 @@ struct {
{ "CURLOPT_FNMATCH_FUNCTION", 200, NULL },
{ "CURLOPT_CHUNK_DATA", 201, NULL },
{ "CURLOPT_FNMATCH_DATA", 202, NULL },
- { "CURLOPT_RESOLVE", 203, NULL },
+ { "CURLOPT_RESOLVE", 203, pycurl_handle_setopt_slist },
{ "CURLOPT_TLSAUTH_USERNAME", 204, pycurl_handle_setopt_string },
{ "CURLOPT_TLSAUTH_PASSWORD", 205, pycurl_handle_setopt_string },
{ "CURLOPT_TLSAUTH_TYPE", 206, pycurl_handle_setopt_string },
@@ -198,7 +214,7 @@ struct {
{ "CURLOPT_SSL_ENABLE_NPN", 225, pycurl_handle_setopt_long },
{ "CURLOPT_SSL_ENABLE_ALPN", 226, pycurl_handle_setopt_long },
{ "CURLOPT_EXPECT_100_TIMEOUT_MS", 227, pycurl_handle_setopt_long },
- { "CURLOPT_PROXYHEADER", 228, NULL },
+ { "CURLOPT_PROXYHEADER", 228, pycurl_handle_setopt_slist },
{ "CURLOPT_HEADEROPT", 229, pycurl_handle_setopt_long },
{ "CURLOPT_PINNEDPUBLICKEY", 230, pycurl_handle_setopt_string },
{ "CURLOPT_UNIX_SOCKET_PATH", 231, pycurl_handle_setopt_string },
@@ -213,7 +229,7 @@ struct {
{ "CURLOPT_STREAM_DEPENDS", 240, NULL },
{ "CURLOPT_STREAM_DEPENDS_E", 241, NULL },
{ "CURLOPT_TFTP_NO_OPTIONS", 242, pycurl_handle_setopt_long },
- { "CURLOPT_CONNECT_TO", 243, NULL },
+ { "CURLOPT_CONNECT_TO", 243, pycurl_handle_setopt_slist },
{ "CURLOPT_TCP_FASTOPEN", 244, pycurl_handle_setopt_long },
{ "CURLOPT_KEEP_SENDING_ON_ERROR", 245, pycurl_handle_setopt_long },
{ "CURLOPT_PROXY_CAINFO", 246, pycurl_handle_setopt_string },
@@ -258,5 +274,15 @@ struct {
{ "CURLOPT_HTTP09_ALLOWED", 285, pycurl_handle_setopt_long },
{ "CURLOPT_ALTSVC_CTRL", 286, pycurl_handle_setopt_long },
{ "CURLOPT_ALTSVC", 287, pycurl_handle_setopt_string },
+ { "CURLOPT_MAXAGE_CONN", 288, pycurl_handle_setopt_long },
+ { "CURLOPT_SASL_AUTHZID", 289, pycurl_handle_setopt_string },
+ { "CURLOPT_MAIL_RCPT_ALLLOWFAILS", 290, pycurl_handle_setopt_long },
+ { "CURLOPT_SSLCERT_BLOB", 291, NULL },
+ { "CURLOPT_SSLKEY_BLOB", 292, NULL },
+ { "CURLOPT_PROXY_SSLCERT_BLOB", 293, NULL },
+ { "CURLOPT_PROXY_SSLKEY_BLOB", 294, NULL },
+ { "CURLOPT_ISSUERCERT_BLOB", 295, NULL },
+ { "CURLOPT_PROXY_ISSUERCERT", 296, pycurl_handle_setopt_string },
+ { "CURLOPT_PROXY_ISSUERCERT_BLOB", 297, NULL },
{ NULL, 0, 0 }
};
diff --git a/src/python.c b/src/python.c
@@ -5404,6 +5404,9 @@ python_kore_curl_handle(PyObject *self, PyObject *args)
handle->url = kore_strdup(url);
memset(&handle->curl, 0, sizeof(handle->curl));
+ handle->body = NULL;
+ LIST_INIT(&handle->slists);
+
if (!kore_curl_init(&handle->curl, handle->url, KORE_CURL_ASYNC)) {
Py_DECREF((PyObject *)handle);
PyErr_SetString(PyExc_RuntimeError, "failed to setup call");
@@ -5416,6 +5419,17 @@ python_kore_curl_handle(PyObject *self, PyObject *args)
static void
pycurl_handle_dealloc(struct pycurl_handle *handle)
{
+ struct pycurl_slist *psl;
+
+ while ((psl = LIST_FIRST(&handle->slists))) {
+ LIST_REMOVE(psl, list);
+ curl_slist_free_all(psl->slist);
+ kore_free(psl);
+ }
+
+ if (handle->body != NULL)
+ kore_buf_free(handle->body);
+
kore_free(handle->url);
kore_curl_cleanup(&handle->curl);
@@ -5423,6 +5437,48 @@ pycurl_handle_dealloc(struct pycurl_handle *handle)
}
static PyObject *
+pycurl_handle_setbody(struct pycurl_handle *handle, PyObject *args)
+{
+ PyObject *obj;
+ char *ptr;
+ Py_ssize_t length;
+
+ if (!PyArg_ParseTuple(args, "O", &obj))
+ return (NULL);
+
+ if (handle->body != NULL) {
+ PyErr_SetString(PyExc_RuntimeError,
+ "curl handle already has body attached");
+ return (NULL);
+ }
+
+ if (!PyBytes_CheckExact(obj)) {
+ PyErr_SetString(PyExc_RuntimeError,
+ "curl.setbody expects bytes");
+ return (NULL);
+ }
+
+ if (PyBytes_AsStringAndSize(obj, &ptr, &length) == -1)
+ return (NULL);
+
+ if (length < 0) {
+ PyErr_SetString(PyExc_TypeError, "invalid length");
+ return (NULL);
+ }
+
+ handle->body = kore_buf_alloc(length);
+ kore_buf_append(handle->body, ptr, length);
+ kore_buf_reset(handle->body);
+
+ curl_easy_setopt(handle->curl.handle,
+ CURLOPT_READFUNCTION, kore_curl_frombuf);
+ curl_easy_setopt(handle->curl.handle, CURLOPT_READDATA, handle->body);
+ curl_easy_setopt(handle->curl.handle, CURLOPT_UPLOAD, 1);
+
+ Py_RETURN_TRUE;
+}
+
+static PyObject *
pycurl_handle_setopt(struct pycurl_handle *handle, PyObject *args)
{
int i, opt;
@@ -5459,14 +5515,15 @@ pycurl_handle_setopt_string(struct pycurl_handle *handle, int idx,
if (!PyUnicode_Check(obj)) {
PyErr_Format(PyExc_RuntimeError,
"option '%s' requires a string as argument",
- py_curlopt[idx]);
+ py_curlopt[idx].name);
return (NULL);
}
if ((str = PyUnicode_AsUTF8(obj)) == NULL)
return (NULL);
- curl_easy_setopt(&handle->curl, py_curlopt[idx].value, str);
+ curl_easy_setopt(handle->curl.handle,
+ CURLOPTTYPE_OBJECTPOINT + py_curlopt[idx].value, str);
Py_RETURN_TRUE;
}
@@ -5478,7 +5535,8 @@ pycurl_handle_setopt_long(struct pycurl_handle *handle, int idx, PyObject *obj)
if (!PyLong_CheckExact(obj)) {
PyErr_Format(PyExc_RuntimeError,
- "option '%s' requires a long as argument", py_curlopt[idx]);
+ "option '%s' requires a long as argument",
+ py_curlopt[idx].name);
return (NULL);
}
@@ -5487,7 +5545,51 @@ pycurl_handle_setopt_long(struct pycurl_handle *handle, int idx, PyObject *obj)
if (val == -1 && PyErr_Occurred())
return (NULL);
- curl_easy_setopt(&handle->curl, py_curlopt[idx].value, val);
+ curl_easy_setopt(handle->curl.handle,
+ CURLOPTTYPE_LONG + py_curlopt[idx].value, val);
+
+ Py_RETURN_TRUE;
+}
+
+static PyObject *
+pycurl_handle_setopt_slist(struct pycurl_handle *handle, int idx, PyObject *obj)
+{
+ struct pycurl_slist *psl;
+ PyObject *item;
+ const char *sval;
+ struct curl_slist *slist;
+ Py_ssize_t list_len, i;
+
+ if (!PyList_CheckExact(obj)) {
+ PyErr_Format(PyExc_RuntimeError,
+ "option '%s' requires a list as argument",
+ py_curlopt[idx].name);
+ return (NULL);
+ }
+
+ slist = NULL;
+ list_len = PyList_Size(obj);
+
+ for (i = 0; i < list_len; i++) {
+ if ((item = PyList_GetItem(obj, i)) == NULL)
+ return (NULL);
+
+ if (!PyUnicode_Check(item))
+ return (NULL);
+
+ if ((sval = PyUnicode_AsUTF8AndSize(item, NULL)) == NULL)
+ return (NULL);
+
+ if ((slist = curl_slist_append(slist, sval)) == NULL)
+ fatal("%s: curl_slist_append failed", __func__);
+ }
+
+ psl = kore_calloc(1, sizeof(*psl));
+ psl->slist = slist;
+ LIST_INSERT_HEAD(&handle->slists, psl, list);
+
+ curl_easy_setopt(handle->curl.handle,
+ CURLOPTTYPE_OBJECTPOINT + py_curlopt[idx].value, slist);
Py_RETURN_TRUE;
}
@@ -5539,6 +5641,11 @@ pycurl_handle_op_iternext(struct pycurl_handle_op *op)
Py_RETURN_NONE;
}
+ if (op->handle->body != NULL) {
+ kore_buf_free(op->handle->body);
+ op->handle->body = NULL;
+ }
+
if (!kore_curl_success(&op->handle->curl)) {
/* Do not log the url here, may contain some sensitive data. */
PyErr_Format(PyExc_RuntimeError, "request failed: %s",
@@ -5684,7 +5791,7 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs)
((headers = PyDict_GetItemString(kwargs, "headers")) != NULL)) {
if (!PyDict_CheckExact(headers)) {
PyErr_SetString(PyExc_RuntimeError,
- "header keyword must be a dict");
+ "headers keyword must be a dict");
return (NULL);
}
}
