kore

An easy to use, scalable and secure web application framework for writing web APIs in C.
Commits | Files | Refs | README | LICENSE | git clone https://git.kore.io/kore.git

commit 2d380cac3f4707e28667030ae449710f28279831
parent 6f31e14e83ff4701532acd02d182990a97f72601
Author: Joris Vink <joris@coders.se>
Date:   Sat, 18 Jan 2020 19:43:38 +0100

Expose our async libcurl support to the Python api.

Kore already exposed parts of this via the kore.httpclient() method but
this commit takes it a bit further and exposes the libcurl interface
completely (including the setopt options).

tldr:

handle = kore.curl("ftp://ftp.eu.openbsd.org/pub/OpenBSD/README")
handle.setopt(kore.CURLOPT_TIMEOUT, 5)

data = await handle.run()
print("%s" % data.decode())

Diffstat:
Makefile | 10++++++++--
include/kore/python_methods.h | 69++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
misc/curl-extract-opt.sh | 46++++++++++++++++++++++++++++++++++++++++++++++
misc/curl/python_curlopt.h | 262+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/python.c | 206++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
5 files changed, 581 insertions(+), 12 deletions(-)

diff --git a/Makefile b/Makefile @@ -11,8 +11,10 @@ MAN_DIR?=$(PREFIX)/share/man SHARE_DIR=$(PREFIX)/share/kore INCLUDE_DIR=$(PREFIX)/include/kore +GENERATED= PLATFORM=platform.h VERSION=src/version.c +PYTHON_CURLOPT=misc/curl/python_curlopt.h S_SRC= src/kore.c src/buf.c src/config.c src/connection.c \ src/domain.c src/filemap.c src/fileref.c src/json.c src/mem.c \ @@ -88,6 +90,7 @@ endif ifneq ("$(PYTHON)", "") S_SRC+=src/python.c + GENERATED+=$(PYTHON_CURLOPT) KORE_PYTHON_LIB?=$(shell ./misc/python3-config.sh --ldflags) KORE_PYTHON_INC?=$(shell ./misc/python3-config.sh --includes) LDFLAGS+=$(KORE_PYTHON_LIB) @@ -142,13 +145,16 @@ endif S_OBJS= $(S_SRC:src/%.c=$(OBJDIR)/%.o) -all: $(PLATFORM) $(VERSION) $(KORE) $(KODEV) +all: $(PLATFORM) $(GENERATED) $(VERSION) $(KORE) $(KODEV) $(PLATFORM): $(OBJDIR) force @if [ -f misc/$(OSNAME)-platform.sh ]; then \ misc/$(OSNAME)-platform.sh > $(OBJDIR)/$(PLATFORM) ; \ fi +$(PYTHON_CURLOPT): $(OBJDIR) force + @cp $(PYTHON_CURLOPT) $(OBJDIR) + $(VERSION): force @if [ -d .git ]; then \ GIT_REVISION=`git rev-parse --short=8 HEAD`; \ @@ -171,7 +177,7 @@ $(KORE): $(OBJDIR) $(S_OBJS) $(CC) $(S_OBJS) $(LDFLAGS) -o $(KORE) @echo $(FEATURES) $(FEATURES_INC) > kore.features -objects: $(OBJDIR) $(PLATFORM) $(S_OBJS) +objects: $(OBJDIR) $(PLATFORM) $(GENERATED) $(S_OBJS) @echo $(LDFLAGS) > $(OBJDIR)/ldflags @echo "$(FEATURES) $(FEATURES_INC)" > $(OBJDIR)/features diff --git a/include/kore/python_methods.h b/include/kore/python_methods.h @@ -67,6 +67,7 @@ static PyObject *python_kore_pgsql_register(PyObject *, PyObject *); #endif #if defined(KORE_USE_CURL) +static PyObject *python_kore_curl_handle(PyObject *, PyObject *); static PyObject *python_kore_httpclient(PyObject *, PyObject *, PyObject *); #endif @@ -109,6 +110,7 @@ static struct PyMethodDef pykore_methods[] = { METH_VARARGS | METH_KEYWORDS), #endif #if defined(KORE_USE_CURL) + METHOD("curl", python_kore_curl_handle, METH_VARARGS), METHOD("httpclient", python_kore_httpclient, METH_VARARGS | METH_KEYWORDS), #endif @@ -788,6 +790,70 @@ static PyTypeObject pyhttp_file_type = { }; #if defined(KORE_USE_CURL) + +#define CURL_CLIENT_OP_RUN 1 +#define CURL_CLIENT_OP_RESULT 2 + +struct pycurl_handle { + PyObject_HEAD + char *url; + struct kore_curl curl; +}; + +struct pycurl_handle_op { + PyObject_HEAD + int state; + struct python_coro *coro; + struct pycurl_handle *handle; +}; + +static PyObject *pycurl_handle_op_await(PyObject *); +static PyObject *pycurl_handle_op_iternext(struct pycurl_handle_op *); + +static void pycurl_handle_dealloc(struct pycurl_handle *); +static void pycurl_handle_op_dealloc(struct pycurl_handle_op *); + +static PyObject *pycurl_handle_run(struct pycurl_handle *, PyObject *); +static PyObject *pycurl_handle_setopt(struct pycurl_handle *, PyObject *); + +static PyObject *pycurl_handle_setopt_string(struct pycurl_handle *, + int, PyObject *); +static PyObject *pycurl_handle_setopt_long(struct pycurl_handle *, + int, PyObject *); + +static PyMethodDef pycurl_handle_methods[] = { + METHOD("run", pycurl_handle_run, METH_VARARGS), + METHOD("setopt", pycurl_handle_setopt, METH_VARARGS), + METHOD(NULL, NULL, -1) +}; + +static PyTypeObject pycurl_handle_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.curl", + .tp_doc = "An asynchronous CURL handle", + .tp_methods = pycurl_handle_methods, + .tp_basicsize = sizeof(struct pycurl_handle), + .tp_dealloc = (destructor)pycurl_handle_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + +static PyAsyncMethods pycurl_handle_op_async = { + (unaryfunc)pycurl_handle_op_await, + NULL, + NULL +}; + +static PyTypeObject pycurl_handle_op_type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "kore.curlop", + .tp_doc = "Asynchronous CURL operation", + .tp_as_async = &pycurl_handle_op_async, + .tp_iternext = (iternextfunc)pycurl_handle_op_iternext, + .tp_basicsize = sizeof(struct pycurl_handle_op), + .tp_dealloc = (destructor)pycurl_handle_op_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +}; + struct pyhttp_client { PyObject_HEAD char *url; @@ -798,9 +864,6 @@ struct pyhttp_client { int tlsverify; }; -#define PYHTTP_CLIENT_OP_RUN 1 -#define PYHTTP_CLIENT_OP_RESULT 2 - struct pyhttp_client_op { PyObject_HEAD int state; diff --git a/misc/curl-extract-opt.sh b/misc/curl-extract-opt.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +if [ $# -ne 1 ]; then + echo "Usage: curl-extract-opt.sh /path/to/include/curl" + exit 1 +fi + +if [ ! -d $1 ]; then + echo "given argument is not a directory" + exit 1 +fi + +if [ ! -f "$1/curl.h" ]; then + echo "$1 does not contain curl.h" + exit 1 +fi + +if [ ! -f "$1/curlver.h" ]; then + echo "$1 does not contain curlver.h" +fi + +version=`egrep "#define LIBCURL_VERSION " "$1/curlver.h" | awk '{ print $3 }' | sed s/\"//g` + +echo "/* Auto generated on `date` from $version */" + +cat << __EOF + +struct { + const char *name; + int value; + PyObject *(*cb)(struct pycurl_handle *, int, PyObject *); +} py_curlopt[] = { +__EOF + +egrep "^.*CINIT\(.*\),$" "$1/curl.h" | \ + cut -d'(' -f 2 | cut -d ')' -f 1 | sed 's/,/ /g' | \ + sed 's/OBJECTPOINT/NULL/g' | \ + sed 's/STRINGPOINT/pycurl_handle_setopt_string/g' | \ + sed 's/LONG/pycurl_handle_setopt_long/g' | \ + sed 's/SLISTPOINT/NULL/g' | \ + sed 's/FUNCTIONPOINT/NULL/g' | \ + sed 's/OFF_T/NULL/g' | \ + awk '{ printf "\t{ \"CURLOPT_%s\", %s, %s },\n", $1, $3, $2 } ' + +echo "\t{ NULL, 0, 0 }" +echo "};" diff --git a/misc/curl/python_curlopt.h b/misc/curl/python_curlopt.h @@ -0,0 +1,262 @@ +/* Auto generated on Sat Jan 18 19:42:48 CET 2020 from 7.64.1 */ + +struct { + const char *name; + int value; + PyObject *(*cb)(struct pycurl_handle *, int, PyObject *); +} py_curlopt[] = { + { "CURLOPT_WRITEDATA", 1, NULL }, + { "CURLOPT_URL", 2, pycurl_handle_setopt_string }, + { "CURLOPT_PORT", 3, pycurl_handle_setopt_long }, + { "CURLOPT_PROXY", 4, pycurl_handle_setopt_string }, + { "CURLOPT_USERPWD", 5, pycurl_handle_setopt_string }, + { "CURLOPT_PROXYUSERPWD", 6, pycurl_handle_setopt_string }, + { "CURLOPT_RANGE", 7, pycurl_handle_setopt_string }, + { "CURLOPT_READDATA", 9, NULL }, + { "CURLOPT_ERRORBUFFER", 10, NULL }, + { "CURLOPT_WRITEFUNCTION", 11, NULL }, + { "CURLOPT_READFUNCTION", 12, NULL }, + { "CURLOPT_TIMEOUT", 13, pycurl_handle_setopt_long }, + { "CURLOPT_INFILESIZE", 14, pycurl_handle_setopt_long }, + { "CURLOPT_POSTFIELDS", 15, NULL }, + { "CURLOPT_REFERER", 16, pycurl_handle_setopt_string }, + { "CURLOPT_FTPPORT", 17, pycurl_handle_setopt_string }, + { "CURLOPT_USERAGENT", 18, pycurl_handle_setopt_string }, + { "CURLOPT_LOW_SPEED_LIMIT", 19, pycurl_handle_setopt_long }, + { "CURLOPT_LOW_SPEED_TIME", 20, pycurl_handle_setopt_long }, + { "CURLOPT_RESUME_FROM", 21, pycurl_handle_setopt_long }, + { "CURLOPT_COOKIE", 22, pycurl_handle_setopt_string }, + { "CURLOPT_HTTPHEADER", 23, NULL }, + { "CURLOPT_HTTPPOST", 24, NULL }, + { "CURLOPT_SSLCERT", 25, pycurl_handle_setopt_string }, + { "CURLOPT_KEYPASSWD", 26, pycurl_handle_setopt_string }, + { "CURLOPT_CRLF", 27, pycurl_handle_setopt_long }, + { "CURLOPT_QUOTE", 28, NULL }, + { "CURLOPT_HEADERDATA", 29, NULL }, + { "CURLOPT_COOKIEFILE", 31, pycurl_handle_setopt_string }, + { "CURLOPT_SSLVERSION", 32, pycurl_handle_setopt_long }, + { "CURLOPT_TIMECONDITION", 33, pycurl_handle_setopt_long }, + { "CURLOPT_TIMEVALUE", 34, pycurl_handle_setopt_long }, + { "CURLOPT_CUSTOMREQUEST", 36, pycurl_handle_setopt_string }, + { "CURLOPT_STDERR", 37, NULL }, + { "CURLOPT_POSTQUOTE", 39, NULL }, + { "CURLOPT_NETRC", 51, pycurl_handle_setopt_long }, + { "CURLOPT_PROGRESSFUNCTION", 56, NULL }, + { "CURLOPT_PROGRESSDATA", 57, NULL }, + { "CURLOPT_AUTOREFERER", 58, pycurl_handle_setopt_long }, + { "CURLOPT_PROXYPORT", 59, pycurl_handle_setopt_long }, + { "CURLOPT_POSTFIELDSIZE", 60, pycurl_handle_setopt_long }, + { "CURLOPT_HTTPPROXYTUNNEL", 61, pycurl_handle_setopt_long }, + { "CURLOPT_INTERFACE", 62, pycurl_handle_setopt_string }, + { "CURLOPT_KRBLEVEL", 63, pycurl_handle_setopt_string }, + { "CURLOPT_SSL_VERIFYPEER", 64, pycurl_handle_setopt_long }, + { "CURLOPT_CAINFO", 65, pycurl_handle_setopt_string }, + { "CURLOPT_MAXREDIRS", 68, pycurl_handle_setopt_long }, + { "CURLOPT_FILETIME", 69, pycurl_handle_setopt_long }, + { "CURLOPT_TELNETOPTIONS", 70, NULL }, + { "CURLOPT_MAXCONNECTS", 71, pycurl_handle_setopt_long }, + { "CURLOPT_FRESH_CONNECT", 74, pycurl_handle_setopt_long }, + { "CURLOPT_FORBID_REUSE", 75, pycurl_handle_setopt_long }, + { "CURLOPT_RANDOM_FILE", 76, pycurl_handle_setopt_string }, + { "CURLOPT_EGDSOCKET", 77, pycurl_handle_setopt_string }, + { "CURLOPT_CONNECTTIMEOUT", 78, pycurl_handle_setopt_long }, + { "CURLOPT_HEADERFUNCTION", 79, NULL }, + { "CURLOPT_HTTPGET", 80, pycurl_handle_setopt_long }, + { "CURLOPT_SSL_VERIFYHOST", 81, pycurl_handle_setopt_long }, + { "CURLOPT_COOKIEJAR", 82, pycurl_handle_setopt_string }, + { "CURLOPT_SSL_CIPHER_LIST", 83, pycurl_handle_setopt_string }, + { "CURLOPT_HTTP_VERSION", 84, pycurl_handle_setopt_long }, + { "CURLOPT_FTP_USE_EPSV", 85, pycurl_handle_setopt_long }, + { "CURLOPT_SSLCERTTYPE", 86, pycurl_handle_setopt_string }, + { "CURLOPT_SSLKEY", 87, pycurl_handle_setopt_string }, + { "CURLOPT_SSLKEYTYPE", 88, pycurl_handle_setopt_string }, + { "CURLOPT_SSLENGINE", 89, pycurl_handle_setopt_string }, + { "CURLOPT_SSLENGINE_DEFAULT", 90, pycurl_handle_setopt_long }, + { "CURLOPT_DNS_CACHE_TIMEOUT", 92, pycurl_handle_setopt_long }, + { "CURLOPT_PREQUOTE", 93, NULL }, + { "CURLOPT_DEBUGFUNCTION", 94, NULL }, + { "CURLOPT_DEBUGDATA", 95, NULL }, + { "CURLOPT_COOKIESESSION", 96, pycurl_handle_setopt_long }, + { "CURLOPT_CAPATH", 97, pycurl_handle_setopt_string }, + { "CURLOPT_BUFFERSIZE", 98, pycurl_handle_setopt_long }, + { "CURLOPT_NOSIGNAL", 99, pycurl_handle_setopt_long }, + { "CURLOPT_SHARE", 100, NULL }, + { "CURLOPT_PROXYTYPE", 101, pycurl_handle_setopt_long }, + { "CURLOPT_ACCEPT_ENCODING", 102, pycurl_handle_setopt_string }, + { "CURLOPT_PRIVATE", 103, NULL }, + { "CURLOPT_HTTP200ALIASES", 104, NULL }, + { "CURLOPT_UNRESTRICTED_AUTH", 105, pycurl_handle_setopt_long }, + { "CURLOPT_FTP_USE_EPRT", 106, pycurl_handle_setopt_long }, + { "CURLOPT_HTTPAUTH", 107, pycurl_handle_setopt_long }, + { "CURLOPT_SSL_CTX_FUNCTION", 108, NULL }, + { "CURLOPT_SSL_CTX_DATA", 109, NULL }, + { "CURLOPT_FTP_CREATE_MISSING_DIRS", 110, pycurl_handle_setopt_long }, + { "CURLOPT_PROXYAUTH", 111, pycurl_handle_setopt_long }, + { "CURLOPT_FTP_RESPONSE_TIMEOUT", 112, pycurl_handle_setopt_long }, + { "CURLOPT_IPRESOLVE", 113, pycurl_handle_setopt_long }, + { "CURLOPT_MAXFILESIZE", 114, pycurl_handle_setopt_long }, + { "CURLOPT_INFILESIZE_LARGE", 115, NULL }, + { "CURLOPT_RESUME_FROM_LARGE", 116, NULL }, + { "CURLOPT_MAXFILESIZE_LARGE", 117, NULL }, + { "CURLOPT_NETRC_FILE", 118, pycurl_handle_setopt_string }, + { "CURLOPT_USE_SSL", 119, pycurl_handle_setopt_long }, + { "CURLOPT_POSTFIELDSIZE_LARGE", 120, NULL }, + { "CURLOPT_TCP_NODELAY", 121, pycurl_handle_setopt_long }, + { "CURLOPT_FTPSSLAUTH", 129, pycurl_handle_setopt_long }, + { "CURLOPT_IOCTLFUNCTION", 130, NULL }, + { "CURLOPT_IOCTLDATA", 131, NULL }, + { "CURLOPT_FTP_ACCOUNT", 134, pycurl_handle_setopt_string }, + { "CURLOPT_COOKIELIST", 135, pycurl_handle_setopt_string }, + { "CURLOPT_IGNORE_CONTENT_LENGTH", 136, pycurl_handle_setopt_long }, + { "CURLOPT_FTP_SKIP_PASV_IP", 137, pycurl_handle_setopt_long }, + { "CURLOPT_FTP_FILEMETHOD", 138, pycurl_handle_setopt_long }, + { "CURLOPT_LOCALPORT", 139, pycurl_handle_setopt_long }, + { "CURLOPT_LOCALPORTRANGE", 140, pycurl_handle_setopt_long }, + { "CURLOPT_CONNECT_ONLY", 141, pycurl_handle_setopt_long }, + { "CURLOPT_CONV_FROM_NETWORK_FUNCTION", 142, NULL }, + { "CURLOPT_CONV_TO_NETWORK_FUNCTION", 143, NULL }, + { "CURLOPT_CONV_FROM_UTF8_FUNCTION", 144, NULL }, + { "CURLOPT_MAX_SEND_SPEED_LARGE", 145, NULL }, + { "CURLOPT_MAX_RECV_SPEED_LARGE", 146, NULL }, + { "CURLOPT_FTP_ALTERNATIVE_TO_USER", 147, pycurl_handle_setopt_string }, + { "CURLOPT_SOCKOPTFUNCTION", 148, NULL }, + { "CURLOPT_SOCKOPTDATA", 149, NULL }, + { "CURLOPT_SSL_SESSIONID_CACHE", 150, pycurl_handle_setopt_long }, + { "CURLOPT_SSH_AUTH_TYPES", 151, pycurl_handle_setopt_long }, + { "CURLOPT_SSH_PUBLIC_KEYFILE", 152, pycurl_handle_setopt_string }, + { "CURLOPT_SSH_PRIVATE_KEYFILE", 153, pycurl_handle_setopt_string }, + { "CURLOPT_FTP_SSL_CCC", 154, pycurl_handle_setopt_long }, + { "CURLOPT_TIMEOUT_MS", 155, pycurl_handle_setopt_long }, + { "CURLOPT_CONNECTTIMEOUT_MS", 156, pycurl_handle_setopt_long }, + { "CURLOPT_HTTP_TRANSFER_DECODING", 157, pycurl_handle_setopt_long }, + { "CURLOPT_HTTP_CONTENT_DECODING", 158, pycurl_handle_setopt_long }, + { "CURLOPT_NEW_FILE_PERMS", 159, pycurl_handle_setopt_long }, + { "CURLOPT_NEW_DIRECTORY_PERMS", 160, pycurl_handle_setopt_long }, + { "CURLOPT_POSTREDIR", 161, pycurl_handle_setopt_long }, + { "CURLOPT_SSH_HOST_PUBLIC_KEY_MD5", 162, pycurl_handle_setopt_string }, + { "CURLOPT_OPENSOCKETFUNCTION", 163, NULL }, + { "CURLOPT_OPENSOCKETDATA", 164, NULL }, + { "CURLOPT_COPYPOSTFIELDS", 165, NULL }, + { "CURLOPT_PROXY_TRANSFER_MODE", 166, pycurl_handle_setopt_long }, + { "CURLOPT_SEEKFUNCTION", 167, NULL }, + { "CURLOPT_SEEKDATA", 168, NULL }, + { "CURLOPT_CRLFILE", 169, pycurl_handle_setopt_string }, + { "CURLOPT_ISSUERCERT", 170, pycurl_handle_setopt_string }, + { "CURLOPT_ADDRESS_SCOPE", 171, pycurl_handle_setopt_long }, + { "CURLOPT_CERTINFO", 172, pycurl_handle_setopt_long }, + { "CURLOPT_USERNAME", 173, pycurl_handle_setopt_string }, + { "CURLOPT_PASSWORD", 174, pycurl_handle_setopt_string }, + { "CURLOPT_PROXYUSERNAME", 175, pycurl_handle_setopt_string }, + { "CURLOPT_PROXYPASSWORD", 176, pycurl_handle_setopt_string }, + { "CURLOPT_NOPROXY", 177, pycurl_handle_setopt_string }, + { "CURLOPT_TFTP_BLKSIZE", 178, pycurl_handle_setopt_long }, + { "CURLOPT_SOCKS5_GSSAPI_NEC", 180, pycurl_handle_setopt_long }, + { "CURLOPT_PROTOCOLS", 181, pycurl_handle_setopt_long }, + { "CURLOPT_REDIR_PROTOCOLS", 182, pycurl_handle_setopt_long }, + { "CURLOPT_SSH_KNOWNHOSTS", 183, pycurl_handle_setopt_string }, + { "CURLOPT_SSH_KEYFUNCTION", 184, NULL }, + { "CURLOPT_SSH_KEYDATA", 185, NULL }, + { "CURLOPT_MAIL_FROM", 186, pycurl_handle_setopt_string }, + { "CURLOPT_MAIL_RCPT", 187, NULL }, + { "CURLOPT_FTP_USE_PRET", 188, pycurl_handle_setopt_long }, + { "CURLOPT_RTSP_REQUEST", 189, pycurl_handle_setopt_long }, + { "CURLOPT_RTSP_SESSION_ID", 190, pycurl_handle_setopt_string }, + { "CURLOPT_RTSP_STREAM_URI", 191, pycurl_handle_setopt_string }, + { "CURLOPT_RTSP_TRANSPORT", 192, pycurl_handle_setopt_string }, + { "CURLOPT_RTSP_CLIENT_CSEQ", 193, pycurl_handle_setopt_long }, + { "CURLOPT_RTSP_SERVER_CSEQ", 194, pycurl_handle_setopt_long }, + { "CURLOPT_INTERLEAVEDATA", 195, NULL }, + { "CURLOPT_INTERLEAVEFUNCTION", 196, NULL }, + { "CURLOPT_WILDCARDMATCH", 197, pycurl_handle_setopt_long }, + { "CURLOPT_CHUNK_BGN_FUNCTION", 198, NULL }, + { "CURLOPT_CHUNK_END_FUNCTION", 199, NULL }, + { "CURLOPT_FNMATCH_FUNCTION", 200, NULL }, + { "CURLOPT_CHUNK_DATA", 201, NULL }, + { "CURLOPT_FNMATCH_DATA", 202, NULL }, + { "CURLOPT_RESOLVE", 203, NULL }, + { "CURLOPT_TLSAUTH_USERNAME", 204, pycurl_handle_setopt_string }, + { "CURLOPT_TLSAUTH_PASSWORD", 205, pycurl_handle_setopt_string }, + { "CURLOPT_TLSAUTH_TYPE", 206, pycurl_handle_setopt_string }, + { "CURLOPT_TRANSFER_ENCODING", 207, pycurl_handle_setopt_long }, + { "CURLOPT_CLOSESOCKETFUNCTION", 208, NULL }, + { "CURLOPT_CLOSESOCKETDATA", 209, NULL }, + { "CURLOPT_GSSAPI_DELEGATION", 210, pycurl_handle_setopt_long }, + { "CURLOPT_DNS_SERVERS", 211, pycurl_handle_setopt_string }, + { "CURLOPT_ACCEPTTIMEOUT_MS", 212, pycurl_handle_setopt_long }, + { "CURLOPT_TCP_KEEPALIVE", 213, pycurl_handle_setopt_long }, + { "CURLOPT_TCP_KEEPIDLE", 214, pycurl_handle_setopt_long }, + { "CURLOPT_TCP_KEEPINTVL", 215, pycurl_handle_setopt_long }, + { "CURLOPT_SSL_OPTIONS", 216, pycurl_handle_setopt_long }, + { "CURLOPT_MAIL_AUTH", 217, pycurl_handle_setopt_string }, + { "CURLOPT_SASL_IR", 218, pycurl_handle_setopt_long }, + { "CURLOPT_XFERINFOFUNCTION", 219, NULL }, + { "CURLOPT_XOAUTH2_BEARER", 220, pycurl_handle_setopt_string }, + { "CURLOPT_DNS_INTERFACE", 221, pycurl_handle_setopt_string }, + { "CURLOPT_DNS_LOCAL_IP4", 222, pycurl_handle_setopt_string }, + { "CURLOPT_DNS_LOCAL_IP6", 223, pycurl_handle_setopt_string }, + { "CURLOPT_LOGIN_OPTIONS", 224, pycurl_handle_setopt_string }, + { "CURLOPT_SSL_ENABLE_NPN", 225, pycurl_handle_setopt_long }, + { "CURLOPT_SSL_ENABLE_ALPN", 226, pycurl_handle_setopt_long }, + { "CURLOPT_EXPECT_100_TIMEOUT_MS", 227, pycurl_handle_setopt_long }, + { "CURLOPT_PROXYHEADER", 228, NULL }, + { "CURLOPT_HEADEROPT", 229, pycurl_handle_setopt_long }, + { "CURLOPT_PINNEDPUBLICKEY", 230, pycurl_handle_setopt_string }, + { "CURLOPT_UNIX_SOCKET_PATH", 231, pycurl_handle_setopt_string }, + { "CURLOPT_SSL_VERIFYSTATUS", 232, pycurl_handle_setopt_long }, + { "CURLOPT_SSL_FALSESTART", 233, pycurl_handle_setopt_long }, + { "CURLOPT_PATH_AS_IS", 234, pycurl_handle_setopt_long }, + { "CURLOPT_PROXY_SERVICE_NAME", 235, pycurl_handle_setopt_string }, + { "CURLOPT_SERVICE_NAME", 236, pycurl_handle_setopt_string }, + { "CURLOPT_PIPEWAIT", 237, pycurl_handle_setopt_long }, + { "CURLOPT_DEFAULT_PROTOCOL", 238, pycurl_handle_setopt_string }, + { "CURLOPT_STREAM_WEIGHT", 239, pycurl_handle_setopt_long }, + { "CURLOPT_STREAM_DEPENDS", 240, NULL }, + { "CURLOPT_STREAM_DEPENDS_E", 241, NULL }, + { "CURLOPT_TFTP_NO_OPTIONS", 242, pycurl_handle_setopt_long }, + { "CURLOPT_CONNECT_TO", 243, NULL }, + { "CURLOPT_TCP_FASTOPEN", 244, pycurl_handle_setopt_long }, + { "CURLOPT_KEEP_SENDING_ON_ERROR", 245, pycurl_handle_setopt_long }, + { "CURLOPT_PROXY_CAINFO", 246, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_CAPATH", 247, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_SSL_VERIFYPEER", 248, pycurl_handle_setopt_long }, + { "CURLOPT_PROXY_SSL_VERIFYHOST", 249, pycurl_handle_setopt_long }, + { "CURLOPT_PROXY_SSLVERSION", 250, pycurl_handle_setopt_long }, + { "CURLOPT_PROXY_TLSAUTH_USERNAME", 251, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_TLSAUTH_PASSWORD", 252, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_TLSAUTH_TYPE", 253, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_SSLCERT", 254, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_SSLCERTTYPE", 255, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_SSLKEY", 256, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_SSLKEYTYPE", 257, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_KEYPASSWD", 258, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_SSL_CIPHER_LIST", 259, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_CRLFILE", 260, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_SSL_OPTIONS", 261, pycurl_handle_setopt_long }, + { "CURLOPT_PRE_PROXY", 262, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_PINNEDPUBLICKEY", 263, pycurl_handle_setopt_string }, + { "CURLOPT_ABSTRACT_UNIX_SOCKET", 264, pycurl_handle_setopt_string }, + { "CURLOPT_SUPPRESS_CONNECT_HEADERS", 265, pycurl_handle_setopt_long }, + { "CURLOPT_REQUEST_TARGET", 266, pycurl_handle_setopt_string }, + { "CURLOPT_SOCKS5_AUTH", 267, pycurl_handle_setopt_long }, + { "CURLOPT_SSH_COMPRESSION", 268, pycurl_handle_setopt_long }, + { "CURLOPT_MIMEPOST", 269, NULL }, + { "CURLOPT_TIMEVALUE_LARGE", 270, NULL }, + { "CURLOPT_HAPPY_EYEBALLS_TIMEOUT_MS", 271, pycurl_handle_setopt_long }, + { "CURLOPT_RESOLVER_START_FUNCTION", 272, NULL }, + { "CURLOPT_RESOLVER_START_DATA", 273, NULL }, + { "CURLOPT_HAPROXYPROTOCOL", 274, pycurl_handle_setopt_long }, + { "CURLOPT_DNS_SHUFFLE_ADDRESSES", 275, pycurl_handle_setopt_long }, + { "CURLOPT_TLS13_CIPHERS", 276, pycurl_handle_setopt_string }, + { "CURLOPT_PROXY_TLS13_CIPHERS", 277, pycurl_handle_setopt_string }, + { "CURLOPT_DISALLOW_USERNAME_IN_URL", 278, pycurl_handle_setopt_long }, + { "CURLOPT_DOH_URL", 279, pycurl_handle_setopt_string }, + { "CURLOPT_UPLOAD_BUFFERSIZE", 280, pycurl_handle_setopt_long }, + { "CURLOPT_UPKEEP_INTERVAL_MS", 281, pycurl_handle_setopt_long }, + { "CURLOPT_CURLU", 282, NULL }, + { "CURLOPT_TRAILERFUNCTION", 283, NULL }, + { "CURLOPT_TRAILERDATA", 284, NULL }, + { "CURLOPT_HTTP09_ALLOWED", 285, pycurl_handle_setopt_long }, + { "CURLOPT_ALTSVC_CTRL", 286, pycurl_handle_setopt_long }, + { "CURLOPT_ALTSVC", 287, pycurl_handle_setopt_string }, + { NULL, 0, 0 } +}; diff --git a/src/python.c b/src/python.c @@ -46,6 +46,10 @@ #include "python_api.h" #include "python_methods.h" +#if defined(KORE_USE_CURL) +#include "python_curlopt.h" +#endif + #include <frameobject.h> struct reqcall { @@ -115,7 +119,8 @@ static int pykore_pgsql_params(struct pykore_pgsql *, PyObject *); #endif #if defined(KORE_USE_CURL) -static void python_curl_callback(struct kore_curl *, void *); +static void python_curl_http_callback(struct kore_curl *, void *); +static void python_curl_handle_callback(struct kore_curl *, void *); static PyObject *pyhttp_client_request(struct pyhttp_client *, int, PyObject *); #endif @@ -383,7 +388,7 @@ kore_python_coro_run(void) #if defined(KORE_USE_CURL) /* - * If a coroutine fired off an httpclient instance, immediately + * If a coroutine fired off a curl instance, immediately * let it make progress. */ kore_curl_do_timeout(); @@ -1506,7 +1511,13 @@ python_module_init(void) #endif #if defined(KORE_USE_CURL) + python_push_type("pycurlhandle", pykore, &pycurl_handle_type); python_push_type("pyhttpclient", pykore, &pyhttp_client_type); + + for (i = 0; py_curlopt[i].name != NULL; i++) { + python_push_integer(pykore, py_curlopt[i].name, + py_curlopt[i].value); + } #endif python_push_type("pyhttp_file", pykore, &pyhttp_file_type); @@ -5349,6 +5360,176 @@ pykore_pgsql_result(struct pykore_pgsql *pysql) #if defined(KORE_USE_CURL) static PyObject * +python_kore_curl_handle(PyObject *self, PyObject *args) +{ + const char *url; + struct pycurl_handle *handle; + + if (!PyArg_ParseTuple(args, "s", &url)) + return (NULL); + + handle = PyObject_New(struct pycurl_handle, &pycurl_handle_type); + if (handle == NULL) + return (NULL); + + handle->url = kore_strdup(url); + memset(&handle->curl, 0, sizeof(handle->curl)); + + if (!kore_curl_init(&handle->curl, handle->url, KORE_CURL_ASYNC)) { + Py_DECREF((PyObject *)handle); + PyErr_SetString(PyExc_RuntimeError, "failed to setup call"); + return (NULL); + } + + return ((PyObject *)handle); +} + +static void +pycurl_handle_dealloc(struct pycurl_handle *handle) +{ + kore_free(handle->url); + kore_curl_cleanup(&handle->curl); + + PyObject_Del((PyObject *)handle); +} + +static PyObject * +pycurl_handle_setopt(struct pycurl_handle *handle, PyObject *args) +{ + int i, opt; + PyObject *value; + + if (!PyArg_ParseTuple(args, "iO", &opt, &value)) + return (NULL); + + for (i = 0; py_curlopt[i].name != NULL; i++) { + if (py_curlopt[i].value == opt) + break; + } + + if (py_curlopt[i].name == NULL) { + PyErr_Format(PyExc_RuntimeError, "invalid option '%d'", opt); + return (NULL); + } + + if (py_curlopt[i].cb == NULL) { + PyErr_Format(PyExc_RuntimeError, "option '%s' not implemented", + py_curlopt[i].name); + return (NULL); + } + + return (py_curlopt[i].cb(handle, i, value)); +} + +static PyObject * +pycurl_handle_setopt_string(struct pycurl_handle *handle, int idx, + PyObject *obj) +{ + const char *str; + + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_RuntimeError, + "option '%s' requires a string as argument", + py_curlopt[idx]); + return (NULL); + } + + if ((str = PyUnicode_AsUTF8(obj)) == NULL) + return (NULL); + + curl_easy_setopt(&handle->curl, py_curlopt[idx].value, str); + + Py_RETURN_TRUE; +} + +static PyObject * +pycurl_handle_setopt_long(struct pycurl_handle *handle, int idx, PyObject *obj) +{ + long val; + + if (!PyLong_CheckExact(obj)) { + PyErr_Format(PyExc_RuntimeError, + "option '%s' requires a long as argument", py_curlopt[idx]); + return (NULL); + } + + PyErr_Clear(); + val = PyLong_AsLong(obj); + if (val == -1 && PyErr_Occurred()) + return (NULL); + + curl_easy_setopt(&handle->curl, py_curlopt[idx].value, val); + + Py_RETURN_TRUE; +} + +static PyObject * +pycurl_handle_run(struct pycurl_handle *handle, PyObject *args) +{ + struct pycurl_handle_op *op; + + op = PyObject_New(struct pycurl_handle_op, &pycurl_handle_op_type); + if (op == NULL) + return (NULL); + + Py_INCREF(handle); + + op->handle = handle; + op->coro = coro_running; + op->state = CURL_CLIENT_OP_RUN; + + kore_curl_bind_callback(&handle->curl, python_curl_handle_callback, op); + + return ((PyObject *)op); +} + +static void +pycurl_handle_op_dealloc(struct pycurl_handle_op *op) +{ + Py_DECREF(op->handle); + PyObject_Del((PyObject *)op); +} + +static PyObject * +pycurl_handle_op_await(PyObject *op) +{ + Py_INCREF(op); + return (op); +} + +static PyObject * +pycurl_handle_op_iternext(struct pycurl_handle_op *op) +{ + size_t len; + PyObject *result; + const u_int8_t *response; + + if (op->state == CURL_CLIENT_OP_RUN) { + kore_curl_run(&op->handle->curl); + op->state = CURL_CLIENT_OP_RESULT; + Py_RETURN_NONE; + } + + if (!kore_curl_success(&op->handle->curl)) { + /* Do not log the url here, may contain some sensitive data. */ + PyErr_Format(PyExc_RuntimeError, "request failed: %s", + kore_curl_strerror(&op->handle->curl)); + return (NULL); + } + + kore_curl_response_as_bytes(&op->handle->curl, &response, &len); + + if ((result = PyBytes_FromStringAndSize((const char *)response, + len)) == NULL) + return (NULL); + + PyErr_SetObject(PyExc_StopIteration, result); + Py_DECREF(result); + + return (NULL); +} + +static PyObject * python_kore_httpclient(PyObject *self, PyObject *args, PyObject *kwargs) { struct pyhttp_client *client; @@ -5524,13 +5705,13 @@ pyhttp_client_request(struct pyhttp_client *client, int m, PyObject *kwargs) op->headers = 0; op->coro = coro_running; - op->state = PYHTTP_CLIENT_OP_RUN; + op->state = CURL_CLIENT_OP_RUN; Py_INCREF(client); op->client = client; kore_curl_http_setup(&op->curl, m, ptr, length); - kore_curl_bind_callback(&op->curl, python_curl_callback, op); + kore_curl_bind_callback(&op->curl, python_curl_http_callback, op); /* Go in with our own bare hands. */ if (client->unix != NULL) { @@ -5611,9 +5792,9 @@ pyhttp_client_op_iternext(struct pyhttp_client_op *op) const u_int8_t *response; PyObject *result, *tuple, *dict, *value; - if (op->state == PYHTTP_CLIENT_OP_RUN) { + if (op->state == CURL_CLIENT_OP_RUN) { kore_curl_run(&op->curl); - op->state = PYHTTP_CLIENT_OP_RESULT; + op->state = CURL_CLIENT_OP_RESULT; Py_RETURN_NONE; } @@ -5673,7 +5854,7 @@ pyhttp_client_op_iternext(struct pyhttp_client_op *op) } static void -python_curl_callback(struct kore_curl *curl, void *arg) +python_curl_http_callback(struct kore_curl *curl, void *arg) { struct pyhttp_client_op *op = arg; @@ -5682,4 +5863,15 @@ python_curl_callback(struct kore_curl *curl, void *arg) else python_coro_wakeup(op->coro); } + +static void +python_curl_handle_callback(struct kore_curl *curl, void *arg) +{ + struct pycurl_handle_op *op = arg; + + if (op->coro->request != NULL) + http_request_wakeup(op->coro->request); + else + python_coro_wakeup(op->coro); +} #endif