kore

An easy to use, scalable and secure web application framework for writing web APIs in C.
Commits | Files | Refs | README | LICENSE | git clone https://git.kore.io/kore.git

worker.c (22418B)



      1 /*
      2  * Copyright (c) 2013-2022 Joris Vink <joris@coders.se>
      3  *
      4  * Permission to use, copy, modify, and distribute this software for any
      5  * purpose with or without fee is hereby granted, provided that the above
      6  * copyright notice and this permission notice appear in all copies.
      7  *
      8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
      9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     15  */
     16 
     17 #include <sys/param.h>
     18 #include <sys/types.h>
     19 #include <sys/stat.h>
     20 #include <sys/shm.h>
     21 #include <sys/wait.h>
     22 #include <sys/time.h>
     23 #include <sys/resource.h>
     24 #include <sys/socket.h>
     25 
     26 #include <fcntl.h>
     27 #include <grp.h>
     28 #include <pwd.h>
     29 #include <signal.h>
     30 #include <unistd.h>
     31 
     32 #include "kore.h"
     33 
     34 #if defined(KORE_USE_ACME)
     35 #include "acme.h"
     36 #endif
     37 
     38 #if !defined(KORE_NO_HTTP)
     39 #include "http.h"
     40 #endif
     41 
     42 #if defined(KORE_USE_PGSQL)
     43 #include "pgsql.h"
     44 #endif
     45 
     46 #if defined(KORE_USE_TASKS)
     47 #include "tasks.h"
     48 #endif
     49 
     50 #if defined(KORE_USE_PYTHON)
     51 #include "python_api.h"
     52 #endif
     53 
     54 #if defined(KORE_USE_CURL)
     55 #include "curl.h"
     56 #endif
     57 
     58 #if defined(__linux__)
     59 #include "seccomp.h"
     60 #endif
     61 
     62 #define WORKER_SOLO_COUNT	3
     63 
     64 #define WORKER(id)						\
     65 	(struct kore_worker *)((u_int8_t *)kore_workers +	\
     66 	    (sizeof(struct kore_worker) * id))
     67 
     68 struct wlock {
     69 	volatile int		lock;
     70 	pid_t			current;
     71 };
     72 
     73 static int	worker_trylock(void);
     74 static void	worker_unlock(void);
     75 static void	worker_reaper(pid_t, int);
     76 static void	worker_runtime_teardown(void);
     77 static void	worker_runtime_configure(void);
     78 static void	worker_domain_check(struct kore_domain *);
     79 
     80 static struct kore_runtime_call	*worker_runtime_signal(void);
     81 
     82 static inline int	worker_acceptlock_obtain(void);
     83 static inline void	worker_acceptlock_release(void);
     84 static void		worker_accept_avail(struct kore_msg *, const void *);
     85 
     86 static void	worker_entropy_recv(struct kore_msg *, const void *);
     87 static void	worker_keymgr_response(struct kore_msg *, const void *);
     88 
     89 static pid_t				worker_pgrp;
     90 static int				accept_avail;
     91 static struct kore_worker		*kore_workers;
     92 static int				worker_no_lock;
     93 static int				shm_accept_key;
     94 static struct wlock			*accept_lock;
     95 
     96 struct kore_worker		*worker = NULL;
     97 u_int8_t			worker_set_affinity = 1;
     98 u_int32_t			worker_accept_threshold = 16;
     99 u_int32_t			worker_rlimit_nofiles = 768;
    100 u_int32_t			worker_max_connections = 512;
    101 u_int32_t			worker_active_connections = 0;
    102 int				worker_policy = KORE_WORKER_POLICY_RESTART;
    103 
    104 int
    105 kore_worker_init(void)
    106 {
    107 	size_t			len;
    108 	struct kore_worker	*kw;
    109 	u_int16_t		idx, id, cpu;
    110 
    111 	worker_no_lock = 0;
    112 
    113 	if (worker_count == 0)
    114 		worker_count = cpu_count;
    115 
    116 	/* Account for the keymgr/acme even if we don't end up starting it. */
    117 	worker_count += 2;
    118 
    119 	len = sizeof(*accept_lock) +
    120 	    (sizeof(struct kore_worker) * worker_count);
    121 
    122 	shm_accept_key = shmget(IPC_PRIVATE, len, IPC_CREAT | IPC_EXCL | 0700);
    123 	if (shm_accept_key == -1)
    124 		fatal("kore_worker_init(): shmget() %s", errno_s);
    125 	if ((accept_lock = shmat(shm_accept_key, NULL, 0)) == (void *)-1)
    126 		fatal("kore_worker_init(): shmat() %s", errno_s);
    127 
    128 	accept_lock->lock = 0;
    129 	accept_lock->current = 0;
    130 
    131 	kore_workers = (struct kore_worker *)((u_int8_t *)accept_lock +
    132 	    sizeof(*accept_lock));
    133 	memset(kore_workers, 0, sizeof(struct kore_worker) * worker_count);
    134 
    135 	if ((worker_count - 2) > cpu_count)
    136 		kore_log(LOG_NOTICE, "more worker processes than cpu cores");
    137 
    138 	/* Setup log buffers. */
    139 	for (idx = KORE_WORKER_BASE; idx < worker_count; idx++) {
    140 		kw = WORKER(idx);
    141 		kw->lb.offset = 0;
    142 	}
    143 
    144 	if (!kore_quiet)
    145 		kore_log(LOG_INFO, "starting worker processes");
    146 
    147 	if ((worker_pgrp = getpgrp()) == -1)
    148 		fatal("%s: getpgrp(): %s", __func__, errno_s);
    149 
    150 	/* Now start all the workers. */
    151 	id = 1;
    152 	cpu = 1;
    153 	for (idx = KORE_WORKER_BASE; idx < worker_count; idx++) {
    154 		if (cpu >= cpu_count)
    155 			cpu = 0;
    156 		if (!kore_worker_spawn(idx, id++, cpu++))
    157 			return (KORE_RESULT_ERROR);
    158 	}
    159 
    160 	if (kore_keymgr_active) {
    161 #if defined(KORE_USE_ACME)
    162 		/* The ACME process is only started if we need it. */
    163 		if (acme_domains) {
    164 			if (!kore_worker_spawn(KORE_WORKER_ACME_IDX,
    165 			    KORE_WORKER_ACME, 0))
    166 				return (KORE_RESULT_ERROR);
    167 		}
    168 #endif
    169 
    170 		/* Now we can start the keymgr. */
    171 		if (!kore_worker_spawn(KORE_WORKER_KEYMGR_IDX,
    172 		    KORE_WORKER_KEYMGR, 0))
    173 			return (KORE_RESULT_ERROR);
    174 	}
    175 
    176 	if (!kore_quiet)
    177 		kore_log(LOG_INFO, "all worker processes started");
    178 
    179 	return (KORE_RESULT_OK);
    180 }
    181 
    182 int
    183 kore_worker_spawn(u_int16_t idx, u_int16_t id, u_int16_t cpu)
    184 {
    185 	int			cnt;
    186 	struct kore_worker	*kw;
    187 #if defined(__linux__)
    188 	int			status;
    189 #endif
    190 
    191 	kw = WORKER(idx);
    192 	kw->id = id;
    193 	kw->cpu = cpu;
    194 	kw->running = 1;
    195 
    196 	kw->ready = 0;
    197 	kw->has_lock = 0;
    198 	kw->active_route = NULL;
    199 
    200 	if (socketpair(AF_UNIX, SOCK_STREAM, 0, kw->pipe) == -1)
    201 		fatal("socketpair(): %s", errno_s);
    202 
    203 	if (!kore_connection_nonblock(kw->pipe[0], 0) ||
    204 	    !kore_connection_nonblock(kw->pipe[1], 0))
    205 		fatal("could not set pipe fds to nonblocking: %s", errno_s);
    206 
    207 	switch (id) {
    208 	case KORE_WORKER_KEYMGR:
    209 		kw->ps = &keymgr_privsep;
    210 		break;
    211 #if defined(KORE_USE_ACME)
    212 	case KORE_WORKER_ACME:
    213 		kw->ps = &acme_privsep;
    214 		break;
    215 #endif
    216 	default:
    217 		kw->ps = &worker_privsep;
    218 		break;
    219 	}
    220 
    221 	kw->pid = fork();
    222 	if (kw->pid == -1)
    223 		fatal("could not spawn worker child: %s", errno_s);
    224 
    225 	if (kw->pid == 0) {
    226 		kw->pid = getpid();
    227 		kore_worker_entry(kw);
    228 		exit(1);
    229 	} else {
    230 		for (cnt = 0; cnt < 50; cnt++) {
    231 			if (kw->ready == 1)
    232 				break;
    233 			usleep(100000);
    234 #if defined(__linux__)
    235 			/*
    236 			 * If seccomp_tracing is enabled, make sure we
    237 			 * handle the SIGSTOP from the child processes.
    238 			 */
    239 			if (kore_seccomp_tracing) {
    240 				if (waitpid(kw->pid, &status, WNOHANG) > 0)
    241 					kore_seccomp_trace(kw->pid, status);
    242 			}
    243 #endif
    244 		}
    245 
    246 		if (kw->ready == 0) {
    247 			kore_log(LOG_NOTICE,
    248 			    "worker %d failed to start, shutting down",
    249 			    kw->id);
    250 
    251 			return (KORE_RESULT_ERROR);
    252 		}
    253 	}
    254 
    255 	return (KORE_RESULT_OK);
    256 }
    257 
    258 struct kore_worker *
    259 kore_worker_data(u_int8_t idx)
    260 {
    261 	if (idx >= worker_count)
    262 		fatal("idx %u too large for worker count", idx);
    263 
    264 	return (WORKER(idx));
    265 }
    266 
    267 struct kore_worker *
    268 kore_worker_data_byid(u_int16_t id)
    269 {
    270 	struct kore_worker	*kw;
    271 	u_int16_t		idx;
    272 
    273 	for (idx = 0; idx < worker_count; idx++) {
    274 		kw = WORKER(idx);
    275 		if (kw->id == id)
    276 			return (kw);
    277 	}
    278 
    279 	return (NULL);
    280 }
    281 
    282 void
    283 kore_worker_shutdown(void)
    284 {
    285 	struct kore_worker	*kw;
    286 	pid_t			pid;
    287 	int			status;
    288 	u_int16_t		idx, done;
    289 
    290 	if (!kore_quiet) {
    291 		kore_log(LOG_NOTICE,
    292 		    "waiting for workers to drain and shutdown");
    293 	}
    294 
    295 	for (;;) {
    296 		for (idx = 0; idx < worker_count; idx++) {
    297 			kw = WORKER(idx);
    298 			if (kw->running == 0)
    299 				continue;
    300 
    301 			if (kw->pid != 0) {
    302 				pid = waitpid(kw->pid, &status, 0);
    303 				if (pid == -1 && errno != ECHILD)
    304 					continue;
    305 
    306 #if defined(__linux__)
    307 				kore_seccomp_trace(kw->pid, status);
    308 #endif
    309 
    310 				kw->pid = 0;
    311 				kw->running = 0;
    312 
    313 				kw->msg[0]->evt.flags |= KORE_EVENT_READ;
    314 				net_recv_flush(kw->msg[0]);
    315 
    316 				if (!kore_quiet) {
    317 					kore_log(LOG_NOTICE,
    318 					    "worker %s exited (%d)",
    319 					    kore_worker_name(kw->id), status);
    320 				}
    321 			}
    322 		}
    323 
    324 		done = 0;
    325 		for (idx = 0; idx < worker_count; idx++) {
    326 			kw = WORKER(idx);
    327 			if (kw->running == 0) {
    328 				done++;
    329 				continue;
    330 			}
    331 		}
    332 
    333 		if (done == worker_count)
    334 			break;
    335 	}
    336 
    337 	if (shmctl(shm_accept_key, IPC_RMID, NULL) == -1) {
    338 		kore_log(LOG_NOTICE,
    339 		    "failed to deleted shm segment: %s", errno_s);
    340 	}
    341 }
    342 
    343 void
    344 kore_worker_dispatch_signal(int sig)
    345 {
    346 	u_int16_t		idx;
    347 	struct kore_worker	*kw;
    348 
    349 	for (idx = 0; idx < worker_count; idx++) {
    350 		kw = WORKER(idx);
    351 
    352 		if (kw->pid == -1 || kw->pid == 0)
    353 			continue;
    354 
    355 		if (kill(kw->pid, sig) == -1) {
    356 			kore_log(LOG_WARNING, "kill(%d, %d): %s",
    357 			    kw->pid, sig, errno_s);
    358 		}
    359 	}
    360 }
    361 
    362 void
    363 kore_worker_privsep(void)
    364 {
    365 	rlim_t			fd;
    366 	struct rlimit		rl;
    367 	struct passwd		*pw;
    368 
    369 	if (worker == NULL)
    370 		fatalx("%s called with no worker", __func__);
    371 
    372 	pw = NULL;
    373 
    374 	/* Must happen before chroot. */
    375 	if (worker->ps->skip_runas == 0) {
    376 		if (worker->ps->runas == NULL) {
    377 			fatalx("no runas user given for %s",
    378 			    kore_worker_name(worker->id));
    379 		}
    380 
    381 		if ((pw = getpwnam(worker->ps->runas)) == NULL) {
    382 			fatalx("cannot getpwnam(\"%s\") for user: %s",
    383 			    worker->ps->runas, errno_s);
    384 		}
    385 	}
    386 
    387 	if (worker->ps->skip_chroot == 0) {
    388 		if (chroot(worker->ps->root) == -1) {
    389 			fatalx("cannot chroot(\"%s\"): %s",
    390 			    worker->ps->root, errno_s);
    391 		}
    392 
    393 		if (chdir("/") == -1)
    394 			fatalx("cannot chdir(\"/\"): %s", errno_s);
    395 	} else {
    396 		if (chdir(worker->ps->root) == -1) {
    397 			fatalx("cannot chdir(\"%s\"): %s",
    398 			    worker->ps->root, errno_s);
    399 		}
    400 	}
    401 
    402 	if (getrlimit(RLIMIT_NOFILE, &rl) == -1) {
    403 		kore_log(LOG_WARNING, "getrlimit(RLIMIT_NOFILE): %s", errno_s);
    404 	} else {
    405 		for (fd = 0; fd < rl.rlim_cur; fd++) {
    406 			if (fcntl(fd, F_GETFD, NULL) != -1) {
    407 				worker_rlimit_nofiles++;
    408 			}
    409 		}
    410 	}
    411 
    412 	rl.rlim_cur = worker_rlimit_nofiles;
    413 	rl.rlim_max = worker_rlimit_nofiles;
    414 	if (setrlimit(RLIMIT_NOFILE, &rl) == -1) {
    415 		kore_log(LOG_ERR, "setrlimit(RLIMIT_NOFILE, %u): %s",
    416 		    worker_rlimit_nofiles, errno_s);
    417 	}
    418 
    419 	if (worker->ps->skip_runas == 0) {
    420 		if (setgroups(1, &pw->pw_gid) ||
    421 #if defined(__MACH__) || defined(NetBSD)
    422 		    setgid(pw->pw_gid) || setegid(pw->pw_gid) ||
    423 		    setuid(pw->pw_uid) || seteuid(pw->pw_uid))
    424 #else
    425 		    setresgid(pw->pw_gid, pw->pw_gid, pw->pw_gid) ||
    426 		    setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid))
    427 #endif
    428 			fatalx("cannot drop privileges (%s)", errno_s);
    429 	}
    430 
    431 	kore_platform_sandbox();
    432 }
    433 
    434 void
    435 kore_worker_entry(struct kore_worker *kw)
    436 {
    437 	struct kore_runtime_call	*sigcall;
    438 	u_int64_t			last_seed;
    439 	int				quit, had_lock, sig;
    440 	u_int64_t			netwait, now, next_timeo;
    441 
    442 	worker = kw;
    443 
    444 	if (!kore_foreground)
    445 		closelog();
    446 
    447 #if defined(__linux__)
    448 	kore_seccomp_traceme();
    449 #endif
    450 
    451 	kore_platform_proctitle(kore_worker_name(kw->id));
    452 
    453 	if (worker_set_affinity == 1)
    454 		kore_platform_worker_setcpu(kw);
    455 
    456 	kore_pid = kw->pid;
    457 	kore_signal_setup();
    458 
    459 	if (kw->id == KORE_WORKER_KEYMGR) {
    460 		kore_keymgr_run();
    461 		exit(0);
    462 	}
    463 
    464 #if defined(KORE_USE_ACME)
    465 	if (kw->id == KORE_WORKER_ACME) {
    466 		kore_acme_run();
    467 		exit(0);
    468 	}
    469 #endif
    470 
    471 	net_init();
    472 	kore_connection_init();
    473 	kore_platform_event_init();
    474 	kore_msg_worker_init();
    475 
    476 #if defined(KORE_USE_TASKS)
    477 	kore_task_init();
    478 #endif
    479 
    480 	kore_worker_privsep();
    481 
    482 #if !defined(KORE_NO_HTTP)
    483 	http_init();
    484 	kore_filemap_resolve_paths();
    485 	kore_accesslog_worker_init();
    486 #endif
    487 	kore_timer_init();
    488 	kore_fileref_init();
    489 	kore_tls_keymgr_init();
    490 
    491 	quit = 0;
    492 	had_lock = 0;
    493 	next_timeo = 0;
    494 	accept_avail = 1;
    495 	worker_active_connections = 0;
    496 
    497 	last_seed = 0;
    498 
    499 	if (kore_keymgr_active) {
    500 		kore_msg_register(KORE_MSG_CRL, worker_keymgr_response);
    501 		kore_msg_register(KORE_MSG_ENTROPY_RESP, worker_entropy_recv);
    502 		kore_msg_register(KORE_MSG_CERTIFICATE, worker_keymgr_response);
    503 
    504 		if (worker->restarted) {
    505 			kore_msg_send(KORE_WORKER_KEYMGR,
    506 			    KORE_MSG_CERTIFICATE_REQ, NULL, 0);
    507 		}
    508 #if defined(KORE_USE_ACME)
    509 		kore_msg_register(KORE_ACME_CHALLENGE_SET_CERT,
    510 		    worker_keymgr_response);
    511 		kore_msg_register(KORE_ACME_CHALLENGE_CLEAR_CERT,
    512 		    worker_keymgr_response);
    513 #endif
    514 	}
    515 
    516 	kore_msg_register(KORE_MSG_ACCEPT_AVAILABLE, worker_accept_avail);
    517 
    518 	if (nlisteners == 0)
    519 		worker_no_lock = 1;
    520 
    521 	worker_runtime_configure();
    522 
    523 	kore_module_onload();
    524 	kore_domain_callback(worker_domain_check);
    525 
    526 	kore_worker_started();
    527 	worker->restarted = 0;
    528 
    529 	sigcall = worker_runtime_signal();
    530 
    531 	for (;;) {
    532 		now = kore_time_ms();
    533 
    534 		if (kore_keymgr_active &&
    535 		    (now - last_seed) > KORE_RESEED_TIME) {
    536 			kore_msg_send(KORE_WORKER_KEYMGR,
    537 			    KORE_MSG_ENTROPY_REQ, NULL, 0);
    538 			last_seed = now;
    539 		}
    540 
    541 		if (!worker->has_lock && accept_avail) {
    542 			if (worker_acceptlock_obtain()) {
    543 				accept_avail = 0;
    544 				if (had_lock == 0) {
    545 					kore_platform_enable_accept();
    546 					had_lock = 1;
    547 				}
    548 			}
    549 		}
    550 
    551 		netwait = kore_timer_next_run(now);
    552 
    553 		if (netwait == KORE_WAIT_INFINITE) {
    554 			if (sig_recv != 0)
    555 				netwait = 10;
    556 #if !defined(KORE_NO_HTTP)
    557 			if (http_request_count > 0)
    558 				netwait = 100;
    559 #endif
    560 		}
    561 
    562 #if defined(KORE_USE_PYTHON)
    563 		if (kore_python_coro_pending())
    564 			netwait = 0;
    565 #endif
    566 
    567 		kore_platform_event_wait(netwait);
    568 		now = kore_time_ms();
    569 
    570 		if (worker->has_lock)
    571 			worker_acceptlock_release();
    572 
    573 		if (!worker->has_lock) {
    574 			if (had_lock == 1) {
    575 				had_lock = 0;
    576 				kore_platform_disable_accept();
    577 			}
    578 		}
    579 
    580 		sig = sig_recv;
    581 		if (sig != 0) {
    582 			switch (sig) {
    583 			case SIGHUP:
    584 				kore_module_reload(1);
    585 				break;
    586 			case SIGQUIT:
    587 			case SIGINT:
    588 			case SIGTERM:
    589 				quit = 1;
    590 				break;
    591 			case SIGCHLD:
    592 #if defined(KORE_USE_PYTHON)
    593 				kore_python_proc_reap();
    594 #endif
    595 				break;
    596 			default:
    597 				break;
    598 			}
    599 
    600 			if (sigcall != NULL)
    601 				kore_runtime_signal(sigcall, sig);
    602 
    603 			if (sig == sig_recv)
    604 				sig_recv = 0;
    605 		}
    606 
    607 		if (quit)
    608 			break;
    609 
    610 		kore_timer_run(now);
    611 #if defined(KORE_USE_CURL)
    612 		kore_curl_run_scheduled();
    613 		kore_curl_do_timeout();
    614 #endif
    615 #if !defined(KORE_NO_HTTP)
    616 		http_process();
    617 #endif
    618 #if defined(KORE_USE_PYTHON)
    619 		kore_python_coro_run();
    620 #endif
    621 		if (next_timeo <= now) {
    622 			kore_connection_check_timeout(now);
    623 			next_timeo = now + 500;
    624 		}
    625 
    626 		kore_connection_prune(KORE_CONNECTION_PRUNE_DISCONNECT);
    627 	}
    628 
    629 	worker_runtime_teardown();
    630 	kore_server_cleanup();
    631 
    632 	kore_platform_event_cleanup();
    633 	kore_connection_cleanup();
    634 	kore_domain_cleanup();
    635 	kore_tls_cleanup();
    636 	kore_module_cleanup();
    637 #if !defined(KORE_NO_HTTP)
    638 	http_cleanup();
    639 #endif
    640 	net_cleanup();
    641 
    642 #if defined(KORE_USE_PYTHON)
    643 	kore_python_cleanup();
    644 #endif
    645 
    646 #if defined(KORE_USE_PGSQL)
    647 	kore_pgsql_sys_cleanup();
    648 #endif
    649 
    650 	kore_mem_cleanup();
    651 	exit(0);
    652 }
    653 
    654 void
    655 kore_worker_reap(void)
    656 {
    657 	pid_t			pid;
    658 	int			status;
    659 
    660 	for (;;) {
    661 		pid = waitpid(-worker_pgrp, &status, WNOHANG);
    662 
    663 		if (pid == -1) {
    664 			if (errno != ECHILD && errno != EINTR) {
    665 				kore_log(LOG_ERR,
    666 				    "%s: waitpid(): %s", __func__, errno_s);
    667 			}
    668 			break;
    669 		}
    670 
    671 		if (pid == 0)
    672 			break;
    673 
    674 		worker_reaper(pid, status);
    675 	}
    676 }
    677 
    678 void
    679 kore_worker_make_busy(void)
    680 {
    681 	if (worker_count == WORKER_SOLO_COUNT || worker_no_lock == 1)
    682 		return;
    683 
    684 	if (worker->has_lock) {
    685 		worker_unlock();
    686 		worker->has_lock = 0;
    687 		kore_msg_send(KORE_MSG_WORKER_ALL,
    688 		    KORE_MSG_ACCEPT_AVAILABLE, NULL, 0);
    689 	}
    690 }
    691 
    692 int
    693 kore_worker_keymgr_response_verify(struct kore_msg *msg, const void *data,
    694     struct kore_domain **out)
    695 {
    696 	struct kore_server		*srv;
    697 	struct kore_domain		*dom;
    698 	const struct kore_x509_msg	*req;
    699 
    700 	if (msg->length < sizeof(*req)) {
    701 		kore_log(LOG_WARNING,
    702 		    "short keymgr message (%zu)", msg->length);
    703 		return (KORE_RESULT_ERROR);
    704 	}
    705 
    706 	req = (const struct kore_x509_msg *)data;
    707 	if (msg->length != (sizeof(*req) + req->data_len)) {
    708 		kore_log(LOG_WARNING,
    709 		    "invalid keymgr payload (%zu)", msg->length);
    710 		return (KORE_RESULT_ERROR);
    711 	}
    712 
    713 	if (req->domain[KORE_DOMAINNAME_LEN] != '\0') {
    714 		kore_log(LOG_WARNING, "domain not NUL-terminated");
    715 		return (KORE_RESULT_ERROR);
    716 
    717 	}
    718 
    719 	if (out == NULL)
    720 		return (KORE_RESULT_OK);
    721 
    722 	dom = NULL;
    723 
    724 	LIST_FOREACH(srv, &kore_servers, list) {
    725 		dom = NULL;
    726 
    727 		if (srv->tls == 0)
    728 			continue;
    729 
    730 		TAILQ_FOREACH(dom, &srv->domains, list) {
    731 			if (!strcmp(dom->domain, req->domain))
    732 				break;
    733 		}
    734 
    735 		if (dom != NULL)
    736 			break;
    737 	}
    738 
    739 	if (dom == NULL) {
    740 		kore_log(LOG_WARNING,
    741 		    "got keymgr response for domain that does not exist");
    742 		return (KORE_RESULT_ERROR);
    743 	}
    744 
    745 	*out = dom;
    746 
    747 	return (KORE_RESULT_OK);
    748 }
    749 
    750 void
    751 kore_worker_started(void)
    752 {
    753 	const char	*chroot;
    754 
    755 	if (worker->ps->skip_chroot)
    756 		chroot = "root";
    757 	else
    758 		chroot = "chroot";
    759 
    760 	if (!kore_quiet) {
    761 		kore_log(LOG_NOTICE,
    762 		    "started (#%d %s=%s%s%s)",
    763 		    getpid(), chroot, worker->ps->root,
    764 		    worker->ps->skip_runas ? "" : " user=",
    765 		    worker->ps->skip_runas ? "" : worker->ps->runas);
    766 	}
    767 
    768 	worker->ready = 1;
    769 }
    770 
    771 static void
    772 worker_runtime_configure(void)
    773 {
    774 	struct kore_runtime_call	*rcall;
    775 
    776 	rcall = NULL;
    777 
    778 #if defined(KORE_USE_PYTHON)
    779 	rcall = kore_runtime_getcall(KORE_PYTHON_WORKER_START_HOOK);
    780 #endif
    781 
    782 	if (rcall == NULL)
    783 		rcall = kore_runtime_getcall("kore_worker_configure");
    784 
    785 	if (rcall != NULL) {
    786 		kore_runtime_execute(rcall);
    787 		kore_free(rcall);
    788 	}
    789 }
    790 
    791 static struct kore_runtime_call *
    792 worker_runtime_signal(void)
    793 {
    794 	struct kore_runtime_call	*rcall;
    795 
    796 	rcall = NULL;
    797 
    798 #if defined(KORE_USE_PYTHON)
    799 	rcall = kore_runtime_getcall(KORE_PYTHON_SIGNAL_HOOK);
    800 #endif
    801 
    802 	if (rcall == NULL)
    803 		rcall = kore_runtime_getcall("kore_worker_signal");
    804 
    805 	return (rcall);
    806 }
    807 
    808 static void
    809 worker_runtime_teardown(void)
    810 {
    811 	struct kore_runtime_call	*rcall;
    812 
    813 	rcall = NULL;
    814 
    815 #if defined(KORE_USE_PYTHON)
    816 	rcall = kore_runtime_getcall(KORE_PYTHON_WORKER_STOP_HOOK);
    817 #endif
    818 
    819 	if (rcall == NULL)
    820 		rcall = kore_runtime_getcall("kore_worker_teardown");
    821 
    822 	if (rcall != NULL) {
    823 		kore_runtime_execute(rcall);
    824 		kore_free(rcall);
    825 	}
    826 }
    827 
    828 static void
    829 worker_domain_check(struct kore_domain *dom)
    830 {
    831 	struct stat	st;
    832 
    833 	if (dom->cafile != NULL) {
    834 		if (stat(dom->cafile, &st) == -1)
    835 			fatalx("'%s': %s", dom->cafile, errno_s);
    836 		if (access(dom->cafile, R_OK) == -1)
    837 			fatalx("'%s': not readable", dom->cafile);
    838 	}
    839 }
    840 
    841 static void
    842 worker_reaper(pid_t pid, int status)
    843 {
    844 	u_int16_t		idx;
    845 	struct kore_worker	*kw;
    846 	const char		*func;
    847 
    848 #if defined(__linux__)
    849 	if (kore_seccomp_trace(pid, status))
    850 		return;
    851 #endif
    852 
    853 	for (idx = 0; idx < worker_count; idx++) {
    854 		kw = WORKER(idx);
    855 		if (kw->pid != pid)
    856 			continue;
    857 
    858 		kw->msg[0]->evt.flags |= KORE_EVENT_READ;
    859 		net_recv_flush(kw->msg[0]);
    860 
    861 		if (!kore_quiet) {
    862 			kore_log(LOG_NOTICE,
    863 			    "worker %s (%d) exited with status %d",
    864 			    kore_worker_name(kw->id), pid, status);
    865 		}
    866 
    867 		kw->running = 0;
    868 
    869 		if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
    870 			kw->pid = 0;
    871 			break;
    872 		}
    873 
    874 		func = "none";
    875 #if !defined(KORE_NO_HTTP)
    876 		if (kw->active_route != NULL)
    877 			func = kw->active_route->func;
    878 #endif
    879 		kore_log(LOG_NOTICE,
    880 		    "worker %d (pid: %d) (hdlr: %s) gone",
    881 		    kw->id, kw->pid, func);
    882 
    883 #if defined(__linux__)
    884 		if (WIFSIGNALED(status) && WTERMSIG(status) == SIGSYS) {
    885 			kore_log(LOG_NOTICE,
    886 			    "worker %d died from sandbox violation", kw->id);
    887 		}
    888 #endif
    889 
    890 		if (kw->id == KORE_WORKER_KEYMGR ||
    891 		    kw->id == KORE_WORKER_ACME) {
    892 			kore_log(LOG_CRIT,
    893 			    "keymgr or acme process gone, stopping");
    894 			kw->pid = 0;
    895 			kore_quit = KORE_QUIT_FATAL;
    896 			break;
    897 		}
    898 
    899 		if (kw->pid == accept_lock->current &&
    900 		    worker_no_lock == 0)
    901 			worker_unlock();
    902 
    903 #if !defined(KORE_NO_HTTP)
    904 		if (kw->active_route != NULL) {
    905 			kw->active_route->errors++;
    906 			kore_log(LOG_NOTICE,
    907 			    "hdlr %s has caused %d error(s)",
    908 			    kw->active_route->func,
    909 			    kw->active_route->errors);
    910 		}
    911 #endif
    912 
    913 		if (worker_policy == KORE_WORKER_POLICY_TERMINATE) {
    914 			kw->pid = 0;
    915 			kore_log(LOG_NOTICE,
    916 			    "worker policy is 'terminate', stopping");
    917 			kore_quit = KORE_QUIT_FATAL;
    918 			break;
    919 		}
    920 
    921 		if (kore_quit == KORE_QUIT_NONE) {
    922 			kore_log(LOG_NOTICE, "restarting worker %d", kw->id);
    923 			kw->restarted = 1;
    924 			kore_msg_parent_remove(kw);
    925 
    926 			if (!kore_worker_spawn(idx, kw->id, kw->cpu)) {
    927 				kore_quit = KORE_QUIT_FATAL;
    928 				kore_log(LOG_ERR, "failed to restart worker");
    929 			} else {
    930 				kore_msg_parent_add(kw);
    931 			}
    932 
    933 			break;
    934 		}
    935 	}
    936 }
    937 
    938 static inline void
    939 worker_acceptlock_release(void)
    940 {
    941 	if (worker_count == WORKER_SOLO_COUNT || worker_no_lock == 1)
    942 		return;
    943 
    944 	if (worker->has_lock != 1)
    945 		return;
    946 
    947 	if (worker_active_connections < worker_max_connections) {
    948 #if !defined(KORE_NO_HTTP)
    949 		if (http_request_count < http_request_limit)
    950 			return;
    951 #else
    952 		return;
    953 #endif
    954 	}
    955 
    956 #if defined(WORKER_DEBUG)
    957 	kore_log(LOG_DEBUG, "worker busy, releasing lock");
    958 #endif
    959 
    960 	worker_unlock();
    961 	worker->has_lock = 0;
    962 
    963 	kore_msg_send(KORE_MSG_WORKER_ALL, KORE_MSG_ACCEPT_AVAILABLE, NULL, 0);
    964 }
    965 
    966 static inline int
    967 worker_acceptlock_obtain(void)
    968 {
    969 	int		r;
    970 
    971 	if (worker->has_lock == 1)
    972 		return (1);
    973 
    974 	if (worker_count == WORKER_SOLO_COUNT || worker_no_lock == 1) {
    975 		worker->has_lock = 1;
    976 		return (1);
    977 	}
    978 
    979 	if (worker_active_connections >= worker_max_connections)
    980 		return (0);
    981 
    982 #if !defined(KORE_NO_HTTP)
    983 	if (http_request_count >= http_request_limit)
    984 		return (0);
    985 #endif
    986 
    987 	r = 0;
    988 	if (worker_trylock()) {
    989 		r = 1;
    990 		worker->has_lock = 1;
    991 #if defined(WORKER_DEBUG)
    992 		kore_log(LOG_DEBUG, "got lock");
    993 #endif
    994 	}
    995 
    996 	return (r);
    997 }
    998 
    999 static int
   1000 worker_trylock(void)
   1001 {
   1002 	if (!__sync_bool_compare_and_swap(&(accept_lock->lock), 0, 1))
   1003 		return (0);
   1004 
   1005 	accept_lock->current = worker->pid;
   1006 
   1007 	return (1);
   1008 }
   1009 
   1010 static void
   1011 worker_unlock(void)
   1012 {
   1013 	accept_lock->current = 0;
   1014 	if (!__sync_bool_compare_and_swap(&(accept_lock->lock), 1, 0))
   1015 		kore_log(LOG_NOTICE, "worker_unlock(): wasn't locked");
   1016 }
   1017 
   1018 static void
   1019 worker_accept_avail(struct kore_msg *msg, const void *data)
   1020 {
   1021 	accept_avail = 1;
   1022 }
   1023 
   1024 static void
   1025 worker_entropy_recv(struct kore_msg *msg, const void *data)
   1026 {
   1027 	if (msg->length != 1024) {
   1028 		kore_log(LOG_WARNING,
   1029 		    "invalid entropy response (got:%zu - wanted:1024)",
   1030 		    msg->length);
   1031 	}
   1032 
   1033 	kore_tls_seed(data, msg->length);
   1034 }
   1035 
   1036 static void
   1037 worker_keymgr_response(struct kore_msg *msg, const void *data)
   1038 {
   1039 	struct kore_domain		*dom;
   1040 	const struct kore_x509_msg	*req;
   1041 
   1042 	if (!kore_worker_keymgr_response_verify(msg, data, &dom))
   1043 		return;
   1044 
   1045 	req = (const struct kore_x509_msg *)data;
   1046 
   1047 	switch (msg->id) {
   1048 	case KORE_MSG_CERTIFICATE:
   1049 		kore_tls_domain_setup(dom, KORE_PEM_CERT_CHAIN,
   1050 		    req->data, req->data_len);
   1051 		break;
   1052 	case KORE_MSG_CRL:
   1053 		kore_tls_domain_crl(dom, req->data, req->data_len);
   1054 		break;
   1055 #if defined(KORE_USE_ACME)
   1056 	case KORE_ACME_CHALLENGE_SET_CERT:
   1057 		if (dom->tls_ctx == NULL) {
   1058 			kore_tls_domain_setup(dom, KORE_DER_CERT_DATA,
   1059 			    req->data, req->data_len);
   1060 		}
   1061 
   1062 		kore_free(dom->acme_cert);
   1063 		dom->acme_cert_len = req->data_len;
   1064 		dom->acme_cert = kore_calloc(1, req->data_len);
   1065 		memcpy(dom->acme_cert, req->data, req->data_len);
   1066 
   1067 		kore_log(LOG_NOTICE, "[%s] tls-alpn-01 challenge active",
   1068 		    dom->domain);
   1069 		dom->acme_challenge = 1;
   1070 		break;
   1071 	case KORE_ACME_CHALLENGE_CLEAR_CERT:
   1072 		dom->acme_cert_len = 0;
   1073 		dom->acme_challenge = 0;
   1074 
   1075 		kore_free(dom->acme_cert);
   1076 		dom->acme_cert = NULL;
   1077 
   1078 		kore_log(LOG_NOTICE, "[%s] tls-alpn-01 challenge disabled",
   1079 		    dom->domain);
   1080 		break;
   1081 #endif
   1082 	default:
   1083 		kore_log(LOG_WARNING, "unknown keymgr request %u", msg->id);
   1084 		break;
   1085 	}
   1086 }