kore

Kore is a web application platform for writing scalable, concurrent web based processes in C or Python.
Commits | Files | Refs | README | LICENSE | git clone https://git.kore.io/kore.git

worker.c (22499B)



      1 /*
      2  * Copyright (c) 2013-2022 Joris Vink <joris@coders.se>
      3  *
      4  * Permission to use, copy, modify, and distribute this software for any
      5  * purpose with or without fee is hereby granted, provided that the above
      6  * copyright notice and this permission notice appear in all copies.
      7  *
      8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
      9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     15  */
     16 
     17 #include <sys/param.h>
     18 #include <sys/types.h>
     19 #include <sys/stat.h>
     20 #include <sys/shm.h>
     21 #include <sys/wait.h>
     22 #include <sys/time.h>
     23 #include <sys/resource.h>
     24 #include <sys/socket.h>
     25 
     26 #include <fcntl.h>
     27 #include <grp.h>
     28 #include <pwd.h>
     29 #include <signal.h>
     30 #include <unistd.h>
     31 
     32 #include "kore.h"
     33 
     34 #if defined(KORE_USE_ACME)
     35 #include "acme.h"
     36 #endif
     37 
     38 #if !defined(KORE_NO_HTTP)
     39 #include "http.h"
     40 #endif
     41 
     42 #if defined(KORE_USE_PGSQL)
     43 #include "pgsql.h"
     44 #endif
     45 
     46 #if defined(KORE_USE_TASKS)
     47 #include "tasks.h"
     48 #endif
     49 
     50 #if defined(KORE_USE_PYTHON)
     51 #include "python_api.h"
     52 #endif
     53 
     54 #if defined(KORE_USE_CURL)
     55 #include "curl.h"
     56 #endif
     57 
     58 #if defined(__linux__)
     59 #include "seccomp.h"
     60 #endif
     61 
     62 #define WORKER_SOLO_COUNT	3
     63 
     64 #define WORKER(id)						\
     65 	(struct kore_worker *)((u_int8_t *)kore_workers +	\
     66 	    (sizeof(struct kore_worker) * id))
     67 
     68 struct wlock {
     69 	volatile int		lock;
     70 	pid_t			current;
     71 };
     72 
     73 static int	worker_trylock(void);
     74 static void	worker_unlock(void);
     75 static void	worker_reaper(pid_t, int);
     76 static void	worker_runtime_teardown(void);
     77 static void	worker_runtime_configure(void);
     78 static void	worker_domain_check(struct kore_domain *);
     79 
     80 static struct kore_runtime_call	*worker_runtime_signal(void);
     81 
     82 static inline int	worker_acceptlock_obtain(void);
     83 static inline void	worker_acceptlock_release(void);
     84 static void		worker_accept_avail(struct kore_msg *, const void *);
     85 
     86 static void	worker_entropy_recv(struct kore_msg *, const void *);
     87 static void	worker_keymgr_response(struct kore_msg *, const void *);
     88 
     89 static pid_t				worker_pgrp;
     90 static int				accept_avail;
     91 static struct kore_worker		*kore_workers;
     92 static int				worker_no_lock;
     93 static int				shm_accept_key;
     94 static struct wlock			*accept_lock;
     95 
     96 struct kore_worker		*worker = NULL;
     97 u_int8_t			worker_set_affinity = 1;
     98 u_int32_t			worker_accept_threshold = 16;
     99 u_int32_t			worker_rlimit_nofiles = 768;
    100 u_int32_t			worker_max_connections = 512;
    101 u_int32_t			worker_active_connections = 0;
    102 int				worker_policy = KORE_WORKER_POLICY_RESTART;
    103 
    104 int
    105 kore_worker_init(void)
    106 {
    107 	size_t			len;
    108 	struct kore_worker	*kw;
    109 	u_int16_t		idx, id, cpu;
    110 
    111 	worker_no_lock = 0;
    112 
    113 	if (worker_count == 0)
    114 		worker_count = cpu_count;
    115 
    116 	/* Account for the keymgr/acme even if we don't end up starting it. */
    117 	worker_count += 2;
    118 
    119 	len = sizeof(*accept_lock) +
    120 	    (sizeof(struct kore_worker) * worker_count);
    121 
    122 	shm_accept_key = shmget(IPC_PRIVATE, len, IPC_CREAT | IPC_EXCL | 0700);
    123 	if (shm_accept_key == -1)
    124 		fatal("kore_worker_init(): shmget() %s", errno_s);
    125 	if ((accept_lock = shmat(shm_accept_key, NULL, 0)) == (void *)-1)
    126 		fatal("kore_worker_init(): shmat() %s", errno_s);
    127 
    128 	accept_lock->lock = 0;
    129 	accept_lock->current = 0;
    130 
    131 	kore_workers = (struct kore_worker *)((u_int8_t *)accept_lock +
    132 	    sizeof(*accept_lock));
    133 	memset(kore_workers, 0, sizeof(struct kore_worker) * worker_count);
    134 
    135 	if ((worker_count - 2) > cpu_count)
    136 		kore_log(LOG_NOTICE, "more worker processes than cpu cores");
    137 
    138 	/* Setup log buffers. */
    139 	for (idx = KORE_WORKER_BASE; idx < worker_count; idx++) {
    140 		kw = WORKER(idx);
    141 		kw->lb.offset = 0;
    142 	}
    143 
    144 	if (!kore_quiet)
    145 		kore_log(LOG_INFO, "starting worker processes");
    146 
    147 	if ((worker_pgrp = getpgrp()) == -1)
    148 		fatal("%s: getpgrp(): %s", __func__, errno_s);
    149 
    150 	/* Now start all the workers. */
    151 	id = 1;
    152 	cpu = 1;
    153 	for (idx = KORE_WORKER_BASE; idx < worker_count; idx++) {
    154 		if (cpu >= cpu_count)
    155 			cpu = 0;
    156 		if (!kore_worker_spawn(idx, id++, cpu++))
    157 			return (KORE_RESULT_ERROR);
    158 	}
    159 
    160 	if (kore_keymgr_active) {
    161 #if defined(KORE_USE_ACME)
    162 		/* The ACME process is only started if we need it. */
    163 		if (acme_domains) {
    164 			if (!kore_worker_spawn(KORE_WORKER_ACME_IDX,
    165 			    KORE_WORKER_ACME, 0))
    166 				return (KORE_RESULT_ERROR);
    167 		}
    168 #endif
    169 
    170 		/* Now we can start the keymgr. */
    171 		if (!kore_worker_spawn(KORE_WORKER_KEYMGR_IDX,
    172 		    KORE_WORKER_KEYMGR, 0))
    173 			return (KORE_RESULT_ERROR);
    174 	}
    175 
    176 	if (!kore_quiet)
    177 		kore_log(LOG_INFO, "all worker processes started");
    178 
    179 	return (KORE_RESULT_OK);
    180 }
    181 
    182 int
    183 kore_worker_spawn(u_int16_t idx, u_int16_t id, u_int16_t cpu)
    184 {
    185 	struct kore_worker	*kw;
    186 	int			cnt, maxcnt;
    187 #if defined(__linux__)
    188 	int			status;
    189 #endif
    190 
    191 	kw = WORKER(idx);
    192 	kw->id = id;
    193 	kw->cpu = cpu;
    194 	kw->running = 1;
    195 
    196 	kw->ready = 0;
    197 	kw->has_lock = 0;
    198 	kw->active_route = NULL;
    199 
    200 	if (socketpair(AF_UNIX, SOCK_STREAM, 0, kw->pipe) == -1)
    201 		fatal("socketpair(): %s", errno_s);
    202 
    203 	if (!kore_connection_nonblock(kw->pipe[0], 0) ||
    204 	    !kore_connection_nonblock(kw->pipe[1], 0))
    205 		fatal("could not set pipe fds to nonblocking: %s", errno_s);
    206 
    207 	maxcnt = 50;
    208 
    209 	switch (id) {
    210 	case KORE_WORKER_KEYMGR:
    211 #if defined(KORE_USE_ACME)
    212 		maxcnt = 600;
    213 #endif
    214 		kw->ps = &keymgr_privsep;
    215 		break;
    216 #if defined(KORE_USE_ACME)
    217 	case KORE_WORKER_ACME:
    218 		kw->ps = &acme_privsep;
    219 		break;
    220 #endif
    221 	default:
    222 		kw->ps = &worker_privsep;
    223 		break;
    224 	}
    225 
    226 	kw->pid = fork();
    227 	if (kw->pid == -1)
    228 		fatal("could not spawn worker child: %s", errno_s);
    229 
    230 	if (kw->pid == 0) {
    231 		kw->pid = getpid();
    232 		kore_worker_entry(kw);
    233 		exit(1);
    234 	} else {
    235 		for (cnt = 0; cnt < maxcnt; cnt++) {
    236 			if (kw->ready == 1)
    237 				break;
    238 			usleep(100000);
    239 #if defined(__linux__)
    240 			/*
    241 			 * If seccomp_tracing is enabled, make sure we
    242 			 * handle the SIGSTOP from the child processes.
    243 			 */
    244 			if (kore_seccomp_tracing) {
    245 				if (waitpid(kw->pid, &status, WNOHANG) > 0)
    246 					kore_seccomp_trace(kw->pid, status);
    247 			}
    248 #endif
    249 		}
    250 
    251 		if (kw->ready == 0) {
    252 			kore_log(LOG_NOTICE,
    253 			    "worker %d failed to start, shutting down",
    254 			    kw->id);
    255 
    256 			return (KORE_RESULT_ERROR);
    257 		}
    258 	}
    259 
    260 	return (KORE_RESULT_OK);
    261 }
    262 
    263 struct kore_worker *
    264 kore_worker_data(u_int8_t idx)
    265 {
    266 	if (idx >= worker_count)
    267 		fatal("idx %u too large for worker count", idx);
    268 
    269 	return (WORKER(idx));
    270 }
    271 
    272 struct kore_worker *
    273 kore_worker_data_byid(u_int16_t id)
    274 {
    275 	struct kore_worker	*kw;
    276 	u_int16_t		idx;
    277 
    278 	for (idx = 0; idx < worker_count; idx++) {
    279 		kw = WORKER(idx);
    280 		if (kw->id == id)
    281 			return (kw);
    282 	}
    283 
    284 	return (NULL);
    285 }
    286 
    287 void
    288 kore_worker_shutdown(void)
    289 {
    290 	struct kore_worker	*kw;
    291 	pid_t			pid;
    292 	int			status;
    293 	u_int16_t		idx, done;
    294 
    295 	if (!kore_quiet) {
    296 		kore_log(LOG_NOTICE,
    297 		    "waiting for workers to drain and shutdown");
    298 	}
    299 
    300 	for (;;) {
    301 		for (idx = 0; idx < worker_count; idx++) {
    302 			kw = WORKER(idx);
    303 			if (kw->running == 0)
    304 				continue;
    305 
    306 			if (kw->pid != 0) {
    307 				pid = waitpid(kw->pid, &status, 0);
    308 				if (pid == -1 && errno != ECHILD)
    309 					continue;
    310 
    311 #if defined(__linux__)
    312 				kore_seccomp_trace(kw->pid, status);
    313 #endif
    314 
    315 				kw->pid = 0;
    316 				kw->running = 0;
    317 
    318 				kw->msg[0]->evt.flags |= KORE_EVENT_READ;
    319 				net_recv_flush(kw->msg[0]);
    320 
    321 				if (!kore_quiet) {
    322 					kore_log(LOG_NOTICE,
    323 					    "worker %s exited (%d)",
    324 					    kore_worker_name(kw->id), status);
    325 				}
    326 			}
    327 		}
    328 
    329 		done = 0;
    330 		for (idx = 0; idx < worker_count; idx++) {
    331 			kw = WORKER(idx);
    332 			if (kw->running == 0) {
    333 				done++;
    334 				continue;
    335 			}
    336 		}
    337 
    338 		if (done == worker_count)
    339 			break;
    340 	}
    341 
    342 	if (shmctl(shm_accept_key, IPC_RMID, NULL) == -1) {
    343 		kore_log(LOG_NOTICE,
    344 		    "failed to deleted shm segment: %s", errno_s);
    345 	}
    346 }
    347 
    348 void
    349 kore_worker_dispatch_signal(int sig)
    350 {
    351 	u_int16_t		idx;
    352 	struct kore_worker	*kw;
    353 
    354 	for (idx = 0; idx < worker_count; idx++) {
    355 		kw = WORKER(idx);
    356 
    357 		if (kw->pid == -1 || kw->pid == 0)
    358 			continue;
    359 
    360 		if (kill(kw->pid, sig) == -1) {
    361 			kore_log(LOG_WARNING, "kill(%d, %d): %s",
    362 			    kw->pid, sig, errno_s);
    363 		}
    364 	}
    365 }
    366 
    367 void
    368 kore_worker_privsep(void)
    369 {
    370 	rlim_t			fd;
    371 	struct rlimit		rl;
    372 	struct passwd		*pw;
    373 
    374 	if (worker == NULL)
    375 		fatalx("%s called with no worker", __func__);
    376 
    377 	if (worker_set_affinity == 1)
    378 		kore_platform_worker_setcpu(worker);
    379 
    380 	pw = NULL;
    381 
    382 	/* Must happen before chroot. */
    383 	if (worker->ps->skip_runas == 0) {
    384 		if (worker->ps->runas == NULL) {
    385 			fatalx("no runas user given for %s",
    386 			    kore_worker_name(worker->id));
    387 		}
    388 
    389 		if ((pw = getpwnam(worker->ps->runas)) == NULL) {
    390 			fatalx("cannot getpwnam(\"%s\") for user: %s",
    391 			    worker->ps->runas, errno_s);
    392 		}
    393 	}
    394 
    395 	if (worker->ps->skip_chroot == 0) {
    396 		if (chroot(worker->ps->root) == -1) {
    397 			fatalx("cannot chroot(\"%s\"): %s",
    398 			    worker->ps->root, errno_s);
    399 		}
    400 
    401 		if (chdir("/") == -1)
    402 			fatalx("cannot chdir(\"/\"): %s", errno_s);
    403 	} else {
    404 		if (chdir(worker->ps->root) == -1) {
    405 			fatalx("cannot chdir(\"%s\"): %s",
    406 			    worker->ps->root, errno_s);
    407 		}
    408 	}
    409 
    410 	if (getrlimit(RLIMIT_NOFILE, &rl) == -1) {
    411 		kore_log(LOG_WARNING, "getrlimit(RLIMIT_NOFILE): %s", errno_s);
    412 	} else {
    413 		for (fd = 0; fd < rl.rlim_cur; fd++) {
    414 			if (fcntl(fd, F_GETFD, NULL) != -1) {
    415 				worker_rlimit_nofiles++;
    416 			}
    417 		}
    418 	}
    419 
    420 	rl.rlim_cur = worker_rlimit_nofiles;
    421 	rl.rlim_max = worker_rlimit_nofiles;
    422 	if (setrlimit(RLIMIT_NOFILE, &rl) == -1) {
    423 		kore_log(LOG_ERR, "setrlimit(RLIMIT_NOFILE, %u): %s",
    424 		    worker_rlimit_nofiles, errno_s);
    425 	}
    426 
    427 	if (worker->ps->skip_runas == 0) {
    428 		if (setgroups(1, &pw->pw_gid) ||
    429 #if defined(__MACH__) || defined(NetBSD)
    430 		    setgid(pw->pw_gid) || setegid(pw->pw_gid) ||
    431 		    setuid(pw->pw_uid) || seteuid(pw->pw_uid))
    432 #else
    433 		    setresgid(pw->pw_gid, pw->pw_gid, pw->pw_gid) ||
    434 		    setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid))
    435 #endif
    436 			fatalx("cannot drop privileges (%s)", errno_s);
    437 	}
    438 
    439 	kore_platform_sandbox();
    440 }
    441 
    442 void
    443 kore_worker_entry(struct kore_worker *kw)
    444 {
    445 	struct kore_runtime_call	*sigcall;
    446 	u_int64_t			last_seed;
    447 	int				quit, had_lock, sig;
    448 	u_int64_t			netwait, now, next_timeo;
    449 
    450 	worker = kw;
    451 
    452 	if (!kore_foreground)
    453 		closelog();
    454 
    455 #if defined(__linux__)
    456 	kore_seccomp_traceme();
    457 #endif
    458 
    459 	kore_platform_proctitle(kore_worker_name(kw->id));
    460 
    461 	kore_pid = kw->pid;
    462 	kore_signal_setup();
    463 
    464 	if (kw->id == KORE_WORKER_KEYMGR) {
    465 		kore_keymgr_run();
    466 		exit(0);
    467 	}
    468 
    469 #if defined(KORE_USE_ACME)
    470 	if (kw->id == KORE_WORKER_ACME) {
    471 		kore_acme_run();
    472 		exit(0);
    473 	}
    474 #endif
    475 
    476 	net_init();
    477 	kore_connection_init();
    478 	kore_platform_event_init();
    479 	kore_msg_worker_init();
    480 
    481 #if defined(KORE_USE_TASKS)
    482 	kore_task_init();
    483 #endif
    484 
    485 	kore_worker_privsep();
    486 
    487 #if !defined(KORE_NO_HTTP)
    488 	http_init();
    489 	kore_filemap_resolve_paths();
    490 	kore_accesslog_worker_init();
    491 #endif
    492 	kore_timer_init();
    493 	kore_fileref_init();
    494 	kore_tls_keymgr_init();
    495 
    496 	quit = 0;
    497 	had_lock = 0;
    498 	next_timeo = 0;
    499 	accept_avail = 1;
    500 	worker_active_connections = 0;
    501 
    502 	last_seed = 0;
    503 
    504 	if (kore_keymgr_active) {
    505 		kore_msg_register(KORE_MSG_CRL, worker_keymgr_response);
    506 		kore_msg_register(KORE_MSG_ENTROPY_RESP, worker_entropy_recv);
    507 		kore_msg_register(KORE_MSG_CERTIFICATE, worker_keymgr_response);
    508 
    509 		if (worker->restarted) {
    510 			kore_msg_send(KORE_WORKER_KEYMGR,
    511 			    KORE_MSG_CERTIFICATE_REQ, NULL, 0);
    512 		}
    513 #if defined(KORE_USE_ACME)
    514 		kore_msg_register(KORE_ACME_CHALLENGE_SET_CERT,
    515 		    worker_keymgr_response);
    516 		kore_msg_register(KORE_ACME_CHALLENGE_CLEAR_CERT,
    517 		    worker_keymgr_response);
    518 #endif
    519 	}
    520 
    521 	kore_msg_register(KORE_MSG_ACCEPT_AVAILABLE, worker_accept_avail);
    522 
    523 	if (nlisteners == 0)
    524 		worker_no_lock = 1;
    525 
    526 	worker_runtime_configure();
    527 
    528 	kore_module_onload();
    529 	kore_domain_callback(worker_domain_check);
    530 
    531 	kore_worker_started();
    532 	worker->restarted = 0;
    533 
    534 	sigcall = worker_runtime_signal();
    535 
    536 	for (;;) {
    537 		now = kore_time_ms();
    538 
    539 		if (kore_keymgr_active &&
    540 		    (now - last_seed) > KORE_RESEED_TIME) {
    541 			kore_msg_send(KORE_WORKER_KEYMGR,
    542 			    KORE_MSG_ENTROPY_REQ, NULL, 0);
    543 			last_seed = now;
    544 		}
    545 
    546 		if (!worker->has_lock && accept_avail) {
    547 			if (worker_acceptlock_obtain()) {
    548 				accept_avail = 0;
    549 				if (had_lock == 0) {
    550 					kore_platform_enable_accept();
    551 					had_lock = 1;
    552 				}
    553 			}
    554 		}
    555 
    556 		netwait = kore_timer_next_run(now);
    557 
    558 		if (netwait == KORE_WAIT_INFINITE) {
    559 			if (sig_recv != 0)
    560 				netwait = 10;
    561 #if !defined(KORE_NO_HTTP)
    562 			if (http_request_count > 0)
    563 				netwait = 100;
    564 #endif
    565 		}
    566 
    567 #if defined(KORE_USE_PYTHON)
    568 		if (kore_python_coro_pending())
    569 			netwait = 0;
    570 #endif
    571 
    572 		kore_platform_event_wait(netwait);
    573 		now = kore_time_ms();
    574 
    575 		if (worker->has_lock)
    576 			worker_acceptlock_release();
    577 
    578 		if (!worker->has_lock) {
    579 			if (had_lock == 1) {
    580 				had_lock = 0;
    581 				kore_platform_disable_accept();
    582 			}
    583 		}
    584 
    585 		sig = sig_recv;
    586 		if (sig != 0) {
    587 			switch (sig) {
    588 			case SIGHUP:
    589 				kore_module_reload(1);
    590 				break;
    591 			case SIGQUIT:
    592 			case SIGINT:
    593 			case SIGTERM:
    594 				quit = 1;
    595 				break;
    596 			case SIGCHLD:
    597 #if defined(KORE_USE_PYTHON)
    598 				kore_python_proc_reap();
    599 #endif
    600 				break;
    601 			default:
    602 				break;
    603 			}
    604 
    605 			if (sigcall != NULL)
    606 				kore_runtime_signal(sigcall, sig);
    607 
    608 			if (sig == sig_recv)
    609 				sig_recv = 0;
    610 		}
    611 
    612 		if (quit)
    613 			break;
    614 
    615 		kore_timer_run(now);
    616 #if defined(KORE_USE_CURL)
    617 		kore_curl_run_scheduled();
    618 		kore_curl_do_timeout();
    619 #endif
    620 #if !defined(KORE_NO_HTTP)
    621 		http_process();
    622 #endif
    623 #if defined(KORE_USE_PYTHON)
    624 		kore_python_coro_run();
    625 #endif
    626 		if (next_timeo <= now) {
    627 			kore_connection_check_timeout(now);
    628 			next_timeo = now + 500;
    629 		}
    630 
    631 		kore_connection_prune(KORE_CONNECTION_PRUNE_DISCONNECT);
    632 	}
    633 
    634 	worker_runtime_teardown();
    635 	kore_server_cleanup();
    636 
    637 	kore_platform_event_cleanup();
    638 	kore_connection_cleanup();
    639 	kore_domain_cleanup();
    640 	kore_tls_cleanup();
    641 	kore_module_cleanup();
    642 #if !defined(KORE_NO_HTTP)
    643 	http_cleanup();
    644 #endif
    645 	net_cleanup();
    646 
    647 #if defined(KORE_USE_PYTHON)
    648 	kore_python_cleanup();
    649 #endif
    650 
    651 #if defined(KORE_USE_PGSQL)
    652 	kore_pgsql_sys_cleanup();
    653 #endif
    654 
    655 	kore_mem_cleanup();
    656 	exit(0);
    657 }
    658 
    659 void
    660 kore_worker_reap(void)
    661 {
    662 	pid_t			pid;
    663 	int			status;
    664 
    665 	for (;;) {
    666 		pid = waitpid(-worker_pgrp, &status, WNOHANG);
    667 
    668 		if (pid == -1) {
    669 			if (errno != ECHILD && errno != EINTR) {
    670 				kore_log(LOG_ERR,
    671 				    "%s: waitpid(): %s", __func__, errno_s);
    672 			}
    673 			break;
    674 		}
    675 
    676 		if (pid == 0)
    677 			break;
    678 
    679 		worker_reaper(pid, status);
    680 	}
    681 }
    682 
    683 void
    684 kore_worker_make_busy(void)
    685 {
    686 	if (worker_count == WORKER_SOLO_COUNT || worker_no_lock == 1)
    687 		return;
    688 
    689 	if (worker->has_lock) {
    690 		worker_unlock();
    691 		worker->has_lock = 0;
    692 		kore_msg_send(KORE_MSG_WORKER_ALL,
    693 		    KORE_MSG_ACCEPT_AVAILABLE, NULL, 0);
    694 	}
    695 }
    696 
    697 int
    698 kore_worker_keymgr_response_verify(struct kore_msg *msg, const void *data,
    699     struct kore_domain **out)
    700 {
    701 	struct kore_server		*srv;
    702 	struct kore_domain		*dom;
    703 	const struct kore_x509_msg	*req;
    704 
    705 	if (msg->length < sizeof(*req)) {
    706 		kore_log(LOG_WARNING,
    707 		    "short keymgr message (%zu)", msg->length);
    708 		return (KORE_RESULT_ERROR);
    709 	}
    710 
    711 	req = (const struct kore_x509_msg *)data;
    712 	if (msg->length != (sizeof(*req) + req->data_len)) {
    713 		kore_log(LOG_WARNING,
    714 		    "invalid keymgr payload (%zu)", msg->length);
    715 		return (KORE_RESULT_ERROR);
    716 	}
    717 
    718 	if (req->domain[KORE_DOMAINNAME_LEN] != '\0') {
    719 		kore_log(LOG_WARNING, "domain not NUL-terminated");
    720 		return (KORE_RESULT_ERROR);
    721 
    722 	}
    723 
    724 	if (out == NULL)
    725 		return (KORE_RESULT_OK);
    726 
    727 	dom = NULL;
    728 
    729 	LIST_FOREACH(srv, &kore_servers, list) {
    730 		dom = NULL;
    731 
    732 		if (srv->tls == 0)
    733 			continue;
    734 
    735 		TAILQ_FOREACH(dom, &srv->domains, list) {
    736 			if (!strcmp(dom->domain, req->domain))
    737 				break;
    738 		}
    739 
    740 		if (dom != NULL)
    741 			break;
    742 	}
    743 
    744 	if (dom == NULL) {
    745 		kore_log(LOG_WARNING,
    746 		    "got keymgr response for domain that does not exist");
    747 		return (KORE_RESULT_ERROR);
    748 	}
    749 
    750 	*out = dom;
    751 
    752 	return (KORE_RESULT_OK);
    753 }
    754 
    755 void
    756 kore_worker_started(void)
    757 {
    758 	const char	*chroot;
    759 
    760 	if (worker->ps->skip_chroot)
    761 		chroot = "root";
    762 	else
    763 		chroot = "chroot";
    764 
    765 	if (!kore_quiet) {
    766 		kore_log(LOG_NOTICE,
    767 		    "started (#%d %s=%s%s%s)",
    768 		    getpid(), chroot, worker->ps->root,
    769 		    worker->ps->skip_runas ? "" : " user=",
    770 		    worker->ps->skip_runas ? "" : worker->ps->runas);
    771 	}
    772 
    773 	worker->ready = 1;
    774 }
    775 
    776 static void
    777 worker_runtime_configure(void)
    778 {
    779 	struct kore_runtime_call	*rcall;
    780 
    781 	rcall = NULL;
    782 
    783 #if defined(KORE_USE_PYTHON)
    784 	rcall = kore_runtime_getcall(KORE_PYTHON_WORKER_START_HOOK);
    785 #endif
    786 
    787 	if (rcall == NULL)
    788 		rcall = kore_runtime_getcall("kore_worker_configure");
    789 
    790 	if (rcall != NULL) {
    791 		kore_runtime_execute(rcall);
    792 		kore_free(rcall);
    793 	}
    794 }
    795 
    796 static struct kore_runtime_call *
    797 worker_runtime_signal(void)
    798 {
    799 	struct kore_runtime_call	*rcall;
    800 
    801 	rcall = NULL;
    802 
    803 #if defined(KORE_USE_PYTHON)
    804 	rcall = kore_runtime_getcall(KORE_PYTHON_SIGNAL_HOOK);
    805 #endif
    806 
    807 	if (rcall == NULL)
    808 		rcall = kore_runtime_getcall("kore_worker_signal");
    809 
    810 	return (rcall);
    811 }
    812 
    813 static void
    814 worker_runtime_teardown(void)
    815 {
    816 	struct kore_runtime_call	*rcall;
    817 
    818 	rcall = NULL;
    819 
    820 #if defined(KORE_USE_PYTHON)
    821 	rcall = kore_runtime_getcall(KORE_PYTHON_WORKER_STOP_HOOK);
    822 #endif
    823 
    824 	if (rcall == NULL)
    825 		rcall = kore_runtime_getcall("kore_worker_teardown");
    826 
    827 	if (rcall != NULL) {
    828 		kore_runtime_execute(rcall);
    829 		kore_free(rcall);
    830 	}
    831 }
    832 
    833 static void
    834 worker_domain_check(struct kore_domain *dom)
    835 {
    836 	struct stat	st;
    837 
    838 	if (dom->cafile != NULL) {
    839 		if (stat(dom->cafile, &st) == -1)
    840 			fatalx("'%s': %s", dom->cafile, errno_s);
    841 		if (access(dom->cafile, R_OK) == -1)
    842 			fatalx("'%s': not readable", dom->cafile);
    843 	}
    844 }
    845 
    846 static void
    847 worker_reaper(pid_t pid, int status)
    848 {
    849 	u_int16_t		idx;
    850 	struct kore_worker	*kw;
    851 	const char		*func;
    852 
    853 #if defined(__linux__)
    854 	if (kore_seccomp_trace(pid, status))
    855 		return;
    856 #endif
    857 
    858 	for (idx = 0; idx < worker_count; idx++) {
    859 		kw = WORKER(idx);
    860 		if (kw->pid != pid)
    861 			continue;
    862 
    863 		kw->msg[0]->evt.flags |= KORE_EVENT_READ;
    864 		net_recv_flush(kw->msg[0]);
    865 
    866 		if (!kore_quiet) {
    867 			kore_log(LOG_NOTICE,
    868 			    "worker %s (%d) exited with status %d",
    869 			    kore_worker_name(kw->id), pid, status);
    870 		}
    871 
    872 		kw->running = 0;
    873 
    874 		if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
    875 			kw->pid = 0;
    876 			break;
    877 		}
    878 
    879 		func = "none";
    880 #if !defined(KORE_NO_HTTP)
    881 		if (kw->active_route != NULL)
    882 			func = kw->active_route->func;
    883 #endif
    884 		kore_log(LOG_NOTICE,
    885 		    "worker %d (pid: %d) (hdlr: %s) gone",
    886 		    kw->id, kw->pid, func);
    887 
    888 #if defined(__linux__)
    889 		if (WIFSIGNALED(status) && WTERMSIG(status) == SIGSYS) {
    890 			kore_log(LOG_NOTICE,
    891 			    "worker %d died from sandbox violation", kw->id);
    892 		}
    893 #endif
    894 
    895 		if (kw->id == KORE_WORKER_KEYMGR ||
    896 		    kw->id == KORE_WORKER_ACME) {
    897 			kore_log(LOG_CRIT,
    898 			    "keymgr or acme process gone, stopping");
    899 			kw->pid = 0;
    900 			kore_quit = KORE_QUIT_FATAL;
    901 			break;
    902 		}
    903 
    904 		if (kw->pid == accept_lock->current &&
    905 		    worker_no_lock == 0)
    906 			worker_unlock();
    907 
    908 #if !defined(KORE_NO_HTTP)
    909 		if (kw->active_route != NULL) {
    910 			kw->active_route->errors++;
    911 			kore_log(LOG_NOTICE,
    912 			    "hdlr %s has caused %d error(s)",
    913 			    kw->active_route->func,
    914 			    kw->active_route->errors);
    915 		}
    916 #endif
    917 
    918 		if (worker_policy == KORE_WORKER_POLICY_TERMINATE) {
    919 			kw->pid = 0;
    920 			kore_log(LOG_NOTICE,
    921 			    "worker policy is 'terminate', stopping");
    922 			kore_quit = KORE_QUIT_FATAL;
    923 			break;
    924 		}
    925 
    926 		if (kore_quit == KORE_QUIT_NONE) {
    927 			kore_log(LOG_NOTICE, "restarting worker %d", kw->id);
    928 			kw->restarted = 1;
    929 			kore_msg_parent_remove(kw);
    930 
    931 			if (!kore_worker_spawn(idx, kw->id, kw->cpu)) {
    932 				kore_quit = KORE_QUIT_FATAL;
    933 				kore_log(LOG_ERR, "failed to restart worker");
    934 			} else {
    935 				kore_msg_parent_add(kw);
    936 			}
    937 
    938 			break;
    939 		}
    940 	}
    941 }
    942 
    943 static inline void
    944 worker_acceptlock_release(void)
    945 {
    946 	if (worker_count == WORKER_SOLO_COUNT || worker_no_lock == 1)
    947 		return;
    948 
    949 	if (worker->has_lock != 1)
    950 		return;
    951 
    952 	if (worker_active_connections < worker_max_connections) {
    953 #if !defined(KORE_NO_HTTP)
    954 		if (http_request_count < http_request_limit)
    955 			return;
    956 #else
    957 		return;
    958 #endif
    959 	}
    960 
    961 #if defined(WORKER_DEBUG)
    962 	kore_log(LOG_DEBUG, "worker busy, releasing lock");
    963 #endif
    964 
    965 	worker_unlock();
    966 	worker->has_lock = 0;
    967 
    968 	kore_msg_send(KORE_MSG_WORKER_ALL, KORE_MSG_ACCEPT_AVAILABLE, NULL, 0);
    969 }
    970 
    971 static inline int
    972 worker_acceptlock_obtain(void)
    973 {
    974 	int		r;
    975 
    976 	if (worker->has_lock == 1)
    977 		return (1);
    978 
    979 	if (worker_count == WORKER_SOLO_COUNT || worker_no_lock == 1) {
    980 		worker->has_lock = 1;
    981 		return (1);
    982 	}
    983 
    984 	if (worker_active_connections >= worker_max_connections)
    985 		return (0);
    986 
    987 #if !defined(KORE_NO_HTTP)
    988 	if (http_request_count >= http_request_limit)
    989 		return (0);
    990 #endif
    991 
    992 	r = 0;
    993 	if (worker_trylock()) {
    994 		r = 1;
    995 		worker->has_lock = 1;
    996 #if defined(WORKER_DEBUG)
    997 		kore_log(LOG_DEBUG, "got lock");
    998 #endif
    999 	}
   1000 
   1001 	return (r);
   1002 }
   1003 
   1004 static int
   1005 worker_trylock(void)
   1006 {
   1007 	if (!__sync_bool_compare_and_swap(&(accept_lock->lock), 0, 1))
   1008 		return (0);
   1009 
   1010 	accept_lock->current = worker->pid;
   1011 
   1012 	return (1);
   1013 }
   1014 
   1015 static void
   1016 worker_unlock(void)
   1017 {
   1018 	accept_lock->current = 0;
   1019 	if (!__sync_bool_compare_and_swap(&(accept_lock->lock), 1, 0))
   1020 		kore_log(LOG_NOTICE, "worker_unlock(): wasn't locked");
   1021 }
   1022 
   1023 static void
   1024 worker_accept_avail(struct kore_msg *msg, const void *data)
   1025 {
   1026 	accept_avail = 1;
   1027 }
   1028 
   1029 static void
   1030 worker_entropy_recv(struct kore_msg *msg, const void *data)
   1031 {
   1032 	if (msg->length != 1024) {
   1033 		kore_log(LOG_WARNING,
   1034 		    "invalid entropy response (got:%zu - wanted:1024)",
   1035 		    msg->length);
   1036 	}
   1037 
   1038 	kore_tls_seed(data, msg->length);
   1039 }
   1040 
   1041 static void
   1042 worker_keymgr_response(struct kore_msg *msg, const void *data)
   1043 {
   1044 	struct kore_domain		*dom;
   1045 	const struct kore_x509_msg	*req;
   1046 
   1047 	if (!kore_worker_keymgr_response_verify(msg, data, &dom))
   1048 		return;
   1049 
   1050 	req = (const struct kore_x509_msg *)data;
   1051 
   1052 	switch (msg->id) {
   1053 	case KORE_MSG_CERTIFICATE:
   1054 		kore_tls_domain_setup(dom, KORE_PEM_CERT_CHAIN,
   1055 		    req->data, req->data_len);
   1056 		break;
   1057 	case KORE_MSG_CRL:
   1058 		kore_tls_domain_crl(dom, req->data, req->data_len);
   1059 		break;
   1060 #if defined(KORE_USE_ACME)
   1061 	case KORE_ACME_CHALLENGE_SET_CERT:
   1062 		if (dom->tls_ctx == NULL) {
   1063 			kore_tls_domain_setup(dom, KORE_DER_CERT_DATA,
   1064 			    req->data, req->data_len);
   1065 		}
   1066 
   1067 		kore_free(dom->acme_cert);
   1068 		dom->acme_cert_len = req->data_len;
   1069 		dom->acme_cert = kore_calloc(1, req->data_len);
   1070 		memcpy(dom->acme_cert, req->data, req->data_len);
   1071 
   1072 		kore_log(LOG_NOTICE, "[%s] tls-alpn-01 challenge active",
   1073 		    dom->domain);
   1074 		dom->acme_challenge = 1;
   1075 		break;
   1076 	case KORE_ACME_CHALLENGE_CLEAR_CERT:
   1077 		dom->acme_cert_len = 0;
   1078 		dom->acme_challenge = 0;
   1079 
   1080 		kore_free(dom->acme_cert);
   1081 		dom->acme_cert = NULL;
   1082 
   1083 		kore_log(LOG_NOTICE, "[%s] tls-alpn-01 challenge disabled",
   1084 		    dom->domain);
   1085 		break;
   1086 #endif
   1087 	default:
   1088 		kore_log(LOG_WARNING, "unknown keymgr request %u", msg->id);
   1089 		break;
   1090 	}
   1091 }