pool.c (7429B)
1 /*
2 * Copyright (c) 2013-2022 Joris Vink <joris@coders.se>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
16
17 /*
18 * A kore_pool is a memory pool containing fixed-sized objects that
19 * can quickly be obtained by a caller via kore_pool_get() and returned
20 * via kore_pool_put().
21 *
22 * Each entry in a pool will have a canary at the end that is used to
23 * catch any potential overruns when the entry is returned to the pool.
24 *
25 * If memory pool guards are enabled three additional things happen:
26 *
27 * 1) The metadata is placed at the start of a page instead
28 * of right before the returned user pointer.
29 *
30 * 2) Each pool entry gets a guard page at the end of its allocation
31 * that is marked as PROT_NONE. Touching a guard page will cause
32 * the application to receive a SIGSEGV.
33 *
34 * 3) Entries are only marked PROT_READ | PROT_WRITE when they are
35 * obtained with kore_pool_get(). Their memory protection is
36 * changed to PROT_NONE when returned to the pool via kore_pool_get().
37 *
38 * Caveats:
39 * Pools are designed to live for the entire lifetime of a Kore process
40 * until it will exit and are therefor not properly cleaned up when exit
41 * time arrives.
42 */
43
44 #include <sys/types.h>
45 #include <sys/mman.h>
46 #include <sys/queue.h>
47
48 #include <stdint.h>
49
50 #include "kore.h"
51
52 #define POOL_MIN_ELEMENTS 16
53
54 #define POOL_ELEMENT_BUSY 0
55 #define POOL_ELEMENT_FREE 1
56
57 #if defined(KORE_USE_TASKS)
58 static void pool_lock(struct kore_pool *);
59 static void pool_unlock(struct kore_pool *);
60 #endif
61
62 static void pool_grow(struct kore_pool *, size_t);
63
64 static void pool_mark_entry_rw(struct kore_pool *, void *);
65 static void pool_mark_entry_none(struct kore_pool *, void *);
66
67 void
68 kore_pool_init(struct kore_pool *pool, const char *name,
69 size_t len, size_t elm)
70 {
71 long pagesz;
72
73 if (elm < POOL_MIN_ELEMENTS)
74 elm = POOL_MIN_ELEMENTS;
75
76 if ((pagesz = sysconf(_SC_PAGESIZE)) == -1)
77 fatal("%s: sysconf: %s", __func__, errno_s);
78
79 if ((pool->name = strdup(name)) == NULL)
80 fatal("kore_pool_init: strdup %s", errno_s);
81
82 pool->uselen = len;
83
84 len = len + sizeof(u_int64_t);
85 len = (len + (16 - 1)) & ~(16 - 1);
86
87 pool->elmlen = len;
88
89 pool->lock = 0;
90 pool->freelist = NULL;
91 pool->pagesz = pagesz;
92 pool->growth = elm * 0.25f;
93 pool->canary = (u_int64_t)kore_platform_random_uint32() << 32 |
94 kore_platform_random_uint32();
95
96 if (kore_mem_guard) {
97 pool->memsz = pool->pagesz * 2;
98
99 while (pool->elmlen >
100 pool->pagesz - sizeof(struct kore_pool_entry)) {
101 pool->memsz += pool->pagesz;
102 pool->elmlen -= MIN(pool->elmlen, pool->pagesz);
103 }
104
105 pool->elmlen = len;
106 } else {
107 pool->memsz = pool->elmlen;
108 }
109
110 pool_grow(pool, elm);
111 }
112
113 void
114 kore_pool_cleanup(struct kore_pool *pool)
115 {
116 struct kore_pool_entry *entry, *next;
117
118 if (kore_mem_guard) {
119 for (entry = pool->freelist; entry != NULL; entry = next) {
120 pool_mark_entry_rw(pool, entry);
121 next = entry->nextfree;
122 (void)munmap(entry, pool->memsz);
123 }
124 }
125
126 free(pool->name);
127 }
128
129 void *
130 kore_pool_get(struct kore_pool *pool)
131 {
132 u_int64_t canary;
133 struct kore_pool_entry *entry;
134
135 #if defined(KORE_USE_TASKS)
136 pool_lock(pool);
137 #endif
138
139 if (pool->freelist == NULL)
140 pool_grow(pool, pool->growth);
141
142 entry = pool->freelist;
143
144 if (kore_mem_guard)
145 pool_mark_entry_rw(pool, entry);
146
147 pool->freelist = entry->nextfree;
148
149 if (entry->state != POOL_ELEMENT_FREE)
150 fatal("%s: element %p was not free", pool->name, (void *)entry);
151
152 entry->nextfree = NULL;
153 entry->state = POOL_ELEMENT_BUSY;
154
155 canary = pool->canary;
156 canary ^= (uintptr_t)entry;
157 canary ^= (uintptr_t)entry->uptr;
158
159 memcpy(entry->canary, &canary, sizeof(canary));
160
161 #if defined(KORE_USE_TASKS)
162 pool_unlock(pool);
163 #endif
164
165 return (entry->uptr);
166 }
167
168 void
169 kore_pool_put(struct kore_pool *pool, void *ptr)
170 {
171 void *base;
172 u_int64_t canary;
173 struct kore_pool_entry *entry;
174
175 #if defined(KORE_USE_TASKS)
176 pool_lock(pool);
177 #endif
178
179 if (kore_mem_guard) {
180 base = (u_int8_t *)ptr - ((uintptr_t)ptr % pool->pagesz);
181 } else {
182 base = (u_int8_t *)ptr - sizeof(*entry);
183 }
184
185 entry = (struct kore_pool_entry *)base;
186
187 if (entry->uptr != ptr) {
188 fatal("%s: uptr mismatch %p != %p",
189 pool->name, entry->uptr, ptr);
190 }
191
192 memcpy(&canary, entry->canary, sizeof(canary));
193 canary ^= (uintptr_t)entry;
194 canary ^= (uintptr_t)ptr;
195
196 if (canary != pool->canary)
197 fatal("%s: memory corruption detected", pool->name);
198
199 if (entry->state != POOL_ELEMENT_BUSY)
200 fatal("%s: element %p was not busy", pool->name, ptr);
201
202 entry->state = POOL_ELEMENT_FREE;
203 entry->nextfree = pool->freelist;
204
205 if (kore_mem_guard)
206 pool_mark_entry_none(pool, entry);
207
208 pool->freelist = entry;
209 #if defined(KORE_USE_TASKS)
210 pool_unlock(pool);
211 #endif
212 }
213
214 static void
215 pool_grow(struct kore_pool *pool, size_t elms)
216 {
217 size_t i;
218 u_int8_t *base, *p;
219 struct kore_pool_entry *entry, *prev;
220
221 prev = pool->freelist;
222
223 if (kore_mem_guard == 0)
224 base = kore_mmap_region(elms * (sizeof(*entry) + pool->elmlen));
225 else
226 base = NULL;
227
228 for (i = 0; i < elms; i++) {
229 if (kore_mem_guard) {
230 base = kore_mmap_region(pool->memsz);
231 p = base + (pool->memsz - pool->pagesz - pool->elmlen);
232 entry = (struct kore_pool_entry *)base;
233 } else {
234 p = base + ((sizeof(*entry) + pool->elmlen) * i);
235 entry = (struct kore_pool_entry *)p;
236 p += sizeof(*entry);
237 }
238
239 entry->uptr = p;
240 entry->nextfree = NULL;
241 entry->state = POOL_ELEMENT_FREE;
242 entry->canary = p + pool->uselen;
243
244 if (prev != NULL) {
245 prev->nextfree = entry;
246 if (kore_mem_guard)
247 pool_mark_entry_none(pool, prev);
248 }
249
250 prev = entry;
251
252 if (pool->freelist == NULL)
253 pool->freelist = entry;
254
255 if (kore_mem_guard) {
256 p += pool->elmlen;
257
258 if (((uintptr_t)p % pool->pagesz) != 0)
259 fatal("%s: misaligned page", __func__);
260
261 if (mprotect(p, pool->pagesz, PROT_NONE) == -1)
262 fatal("%s: mprotect: %s", __func__, errno_s);
263
264 if (madvise(p, pool->pagesz, MADV_FREE) == -1)
265 fatal("%s: madvise: %s", __func__, errno_s);
266 }
267 }
268
269 if (prev != NULL && kore_mem_guard)
270 pool_mark_entry_none(pool, prev);
271 }
272
273 static void
274 pool_mark_entry_none(struct kore_pool *pool, void *ptr)
275 {
276 if (mprotect(ptr, pool->memsz - pool->pagesz, PROT_NONE) == -1)
277 fatal("%s: mprotect: %s", __func__, errno_s);
278 }
279
280 static void
281 pool_mark_entry_rw(struct kore_pool *pool, void *ptr)
282 {
283 if (mprotect(ptr, pool->memsz - pool->pagesz,
284 PROT_READ | PROT_WRITE) == -1)
285 fatal("%s: mprotect: %s", __func__, errno_s);
286 }
287
288 #if defined(KORE_USE_TASKS)
289 static void
290 pool_lock(struct kore_pool *pool)
291 {
292 for (;;) {
293 if (__sync_bool_compare_and_swap(&pool->lock, 0, 1))
294 break;
295 }
296 }
297
298 static void
299 pool_unlock(struct kore_pool *pool)
300 {
301 if (!__sync_bool_compare_and_swap(&pool->lock, 1, 0))
302 fatal("pool_unlock: failed to release %s", pool->name);
303 }
304 #endif