thr_stack.c revision 331722 
1/*
2 * Copyright (c) 2001 Daniel Eischen <deischen@freebsd.org>
3 * Copyright (c) 2000-2001 Jason Evans <jasone@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: stable/11/lib/libthr/thread/thr_stack.c 331722 2018-03-29 02:50:57Z eadler $");
30
31#include <sys/types.h>
32#include <sys/mman.h>
33#include <sys/queue.h>
34#include <sys/resource.h>
35#include <sys/sysctl.h>
36#include <stdlib.h>
37#include <pthread.h>
38#include <link.h>
39
40#include "thr_private.h"
41
42/* Spare thread stack. */
43struct stack {
44	LIST_ENTRY(stack)	qe;		/* Stack queue linkage. */
45	size_t			stacksize;	/* Stack size (rounded up). */
46	size_t			guardsize;	/* Guard size. */
47	void			*stackaddr;	/* Stack address. */
48};
49
50/*
51 * Default sized (stack and guard) spare stack queue.  Stacks are cached
52 * to avoid additional complexity managing mmap()ed stack regions.  Spare
53 * stacks are used in LIFO order to increase cache locality.
54 */
55static LIST_HEAD(, stack)	dstackq = LIST_HEAD_INITIALIZER(dstackq);
56
57/*
58 * Miscellaneous sized (non-default stack and/or guard) spare stack queue.
59 * Stacks are cached to avoid additional complexity managing mmap()ed
60 * stack regions.  This list is unordered, since ordering on both stack
61 * size and guard size would be more trouble than it's worth.  Stacks are
62 * allocated from this cache on a first size match basis.
63 */
64static LIST_HEAD(, stack)	mstackq = LIST_HEAD_INITIALIZER(mstackq);
65
66/**
67 * Base address of the last stack allocated (including its red zone, if
68 * there is one).  Stacks are allocated contiguously, starting beyond the
69 * top of the main stack.  When a new stack is created, a red zone is
70 * typically created (actually, the red zone is mapped with PROT_NONE) above
71 * the top of the stack, such that the stack will not be able to grow all
72 * the way to the bottom of the next stack.  This isn't fool-proof.  It is
73 * possible for a stack to grow by a large amount, such that it grows into
74 * the next stack, and as long as the memory within the red zone is never
75 * accessed, nothing will prevent one thread stack from trouncing all over
76 * the next.
77 *
78 * low memory
79 *     . . . . . . . . . . . . . . . . . .
80 *    |                                   |
81 *    |             stack 3               | start of 3rd thread stack
82 *    +-----------------------------------+
83 *    |                                   |
84 *    |       Red Zone (guard page)       | red zone for 2nd thread
85 *    |                                   |
86 *    +-----------------------------------+
87 *    |  stack 2 - _thr_stack_default     | top of 2nd thread stack
88 *    |                                   |
89 *    |                                   |
90 *    |                                   |
91 *    |                                   |
92 *    |             stack 2               |
93 *    +-----------------------------------+ <-- start of 2nd thread stack
94 *    |                                   |
95 *    |       Red Zone                    | red zone for 1st thread
96 *    |                                   |
97 *    +-----------------------------------+
98 *    |  stack 1 - _thr_stack_default     | top of 1st thread stack
99 *    |                                   |
100 *    |                                   |
101 *    |                                   |
102 *    |                                   |
103 *    |             stack 1               |
104 *    +-----------------------------------+ <-- start of 1st thread stack
105 *    |                                   |   (initial value of last_stack)
106 *    |       Red Zone                    |
107 *    |                                   | red zone for main thread
108 *    +-----------------------------------+
109 *    | USRSTACK - _thr_stack_initial     | top of main thread stack
110 *    |                                   | ^
111 *    |                                   | |
112 *    |                                   | |
113 *    |                                   | | stack growth
114 *    |                                   |
115 *    +-----------------------------------+ <-- start of main thread stack
116 *                                              (USRSTACK)
117 * high memory
118 *
119 */
120static char *last_stack = NULL;
121
122/*
123 * Round size up to the nearest multiple of
124 * _thr_page_size.
125 */
126static inline size_t
127round_up(size_t size)
128{
129	if (size % _thr_page_size != 0)
130		size = ((size / _thr_page_size) + 1) *
131		    _thr_page_size;
132	return size;
133}
134
135void
136_thr_stack_fix_protection(struct pthread *thrd)
137{
138
139	mprotect((char *)thrd->attr.stackaddr_attr +
140	    round_up(thrd->attr.guardsize_attr),
141	    round_up(thrd->attr.stacksize_attr),
142	    _rtld_get_stack_prot());
143}
144
145static void
146singlethread_map_stacks_exec(void)
147{
148	int mib[2];
149	struct rlimit rlim;
150	u_long usrstack;
151	size_t len;
152
153	mib[0] = CTL_KERN;
154	mib[1] = KERN_USRSTACK;
155	len = sizeof(usrstack);
156	if (sysctl(mib, sizeof(mib) / sizeof(mib[0]), &usrstack, &len, NULL, 0)
157	    == -1)
158		return;
159	if (getrlimit(RLIMIT_STACK, &rlim) == -1)
160		return;
161	mprotect((void *)(uintptr_t)(usrstack - rlim.rlim_cur),
162	    rlim.rlim_cur, _rtld_get_stack_prot());
163}
164
165void
166__thr_map_stacks_exec(void)
167{
168	struct pthread *curthread, *thrd;
169	struct stack *st;
170
171	if (!_thr_is_inited()) {
172		singlethread_map_stacks_exec();
173		return;
174	}
175	curthread = _get_curthread();
176	THREAD_LIST_RDLOCK(curthread);
177	LIST_FOREACH(st, &mstackq, qe)
178		mprotect((char *)st->stackaddr + st->guardsize, st->stacksize,
179		    _rtld_get_stack_prot());
180	LIST_FOREACH(st, &dstackq, qe)
181		mprotect((char *)st->stackaddr + st->guardsize, st->stacksize,
182		    _rtld_get_stack_prot());
183	TAILQ_FOREACH(thrd, &_thread_gc_list, gcle)
184		_thr_stack_fix_protection(thrd);
185	TAILQ_FOREACH(thrd, &_thread_list, tle)
186		_thr_stack_fix_protection(thrd);
187	THREAD_LIST_UNLOCK(curthread);
188}
189
190int
191_thr_stack_alloc(struct pthread_attr *attr)
192{
193	struct pthread *curthread = _get_curthread();
194	struct stack *spare_stack;
195	size_t stacksize;
196	size_t guardsize;
197	char *stackaddr;
198
199	/*
200	 * Round up stack size to nearest multiple of _thr_page_size so
201	 * that mmap() * will work.  If the stack size is not an even
202	 * multiple, we end up initializing things such that there is
203	 * unused space above the beginning of the stack, so the stack
204	 * sits snugly against its guard.
205	 */
206	stacksize = round_up(attr->stacksize_attr);
207	guardsize = round_up(attr->guardsize_attr);
208
209	attr->stackaddr_attr = NULL;
210	attr->flags &= ~THR_STACK_USER;
211
212	/*
213	 * Use the garbage collector lock for synchronization of the
214	 * spare stack lists and allocations from usrstack.
215	 */
216	THREAD_LIST_WRLOCK(curthread);
217	/*
218	 * If the stack and guard sizes are default, try to allocate a stack
219	 * from the default-size stack cache:
220	 */
221	if ((stacksize == THR_STACK_DEFAULT) &&
222	    (guardsize == _thr_guard_default)) {
223		if ((spare_stack = LIST_FIRST(&dstackq)) != NULL) {
224			/* Use the spare stack. */
225			LIST_REMOVE(spare_stack, qe);
226			attr->stackaddr_attr = spare_stack->stackaddr;
227		}
228	}
229	/*
230	 * The user specified a non-default stack and/or guard size, so try to
231	 * allocate a stack from the non-default size stack cache, using the
232	 * rounded up stack size (stack_size) in the search:
233	 */
234	else {
235		LIST_FOREACH(spare_stack, &mstackq, qe) {
236			if (spare_stack->stacksize == stacksize &&
237			    spare_stack->guardsize == guardsize) {
238				LIST_REMOVE(spare_stack, qe);
239				attr->stackaddr_attr = spare_stack->stackaddr;
240				break;
241			}
242		}
243	}
244	if (attr->stackaddr_attr != NULL) {
245		/* A cached stack was found.  Release the lock. */
246		THREAD_LIST_UNLOCK(curthread);
247	}
248	else {
249		/*
250		 * Allocate a stack from or below usrstack, depending
251		 * on the LIBPTHREAD_BIGSTACK_MAIN env variable.
252		 */
253		if (last_stack == NULL)
254			last_stack = _usrstack - _thr_stack_initial -
255			    _thr_guard_default;
256
257		/* Allocate a new stack. */
258		stackaddr = last_stack - stacksize - guardsize;
259
260		/*
261		 * Even if stack allocation fails, we don't want to try to
262		 * use this location again, so unconditionally decrement
263		 * last_stack.  Under normal operating conditions, the most
264		 * likely reason for an mmap() error is a stack overflow of
265		 * the adjacent thread stack.
266		 */
267		last_stack -= (stacksize + guardsize);
268
269		/* Release the lock before mmap'ing it. */
270		THREAD_LIST_UNLOCK(curthread);
271
272		/* Map the stack and guard page together, and split guard
273		   page from allocated space: */
274		if ((stackaddr = mmap(stackaddr, stacksize + guardsize,
275		     _rtld_get_stack_prot(), MAP_STACK,
276		     -1, 0)) != MAP_FAILED &&
277		    (guardsize == 0 ||
278		     mprotect(stackaddr, guardsize, PROT_NONE) == 0)) {
279			stackaddr += guardsize;
280		} else {
281			if (stackaddr != MAP_FAILED)
282				munmap(stackaddr, stacksize + guardsize);
283			stackaddr = NULL;
284		}
285		attr->stackaddr_attr = stackaddr;
286	}
287	if (attr->stackaddr_attr != NULL)
288		return (0);
289	else
290		return (-1);
291}
292
293/* This function must be called with _thread_list_lock held. */
294void
295_thr_stack_free(struct pthread_attr *attr)
296{
297	struct stack *spare_stack;
298
299	if ((attr != NULL) && ((attr->flags & THR_STACK_USER) == 0)
300	    && (attr->stackaddr_attr != NULL)) {
301		spare_stack = (struct stack *)
302			((char *)attr->stackaddr_attr +
303			attr->stacksize_attr - sizeof(struct stack));
304		spare_stack->stacksize = round_up(attr->stacksize_attr);
305		spare_stack->guardsize = round_up(attr->guardsize_attr);
306		spare_stack->stackaddr = attr->stackaddr_attr;
307
308		if (spare_stack->stacksize == THR_STACK_DEFAULT &&
309		    spare_stack->guardsize == _thr_guard_default) {
310			/* Default stack/guard size. */
311			LIST_INSERT_HEAD(&dstackq, spare_stack, qe);
312		} else {
313			/* Non-default stack/guard size. */
314			LIST_INSERT_HEAD(&mstackq, spare_stack, qe);
315		}
316		attr->stackaddr_attr = NULL;
317	}
318}
319