1112918Sjeff/*
2112918Sjeff * Copyright (c) 2001 Daniel Eischen <deischen@freebsd.org>
3112918Sjeff * Copyright (c) 2000-2001 Jason Evans <jasone@freebsd.org>
4112918Sjeff * All rights reserved.
5112918Sjeff *
6112918Sjeff * Redistribution and use in source and binary forms, with or without
7112918Sjeff * modification, are permitted provided that the following conditions
8112918Sjeff * are met:
9112918Sjeff * 1. Redistributions of source code must retain the above copyright
10112918Sjeff *    notice, this list of conditions and the following disclaimer.
11112918Sjeff * 2. Redistributions in binary form must reproduce the above copyright
12112918Sjeff *    notice, this list of conditions and the following disclaimer in the
13112918Sjeff *    documentation and/or other materials provided with the distribution.
14112918Sjeff *
15112918Sjeff * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
16112918Sjeff * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17112918Sjeff * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18112918Sjeff * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
19112918Sjeff * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20112918Sjeff * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21112918Sjeff * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22112918Sjeff * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23112918Sjeff * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24112918Sjeff * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25112918Sjeff * SUCH DAMAGE.
26112918Sjeff *
27112918Sjeff * $FreeBSD$
28112918Sjeff */
29144518Sdavidxu
30112918Sjeff#include <sys/types.h>
31112918Sjeff#include <sys/mman.h>
32112918Sjeff#include <sys/queue.h>
33217224Skib#include <sys/resource.h>
34217224Skib#include <sys/sysctl.h>
35112918Sjeff#include <stdlib.h>
36112918Sjeff#include <pthread.h>
37217191Skib#include <link.h>
38144518Sdavidxu
39112918Sjeff#include "thr_private.h"
40112918Sjeff
41112918Sjeff/* Spare thread stack. */
42112918Sjeffstruct stack {
43112918Sjeff	LIST_ENTRY(stack)	qe;		/* Stack queue linkage. */
44112918Sjeff	size_t			stacksize;	/* Stack size (rounded up). */
45112918Sjeff	size_t			guardsize;	/* Guard size. */
46112918Sjeff	void			*stackaddr;	/* Stack address. */
47112918Sjeff};
48112918Sjeff
49112918Sjeff/*
50144518Sdavidxu * Default sized (stack and guard) spare stack queue.  Stacks are cached
51144518Sdavidxu * to avoid additional complexity managing mmap()ed stack regions.  Spare
52144518Sdavidxu * stacks are used in LIFO order to increase cache locality.
53112918Sjeff */
54144518Sdavidxustatic LIST_HEAD(, stack)	dstackq = LIST_HEAD_INITIALIZER(dstackq);
55112918Sjeff
56112918Sjeff/*
57112918Sjeff * Miscellaneous sized (non-default stack and/or guard) spare stack queue.
58144518Sdavidxu * Stacks are cached to avoid additional complexity managing mmap()ed
59144518Sdavidxu * stack regions.  This list is unordered, since ordering on both stack
60144518Sdavidxu * size and guard size would be more trouble than it's worth.  Stacks are
61144518Sdavidxu * allocated from this cache on a first size match basis.
62112918Sjeff */
63144518Sdavidxustatic LIST_HEAD(, stack)	mstackq = LIST_HEAD_INITIALIZER(mstackq);
64112918Sjeff
65112918Sjeff/**
66144518Sdavidxu * Base address of the last stack allocated (including its red zone, if
67144518Sdavidxu * there is one).  Stacks are allocated contiguously, starting beyond the
68144518Sdavidxu * top of the main stack.  When a new stack is created, a red zone is
69144518Sdavidxu * typically created (actually, the red zone is mapped with PROT_NONE) above
70144518Sdavidxu * the top of the stack, such that the stack will not be able to grow all
71144518Sdavidxu * the way to the bottom of the next stack.  This isn't fool-proof.  It is
72144518Sdavidxu * possible for a stack to grow by a large amount, such that it grows into
73144518Sdavidxu * the next stack, and as long as the memory within the red zone is never
74144518Sdavidxu * accessed, nothing will prevent one thread stack from trouncing all over
75144518Sdavidxu * the next.
76112918Sjeff *
77112918Sjeff * low memory
78112918Sjeff *     . . . . . . . . . . . . . . . . . .
79112918Sjeff *    |                                   |
80112918Sjeff *    |             stack 3               | start of 3rd thread stack
81112918Sjeff *    +-----------------------------------+
82112918Sjeff *    |                                   |
83112918Sjeff *    |       Red Zone (guard page)       | red zone for 2nd thread
84112918Sjeff *    |                                   |
85112918Sjeff *    +-----------------------------------+
86144518Sdavidxu *    |  stack 2 - _thr_stack_default     | top of 2nd thread stack
87112918Sjeff *    |                                   |
88112918Sjeff *    |                                   |
89112918Sjeff *    |                                   |
90112918Sjeff *    |                                   |
91112918Sjeff *    |             stack 2               |
92112918Sjeff *    +-----------------------------------+ <-- start of 2nd thread stack
93112918Sjeff *    |                                   |
94112918Sjeff *    |       Red Zone                    | red zone for 1st thread
95112918Sjeff *    |                                   |
96112918Sjeff *    +-----------------------------------+
97144518Sdavidxu *    |  stack 1 - _thr_stack_default     | top of 1st thread stack
98112918Sjeff *    |                                   |
99112918Sjeff *    |                                   |
100112918Sjeff *    |                                   |
101112918Sjeff *    |                                   |
102112918Sjeff *    |             stack 1               |
103112918Sjeff *    +-----------------------------------+ <-- start of 1st thread stack
104112918Sjeff *    |                                   |   (initial value of last_stack)
105112918Sjeff *    |       Red Zone                    |
106112918Sjeff *    |                                   | red zone for main thread
107112918Sjeff *    +-----------------------------------+
108144518Sdavidxu *    | USRSTACK - _thr_stack_initial     | top of main thread stack
109112918Sjeff *    |                                   | ^
110112918Sjeff *    |                                   | |
111112918Sjeff *    |                                   | |
112112918Sjeff *    |                                   | | stack growth
113112918Sjeff *    |                                   |
114112918Sjeff *    +-----------------------------------+ <-- start of main thread stack
115112918Sjeff *                                              (USRSTACK)
116112918Sjeff * high memory
117112918Sjeff *
118112918Sjeff */
119157457Sdavidxustatic char *last_stack = NULL;
120112918Sjeff
121144518Sdavidxu/*
122144518Sdavidxu * Round size up to the nearest multiple of
123144518Sdavidxu * _thr_page_size.
124144518Sdavidxu */
125144518Sdavidxustatic inline size_t
126144518Sdavidxuround_up(size_t size)
127112918Sjeff{
128144518Sdavidxu	if (size % _thr_page_size != 0)
129144518Sdavidxu		size = ((size / _thr_page_size) + 1) *
130144518Sdavidxu		    _thr_page_size;
131144518Sdavidxu	return size;
132144518Sdavidxu}
133112918Sjeff
134217191Skibvoid
135217191Skib_thr_stack_fix_protection(struct pthread *thrd)
136217191Skib{
137217191Skib
138217191Skib	mprotect((char *)thrd->attr.stackaddr_attr +
139217191Skib	    round_up(thrd->attr.guardsize_attr),
140217191Skib	    round_up(thrd->attr.stacksize_attr),
141217191Skib	    _rtld_get_stack_prot());
142217191Skib}
143217191Skib
144217224Skibstatic void
145217224Skibsinglethread_map_stacks_exec(void)
146217224Skib{
147217224Skib	int mib[2];
148217224Skib	struct rlimit rlim;
149217224Skib	u_long usrstack;
150217224Skib	size_t len;
151217224Skib
152217224Skib	mib[0] = CTL_KERN;
153217224Skib	mib[1] = KERN_USRSTACK;
154217224Skib	len = sizeof(usrstack);
155217224Skib	if (sysctl(mib, sizeof(mib) / sizeof(mib[0]), &usrstack, &len, NULL, 0)
156217224Skib	    == -1)
157217224Skib		return;
158217224Skib	if (getrlimit(RLIMIT_STACK, &rlim) == -1)
159217224Skib		return;
160217224Skib	mprotect((void *)(uintptr_t)(usrstack - rlim.rlim_cur),
161217224Skib	    rlim.rlim_cur, _rtld_get_stack_prot());
162217224Skib}
163217224Skib
164217191Skibvoid
165296732Skib__thr_map_stacks_exec(void)
166217191Skib{
167217191Skib	struct pthread *curthread, *thrd;
168217191Skib	struct stack *st;
169217191Skib
170217224Skib	if (!_thr_is_inited()) {
171217224Skib		singlethread_map_stacks_exec();
172217224Skib		return;
173217224Skib	}
174217191Skib	curthread = _get_curthread();
175217191Skib	THREAD_LIST_RDLOCK(curthread);
176217191Skib	LIST_FOREACH(st, &mstackq, qe)
177217191Skib		mprotect((char *)st->stackaddr + st->guardsize, st->stacksize,
178217191Skib		    _rtld_get_stack_prot());
179217191Skib	LIST_FOREACH(st, &dstackq, qe)
180217191Skib		mprotect((char *)st->stackaddr + st->guardsize, st->stacksize,
181217191Skib		    _rtld_get_stack_prot());
182217191Skib	TAILQ_FOREACH(thrd, &_thread_gc_list, gcle)
183217191Skib		_thr_stack_fix_protection(thrd);
184217191Skib	TAILQ_FOREACH(thrd, &_thread_list, tle)
185217191Skib		_thr_stack_fix_protection(thrd);
186217191Skib	THREAD_LIST_UNLOCK(curthread);
187217191Skib}
188217191Skib
189144518Sdavidxuint
190144518Sdavidxu_thr_stack_alloc(struct pthread_attr *attr)
191144518Sdavidxu{
192144518Sdavidxu	struct pthread *curthread = _get_curthread();
193144518Sdavidxu	struct stack *spare_stack;
194144518Sdavidxu	size_t stacksize;
195144518Sdavidxu	size_t guardsize;
196144518Sdavidxu	char *stackaddr;
197144518Sdavidxu
198112918Sjeff	/*
199144518Sdavidxu	 * Round up stack size to nearest multiple of _thr_page_size so
200144518Sdavidxu	 * that mmap() * will work.  If the stack size is not an even
201144518Sdavidxu	 * multiple, we end up initializing things such that there is
202144518Sdavidxu	 * unused space above the beginning of the stack, so the stack
203144518Sdavidxu	 * sits snugly against its guard.
204112918Sjeff	 */
205144518Sdavidxu	stacksize = round_up(attr->stacksize_attr);
206144518Sdavidxu	guardsize = round_up(attr->guardsize_attr);
207112918Sjeff
208144518Sdavidxu	attr->stackaddr_attr = NULL;
209144518Sdavidxu	attr->flags &= ~THR_STACK_USER;
210144518Sdavidxu
211112918Sjeff	/*
212144518Sdavidxu	 * Use the garbage collector lock for synchronization of the
213144518Sdavidxu	 * spare stack lists and allocations from usrstack.
214144518Sdavidxu	 */
215212536Sdavidxu	THREAD_LIST_WRLOCK(curthread);
216144518Sdavidxu	/*
217112918Sjeff	 * If the stack and guard sizes are default, try to allocate a stack
218112918Sjeff	 * from the default-size stack cache:
219112918Sjeff	 */
220144518Sdavidxu	if ((stacksize == THR_STACK_DEFAULT) &&
221144518Sdavidxu	    (guardsize == _thr_guard_default)) {
222144518Sdavidxu		if ((spare_stack = LIST_FIRST(&dstackq)) != NULL) {
223144518Sdavidxu			/* Use the spare stack. */
224112918Sjeff			LIST_REMOVE(spare_stack, qe);
225144518Sdavidxu			attr->stackaddr_attr = spare_stack->stackaddr;
226112918Sjeff		}
227112918Sjeff	}
228112918Sjeff	/*
229112918Sjeff	 * The user specified a non-default stack and/or guard size, so try to
230112918Sjeff	 * allocate a stack from the non-default size stack cache, using the
231112918Sjeff	 * rounded up stack size (stack_size) in the search:
232112918Sjeff	 */
233112918Sjeff	else {
234144518Sdavidxu		LIST_FOREACH(spare_stack, &mstackq, qe) {
235144518Sdavidxu			if (spare_stack->stacksize == stacksize &&
236112918Sjeff			    spare_stack->guardsize == guardsize) {
237112918Sjeff				LIST_REMOVE(spare_stack, qe);
238144518Sdavidxu				attr->stackaddr_attr = spare_stack->stackaddr;
239112918Sjeff				break;
240112918Sjeff			}
241112918Sjeff		}
242112918Sjeff	}
243144518Sdavidxu	if (attr->stackaddr_attr != NULL) {
244144518Sdavidxu		/* A cached stack was found.  Release the lock. */
245144518Sdavidxu		THREAD_LIST_UNLOCK(curthread);
246144518Sdavidxu	}
247144518Sdavidxu	else {
248270040Skib		/*
249270040Skib		 * Allocate a stack from or below usrstack, depending
250270040Skib		 * on the LIBPTHREAD_BIGSTACK_MAIN env variable.
251270040Skib		 */
252112918Sjeff		if (last_stack == NULL)
253144518Sdavidxu			last_stack = _usrstack - _thr_stack_initial -
254144518Sdavidxu			    _thr_guard_default;
255112918Sjeff
256112918Sjeff		/* Allocate a new stack. */
257144518Sdavidxu		stackaddr = last_stack - stacksize - guardsize;
258112918Sjeff
259112918Sjeff		/*
260144518Sdavidxu		 * Even if stack allocation fails, we don't want to try to
261144518Sdavidxu		 * use this location again, so unconditionally decrement
262112918Sjeff		 * last_stack.  Under normal operating conditions, the most
263144518Sdavidxu		 * likely reason for an mmap() error is a stack overflow of
264144518Sdavidxu		 * the adjacent thread stack.
265112918Sjeff		 */
266144518Sdavidxu		last_stack -= (stacksize + guardsize);
267112918Sjeff
268144518Sdavidxu		/* Release the lock before mmap'ing it. */
269144518Sdavidxu		THREAD_LIST_UNLOCK(curthread);
270144518Sdavidxu
271144518Sdavidxu		/* Map the stack and guard page together, and split guard
272144518Sdavidxu		   page from allocated space: */
273270039Skib		if ((stackaddr = mmap(stackaddr, stacksize + guardsize,
274217191Skib		     _rtld_get_stack_prot(), MAP_STACK,
275144518Sdavidxu		     -1, 0)) != MAP_FAILED &&
276144518Sdavidxu		    (guardsize == 0 ||
277144518Sdavidxu		     mprotect(stackaddr, guardsize, PROT_NONE) == 0)) {
278144518Sdavidxu			stackaddr += guardsize;
279144518Sdavidxu		} else {
280144518Sdavidxu			if (stackaddr != MAP_FAILED)
281144518Sdavidxu				munmap(stackaddr, stacksize + guardsize);
282144518Sdavidxu			stackaddr = NULL;
283144518Sdavidxu		}
284144518Sdavidxu		attr->stackaddr_attr = stackaddr;
285112918Sjeff	}
286144518Sdavidxu	if (attr->stackaddr_attr != NULL)
287144518Sdavidxu		return (0);
288144518Sdavidxu	else
289144518Sdavidxu		return (-1);
290112918Sjeff}
291112918Sjeff
292144518Sdavidxu/* This function must be called with _thread_list_lock held. */
293112918Sjeffvoid
294144518Sdavidxu_thr_stack_free(struct pthread_attr *attr)
295112918Sjeff{
296144518Sdavidxu	struct stack *spare_stack;
297112918Sjeff
298144518Sdavidxu	if ((attr != NULL) && ((attr->flags & THR_STACK_USER) == 0)
299144518Sdavidxu	    && (attr->stackaddr_attr != NULL)) {
300157457Sdavidxu		spare_stack = (struct stack *)
301157457Sdavidxu			((char *)attr->stackaddr_attr +
302157457Sdavidxu			attr->stacksize_attr - sizeof(struct stack));
303144518Sdavidxu		spare_stack->stacksize = round_up(attr->stacksize_attr);
304144518Sdavidxu		spare_stack->guardsize = round_up(attr->guardsize_attr);
305144518Sdavidxu		spare_stack->stackaddr = attr->stackaddr_attr;
306112918Sjeff
307144518Sdavidxu		if (spare_stack->stacksize == THR_STACK_DEFAULT &&
308144518Sdavidxu		    spare_stack->guardsize == _thr_guard_default) {
309144518Sdavidxu			/* Default stack/guard size. */
310144518Sdavidxu			LIST_INSERT_HEAD(&dstackq, spare_stack, qe);
311144518Sdavidxu		} else {
312144518Sdavidxu			/* Non-default stack/guard size. */
313144518Sdavidxu			LIST_INSERT_HEAD(&mstackq, spare_stack, qe);
314144518Sdavidxu		}
315144518Sdavidxu		attr->stackaddr_attr = NULL;
316112918Sjeff	}
317112918Sjeff}
318