1/* Library support for -fsplit-stack. */ 2/* Copyright (C) 2009-2020 Free Software Foundation, Inc. 3 Contributed by Ian Lance Taylor <iant@google.com>. 4 5This file is part of GCC. 6 7GCC is free software; you can redistribute it and/or modify it under 8the terms of the GNU General Public License as published by the Free 9Software Foundation; either version 3, or (at your option) any later 10version. 11 12GCC is distributed in the hope that it will be useful, but WITHOUT ANY 13WARRANTY; without even the implied warranty of MERCHANTABILITY or 14FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 15for more details. 16 17Under Section 7 of GPL version 3, you are granted additional 18permissions described in the GCC Runtime Library Exception, version 193.1, as published by the Free Software Foundation. 20 21You should have received a copy of the GNU General Public License and 22a copy of the GCC Runtime Library Exception along with this program; 23see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 24<http://www.gnu.org/licenses/>. */ 25 26#pragma GCC optimize ("no-isolate-erroneous-paths-dereference") 27 28/* powerpc 32-bit not supported. */ 29#if !defined __powerpc__ || defined __powerpc64__ 30 31#include "tconfig.h" 32#include "tsystem.h" 33#include "coretypes.h" 34#include "tm.h" 35#include "libgcc_tm.h" 36 37/* If inhibit_libc is defined, we cannot compile this file. The 38 effect is that people will not be able to use -fsplit-stack. That 39 is much better than failing the build particularly since people 40 will want to define inhibit_libc while building a compiler which 41 can build glibc. */ 42 43#ifndef inhibit_libc 44 45#include <assert.h> 46#include <errno.h> 47#include <signal.h> 48#include <stdlib.h> 49#include <string.h> 50#include <unistd.h> 51#include <sys/mman.h> 52#include <sys/uio.h> 53 54#include "generic-morestack.h" 55 56/* Some systems use LD_PRELOAD or similar tricks to add hooks to 57 mmap/munmap. That breaks this code, because when we call mmap 58 there is enough stack space for the system call but there is not, 59 in general, enough stack space to run a hook. Try to avoid the 60 problem by calling syscall directly. We only do this on GNU/Linux 61 for now, but it should be easy to add support for more systems with 62 testing. */ 63 64#if defined(__gnu_linux__) 65 66#include <sys/syscall.h> 67 68#if defined(SYS_mmap) || defined(SYS_mmap2) 69 70#ifdef SYS_mmap2 71#define MORESTACK_MMAP SYS_mmap2 72#define MORESTACK_ADJUST_OFFSET(x) ((x) / 4096ULL) 73#else 74#define MORESTACK_MMAP SYS_mmap 75#define MORESTACK_ADJUST_OFFSET(x) (x) 76#endif 77 78static void * 79morestack_mmap (void *addr, size_t length, int prot, int flags, int fd, 80 off_t offset) 81{ 82 offset = MORESTACK_ADJUST_OFFSET (offset); 83 84#ifdef __s390__ 85 long args[6] = { (long) addr, (long) length, (long) prot, (long) flags, 86 (long) fd, (long) offset }; 87 return (void *) syscall (MORESTACK_MMAP, args); 88#else 89 return (void *) syscall (MORESTACK_MMAP, addr, length, prot, flags, fd, 90 offset); 91#endif 92} 93 94#define mmap morestack_mmap 95 96#endif /* defined(SYS_MMAP) || defined(SYS_mmap2) */ 97 98#if defined(SYS_munmap) 99 100static int 101morestack_munmap (void * addr, size_t length) 102{ 103 return (int) syscall (SYS_munmap, addr, length); 104} 105 106#define munmap morestack_munmap 107 108#endif /* defined(SYS_munmap) */ 109 110#endif /* defined(__gnu_linux__) */ 111 112typedef unsigned uintptr_type __attribute__ ((mode (pointer))); 113 114/* This file contains subroutines that are used by code compiled with 115 -fsplit-stack. */ 116 117/* Declare functions to avoid warnings--there is no header file for 118 these internal functions. We give most of these functions the 119 flatten attribute in order to minimize their stack usage--here we 120 must minimize stack usage even at the cost of code size, and in 121 general inlining everything will do that. */ 122 123extern void 124__generic_morestack_set_initial_sp (void *sp, size_t len) 125 __attribute__ ((no_split_stack, flatten, visibility ("hidden"))); 126 127extern void * 128__generic_morestack (size_t *frame_size, void *old_stack, size_t param_size) 129 __attribute__ ((no_split_stack, flatten, visibility ("hidden"))); 130 131extern void * 132__generic_releasestack (size_t *pavailable) 133 __attribute__ ((no_split_stack, flatten, visibility ("hidden"))); 134 135extern void 136__morestack_block_signals (void) 137 __attribute__ ((no_split_stack, flatten, visibility ("hidden"))); 138 139extern void 140__morestack_unblock_signals (void) 141 __attribute__ ((no_split_stack, flatten, visibility ("hidden"))); 142 143extern size_t 144__generic_findstack (void *stack) 145 __attribute__ ((no_split_stack, flatten, visibility ("hidden"))); 146 147extern void 148__morestack_load_mmap (void) 149 __attribute__ ((no_split_stack, visibility ("hidden"))); 150 151extern void * 152__morestack_allocate_stack_space (size_t size) 153 __attribute__ ((visibility ("hidden"))); 154 155/* These are functions which -fsplit-stack code can call. These are 156 not called by the compiler, and are not hidden. FIXME: These 157 should be in some header file somewhere, somehow. */ 158 159extern void * 160__splitstack_find (void *, void *, size_t *, void **, void **, void **) 161 __attribute__ ((visibility ("default"))); 162 163extern void 164__splitstack_block_signals (int *, int *) 165 __attribute__ ((visibility ("default"))); 166 167extern void 168__splitstack_getcontext (void *context[10]) 169 __attribute__ ((no_split_stack, visibility ("default"))); 170 171extern void 172__splitstack_setcontext (void *context[10]) 173 __attribute__ ((no_split_stack, visibility ("default"))); 174 175extern void * 176__splitstack_makecontext (size_t, void *context[10], size_t *) 177 __attribute__ ((visibility ("default"))); 178 179extern void * 180__splitstack_resetcontext (void *context[10], size_t *) 181 __attribute__ ((visibility ("default"))); 182 183extern void 184__splitstack_releasecontext (void *context[10]) 185 __attribute__ ((visibility ("default"))); 186 187extern void 188__splitstack_block_signals_context (void *context[10], int *, int *) 189 __attribute__ ((visibility ("default"))); 190 191extern void * 192__splitstack_find_context (void *context[10], size_t *, void **, void **, 193 void **) 194 __attribute__ ((visibility ("default"))); 195 196/* These functions must be defined by the processor specific code. */ 197 198extern void *__morestack_get_guard (void) 199 __attribute__ ((no_split_stack, visibility ("hidden"))); 200 201extern void __morestack_set_guard (void *) 202 __attribute__ ((no_split_stack, visibility ("hidden"))); 203 204extern void *__morestack_make_guard (void *, size_t) 205 __attribute__ ((no_split_stack, visibility ("hidden"))); 206 207/* When we allocate a stack segment we put this header at the 208 start. */ 209 210struct stack_segment 211{ 212 /* The previous stack segment--when a function running on this stack 213 segment returns, it will run on the previous one. */ 214 struct stack_segment *prev; 215 /* The next stack segment, if it has been allocated--when a function 216 is running on this stack segment, the next one is not being 217 used. */ 218 struct stack_segment *next; 219 /* The total size of this stack segment. */ 220 size_t size; 221 /* The stack address when this stack was created. This is used when 222 popping the stack. */ 223 void *old_stack; 224 /* A list of memory blocks allocated by dynamic stack 225 allocation. */ 226 struct dynamic_allocation_blocks *dynamic_allocation; 227 /* A list of dynamic memory blocks no longer needed. */ 228 struct dynamic_allocation_blocks *free_dynamic_allocation; 229 /* An extra pointer in case we need some more information some 230 day. */ 231 void *extra; 232}; 233 234/* This structure holds the (approximate) initial stack pointer and 235 size for the system supplied stack for a thread. This is set when 236 the thread is created. We also store a sigset_t here to hold the 237 signal mask while splitting the stack, since we don't want to store 238 that on the stack. */ 239 240struct initial_sp 241{ 242 /* The initial stack pointer. */ 243 void *sp; 244 /* The stack length. */ 245 size_t len; 246 /* A signal mask, put here so that the thread can use it without 247 needing stack space. */ 248 sigset_t mask; 249 /* Non-zero if we should not block signals. This is a reversed flag 250 so that the default zero value is the safe value. The type is 251 uintptr_type because it replaced one of the void * pointers in 252 extra. */ 253 uintptr_type dont_block_signals; 254 /* Some extra space for later extensibility. */ 255 void *extra[4]; 256}; 257 258/* A list of memory blocks allocated by dynamic stack allocation. 259 This is used for code that calls alloca or uses variably sized 260 arrays. */ 261 262struct dynamic_allocation_blocks 263{ 264 /* The next block in the list. */ 265 struct dynamic_allocation_blocks *next; 266 /* The size of the allocated memory. */ 267 size_t size; 268 /* The allocated memory. */ 269 void *block; 270}; 271 272/* These thread local global variables must be shared by all split 273 stack code across shared library boundaries. Therefore, they have 274 default visibility. They have extensibility fields if needed for 275 new versions. If more radical changes are needed, new code can be 276 written using new variable names, while still using the existing 277 variables in a backward compatible manner. Symbol versioning is 278 also used, although, since these variables are only referenced by 279 code in this file and generic-morestack-thread.c, it is likely that 280 simply using new names will suffice. */ 281 282/* The first stack segment allocated for this thread. */ 283 284__thread struct stack_segment *__morestack_segments 285 __attribute__ ((visibility ("default"))); 286 287/* The stack segment that we think we are currently using. This will 288 be correct in normal usage, but will be incorrect if an exception 289 unwinds into a different stack segment or if longjmp jumps to a 290 different stack segment. */ 291 292__thread struct stack_segment *__morestack_current_segment 293 __attribute__ ((visibility ("default"))); 294 295/* The initial stack pointer and size for this thread. */ 296 297__thread struct initial_sp __morestack_initial_sp 298 __attribute__ ((visibility ("default"))); 299 300/* A static signal mask, to avoid taking up stack space. */ 301 302static sigset_t __morestack_fullmask; 303 304/* Page size, as returned from getpagesize(). Set on startup. */ 305static unsigned int static_pagesize; 306 307/* Set on startup to non-zero value if SPLIT_STACK_GUARD env var is set. */ 308static int use_guard_page; 309 310/* Convert an integer to a decimal string without using much stack 311 space. Return a pointer to the part of the buffer to use. We this 312 instead of sprintf because sprintf will require too much stack 313 space. */ 314 315static char * 316print_int (int val, char *buf, int buflen, size_t *print_len) 317{ 318 int is_negative; 319 int i; 320 unsigned int uval; 321 322 uval = (unsigned int) val; 323 if (val >= 0) 324 is_negative = 0; 325 else 326 { 327 is_negative = 1; 328 uval = - uval; 329 } 330 331 i = buflen; 332 do 333 { 334 --i; 335 buf[i] = '0' + (uval % 10); 336 uval /= 10; 337 } 338 while (uval != 0 && i > 0); 339 340 if (is_negative) 341 { 342 if (i > 0) 343 --i; 344 buf[i] = '-'; 345 } 346 347 *print_len = buflen - i; 348 return buf + i; 349} 350 351/* Print the string MSG/LEN, the errno number ERR, and a newline on 352 stderr. Then crash. */ 353 354void 355__morestack_fail (const char *, size_t, int) __attribute__ ((noreturn)); 356 357void 358__morestack_fail (const char *msg, size_t len, int err) 359{ 360 char buf[24]; 361 static const char nl[] = "\n"; 362 struct iovec iov[3]; 363 union { char *p; const char *cp; } const_cast; 364 365 const_cast.cp = msg; 366 iov[0].iov_base = const_cast.p; 367 iov[0].iov_len = len; 368 /* We can't call strerror, because it may try to translate the error 369 message, and that would use too much stack space. */ 370 iov[1].iov_base = print_int (err, buf, sizeof buf, &iov[1].iov_len); 371 const_cast.cp = &nl[0]; 372 iov[2].iov_base = const_cast.p; 373 iov[2].iov_len = sizeof nl - 1; 374 /* FIXME: On systems without writev we need to issue three write 375 calls, or punt on printing errno. For now this is irrelevant 376 since stack splitting only works on GNU/Linux anyhow. */ 377 writev (2, iov, 3); 378 abort (); 379} 380 381/* Allocate a new stack segment. FRAME_SIZE is the required frame 382 size. */ 383 384static struct stack_segment * 385allocate_segment (size_t frame_size) 386{ 387 unsigned int pagesize; 388 unsigned int overhead; 389 unsigned int allocate; 390 void *space; 391 struct stack_segment *pss; 392 393 pagesize = static_pagesize; 394 overhead = sizeof (struct stack_segment); 395 396 allocate = pagesize; 397 if (allocate < MINSIGSTKSZ) 398 allocate = ((MINSIGSTKSZ + overhead + pagesize - 1) 399 & ~ (pagesize - 1)); 400 if (allocate < frame_size) 401 allocate = ((frame_size + overhead + pagesize - 1) 402 & ~ (pagesize - 1)); 403 404 if (use_guard_page) 405 allocate += pagesize; 406 407 /* FIXME: If this binary requires an executable stack, then we need 408 to set PROT_EXEC. Unfortunately figuring that out is complicated 409 and target dependent. We would need to use dl_iterate_phdr to 410 see if there is any object which does not have a PT_GNU_STACK 411 phdr, though only for architectures which use that mechanism. */ 412 space = mmap (NULL, allocate, PROT_READ | PROT_WRITE, 413 MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); 414 if (space == MAP_FAILED) 415 { 416 static const char msg[] = 417 "unable to allocate additional stack space: errno "; 418 __morestack_fail (msg, sizeof msg - 1, errno); 419 } 420 421 if (use_guard_page) 422 { 423 void *guard; 424 425#ifdef __LIBGCC_STACK_GROWS_DOWNWARD__ 426 guard = space; 427 space = (char *) space + pagesize; 428#else 429 guard = space + allocate - pagesize; 430#endif 431 432 mprotect (guard, pagesize, PROT_NONE); 433 allocate -= pagesize; 434 } 435 436 pss = (struct stack_segment *) space; 437 438 pss->prev = NULL; 439 pss->next = NULL; 440 pss->size = allocate - overhead; 441 pss->dynamic_allocation = NULL; 442 pss->free_dynamic_allocation = NULL; 443 pss->extra = NULL; 444 445 return pss; 446} 447 448/* Free a list of dynamic blocks. */ 449 450static void 451free_dynamic_blocks (struct dynamic_allocation_blocks *p) 452{ 453 while (p != NULL) 454 { 455 struct dynamic_allocation_blocks *next; 456 457 next = p->next; 458 free (p->block); 459 free (p); 460 p = next; 461 } 462} 463 464/* Merge two lists of dynamic blocks. */ 465 466static struct dynamic_allocation_blocks * 467merge_dynamic_blocks (struct dynamic_allocation_blocks *a, 468 struct dynamic_allocation_blocks *b) 469{ 470 struct dynamic_allocation_blocks **pp; 471 472 if (a == NULL) 473 return b; 474 if (b == NULL) 475 return a; 476 for (pp = &a->next; *pp != NULL; pp = &(*pp)->next) 477 ; 478 *pp = b; 479 return a; 480} 481 482/* Release stack segments. If FREE_DYNAMIC is non-zero, we also free 483 any dynamic blocks. Otherwise we return them. */ 484 485struct dynamic_allocation_blocks * 486__morestack_release_segments (struct stack_segment **pp, int free_dynamic) 487{ 488 struct dynamic_allocation_blocks *ret; 489 struct stack_segment *pss; 490 491 ret = NULL; 492 pss = *pp; 493 while (pss != NULL) 494 { 495 struct stack_segment *next; 496 unsigned int allocate; 497 498 next = pss->next; 499 500 if (pss->dynamic_allocation != NULL 501 || pss->free_dynamic_allocation != NULL) 502 { 503 if (free_dynamic) 504 { 505 free_dynamic_blocks (pss->dynamic_allocation); 506 free_dynamic_blocks (pss->free_dynamic_allocation); 507 } 508 else 509 { 510 ret = merge_dynamic_blocks (pss->dynamic_allocation, ret); 511 ret = merge_dynamic_blocks (pss->free_dynamic_allocation, ret); 512 } 513 } 514 515 allocate = pss->size + sizeof (struct stack_segment); 516 if (munmap (pss, allocate) < 0) 517 { 518 static const char msg[] = "munmap of stack space failed: errno "; 519 __morestack_fail (msg, sizeof msg - 1, errno); 520 } 521 522 pss = next; 523 } 524 *pp = NULL; 525 526 return ret; 527} 528 529/* This function is called by a processor specific function to set the 530 initial stack pointer for a thread. The operating system will 531 always create a stack for a thread. Here we record a stack pointer 532 near the base of that stack. The size argument lets the processor 533 specific code estimate how much stack space is available on this 534 initial stack. */ 535 536void 537__generic_morestack_set_initial_sp (void *sp, size_t len) 538{ 539 /* The stack pointer most likely starts on a page boundary. Adjust 540 to the nearest 512 byte boundary. It's not essential that we be 541 precise here; getting it wrong will just leave some stack space 542 unused. */ 543#ifdef __LIBGCC_STACK_GROWS_DOWNWARD__ 544 sp = (void *) ((((__UINTPTR_TYPE__) sp + 511U) / 512U) * 512U); 545#else 546 sp = (void *) ((((__UINTPTR_TYPE__) sp - 511U) / 512U) * 512U); 547#endif 548 549 __morestack_initial_sp.sp = sp; 550 __morestack_initial_sp.len = len; 551 sigemptyset (&__morestack_initial_sp.mask); 552 553 sigfillset (&__morestack_fullmask); 554#if defined(__GLIBC__) && defined(__linux__) 555 /* In glibc, the first two real time signals are used by the NPTL 556 threading library. By taking them out of the set of signals, we 557 avoiding copying the signal mask in pthread_sigmask. More 558 importantly, pthread_sigmask uses less stack space on x86_64. */ 559 sigdelset (&__morestack_fullmask, __SIGRTMIN); 560 sigdelset (&__morestack_fullmask, __SIGRTMIN + 1); 561#endif 562} 563 564/* This function is called by a processor specific function which is 565 run in the prologue when more stack is needed. The processor 566 specific function handles the details of saving registers and 567 frobbing the actual stack pointer. This function is responsible 568 for allocating a new stack segment and for copying a parameter 569 block from the old stack to the new one. On function entry 570 *PFRAME_SIZE is the size of the required stack frame--the returned 571 stack must be at least this large. On function exit *PFRAME_SIZE 572 is the amount of space remaining on the allocated stack. OLD_STACK 573 points at the parameters the old stack (really the current one 574 while this function is running). OLD_STACK is saved so that it can 575 be returned by a later call to __generic_releasestack. PARAM_SIZE 576 is the size in bytes of parameters to copy to the new stack. This 577 function returns a pointer to the new stack segment, pointing to 578 the memory after the parameters have been copied. The returned 579 value minus the returned *PFRAME_SIZE (or plus if the stack grows 580 upward) is the first address on the stack which should not be used. 581 582 This function is running on the old stack and has only a limited 583 amount of stack space available. */ 584 585void * 586__generic_morestack (size_t *pframe_size, void *old_stack, size_t param_size) 587{ 588 size_t frame_size = *pframe_size; 589 struct stack_segment *current; 590 struct stack_segment **pp; 591 struct dynamic_allocation_blocks *dynamic; 592 char *from; 593 char *to; 594 void *ret; 595 size_t i; 596 size_t aligned; 597 598 current = __morestack_current_segment; 599 600 pp = current != NULL ? ¤t->next : &__morestack_segments; 601 if (*pp != NULL && (*pp)->size < frame_size) 602 dynamic = __morestack_release_segments (pp, 0); 603 else 604 dynamic = NULL; 605 current = *pp; 606 607 if (current == NULL) 608 { 609 current = allocate_segment (frame_size + param_size); 610 current->prev = __morestack_current_segment; 611 *pp = current; 612 } 613 614 current->old_stack = old_stack; 615 616 __morestack_current_segment = current; 617 618 if (dynamic != NULL) 619 { 620 /* Move the free blocks onto our list. We don't want to call 621 free here, as we are short on stack space. */ 622 current->free_dynamic_allocation = 623 merge_dynamic_blocks (dynamic, current->free_dynamic_allocation); 624 } 625 626 *pframe_size = current->size - param_size; 627 628 /* Align the returned stack to a 32-byte boundary. */ 629 aligned = (param_size + 31) & ~ (size_t) 31; 630 631#ifdef __LIBGCC_STACK_GROWS_DOWNWARD__ 632 { 633 char *bottom = (char *) (current + 1) + current->size; 634 to = bottom - aligned; 635 ret = bottom - aligned; 636 } 637#else 638 to = current + 1; 639 to += aligned - param_size; 640 ret = (char *) (current + 1) + aligned; 641#endif 642 643 /* We don't call memcpy to avoid worrying about the dynamic linker 644 trying to resolve it. */ 645 from = (char *) old_stack; 646 for (i = 0; i < param_size; i++) 647 *to++ = *from++; 648 649 return ret; 650} 651 652/* This function is called by a processor specific function when it is 653 ready to release a stack segment. We don't actually release the 654 stack segment, we just move back to the previous one. The current 655 stack segment will still be available if we need it in 656 __generic_morestack. This returns a pointer to the new stack 657 segment to use, which is the one saved by a previous call to 658 __generic_morestack. The processor specific function is then 659 responsible for actually updating the stack pointer. This sets 660 *PAVAILABLE to the amount of stack space now available. */ 661 662void * 663__generic_releasestack (size_t *pavailable) 664{ 665 struct stack_segment *current; 666 void *old_stack; 667 668 current = __morestack_current_segment; 669 old_stack = current->old_stack; 670 current = current->prev; 671 __morestack_current_segment = current; 672 673 if (current != NULL) 674 { 675#ifdef __LIBGCC_STACK_GROWS_DOWNWARD__ 676 *pavailable = (char *) old_stack - (char *) (current + 1); 677#else 678 *pavailable = (char *) (current + 1) + current->size - (char *) old_stack; 679#endif 680 } 681 else 682 { 683 size_t used; 684 685 /* We have popped back to the original stack. */ 686#ifdef __LIBGCC_STACK_GROWS_DOWNWARD__ 687 if ((char *) old_stack >= (char *) __morestack_initial_sp.sp) 688 used = 0; 689 else 690 used = (char *) __morestack_initial_sp.sp - (char *) old_stack; 691#else 692 if ((char *) old_stack <= (char *) __morestack_initial_sp.sp) 693 used = 0; 694 else 695 used = (char *) old_stack - (char *) __morestack_initial_sp.sp; 696#endif 697 698 if (used > __morestack_initial_sp.len) 699 *pavailable = 0; 700 else 701 *pavailable = __morestack_initial_sp.len - used; 702 } 703 704 return old_stack; 705} 706 707/* Block signals while splitting the stack. This avoids trouble if we 708 try to invoke a signal handler which itself wants to split the 709 stack. */ 710 711extern int pthread_sigmask (int, const sigset_t *, sigset_t *) 712 __attribute__ ((weak)); 713 714void 715__morestack_block_signals (void) 716{ 717 if (__morestack_initial_sp.dont_block_signals) 718 ; 719 else if (pthread_sigmask) 720 pthread_sigmask (SIG_BLOCK, &__morestack_fullmask, 721 &__morestack_initial_sp.mask); 722 else 723 sigprocmask (SIG_BLOCK, &__morestack_fullmask, 724 &__morestack_initial_sp.mask); 725} 726 727/* Unblock signals while splitting the stack. */ 728 729void 730__morestack_unblock_signals (void) 731{ 732 if (__morestack_initial_sp.dont_block_signals) 733 ; 734 else if (pthread_sigmask) 735 pthread_sigmask (SIG_SETMASK, &__morestack_initial_sp.mask, NULL); 736 else 737 sigprocmask (SIG_SETMASK, &__morestack_initial_sp.mask, NULL); 738} 739 740/* This function is called to allocate dynamic stack space, for alloca 741 or a variably sized array. This is a regular function with 742 sufficient stack space, so we just use malloc to allocate the 743 space. We attach the allocated blocks to the current stack 744 segment, so that they will eventually be reused or freed. */ 745 746void * 747__morestack_allocate_stack_space (size_t size) 748{ 749 struct stack_segment *seg, *current; 750 struct dynamic_allocation_blocks *p; 751 752 /* We have to block signals to avoid getting confused if we get 753 interrupted by a signal whose handler itself uses alloca or a 754 variably sized array. */ 755 __morestack_block_signals (); 756 757 /* Since we don't want to call free while we are low on stack space, 758 we may have a list of already allocated blocks waiting to be 759 freed. Release them all, unless we find one that is large 760 enough. We don't look at every block to see if one is large 761 enough, just the first one, because we aren't trying to build a 762 memory allocator here, we're just trying to speed up common 763 cases. */ 764 765 current = __morestack_current_segment; 766 p = NULL; 767 for (seg = __morestack_segments; seg != NULL; seg = seg->next) 768 { 769 p = seg->free_dynamic_allocation; 770 if (p != NULL) 771 { 772 if (p->size >= size) 773 { 774 seg->free_dynamic_allocation = p->next; 775 break; 776 } 777 778 free_dynamic_blocks (p); 779 seg->free_dynamic_allocation = NULL; 780 p = NULL; 781 } 782 } 783 784 if (p == NULL) 785 { 786 /* We need to allocate additional memory. */ 787 p = malloc (sizeof (*p)); 788 if (p == NULL) 789 abort (); 790 p->size = size; 791 p->block = malloc (size); 792 if (p->block == NULL) 793 abort (); 794 } 795 796 /* If we are still on the initial stack, then we have a space leak. 797 FIXME. */ 798 if (current != NULL) 799 { 800 p->next = current->dynamic_allocation; 801 current->dynamic_allocation = p; 802 } 803 804 __morestack_unblock_signals (); 805 806 return p->block; 807} 808 809/* Find the stack segment for STACK and return the amount of space 810 available. This is used when unwinding the stack because of an 811 exception, in order to reset the stack guard correctly. */ 812 813size_t 814__generic_findstack (void *stack) 815{ 816 struct stack_segment *pss; 817 size_t used; 818 819 for (pss = __morestack_current_segment; pss != NULL; pss = pss->prev) 820 { 821 if ((char *) pss < (char *) stack 822 && (char *) pss + pss->size > (char *) stack) 823 { 824 __morestack_current_segment = pss; 825#ifdef __LIBGCC_STACK_GROWS_DOWNWARD__ 826 return (char *) stack - (char *) (pss + 1); 827#else 828 return (char *) (pss + 1) + pss->size - (char *) stack; 829#endif 830 } 831 } 832 833 /* We have popped back to the original stack. */ 834 835 if (__morestack_initial_sp.sp == NULL) 836 return 0; 837 838#ifdef __LIBGCC_STACK_GROWS_DOWNWARD__ 839 if ((char *) stack >= (char *) __morestack_initial_sp.sp) 840 used = 0; 841 else 842 used = (char *) __morestack_initial_sp.sp - (char *) stack; 843#else 844 if ((char *) stack <= (char *) __morestack_initial_sp.sp) 845 used = 0; 846 else 847 used = (char *) stack - (char *) __morestack_initial_sp.sp; 848#endif 849 850 if (used > __morestack_initial_sp.len) 851 return 0; 852 else 853 return __morestack_initial_sp.len - used; 854} 855 856/* This function is called at program startup time to make sure that 857 mmap, munmap, and getpagesize are resolved if linking dynamically. 858 We want to resolve them while we have enough stack for them, rather 859 than calling into the dynamic linker while low on stack space. 860 Similarly, invoke getenv here to check for split-stack related control 861 variables, since doing do as part of the __morestack path can result 862 in unwanted use of SSE/AVX registers (see GCC PR 86213). */ 863 864void 865__morestack_load_mmap (void) 866{ 867 /* Call with bogus values to run faster. We don't care if the call 868 fails. Pass __MORESTACK_CURRENT_SEGMENT to make sure that any 869 TLS accessor function is resolved. */ 870 mmap (__morestack_current_segment, 0, PROT_READ, MAP_ANONYMOUS, -1, 0); 871 mprotect (NULL, 0, 0); 872 munmap (0, static_pagesize); 873 874 /* Initialize these values here, so as to avoid dynamic linker 875 activity as part of a __morestack call. */ 876 static_pagesize = getpagesize(); 877 use_guard_page = getenv ("SPLIT_STACK_GUARD") != 0; 878} 879 880/* This function may be used to iterate over the stack segments. 881 This can be called like this. 882 void *next_segment = NULL; 883 void *next_sp = NULL; 884 void *initial_sp = NULL; 885 void *stack; 886 size_t stack_size; 887 while ((stack = __splitstack_find (next_segment, next_sp, &stack_size, 888 &next_segment, &next_sp, 889 &initial_sp)) != NULL) 890 { 891 // Stack segment starts at stack and is stack_size bytes long. 892 } 893 894 There is no way to iterate over the stack segments of a different 895 thread. However, what is permitted is for one thread to call this 896 with the first two values NULL, to pass next_segment, next_sp, and 897 initial_sp to a different thread, and then to suspend one way or 898 another. A different thread may run the subsequent 899 __morestack_find iterations. Of course, this will only work if the 900 first thread is suspended during the __morestack_find iterations. 901 If not, the second thread will be looking at the stack while it is 902 changing, and anything could happen. 903 904 FIXME: This should be declared in some header file, but where? */ 905 906void * 907__splitstack_find (void *segment_arg, void *sp, size_t *len, 908 void **next_segment, void **next_sp, 909 void **initial_sp) 910{ 911 struct stack_segment *segment; 912 void *ret; 913 char *nsp; 914 915 if (segment_arg == (void *) (uintptr_type) 1) 916 { 917 char *isp = (char *) *initial_sp; 918 919 if (isp == NULL) 920 return NULL; 921 922 *next_segment = (void *) (uintptr_type) 2; 923 *next_sp = NULL; 924#ifdef __LIBGCC_STACK_GROWS_DOWNWARD__ 925 if ((char *) sp >= isp) 926 return NULL; 927 *len = (char *) isp - (char *) sp; 928 return sp; 929#else 930 if ((char *) sp <= (char *) isp) 931 return NULL; 932 *len = (char *) sp - (char *) isp; 933 return (void *) isp; 934#endif 935 } 936 else if (segment_arg == (void *) (uintptr_type) 2) 937 return NULL; 938 else if (segment_arg != NULL) 939 segment = (struct stack_segment *) segment_arg; 940 else 941 { 942 *initial_sp = __morestack_initial_sp.sp; 943 segment = __morestack_current_segment; 944 sp = (void *) &segment; 945 while (1) 946 { 947 if (segment == NULL) 948 return __splitstack_find ((void *) (uintptr_type) 1, sp, len, 949 next_segment, next_sp, initial_sp); 950 if ((char *) sp >= (char *) (segment + 1) 951 && (char *) sp <= (char *) (segment + 1) + segment->size) 952 break; 953 segment = segment->prev; 954 } 955 } 956 957 if (segment->prev == NULL) 958 *next_segment = (void *) (uintptr_type) 1; 959 else 960 *next_segment = segment->prev; 961 962 /* The old_stack value is the address of the function parameters of 963 the function which called __morestack. So if f1 called f2 which 964 called __morestack, the stack looks like this: 965 966 parameters <- old_stack 967 return in f1 968 return in f2 969 registers pushed by __morestack 970 971 The registers pushed by __morestack may not be visible on any 972 other stack, if we are being called by a signal handler 973 immediately after the call to __morestack_unblock_signals. We 974 want to adjust our return value to include those registers. This 975 is target dependent. */ 976 977 nsp = (char *) segment->old_stack; 978 979 if (nsp == NULL) 980 { 981 /* We've reached the top of the stack. */ 982 *next_segment = (void *) (uintptr_type) 2; 983 } 984 else 985 { 986#if defined (__x86_64__) 987 nsp -= 12 * sizeof (void *); 988#elif defined (__i386__) 989 nsp -= 6 * sizeof (void *); 990#elif defined __powerpc64__ 991#elif defined __s390x__ 992 nsp -= 2 * 160; 993#elif defined __s390__ 994 nsp -= 2 * 96; 995#else 996#error "unrecognized target" 997#endif 998 999 *next_sp = (void *) nsp; 1000 } 1001 1002#ifdef __LIBGCC_STACK_GROWS_DOWNWARD__ 1003 *len = (char *) (segment + 1) + segment->size - (char *) sp; 1004 ret = (void *) sp; 1005#else 1006 *len = (char *) sp - (char *) (segment + 1); 1007 ret = (void *) (segment + 1); 1008#endif 1009 1010 return ret; 1011} 1012 1013/* Tell the split stack code whether it has to block signals while 1014 manipulating the stack. This is for programs in which some threads 1015 block all signals. If a thread already blocks signals, there is no 1016 need for the split stack code to block them as well. If NEW is not 1017 NULL, then if *NEW is non-zero signals will be blocked while 1018 splitting the stack, otherwise they will not. If OLD is not NULL, 1019 *OLD will be set to the old value. */ 1020 1021void 1022__splitstack_block_signals (int *new, int *old) 1023{ 1024 if (old != NULL) 1025 *old = __morestack_initial_sp.dont_block_signals ? 0 : 1; 1026 if (new != NULL) 1027 __morestack_initial_sp.dont_block_signals = *new ? 0 : 1; 1028} 1029 1030/* The offsets into the arrays used by __splitstack_getcontext and 1031 __splitstack_setcontext. */ 1032 1033enum __splitstack_context_offsets 1034{ 1035 MORESTACK_SEGMENTS = 0, 1036 CURRENT_SEGMENT = 1, 1037 CURRENT_STACK = 2, 1038 STACK_GUARD = 3, 1039 INITIAL_SP = 4, 1040 INITIAL_SP_LEN = 5, 1041 BLOCK_SIGNALS = 6, 1042 1043 NUMBER_OFFSETS = 10 1044}; 1045 1046/* Get the current split stack context. This may be used for 1047 coroutine switching, similar to getcontext. The argument should 1048 have at least 10 void *pointers for extensibility, although we 1049 don't currently use all of them. This would normally be called 1050 immediately before a call to getcontext or swapcontext or 1051 setjmp. */ 1052 1053void 1054__splitstack_getcontext (void *context[NUMBER_OFFSETS]) 1055{ 1056 memset (context, 0, NUMBER_OFFSETS * sizeof (void *)); 1057 context[MORESTACK_SEGMENTS] = (void *) __morestack_segments; 1058 context[CURRENT_SEGMENT] = (void *) __morestack_current_segment; 1059 context[CURRENT_STACK] = (void *) &context; 1060 context[STACK_GUARD] = __morestack_get_guard (); 1061 context[INITIAL_SP] = (void *) __morestack_initial_sp.sp; 1062 context[INITIAL_SP_LEN] = (void *) (uintptr_type) __morestack_initial_sp.len; 1063 context[BLOCK_SIGNALS] = (void *) __morestack_initial_sp.dont_block_signals; 1064} 1065 1066/* Set the current split stack context. The argument should be a 1067 context previously passed to __splitstack_getcontext. This would 1068 normally be called immediately after a call to getcontext or 1069 swapcontext or setjmp if something jumped to it. */ 1070 1071void 1072__splitstack_setcontext (void *context[NUMBER_OFFSETS]) 1073{ 1074 __morestack_segments = (struct stack_segment *) context[MORESTACK_SEGMENTS]; 1075 __morestack_current_segment = 1076 (struct stack_segment *) context[CURRENT_SEGMENT]; 1077 __morestack_set_guard (context[STACK_GUARD]); 1078 __morestack_initial_sp.sp = context[INITIAL_SP]; 1079 __morestack_initial_sp.len = (size_t) context[INITIAL_SP_LEN]; 1080 __morestack_initial_sp.dont_block_signals = 1081 (uintptr_type) context[BLOCK_SIGNALS]; 1082} 1083 1084/* Create a new split stack context. This will allocate a new stack 1085 segment which may be used by a coroutine. STACK_SIZE is the 1086 minimum size of the new stack. The caller is responsible for 1087 actually setting the stack pointer. This would normally be called 1088 before a call to makecontext, and the returned stack pointer and 1089 size would be used to set the uc_stack field. A function called 1090 via makecontext on a stack created by __splitstack_makecontext may 1091 not return. Note that the returned pointer points to the lowest 1092 address in the stack space, and thus may not be the value to which 1093 to set the stack pointer. */ 1094 1095void * 1096__splitstack_makecontext (size_t stack_size, void *context[NUMBER_OFFSETS], 1097 size_t *size) 1098{ 1099 struct stack_segment *segment; 1100 void *initial_sp; 1101 1102 memset (context, 0, NUMBER_OFFSETS * sizeof (void *)); 1103 segment = allocate_segment (stack_size); 1104 context[MORESTACK_SEGMENTS] = segment; 1105 context[CURRENT_SEGMENT] = segment; 1106#ifdef __LIBGCC_STACK_GROWS_DOWNWARD__ 1107 initial_sp = (void *) ((char *) (segment + 1) + segment->size); 1108#else 1109 initial_sp = (void *) (segment + 1); 1110#endif 1111 context[STACK_GUARD] = __morestack_make_guard (initial_sp, segment->size); 1112 context[INITIAL_SP] = NULL; 1113 context[INITIAL_SP_LEN] = 0; 1114 *size = segment->size; 1115 return (void *) (segment + 1); 1116} 1117 1118/* Given an existing split stack context, reset it back to the start 1119 of the stack. Return the stack pointer and size, appropriate for 1120 use with makecontext. This may be used if a coroutine exits, in 1121 order to reuse the stack segments for a new coroutine. */ 1122 1123void * 1124__splitstack_resetcontext (void *context[10], size_t *size) 1125{ 1126 struct stack_segment *segment; 1127 void *initial_sp; 1128 size_t initial_size; 1129 void *ret; 1130 1131 /* Reset the context assuming that MORESTACK_SEGMENTS, INITIAL_SP 1132 and INITIAL_SP_LEN are correct. */ 1133 1134 segment = context[MORESTACK_SEGMENTS]; 1135 context[CURRENT_SEGMENT] = segment; 1136 context[CURRENT_STACK] = NULL; 1137 if (segment == NULL) 1138 { 1139 initial_sp = context[INITIAL_SP]; 1140 initial_size = (uintptr_type) context[INITIAL_SP_LEN]; 1141 ret = initial_sp; 1142#ifdef __LIBGCC_STACK_GROWS_DOWNWARD__ 1143 ret = (void *) ((char *) ret - initial_size); 1144#endif 1145 } 1146 else 1147 { 1148#ifdef __LIBGCC_STACK_GROWS_DOWNWARD__ 1149 initial_sp = (void *) ((char *) (segment + 1) + segment->size); 1150#else 1151 initial_sp = (void *) (segment + 1); 1152#endif 1153 initial_size = segment->size; 1154 ret = (void *) (segment + 1); 1155 } 1156 context[STACK_GUARD] = __morestack_make_guard (initial_sp, initial_size); 1157 context[BLOCK_SIGNALS] = NULL; 1158 *size = initial_size; 1159 return ret; 1160} 1161 1162/* Release all the memory associated with a splitstack context. This 1163 may be used if a coroutine exits and the associated stack should be 1164 freed. */ 1165 1166void 1167__splitstack_releasecontext (void *context[10]) 1168{ 1169 __morestack_release_segments (((struct stack_segment **) 1170 &context[MORESTACK_SEGMENTS]), 1171 1); 1172} 1173 1174/* Like __splitstack_block_signals, but operating on CONTEXT, rather 1175 than on the current state. */ 1176 1177void 1178__splitstack_block_signals_context (void *context[NUMBER_OFFSETS], int *new, 1179 int *old) 1180{ 1181 if (old != NULL) 1182 *old = ((uintptr_type) context[BLOCK_SIGNALS]) != 0 ? 0 : 1; 1183 if (new != NULL) 1184 context[BLOCK_SIGNALS] = (void *) (uintptr_type) (*new ? 0 : 1); 1185} 1186 1187/* Find the stack segments associated with a split stack context. 1188 This will return the address of the first stack segment and set 1189 *STACK_SIZE to its size. It will set next_segment, next_sp, and 1190 initial_sp which may be passed to __splitstack_find to find the 1191 remaining segments. */ 1192 1193void * 1194__splitstack_find_context (void *context[NUMBER_OFFSETS], size_t *stack_size, 1195 void **next_segment, void **next_sp, 1196 void **initial_sp) 1197{ 1198 void *sp; 1199 struct stack_segment *segment; 1200 1201 *initial_sp = context[INITIAL_SP]; 1202 1203 sp = context[CURRENT_STACK]; 1204 if (sp == NULL) 1205 { 1206 /* Most likely this context was created but was never used. The 1207 value 2 is a code used by __splitstack_find to mean that we 1208 have reached the end of the list of stacks. */ 1209 *next_segment = (void *) (uintptr_type) 2; 1210 *next_sp = NULL; 1211 *initial_sp = NULL; 1212 return NULL; 1213 } 1214 1215 segment = context[CURRENT_SEGMENT]; 1216 if (segment == NULL) 1217 { 1218 /* Most likely this context was saved by a thread which was not 1219 created using __splistack_makecontext and which has never 1220 split the stack. The value 1 is a code used by 1221 __splitstack_find to look at the initial stack. */ 1222 segment = (struct stack_segment *) (uintptr_type) 1; 1223 } 1224 1225 return __splitstack_find (segment, sp, stack_size, next_segment, next_sp, 1226 initial_sp); 1227} 1228 1229#endif /* !defined (inhibit_libc) */ 1230#endif /* not powerpc 32-bit */ 1231