1// SPDX-License-Identifier: GPL-2.0-only 2#include <linux/objtool.h> 3#include <linux/module.h> 4#include <linux/sort.h> 5#include <asm/ptrace.h> 6#include <asm/stacktrace.h> 7#include <asm/unwind.h> 8#include <asm/orc_types.h> 9#include <asm/orc_lookup.h> 10#include <asm/orc_header.h> 11 12ORC_HEADER; 13 14#define orc_warn(fmt, ...) \ 15 printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__) 16 17#define orc_warn_current(args...) \ 18({ \ 19 static bool dumped_before; \ 20 if (state->task == current && !state->error) { \ 21 orc_warn(args); \ 22 if (unwind_debug && !dumped_before) { \ 23 dumped_before = true; \ 24 unwind_dump(state); \ 25 } \ 26 } \ 27}) 28 29extern int __start_orc_unwind_ip[]; 30extern int __stop_orc_unwind_ip[]; 31extern struct orc_entry __start_orc_unwind[]; 32extern struct orc_entry __stop_orc_unwind[]; 33 34static bool orc_init __ro_after_init; 35static bool unwind_debug __ro_after_init; 36static unsigned int lookup_num_blocks __ro_after_init; 37 38static int __init unwind_debug_cmdline(char *str) 39{ 40 unwind_debug = true; 41 42 return 0; 43} 44early_param("unwind_debug", unwind_debug_cmdline); 45 46static void unwind_dump(struct unwind_state *state) 47{ 48 static bool dumped_before; 49 unsigned long word, *sp; 50 struct stack_info stack_info = {0}; 51 unsigned long visit_mask = 0; 52 53 if (dumped_before) 54 return; 55 56 dumped_before = true; 57 58 printk_deferred("unwind stack type:%d next_sp:%p mask:0x%lx graph_idx:%d\n", 59 state->stack_info.type, state->stack_info.next_sp, 60 state->stack_mask, state->graph_idx); 61 62 for (sp = __builtin_frame_address(0); sp; 63 sp = PTR_ALIGN(stack_info.next_sp, sizeof(long))) { 64 if (get_stack_info(sp, state->task, &stack_info, &visit_mask)) 65 break; 66 67 for (; sp < stack_info.end; sp++) { 68 69 word = READ_ONCE_NOCHECK(*sp); 70 71 printk_deferred("%0*lx: %0*lx (%pB)\n", BITS_PER_LONG/4, 72 (unsigned long)sp, BITS_PER_LONG/4, 73 word, (void *)word); 74 } 75 } 76} 77 78static inline unsigned long orc_ip(const int *ip) 79{ 80 return (unsigned long)ip + *ip; 81} 82 83static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table, 84 unsigned int num_entries, unsigned long ip) 85{ 86 int *first = ip_table; 87 int *last = ip_table + num_entries - 1; 88 int *mid, *found = first; 89 90 if (!num_entries) 91 return NULL; 92 93 /* 94 * Do a binary range search to find the rightmost duplicate of a given 95 * starting address. Some entries are section terminators which are 96 * "weak" entries for ensuring there are no gaps. They should be 97 * ignored when they conflict with a real entry. 98 */ 99 while (first <= last) { 100 mid = first + ((last - first) / 2); 101 102 if (orc_ip(mid) <= ip) { 103 found = mid; 104 first = mid + 1; 105 } else 106 last = mid - 1; 107 } 108 109 return u_table + (found - ip_table); 110} 111 112#ifdef CONFIG_MODULES 113static struct orc_entry *orc_module_find(unsigned long ip) 114{ 115 struct module *mod; 116 117 mod = __module_address(ip); 118 if (!mod || !mod->arch.orc_unwind || !mod->arch.orc_unwind_ip) 119 return NULL; 120 return __orc_find(mod->arch.orc_unwind_ip, mod->arch.orc_unwind, 121 mod->arch.num_orcs, ip); 122} 123#else 124static struct orc_entry *orc_module_find(unsigned long ip) 125{ 126 return NULL; 127} 128#endif 129 130#ifdef CONFIG_DYNAMIC_FTRACE 131static struct orc_entry *orc_find(unsigned long ip); 132 133/* 134 * Ftrace dynamic trampolines do not have orc entries of their own. 135 * But they are copies of the ftrace entries that are static and 136 * defined in ftrace_*.S, which do have orc entries. 137 * 138 * If the unwinder comes across a ftrace trampoline, then find the 139 * ftrace function that was used to create it, and use that ftrace 140 * function's orc entry, as the placement of the return code in 141 * the stack will be identical. 142 */ 143static struct orc_entry *orc_ftrace_find(unsigned long ip) 144{ 145 struct ftrace_ops *ops; 146 unsigned long tramp_addr, offset; 147 148 ops = ftrace_ops_trampoline(ip); 149 if (!ops) 150 return NULL; 151 152 /* Set tramp_addr to the start of the code copied by the trampoline */ 153 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) 154 tramp_addr = (unsigned long)ftrace_regs_caller; 155 else 156 tramp_addr = (unsigned long)ftrace_caller; 157 158 /* Now place tramp_addr to the location within the trampoline ip is at */ 159 offset = ip - ops->trampoline; 160 tramp_addr += offset; 161 162 /* Prevent unlikely recursion */ 163 if (ip == tramp_addr) 164 return NULL; 165 166 return orc_find(tramp_addr); 167} 168#else 169static struct orc_entry *orc_ftrace_find(unsigned long ip) 170{ 171 return NULL; 172} 173#endif 174 175/* 176 * If we crash with IP==0, the last successfully executed instruction 177 * was probably an indirect function call with a NULL function pointer, 178 * and we don't have unwind information for NULL. 179 * This hardcoded ORC entry for IP==0 allows us to unwind from a NULL function 180 * pointer into its parent and then continue normally from there. 181 */ 182static struct orc_entry null_orc_entry = { 183 .sp_offset = sizeof(long), 184 .sp_reg = ORC_REG_SP, 185 .bp_reg = ORC_REG_UNDEFINED, 186 .type = ORC_TYPE_CALL 187}; 188 189/* Fake frame pointer entry -- used as a fallback for generated code */ 190static struct orc_entry orc_fp_entry = { 191 .type = ORC_TYPE_CALL, 192 .sp_reg = ORC_REG_BP, 193 .sp_offset = 16, 194 .bp_reg = ORC_REG_PREV_SP, 195 .bp_offset = -16, 196}; 197 198static struct orc_entry *orc_find(unsigned long ip) 199{ 200 static struct orc_entry *orc; 201 202 if (ip == 0) 203 return &null_orc_entry; 204 205 /* For non-init vmlinux addresses, use the fast lookup table: */ 206 if (ip >= LOOKUP_START_IP && ip < LOOKUP_STOP_IP) { 207 unsigned int idx, start, stop; 208 209 idx = (ip - LOOKUP_START_IP) / LOOKUP_BLOCK_SIZE; 210 211 if (unlikely((idx >= lookup_num_blocks-1))) { 212 orc_warn("WARNING: bad lookup idx: idx=%u num=%u ip=%pB\n", 213 idx, lookup_num_blocks, (void *)ip); 214 return NULL; 215 } 216 217 start = orc_lookup[idx]; 218 stop = orc_lookup[idx + 1] + 1; 219 220 if (unlikely((__start_orc_unwind + start >= __stop_orc_unwind) || 221 (__start_orc_unwind + stop > __stop_orc_unwind))) { 222 orc_warn("WARNING: bad lookup value: idx=%u num=%u start=%u stop=%u ip=%pB\n", 223 idx, lookup_num_blocks, start, stop, (void *)ip); 224 return NULL; 225 } 226 227 return __orc_find(__start_orc_unwind_ip + start, 228 __start_orc_unwind + start, stop - start, ip); 229 } 230 231 /* vmlinux .init slow lookup: */ 232 if (is_kernel_inittext(ip)) 233 return __orc_find(__start_orc_unwind_ip, __start_orc_unwind, 234 __stop_orc_unwind_ip - __start_orc_unwind_ip, ip); 235 236 /* Module lookup: */ 237 orc = orc_module_find(ip); 238 if (orc) 239 return orc; 240 241 return orc_ftrace_find(ip); 242} 243 244#ifdef CONFIG_MODULES 245 246static DEFINE_MUTEX(sort_mutex); 247static int *cur_orc_ip_table = __start_orc_unwind_ip; 248static struct orc_entry *cur_orc_table = __start_orc_unwind; 249 250static void orc_sort_swap(void *_a, void *_b, int size) 251{ 252 struct orc_entry *orc_a, *orc_b; 253 int *a = _a, *b = _b, tmp; 254 int delta = _b - _a; 255 256 /* Swap the .orc_unwind_ip entries: */ 257 tmp = *a; 258 *a = *b + delta; 259 *b = tmp - delta; 260 261 /* Swap the corresponding .orc_unwind entries: */ 262 orc_a = cur_orc_table + (a - cur_orc_ip_table); 263 orc_b = cur_orc_table + (b - cur_orc_ip_table); 264 swap(*orc_a, *orc_b); 265} 266 267static int orc_sort_cmp(const void *_a, const void *_b) 268{ 269 struct orc_entry *orc_a; 270 const int *a = _a, *b = _b; 271 unsigned long a_val = orc_ip(a); 272 unsigned long b_val = orc_ip(b); 273 274 if (a_val > b_val) 275 return 1; 276 if (a_val < b_val) 277 return -1; 278 279 /* 280 * The "weak" section terminator entries need to always be first 281 * to ensure the lookup code skips them in favor of real entries. 282 * These terminator entries exist to handle any gaps created by 283 * whitelisted .o files which didn't get objtool generation. 284 */ 285 orc_a = cur_orc_table + (a - cur_orc_ip_table); 286 return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; 287} 288 289void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size, 290 void *_orc, size_t orc_size) 291{ 292 int *orc_ip = _orc_ip; 293 struct orc_entry *orc = _orc; 294 unsigned int num_entries = orc_ip_size / sizeof(int); 295 296 WARN_ON_ONCE(orc_ip_size % sizeof(int) != 0 || 297 orc_size % sizeof(*orc) != 0 || 298 num_entries != orc_size / sizeof(*orc)); 299 300 /* 301 * The 'cur_orc_*' globals allow the orc_sort_swap() callback to 302 * associate an .orc_unwind_ip table entry with its corresponding 303 * .orc_unwind entry so they can both be swapped. 304 */ 305 mutex_lock(&sort_mutex); 306 cur_orc_ip_table = orc_ip; 307 cur_orc_table = orc; 308 sort(orc_ip, num_entries, sizeof(int), orc_sort_cmp, orc_sort_swap); 309 mutex_unlock(&sort_mutex); 310 311 mod->arch.orc_unwind_ip = orc_ip; 312 mod->arch.orc_unwind = orc; 313 mod->arch.num_orcs = num_entries; 314} 315#endif 316 317void __init unwind_init(void) 318{ 319 size_t orc_ip_size = (void *)__stop_orc_unwind_ip - (void *)__start_orc_unwind_ip; 320 size_t orc_size = (void *)__stop_orc_unwind - (void *)__start_orc_unwind; 321 size_t num_entries = orc_ip_size / sizeof(int); 322 struct orc_entry *orc; 323 int i; 324 325 if (!num_entries || orc_ip_size % sizeof(int) != 0 || 326 orc_size % sizeof(struct orc_entry) != 0 || 327 num_entries != orc_size / sizeof(struct orc_entry)) { 328 orc_warn("WARNING: Bad or missing .orc_unwind table. Disabling unwinder.\n"); 329 return; 330 } 331 332 /* 333 * Note, the orc_unwind and orc_unwind_ip tables were already 334 * sorted at build time via the 'sorttable' tool. 335 * It's ready for binary search straight away, no need to sort it. 336 */ 337 338 /* Initialize the fast lookup table: */ 339 lookup_num_blocks = orc_lookup_end - orc_lookup; 340 for (i = 0; i < lookup_num_blocks-1; i++) { 341 orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, 342 num_entries, 343 LOOKUP_START_IP + (LOOKUP_BLOCK_SIZE * i)); 344 if (!orc) { 345 orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n"); 346 return; 347 } 348 349 orc_lookup[i] = orc - __start_orc_unwind; 350 } 351 352 /* Initialize the ending block: */ 353 orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, num_entries, 354 LOOKUP_STOP_IP); 355 if (!orc) { 356 orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n"); 357 return; 358 } 359 orc_lookup[lookup_num_blocks-1] = orc - __start_orc_unwind; 360 361 orc_init = true; 362} 363 364unsigned long unwind_get_return_address(struct unwind_state *state) 365{ 366 if (unwind_done(state)) 367 return 0; 368 369 return __kernel_text_address(state->ip) ? state->ip : 0; 370} 371EXPORT_SYMBOL_GPL(unwind_get_return_address); 372 373unsigned long *unwind_get_return_address_ptr(struct unwind_state *state) 374{ 375 if (unwind_done(state)) 376 return NULL; 377 378 if (state->regs) 379 return &state->regs->ip; 380 381 if (state->sp) 382 return (unsigned long *)state->sp - 1; 383 384 return NULL; 385} 386 387static bool stack_access_ok(struct unwind_state *state, unsigned long _addr, 388 size_t len) 389{ 390 struct stack_info *info = &state->stack_info; 391 void *addr = (void *)_addr; 392 393 if (on_stack(info, addr, len)) 394 return true; 395 396 return !get_stack_info(addr, state->task, info, &state->stack_mask) && 397 on_stack(info, addr, len); 398} 399 400static bool deref_stack_reg(struct unwind_state *state, unsigned long addr, 401 unsigned long *val) 402{ 403 if (!stack_access_ok(state, addr, sizeof(long))) 404 return false; 405 406 *val = READ_ONCE_NOCHECK(*(unsigned long *)addr); 407 return true; 408} 409 410static bool deref_stack_regs(struct unwind_state *state, unsigned long addr, 411 unsigned long *ip, unsigned long *sp) 412{ 413 struct pt_regs *regs = (struct pt_regs *)addr; 414 415 /* x86-32 support will be more complicated due to the ®s->sp hack */ 416 BUILD_BUG_ON(IS_ENABLED(CONFIG_X86_32)); 417 418 if (!stack_access_ok(state, addr, sizeof(struct pt_regs))) 419 return false; 420 421 *ip = READ_ONCE_NOCHECK(regs->ip); 422 *sp = READ_ONCE_NOCHECK(regs->sp); 423 return true; 424} 425 426static bool deref_stack_iret_regs(struct unwind_state *state, unsigned long addr, 427 unsigned long *ip, unsigned long *sp) 428{ 429 struct pt_regs *regs = (void *)addr - IRET_FRAME_OFFSET; 430 431 if (!stack_access_ok(state, addr, IRET_FRAME_SIZE)) 432 return false; 433 434 *ip = READ_ONCE_NOCHECK(regs->ip); 435 *sp = READ_ONCE_NOCHECK(regs->sp); 436 return true; 437} 438 439/* 440 * If state->regs is non-NULL, and points to a full pt_regs, just get the reg 441 * value from state->regs. 442 * 443 * Otherwise, if state->regs just points to IRET regs, and the previous frame 444 * had full regs, it's safe to get the value from the previous regs. This can 445 * happen when early/late IRQ entry code gets interrupted by an NMI. 446 */ 447static bool get_reg(struct unwind_state *state, unsigned int reg_off, 448 unsigned long *val) 449{ 450 unsigned int reg = reg_off/8; 451 452 if (!state->regs) 453 return false; 454 455 if (state->full_regs) { 456 *val = READ_ONCE_NOCHECK(((unsigned long *)state->regs)[reg]); 457 return true; 458 } 459 460 if (state->prev_regs) { 461 *val = READ_ONCE_NOCHECK(((unsigned long *)state->prev_regs)[reg]); 462 return true; 463 } 464 465 return false; 466} 467 468bool unwind_next_frame(struct unwind_state *state) 469{ 470 unsigned long ip_p, sp, tmp, orig_ip = state->ip, prev_sp = state->sp; 471 enum stack_type prev_type = state->stack_info.type; 472 struct orc_entry *orc; 473 bool indirect = false; 474 475 if (unwind_done(state)) 476 return false; 477 478 /* Don't let modules unload while we're reading their ORC data. */ 479 preempt_disable(); 480 481 /* End-of-stack check for user tasks: */ 482 if (state->regs && user_mode(state->regs)) 483 goto the_end; 484 485 /* 486 * Find the orc_entry associated with the text address. 487 * 488 * For a call frame (as opposed to a signal frame), state->ip points to 489 * the instruction after the call. That instruction's stack layout 490 * could be different from the call instruction's layout, for example 491 * if the call was to a noreturn function. So get the ORC data for the 492 * call instruction itself. 493 */ 494 orc = orc_find(state->signal ? state->ip : state->ip - 1); 495 if (!orc) { 496 /* 497 * As a fallback, try to assume this code uses a frame pointer. 498 * This is useful for generated code, like BPF, which ORC 499 * doesn't know about. This is just a guess, so the rest of 500 * the unwind is no longer considered reliable. 501 */ 502 orc = &orc_fp_entry; 503 state->error = true; 504 } else { 505 if (orc->type == ORC_TYPE_UNDEFINED) 506 goto err; 507 508 if (orc->type == ORC_TYPE_END_OF_STACK) 509 goto the_end; 510 } 511 512 state->signal = orc->signal; 513 514 /* Find the previous frame's stack: */ 515 switch (orc->sp_reg) { 516 case ORC_REG_SP: 517 sp = state->sp + orc->sp_offset; 518 break; 519 520 case ORC_REG_BP: 521 sp = state->bp + orc->sp_offset; 522 break; 523 524 case ORC_REG_SP_INDIRECT: 525 sp = state->sp; 526 indirect = true; 527 break; 528 529 case ORC_REG_BP_INDIRECT: 530 sp = state->bp + orc->sp_offset; 531 indirect = true; 532 break; 533 534 case ORC_REG_R10: 535 if (!get_reg(state, offsetof(struct pt_regs, r10), &sp)) { 536 orc_warn_current("missing R10 value at %pB\n", 537 (void *)state->ip); 538 goto err; 539 } 540 break; 541 542 case ORC_REG_R13: 543 if (!get_reg(state, offsetof(struct pt_regs, r13), &sp)) { 544 orc_warn_current("missing R13 value at %pB\n", 545 (void *)state->ip); 546 goto err; 547 } 548 break; 549 550 case ORC_REG_DI: 551 if (!get_reg(state, offsetof(struct pt_regs, di), &sp)) { 552 orc_warn_current("missing RDI value at %pB\n", 553 (void *)state->ip); 554 goto err; 555 } 556 break; 557 558 case ORC_REG_DX: 559 if (!get_reg(state, offsetof(struct pt_regs, dx), &sp)) { 560 orc_warn_current("missing DX value at %pB\n", 561 (void *)state->ip); 562 goto err; 563 } 564 break; 565 566 default: 567 orc_warn("unknown SP base reg %d at %pB\n", 568 orc->sp_reg, (void *)state->ip); 569 goto err; 570 } 571 572 if (indirect) { 573 if (!deref_stack_reg(state, sp, &sp)) 574 goto err; 575 576 if (orc->sp_reg == ORC_REG_SP_INDIRECT) 577 sp += orc->sp_offset; 578 } 579 580 /* Find IP, SP and possibly regs: */ 581 switch (orc->type) { 582 case ORC_TYPE_CALL: 583 ip_p = sp - sizeof(long); 584 585 if (!deref_stack_reg(state, ip_p, &state->ip)) 586 goto err; 587 588 state->ip = unwind_recover_ret_addr(state, state->ip, 589 (unsigned long *)ip_p); 590 state->sp = sp; 591 state->regs = NULL; 592 state->prev_regs = NULL; 593 break; 594 595 case ORC_TYPE_REGS: 596 if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) { 597 orc_warn_current("can't access registers at %pB\n", 598 (void *)orig_ip); 599 goto err; 600 } 601 /* 602 * There is a small chance to interrupt at the entry of 603 * arch_rethook_trampoline() where the ORC info doesn't exist. 604 * That point is right after the RET to arch_rethook_trampoline() 605 * which was modified return address. 606 * At that point, the @addr_p of the unwind_recover_rethook() 607 * (this has to point the address of the stack entry storing 608 * the modified return address) must be "SP - (a stack entry)" 609 * because SP is incremented by the RET. 610 */ 611 state->ip = unwind_recover_rethook(state, state->ip, 612 (unsigned long *)(state->sp - sizeof(long))); 613 state->regs = (struct pt_regs *)sp; 614 state->prev_regs = NULL; 615 state->full_regs = true; 616 break; 617 618 case ORC_TYPE_REGS_PARTIAL: 619 if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) { 620 orc_warn_current("can't access iret registers at %pB\n", 621 (void *)orig_ip); 622 goto err; 623 } 624 /* See ORC_TYPE_REGS case comment. */ 625 state->ip = unwind_recover_rethook(state, state->ip, 626 (unsigned long *)(state->sp - sizeof(long))); 627 628 if (state->full_regs) 629 state->prev_regs = state->regs; 630 state->regs = (void *)sp - IRET_FRAME_OFFSET; 631 state->full_regs = false; 632 break; 633 634 default: 635 orc_warn("unknown .orc_unwind entry type %d at %pB\n", 636 orc->type, (void *)orig_ip); 637 goto err; 638 } 639 640 /* Find BP: */ 641 switch (orc->bp_reg) { 642 case ORC_REG_UNDEFINED: 643 if (get_reg(state, offsetof(struct pt_regs, bp), &tmp)) 644 state->bp = tmp; 645 break; 646 647 case ORC_REG_PREV_SP: 648 if (!deref_stack_reg(state, sp + orc->bp_offset, &state->bp)) 649 goto err; 650 break; 651 652 case ORC_REG_BP: 653 if (!deref_stack_reg(state, state->bp + orc->bp_offset, &state->bp)) 654 goto err; 655 break; 656 657 default: 658 orc_warn("unknown BP base reg %d for ip %pB\n", 659 orc->bp_reg, (void *)orig_ip); 660 goto err; 661 } 662 663 /* Prevent a recursive loop due to bad ORC data: */ 664 if (state->stack_info.type == prev_type && 665 on_stack(&state->stack_info, (void *)state->sp, sizeof(long)) && 666 state->sp <= prev_sp) { 667 orc_warn_current("stack going in the wrong direction? at %pB\n", 668 (void *)orig_ip); 669 goto err; 670 } 671 672 preempt_enable(); 673 return true; 674 675err: 676 state->error = true; 677 678the_end: 679 preempt_enable(); 680 state->stack_info.type = STACK_TYPE_UNKNOWN; 681 return false; 682} 683EXPORT_SYMBOL_GPL(unwind_next_frame); 684 685void __unwind_start(struct unwind_state *state, struct task_struct *task, 686 struct pt_regs *regs, unsigned long *first_frame) 687{ 688 memset(state, 0, sizeof(*state)); 689 state->task = task; 690 691 if (!orc_init) 692 goto err; 693 694 /* 695 * Refuse to unwind the stack of a task while it's executing on another 696 * CPU. This check is racy, but that's ok: the unwinder has other 697 * checks to prevent it from going off the rails. 698 */ 699 if (task_on_another_cpu(task)) 700 goto err; 701 702 if (regs) { 703 if (user_mode(regs)) 704 goto the_end; 705 706 state->ip = regs->ip; 707 state->sp = regs->sp; 708 state->bp = regs->bp; 709 state->regs = regs; 710 state->full_regs = true; 711 state->signal = true; 712 713 } else if (task == current) { 714 asm volatile("lea (%%rip), %0\n\t" 715 "mov %%rsp, %1\n\t" 716 "mov %%rbp, %2\n\t" 717 : "=r" (state->ip), "=r" (state->sp), 718 "=r" (state->bp)); 719 720 } else { 721 struct inactive_task_frame *frame = (void *)task->thread.sp; 722 723 state->sp = task->thread.sp + sizeof(*frame); 724 state->bp = READ_ONCE_NOCHECK(frame->bp); 725 state->ip = READ_ONCE_NOCHECK(frame->ret_addr); 726 state->signal = (void *)state->ip == ret_from_fork; 727 } 728 729 if (get_stack_info((unsigned long *)state->sp, state->task, 730 &state->stack_info, &state->stack_mask)) { 731 /* 732 * We weren't on a valid stack. It's possible that 733 * we overflowed a valid stack into a guard page. 734 * See if the next page up is valid so that we can 735 * generate some kind of backtrace if this happens. 736 */ 737 void *next_page = (void *)PAGE_ALIGN((unsigned long)state->sp); 738 state->error = true; 739 if (get_stack_info(next_page, state->task, &state->stack_info, 740 &state->stack_mask)) 741 return; 742 } 743 744 /* 745 * The caller can provide the address of the first frame directly 746 * (first_frame) or indirectly (regs->sp) to indicate which stack frame 747 * to start unwinding at. Skip ahead until we reach it. 748 */ 749 750 /* When starting from regs, skip the regs frame: */ 751 if (regs) { 752 unwind_next_frame(state); 753 return; 754 } 755 756 /* Otherwise, skip ahead to the user-specified starting frame: */ 757 while (!unwind_done(state) && 758 (!on_stack(&state->stack_info, first_frame, sizeof(long)) || 759 state->sp <= (unsigned long)first_frame)) 760 unwind_next_frame(state); 761 762 return; 763 764err: 765 state->error = true; 766the_end: 767 state->stack_info.type = STACK_TYPE_UNKNOWN; 768} 769EXPORT_SYMBOL_GPL(__unwind_start); 770