x86/kernel/unwind_orc.c

20 	if (state->task == current && !state->error) {			\
24 			unwind_dump(state);				\
46 static void unwind_dump(struct unwind_state *state)
59 			state->stack_info.type, state->stack_info.next_sp,
60 			state->stack_mask, state->graph_idx);
64 		if (get_stack_info(sp, state->task, &stack_info, &visit_mask))
364 unsigned long unwind_get_return_address(struct unwind_state *state)
366 	if (unwind_done(state))
369 	return __kernel_text_address(state->ip) ? state->ip : 0;
373 unsigned long *unwind_get_return_address_ptr(struct unwind_state *state)
375 	if (unwind_done(state))
378 	if (state->regs)
379 		return &state->regs->ip;
381 	if (state->sp)
382 		return (unsigned long *)state->sp - 1;
387 static bool stack_access_ok(struct unwind_state *state, unsigned long _addr,
390 	struct stack_info *info = &state->stack_info;
396 	return !get_stack_info(addr, state->task, info, &state->stack_mask) &&
400 static bool deref_stack_reg(struct unwind_state *state, unsigned long addr,
403 	if (!stack_access_ok(state, addr, sizeof(long)))
410 static bool deref_stack_regs(struct unwind_state *state, unsigned long addr,
418 	if (!stack_access_ok(state, addr, sizeof(struct pt_regs)))
426 static bool deref_stack_iret_regs(struct unwind_state *state, unsigned long addr,
431 	if (!stack_access_ok(state, addr, IRET_FRAME_SIZE))
440  * If state->regs is non-NULL, and points to a full pt_regs, just get the reg
441  * value from state->regs.
443  * Otherwise, if state->regs just points to IRET regs, and the previous frame
447 static bool get_reg(struct unwind_state *state, unsigned int reg_off,
452 	if (!state->regs)
455 	if (state->full_regs) {
456 		*val = READ_ONCE_NOCHECK(((unsigned long *)state->regs)[reg]);
460 	if (state->prev_regs) {
461 		*val = READ_ONCE_NOCHECK(((unsigned long *)state->prev_regs)[reg]);
468 bool unwind_next_frame(struct unwind_state *state)
470 	unsigned long ip_p, sp, tmp, orig_ip = state->ip, prev_sp = state->sp;
471 	enum stack_type prev_type = state->stack_info.type;
475 	if (unwind_done(state))
482 	if (state->regs && user_mode(state->regs))
488 	 * For a call frame (as opposed to a signal frame), state->ip points to
494 	orc = orc_find(state->signal ? state->ip : state->ip - 1);
503 		state->error = true;
512 	state->signal = orc->signal;
517 		sp = state->sp + orc->sp_offset;
521 		sp = state->bp + orc->sp_offset;
525 		sp = state->sp;
530 		sp = state->bp + orc->sp_offset;
535 		if (!get_reg(state, offsetof(struct pt_regs, r10), &sp)) {
537 					 (void *)state->ip);
543 		if (!get_reg(state, offsetof(struct pt_regs, r13), &sp)) {
545 					 (void *)state->ip);
551 		if (!get_reg(state, offsetof(struct pt_regs, di), &sp)) {
553 					 (void *)state->ip);
559 		if (!get_reg(state, offsetof(struct pt_regs, dx), &sp)) {
561 					 (void *)state->ip);
568 			 orc->sp_reg, (void *)state->ip);
573 		if (!deref_stack_reg(state, sp, &sp))
585 		if (!deref_stack_reg(state, ip_p, &state->ip))
588 		state->ip = unwind_recover_ret_addr(state, state->ip,
590 		state->sp = sp;
591 		state->regs = NULL;
592 		state->prev_regs = NULL;
596 		if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) {
611 		state->ip = unwind_recover_rethook(state, state->ip,
612 				(unsigned long *)(state->sp - sizeof(long)));
613 		state->regs = (struct pt_regs *)sp;
614 		state->prev_regs = NULL;
615 		state->full_regs = true;
619 		if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) {
625 		state->ip = unwind_recover_rethook(state, state->ip,
626 				(unsigned long *)(state->sp - sizeof(long)));
628 		if (state->full_regs)
629 			state->prev_regs = state->regs;
630 		state->regs = (void *)sp - IRET_FRAME_OFFSET;
631 		state->full_regs = false;
643 		if (get_reg(state, offsetof(struct pt_regs, bp), &tmp))
644 			state->bp = tmp;
648 		if (!deref_stack_reg(state, sp + orc->bp_offset, &state->bp))
653 		if (!deref_stack_reg(state, state->bp + orc->bp_offset, &state->bp))
664 	if (state->stack_info.type == prev_type &&
665 	    on_stack(&state->stack_info, (void *)state->sp, sizeof(long)) &&
666 	    state->sp <= prev_sp) {
676 	state->error = true;
680 	state->stack_info.type = STACK_TYPE_UNKNOWN;
685 void __unwind_start(struct unwind_state *state, struct task_struct *task,
688 	memset(state, 0, sizeof(*state));
689 	state->task = task;
706 		state->ip = regs->ip;
707 		state->sp = regs->sp;
708 		state->bp = regs->bp;
709 		state->regs = regs;
710 		state->full_regs = true;
711 		state->signal = true;
717 			     : "=r" (state->ip), "=r" (state->sp),
718 			       "=r" (state->bp));
723 		state->sp = task->thread.sp + sizeof(*frame);
724 		state->bp = READ_ONCE_NOCHECK(frame->bp);
725 		state->ip = READ_ONCE_NOCHECK(frame->ret_addr);
726 		state->signal = (void *)state->ip == ret_from_fork;
729 	if (get_stack_info((unsigned long *)state->sp, state->task,
730 			   &state->stack_info, &state->stack_mask)) {
737 		void *next_page = (void *)PAGE_ALIGN((unsigned long)state->sp);
738 		state->error = true;
739 		if (get_stack_info(next_page, state->task, &state->stack_info,
740 				   &state->stack_mask))
752 		unwind_next_frame(state);
757 	while (!unwind_done(state) &&
758 	       (!on_stack(&state->stack_info, first_frame, sizeof(long)) ||
759 			state->sp <= (unsigned long)first_frame))
760 		unwind_next_frame(state);
765 	state->error = true;
767 	state->stack_info.type = STACK_TYPE_UNKNOWN;