1/* 2 * Asm versions of Xen pv-ops, suitable for either direct use or 3 * inlining. The inline versions are the same as the direct-use 4 * versions, with the pre- and post-amble chopped off. 5 * 6 * This code is encoded for size rather than absolute efficiency, with 7 * a view to being able to inline as much as possible. 8 * 9 * We only bother with direct forms (ie, vcpu in pda) of the 10 * operations here; the indirect forms are better handled in C, since 11 * they're generally too large to inline anyway. 12 */ 13 14#include <asm/errno.h> 15#include <asm/percpu.h> 16#include <asm/processor-flags.h> 17#include <asm/segment.h> 18 19#include <xen/interface/xen.h> 20 21#include "xen-asm.h" 22 23ENTRY(xen_adjust_exception_frame) 24 mov 8+0(%rsp), %rcx 25 mov 8+8(%rsp), %r11 26 ret $16 27 28hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 29/* 30 * Xen64 iret frame: 31 * 32 * ss 33 * rsp 34 * rflags 35 * cs 36 * rip <-- standard iret frame 37 * 38 * flags 39 * 40 * rcx } 41 * r11 }<-- pushed by hypercall page 42 * rsp->rax } 43 */ 44ENTRY(xen_iret) 45 pushq $0 461: jmp hypercall_iret 47ENDPATCH(xen_iret) 48RELOC(xen_iret, 1b+1) 49 50/* 51 * sysexit is not used for 64-bit processes, so it's only ever used to 52 * return to 32-bit compat userspace. 53 */ 54ENTRY(xen_sysexit) 55 pushq $__USER32_DS 56 pushq %rcx 57 pushq $X86_EFLAGS_IF 58 pushq $__USER32_CS 59 pushq %rdx 60 61 pushq $0 621: jmp hypercall_iret 63ENDPATCH(xen_sysexit) 64RELOC(xen_sysexit, 1b+1) 65 66ENTRY(xen_sysret64) 67 /* 68 * We're already on the usermode stack at this point, but 69 * still with the kernel gs, so we can easily switch back 70 */ 71 movq %rsp, PER_CPU_VAR(old_rsp) 72 movq PER_CPU_VAR(kernel_stack), %rsp 73 74 pushq $__USER_DS 75 pushq PER_CPU_VAR(old_rsp) 76 pushq %r11 77 pushq $__USER_CS 78 pushq %rcx 79 80 pushq $VGCF_in_syscall 811: jmp hypercall_iret 82ENDPATCH(xen_sysret64) 83RELOC(xen_sysret64, 1b+1) 84 85ENTRY(xen_sysret32) 86 /* 87 * We're already on the usermode stack at this point, but 88 * still with the kernel gs, so we can easily switch back 89 */ 90 movq %rsp, PER_CPU_VAR(old_rsp) 91 movq PER_CPU_VAR(kernel_stack), %rsp 92 93 pushq $__USER32_DS 94 pushq PER_CPU_VAR(old_rsp) 95 pushq %r11 96 pushq $__USER32_CS 97 pushq %rcx 98 99 pushq $0 1001: jmp hypercall_iret 101ENDPATCH(xen_sysret32) 102RELOC(xen_sysret32, 1b+1) 103 104/* 105 * Xen handles syscall callbacks much like ordinary exceptions, which 106 * means we have: 107 * - kernel gs 108 * - kernel rsp 109 * - an iret-like stack frame on the stack (including rcx and r11): 110 * ss 111 * rsp 112 * rflags 113 * cs 114 * rip 115 * r11 116 * rsp->rcx 117 * 118 * In all the entrypoints, we undo all that to make it look like a 119 * CPU-generated syscall/sysenter and jump to the normal entrypoint. 120 */ 121 122.macro undo_xen_syscall 123 mov 0*8(%rsp), %rcx 124 mov 1*8(%rsp), %r11 125 mov 5*8(%rsp), %rsp 126.endm 127 128/* Normal 64-bit system call target */ 129ENTRY(xen_syscall_target) 130 undo_xen_syscall 131 jmp system_call_after_swapgs 132ENDPROC(xen_syscall_target) 133 134#ifdef CONFIG_IA32_EMULATION 135 136/* 32-bit compat syscall target */ 137ENTRY(xen_syscall32_target) 138 undo_xen_syscall 139 jmp ia32_cstar_target 140ENDPROC(xen_syscall32_target) 141 142/* 32-bit compat sysenter target */ 143ENTRY(xen_sysenter_target) 144 undo_xen_syscall 145 jmp ia32_sysenter_target 146ENDPROC(xen_sysenter_target) 147 148#else /* !CONFIG_IA32_EMULATION */ 149 150ENTRY(xen_syscall32_target) 151ENTRY(xen_sysenter_target) 152 lea 16(%rsp), %rsp /* strip %rcx, %r11 */ 153 mov $-ENOSYS, %rax 154 pushq $0 155 jmp hypercall_iret 156ENDPROC(xen_syscall32_target) 157ENDPROC(xen_sysenter_target) 158 159#endif /* CONFIG_IA32_EMULATION */ 160