1#ifndef __ASM_PARAVIRT_H 2#define __ASM_PARAVIRT_H 3/* Various instructions on x86 need to be replaced for 4 * para-virtualization: those hooks are defined here. */ 5 6#ifdef CONFIG_PARAVIRT 7#include <asm/page.h> 8 9/* Bitmask of what can be clobbered: usually at least eax. */ 10#define CLBR_NONE 0x0 11#define CLBR_EAX 0x1 12#define CLBR_ECX 0x2 13#define CLBR_EDX 0x4 14#define CLBR_ANY 0x7 15 16#ifndef __ASSEMBLY__ 17#include <linux/types.h> 18#include <linux/cpumask.h> 19#include <asm/kmap_types.h> 20 21struct page; 22struct thread_struct; 23struct Xgt_desc_struct; 24struct tss_struct; 25struct mm_struct; 26struct desc_struct; 27 28/* Lazy mode for batching updates / context switch */ 29enum paravirt_lazy_mode { 30 PARAVIRT_LAZY_NONE = 0, 31 PARAVIRT_LAZY_MMU = 1, 32 PARAVIRT_LAZY_CPU = 2, 33 PARAVIRT_LAZY_FLUSH = 3, 34}; 35 36struct paravirt_ops 37{ 38 unsigned int kernel_rpl; 39 int shared_kernel_pmd; 40 int paravirt_enabled; 41 const char *name; 42 43 /* 44 * Patch may replace one of the defined code sequences with arbitrary 45 * code, subject to the same register constraints. This generally 46 * means the code is not free to clobber any registers other than EAX. 47 * The patch function should return the number of bytes of code 48 * generated, as we nop pad the rest in generic code. 49 */ 50 unsigned (*patch)(u8 type, u16 clobber, void *firstinsn, unsigned len); 51 52 /* Basic arch-specific setup */ 53 void (*arch_setup)(void); 54 char *(*memory_setup)(void); 55 void (*init_IRQ)(void); 56 void (*time_init)(void); 57 58 /* 59 * Called before/after init_mm pagetable setup. setup_start 60 * may reset %cr3, and may pre-install parts of the pagetable; 61 * pagetable setup is expected to preserve any existing 62 * mapping. 63 */ 64 void (*pagetable_setup_start)(pgd_t *pgd_base); 65 void (*pagetable_setup_done)(pgd_t *pgd_base); 66 67 /* Print a banner to identify the environment */ 68 void (*banner)(void); 69 70 /* Set and set time of day */ 71 unsigned long (*get_wallclock)(void); 72 int (*set_wallclock)(unsigned long); 73 74 /* cpuid emulation, mostly so that caps bits can be disabled */ 75 void (*cpuid)(unsigned int *eax, unsigned int *ebx, 76 unsigned int *ecx, unsigned int *edx); 77 78 /* hooks for various privileged instructions */ 79 unsigned long (*get_debugreg)(int regno); 80 void (*set_debugreg)(int regno, unsigned long value); 81 82 void (*clts)(void); 83 84 unsigned long (*read_cr0)(void); 85 void (*write_cr0)(unsigned long); 86 87 unsigned long (*read_cr2)(void); 88 void (*write_cr2)(unsigned long); 89 90 unsigned long (*read_cr3)(void); 91 void (*write_cr3)(unsigned long); 92 93 unsigned long (*read_cr4_safe)(void); 94 unsigned long (*read_cr4)(void); 95 void (*write_cr4)(unsigned long); 96 97 /* 98 * Get/set interrupt state. save_fl and restore_fl are only 99 * expected to use X86_EFLAGS_IF; all other bits 100 * returned from save_fl are undefined, and may be ignored by 101 * restore_fl. 102 */ 103 unsigned long (*save_fl)(void); 104 void (*restore_fl)(unsigned long); 105 void (*irq_disable)(void); 106 void (*irq_enable)(void); 107 void (*safe_halt)(void); 108 void (*halt)(void); 109 110 void (*wbinvd)(void); 111 112 /* MSR, PMC and TSR operations. 113 err = 0/-EFAULT. wrmsr returns 0/-EFAULT. */ 114 u64 (*read_msr)(unsigned int msr, int *err); 115 int (*write_msr)(unsigned int msr, u64 val); 116 117 u64 (*read_tsc)(void); 118 u64 (*read_pmc)(void); 119 u64 (*get_scheduled_cycles)(void); 120 unsigned long (*get_cpu_khz)(void); 121 122 /* Segment descriptor handling */ 123 void (*load_tr_desc)(void); 124 void (*load_gdt)(const struct Xgt_desc_struct *); 125 void (*load_idt)(const struct Xgt_desc_struct *); 126 void (*store_gdt)(struct Xgt_desc_struct *); 127 void (*store_idt)(struct Xgt_desc_struct *); 128 void (*set_ldt)(const void *desc, unsigned entries); 129 unsigned long (*store_tr)(void); 130 void (*load_tls)(struct thread_struct *t, unsigned int cpu); 131 void (*write_ldt_entry)(struct desc_struct *, 132 int entrynum, u32 low, u32 high); 133 void (*write_gdt_entry)(struct desc_struct *, 134 int entrynum, u32 low, u32 high); 135 void (*write_idt_entry)(struct desc_struct *, 136 int entrynum, u32 low, u32 high); 137 void (*load_esp0)(struct tss_struct *tss, struct thread_struct *t); 138 139 void (*set_iopl_mask)(unsigned mask); 140 void (*io_delay)(void); 141 142 /* 143 * Hooks for intercepting the creation/use/destruction of an 144 * mm_struct. 145 */ 146 void (*activate_mm)(struct mm_struct *prev, 147 struct mm_struct *next); 148 void (*dup_mmap)(struct mm_struct *oldmm, 149 struct mm_struct *mm); 150 void (*exit_mmap)(struct mm_struct *mm); 151 152#ifdef CONFIG_X86_LOCAL_APIC 153 /* 154 * Direct APIC operations, principally for VMI. Ideally 155 * these shouldn't be in this interface. 156 */ 157 void (*apic_write)(unsigned long reg, unsigned long v); 158 void (*apic_write_atomic)(unsigned long reg, unsigned long v); 159 unsigned long (*apic_read)(unsigned long reg); 160 void (*setup_boot_clock)(void); 161 void (*setup_secondary_clock)(void); 162 163 void (*startup_ipi_hook)(int phys_apicid, 164 unsigned long start_eip, 165 unsigned long start_esp); 166#endif 167 168 /* TLB operations */ 169 void (*flush_tlb_user)(void); 170 void (*flush_tlb_kernel)(void); 171 void (*flush_tlb_single)(unsigned long addr); 172 void (*flush_tlb_others)(const cpumask_t *cpus, struct mm_struct *mm, 173 unsigned long va); 174 175 /* Hooks for allocating/releasing pagetable pages */ 176 void (*alloc_pt)(u32 pfn); 177 void (*alloc_pd)(u32 pfn); 178 void (*alloc_pd_clone)(u32 pfn, u32 clonepfn, u32 start, u32 count); 179 void (*release_pt)(u32 pfn); 180 void (*release_pd)(u32 pfn); 181 182 /* Pagetable manipulation functions */ 183 void (*set_pte)(pte_t *ptep, pte_t pteval); 184 void (*set_pte_at)(struct mm_struct *mm, unsigned long addr, 185 pte_t *ptep, pte_t pteval); 186 void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval); 187 void (*pte_update)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); 188 void (*pte_update_defer)(struct mm_struct *mm, 189 unsigned long addr, pte_t *ptep); 190 191#ifdef CONFIG_HIGHPTE 192 void *(*kmap_atomic_pte)(struct page *page, enum km_type type); 193#endif 194 195#ifdef CONFIG_X86_PAE 196 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval); 197 void (*set_pte_present)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte); 198 void (*set_pud)(pud_t *pudp, pud_t pudval); 199 void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); 200 void (*pmd_clear)(pmd_t *pmdp); 201 202 unsigned long long (*pte_val)(pte_t); 203 unsigned long long (*pmd_val)(pmd_t); 204 unsigned long long (*pgd_val)(pgd_t); 205 206 pte_t (*make_pte)(unsigned long long pte); 207 pmd_t (*make_pmd)(unsigned long long pmd); 208 pgd_t (*make_pgd)(unsigned long long pgd); 209#else 210 unsigned long (*pte_val)(pte_t); 211 unsigned long (*pgd_val)(pgd_t); 212 213 pte_t (*make_pte)(unsigned long pte); 214 pgd_t (*make_pgd)(unsigned long pgd); 215#endif 216 217 /* Set deferred update mode, used for batching operations. */ 218 void (*set_lazy_mode)(enum paravirt_lazy_mode mode); 219 220 /* These two are jmp to, not actually called. */ 221 void (*irq_enable_sysexit)(void); 222 void (*iret)(void); 223}; 224 225extern struct paravirt_ops paravirt_ops; 226 227#define PARAVIRT_PATCH(x) \ 228 (offsetof(struct paravirt_ops, x) / sizeof(void *)) 229 230#define paravirt_type(type) \ 231 [paravirt_typenum] "i" (PARAVIRT_PATCH(type)) 232#define paravirt_clobber(clobber) \ 233 [paravirt_clobber] "i" (clobber) 234 235/* 236 * Generate some code, and mark it as patchable by the 237 * apply_paravirt() alternate instruction patcher. 238 */ 239#define _paravirt_alt(insn_string, type, clobber) \ 240 "771:\n\t" insn_string "\n" "772:\n" \ 241 ".pushsection .parainstructions,\"a\"\n" \ 242 " .long 771b\n" \ 243 " .byte " type "\n" \ 244 " .byte 772b-771b\n" \ 245 " .short " clobber "\n" \ 246 ".popsection\n" 247 248/* Generate patchable code, with the default asm parameters. */ 249#define paravirt_alt(insn_string) \ 250 _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]") 251 252unsigned paravirt_patch_nop(void); 253unsigned paravirt_patch_ignore(unsigned len); 254unsigned paravirt_patch_call(void *target, u16 tgt_clobbers, 255 void *site, u16 site_clobbers, 256 unsigned len); 257unsigned paravirt_patch_jmp(void *target, void *site, unsigned len); 258unsigned paravirt_patch_default(u8 type, u16 clobbers, void *site, unsigned len); 259 260unsigned paravirt_patch_insns(void *site, unsigned len, 261 const char *start, const char *end); 262 263 264/* 265 * This generates an indirect call based on the operation type number. 266 * The type number, computed in PARAVIRT_PATCH, is derived from the 267 * offset into the paravirt_ops structure, and can therefore be freely 268 * converted back into a structure offset. 269 */ 270#define PARAVIRT_CALL "call *(paravirt_ops+%c[paravirt_typenum]*4);" 271 272/* 273 * These macros are intended to wrap calls into a paravirt_ops 274 * operation, so that they can be later identified and patched at 275 * runtime. 276 * 277 * Normally, a call to a pv_op function is a simple indirect call: 278 * (paravirt_ops.operations)(args...). 279 * 280 * Unfortunately, this is a relatively slow operation for modern CPUs, 281 * because it cannot necessarily determine what the destination 282 * address is. In this case, the address is a runtime constant, so at 283 * the very least we can patch the call to e a simple direct call, or 284 * ideally, patch an inline implementation into the callsite. (Direct 285 * calls are essentially free, because the call and return addresses 286 * are completely predictable.) 287 * 288 * These macros rely on the standard gcc "regparm(3)" calling 289 * convention, in which the first three arguments are placed in %eax, 290 * %edx, %ecx (in that order), and the remaining arguments are placed 291 * on the stack. All caller-save registers (eax,edx,ecx) are expected 292 * to be modified (either clobbered or used for return values). 293 * 294 * The call instruction itself is marked by placing its start address 295 * and size into the .parainstructions section, so that 296 * apply_paravirt() in arch/i386/kernel/alternative.c can do the 297 * appropriate patching under the control of the backend paravirt_ops 298 * implementation. 299 * 300 * Unfortunately there's no way to get gcc to generate the args setup 301 * for the call, and then allow the call itself to be generated by an 302 * inline asm. Because of this, we must do the complete arg setup and 303 * return value handling from within these macros. This is fairly 304 * cumbersome. 305 * 306 * There are 5 sets of PVOP_* macros for dealing with 0-4 arguments. 307 * It could be extended to more arguments, but there would be little 308 * to be gained from that. For each number of arguments, there are 309 * the two VCALL and CALL variants for void and non-void functions. 310 * 311 * When there is a return value, the invoker of the macro must specify 312 * the return type. The macro then uses sizeof() on that type to 313 * determine whether its a 32 or 64 bit value, and places the return 314 * in the right register(s) (just %eax for 32-bit, and %edx:%eax for 315 * 64-bit). 316 * 317 * 64-bit arguments are passed as a pair of adjacent 32-bit arguments 318 * in low,high order. 319 * 320 * Small structures are passed and returned in registers. The macro 321 * calling convention can't directly deal with this, so the wrapper 322 * functions must do this. 323 * 324 * These PVOP_* macros are only defined within this header. This 325 * means that all uses must be wrapped in inline functions. This also 326 * makes sure the incoming and outgoing types are always correct. 327 */ 328#define __PVOP_CALL(rettype, op, pre, post, ...) \ 329 ({ \ 330 rettype __ret; \ 331 unsigned long __eax, __edx, __ecx; \ 332 if (sizeof(rettype) > sizeof(unsigned long)) { \ 333 asm volatile(pre \ 334 paravirt_alt(PARAVIRT_CALL) \ 335 post \ 336 : "=a" (__eax), "=d" (__edx), \ 337 "=c" (__ecx) \ 338 : paravirt_type(op), \ 339 paravirt_clobber(CLBR_ANY), \ 340 ##__VA_ARGS__ \ 341 : "memory", "cc"); \ 342 __ret = (rettype)((((u64)__edx) << 32) | __eax); \ 343 } else { \ 344 asm volatile(pre \ 345 paravirt_alt(PARAVIRT_CALL) \ 346 post \ 347 : "=a" (__eax), "=d" (__edx), \ 348 "=c" (__ecx) \ 349 : paravirt_type(op), \ 350 paravirt_clobber(CLBR_ANY), \ 351 ##__VA_ARGS__ \ 352 : "memory", "cc"); \ 353 __ret = (rettype)__eax; \ 354 } \ 355 __ret; \ 356 }) 357#define __PVOP_VCALL(op, pre, post, ...) \ 358 ({ \ 359 unsigned long __eax, __edx, __ecx; \ 360 asm volatile(pre \ 361 paravirt_alt(PARAVIRT_CALL) \ 362 post \ 363 : "=a" (__eax), "=d" (__edx), "=c" (__ecx) \ 364 : paravirt_type(op), \ 365 paravirt_clobber(CLBR_ANY), \ 366 ##__VA_ARGS__ \ 367 : "memory", "cc"); \ 368 }) 369 370#define PVOP_CALL0(rettype, op) \ 371 __PVOP_CALL(rettype, op, "", "") 372#define PVOP_VCALL0(op) \ 373 __PVOP_VCALL(op, "", "") 374 375#define PVOP_CALL1(rettype, op, arg1) \ 376 __PVOP_CALL(rettype, op, "", "", "0" ((u32)(arg1))) 377#define PVOP_VCALL1(op, arg1) \ 378 __PVOP_VCALL(op, "", "", "0" ((u32)(arg1))) 379 380#define PVOP_CALL2(rettype, op, arg1, arg2) \ 381 __PVOP_CALL(rettype, op, "", "", "0" ((u32)(arg1)), "1" ((u32)(arg2))) 382#define PVOP_VCALL2(op, arg1, arg2) \ 383 __PVOP_VCALL(op, "", "", "0" ((u32)(arg1)), "1" ((u32)(arg2))) 384 385#define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \ 386 __PVOP_CALL(rettype, op, "", "", "0" ((u32)(arg1)), \ 387 "1"((u32)(arg2)), "2"((u32)(arg3))) 388#define PVOP_VCALL3(op, arg1, arg2, arg3) \ 389 __PVOP_VCALL(op, "", "", "0" ((u32)(arg1)), "1"((u32)(arg2)), \ 390 "2"((u32)(arg3))) 391 392#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \ 393 __PVOP_CALL(rettype, op, \ 394 "push %[_arg4];", "lea 4(%%esp),%%esp;", \ 395 "0" ((u32)(arg1)), "1" ((u32)(arg2)), \ 396 "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4))) 397#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \ 398 __PVOP_VCALL(op, \ 399 "push %[_arg4];", "lea 4(%%esp),%%esp;", \ 400 "0" ((u32)(arg1)), "1" ((u32)(arg2)), \ 401 "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4))) 402 403static inline int paravirt_enabled(void) 404{ 405 return paravirt_ops.paravirt_enabled; 406} 407 408static inline void load_esp0(struct tss_struct *tss, 409 struct thread_struct *thread) 410{ 411 PVOP_VCALL2(load_esp0, tss, thread); 412} 413 414#define ARCH_SETUP paravirt_ops.arch_setup(); 415static inline unsigned long get_wallclock(void) 416{ 417 return PVOP_CALL0(unsigned long, get_wallclock); 418} 419 420static inline int set_wallclock(unsigned long nowtime) 421{ 422 return PVOP_CALL1(int, set_wallclock, nowtime); 423} 424 425static inline void (*choose_time_init(void))(void) 426{ 427 return paravirt_ops.time_init; 428} 429 430/* The paravirtualized CPUID instruction. */ 431static inline void __cpuid(unsigned int *eax, unsigned int *ebx, 432 unsigned int *ecx, unsigned int *edx) 433{ 434 PVOP_VCALL4(cpuid, eax, ebx, ecx, edx); 435} 436 437/* 438 * These special macros can be used to get or set a debugging register 439 */ 440static inline unsigned long paravirt_get_debugreg(int reg) 441{ 442 return PVOP_CALL1(unsigned long, get_debugreg, reg); 443} 444#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg) 445static inline void set_debugreg(unsigned long val, int reg) 446{ 447 PVOP_VCALL2(set_debugreg, reg, val); 448} 449 450static inline void clts(void) 451{ 452 PVOP_VCALL0(clts); 453} 454 455static inline unsigned long read_cr0(void) 456{ 457 return PVOP_CALL0(unsigned long, read_cr0); 458} 459 460static inline void write_cr0(unsigned long x) 461{ 462 PVOP_VCALL1(write_cr0, x); 463} 464 465static inline unsigned long read_cr2(void) 466{ 467 return PVOP_CALL0(unsigned long, read_cr2); 468} 469 470static inline void write_cr2(unsigned long x) 471{ 472 PVOP_VCALL1(write_cr2, x); 473} 474 475static inline unsigned long read_cr3(void) 476{ 477 return PVOP_CALL0(unsigned long, read_cr3); 478} 479 480static inline void write_cr3(unsigned long x) 481{ 482 PVOP_VCALL1(write_cr3, x); 483} 484 485static inline unsigned long read_cr4(void) 486{ 487 return PVOP_CALL0(unsigned long, read_cr4); 488} 489static inline unsigned long read_cr4_safe(void) 490{ 491 return PVOP_CALL0(unsigned long, read_cr4_safe); 492} 493 494static inline void write_cr4(unsigned long x) 495{ 496 PVOP_VCALL1(write_cr4, x); 497} 498 499static inline void raw_safe_halt(void) 500{ 501 PVOP_VCALL0(safe_halt); 502} 503 504static inline void halt(void) 505{ 506 PVOP_VCALL0(safe_halt); 507} 508 509static inline void wbinvd(void) 510{ 511 PVOP_VCALL0(wbinvd); 512} 513 514#define get_kernel_rpl() (paravirt_ops.kernel_rpl) 515 516static inline u64 paravirt_read_msr(unsigned msr, int *err) 517{ 518 return PVOP_CALL2(u64, read_msr, msr, err); 519} 520static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high) 521{ 522 return PVOP_CALL3(int, write_msr, msr, low, high); 523} 524 525/* These should all do BUG_ON(_err), but our headers are too tangled. */ 526#define rdmsr(msr,val1,val2) do { \ 527 int _err; \ 528 u64 _l = paravirt_read_msr(msr, &_err); \ 529 val1 = (u32)_l; \ 530 val2 = _l >> 32; \ 531} while(0) 532 533#define wrmsr(msr,val1,val2) do { \ 534 paravirt_write_msr(msr, val1, val2); \ 535} while(0) 536 537#define rdmsrl(msr,val) do { \ 538 int _err; \ 539 val = paravirt_read_msr(msr, &_err); \ 540} while(0) 541 542#define wrmsrl(msr,val) wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32) 543#define wrmsr_safe(msr,a,b) paravirt_write_msr(msr, a, b) 544 545/* rdmsr with exception handling */ 546#define rdmsr_safe(msr,a,b) ({ \ 547 int _err; \ 548 u64 _l = paravirt_read_msr(msr, &_err); \ 549 (*a) = (u32)_l; \ 550 (*b) = _l >> 32; \ 551 _err; }) 552 553 554static inline u64 paravirt_read_tsc(void) 555{ 556 return PVOP_CALL0(u64, read_tsc); 557} 558 559#define rdtscl(low) do { \ 560 u64 _l = paravirt_read_tsc(); \ 561 low = (int)_l; \ 562} while(0) 563 564#define rdtscll(val) (val = paravirt_read_tsc()) 565 566#define get_scheduled_cycles(val) (val = paravirt_ops.get_scheduled_cycles()) 567#define calculate_cpu_khz() (paravirt_ops.get_cpu_khz()) 568 569#define write_tsc(val1,val2) wrmsr(0x10, val1, val2) 570 571static inline unsigned long long paravirt_read_pmc(int counter) 572{ 573 return PVOP_CALL1(u64, read_pmc, counter); 574} 575 576#define rdpmc(counter,low,high) do { \ 577 u64 _l = paravirt_read_pmc(counter); \ 578 low = (u32)_l; \ 579 high = _l >> 32; \ 580} while(0) 581 582static inline void load_TR_desc(void) 583{ 584 PVOP_VCALL0(load_tr_desc); 585} 586static inline void load_gdt(const struct Xgt_desc_struct *dtr) 587{ 588 PVOP_VCALL1(load_gdt, dtr); 589} 590static inline void load_idt(const struct Xgt_desc_struct *dtr) 591{ 592 PVOP_VCALL1(load_idt, dtr); 593} 594static inline void set_ldt(const void *addr, unsigned entries) 595{ 596 PVOP_VCALL2(set_ldt, addr, entries); 597} 598static inline void store_gdt(struct Xgt_desc_struct *dtr) 599{ 600 PVOP_VCALL1(store_gdt, dtr); 601} 602static inline void store_idt(struct Xgt_desc_struct *dtr) 603{ 604 PVOP_VCALL1(store_idt, dtr); 605} 606static inline unsigned long paravirt_store_tr(void) 607{ 608 return PVOP_CALL0(unsigned long, store_tr); 609} 610#define store_tr(tr) ((tr) = paravirt_store_tr()) 611static inline void load_TLS(struct thread_struct *t, unsigned cpu) 612{ 613 PVOP_VCALL2(load_tls, t, cpu); 614} 615static inline void write_ldt_entry(void *dt, int entry, u32 low, u32 high) 616{ 617 PVOP_VCALL4(write_ldt_entry, dt, entry, low, high); 618} 619static inline void write_gdt_entry(void *dt, int entry, u32 low, u32 high) 620{ 621 PVOP_VCALL4(write_gdt_entry, dt, entry, low, high); 622} 623static inline void write_idt_entry(void *dt, int entry, u32 low, u32 high) 624{ 625 PVOP_VCALL4(write_idt_entry, dt, entry, low, high); 626} 627static inline void set_iopl_mask(unsigned mask) 628{ 629 PVOP_VCALL1(set_iopl_mask, mask); 630} 631 632/* The paravirtualized I/O functions */ 633static inline void slow_down_io(void) { 634 paravirt_ops.io_delay(); 635#ifdef REALLY_SLOW_IO 636 paravirt_ops.io_delay(); 637 paravirt_ops.io_delay(); 638 paravirt_ops.io_delay(); 639#endif 640} 641 642#ifdef CONFIG_X86_LOCAL_APIC 643/* 644 * Basic functions accessing APICs. 645 */ 646static inline void apic_write(unsigned long reg, unsigned long v) 647{ 648 PVOP_VCALL2(apic_write, reg, v); 649} 650 651static inline void apic_write_atomic(unsigned long reg, unsigned long v) 652{ 653 PVOP_VCALL2(apic_write_atomic, reg, v); 654} 655 656static inline unsigned long apic_read(unsigned long reg) 657{ 658 return PVOP_CALL1(unsigned long, apic_read, reg); 659} 660 661static inline void setup_boot_clock(void) 662{ 663 PVOP_VCALL0(setup_boot_clock); 664} 665 666static inline void setup_secondary_clock(void) 667{ 668 PVOP_VCALL0(setup_secondary_clock); 669} 670#endif 671 672static inline void paravirt_pagetable_setup_start(pgd_t *base) 673{ 674 if (paravirt_ops.pagetable_setup_start) 675 (*paravirt_ops.pagetable_setup_start)(base); 676} 677 678static inline void paravirt_pagetable_setup_done(pgd_t *base) 679{ 680 if (paravirt_ops.pagetable_setup_done) 681 (*paravirt_ops.pagetable_setup_done)(base); 682} 683 684#ifdef CONFIG_SMP 685static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip, 686 unsigned long start_esp) 687{ 688 PVOP_VCALL3(startup_ipi_hook, phys_apicid, start_eip, start_esp); 689} 690#endif 691 692static inline void paravirt_activate_mm(struct mm_struct *prev, 693 struct mm_struct *next) 694{ 695 PVOP_VCALL2(activate_mm, prev, next); 696} 697 698static inline void arch_dup_mmap(struct mm_struct *oldmm, 699 struct mm_struct *mm) 700{ 701 PVOP_VCALL2(dup_mmap, oldmm, mm); 702} 703 704static inline void arch_exit_mmap(struct mm_struct *mm) 705{ 706 PVOP_VCALL1(exit_mmap, mm); 707} 708 709static inline void __flush_tlb(void) 710{ 711 PVOP_VCALL0(flush_tlb_user); 712} 713static inline void __flush_tlb_global(void) 714{ 715 PVOP_VCALL0(flush_tlb_kernel); 716} 717static inline void __flush_tlb_single(unsigned long addr) 718{ 719 PVOP_VCALL1(flush_tlb_single, addr); 720} 721 722static inline void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, 723 unsigned long va) 724{ 725 PVOP_VCALL3(flush_tlb_others, &cpumask, mm, va); 726} 727 728static inline void paravirt_alloc_pt(unsigned pfn) 729{ 730 PVOP_VCALL1(alloc_pt, pfn); 731} 732static inline void paravirt_release_pt(unsigned pfn) 733{ 734 PVOP_VCALL1(release_pt, pfn); 735} 736 737static inline void paravirt_alloc_pd(unsigned pfn) 738{ 739 PVOP_VCALL1(alloc_pd, pfn); 740} 741 742static inline void paravirt_alloc_pd_clone(unsigned pfn, unsigned clonepfn, 743 unsigned start, unsigned count) 744{ 745 PVOP_VCALL4(alloc_pd_clone, pfn, clonepfn, start, count); 746} 747static inline void paravirt_release_pd(unsigned pfn) 748{ 749 PVOP_VCALL1(release_pd, pfn); 750} 751 752#ifdef CONFIG_HIGHPTE 753static inline void *kmap_atomic_pte(struct page *page, enum km_type type) 754{ 755 unsigned long ret; 756 ret = PVOP_CALL2(unsigned long, kmap_atomic_pte, page, type); 757 return (void *)ret; 758} 759#endif 760 761static inline void pte_update(struct mm_struct *mm, unsigned long addr, 762 pte_t *ptep) 763{ 764 PVOP_VCALL3(pte_update, mm, addr, ptep); 765} 766 767static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr, 768 pte_t *ptep) 769{ 770 PVOP_VCALL3(pte_update_defer, mm, addr, ptep); 771} 772 773#ifdef CONFIG_X86_PAE 774static inline pte_t __pte(unsigned long long val) 775{ 776 unsigned long long ret = PVOP_CALL2(unsigned long long, make_pte, 777 val, val >> 32); 778 return (pte_t) { ret, ret >> 32 }; 779} 780 781static inline pmd_t __pmd(unsigned long long val) 782{ 783 return (pmd_t) { PVOP_CALL2(unsigned long long, make_pmd, val, val >> 32) }; 784} 785 786static inline pgd_t __pgd(unsigned long long val) 787{ 788 return (pgd_t) { PVOP_CALL2(unsigned long long, make_pgd, val, val >> 32) }; 789} 790 791static inline unsigned long long pte_val(pte_t x) 792{ 793 return PVOP_CALL2(unsigned long long, pte_val, x.pte_low, x.pte_high); 794} 795 796static inline unsigned long long pmd_val(pmd_t x) 797{ 798 return PVOP_CALL2(unsigned long long, pmd_val, x.pmd, x.pmd >> 32); 799} 800 801static inline unsigned long long pgd_val(pgd_t x) 802{ 803 return PVOP_CALL2(unsigned long long, pgd_val, x.pgd, x.pgd >> 32); 804} 805 806static inline void set_pte(pte_t *ptep, pte_t pteval) 807{ 808 PVOP_VCALL3(set_pte, ptep, pteval.pte_low, pteval.pte_high); 809} 810 811static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, 812 pte_t *ptep, pte_t pteval) 813{ 814 /* 5 arg words */ 815 paravirt_ops.set_pte_at(mm, addr, ptep, pteval); 816} 817 818static inline void set_pte_atomic(pte_t *ptep, pte_t pteval) 819{ 820 PVOP_VCALL3(set_pte_atomic, ptep, pteval.pte_low, pteval.pte_high); 821} 822 823static inline void set_pte_present(struct mm_struct *mm, unsigned long addr, 824 pte_t *ptep, pte_t pte) 825{ 826 /* 5 arg words */ 827 paravirt_ops.set_pte_present(mm, addr, ptep, pte); 828} 829 830static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) 831{ 832 PVOP_VCALL3(set_pmd, pmdp, pmdval.pmd, pmdval.pmd >> 32); 833} 834 835static inline void set_pud(pud_t *pudp, pud_t pudval) 836{ 837 PVOP_VCALL3(set_pud, pudp, pudval.pgd.pgd, pudval.pgd.pgd >> 32); 838} 839 840static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) 841{ 842 PVOP_VCALL3(pte_clear, mm, addr, ptep); 843} 844 845static inline void pmd_clear(pmd_t *pmdp) 846{ 847 PVOP_VCALL1(pmd_clear, pmdp); 848} 849 850#else /* !CONFIG_X86_PAE */ 851 852static inline pte_t __pte(unsigned long val) 853{ 854 return (pte_t) { PVOP_CALL1(unsigned long, make_pte, val) }; 855} 856 857static inline pgd_t __pgd(unsigned long val) 858{ 859 return (pgd_t) { PVOP_CALL1(unsigned long, make_pgd, val) }; 860} 861 862static inline unsigned long pte_val(pte_t x) 863{ 864 return PVOP_CALL1(unsigned long, pte_val, x.pte_low); 865} 866 867static inline unsigned long pgd_val(pgd_t x) 868{ 869 return PVOP_CALL1(unsigned long, pgd_val, x.pgd); 870} 871 872static inline void set_pte(pte_t *ptep, pte_t pteval) 873{ 874 PVOP_VCALL2(set_pte, ptep, pteval.pte_low); 875} 876 877static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, 878 pte_t *ptep, pte_t pteval) 879{ 880 PVOP_VCALL4(set_pte_at, mm, addr, ptep, pteval.pte_low); 881} 882 883static inline void set_pmd(pmd_t *pmdp, pmd_t pmdval) 884{ 885 PVOP_VCALL2(set_pmd, pmdp, pmdval.pud.pgd.pgd); 886} 887#endif /* CONFIG_X86_PAE */ 888 889#define __HAVE_ARCH_ENTER_LAZY_CPU_MODE 890static inline void arch_enter_lazy_cpu_mode(void) 891{ 892 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_CPU); 893} 894 895static inline void arch_leave_lazy_cpu_mode(void) 896{ 897 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_NONE); 898} 899 900static inline void arch_flush_lazy_cpu_mode(void) 901{ 902 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_FLUSH); 903} 904 905 906#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE 907static inline void arch_enter_lazy_mmu_mode(void) 908{ 909 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_MMU); 910} 911 912static inline void arch_leave_lazy_mmu_mode(void) 913{ 914 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_NONE); 915} 916 917static inline void arch_flush_lazy_mmu_mode(void) 918{ 919 PVOP_VCALL1(set_lazy_mode, PARAVIRT_LAZY_FLUSH); 920} 921 922void _paravirt_nop(void); 923#define paravirt_nop ((void *)_paravirt_nop) 924 925/* These all sit in the .parainstructions section to tell us what to patch. */ 926struct paravirt_patch_site { 927 u8 *instr; /* original instructions */ 928 u8 instrtype; /* type of this instruction */ 929 u8 len; /* length of original instruction */ 930 u16 clobbers; /* what registers you may clobber */ 931}; 932 933extern struct paravirt_patch_site __parainstructions[], 934 __parainstructions_end[]; 935 936static inline unsigned long __raw_local_save_flags(void) 937{ 938 unsigned long f; 939 940 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;" 941 PARAVIRT_CALL 942 "popl %%edx; popl %%ecx") 943 : "=a"(f) 944 : paravirt_type(save_fl), 945 paravirt_clobber(CLBR_EAX) 946 : "memory", "cc"); 947 return f; 948} 949 950static inline void raw_local_irq_restore(unsigned long f) 951{ 952 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;" 953 PARAVIRT_CALL 954 "popl %%edx; popl %%ecx") 955 : "=a"(f) 956 : "0"(f), 957 paravirt_type(restore_fl), 958 paravirt_clobber(CLBR_EAX) 959 : "memory", "cc"); 960} 961 962static inline void raw_local_irq_disable(void) 963{ 964 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;" 965 PARAVIRT_CALL 966 "popl %%edx; popl %%ecx") 967 : 968 : paravirt_type(irq_disable), 969 paravirt_clobber(CLBR_EAX) 970 : "memory", "eax", "cc"); 971} 972 973static inline void raw_local_irq_enable(void) 974{ 975 asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;" 976 PARAVIRT_CALL 977 "popl %%edx; popl %%ecx") 978 : 979 : paravirt_type(irq_enable), 980 paravirt_clobber(CLBR_EAX) 981 : "memory", "eax", "cc"); 982} 983 984static inline unsigned long __raw_local_irq_save(void) 985{ 986 unsigned long f; 987 988 f = __raw_local_save_flags(); 989 raw_local_irq_disable(); 990 return f; 991} 992 993#define CLI_STRING \ 994 _paravirt_alt("pushl %%ecx; pushl %%edx;" \ 995 "call *paravirt_ops+%c[paravirt_cli_type]*4;" \ 996 "popl %%edx; popl %%ecx", \ 997 "%c[paravirt_cli_type]", "%c[paravirt_clobber]") 998 999#define STI_STRING \ 1000 _paravirt_alt("pushl %%ecx; pushl %%edx;" \ 1001 "call *paravirt_ops+%c[paravirt_sti_type]*4;" \ 1002 "popl %%edx; popl %%ecx", \ 1003 "%c[paravirt_sti_type]", "%c[paravirt_clobber]") 1004 1005#define CLI_STI_CLOBBERS , "%eax" 1006#define CLI_STI_INPUT_ARGS \ 1007 , \ 1008 [paravirt_cli_type] "i" (PARAVIRT_PATCH(irq_disable)), \ 1009 [paravirt_sti_type] "i" (PARAVIRT_PATCH(irq_enable)), \ 1010 paravirt_clobber(CLBR_EAX) 1011 1012/* Make sure as little as possible of this mess escapes. */ 1013#undef PARAVIRT_CALL 1014#undef __PVOP_CALL 1015#undef __PVOP_VCALL 1016#undef PVOP_VCALL0 1017#undef PVOP_CALL0 1018#undef PVOP_VCALL1 1019#undef PVOP_CALL1 1020#undef PVOP_VCALL2 1021#undef PVOP_CALL2 1022#undef PVOP_VCALL3 1023#undef PVOP_CALL3 1024#undef PVOP_VCALL4 1025#undef PVOP_CALL4 1026 1027#else /* __ASSEMBLY__ */ 1028 1029#define PARA_PATCH(off) ((off) / 4) 1030 1031#define PARA_SITE(ptype, clobbers, ops) \ 1032771:; \ 1033 ops; \ 1034772:; \ 1035 .pushsection .parainstructions,"a"; \ 1036 .long 771b; \ 1037 .byte ptype; \ 1038 .byte 772b-771b; \ 1039 .short clobbers; \ 1040 .popsection 1041 1042#define INTERRUPT_RETURN \ 1043 PARA_SITE(PARA_PATCH(PARAVIRT_iret), CLBR_NONE, \ 1044 jmp *%cs:paravirt_ops+PARAVIRT_iret) 1045 1046#define DISABLE_INTERRUPTS(clobbers) \ 1047 PARA_SITE(PARA_PATCH(PARAVIRT_irq_disable), clobbers, \ 1048 pushl %eax; pushl %ecx; pushl %edx; \ 1049 call *%cs:paravirt_ops+PARAVIRT_irq_disable; \ 1050 popl %edx; popl %ecx; popl %eax) \ 1051 1052#define ENABLE_INTERRUPTS(clobbers) \ 1053 PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable), clobbers, \ 1054 pushl %eax; pushl %ecx; pushl %edx; \ 1055 call *%cs:paravirt_ops+PARAVIRT_irq_enable; \ 1056 popl %edx; popl %ecx; popl %eax) 1057 1058#define ENABLE_INTERRUPTS_SYSEXIT \ 1059 PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable_sysexit), CLBR_NONE, \ 1060 jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit) 1061 1062#define GET_CR0_INTO_EAX \ 1063 push %ecx; push %edx; \ 1064 call *paravirt_ops+PARAVIRT_read_cr0; \ 1065 pop %edx; pop %ecx 1066 1067#endif /* __ASSEMBLY__ */ 1068#endif /* CONFIG_PARAVIRT */ 1069#endif /* __ASM_PARAVIRT_H */ 1070