1/*- 2 * Copyright (c) 2003 Peter Wemm. 3 * Copyright (c) 1993 The Regents of the University of California. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 4. Neither the name of the University nor the names of its contributors 15 * may be used to endorse or promote products derived from this software 16 * without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * $FreeBSD: stable/10/sys/amd64/include/cpufunc.h 313150 2017-02-03 12:20:44Z kib $ 31 */ 32 33/* 34 * Functions to provide access to special i386 instructions. 35 * This in included in sys/systm.h, and that file should be 36 * used in preference to this. 37 */ 38 39#ifndef _MACHINE_CPUFUNC_H_ 40#define _MACHINE_CPUFUNC_H_ 41 42#ifndef _SYS_CDEFS_H_ 43#error this file needs sys/cdefs.h as a prerequisite 44#endif 45 46struct region_descriptor; 47 48#define readb(va) (*(volatile uint8_t *) (va)) 49#define readw(va) (*(volatile uint16_t *) (va)) 50#define readl(va) (*(volatile uint32_t *) (va)) 51#define readq(va) (*(volatile uint64_t *) (va)) 52 53#define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 54#define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 55#define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 56#define writeq(va, d) (*(volatile uint64_t *) (va) = (d)) 57 58#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 59 60static __inline void 61breakpoint(void) 62{ 63 __asm __volatile("int $3"); 64} 65 66static __inline u_int 67bsfl(u_int mask) 68{ 69 u_int result; 70 71 __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 72 return (result); 73} 74 75static __inline u_long 76bsfq(u_long mask) 77{ 78 u_long result; 79 80 __asm __volatile("bsfq %1,%0" : "=r" (result) : "rm" (mask)); 81 return (result); 82} 83 84static __inline u_int 85bsrl(u_int mask) 86{ 87 u_int result; 88 89 __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 90 return (result); 91} 92 93static __inline u_long 94bsrq(u_long mask) 95{ 96 u_long result; 97 98 __asm __volatile("bsrq %1,%0" : "=r" (result) : "rm" (mask)); 99 return (result); 100} 101 102static __inline void 103clflush(u_long addr) 104{ 105 106 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 107} 108 109static __inline void 110clflushopt(u_long addr) 111{ 112 113 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 114} 115 116static __inline void 117clts(void) 118{ 119 120 __asm __volatile("clts"); 121} 122 123static __inline void 124disable_intr(void) 125{ 126 __asm __volatile("cli" : : : "memory"); 127} 128 129static __inline void 130do_cpuid(u_int ax, u_int *p) 131{ 132 __asm __volatile("cpuid" 133 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 134 : "0" (ax)); 135} 136 137static __inline void 138cpuid_count(u_int ax, u_int cx, u_int *p) 139{ 140 __asm __volatile("cpuid" 141 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 142 : "0" (ax), "c" (cx)); 143} 144 145static __inline void 146enable_intr(void) 147{ 148 __asm __volatile("sti"); 149} 150 151#ifdef _KERNEL 152 153#define HAVE_INLINE_FFS 154#define ffs(x) __builtin_ffs(x) 155 156#define HAVE_INLINE_FFSL 157 158static __inline int 159ffsl(long mask) 160{ 161 return (mask == 0 ? mask : (int)bsfq((u_long)mask) + 1); 162} 163 164#define HAVE_INLINE_FFSLL 165 166static __inline int 167ffsll(long long mask) 168{ 169 return (ffsl((long)mask)); 170} 171 172#define HAVE_INLINE_FLS 173 174static __inline int 175fls(int mask) 176{ 177 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 178} 179 180#define HAVE_INLINE_FLSL 181 182static __inline int 183flsl(long mask) 184{ 185 return (mask == 0 ? mask : (int)bsrq((u_long)mask) + 1); 186} 187 188#define HAVE_INLINE_FLSLL 189 190static __inline int 191flsll(long long mask) 192{ 193 return (flsl((long)mask)); 194} 195 196#endif /* _KERNEL */ 197 198static __inline void 199halt(void) 200{ 201 __asm __volatile("hlt"); 202} 203 204static __inline u_char 205inb(u_int port) 206{ 207 u_char data; 208 209 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 210 return (data); 211} 212 213static __inline u_int 214inl(u_int port) 215{ 216 u_int data; 217 218 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 219 return (data); 220} 221 222static __inline void 223insb(u_int port, void *addr, size_t count) 224{ 225 __asm __volatile("cld; rep; insb" 226 : "+D" (addr), "+c" (count) 227 : "d" (port) 228 : "memory"); 229} 230 231static __inline void 232insw(u_int port, void *addr, size_t count) 233{ 234 __asm __volatile("cld; rep; insw" 235 : "+D" (addr), "+c" (count) 236 : "d" (port) 237 : "memory"); 238} 239 240static __inline void 241insl(u_int port, void *addr, size_t count) 242{ 243 __asm __volatile("cld; rep; insl" 244 : "+D" (addr), "+c" (count) 245 : "d" (port) 246 : "memory"); 247} 248 249static __inline void 250invd(void) 251{ 252 __asm __volatile("invd"); 253} 254 255static __inline u_short 256inw(u_int port) 257{ 258 u_short data; 259 260 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 261 return (data); 262} 263 264static __inline void 265outb(u_int port, u_char data) 266{ 267 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 268} 269 270static __inline void 271outl(u_int port, u_int data) 272{ 273 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 274} 275 276static __inline void 277outsb(u_int port, const void *addr, size_t count) 278{ 279 __asm __volatile("cld; rep; outsb" 280 : "+S" (addr), "+c" (count) 281 : "d" (port)); 282} 283 284static __inline void 285outsw(u_int port, const void *addr, size_t count) 286{ 287 __asm __volatile("cld; rep; outsw" 288 : "+S" (addr), "+c" (count) 289 : "d" (port)); 290} 291 292static __inline void 293outsl(u_int port, const void *addr, size_t count) 294{ 295 __asm __volatile("cld; rep; outsl" 296 : "+S" (addr), "+c" (count) 297 : "d" (port)); 298} 299 300static __inline void 301outw(u_int port, u_short data) 302{ 303 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 304} 305 306static __inline u_long 307popcntq(u_long mask) 308{ 309 u_long result; 310 311 __asm __volatile("popcntq %1,%0" : "=r" (result) : "rm" (mask)); 312 return (result); 313} 314 315static __inline void 316lfence(void) 317{ 318 319 __asm __volatile("lfence" : : : "memory"); 320} 321 322static __inline void 323mfence(void) 324{ 325 326 __asm __volatile("mfence" : : : "memory"); 327} 328 329static __inline void 330sfence(void) 331{ 332 333 __asm __volatile("sfence" : : : "memory"); 334} 335 336static __inline void 337ia32_pause(void) 338{ 339 __asm __volatile("pause"); 340} 341 342static __inline u_long 343read_rflags(void) 344{ 345 u_long rf; 346 347 __asm __volatile("pushfq; popq %0" : "=r" (rf)); 348 return (rf); 349} 350 351static __inline uint64_t 352rdmsr(u_int msr) 353{ 354 uint32_t low, high; 355 356 __asm __volatile("rdmsr" : "=a" (low), "=d" (high) : "c" (msr)); 357 return (low | ((uint64_t)high << 32)); 358} 359 360static __inline uint64_t 361rdpmc(u_int pmc) 362{ 363 uint32_t low, high; 364 365 __asm __volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (pmc)); 366 return (low | ((uint64_t)high << 32)); 367} 368 369static __inline uint64_t 370rdtsc(void) 371{ 372 uint32_t low, high; 373 374 __asm __volatile("rdtsc" : "=a" (low), "=d" (high)); 375 return (low | ((uint64_t)high << 32)); 376} 377 378static __inline uint32_t 379rdtsc32(void) 380{ 381 uint32_t rv; 382 383 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 384 return (rv); 385} 386 387static __inline void 388wbinvd(void) 389{ 390 __asm __volatile("wbinvd"); 391} 392 393static __inline void 394write_rflags(u_long rf) 395{ 396 __asm __volatile("pushq %0; popfq" : : "r" (rf)); 397} 398 399static __inline void 400wrmsr(u_int msr, uint64_t newval) 401{ 402 uint32_t low, high; 403 404 low = newval; 405 high = newval >> 32; 406 __asm __volatile("wrmsr" : : "a" (low), "d" (high), "c" (msr)); 407} 408 409static __inline void 410load_cr0(u_long data) 411{ 412 413 __asm __volatile("movq %0,%%cr0" : : "r" (data)); 414} 415 416static __inline u_long 417rcr0(void) 418{ 419 u_long data; 420 421 __asm __volatile("movq %%cr0,%0" : "=r" (data)); 422 return (data); 423} 424 425static __inline u_long 426rcr2(void) 427{ 428 u_long data; 429 430 __asm __volatile("movq %%cr2,%0" : "=r" (data)); 431 return (data); 432} 433 434static __inline void 435load_cr3(u_long data) 436{ 437 438 __asm __volatile("movq %0,%%cr3" : : "r" (data) : "memory"); 439} 440 441static __inline u_long 442rcr3(void) 443{ 444 u_long data; 445 446 __asm __volatile("movq %%cr3,%0" : "=r" (data)); 447 return (data); 448} 449 450static __inline void 451load_cr4(u_long data) 452{ 453 __asm __volatile("movq %0,%%cr4" : : "r" (data)); 454} 455 456static __inline u_long 457rcr4(void) 458{ 459 u_long data; 460 461 __asm __volatile("movq %%cr4,%0" : "=r" (data)); 462 return (data); 463} 464 465static __inline u_long 466rxcr(u_int reg) 467{ 468 u_int low, high; 469 470 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 471 return (low | ((uint64_t)high << 32)); 472} 473 474static __inline void 475load_xcr(u_int reg, u_long val) 476{ 477 u_int low, high; 478 479 low = val; 480 high = val >> 32; 481 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 482} 483 484/* 485 * Global TLB flush (except for thise for pages marked PG_G) 486 */ 487static __inline void 488invltlb(void) 489{ 490 491 load_cr3(rcr3()); 492} 493 494#ifndef CR4_PGE 495#define CR4_PGE 0x00000080 /* Page global enable */ 496#endif 497 498/* 499 * Perform the guaranteed invalidation of all TLB entries. This 500 * includes the global entries, and entries in all PCIDs, not only the 501 * current context. The function works both on non-PCID CPUs and CPUs 502 * with the PCID turned off or on. See IA-32 SDM Vol. 3a 4.10.4.1 503 * Operations that Invalidate TLBs and Paging-Structure Caches. 504 */ 505static __inline void 506invltlb_globpcid(void) 507{ 508 uint64_t cr4; 509 510 cr4 = rcr4(); 511 load_cr4(cr4 & ~CR4_PGE); 512 /* 513 * Although preemption at this point could be detrimental to 514 * performance, it would not lead to an error. PG_G is simply 515 * ignored if CR4.PGE is clear. Moreover, in case this block 516 * is re-entered, the load_cr4() either above or below will 517 * modify CR4.PGE flushing the TLB. 518 */ 519 load_cr4(cr4 | CR4_PGE); 520} 521 522/* 523 * TLB flush for an individual page (even if it has PG_G). 524 * Only works on 486+ CPUs (i386 does not have PG_G). 525 */ 526static __inline void 527invlpg(u_long addr) 528{ 529 530 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 531} 532 533#define INVPCID_ADDR 0 534#define INVPCID_CTX 1 535#define INVPCID_CTXGLOB 2 536#define INVPCID_ALLCTX 3 537 538struct invpcid_descr { 539 uint64_t pcid:12 __packed; 540 uint64_t pad:52 __packed; 541 uint64_t addr; 542} __packed; 543 544static __inline void 545invpcid(struct invpcid_descr *d, int type) 546{ 547 548 /* invpcid (%rdx),%rax */ 549 __asm __volatile(".byte 0x66,0x0f,0x38,0x82,0x02" 550 : : "d" (d), "a" ((u_long)type) : "memory"); 551} 552 553static __inline u_short 554rfs(void) 555{ 556 u_short sel; 557 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 558 return (sel); 559} 560 561static __inline u_short 562rgs(void) 563{ 564 u_short sel; 565 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 566 return (sel); 567} 568 569static __inline u_short 570rss(void) 571{ 572 u_short sel; 573 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 574 return (sel); 575} 576 577static __inline void 578load_ds(u_short sel) 579{ 580 __asm __volatile("movw %0,%%ds" : : "rm" (sel)); 581} 582 583static __inline void 584load_es(u_short sel) 585{ 586 __asm __volatile("movw %0,%%es" : : "rm" (sel)); 587} 588 589static __inline void 590cpu_monitor(const void *addr, u_long extensions, u_int hints) 591{ 592 593 __asm __volatile("monitor" 594 : : "a" (addr), "c" (extensions), "d" (hints)); 595} 596 597static __inline void 598cpu_mwait(u_long extensions, u_int hints) 599{ 600 601 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 602} 603 604#ifdef _KERNEL 605/* This is defined in <machine/specialreg.h> but is too painful to get to */ 606#ifndef MSR_FSBASE 607#define MSR_FSBASE 0xc0000100 608#endif 609static __inline void 610load_fs(u_short sel) 611{ 612 /* Preserve the fsbase value across the selector load */ 613 __asm __volatile("rdmsr; movw %0,%%fs; wrmsr" 614 : : "rm" (sel), "c" (MSR_FSBASE) : "eax", "edx"); 615} 616 617#ifndef MSR_GSBASE 618#define MSR_GSBASE 0xc0000101 619#endif 620static __inline void 621load_gs(u_short sel) 622{ 623 /* 624 * Preserve the gsbase value across the selector load. 625 * Note that we have to disable interrupts because the gsbase 626 * being trashed happens to be the kernel gsbase at the time. 627 */ 628 __asm __volatile("pushfq; cli; rdmsr; movw %0,%%gs; wrmsr; popfq" 629 : : "rm" (sel), "c" (MSR_GSBASE) : "eax", "edx"); 630} 631#else 632/* Usable by userland */ 633static __inline void 634load_fs(u_short sel) 635{ 636 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 637} 638 639static __inline void 640load_gs(u_short sel) 641{ 642 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 643} 644#endif 645 646static __inline void 647lidt(struct region_descriptor *addr) 648{ 649 __asm __volatile("lidt (%0)" : : "r" (addr)); 650} 651 652static __inline void 653lldt(u_short sel) 654{ 655 __asm __volatile("lldt %0" : : "r" (sel)); 656} 657 658static __inline void 659ltr(u_short sel) 660{ 661 __asm __volatile("ltr %0" : : "r" (sel)); 662} 663 664static __inline uint64_t 665rdr0(void) 666{ 667 uint64_t data; 668 __asm __volatile("movq %%dr0,%0" : "=r" (data)); 669 return (data); 670} 671 672static __inline void 673load_dr0(uint64_t dr0) 674{ 675 __asm __volatile("movq %0,%%dr0" : : "r" (dr0)); 676} 677 678static __inline uint64_t 679rdr1(void) 680{ 681 uint64_t data; 682 __asm __volatile("movq %%dr1,%0" : "=r" (data)); 683 return (data); 684} 685 686static __inline void 687load_dr1(uint64_t dr1) 688{ 689 __asm __volatile("movq %0,%%dr1" : : "r" (dr1)); 690} 691 692static __inline uint64_t 693rdr2(void) 694{ 695 uint64_t data; 696 __asm __volatile("movq %%dr2,%0" : "=r" (data)); 697 return (data); 698} 699 700static __inline void 701load_dr2(uint64_t dr2) 702{ 703 __asm __volatile("movq %0,%%dr2" : : "r" (dr2)); 704} 705 706static __inline uint64_t 707rdr3(void) 708{ 709 uint64_t data; 710 __asm __volatile("movq %%dr3,%0" : "=r" (data)); 711 return (data); 712} 713 714static __inline void 715load_dr3(uint64_t dr3) 716{ 717 __asm __volatile("movq %0,%%dr3" : : "r" (dr3)); 718} 719 720static __inline uint64_t 721rdr4(void) 722{ 723 uint64_t data; 724 __asm __volatile("movq %%dr4,%0" : "=r" (data)); 725 return (data); 726} 727 728static __inline void 729load_dr4(uint64_t dr4) 730{ 731 __asm __volatile("movq %0,%%dr4" : : "r" (dr4)); 732} 733 734static __inline uint64_t 735rdr5(void) 736{ 737 uint64_t data; 738 __asm __volatile("movq %%dr5,%0" : "=r" (data)); 739 return (data); 740} 741 742static __inline void 743load_dr5(uint64_t dr5) 744{ 745 __asm __volatile("movq %0,%%dr5" : : "r" (dr5)); 746} 747 748static __inline uint64_t 749rdr6(void) 750{ 751 uint64_t data; 752 __asm __volatile("movq %%dr6,%0" : "=r" (data)); 753 return (data); 754} 755 756static __inline void 757load_dr6(uint64_t dr6) 758{ 759 __asm __volatile("movq %0,%%dr6" : : "r" (dr6)); 760} 761 762static __inline uint64_t 763rdr7(void) 764{ 765 uint64_t data; 766 __asm __volatile("movq %%dr7,%0" : "=r" (data)); 767 return (data); 768} 769 770static __inline void 771load_dr7(uint64_t dr7) 772{ 773 __asm __volatile("movq %0,%%dr7" : : "r" (dr7)); 774} 775 776static __inline register_t 777intr_disable(void) 778{ 779 register_t rflags; 780 781 rflags = read_rflags(); 782 disable_intr(); 783 return (rflags); 784} 785 786static __inline void 787intr_restore(register_t rflags) 788{ 789 write_rflags(rflags); 790} 791 792#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 793 794int breakpoint(void); 795u_int bsfl(u_int mask); 796u_int bsrl(u_int mask); 797void clflush(u_long addr); 798void clts(void); 799void cpuid_count(u_int ax, u_int cx, u_int *p); 800void disable_intr(void); 801void do_cpuid(u_int ax, u_int *p); 802void enable_intr(void); 803void halt(void); 804void ia32_pause(void); 805u_char inb(u_int port); 806u_int inl(u_int port); 807void insb(u_int port, void *addr, size_t count); 808void insl(u_int port, void *addr, size_t count); 809void insw(u_int port, void *addr, size_t count); 810register_t intr_disable(void); 811void intr_restore(register_t rf); 812void invd(void); 813void invlpg(u_int addr); 814void invltlb(void); 815u_short inw(u_int port); 816void lidt(struct region_descriptor *addr); 817void lldt(u_short sel); 818void load_cr0(u_long cr0); 819void load_cr3(u_long cr3); 820void load_cr4(u_long cr4); 821void load_dr0(uint64_t dr0); 822void load_dr1(uint64_t dr1); 823void load_dr2(uint64_t dr2); 824void load_dr3(uint64_t dr3); 825void load_dr4(uint64_t dr4); 826void load_dr5(uint64_t dr5); 827void load_dr6(uint64_t dr6); 828void load_dr7(uint64_t dr7); 829void load_fs(u_short sel); 830void load_gs(u_short sel); 831void ltr(u_short sel); 832void outb(u_int port, u_char data); 833void outl(u_int port, u_int data); 834void outsb(u_int port, const void *addr, size_t count); 835void outsl(u_int port, const void *addr, size_t count); 836void outsw(u_int port, const void *addr, size_t count); 837void outw(u_int port, u_short data); 838u_long rcr0(void); 839u_long rcr2(void); 840u_long rcr3(void); 841u_long rcr4(void); 842uint64_t rdmsr(u_int msr); 843uint64_t rdpmc(u_int pmc); 844uint64_t rdr0(void); 845uint64_t rdr1(void); 846uint64_t rdr2(void); 847uint64_t rdr3(void); 848uint64_t rdr4(void); 849uint64_t rdr5(void); 850uint64_t rdr6(void); 851uint64_t rdr7(void); 852uint64_t rdtsc(void); 853u_long read_rflags(void); 854u_int rfs(void); 855u_int rgs(void); 856void wbinvd(void); 857void write_rflags(u_int rf); 858void wrmsr(u_int msr, uint64_t newval); 859 860#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 861 862void reset_dbregs(void); 863 864#ifdef _KERNEL 865int rdmsr_safe(u_int msr, uint64_t *val); 866int wrmsr_safe(u_int msr, uint64_t newval); 867#endif 868 869#endif /* !_MACHINE_CPUFUNC_H_ */ 870