1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD: stable/10/sys/i386/include/cpufunc.h 313150 2017-02-03 12:20:44Z kib $ 30 */ 31 32/* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38#ifndef _MACHINE_CPUFUNC_H_ 39#define _MACHINE_CPUFUNC_H_ 40 41#ifndef _SYS_CDEFS_H_ 42#error this file needs sys/cdefs.h as a prerequisite 43#endif 44 45#ifdef XEN 46extern void xen_cli(void); 47extern void xen_sti(void); 48extern u_int xen_rcr2(void); 49extern void xen_load_cr3(u_int data); 50extern void xen_tlb_flush(void); 51extern void xen_invlpg(u_int addr); 52extern void write_eflags(u_int eflags); 53extern u_int read_eflags(void); 54#endif 55 56struct region_descriptor; 57 58#define readb(va) (*(volatile uint8_t *) (va)) 59#define readw(va) (*(volatile uint16_t *) (va)) 60#define readl(va) (*(volatile uint32_t *) (va)) 61 62#define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 63#define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 64#define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 65 66#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 67 68static __inline void 69breakpoint(void) 70{ 71 __asm __volatile("int $3"); 72} 73 74static __inline u_int 75bsfl(u_int mask) 76{ 77 u_int result; 78 79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81} 82 83static __inline u_int 84bsrl(u_int mask) 85{ 86 u_int result; 87 88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 89 return (result); 90} 91 92static __inline void 93clflush(u_long addr) 94{ 95 96 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 97} 98 99static __inline void 100clflushopt(u_long addr) 101{ 102 103 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 104} 105 106static __inline void 107clts(void) 108{ 109 110 __asm __volatile("clts"); 111} 112 113static __inline void 114disable_intr(void) 115{ 116#ifdef XEN 117 xen_cli(); 118#else 119 __asm __volatile("cli" : : : "memory"); 120#endif 121} 122 123static __inline void 124do_cpuid(u_int ax, u_int *p) 125{ 126 __asm __volatile("cpuid" 127 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 128 : "0" (ax)); 129} 130 131static __inline void 132cpuid_count(u_int ax, u_int cx, u_int *p) 133{ 134 __asm __volatile("cpuid" 135 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 136 : "0" (ax), "c" (cx)); 137} 138 139static __inline void 140enable_intr(void) 141{ 142#ifdef XEN 143 xen_sti(); 144#else 145 __asm __volatile("sti"); 146#endif 147} 148 149static __inline void 150cpu_monitor(const void *addr, u_long extensions, u_int hints) 151{ 152 153 __asm __volatile("monitor" 154 : : "a" (addr), "c" (extensions), "d" (hints)); 155} 156 157static __inline void 158cpu_mwait(u_long extensions, u_int hints) 159{ 160 161 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 162} 163 164static __inline void 165lfence(void) 166{ 167 168 __asm __volatile("lfence" : : : "memory"); 169} 170 171static __inline void 172mfence(void) 173{ 174 175 __asm __volatile("mfence" : : : "memory"); 176} 177 178static __inline void 179sfence(void) 180{ 181 182 __asm __volatile("sfence" : : : "memory"); 183} 184 185#ifdef _KERNEL 186 187#define HAVE_INLINE_FFS 188 189static __inline int 190ffs(int mask) 191{ 192 /* 193 * Note that gcc-2's builtin ffs would be used if we didn't declare 194 * this inline or turn off the builtin. The builtin is faster but 195 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 196 * versions. 197 */ 198 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 199} 200 201#define HAVE_INLINE_FFSL 202 203static __inline int 204ffsl(long mask) 205{ 206 return (ffs((int)mask)); 207} 208 209#define HAVE_INLINE_FLS 210 211static __inline int 212fls(int mask) 213{ 214 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 215} 216 217#define HAVE_INLINE_FLSL 218 219static __inline int 220flsl(long mask) 221{ 222 return (fls((int)mask)); 223} 224 225#endif /* _KERNEL */ 226 227static __inline void 228halt(void) 229{ 230 __asm __volatile("hlt"); 231} 232 233static __inline u_char 234inb(u_int port) 235{ 236 u_char data; 237 238 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 239 return (data); 240} 241 242static __inline u_int 243inl(u_int port) 244{ 245 u_int data; 246 247 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 248 return (data); 249} 250 251static __inline void 252insb(u_int port, void *addr, size_t count) 253{ 254 __asm __volatile("cld; rep; insb" 255 : "+D" (addr), "+c" (count) 256 : "d" (port) 257 : "memory"); 258} 259 260static __inline void 261insw(u_int port, void *addr, size_t count) 262{ 263 __asm __volatile("cld; rep; insw" 264 : "+D" (addr), "+c" (count) 265 : "d" (port) 266 : "memory"); 267} 268 269static __inline void 270insl(u_int port, void *addr, size_t count) 271{ 272 __asm __volatile("cld; rep; insl" 273 : "+D" (addr), "+c" (count) 274 : "d" (port) 275 : "memory"); 276} 277 278static __inline void 279invd(void) 280{ 281 __asm __volatile("invd"); 282} 283 284static __inline u_short 285inw(u_int port) 286{ 287 u_short data; 288 289 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 290 return (data); 291} 292 293static __inline void 294outb(u_int port, u_char data) 295{ 296 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 297} 298 299static __inline void 300outl(u_int port, u_int data) 301{ 302 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 303} 304 305static __inline void 306outsb(u_int port, const void *addr, size_t count) 307{ 308 __asm __volatile("cld; rep; outsb" 309 : "+S" (addr), "+c" (count) 310 : "d" (port)); 311} 312 313static __inline void 314outsw(u_int port, const void *addr, size_t count) 315{ 316 __asm __volatile("cld; rep; outsw" 317 : "+S" (addr), "+c" (count) 318 : "d" (port)); 319} 320 321static __inline void 322outsl(u_int port, const void *addr, size_t count) 323{ 324 __asm __volatile("cld; rep; outsl" 325 : "+S" (addr), "+c" (count) 326 : "d" (port)); 327} 328 329static __inline void 330outw(u_int port, u_short data) 331{ 332 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 333} 334 335static __inline void 336ia32_pause(void) 337{ 338 __asm __volatile("pause"); 339} 340 341static __inline u_int 342#ifdef XEN 343_read_eflags(void) 344#else 345read_eflags(void) 346#endif 347{ 348 u_int ef; 349 350 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 351 return (ef); 352} 353 354static __inline uint64_t 355rdmsr(u_int msr) 356{ 357 uint64_t rv; 358 359 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 360 return (rv); 361} 362 363static __inline uint64_t 364rdpmc(u_int pmc) 365{ 366 uint64_t rv; 367 368 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 369 return (rv); 370} 371 372static __inline uint64_t 373rdtsc(void) 374{ 375 uint64_t rv; 376 377 __asm __volatile("rdtsc" : "=A" (rv)); 378 return (rv); 379} 380 381static __inline uint32_t 382rdtsc32(void) 383{ 384 uint32_t rv; 385 386 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 387 return (rv); 388} 389 390static __inline void 391wbinvd(void) 392{ 393 __asm __volatile("wbinvd"); 394} 395 396static __inline void 397#ifdef XEN 398_write_eflags(u_int ef) 399#else 400write_eflags(u_int ef) 401#endif 402{ 403 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 404} 405 406static __inline void 407wrmsr(u_int msr, uint64_t newval) 408{ 409 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 410} 411 412static __inline void 413load_cr0(u_int data) 414{ 415 416 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 417} 418 419static __inline u_int 420rcr0(void) 421{ 422 u_int data; 423 424 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 425 return (data); 426} 427 428static __inline u_int 429rcr2(void) 430{ 431 u_int data; 432 433#ifdef XEN 434 return (xen_rcr2()); 435#endif 436 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 437 return (data); 438} 439 440static __inline void 441load_cr3(u_int data) 442{ 443#ifdef XEN 444 xen_load_cr3(data); 445#else 446 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 447#endif 448} 449 450static __inline u_int 451rcr3(void) 452{ 453 u_int data; 454 455 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 456 return (data); 457} 458 459static __inline void 460load_cr4(u_int data) 461{ 462 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 463} 464 465static __inline u_int 466rcr4(void) 467{ 468 u_int data; 469 470 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 471 return (data); 472} 473 474static __inline uint64_t 475rxcr(u_int reg) 476{ 477 u_int low, high; 478 479 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 480 return (low | ((uint64_t)high << 32)); 481} 482 483static __inline void 484load_xcr(u_int reg, uint64_t val) 485{ 486 u_int low, high; 487 488 low = val; 489 high = val >> 32; 490 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 491} 492 493/* 494 * Global TLB flush (except for thise for pages marked PG_G) 495 */ 496static __inline void 497invltlb(void) 498{ 499#ifdef XEN 500 xen_tlb_flush(); 501#else 502 load_cr3(rcr3()); 503#endif 504} 505 506/* 507 * TLB flush for an individual page (even if it has PG_G). 508 * Only works on 486+ CPUs (i386 does not have PG_G). 509 */ 510static __inline void 511invlpg(u_int addr) 512{ 513 514#ifdef XEN 515 xen_invlpg(addr); 516#else 517 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 518#endif 519} 520 521static __inline u_short 522rfs(void) 523{ 524 u_short sel; 525 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 526 return (sel); 527} 528 529static __inline uint64_t 530rgdt(void) 531{ 532 uint64_t gdtr; 533 __asm __volatile("sgdt %0" : "=m" (gdtr)); 534 return (gdtr); 535} 536 537static __inline u_short 538rgs(void) 539{ 540 u_short sel; 541 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 542 return (sel); 543} 544 545static __inline uint64_t 546ridt(void) 547{ 548 uint64_t idtr; 549 __asm __volatile("sidt %0" : "=m" (idtr)); 550 return (idtr); 551} 552 553static __inline u_short 554rldt(void) 555{ 556 u_short ldtr; 557 __asm __volatile("sldt %0" : "=g" (ldtr)); 558 return (ldtr); 559} 560 561static __inline u_short 562rss(void) 563{ 564 u_short sel; 565 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 566 return (sel); 567} 568 569static __inline u_short 570rtr(void) 571{ 572 u_short tr; 573 __asm __volatile("str %0" : "=g" (tr)); 574 return (tr); 575} 576 577static __inline void 578load_fs(u_short sel) 579{ 580 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 581} 582 583static __inline void 584load_gs(u_short sel) 585{ 586 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 587} 588 589static __inline void 590lidt(struct region_descriptor *addr) 591{ 592 __asm __volatile("lidt (%0)" : : "r" (addr)); 593} 594 595static __inline void 596lldt(u_short sel) 597{ 598 __asm __volatile("lldt %0" : : "r" (sel)); 599} 600 601static __inline void 602ltr(u_short sel) 603{ 604 __asm __volatile("ltr %0" : : "r" (sel)); 605} 606 607static __inline u_int 608rdr0(void) 609{ 610 u_int data; 611 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 612 return (data); 613} 614 615static __inline void 616load_dr0(u_int dr0) 617{ 618 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 619} 620 621static __inline u_int 622rdr1(void) 623{ 624 u_int data; 625 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 626 return (data); 627} 628 629static __inline void 630load_dr1(u_int dr1) 631{ 632 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 633} 634 635static __inline u_int 636rdr2(void) 637{ 638 u_int data; 639 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 640 return (data); 641} 642 643static __inline void 644load_dr2(u_int dr2) 645{ 646 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 647} 648 649static __inline u_int 650rdr3(void) 651{ 652 u_int data; 653 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 654 return (data); 655} 656 657static __inline void 658load_dr3(u_int dr3) 659{ 660 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 661} 662 663static __inline u_int 664rdr4(void) 665{ 666 u_int data; 667 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 668 return (data); 669} 670 671static __inline void 672load_dr4(u_int dr4) 673{ 674 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 675} 676 677static __inline u_int 678rdr5(void) 679{ 680 u_int data; 681 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 682 return (data); 683} 684 685static __inline void 686load_dr5(u_int dr5) 687{ 688 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 689} 690 691static __inline u_int 692rdr6(void) 693{ 694 u_int data; 695 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 696 return (data); 697} 698 699static __inline void 700load_dr6(u_int dr6) 701{ 702 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 703} 704 705static __inline u_int 706rdr7(void) 707{ 708 u_int data; 709 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 710 return (data); 711} 712 713static __inline void 714load_dr7(u_int dr7) 715{ 716 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 717} 718 719static __inline u_char 720read_cyrix_reg(u_char reg) 721{ 722 outb(0x22, reg); 723 return inb(0x23); 724} 725 726static __inline void 727write_cyrix_reg(u_char reg, u_char data) 728{ 729 outb(0x22, reg); 730 outb(0x23, data); 731} 732 733static __inline register_t 734intr_disable(void) 735{ 736 register_t eflags; 737 738 eflags = read_eflags(); 739 disable_intr(); 740 return (eflags); 741} 742 743static __inline void 744intr_restore(register_t eflags) 745{ 746 write_eflags(eflags); 747} 748 749#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 750 751int breakpoint(void); 752u_int bsfl(u_int mask); 753u_int bsrl(u_int mask); 754void clflush(u_long addr); 755void clts(void); 756void cpuid_count(u_int ax, u_int cx, u_int *p); 757void disable_intr(void); 758void do_cpuid(u_int ax, u_int *p); 759void enable_intr(void); 760void halt(void); 761void ia32_pause(void); 762u_char inb(u_int port); 763u_int inl(u_int port); 764void insb(u_int port, void *addr, size_t count); 765void insl(u_int port, void *addr, size_t count); 766void insw(u_int port, void *addr, size_t count); 767register_t intr_disable(void); 768void intr_restore(register_t ef); 769void invd(void); 770void invlpg(u_int addr); 771void invltlb(void); 772u_short inw(u_int port); 773void lidt(struct region_descriptor *addr); 774void lldt(u_short sel); 775void load_cr0(u_int cr0); 776void load_cr3(u_int cr3); 777void load_cr4(u_int cr4); 778void load_dr0(u_int dr0); 779void load_dr1(u_int dr1); 780void load_dr2(u_int dr2); 781void load_dr3(u_int dr3); 782void load_dr4(u_int dr4); 783void load_dr5(u_int dr5); 784void load_dr6(u_int dr6); 785void load_dr7(u_int dr7); 786void load_fs(u_short sel); 787void load_gs(u_short sel); 788void ltr(u_short sel); 789void outb(u_int port, u_char data); 790void outl(u_int port, u_int data); 791void outsb(u_int port, const void *addr, size_t count); 792void outsl(u_int port, const void *addr, size_t count); 793void outsw(u_int port, const void *addr, size_t count); 794void outw(u_int port, u_short data); 795u_int rcr0(void); 796u_int rcr2(void); 797u_int rcr3(void); 798u_int rcr4(void); 799uint64_t rdmsr(u_int msr); 800uint64_t rdpmc(u_int pmc); 801u_int rdr0(void); 802u_int rdr1(void); 803u_int rdr2(void); 804u_int rdr3(void); 805u_int rdr4(void); 806u_int rdr5(void); 807u_int rdr6(void); 808u_int rdr7(void); 809uint64_t rdtsc(void); 810u_char read_cyrix_reg(u_char reg); 811u_int read_eflags(void); 812u_int rfs(void); 813uint64_t rgdt(void); 814u_int rgs(void); 815uint64_t ridt(void); 816u_short rldt(void); 817u_short rtr(void); 818void wbinvd(void); 819void write_cyrix_reg(u_char reg, u_char data); 820void write_eflags(u_int ef); 821void wrmsr(u_int msr, uint64_t newval); 822 823#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 824 825void reset_dbregs(void); 826 827#ifdef _KERNEL 828int rdmsr_safe(u_int msr, uint64_t *val); 829int wrmsr_safe(u_int msr, uint64_t newval); 830#endif 831 832#endif /* !_MACHINE_CPUFUNC_H_ */ 833