cpufunc.h revision 181911
1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD: head/sys/i386/include/cpufunc.h 181911 2008-08-20 09:16:46Z kmacy $ 30 */ 31 32/* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38#ifndef _MACHINE_CPUFUNC_H_ 39#define _MACHINE_CPUFUNC_H_ 40 41#ifndef _SYS_CDEFS_H_ 42#error this file needs sys/cdefs.h as a prerequisite 43#endif 44 45#ifdef XEN 46extern void xen_cli(void); 47extern void xen_sti(void); 48extern void xen_load_cr3(u_int data); 49extern void xen_tlb_flush(void); 50extern void xen_invlpg(u_int addr); 51extern int xen_save_and_cli(void); 52extern void xen_restore_flags(u_int eflags); 53#endif 54 55struct region_descriptor; 56 57#define readb(va) (*(volatile u_int8_t *) (va)) 58#define readw(va) (*(volatile u_int16_t *) (va)) 59#define readl(va) (*(volatile u_int32_t *) (va)) 60 61#define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 62#define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 63#define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 64 65#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 66 67static __inline void 68breakpoint(void) 69{ 70 __asm __volatile("int $3"); 71} 72 73static __inline u_int 74bsfl(u_int mask) 75{ 76 u_int result; 77 78 __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 79 return (result); 80} 81 82static __inline u_int 83bsrl(u_int mask) 84{ 85 u_int result; 86 87 __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 88 return (result); 89} 90 91static __inline void 92disable_intr(void) 93{ 94#ifdef XEN 95 xen_cli(); 96#else 97 __asm __volatile("cli" : : : "memory"); 98#endif 99} 100 101static __inline void 102do_cpuid(u_int ax, u_int *p) 103{ 104 __asm __volatile("cpuid" 105 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 106 : "0" (ax)); 107} 108 109static __inline void 110cpuid_count(u_int ax, u_int cx, u_int *p) 111{ 112 __asm __volatile("cpuid" 113 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 114 : "0" (ax), "c" (cx)); 115} 116 117static __inline void 118enable_intr(void) 119{ 120#ifdef XEN 121 xen_sti(); 122#else 123 __asm __volatile("sti"); 124#endif 125} 126 127static inline void 128cpu_monitor(const void *addr, int extensions, int hints) 129{ 130 __asm __volatile("monitor;" 131 : :"a" (addr), "c" (extensions), "d"(hints)); 132} 133 134static inline void 135cpu_mwait(int extensions, int hints) 136{ 137 __asm __volatile("mwait;" : :"a" (hints), "c" (extensions)); 138} 139 140#ifdef _KERNEL 141 142#define HAVE_INLINE_FFS 143 144static __inline int 145ffs(int mask) 146{ 147 /* 148 * Note that gcc-2's builtin ffs would be used if we didn't declare 149 * this inline or turn off the builtin. The builtin is faster but 150 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 151 * versions. 152 */ 153 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 154} 155 156#define HAVE_INLINE_FLS 157 158static __inline int 159fls(int mask) 160{ 161 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 162} 163 164#endif /* _KERNEL */ 165 166static __inline void 167halt(void) 168{ 169 __asm __volatile("hlt"); 170} 171 172#if !defined(__GNUCLIKE_BUILTIN_CONSTANT_P) || __GNUCLIKE_ASM < 3 173 174#define inb(port) inbv(port) 175#define outb(port, data) outbv(port, data) 176 177#else /* __GNUCLIKE_BUILTIN_CONSTANT_P && __GNUCLIKE_ASM >= 3 */ 178 179/* 180 * The following complications are to get around gcc not having a 181 * constraint letter for the range 0..255. We still put "d" in the 182 * constraint because "i" isn't a valid constraint when the port 183 * isn't constant. This only matters for -O0 because otherwise 184 * the non-working version gets optimized away. 185 * 186 * Use an expression-statement instead of a conditional expression 187 * because gcc-2.6.0 would promote the operands of the conditional 188 * and produce poor code for "if ((inb(var) & const1) == const2)". 189 * 190 * The unnecessary test `(port) < 0x10000' is to generate a warning if 191 * the `port' has type u_short or smaller. Such types are pessimal. 192 * This actually only works for signed types. The range check is 193 * careful to avoid generating warnings. 194 */ 195#define inb(port) __extension__ ({ \ 196 u_char _data; \ 197 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 198 && (port) < 0x10000) \ 199 _data = inbc(port); \ 200 else \ 201 _data = inbv(port); \ 202 _data; }) 203 204#define outb(port, data) ( \ 205 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 206 && (port) < 0x10000 \ 207 ? outbc(port, data) : outbv(port, data)) 208 209static __inline u_char 210inbc(u_int port) 211{ 212 u_char data; 213 214 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 215 return (data); 216} 217 218static __inline void 219outbc(u_int port, u_char data) 220{ 221 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 222} 223 224#endif /* __GNUCLIKE_BUILTIN_CONSTANT_P && __GNUCLIKE_ASM >= 3*/ 225 226static __inline u_char 227inbv(u_int port) 228{ 229 u_char data; 230 /* 231 * We use %%dx and not %1 here because i/o is done at %dx and not at 232 * %edx, while gcc generates inferior code (movw instead of movl) 233 * if we tell it to load (u_short) port. 234 */ 235 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 236 return (data); 237} 238 239static __inline u_int 240inl(u_int port) 241{ 242 u_int data; 243 244 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 245 return (data); 246} 247 248static __inline void 249insb(u_int port, void *addr, size_t cnt) 250{ 251 __asm __volatile("cld; rep; insb" 252 : "+D" (addr), "+c" (cnt) 253 : "d" (port) 254 : "memory"); 255} 256 257static __inline void 258insw(u_int port, void *addr, size_t cnt) 259{ 260 __asm __volatile("cld; rep; insw" 261 : "+D" (addr), "+c" (cnt) 262 : "d" (port) 263 : "memory"); 264} 265 266static __inline void 267insl(u_int port, void *addr, size_t cnt) 268{ 269 __asm __volatile("cld; rep; insl" 270 : "+D" (addr), "+c" (cnt) 271 : "d" (port) 272 : "memory"); 273} 274 275static __inline void 276invd(void) 277{ 278 __asm __volatile("invd"); 279} 280 281static __inline u_short 282inw(u_int port) 283{ 284 u_short data; 285 286 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 287 return (data); 288} 289 290static __inline void 291outbv(u_int port, u_char data) 292{ 293 u_char al; 294 /* 295 * Use an unnecessary assignment to help gcc's register allocator. 296 * This make a large difference for gcc-1.40 and a tiny difference 297 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 298 * best results. gcc-2.6.0 can't handle this. 299 */ 300 al = data; 301 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 302} 303 304static __inline void 305outl(u_int port, u_int data) 306{ 307 /* 308 * outl() and outw() aren't used much so we haven't looked at 309 * possible micro-optimizations such as the unnecessary 310 * assignment for them. 311 */ 312 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 313} 314 315static __inline void 316outsb(u_int port, const void *addr, size_t cnt) 317{ 318 __asm __volatile("cld; rep; outsb" 319 : "+S" (addr), "+c" (cnt) 320 : "d" (port)); 321} 322 323static __inline void 324outsw(u_int port, const void *addr, size_t cnt) 325{ 326 __asm __volatile("cld; rep; outsw" 327 : "+S" (addr), "+c" (cnt) 328 : "d" (port)); 329} 330 331static __inline void 332outsl(u_int port, const void *addr, size_t cnt) 333{ 334 __asm __volatile("cld; rep; outsl" 335 : "+S" (addr), "+c" (cnt) 336 : "d" (port)); 337} 338 339static __inline void 340outw(u_int port, u_short data) 341{ 342 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 343} 344 345static __inline void 346ia32_pause(void) 347{ 348 __asm __volatile("pause"); 349} 350 351static __inline u_int 352read_eflags(void) 353{ 354 u_int ef; 355 356 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 357 return (ef); 358} 359 360static __inline uint64_t 361rdmsr(u_int msr) 362{ 363 uint64_t rv; 364 365 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 366 return (rv); 367} 368 369static __inline uint64_t 370rdpmc(u_int pmc) 371{ 372 uint64_t rv; 373 374 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 375 return (rv); 376} 377 378static __inline uint64_t 379rdtsc(void) 380{ 381 uint64_t rv; 382 383 __asm __volatile("rdtsc" : "=A" (rv)); 384 return (rv); 385} 386 387static __inline void 388wbinvd(void) 389{ 390 __asm __volatile("wbinvd"); 391} 392 393static __inline void 394write_eflags(u_int ef) 395{ 396 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 397} 398 399static __inline void 400wrmsr(u_int msr, uint64_t newval) 401{ 402 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 403} 404 405static __inline void 406load_cr0(u_int data) 407{ 408 409 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 410} 411 412static __inline u_int 413rcr0(void) 414{ 415 u_int data; 416 417 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 418 return (data); 419} 420 421static __inline u_int 422rcr2(void) 423{ 424 u_int data; 425 426 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 427 return (data); 428} 429 430static __inline void 431load_cr3(u_int data) 432{ 433#ifdef XEN 434 xen_load_cr3(data); 435#else 436 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 437#endif 438} 439 440static __inline u_int 441rcr3(void) 442{ 443 u_int data; 444 445 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 446 return (data); 447} 448 449static __inline void 450load_cr4(u_int data) 451{ 452 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 453} 454 455static __inline u_int 456rcr4(void) 457{ 458 u_int data; 459 460 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 461 return (data); 462} 463 464/* 465 * Global TLB flush (except for thise for pages marked PG_G) 466 */ 467static __inline void 468invltlb(void) 469{ 470#ifdef XEN 471 xen_tlb_flush(); 472#else 473 load_cr3(rcr3()); 474#endif 475} 476 477/* 478 * TLB flush for an individual page (even if it has PG_G). 479 * Only works on 486+ CPUs (i386 does not have PG_G). 480 */ 481static __inline void 482invlpg(u_int addr) 483{ 484 485#ifdef XEN 486 xen_invlpg(addr); 487#else 488 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 489#endif 490} 491 492static __inline u_int 493rfs(void) 494{ 495 u_int sel; 496 __asm __volatile("movl %%fs,%0" : "=rm" (sel)); 497 return (sel); 498} 499 500static __inline uint64_t 501rgdt(void) 502{ 503 uint64_t gdtr; 504 __asm __volatile("sgdt %0" : "=m" (gdtr)); 505 return (gdtr); 506} 507 508static __inline u_int 509rgs(void) 510{ 511 u_int sel; 512 __asm __volatile("movl %%gs,%0" : "=rm" (sel)); 513 return (sel); 514} 515 516static __inline uint64_t 517ridt(void) 518{ 519 uint64_t idtr; 520 __asm __volatile("sidt %0" : "=m" (idtr)); 521 return (idtr); 522} 523 524static __inline u_short 525rldt(void) 526{ 527 u_short ldtr; 528 __asm __volatile("sldt %0" : "=g" (ldtr)); 529 return (ldtr); 530} 531 532static __inline u_int 533rss(void) 534{ 535 u_int sel; 536 __asm __volatile("movl %%ss,%0" : "=rm" (sel)); 537 return (sel); 538} 539 540static __inline u_short 541rtr(void) 542{ 543 u_short tr; 544 __asm __volatile("str %0" : "=g" (tr)); 545 return (tr); 546} 547 548static __inline void 549load_fs(u_int sel) 550{ 551 __asm __volatile("movl %0,%%fs" : : "rm" (sel)); 552} 553 554static __inline void 555load_gs(u_int sel) 556{ 557 __asm __volatile("movl %0,%%gs" : : "rm" (sel)); 558} 559 560static __inline void 561lidt(struct region_descriptor *addr) 562{ 563 __asm __volatile("lidt (%0)" : : "r" (addr)); 564} 565 566static __inline void 567lldt(u_short sel) 568{ 569 __asm __volatile("lldt %0" : : "r" (sel)); 570} 571 572static __inline void 573ltr(u_short sel) 574{ 575 __asm __volatile("ltr %0" : : "r" (sel)); 576} 577 578static __inline u_int 579rdr0(void) 580{ 581 u_int data; 582 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 583 return (data); 584} 585 586static __inline void 587load_dr0(u_int dr0) 588{ 589 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 590} 591 592static __inline u_int 593rdr1(void) 594{ 595 u_int data; 596 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 597 return (data); 598} 599 600static __inline void 601load_dr1(u_int dr1) 602{ 603 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 604} 605 606static __inline u_int 607rdr2(void) 608{ 609 u_int data; 610 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 611 return (data); 612} 613 614static __inline void 615load_dr2(u_int dr2) 616{ 617 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 618} 619 620static __inline u_int 621rdr3(void) 622{ 623 u_int data; 624 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 625 return (data); 626} 627 628static __inline void 629load_dr3(u_int dr3) 630{ 631 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 632} 633 634static __inline u_int 635rdr4(void) 636{ 637 u_int data; 638 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 639 return (data); 640} 641 642static __inline void 643load_dr4(u_int dr4) 644{ 645 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 646} 647 648static __inline u_int 649rdr5(void) 650{ 651 u_int data; 652 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 653 return (data); 654} 655 656static __inline void 657load_dr5(u_int dr5) 658{ 659 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 660} 661 662static __inline u_int 663rdr6(void) 664{ 665 u_int data; 666 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 667 return (data); 668} 669 670static __inline void 671load_dr6(u_int dr6) 672{ 673 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 674} 675 676static __inline u_int 677rdr7(void) 678{ 679 u_int data; 680 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 681 return (data); 682} 683 684static __inline void 685load_dr7(u_int dr7) 686{ 687 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 688} 689 690static __inline register_t 691intr_disable(void) 692{ 693 register_t eflags; 694 695#ifdef XEN 696 eflags = xen_save_and_cli(); 697#else 698 eflags = read_eflags(); 699 disable_intr(); 700#endif 701 return (eflags); 702} 703 704static __inline void 705intr_restore(register_t eflags) 706{ 707#ifdef XEN 708 xen_restore_flags(eflags); 709#else 710 write_eflags(eflags); 711#endif 712} 713 714#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 715 716int breakpoint(void); 717u_int bsfl(u_int mask); 718u_int bsrl(u_int mask); 719void disable_intr(void); 720void do_cpuid(u_int ax, u_int *p); 721void enable_intr(void); 722void halt(void); 723void ia32_pause(void); 724u_char inb(u_int port); 725u_int inl(u_int port); 726void insb(u_int port, void *addr, size_t cnt); 727void insl(u_int port, void *addr, size_t cnt); 728void insw(u_int port, void *addr, size_t cnt); 729register_t intr_disable(void); 730void intr_restore(register_t ef); 731void invd(void); 732void invlpg(u_int addr); 733void invltlb(void); 734u_short inw(u_int port); 735void lidt(struct region_descriptor *addr); 736void lldt(u_short sel); 737void load_cr0(u_int cr0); 738void load_cr3(u_int cr3); 739void load_cr4(u_int cr4); 740void load_dr0(u_int dr0); 741void load_dr1(u_int dr1); 742void load_dr2(u_int dr2); 743void load_dr3(u_int dr3); 744void load_dr4(u_int dr4); 745void load_dr5(u_int dr5); 746void load_dr6(u_int dr6); 747void load_dr7(u_int dr7); 748void load_fs(u_int sel); 749void load_gs(u_int sel); 750void ltr(u_short sel); 751void outb(u_int port, u_char data); 752void outl(u_int port, u_int data); 753void outsb(u_int port, const void *addr, size_t cnt); 754void outsl(u_int port, const void *addr, size_t cnt); 755void outsw(u_int port, const void *addr, size_t cnt); 756void outw(u_int port, u_short data); 757u_int rcr0(void); 758u_int rcr2(void); 759u_int rcr3(void); 760u_int rcr4(void); 761uint64_t rdmsr(u_int msr); 762uint64_t rdpmc(u_int pmc); 763u_int rdr0(void); 764u_int rdr1(void); 765u_int rdr2(void); 766u_int rdr3(void); 767u_int rdr4(void); 768u_int rdr5(void); 769u_int rdr6(void); 770u_int rdr7(void); 771uint64_t rdtsc(void); 772u_int read_eflags(void); 773u_int rfs(void); 774uint64_t rgdt(void); 775u_int rgs(void); 776uint64_t ridt(void); 777u_short rldt(void); 778u_short rtr(void); 779void wbinvd(void); 780void write_eflags(u_int ef); 781void wrmsr(u_int msr, uint64_t newval); 782 783#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 784 785void reset_dbregs(void); 786 787#ifdef _KERNEL 788int rdmsr_safe(u_int msr, uint64_t *val); 789int wrmsr_safe(u_int msr, uint64_t newval); 790#endif 791 792#endif /* !_MACHINE_CPUFUNC_H_ */ 793