cpufunc.h revision 178299
1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD: head/sys/i386/include/cpufunc.h 178299 2008-04-18 05:47:56Z jeff $ 30 */ 31 32/* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38#ifndef _MACHINE_CPUFUNC_H_ 39#define _MACHINE_CPUFUNC_H_ 40 41#ifndef _SYS_CDEFS_H_ 42#error this file needs sys/cdefs.h as a prerequisite 43#endif 44 45struct region_descriptor; 46 47#define readb(va) (*(volatile u_int8_t *) (va)) 48#define readw(va) (*(volatile u_int16_t *) (va)) 49#define readl(va) (*(volatile u_int32_t *) (va)) 50 51#define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 52#define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 53#define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 54 55#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 56 57static __inline void 58breakpoint(void) 59{ 60 __asm __volatile("int $3"); 61} 62 63static __inline u_int 64bsfl(u_int mask) 65{ 66 u_int result; 67 68 __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 69 return (result); 70} 71 72static __inline u_int 73bsrl(u_int mask) 74{ 75 u_int result; 76 77 __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 78 return (result); 79} 80 81static __inline void 82disable_intr(void) 83{ 84 __asm __volatile("cli" : : : "memory"); 85} 86 87static __inline void 88do_cpuid(u_int ax, u_int *p) 89{ 90 __asm __volatile("cpuid" 91 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 92 : "0" (ax)); 93} 94 95static __inline void 96cpuid_count(u_int ax, u_int cx, u_int *p) 97{ 98 __asm __volatile("cpuid" 99 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 100 : "0" (ax), "c" (cx)); 101} 102 103static __inline void 104enable_intr(void) 105{ 106 __asm __volatile("sti"); 107} 108 109static inline void 110cpu_monitor(const void *addr, int extensions, int hints) 111{ 112 __asm __volatile("monitor;" 113 : :"a" (addr), "c" (extensions), "d"(hints)); 114} 115 116static inline void 117cpu_mwait(int extensions, int hints) 118{ 119 __asm __volatile("mwait;" : :"a" (hints), "c" (extensions)); 120} 121 122#ifdef _KERNEL 123 124#define HAVE_INLINE_FFS 125 126static __inline int 127ffs(int mask) 128{ 129 /* 130 * Note that gcc-2's builtin ffs would be used if we didn't declare 131 * this inline or turn off the builtin. The builtin is faster but 132 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 133 * versions. 134 */ 135 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 136} 137 138#define HAVE_INLINE_FLS 139 140static __inline int 141fls(int mask) 142{ 143 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 144} 145 146#endif /* _KERNEL */ 147 148static __inline void 149halt(void) 150{ 151 __asm __volatile("hlt"); 152} 153 154#if !defined(__GNUCLIKE_BUILTIN_CONSTANT_P) || __GNUCLIKE_ASM < 3 155 156#define inb(port) inbv(port) 157#define outb(port, data) outbv(port, data) 158 159#else /* __GNUCLIKE_BUILTIN_CONSTANT_P && __GNUCLIKE_ASM >= 3 */ 160 161/* 162 * The following complications are to get around gcc not having a 163 * constraint letter for the range 0..255. We still put "d" in the 164 * constraint because "i" isn't a valid constraint when the port 165 * isn't constant. This only matters for -O0 because otherwise 166 * the non-working version gets optimized away. 167 * 168 * Use an expression-statement instead of a conditional expression 169 * because gcc-2.6.0 would promote the operands of the conditional 170 * and produce poor code for "if ((inb(var) & const1) == const2)". 171 * 172 * The unnecessary test `(port) < 0x10000' is to generate a warning if 173 * the `port' has type u_short or smaller. Such types are pessimal. 174 * This actually only works for signed types. The range check is 175 * careful to avoid generating warnings. 176 */ 177#define inb(port) __extension__ ({ \ 178 u_char _data; \ 179 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 180 && (port) < 0x10000) \ 181 _data = inbc(port); \ 182 else \ 183 _data = inbv(port); \ 184 _data; }) 185 186#define outb(port, data) ( \ 187 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 188 && (port) < 0x10000 \ 189 ? outbc(port, data) : outbv(port, data)) 190 191static __inline u_char 192inbc(u_int port) 193{ 194 u_char data; 195 196 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 197 return (data); 198} 199 200static __inline void 201outbc(u_int port, u_char data) 202{ 203 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 204} 205 206#endif /* __GNUCLIKE_BUILTIN_CONSTANT_P && __GNUCLIKE_ASM >= 3*/ 207 208static __inline u_char 209inbv(u_int port) 210{ 211 u_char data; 212 /* 213 * We use %%dx and not %1 here because i/o is done at %dx and not at 214 * %edx, while gcc generates inferior code (movw instead of movl) 215 * if we tell it to load (u_short) port. 216 */ 217 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 218 return (data); 219} 220 221static __inline u_int 222inl(u_int port) 223{ 224 u_int data; 225 226 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 227 return (data); 228} 229 230static __inline void 231insb(u_int port, void *addr, size_t cnt) 232{ 233 __asm __volatile("cld; rep; insb" 234 : "+D" (addr), "+c" (cnt) 235 : "d" (port) 236 : "memory"); 237} 238 239static __inline void 240insw(u_int port, void *addr, size_t cnt) 241{ 242 __asm __volatile("cld; rep; insw" 243 : "+D" (addr), "+c" (cnt) 244 : "d" (port) 245 : "memory"); 246} 247 248static __inline void 249insl(u_int port, void *addr, size_t cnt) 250{ 251 __asm __volatile("cld; rep; insl" 252 : "+D" (addr), "+c" (cnt) 253 : "d" (port) 254 : "memory"); 255} 256 257static __inline void 258invd(void) 259{ 260 __asm __volatile("invd"); 261} 262 263static __inline u_short 264inw(u_int port) 265{ 266 u_short data; 267 268 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 269 return (data); 270} 271 272static __inline void 273outbv(u_int port, u_char data) 274{ 275 u_char al; 276 /* 277 * Use an unnecessary assignment to help gcc's register allocator. 278 * This make a large difference for gcc-1.40 and a tiny difference 279 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 280 * best results. gcc-2.6.0 can't handle this. 281 */ 282 al = data; 283 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 284} 285 286static __inline void 287outl(u_int port, u_int data) 288{ 289 /* 290 * outl() and outw() aren't used much so we haven't looked at 291 * possible micro-optimizations such as the unnecessary 292 * assignment for them. 293 */ 294 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 295} 296 297static __inline void 298outsb(u_int port, const void *addr, size_t cnt) 299{ 300 __asm __volatile("cld; rep; outsb" 301 : "+S" (addr), "+c" (cnt) 302 : "d" (port)); 303} 304 305static __inline void 306outsw(u_int port, const void *addr, size_t cnt) 307{ 308 __asm __volatile("cld; rep; outsw" 309 : "+S" (addr), "+c" (cnt) 310 : "d" (port)); 311} 312 313static __inline void 314outsl(u_int port, const void *addr, size_t cnt) 315{ 316 __asm __volatile("cld; rep; outsl" 317 : "+S" (addr), "+c" (cnt) 318 : "d" (port)); 319} 320 321static __inline void 322outw(u_int port, u_short data) 323{ 324 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 325} 326 327static __inline void 328ia32_pause(void) 329{ 330 __asm __volatile("pause"); 331} 332 333static __inline u_int 334read_eflags(void) 335{ 336 u_int ef; 337 338 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 339 return (ef); 340} 341 342static __inline uint64_t 343rdmsr(u_int msr) 344{ 345 uint64_t rv; 346 347 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 348 return (rv); 349} 350 351static __inline uint64_t 352rdpmc(u_int pmc) 353{ 354 uint64_t rv; 355 356 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 357 return (rv); 358} 359 360static __inline uint64_t 361rdtsc(void) 362{ 363 uint64_t rv; 364 365 __asm __volatile("rdtsc" : "=A" (rv)); 366 return (rv); 367} 368 369static __inline void 370wbinvd(void) 371{ 372 __asm __volatile("wbinvd"); 373} 374 375static __inline void 376write_eflags(u_int ef) 377{ 378 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 379} 380 381static __inline void 382wrmsr(u_int msr, uint64_t newval) 383{ 384 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 385} 386 387static __inline void 388load_cr0(u_int data) 389{ 390 391 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 392} 393 394static __inline u_int 395rcr0(void) 396{ 397 u_int data; 398 399 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 400 return (data); 401} 402 403static __inline u_int 404rcr2(void) 405{ 406 u_int data; 407 408 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 409 return (data); 410} 411 412static __inline void 413load_cr3(u_int data) 414{ 415 416 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 417} 418 419static __inline u_int 420rcr3(void) 421{ 422 u_int data; 423 424 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 425 return (data); 426} 427 428static __inline void 429load_cr4(u_int data) 430{ 431 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 432} 433 434static __inline u_int 435rcr4(void) 436{ 437 u_int data; 438 439 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 440 return (data); 441} 442 443/* 444 * Global TLB flush (except for thise for pages marked PG_G) 445 */ 446static __inline void 447invltlb(void) 448{ 449 450 load_cr3(rcr3()); 451} 452 453/* 454 * TLB flush for an individual page (even if it has PG_G). 455 * Only works on 486+ CPUs (i386 does not have PG_G). 456 */ 457static __inline void 458invlpg(u_int addr) 459{ 460 461 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 462} 463 464static __inline u_int 465rfs(void) 466{ 467 u_int sel; 468 __asm __volatile("movl %%fs,%0" : "=rm" (sel)); 469 return (sel); 470} 471 472static __inline uint64_t 473rgdt(void) 474{ 475 uint64_t gdtr; 476 __asm __volatile("sgdt %0" : "=m" (gdtr)); 477 return (gdtr); 478} 479 480static __inline u_int 481rgs(void) 482{ 483 u_int sel; 484 __asm __volatile("movl %%gs,%0" : "=rm" (sel)); 485 return (sel); 486} 487 488static __inline uint64_t 489ridt(void) 490{ 491 uint64_t idtr; 492 __asm __volatile("sidt %0" : "=m" (idtr)); 493 return (idtr); 494} 495 496static __inline u_short 497rldt(void) 498{ 499 u_short ldtr; 500 __asm __volatile("sldt %0" : "=g" (ldtr)); 501 return (ldtr); 502} 503 504static __inline u_int 505rss(void) 506{ 507 u_int sel; 508 __asm __volatile("movl %%ss,%0" : "=rm" (sel)); 509 return (sel); 510} 511 512static __inline u_short 513rtr(void) 514{ 515 u_short tr; 516 __asm __volatile("str %0" : "=g" (tr)); 517 return (tr); 518} 519 520static __inline void 521load_fs(u_int sel) 522{ 523 __asm __volatile("movl %0,%%fs" : : "rm" (sel)); 524} 525 526static __inline void 527load_gs(u_int sel) 528{ 529 __asm __volatile("movl %0,%%gs" : : "rm" (sel)); 530} 531 532static __inline void 533lidt(struct region_descriptor *addr) 534{ 535 __asm __volatile("lidt (%0)" : : "r" (addr)); 536} 537 538static __inline void 539lldt(u_short sel) 540{ 541 __asm __volatile("lldt %0" : : "r" (sel)); 542} 543 544static __inline void 545ltr(u_short sel) 546{ 547 __asm __volatile("ltr %0" : : "r" (sel)); 548} 549 550static __inline u_int 551rdr0(void) 552{ 553 u_int data; 554 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 555 return (data); 556} 557 558static __inline void 559load_dr0(u_int dr0) 560{ 561 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 562} 563 564static __inline u_int 565rdr1(void) 566{ 567 u_int data; 568 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 569 return (data); 570} 571 572static __inline void 573load_dr1(u_int dr1) 574{ 575 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 576} 577 578static __inline u_int 579rdr2(void) 580{ 581 u_int data; 582 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 583 return (data); 584} 585 586static __inline void 587load_dr2(u_int dr2) 588{ 589 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 590} 591 592static __inline u_int 593rdr3(void) 594{ 595 u_int data; 596 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 597 return (data); 598} 599 600static __inline void 601load_dr3(u_int dr3) 602{ 603 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 604} 605 606static __inline u_int 607rdr4(void) 608{ 609 u_int data; 610 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 611 return (data); 612} 613 614static __inline void 615load_dr4(u_int dr4) 616{ 617 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 618} 619 620static __inline u_int 621rdr5(void) 622{ 623 u_int data; 624 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 625 return (data); 626} 627 628static __inline void 629load_dr5(u_int dr5) 630{ 631 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 632} 633 634static __inline u_int 635rdr6(void) 636{ 637 u_int data; 638 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 639 return (data); 640} 641 642static __inline void 643load_dr6(u_int dr6) 644{ 645 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 646} 647 648static __inline u_int 649rdr7(void) 650{ 651 u_int data; 652 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 653 return (data); 654} 655 656static __inline void 657load_dr7(u_int dr7) 658{ 659 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 660} 661 662static __inline register_t 663intr_disable(void) 664{ 665 register_t eflags; 666 667 eflags = read_eflags(); 668 disable_intr(); 669 return (eflags); 670} 671 672static __inline void 673intr_restore(register_t eflags) 674{ 675 write_eflags(eflags); 676} 677 678#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 679 680int breakpoint(void); 681u_int bsfl(u_int mask); 682u_int bsrl(u_int mask); 683void disable_intr(void); 684void do_cpuid(u_int ax, u_int *p); 685void enable_intr(void); 686void halt(void); 687void ia32_pause(void); 688u_char inb(u_int port); 689u_int inl(u_int port); 690void insb(u_int port, void *addr, size_t cnt); 691void insl(u_int port, void *addr, size_t cnt); 692void insw(u_int port, void *addr, size_t cnt); 693register_t intr_disable(void); 694void intr_restore(register_t ef); 695void invd(void); 696void invlpg(u_int addr); 697void invltlb(void); 698u_short inw(u_int port); 699void lidt(struct region_descriptor *addr); 700void lldt(u_short sel); 701void load_cr0(u_int cr0); 702void load_cr3(u_int cr3); 703void load_cr4(u_int cr4); 704void load_dr0(u_int dr0); 705void load_dr1(u_int dr1); 706void load_dr2(u_int dr2); 707void load_dr3(u_int dr3); 708void load_dr4(u_int dr4); 709void load_dr5(u_int dr5); 710void load_dr6(u_int dr6); 711void load_dr7(u_int dr7); 712void load_fs(u_int sel); 713void load_gs(u_int sel); 714void ltr(u_short sel); 715void outb(u_int port, u_char data); 716void outl(u_int port, u_int data); 717void outsb(u_int port, const void *addr, size_t cnt); 718void outsl(u_int port, const void *addr, size_t cnt); 719void outsw(u_int port, const void *addr, size_t cnt); 720void outw(u_int port, u_short data); 721u_int rcr0(void); 722u_int rcr2(void); 723u_int rcr3(void); 724u_int rcr4(void); 725uint64_t rdmsr(u_int msr); 726uint64_t rdpmc(u_int pmc); 727u_int rdr0(void); 728u_int rdr1(void); 729u_int rdr2(void); 730u_int rdr3(void); 731u_int rdr4(void); 732u_int rdr5(void); 733u_int rdr6(void); 734u_int rdr7(void); 735uint64_t rdtsc(void); 736u_int read_eflags(void); 737u_int rfs(void); 738uint64_t rgdt(void); 739u_int rgs(void); 740uint64_t ridt(void); 741u_short rldt(void); 742u_short rtr(void); 743void wbinvd(void); 744void write_eflags(u_int ef); 745void wrmsr(u_int msr, uint64_t newval); 746 747#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 748 749void reset_dbregs(void); 750 751#endif /* !_MACHINE_CPUFUNC_H_ */ 752