cpufunc.h revision 126891
1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by the University of 16 * California, Berkeley and its contributors. 17 * 4. Neither the name of the University nor the names of its contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 * 33 * $FreeBSD: head/sys/i386/include/cpufunc.h 126891 2004-03-12 21:45:33Z trhodes $ 34 */ 35 36/* 37 * Functions to provide access to special i386 instructions. 38 * This in included in sys/systm.h, and that file should be 39 * used in preference to this. 40 */ 41 42#ifndef _MACHINE_CPUFUNC_H_ 43#define _MACHINE_CPUFUNC_H_ 44 45struct region_descriptor; 46 47#define readb(va) (*(volatile u_int8_t *) (va)) 48#define readw(va) (*(volatile u_int16_t *) (va)) 49#define readl(va) (*(volatile u_int32_t *) (va)) 50 51#define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 52#define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 53#define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 54 55#if defined(__GNUC__) || defined(__INTEL_COMPILER) 56 57static __inline void 58breakpoint(void) 59{ 60 __asm __volatile("int $3"); 61} 62 63static __inline u_int 64bsfl(u_int mask) 65{ 66 u_int result; 67 68 __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 69 return (result); 70} 71 72static __inline u_int 73bsrl(u_int mask) 74{ 75 u_int result; 76 77 __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 78 return (result); 79} 80 81static __inline void 82disable_intr(void) 83{ 84 __asm __volatile("cli" : : : "memory"); 85} 86 87static __inline void 88do_cpuid(u_int ax, u_int *p) 89{ 90 __asm __volatile("cpuid" 91 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 92 : "0" (ax)); 93} 94 95static __inline void 96enable_intr(void) 97{ 98 __asm __volatile("sti"); 99} 100 101#ifdef _KERNEL 102 103#define HAVE_INLINE_FFS 104 105static __inline int 106ffs(int mask) 107{ 108 /* 109 * Note that gcc-2's builtin ffs would be used if we didn't declare 110 * this inline or turn off the builtin. The builtin is faster but 111 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 112 * versions. 113 */ 114 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 115} 116 117#define HAVE_INLINE_FLS 118 119static __inline int 120fls(int mask) 121{ 122 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 123} 124 125#endif /* _KERNEL */ 126 127static __inline void 128halt(void) 129{ 130 __asm __volatile("hlt"); 131} 132 133#if __GNUC__ < 2 134 135#define inb(port) inbv(port) 136#define outb(port, data) outbv(port, data) 137 138#else /* __GNUC >= 2 */ 139 140/* 141 * The following complications are to get around gcc not having a 142 * constraint letter for the range 0..255. We still put "d" in the 143 * constraint because "i" isn't a valid constraint when the port 144 * isn't constant. This only matters for -O0 because otherwise 145 * the non-working version gets optimized away. 146 * 147 * Use an expression-statement instead of a conditional expression 148 * because gcc-2.6.0 would promote the operands of the conditional 149 * and produce poor code for "if ((inb(var) & const1) == const2)". 150 * 151 * The unnecessary test `(port) < 0x10000' is to generate a warning if 152 * the `port' has type u_short or smaller. Such types are pessimal. 153 * This actually only works for signed types. The range check is 154 * careful to avoid generating warnings. 155 */ 156#define inb(port) __extension__ ({ \ 157 u_char _data; \ 158 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 159 && (port) < 0x10000) \ 160 _data = inbc(port); \ 161 else \ 162 _data = inbv(port); \ 163 _data; }) 164 165#define outb(port, data) ( \ 166 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 167 && (port) < 0x10000 \ 168 ? outbc(port, data) : outbv(port, data)) 169 170static __inline u_char 171inbc(u_int port) 172{ 173 u_char data; 174 175 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 176 return (data); 177} 178 179static __inline void 180outbc(u_int port, u_char data) 181{ 182 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 183} 184 185#endif /* __GNUC <= 2 */ 186 187static __inline u_char 188inbv(u_int port) 189{ 190 u_char data; 191 /* 192 * We use %%dx and not %1 here because i/o is done at %dx and not at 193 * %edx, while gcc generates inferior code (movw instead of movl) 194 * if we tell it to load (u_short) port. 195 */ 196 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 197 return (data); 198} 199 200static __inline u_int 201inl(u_int port) 202{ 203 u_int data; 204 205 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 206 return (data); 207} 208 209static __inline void 210insb(u_int port, void *addr, size_t cnt) 211{ 212 __asm __volatile("cld; rep; insb" 213 : "+D" (addr), "+c" (cnt) 214 : "d" (port) 215 : "memory"); 216} 217 218static __inline void 219insw(u_int port, void *addr, size_t cnt) 220{ 221 __asm __volatile("cld; rep; insw" 222 : "+D" (addr), "+c" (cnt) 223 : "d" (port) 224 : "memory"); 225} 226 227static __inline void 228insl(u_int port, void *addr, size_t cnt) 229{ 230 __asm __volatile("cld; rep; insl" 231 : "+D" (addr), "+c" (cnt) 232 : "d" (port) 233 : "memory"); 234} 235 236static __inline void 237invd(void) 238{ 239 __asm __volatile("invd"); 240} 241 242static __inline u_short 243inw(u_int port) 244{ 245 u_short data; 246 247 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 248 return (data); 249} 250 251static __inline void 252outbv(u_int port, u_char data) 253{ 254 u_char al; 255 /* 256 * Use an unnecessary assignment to help gcc's register allocator. 257 * This make a large difference for gcc-1.40 and a tiny difference 258 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 259 * best results. gcc-2.6.0 can't handle this. 260 */ 261 al = data; 262 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 263} 264 265static __inline void 266outl(u_int port, u_int data) 267{ 268 /* 269 * outl() and outw() aren't used much so we haven't looked at 270 * possible micro-optimizations such as the unnecessary 271 * assignment for them. 272 */ 273 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 274} 275 276static __inline void 277outsb(u_int port, const void *addr, size_t cnt) 278{ 279 __asm __volatile("cld; rep; outsb" 280 : "+S" (addr), "+c" (cnt) 281 : "d" (port)); 282} 283 284static __inline void 285outsw(u_int port, const void *addr, size_t cnt) 286{ 287 __asm __volatile("cld; rep; outsw" 288 : "+S" (addr), "+c" (cnt) 289 : "d" (port)); 290} 291 292static __inline void 293outsl(u_int port, const void *addr, size_t cnt) 294{ 295 __asm __volatile("cld; rep; outsl" 296 : "+S" (addr), "+c" (cnt) 297 : "d" (port)); 298} 299 300static __inline void 301outw(u_int port, u_short data) 302{ 303 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 304} 305 306static __inline void 307ia32_pause(void) 308{ 309 __asm __volatile("pause"); 310} 311 312static __inline u_int 313read_eflags(void) 314{ 315 u_int ef; 316 317 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 318 return (ef); 319} 320 321static __inline u_int64_t 322rdmsr(u_int msr) 323{ 324 u_int64_t rv; 325 326 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 327 return (rv); 328} 329 330static __inline u_int64_t 331rdpmc(u_int pmc) 332{ 333 u_int64_t rv; 334 335 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 336 return (rv); 337} 338 339static __inline u_int64_t 340rdtsc(void) 341{ 342 u_int64_t rv; 343 344 __asm __volatile("rdtsc" : "=A" (rv)); 345 return (rv); 346} 347 348static __inline void 349wbinvd(void) 350{ 351 __asm __volatile("wbinvd"); 352} 353 354static __inline void 355write_eflags(u_int ef) 356{ 357 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 358} 359 360static __inline void 361wrmsr(u_int msr, u_int64_t newval) 362{ 363 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 364} 365 366static __inline void 367load_cr0(u_int data) 368{ 369 370 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 371} 372 373static __inline u_int 374rcr0(void) 375{ 376 u_int data; 377 378 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 379 return (data); 380} 381 382static __inline u_int 383rcr2(void) 384{ 385 u_int data; 386 387 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 388 return (data); 389} 390 391static __inline void 392load_cr3(u_int data) 393{ 394 395 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 396} 397 398static __inline u_int 399rcr3(void) 400{ 401 u_int data; 402 403 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 404 return (data); 405} 406 407static __inline void 408load_cr4(u_int data) 409{ 410 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 411} 412 413static __inline u_int 414rcr4(void) 415{ 416 u_int data; 417 418 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 419 return (data); 420} 421 422/* 423 * Global TLB flush (except for thise for pages marked PG_G) 424 */ 425static __inline void 426invltlb(void) 427{ 428 429 load_cr3(rcr3()); 430} 431 432/* 433 * TLB flush for an individual page (even if it has PG_G). 434 * Only works on 486+ CPUs (i386 does not have PG_G). 435 */ 436static __inline void 437invlpg(u_int addr) 438{ 439 440 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 441} 442 443static __inline u_int 444rfs(void) 445{ 446 u_int sel; 447 __asm __volatile("movl %%fs,%0" : "=rm" (sel)); 448 return (sel); 449} 450 451static __inline u_int 452rgs(void) 453{ 454 u_int sel; 455 __asm __volatile("movl %%gs,%0" : "=rm" (sel)); 456 return (sel); 457} 458 459static __inline void 460load_fs(u_int sel) 461{ 462 __asm __volatile("movl %0,%%fs" : : "rm" (sel)); 463} 464 465static __inline void 466load_gs(u_int sel) 467{ 468 __asm __volatile("movl %0,%%gs" : : "rm" (sel)); 469} 470 471static __inline void 472lidt(struct region_descriptor *addr) 473{ 474 __asm __volatile("lidt (%0)" : : "r" (addr)); 475} 476 477static __inline void 478lldt(u_short sel) 479{ 480 __asm __volatile("lldt %0" : : "r" (sel)); 481} 482 483static __inline void 484ltr(u_short sel) 485{ 486 __asm __volatile("ltr %0" : : "r" (sel)); 487} 488 489static __inline u_int 490rdr0(void) 491{ 492 u_int data; 493 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 494 return (data); 495} 496 497static __inline void 498load_dr0(u_int dr0) 499{ 500 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 501} 502 503static __inline u_int 504rdr1(void) 505{ 506 u_int data; 507 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 508 return (data); 509} 510 511static __inline void 512load_dr1(u_int dr1) 513{ 514 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 515} 516 517static __inline u_int 518rdr2(void) 519{ 520 u_int data; 521 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 522 return (data); 523} 524 525static __inline void 526load_dr2(u_int dr2) 527{ 528 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 529} 530 531static __inline u_int 532rdr3(void) 533{ 534 u_int data; 535 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 536 return (data); 537} 538 539static __inline void 540load_dr3(u_int dr3) 541{ 542 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 543} 544 545static __inline u_int 546rdr4(void) 547{ 548 u_int data; 549 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 550 return (data); 551} 552 553static __inline void 554load_dr4(u_int dr4) 555{ 556 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 557} 558 559static __inline u_int 560rdr5(void) 561{ 562 u_int data; 563 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 564 return (data); 565} 566 567static __inline void 568load_dr5(u_int dr5) 569{ 570 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 571} 572 573static __inline u_int 574rdr6(void) 575{ 576 u_int data; 577 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 578 return (data); 579} 580 581static __inline void 582load_dr6(u_int dr6) 583{ 584 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 585} 586 587static __inline u_int 588rdr7(void) 589{ 590 u_int data; 591 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 592 return (data); 593} 594 595static __inline void 596load_dr7(u_int dr7) 597{ 598 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 599} 600 601static __inline register_t 602intr_disable(void) 603{ 604 register_t eflags; 605 606 eflags = read_eflags(); 607 disable_intr(); 608 return (eflags); 609} 610 611static __inline void 612intr_restore(register_t eflags) 613{ 614 write_eflags(eflags); 615} 616 617#else /* !(__GNUC__ || __INTEL_COMPILER) */ 618 619int breakpoint(void); 620u_int bsfl(u_int mask); 621u_int bsrl(u_int mask); 622void disable_intr(void); 623void do_cpuid(u_int ax, u_int *p); 624void enable_intr(void); 625void halt(void); 626void ia32_pause(void); 627u_char inb(u_int port); 628u_int inl(u_int port); 629void insb(u_int port, void *addr, size_t cnt); 630void insl(u_int port, void *addr, size_t cnt); 631void insw(u_int port, void *addr, size_t cnt); 632register_t intr_disable(void); 633void intr_restore(register_t ef); 634void invd(void); 635void invlpg(u_int addr); 636void invltlb(void); 637u_short inw(u_int port); 638void lidt(struct region_descriptor *addr); 639void lldt(u_short sel); 640void load_cr0(u_int cr0); 641void load_cr3(u_int cr3); 642void load_cr4(u_int cr4); 643void load_dr0(u_int dr0); 644void load_dr1(u_int dr1); 645void load_dr2(u_int dr2); 646void load_dr3(u_int dr3); 647void load_dr4(u_int dr4); 648void load_dr5(u_int dr5); 649void load_dr6(u_int dr6); 650void load_dr7(u_int dr7); 651void load_fs(u_int sel); 652void load_gs(u_int sel); 653void ltr(u_short sel); 654void outb(u_int port, u_char data); 655void outl(u_int port, u_int data); 656void outsb(u_int port, const void *addr, size_t cnt); 657void outsl(u_int port, const void *addr, size_t cnt); 658void outsw(u_int port, const void *addr, size_t cnt); 659void outw(u_int port, u_short data); 660u_int rcr0(void); 661u_int rcr2(void); 662u_int rcr3(void); 663u_int rcr4(void); 664u_int64_t rdmsr(u_int msr); 665u_int64_t rdpmc(u_int pmc); 666u_int rdr0(void); 667u_int rdr1(void); 668u_int rdr2(void); 669u_int rdr3(void); 670u_int rdr4(void); 671u_int rdr5(void); 672u_int rdr6(void); 673u_int rdr7(void); 674u_int64_t rdtsc(void); 675u_int read_eflags(void); 676u_int rfs(void); 677u_int rgs(void); 678void wbinvd(void); 679void write_eflags(u_int ef); 680void wrmsr(u_int msr, u_int64_t newval); 681 682#endif /* __GNUC__ || __INTEL_COMPILER */ 683 684void reset_dbregs(void); 685 686#endif /* !_MACHINE_CPUFUNC_H_ */ 687