cpufunc.h revision 92761
1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by the University of 16 * California, Berkeley and its contributors. 17 * 4. Neither the name of the University nor the names of its contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 * 33 * $FreeBSD: head/sys/i386/include/cpufunc.h 92761 2002-03-20 05:48:58Z alfred $ 34 */ 35 36/* 37 * Functions to provide access to special i386 instructions. 38 * This in included in sys/systm.h, and that file should be 39 * used in preference to this. 40 */ 41 42#ifndef _MACHINE_CPUFUNC_H_ 43#define _MACHINE_CPUFUNC_H_ 44 45#include <sys/cdefs.h> 46#include <machine/psl.h> 47 48__BEGIN_DECLS 49#define readb(va) (*(volatile u_int8_t *) (va)) 50#define readw(va) (*(volatile u_int16_t *) (va)) 51#define readl(va) (*(volatile u_int32_t *) (va)) 52 53#define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 54#define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 55#define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 56 57#define CRITICAL_FORK (read_eflags() | PSL_I) 58 59#ifdef __GNUC__ 60 61#ifdef SWTCH_OPTIM_STATS 62extern int tlb_flush_count; /* XXX */ 63#endif 64 65static __inline void 66breakpoint(void) 67{ 68 __asm __volatile("int $3"); 69} 70 71static __inline u_int 72bsfl(u_int mask) 73{ 74 u_int result; 75 76 __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 77 return (result); 78} 79 80static __inline u_int 81bsrl(u_int mask) 82{ 83 u_int result; 84 85 __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 86 return (result); 87} 88 89static __inline void 90disable_intr(void) 91{ 92 __asm __volatile("cli" : : : "memory"); 93} 94 95static __inline void 96enable_intr(void) 97{ 98 __asm __volatile("sti"); 99} 100 101#define HAVE_INLINE_FFS 102 103static __inline int 104ffs(int mask) 105{ 106 /* 107 * Note that gcc-2's builtin ffs would be used if we didn't declare 108 * this inline or turn off the builtin. The builtin is faster but 109 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 110 * versions. 111 */ 112 return (mask == 0 ? mask : bsfl((u_int)mask) + 1); 113} 114 115#define HAVE_INLINE_FLS 116 117static __inline int 118fls(int mask) 119{ 120 return (mask == 0 ? mask : bsrl((u_int)mask) + 1); 121} 122 123#if __GNUC__ < 2 124 125#define inb(port) inbv(port) 126#define outb(port, data) outbv(port, data) 127 128#else /* __GNUC >= 2 */ 129 130/* 131 * The following complications are to get around gcc not having a 132 * constraint letter for the range 0..255. We still put "d" in the 133 * constraint because "i" isn't a valid constraint when the port 134 * isn't constant. This only matters for -O0 because otherwise 135 * the non-working version gets optimized away. 136 * 137 * Use an expression-statement instead of a conditional expression 138 * because gcc-2.6.0 would promote the operands of the conditional 139 * and produce poor code for "if ((inb(var) & const1) == const2)". 140 * 141 * The unnecessary test `(port) < 0x10000' is to generate a warning if 142 * the `port' has type u_short or smaller. Such types are pessimal. 143 * This actually only works for signed types. The range check is 144 * careful to avoid generating warnings. 145 */ 146#define inb(port) __extension__ ({ \ 147 u_char _data; \ 148 if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 149 && (port) < 0x10000) \ 150 _data = inbc(port); \ 151 else \ 152 _data = inbv(port); \ 153 _data; }) 154 155#define outb(port, data) ( \ 156 __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 157 && (port) < 0x10000 \ 158 ? outbc(port, data) : outbv(port, data)) 159 160static __inline u_char 161inbc(u_int port) 162{ 163 u_char data; 164 165 __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 166 return (data); 167} 168 169static __inline void 170outbc(u_int port, u_char data) 171{ 172 __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 173} 174 175#endif /* __GNUC <= 2 */ 176 177static __inline u_char 178inbv(u_int port) 179{ 180 u_char data; 181 /* 182 * We use %%dx and not %1 here because i/o is done at %dx and not at 183 * %edx, while gcc generates inferior code (movw instead of movl) 184 * if we tell it to load (u_short) port. 185 */ 186 __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 187 return (data); 188} 189 190static __inline u_int 191inl(u_int port) 192{ 193 u_int data; 194 195 __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 196 return (data); 197} 198 199static __inline void 200insb(u_int port, void *addr, size_t cnt) 201{ 202 __asm __volatile("cld; rep; insb" 203 : "+D" (addr), "+c" (cnt) 204 : "d" (port) 205 : "memory"); 206} 207 208static __inline void 209insw(u_int port, void *addr, size_t cnt) 210{ 211 __asm __volatile("cld; rep; insw" 212 : "+D" (addr), "+c" (cnt) 213 : "d" (port) 214 : "memory"); 215} 216 217static __inline void 218insl(u_int port, void *addr, size_t cnt) 219{ 220 __asm __volatile("cld; rep; insl" 221 : "+D" (addr), "+c" (cnt) 222 : "d" (port) 223 : "memory"); 224} 225 226static __inline void 227invd(void) 228{ 229 __asm __volatile("invd"); 230} 231 232#if defined(SMP) && defined(_KERNEL) 233 234/* 235 * When using APIC IPI's, invlpg() is not simply the invlpg instruction 236 * (this is a bug) and the inlining cost is prohibitive since the call 237 * executes into the IPI transmission system. 238 */ 239void invlpg (u_int addr); 240void invltlb (void); 241 242static __inline void 243cpu_invlpg(void *addr) 244{ 245 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 246} 247 248static __inline void 249cpu_invltlb(void) 250{ 251 u_int temp; 252 /* 253 * This should be implemented as load_cr3(rcr3()) when load_cr3() 254 * is inlined. 255 */ 256 __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp) 257 : : "memory"); 258#if defined(SWTCH_OPTIM_STATS) 259 ++tlb_flush_count; 260#endif 261} 262 263#else /* !(SMP && _KERNEL) */ 264 265static __inline void 266invlpg(u_int addr) 267{ 268 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 269} 270 271static __inline void 272invltlb(void) 273{ 274 u_int temp; 275 /* 276 * This should be implemented as load_cr3(rcr3()) when load_cr3() 277 * is inlined. 278 */ 279 __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp) 280 : : "memory"); 281#ifdef SWTCH_OPTIM_STATS 282 ++tlb_flush_count; 283#endif 284} 285 286#endif /* SMP && _KERNEL */ 287 288static __inline u_short 289inw(u_int port) 290{ 291 u_short data; 292 293 __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 294 return (data); 295} 296 297static __inline void 298outbv(u_int port, u_char data) 299{ 300 u_char al; 301 /* 302 * Use an unnecessary assignment to help gcc's register allocator. 303 * This make a large difference for gcc-1.40 and a tiny difference 304 * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 305 * best results. gcc-2.6.0 can't handle this. 306 */ 307 al = data; 308 __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 309} 310 311static __inline void 312outl(u_int port, u_int data) 313{ 314 /* 315 * outl() and outw() aren't used much so we haven't looked at 316 * possible micro-optimizations such as the unnecessary 317 * assignment for them. 318 */ 319 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 320} 321 322static __inline void 323outsb(u_int port, const void *addr, size_t cnt) 324{ 325 __asm __volatile("cld; rep; outsb" 326 : "+S" (addr), "+c" (cnt) 327 : "d" (port)); 328} 329 330static __inline void 331outsw(u_int port, const void *addr, size_t cnt) 332{ 333 __asm __volatile("cld; rep; outsw" 334 : "+S" (addr), "+c" (cnt) 335 : "d" (port)); 336} 337 338static __inline void 339outsl(u_int port, const void *addr, size_t cnt) 340{ 341 __asm __volatile("cld; rep; outsl" 342 : "+S" (addr), "+c" (cnt) 343 : "d" (port)); 344} 345 346static __inline void 347outw(u_int port, u_short data) 348{ 349 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 350} 351 352static __inline u_int 353rcr2(void) 354{ 355 u_int data; 356 357 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 358 return (data); 359} 360 361static __inline u_int 362read_eflags(void) 363{ 364 u_int ef; 365 366 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 367 return (ef); 368} 369 370static __inline void 371do_cpuid(u_int ax, u_int *p) 372{ 373 __asm __volatile( 374 "cpuid" 375 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 376 : "0" (ax) 377 ); 378} 379 380static __inline u_int64_t 381rdmsr(u_int msr) 382{ 383 u_int64_t rv; 384 385 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 386 return (rv); 387} 388 389static __inline u_int64_t 390rdpmc(u_int pmc) 391{ 392 u_int64_t rv; 393 394 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 395 return (rv); 396} 397 398static __inline u_int64_t 399rdtsc(void) 400{ 401 u_int64_t rv; 402 403 __asm __volatile("rdtsc" : "=A" (rv)); 404 return (rv); 405} 406 407static __inline void 408wbinvd(void) 409{ 410 __asm __volatile("wbinvd"); 411} 412 413static __inline void 414write_eflags(u_int ef) 415{ 416 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 417} 418 419static __inline void 420wrmsr(u_int msr, u_int64_t newval) 421{ 422 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 423} 424 425static __inline u_int 426rfs(void) 427{ 428 u_int sel; 429 __asm __volatile("movl %%fs,%0" : "=rm" (sel)); 430 return (sel); 431} 432 433static __inline u_int 434rgs(void) 435{ 436 u_int sel; 437 __asm __volatile("movl %%gs,%0" : "=rm" (sel)); 438 return (sel); 439} 440 441static __inline void 442load_fs(u_int sel) 443{ 444 __asm __volatile("movl %0,%%fs" : : "rm" (sel)); 445} 446 447static __inline void 448load_gs(u_int sel) 449{ 450 __asm __volatile("movl %0,%%gs" : : "rm" (sel)); 451} 452 453static __inline u_int 454rdr0(void) 455{ 456 u_int data; 457 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 458 return (data); 459} 460 461static __inline void 462load_dr0(u_int sel) 463{ 464 __asm __volatile("movl %0,%%dr0" : : "r" (sel)); 465} 466 467static __inline u_int 468rdr1(void) 469{ 470 u_int data; 471 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 472 return (data); 473} 474 475static __inline void 476load_dr1(u_int sel) 477{ 478 __asm __volatile("movl %0,%%dr1" : : "r" (sel)); 479} 480 481static __inline u_int 482rdr2(void) 483{ 484 u_int data; 485 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 486 return (data); 487} 488 489static __inline void 490load_dr2(u_int sel) 491{ 492 __asm __volatile("movl %0,%%dr2" : : "r" (sel)); 493} 494 495static __inline u_int 496rdr3(void) 497{ 498 u_int data; 499 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 500 return (data); 501} 502 503static __inline void 504load_dr3(u_int sel) 505{ 506 __asm __volatile("movl %0,%%dr3" : : "r" (sel)); 507} 508 509static __inline u_int 510rdr4(void) 511{ 512 u_int data; 513 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 514 return (data); 515} 516 517static __inline void 518load_dr4(u_int sel) 519{ 520 __asm __volatile("movl %0,%%dr4" : : "r" (sel)); 521} 522 523static __inline u_int 524rdr5(void) 525{ 526 u_int data; 527 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 528 return (data); 529} 530 531static __inline void 532load_dr5(u_int sel) 533{ 534 __asm __volatile("movl %0,%%dr5" : : "r" (sel)); 535} 536 537static __inline u_int 538rdr6(void) 539{ 540 u_int data; 541 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 542 return (data); 543} 544 545static __inline void 546load_dr6(u_int sel) 547{ 548 __asm __volatile("movl %0,%%dr6" : : "r" (sel)); 549} 550 551static __inline u_int 552rdr7(void) 553{ 554 u_int data; 555 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 556 return (data); 557} 558 559static __inline void 560load_dr7(u_int sel) 561{ 562 __asm __volatile("movl %0,%%dr7" : : "r" (sel)); 563} 564 565static __inline critical_t 566cpu_critical_enter(void) 567{ 568 critical_t eflags; 569 570 eflags = read_eflags(); 571 disable_intr(); 572 return (eflags); 573} 574 575static __inline void 576cpu_critical_exit(critical_t eflags) 577{ 578 write_eflags(eflags); 579} 580 581#else /* !__GNUC__ */ 582 583int breakpoint (void); 584u_int bsfl (u_int mask); 585u_int bsrl (u_int mask); 586void disable_intr (void); 587void do_cpuid (u_int ax, u_int *p); 588void enable_intr (void); 589u_char inb (u_int port); 590u_int inl (u_int port); 591void insb (u_int port, void *addr, size_t cnt); 592void insl (u_int port, void *addr, size_t cnt); 593void insw (u_int port, void *addr, size_t cnt); 594void invd (void); 595void invlpg (u_int addr); 596void invltlb (void); 597u_short inw (u_int port); 598void outb (u_int port, u_char data); 599void outl (u_int port, u_int data); 600void outsb (u_int port, void *addr, size_t cnt); 601void outsl (u_int port, void *addr, size_t cnt); 602void outsw (u_int port, void *addr, size_t cnt); 603void outw (u_int port, u_short data); 604u_int rcr2 (void); 605u_int64_t rdmsr (u_int msr); 606u_int64_t rdpmc (u_int pmc); 607u_int64_t rdtsc (void); 608u_int read_eflags (void); 609void wbinvd (void); 610void write_eflags (u_int ef); 611void wrmsr (u_int msr, u_int64_t newval); 612u_int rfs (void); 613u_int rgs (void); 614void load_fs (u_int sel); 615void load_gs (u_int sel); 616critical_t cpu_critical_enter(void); 617void cpu_critical_exit(critical_t eflags); 618 619#endif /* __GNUC__ */ 620 621void load_cr0 (u_int cr0); 622void load_cr3 (u_int cr3); 623void load_cr4 (u_int cr4); 624void ltr (u_short sel); 625u_int rcr0 (void); 626u_int rcr3 (void); 627u_int rcr4 (void); 628void reset_dbregs(void); 629__END_DECLS 630 631#endif /* !_MACHINE_CPUFUNC_H_ */ 632