cpufunc.h revision 128019
12736Sdfr/*- 22736Sdfr * Copyright (c) 1993 The Regents of the University of California. 32736Sdfr * All rights reserved. 42736Sdfr * 52736Sdfr * Redistribution and use in source and binary forms, with or without 62736Sdfr * modification, are permitted provided that the following conditions 72736Sdfr * are met: 82736Sdfr * 1. Redistributions of source code must retain the above copyright 92736Sdfr * notice, this list of conditions and the following disclaimer. 102736Sdfr * 2. Redistributions in binary form must reproduce the above copyright 112736Sdfr * notice, this list of conditions and the following disclaimer in the 122736Sdfr * documentation and/or other materials provided with the distribution. 132736Sdfr * 4. Neither the name of the University nor the names of its contributors 142736Sdfr * may be used to endorse or promote products derived from this software 152736Sdfr * without specific prior written permission. 162736Sdfr * 172736Sdfr * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 182736Sdfr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 192736Sdfr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 202736Sdfr * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 212736Sdfr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 222736Sdfr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 232736Sdfr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 242736Sdfr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 252736Sdfr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 262736Sdfr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 272736Sdfr * SUCH DAMAGE. 282736Sdfr * 292736Sdfr * $FreeBSD: head/sys/i386/include/cpufunc.h 128019 2004-04-07 20:46:16Z imp $ 30174891Sedwin */ 312736Sdfr 322736Sdfr/* 3399112Sobrien * Functions to provide access to special i386 instructions. 3499112Sobrien * This in included in sys/systm.h, and that file should be 3527420Scharnier * used in preference to this. 36174891Sedwin */ 37174891Sedwin 38174750Sedwin#ifndef _MACHINE_CPUFUNC_H_ 39174750Sedwin#define _MACHINE_CPUFUNC_H_ 40174891Sedwin 41174891Sedwinstruct region_descriptor; 42174750Sedwin 4327420Scharnier#define readb(va) (*(volatile u_int8_t *) (va)) 4427420Scharnier#define readw(va) (*(volatile u_int16_t *) (va)) 45174891Sedwin#define readl(va) (*(volatile u_int32_t *) (va)) 46174891Sedwin 472736Sdfr#define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 482736Sdfr#define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 492736Sdfr#define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 502736Sdfr 51174891Sedwin#if defined(__GNUC__) || defined(__INTEL_COMPILER) 522736Sdfr 53174891Sedwinstatic __inline void 54174891Sedwinbreakpoint(void) 55174891Sedwin{ 562736Sdfr __asm __volatile("int $3"); 57174891Sedwin} 5887285Sdwmalone 59174891Sedwinstatic __inline u_int 60174891Sedwinbsfl(u_int mask) 61174891Sedwin{ 62174891Sedwin u_int result; 63174891Sedwin 64174750Sedwin __asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask)); 65174750Sedwin return (result); 662736Sdfr} 67174750Sedwin 68174750Sedwinstatic __inline u_int 69174891Sedwinbsrl(u_int mask) 70174891Sedwin{ 71174750Sedwin u_int result; 722736Sdfr 732736Sdfr __asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask)); 742736Sdfr return (result); 75174750Sedwin} 76174750Sedwin 772736Sdfrstatic __inline void 78174750Sedwindisable_intr(void) 79174891Sedwin{ 80174891Sedwin __asm __volatile("cli" : : : "memory"); 81174891Sedwin} 82174891Sedwin 83174891Sedwinstatic __inline void 84174891Sedwindo_cpuid(u_int ax, u_int *p) 85174891Sedwin{ 86174891Sedwin __asm __volatile("cpuid" 87174891Sedwin : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 88174891Sedwin : "0" (ax)); 89174891Sedwin} 90174891Sedwin 91174891Sedwinstatic __inline void 92174891Sedwinenable_intr(void) 93174891Sedwin{ 94174891Sedwin __asm __volatile("sti"); 95174891Sedwin} 96174891Sedwin 97174891Sedwin#ifdef _KERNEL 98174891Sedwin 99174891Sedwin#define HAVE_INLINE_FFS 100174891Sedwin 101174891Sedwinstatic __inline int 102174891Sedwinffs(int mask) 103174891Sedwin{ 104174891Sedwin /* 105174891Sedwin * Note that gcc-2's builtin ffs would be used if we didn't declare 106174891Sedwin * this inline or turn off the builtin. The builtin is faster but 107174750Sedwin * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 108174750Sedwin * versions. 109174750Sedwin */ 110174750Sedwin return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 111174750Sedwin} 112174891Sedwin 113174750Sedwin#define HAVE_INLINE_FLS 1142736Sdfr 1152736Sdfrstatic __inline int 116174750Sedwinfls(int mask) 117174750Sedwin{ 1182736Sdfr return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 119174750Sedwin} 120174891Sedwin 121174891Sedwin#endif /* _KERNEL */ 122174891Sedwin 123174891Sedwinstatic __inline void 124174891Sedwinhalt(void) 125174891Sedwin{ 126174891Sedwin __asm __volatile("hlt"); 127174891Sedwin} 128174891Sedwin 129174891Sedwin#if __GNUC__ < 2 130174891Sedwin 131174891Sedwin#define inb(port) inbv(port) 132174891Sedwin#define outb(port, data) outbv(port, data) 133174891Sedwin 134174891Sedwin#else /* __GNUC >= 2 */ 135174891Sedwin 136174891Sedwin/* 137174891Sedwin * The following complications are to get around gcc not having a 138174891Sedwin * constraint letter for the range 0..255. We still put "d" in the 139174891Sedwin * constraint because "i" isn't a valid constraint when the port 140174891Sedwin * isn't constant. This only matters for -O0 because otherwise 141174891Sedwin * the non-working version gets optimized away. 142174891Sedwin * 143174891Sedwin * Use an expression-statement instead of a conditional expression 144174891Sedwin * because gcc-2.6.0 would promote the operands of the conditional 145174891Sedwin * and produce poor code for "if ((inb(var) & const1) == const2)". 146174891Sedwin * 147174891Sedwin * The unnecessary test `(port) < 0x10000' is to generate a warning if 148174750Sedwin * the `port' has type u_short or smaller. Such types are pessimal. 149174750Sedwin * This actually only works for signed types. The range check is 150174750Sedwin * careful to avoid generating warnings. 151174750Sedwin */ 152174750Sedwin#define inb(port) __extension__ ({ \ 153174891Sedwin u_char _data; \ 154174750Sedwin if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 1552736Sdfr && (port) < 0x10000) \ 1562736Sdfr _data = inbc(port); \ 157174750Sedwin else \ 158174750Sedwin _data = inbv(port); \ 1592736Sdfr _data; }) 160174750Sedwin 1612736Sdfr#define outb(port, data) ( \ 162174891Sedwin __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 163174891Sedwin && (port) < 0x10000 \ 164174891Sedwin ? outbc(port, data) : outbv(port, data)) 165174891Sedwin 166174891Sedwinstatic __inline u_char 167174891Sedwininbc(u_int port) 168174891Sedwin{ 169174891Sedwin u_char data; 170174891Sedwin 171174891Sedwin __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 172174891Sedwin return (data); 173174891Sedwin} 174174891Sedwin 175174891Sedwinstatic __inline void 176174891Sedwinoutbc(u_int port, u_char data) 177174891Sedwin{ 178174891Sedwin __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 179174891Sedwin} 180174891Sedwin 181174891Sedwin#endif /* __GNUC <= 2 */ 182174891Sedwin 183174891Sedwinstatic __inline u_char 184174891Sedwininbv(u_int port) 185174891Sedwin{ 186174891Sedwin u_char data; 187174891Sedwin /* 188174891Sedwin * We use %%dx and not %1 here because i/o is done at %dx and not at 189174891Sedwin * %edx, while gcc generates inferior code (movw instead of movl) 190174750Sedwin * if we tell it to load (u_short) port. 191174750Sedwin */ 192174750Sedwin __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 193174750Sedwin return (data); 194174750Sedwin} 195174891Sedwin 196174750Sedwinstatic __inline u_int 1972736Sdfrinl(u_int port) 1982736Sdfr{ 199174750Sedwin u_int data; 200174750Sedwin 2012736Sdfr __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 202174750Sedwin return (data); 203174750Sedwin} 2042736Sdfr 2058874Srgrimesstatic __inline void 206174750Sedwininsb(u_int port, void *addr, size_t cnt) 207174750Sedwin{ 2082736Sdfr __asm __volatile("cld; rep; insb" 209174891Sedwin : "+D" (addr), "+c" (cnt) 210174750Sedwin : "d" (port) 2112736Sdfr : "memory"); 212174891Sedwin} 213174891Sedwin 214174891Sedwinstatic __inline void 215174891Sedwininsw(u_int port, void *addr, size_t cnt) 216174891Sedwin{ 217174891Sedwin __asm __volatile("cld; rep; insw" 218174891Sedwin : "+D" (addr), "+c" (cnt) 219174891Sedwin : "d" (port) 220174891Sedwin : "memory"); 221174891Sedwin} 222174891Sedwin 223174891Sedwinstatic __inline void 224174891Sedwininsl(u_int port, void *addr, size_t cnt) 225174891Sedwin{ 226174750Sedwin __asm __volatile("cld; rep; insl" 227174750Sedwin : "+D" (addr), "+c" (cnt) 228174891Sedwin : "d" (port) 2292736Sdfr : "memory"); 230174750Sedwin} 231174750Sedwin 232174750Sedwinstatic __inline void 233174750Sedwininvd(void) 234174750Sedwin{ 235174750Sedwin __asm __volatile("invd"); 236174750Sedwin} 237174750Sedwin 238174750Sedwinstatic __inline u_short 239174750Sedwininw(u_int port) 240174750Sedwin{ 241174750Sedwin u_short data; 242174750Sedwin 243174750Sedwin __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 244174750Sedwin return (data); 245174750Sedwin} 246174750Sedwin 247174750Sedwinstatic __inline void 248174750Sedwinoutbv(u_int port, u_char data) 249174750Sedwin{ 250174750Sedwin u_char al; 251174750Sedwin /* 252174750Sedwin * Use an unnecessary assignment to help gcc's register allocator. 253174750Sedwin * This make a large difference for gcc-1.40 and a tiny difference 254174750Sedwin * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 255174750Sedwin * best results. gcc-2.6.0 can't handle this. 256174750Sedwin */ 257174750Sedwin al = data; 258174750Sedwin __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 259174750Sedwin} 260174750Sedwin 261174750Sedwinstatic __inline void 262174750Sedwinoutl(u_int port, u_int data) 263174750Sedwin{ 264174750Sedwin /* 265174750Sedwin * outl() and outw() aren't used much so we haven't looked at 266174750Sedwin * possible micro-optimizations such as the unnecessary 267174750Sedwin * assignment for them. 268174750Sedwin */ 269174750Sedwin __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 270174750Sedwin} 271174750Sedwin 272174750Sedwinstatic __inline void 273174750Sedwinoutsb(u_int port, const void *addr, size_t cnt) 274174750Sedwin{ 275174750Sedwin __asm __volatile("cld; rep; outsb" 276174750Sedwin : "+S" (addr), "+c" (cnt) 277174750Sedwin : "d" (port)); 278174750Sedwin} 279174750Sedwin 280174891Sedwinstatic __inline void 281174891Sedwinoutsw(u_int port, const void *addr, size_t cnt) 282174891Sedwin{ 283174891Sedwin __asm __volatile("cld; rep; outsw" 284174891Sedwin : "+S" (addr), "+c" (cnt) 285174891Sedwin : "d" (port)); 286174891Sedwin} 287174891Sedwin 288174891Sedwinstatic __inline void 289174750Sedwinoutsl(u_int port, const void *addr, size_t cnt) 290174750Sedwin{ 291174750Sedwin __asm __volatile("cld; rep; outsl" 292174750Sedwin : "+S" (addr), "+c" (cnt) 293174750Sedwin : "d" (port)); 294174750Sedwin} 295174750Sedwin 296174750Sedwinstatic __inline void 2972736Sdfroutw(u_int port, u_short data) 2982736Sdfr{ 299174750Sedwin __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 300174750Sedwin} 301174750Sedwin 302174750Sedwinstatic __inline void 303174750Sedwinia32_pause(void) 3042736Sdfr{ 305 __asm __volatile("pause"); 306} 307 308static __inline u_int 309read_eflags(void) 310{ 311 u_int ef; 312 313 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 314 return (ef); 315} 316 317static __inline u_int64_t 318rdmsr(u_int msr) 319{ 320 u_int64_t rv; 321 322 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 323 return (rv); 324} 325 326static __inline u_int64_t 327rdpmc(u_int pmc) 328{ 329 u_int64_t rv; 330 331 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 332 return (rv); 333} 334 335static __inline u_int64_t 336rdtsc(void) 337{ 338 u_int64_t rv; 339 340 __asm __volatile("rdtsc" : "=A" (rv)); 341 return (rv); 342} 343 344static __inline void 345wbinvd(void) 346{ 347 __asm __volatile("wbinvd"); 348} 349 350static __inline void 351write_eflags(u_int ef) 352{ 353 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 354} 355 356static __inline void 357wrmsr(u_int msr, u_int64_t newval) 358{ 359 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 360} 361 362static __inline void 363load_cr0(u_int data) 364{ 365 366 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 367} 368 369static __inline u_int 370rcr0(void) 371{ 372 u_int data; 373 374 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 375 return (data); 376} 377 378static __inline u_int 379rcr2(void) 380{ 381 u_int data; 382 383 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 384 return (data); 385} 386 387static __inline void 388load_cr3(u_int data) 389{ 390 391 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 392} 393 394static __inline u_int 395rcr3(void) 396{ 397 u_int data; 398 399 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 400 return (data); 401} 402 403static __inline void 404load_cr4(u_int data) 405{ 406 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 407} 408 409static __inline u_int 410rcr4(void) 411{ 412 u_int data; 413 414 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 415 return (data); 416} 417 418/* 419 * Global TLB flush (except for thise for pages marked PG_G) 420 */ 421static __inline void 422invltlb(void) 423{ 424 425 load_cr3(rcr3()); 426} 427 428/* 429 * TLB flush for an individual page (even if it has PG_G). 430 * Only works on 486+ CPUs (i386 does not have PG_G). 431 */ 432static __inline void 433invlpg(u_int addr) 434{ 435 436 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 437} 438 439static __inline u_int 440rfs(void) 441{ 442 u_int sel; 443 __asm __volatile("movl %%fs,%0" : "=rm" (sel)); 444 return (sel); 445} 446 447static __inline u_int 448rgs(void) 449{ 450 u_int sel; 451 __asm __volatile("movl %%gs,%0" : "=rm" (sel)); 452 return (sel); 453} 454 455static __inline u_int 456rss(void) 457{ 458 u_int sel; 459 __asm __volatile("movl %%ss,%0" : "=rm" (sel)); 460 return (sel); 461} 462 463static __inline void 464load_fs(u_int sel) 465{ 466 __asm __volatile("movl %0,%%fs" : : "rm" (sel)); 467} 468 469static __inline void 470load_gs(u_int sel) 471{ 472 __asm __volatile("movl %0,%%gs" : : "rm" (sel)); 473} 474 475static __inline void 476lidt(struct region_descriptor *addr) 477{ 478 __asm __volatile("lidt (%0)" : : "r" (addr)); 479} 480 481static __inline void 482lldt(u_short sel) 483{ 484 __asm __volatile("lldt %0" : : "r" (sel)); 485} 486 487static __inline void 488ltr(u_short sel) 489{ 490 __asm __volatile("ltr %0" : : "r" (sel)); 491} 492 493static __inline u_int 494rdr0(void) 495{ 496 u_int data; 497 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 498 return (data); 499} 500 501static __inline void 502load_dr0(u_int dr0) 503{ 504 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 505} 506 507static __inline u_int 508rdr1(void) 509{ 510 u_int data; 511 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 512 return (data); 513} 514 515static __inline void 516load_dr1(u_int dr1) 517{ 518 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 519} 520 521static __inline u_int 522rdr2(void) 523{ 524 u_int data; 525 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 526 return (data); 527} 528 529static __inline void 530load_dr2(u_int dr2) 531{ 532 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 533} 534 535static __inline u_int 536rdr3(void) 537{ 538 u_int data; 539 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 540 return (data); 541} 542 543static __inline void 544load_dr3(u_int dr3) 545{ 546 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 547} 548 549static __inline u_int 550rdr4(void) 551{ 552 u_int data; 553 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 554 return (data); 555} 556 557static __inline void 558load_dr4(u_int dr4) 559{ 560 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 561} 562 563static __inline u_int 564rdr5(void) 565{ 566 u_int data; 567 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 568 return (data); 569} 570 571static __inline void 572load_dr5(u_int dr5) 573{ 574 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 575} 576 577static __inline u_int 578rdr6(void) 579{ 580 u_int data; 581 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 582 return (data); 583} 584 585static __inline void 586load_dr6(u_int dr6) 587{ 588 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 589} 590 591static __inline u_int 592rdr7(void) 593{ 594 u_int data; 595 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 596 return (data); 597} 598 599static __inline void 600load_dr7(u_int dr7) 601{ 602 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 603} 604 605static __inline register_t 606intr_disable(void) 607{ 608 register_t eflags; 609 610 eflags = read_eflags(); 611 disable_intr(); 612 return (eflags); 613} 614 615static __inline void 616intr_restore(register_t eflags) 617{ 618 write_eflags(eflags); 619} 620 621#else /* !(__GNUC__ || __INTEL_COMPILER) */ 622 623int breakpoint(void); 624u_int bsfl(u_int mask); 625u_int bsrl(u_int mask); 626void disable_intr(void); 627void do_cpuid(u_int ax, u_int *p); 628void enable_intr(void); 629void halt(void); 630void ia32_pause(void); 631u_char inb(u_int port); 632u_int inl(u_int port); 633void insb(u_int port, void *addr, size_t cnt); 634void insl(u_int port, void *addr, size_t cnt); 635void insw(u_int port, void *addr, size_t cnt); 636register_t intr_disable(void); 637void intr_restore(register_t ef); 638void invd(void); 639void invlpg(u_int addr); 640void invltlb(void); 641u_short inw(u_int port); 642void lidt(struct region_descriptor *addr); 643void lldt(u_short sel); 644void load_cr0(u_int cr0); 645void load_cr3(u_int cr3); 646void load_cr4(u_int cr4); 647void load_dr0(u_int dr0); 648void load_dr1(u_int dr1); 649void load_dr2(u_int dr2); 650void load_dr3(u_int dr3); 651void load_dr4(u_int dr4); 652void load_dr5(u_int dr5); 653void load_dr6(u_int dr6); 654void load_dr7(u_int dr7); 655void load_fs(u_int sel); 656void load_gs(u_int sel); 657void ltr(u_short sel); 658void outb(u_int port, u_char data); 659void outl(u_int port, u_int data); 660void outsb(u_int port, const void *addr, size_t cnt); 661void outsl(u_int port, const void *addr, size_t cnt); 662void outsw(u_int port, const void *addr, size_t cnt); 663void outw(u_int port, u_short data); 664u_int rcr0(void); 665u_int rcr2(void); 666u_int rcr3(void); 667u_int rcr4(void); 668u_int64_t rdmsr(u_int msr); 669u_int64_t rdpmc(u_int pmc); 670u_int rdr0(void); 671u_int rdr1(void); 672u_int rdr2(void); 673u_int rdr3(void); 674u_int rdr4(void); 675u_int rdr5(void); 676u_int rdr6(void); 677u_int rdr7(void); 678u_int64_t rdtsc(void); 679u_int read_eflags(void); 680u_int rfs(void); 681u_int rgs(void); 682void wbinvd(void); 683void write_eflags(u_int ef); 684void wrmsr(u_int msr, u_int64_t newval); 685 686#endif /* __GNUC__ || __INTEL_COMPILER */ 687 688void reset_dbregs(void); 689 690#endif /* !_MACHINE_CPUFUNC_H_ */ 691