cpufunc.h revision 42410
1221163Sadrian/*- 2221163Sadrian * Copyright (c) 1993 The Regents of the University of California. 3221163Sadrian * All rights reserved. 4221163Sadrian * 5221163Sadrian * Redistribution and use in source and binary forms, with or without 6221163Sadrian * modification, are permitted provided that the following conditions 7221163Sadrian * are met: 8221163Sadrian * 1. Redistributions of source code must retain the above copyright 9221163Sadrian * notice, this list of conditions and the following disclaimer. 10221163Sadrian * 2. Redistributions in binary form must reproduce the above copyright 11221163Sadrian * notice, this list of conditions and the following disclaimer in the 12221163Sadrian * documentation and/or other materials provided with the distribution. 13221163Sadrian * 3. All advertising materials mentioning features or use of this software 14221163Sadrian * must display the following acknowledgement: 15221163Sadrian * This product includes software developed by the University of 16221163Sadrian * California, Berkeley and its contributors. 17221163Sadrian * 4. Neither the name of the University nor the names of its contributors 18221163Sadrian * may be used to endorse or promote products derived from this software 19221163Sadrian * without specific prior written permission. 20221163Sadrian * 21221163Sadrian * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22221163Sadrian * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23221163Sadrian * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24221163Sadrian * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25221163Sadrian * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26221163Sadrian * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27221163Sadrian * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28221163Sadrian * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29221163Sadrian * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30221163Sadrian * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31221163Sadrian * SUCH DAMAGE. 32221163Sadrian * 33221163Sadrian * $Id: cpufunc.h,v 1.82 1999/01/08 16:29:57 bde Exp $ 34221163Sadrian */ 35221163Sadrian 36221163Sadrian/* 37221163Sadrian * Functions to provide access to special i386 instructions. 38221163Sadrian */ 39221163Sadrian 40221163Sadrian#ifndef _MACHINE_CPUFUNC_H_ 41221163Sadrian#define _MACHINE_CPUFUNC_H_ 42221163Sadrian 43221163Sadrian#define readb(va) (*(volatile u_int8_t *) (va)) 44221163Sadrian#define readw(va) (*(volatile u_int16_t *) (va)) 45221163Sadrian#define readl(va) (*(volatile u_int32_t *) (va)) 46221163Sadrian 47221163Sadrian#define writeb(va, d) (*(volatile u_int8_t *) (va) = (d)) 48221163Sadrian#define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) 49221163Sadrian#define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) 50221163Sadrian 51221163Sadrian#ifdef __GNUC__ 52221163Sadrian 53221163Sadrian#ifdef SMP 54221163Sadrian#include <machine/lock.h> /* XXX */ 55221163Sadrian#endif 56221163Sadrian 57221163Sadrian#ifdef SWTCH_OPTIM_STATS 58221163Sadrianextern int tlb_flush_count; /* XXX */ 59221163Sadrian#endif 60221163Sadrian 61221163Sadrianstatic __inline void 62221163Sadrianbreakpoint(void) 63221163Sadrian{ 64221163Sadrian __asm __volatile("int $3"); 65221163Sadrian} 66221163Sadrian 67221163Sadrianstatic __inline void 68221163Sadriandisable_intr(void) 69221163Sadrian{ 70221163Sadrian __asm __volatile("cli" : : : "memory"); 71221163Sadrian#ifdef SMP 72221163Sadrian MPINTR_LOCK(); 73221163Sadrian#endif 74221163Sadrian} 75221163Sadrian 76221163Sadrianstatic __inline void 77221163Sadrianenable_intr(void) 78221163Sadrian{ 79221163Sadrian#ifdef SMP 80221163Sadrian MPINTR_UNLOCK(); 81225883Sadrian#endif 82221163Sadrian __asm __volatile("sti"); 83221163Sadrian} 84221163Sadrian 85221163Sadrian#define HAVE_INLINE_FFS 86221163Sadrian 87225883Sadrianstatic __inline int 88221163Sadrianffs(int mask) 89221163Sadrian{ 90221163Sadrian int result; 91221163Sadrian /* 92221163Sadrian * bsfl turns out to be not all that slow on 486's. It can beaten 93221163Sadrian * using a binary search to reduce to 4 bits and then a table lookup, 94221163Sadrian * but only if the code is inlined and in the cache, and the code 95221163Sadrian * is quite large so inlining it probably busts the cache. 96221163Sadrian * 97221163Sadrian * Note that gcc-2's builtin ffs would be used if we didn't declare 98221163Sadrian * this inline or turn off the builtin. The builtin is faster but 99221163Sadrian * broken in gcc-2.4.5 and slower but working in gcc-2.5 and 2.6. 100221163Sadrian */ 101221163Sadrian __asm __volatile("testl %0,%0; je 1f; bsfl %0,%0; incl %0; 1:" 102221163Sadrian : "=r" (result) : "0" (mask)); 103221163Sadrian return (result); 104221163Sadrian} 105221163Sadrian 106221163Sadrian#define HAVE_INLINE_FLS 107221163Sadrian 108221163Sadrianstatic __inline int 109221163Sadrianfls(int mask) 110221163Sadrian{ 111221163Sadrian int result; 112221163Sadrian __asm __volatile("testl %0,%0; je 1f; bsrl %0,%0; incl %0; 1:" 113221163Sadrian : "=r" (result) : "0" (mask)); 114221163Sadrian return (result); 115221163Sadrian} 116221163Sadrian 117221163Sadrian#if __GNUC__ < 2 118221163Sadrian 119221163Sadrian#define inb(port) inbv(port) 120221163Sadrian#define outb(port, data) outbv(port, data) 121221163Sadrian 122221163Sadrian#else /* __GNUC >= 2 */ 123221163Sadrian 124221163Sadrian/* 125221163Sadrian * The following complications are to get around gcc not having a 126221163Sadrian * constraint letter for the range 0..255. We still put "d" in the 127221163Sadrian * constraint because "i" isn't a valid constraint when the port 128221163Sadrian * isn't constant. This only matters for -O0 because otherwise 129221163Sadrian * the non-working version gets optimized away. 130221163Sadrian * 131221163Sadrian * Use an expression-statement instead of a conditional expression 132221163Sadrian * because gcc-2.6.0 would promote the operands of the conditional 133221163Sadrian * and produce poor code for "if ((inb(var) & const1) == const2)". 134221163Sadrian * 135221163Sadrian * The unnecessary test `(port) < 0x10000' is to generate a warning if 136221163Sadrian * the `port' has type u_short or smaller. Such types are pessimal. 137221163Sadrian * This actually only works for signed types. The range check is 138221163Sadrian * careful to avoid generating warnings. 139221163Sadrian */ 140221163Sadrian#define inb(port) __extension__ ({ \ 141221163Sadrian u_char _data; \ 142221163Sadrian if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 143221163Sadrian && (port) < 0x10000) \ 144221163Sadrian _data = inbc(port); \ 145221163Sadrian else \ 146221163Sadrian _data = inbv(port); \ 147221163Sadrian _data; }) 148221163Sadrian 149221163Sadrian#define outb(port, data) ( \ 150221163Sadrian __builtin_constant_p(port) && ((port) & 0xffff) < 0x100 \ 151221163Sadrian && (port) < 0x10000 \ 152221163Sadrian ? outbc(port, data) : outbv(port, data)) 153221163Sadrian 154221163Sadrianstatic __inline u_char 155221163Sadrianinbc(u_int port) 156221163Sadrian{ 157221163Sadrian u_char data; 158221163Sadrian 159221163Sadrian __asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port))); 160221163Sadrian return (data); 161221163Sadrian} 162221163Sadrian 163221163Sadrianstatic __inline void 164221163Sadrianoutbc(u_int port, u_char data) 165221163Sadrian{ 166221163Sadrian __asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port))); 167221163Sadrian} 168221163Sadrian 169221163Sadrian#endif /* __GNUC <= 2 */ 170221163Sadrian 171221163Sadrianstatic __inline u_char 172221163Sadrianinbv(u_int port) 173221163Sadrian{ 174221163Sadrian u_char data; 175221163Sadrian /* 176221163Sadrian * We use %%dx and not %1 here because i/o is done at %dx and not at 177221163Sadrian * %edx, while gcc generates inferior code (movw instead of movl) 178221163Sadrian * if we tell it to load (u_short) port. 179221163Sadrian */ 180221163Sadrian __asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port)); 181221163Sadrian return (data); 182221163Sadrian} 183221163Sadrian 184221163Sadrianstatic __inline u_int 185221163Sadrianinl(u_int port) 186221163Sadrian{ 187221163Sadrian u_int data; 188221163Sadrian 189221163Sadrian __asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port)); 190221163Sadrian return (data); 191221163Sadrian} 192221163Sadrian 193221163Sadrianstatic __inline void 194221163Sadrianinsb(u_int port, void *addr, size_t cnt) 195221163Sadrian{ 196221163Sadrian __asm __volatile("cld; rep; insb" 197221163Sadrian : : "d" (port), "D" (addr), "c" (cnt) 198221163Sadrian : "di", "cx", "memory"); 199221163Sadrian} 200221163Sadrian 201221163Sadrianstatic __inline void 202221163Sadrianinsw(u_int port, void *addr, size_t cnt) 203221163Sadrian{ 204221163Sadrian __asm __volatile("cld; rep; insw" 205221163Sadrian : : "d" (port), "D" (addr), "c" (cnt) 206221163Sadrian : "di", "cx", "memory"); 207221163Sadrian} 208221163Sadrian 209221163Sadrianstatic __inline void 210221163Sadrianinsl(u_int port, void *addr, size_t cnt) 211221163Sadrian{ 212221163Sadrian __asm __volatile("cld; rep; insl" 213221163Sadrian : : "d" (port), "D" (addr), "c" (cnt) 214221163Sadrian : "di", "cx", "memory"); 215221163Sadrian} 216221163Sadrian 217221163Sadrianstatic __inline void 218221163Sadrianinvd(void) 219221163Sadrian{ 220221163Sadrian __asm __volatile("invd"); 221221163Sadrian} 222221163Sadrian 223221163Sadrian#ifdef KERNEL 224221163Sadrian#ifdef SMP 225221163Sadrian 226221163Sadrian/* 227221163Sadrian * When using APIC IPI's, the inlining cost is prohibitive since the call 228221163Sadrian * executes into the IPI transmission system. 229221163Sadrian */ 230221163Sadrianvoid invlpg __P((u_int addr)); 231221163Sadrianvoid invltlb __P((void)); 232221163Sadrian 233221163Sadrianstatic __inline void 234221163Sadriancpu_invlpg(void *addr) 235221163Sadrian{ 236221163Sadrian __asm __volatile("invlpg %0"::"m"(*(char *)addr):"memory"); 237221163Sadrian} 238221163Sadrian 239221596Sadrianstatic __inline void 240221596Sadriancpu_invltlb(void) 241221163Sadrian{ 242221596Sadrian u_int temp; 243221163Sadrian /* 244221163Sadrian * This should be implemented as load_cr3(rcr3()) when load_cr3() 245221163Sadrian * is inlined. 246221163Sadrian */ 247221163Sadrian __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp) 248221163Sadrian : : "memory"); 249221163Sadrian#if defined(SWTCH_OPTIM_STATS) 250221163Sadrian ++tlb_flush_count; 251221163Sadrian#endif 252221163Sadrian} 253221163Sadrian#else /* !SMP */ 254221163Sadrian 255221163Sadrianstatic __inline void 256221163Sadrianinvlpg(u_int addr) 257221163Sadrian{ 258221163Sadrian __asm __volatile("invlpg %0"::"m"(*(char *)addr):"memory"); 259221163Sadrian} 260221163Sadrian 261221163Sadrian 262221163Sadrianstatic __inline void 263221163Sadrianinvltlb(void) 264221163Sadrian{ 265221163Sadrian u_int temp; 266221163Sadrian /* 267221163Sadrian * This should be implemented as load_cr3(rcr3()) when load_cr3() 268221163Sadrian * is inlined. 269221163Sadrian */ 270221163Sadrian __asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp) 271221163Sadrian : : "memory"); 272221163Sadrian#if defined(SWTCH_OPTIM_STATS) 273221163Sadrian ++tlb_flush_count; 274221163Sadrian#endif 275221163Sadrian} 276221163Sadrian 277221163Sadrian#endif /* SMP */ 278221163Sadrian#endif /* KERNEL */ 279221163Sadrian 280221163Sadrianstatic __inline u_short 281221163Sadrianinw(u_int port) 282221163Sadrian{ 283221163Sadrian u_short data; 284221163Sadrian 285221163Sadrian __asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port)); 286221163Sadrian return (data); 287221163Sadrian} 288221163Sadrian 289221163Sadrianstatic __inline u_int 290221163Sadrianloadandclear(u_int *addr) 291221163Sadrian{ 292222584Sadrian u_int result; 293221163Sadrian 294221603Sadrian __asm __volatile("xorl %0,%0; xchgl %1,%0" 295221603Sadrian : "=&r" (result) : "m" (*addr)); 296221603Sadrian return (result); 297221603Sadrian} 298221603Sadrian 299221603Sadrianstatic __inline void 300221603Sadrianoutbv(u_int port, u_char data) 301226488Sadrian{ 302226488Sadrian u_char al; 303221603Sadrian /* 304221163Sadrian * Use an unnecessary assignment to help gcc's register allocator. 305221163Sadrian * This make a large difference for gcc-1.40 and a tiny difference 306221163Sadrian * for gcc-2.6.0. For gcc-1.40, al had to be ``asm("ax")'' for 307221163Sadrian * best results. gcc-2.6.0 can't handle this. 308221163Sadrian */ 309221163Sadrian al = data; 310221163Sadrian __asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port)); 311221163Sadrian} 312221163Sadrian 313221163Sadrianstatic __inline void 314221163Sadrianoutl(u_int port, u_int data) 315{ 316 /* 317 * outl() and outw() aren't used much so we haven't looked at 318 * possible micro-optimizations such as the unnecessary 319 * assignment for them. 320 */ 321 __asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port)); 322} 323 324static __inline void 325outsb(u_int port, const void *addr, size_t cnt) 326{ 327 __asm __volatile("cld; rep; outsb" 328 : : "d" (port), "S" (addr), "c" (cnt) 329 : "si", "cx"); 330} 331 332static __inline void 333outsw(u_int port, const void *addr, size_t cnt) 334{ 335 __asm __volatile("cld; rep; outsw" 336 : : "d" (port), "S" (addr), "c" (cnt) 337 : "si", "cx"); 338} 339 340static __inline void 341outsl(u_int port, const void *addr, size_t cnt) 342{ 343 __asm __volatile("cld; rep; outsl" 344 : : "d" (port), "S" (addr), "c" (cnt) 345 : "si", "cx"); 346} 347 348static __inline void 349outw(u_int port, u_short data) 350{ 351 __asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port)); 352} 353 354static __inline u_int 355rcr2(void) 356{ 357 u_int data; 358 359 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 360 return (data); 361} 362 363static __inline u_int 364read_eflags(void) 365{ 366 u_int ef; 367 368 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 369 return (ef); 370} 371 372static __inline u_int64_t 373rdmsr(u_int msr) 374{ 375 u_int64_t rv; 376 377 __asm __volatile(".byte 0x0f, 0x32" : "=A" (rv) : "c" (msr)); 378 return (rv); 379} 380 381static __inline u_int64_t 382rdpmc(u_int pmc) 383{ 384 u_int64_t rv; 385 386 __asm __volatile(".byte 0x0f, 0x33" : "=A" (rv) : "c" (pmc)); 387 return (rv); 388} 389 390static __inline u_int64_t 391rdtsc(void) 392{ 393 u_int64_t rv; 394 395 __asm __volatile(".byte 0x0f, 0x31" : "=A" (rv)); 396 return (rv); 397} 398 399static __inline void 400setbits(volatile unsigned *addr, u_int bits) 401{ 402 __asm __volatile( 403#ifdef SMP 404 "lock; " 405#endif 406 "orl %1,%0" : "=m" (*addr) : "ir" (bits)); 407} 408 409static __inline void 410wbinvd(void) 411{ 412 __asm __volatile("wbinvd"); 413} 414 415static __inline void 416write_eflags(u_int ef) 417{ 418 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 419} 420 421static __inline void 422wrmsr(u_int msr, u_int64_t newval) 423{ 424 __asm __volatile(".byte 0x0f, 0x30" : : "A" (newval), "c" (msr)); 425} 426 427#else /* !__GNUC__ */ 428 429int breakpoint __P((void)); 430void disable_intr __P((void)); 431void enable_intr __P((void)); 432u_char inb __P((u_int port)); 433u_int inl __P((u_int port)); 434void insb __P((u_int port, void *addr, size_t cnt)); 435void insl __P((u_int port, void *addr, size_t cnt)); 436void insw __P((u_int port, void *addr, size_t cnt)); 437void invd __P((void)); 438void invlpg __P((u_int addr)); 439void invltlb __P((void)); 440u_short inw __P((u_int port)); 441u_int loadandclear __P((u_int *addr)); 442void outb __P((u_int port, u_char data)); 443void outl __P((u_int port, u_int data)); 444void outsb __P((u_int port, void *addr, size_t cnt)); 445void outsl __P((u_int port, void *addr, size_t cnt)); 446void outsw __P((u_int port, void *addr, size_t cnt)); 447void outw __P((u_int port, u_short data)); 448u_int rcr2 __P((void)); 449u_int64_t rdmsr __P((u_int msr)); 450u_int64_t rdpmc __P((u_int pmc)); 451u_int64_t rdtsc __P((void)); 452u_int read_eflags __P((void)); 453void setbits __P((volatile unsigned *addr, u_int bits)); 454void wbinvd __P((void)); 455void write_eflags __P((u_int ef)); 456void wrmsr __P((u_int msr, u_int64_t newval)); 457 458#endif /* __GNUC__ */ 459 460void load_cr0 __P((u_int cr0)); 461void load_cr3 __P((u_int cr3)); 462void load_cr4 __P((u_int cr4)); 463void ltr __P((u_short sel)); 464u_int rcr0 __P((void)); 465u_int rcr3 __P((void)); 466u_int rcr4 __P((void)); 467 468#endif /* !_MACHINE_CPUFUNC_H_ */ 469