1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD: releng/11.1/sys/i386/include/cpufunc.h 313148 2017-02-03 12:03:10Z kib $ 30 */ 31 32/* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38#ifndef _MACHINE_CPUFUNC_H_ 39#define _MACHINE_CPUFUNC_H_ 40 41struct region_descriptor; 42 43#define readb(va) (*(volatile uint8_t *) (va)) 44#define readw(va) (*(volatile uint16_t *) (va)) 45#define readl(va) (*(volatile uint32_t *) (va)) 46 47#define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 48#define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 49#define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 50 51#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 52 53static __inline void 54breakpoint(void) 55{ 56 __asm __volatile("int $3"); 57} 58 59static __inline u_int 60bsfl(u_int mask) 61{ 62 u_int result; 63 64 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 65 return (result); 66} 67 68static __inline u_int 69bsrl(u_int mask) 70{ 71 u_int result; 72 73 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 74 return (result); 75} 76 77static __inline void 78clflush(u_long addr) 79{ 80 81 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 82} 83 84static __inline void 85clflushopt(u_long addr) 86{ 87 88 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 89} 90 91static __inline void 92clts(void) 93{ 94 95 __asm __volatile("clts"); 96} 97 98static __inline void 99disable_intr(void) 100{ 101 102 __asm __volatile("cli" : : : "memory"); 103} 104 105static __inline void 106do_cpuid(u_int ax, u_int *p) 107{ 108 __asm __volatile("cpuid" 109 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 110 : "0" (ax)); 111} 112 113static __inline void 114cpuid_count(u_int ax, u_int cx, u_int *p) 115{ 116 __asm __volatile("cpuid" 117 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 118 : "0" (ax), "c" (cx)); 119} 120 121static __inline void 122enable_intr(void) 123{ 124 125 __asm __volatile("sti"); 126} 127 128static __inline void 129cpu_monitor(const void *addr, u_long extensions, u_int hints) 130{ 131 132 __asm __volatile("monitor" 133 : : "a" (addr), "c" (extensions), "d" (hints)); 134} 135 136static __inline void 137cpu_mwait(u_long extensions, u_int hints) 138{ 139 140 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 141} 142 143static __inline void 144lfence(void) 145{ 146 147 __asm __volatile("lfence" : : : "memory"); 148} 149 150static __inline void 151mfence(void) 152{ 153 154 __asm __volatile("mfence" : : : "memory"); 155} 156 157static __inline void 158sfence(void) 159{ 160 161 __asm __volatile("sfence" : : : "memory"); 162} 163 164#ifdef _KERNEL 165 166#ifndef __HAIKU__ 167#define HAVE_INLINE_FFS 168 169static __inline int 170ffs(int mask) 171{ 172 /* 173 * Note that gcc-2's builtin ffs would be used if we didn't declare 174 * this inline or turn off the builtin. The builtin is faster but 175 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 176 * versions. 177 */ 178 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 179} 180#endif 181 182#define HAVE_INLINE_FFSL 183 184static __inline int 185ffsl(long mask) 186{ 187 return (ffs((int)mask)); 188} 189 190#define HAVE_INLINE_FLS 191 192static __inline int 193fls(int mask) 194{ 195 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 196} 197 198#define HAVE_INLINE_FLSL 199 200static __inline int 201flsl(long mask) 202{ 203 return (fls((int)mask)); 204} 205 206#endif /* _KERNEL */ 207 208static __inline void 209halt(void) 210{ 211 __asm __volatile("hlt"); 212} 213 214static __inline u_char 215inb(u_int port) 216{ 217 u_char data; 218 219 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 220 return (data); 221} 222 223static __inline u_int 224inl(u_int port) 225{ 226 u_int data; 227 228 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 229 return (data); 230} 231 232static __inline void 233insb(u_int port, void *addr, size_t count) 234{ 235 __asm __volatile("cld; rep; insb" 236 : "+D" (addr), "+c" (count) 237 : "d" (port) 238 : "memory"); 239} 240 241static __inline void 242insw(u_int port, void *addr, size_t count) 243{ 244 __asm __volatile("cld; rep; insw" 245 : "+D" (addr), "+c" (count) 246 : "d" (port) 247 : "memory"); 248} 249 250static __inline void 251insl(u_int port, void *addr, size_t count) 252{ 253 __asm __volatile("cld; rep; insl" 254 : "+D" (addr), "+c" (count) 255 : "d" (port) 256 : "memory"); 257} 258 259static __inline void 260invd(void) 261{ 262 __asm __volatile("invd"); 263} 264 265static __inline u_short 266inw(u_int port) 267{ 268 u_short data; 269 270 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 271 return (data); 272} 273 274static __inline void 275outb(u_int port, u_char data) 276{ 277 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 278} 279 280static __inline void 281outl(u_int port, u_int data) 282{ 283 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 284} 285 286static __inline void 287outsb(u_int port, const void *addr, size_t count) 288{ 289 __asm __volatile("cld; rep; outsb" 290 : "+S" (addr), "+c" (count) 291 : "d" (port)); 292} 293 294static __inline void 295outsw(u_int port, const void *addr, size_t count) 296{ 297 __asm __volatile("cld; rep; outsw" 298 : "+S" (addr), "+c" (count) 299 : "d" (port)); 300} 301 302static __inline void 303outsl(u_int port, const void *addr, size_t count) 304{ 305 __asm __volatile("cld; rep; outsl" 306 : "+S" (addr), "+c" (count) 307 : "d" (port)); 308} 309 310static __inline void 311outw(u_int port, u_short data) 312{ 313 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 314} 315 316static __inline void 317ia32_pause(void) 318{ 319 __asm __volatile("pause"); 320} 321 322static __inline u_int 323read_eflags(void) 324{ 325 u_int ef; 326 327 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 328 return (ef); 329} 330 331static __inline uint64_t 332rdmsr(u_int msr) 333{ 334 uint64_t rv; 335 336 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 337 return (rv); 338} 339 340static __inline uint32_t 341rdmsr32(u_int msr) 342{ 343 uint32_t low; 344 345 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx"); 346 return (low); 347} 348 349static __inline uint64_t 350rdpmc(u_int pmc) 351{ 352 uint64_t rv; 353 354 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 355 return (rv); 356} 357 358static __inline uint64_t 359rdtsc(void) 360{ 361 uint64_t rv; 362 363 __asm __volatile("rdtsc" : "=A" (rv)); 364 return (rv); 365} 366 367static __inline uint32_t 368rdtsc32(void) 369{ 370 uint32_t rv; 371 372 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 373 return (rv); 374} 375 376#ifndef wbinvd 377static __inline void 378wbinvd(void) 379{ 380 __asm __volatile("wbinvd"); 381} 382#endif 383 384static __inline void 385write_eflags(u_int ef) 386{ 387 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 388} 389 390static __inline void 391wrmsr(u_int msr, uint64_t newval) 392{ 393 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 394} 395 396static __inline void 397load_cr0(u_int data) 398{ 399 400 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 401} 402 403static __inline u_int 404rcr0(void) 405{ 406 u_int data; 407 408 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 409 return (data); 410} 411 412static __inline u_int 413rcr2(void) 414{ 415 u_int data; 416 417 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 418 return (data); 419} 420 421static __inline void 422load_cr3(u_int data) 423{ 424 425 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 426} 427 428static __inline u_int 429rcr3(void) 430{ 431 u_int data; 432 433 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 434 return (data); 435} 436 437static __inline void 438load_cr4(u_int data) 439{ 440 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 441} 442 443static __inline u_int 444rcr4(void) 445{ 446 u_int data; 447 448 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 449 return (data); 450} 451 452static __inline uint64_t 453rxcr(u_int reg) 454{ 455 u_int low, high; 456 457 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 458 return (low | ((uint64_t)high << 32)); 459} 460 461static __inline void 462load_xcr(u_int reg, uint64_t val) 463{ 464 u_int low, high; 465 466 low = val; 467 high = val >> 32; 468 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 469} 470 471/* 472 * Global TLB flush (except for thise for pages marked PG_G) 473 */ 474static __inline void 475invltlb(void) 476{ 477 478 load_cr3(rcr3()); 479} 480 481/* 482 * TLB flush for an individual page (even if it has PG_G). 483 * Only works on 486+ CPUs (i386 does not have PG_G). 484 */ 485static __inline void 486invlpg(u_int addr) 487{ 488 489 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 490} 491 492static __inline u_short 493rfs(void) 494{ 495 u_short sel; 496 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 497 return (sel); 498} 499 500static __inline uint64_t 501rgdt(void) 502{ 503 uint64_t gdtr; 504 __asm __volatile("sgdt %0" : "=m" (gdtr)); 505 return (gdtr); 506} 507 508static __inline u_short 509rgs(void) 510{ 511 u_short sel; 512 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 513 return (sel); 514} 515 516static __inline uint64_t 517ridt(void) 518{ 519 uint64_t idtr; 520 __asm __volatile("sidt %0" : "=m" (idtr)); 521 return (idtr); 522} 523 524static __inline u_short 525rldt(void) 526{ 527 u_short ldtr; 528 __asm __volatile("sldt %0" : "=g" (ldtr)); 529 return (ldtr); 530} 531 532static __inline u_short 533rss(void) 534{ 535 u_short sel; 536 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 537 return (sel); 538} 539 540static __inline u_short 541rtr(void) 542{ 543 u_short tr; 544 __asm __volatile("str %0" : "=g" (tr)); 545 return (tr); 546} 547 548static __inline void 549load_fs(u_short sel) 550{ 551 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 552} 553 554static __inline void 555load_gs(u_short sel) 556{ 557 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 558} 559 560static __inline void 561lidt(struct region_descriptor *addr) 562{ 563 __asm __volatile("lidt (%0)" : : "r" (addr)); 564} 565 566static __inline void 567lldt(u_short sel) 568{ 569 __asm __volatile("lldt %0" : : "r" (sel)); 570} 571 572static __inline void 573ltr(u_short sel) 574{ 575 __asm __volatile("ltr %0" : : "r" (sel)); 576} 577 578static __inline u_int 579rdr0(void) 580{ 581 u_int data; 582 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 583 return (data); 584} 585 586static __inline void 587load_dr0(u_int dr0) 588{ 589 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 590} 591 592static __inline u_int 593rdr1(void) 594{ 595 u_int data; 596 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 597 return (data); 598} 599 600static __inline void 601load_dr1(u_int dr1) 602{ 603 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 604} 605 606static __inline u_int 607rdr2(void) 608{ 609 u_int data; 610 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 611 return (data); 612} 613 614static __inline void 615load_dr2(u_int dr2) 616{ 617 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 618} 619 620static __inline u_int 621rdr3(void) 622{ 623 u_int data; 624 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 625 return (data); 626} 627 628static __inline void 629load_dr3(u_int dr3) 630{ 631 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 632} 633 634static __inline u_int 635rdr4(void) 636{ 637 u_int data; 638 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 639 return (data); 640} 641 642static __inline void 643load_dr4(u_int dr4) 644{ 645 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 646} 647 648static __inline u_int 649rdr5(void) 650{ 651 u_int data; 652 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 653 return (data); 654} 655 656static __inline void 657load_dr5(u_int dr5) 658{ 659 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 660} 661 662static __inline u_int 663rdr6(void) 664{ 665 u_int data; 666 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 667 return (data); 668} 669 670static __inline void 671load_dr6(u_int dr6) 672{ 673 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 674} 675 676static __inline u_int 677rdr7(void) 678{ 679 u_int data; 680 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 681 return (data); 682} 683 684static __inline void 685load_dr7(u_int dr7) 686{ 687 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 688} 689 690static __inline u_char 691read_cyrix_reg(u_char reg) 692{ 693 outb(0x22, reg); 694 return inb(0x23); 695} 696 697static __inline void 698write_cyrix_reg(u_char reg, u_char data) 699{ 700 outb(0x22, reg); 701 outb(0x23, data); 702} 703 704#ifndef __HAIKU__ 705static __inline register_t 706intr_disable(void) 707{ 708 register_t eflags; 709 710 eflags = read_eflags(); 711 disable_intr(); 712 return (eflags); 713} 714 715static __inline void 716intr_restore(register_t eflags) 717{ 718 write_eflags(eflags); 719} 720#endif 721 722#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 723 724int breakpoint(void); 725u_int bsfl(u_int mask); 726u_int bsrl(u_int mask); 727void clflush(u_long addr); 728void clts(void); 729void cpuid_count(u_int ax, u_int cx, u_int *p); 730void disable_intr(void); 731void do_cpuid(u_int ax, u_int *p); 732void enable_intr(void); 733void halt(void); 734void ia32_pause(void); 735u_char inb(u_int port); 736u_int inl(u_int port); 737void insb(u_int port, void *addr, size_t count); 738void insl(u_int port, void *addr, size_t count); 739void insw(u_int port, void *addr, size_t count); 740register_t intr_disable(void); 741void intr_restore(register_t ef); 742void invd(void); 743void invlpg(u_int addr); 744void invltlb(void); 745u_short inw(u_int port); 746void lidt(struct region_descriptor *addr); 747void lldt(u_short sel); 748void load_cr0(u_int cr0); 749void load_cr3(u_int cr3); 750void load_cr4(u_int cr4); 751void load_dr0(u_int dr0); 752void load_dr1(u_int dr1); 753void load_dr2(u_int dr2); 754void load_dr3(u_int dr3); 755void load_dr4(u_int dr4); 756void load_dr5(u_int dr5); 757void load_dr6(u_int dr6); 758void load_dr7(u_int dr7); 759void load_fs(u_short sel); 760void load_gs(u_short sel); 761void ltr(u_short sel); 762void outb(u_int port, u_char data); 763void outl(u_int port, u_int data); 764void outsb(u_int port, const void *addr, size_t count); 765void outsl(u_int port, const void *addr, size_t count); 766void outsw(u_int port, const void *addr, size_t count); 767void outw(u_int port, u_short data); 768u_int rcr0(void); 769u_int rcr2(void); 770u_int rcr3(void); 771u_int rcr4(void); 772uint64_t rdmsr(u_int msr); 773uint64_t rdpmc(u_int pmc); 774u_int rdr0(void); 775u_int rdr1(void); 776u_int rdr2(void); 777u_int rdr3(void); 778u_int rdr4(void); 779u_int rdr5(void); 780u_int rdr6(void); 781u_int rdr7(void); 782uint64_t rdtsc(void); 783u_char read_cyrix_reg(u_char reg); 784u_int read_eflags(void); 785u_int rfs(void); 786uint64_t rgdt(void); 787u_int rgs(void); 788uint64_t ridt(void); 789u_short rldt(void); 790u_short rtr(void); 791void wbinvd(void); 792void write_cyrix_reg(u_char reg, u_char data); 793void write_eflags(u_int ef); 794void wrmsr(u_int msr, uint64_t newval); 795 796#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 797 798void reset_dbregs(void); 799 800#ifdef _KERNEL 801int rdmsr_safe(u_int msr, uint64_t *val); 802int wrmsr_safe(u_int msr, uint64_t newval); 803#endif 804 805#endif /* !_MACHINE_CPUFUNC_H_ */ 806