1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD: releng/10.3/sys/i386/include/cpufunc.h 290189 2015-10-30 10:02:57Z kib $ 30 */ 31 32/* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38#ifndef _MACHINE_CPUFUNC_H_ 39#define _MACHINE_CPUFUNC_H_ 40 41#ifndef _SYS_CDEFS_H_ 42#error this file needs sys/cdefs.h as a prerequisite 43#endif 44 45#ifdef XEN 46extern void xen_cli(void); 47extern void xen_sti(void); 48extern u_int xen_rcr2(void); 49extern void xen_load_cr3(u_int data); 50extern void xen_tlb_flush(void); 51extern void xen_invlpg(u_int addr); 52extern void write_eflags(u_int eflags); 53extern u_int read_eflags(void); 54#endif 55 56struct region_descriptor; 57 58#define readb(va) (*(volatile uint8_t *) (va)) 59#define readw(va) (*(volatile uint16_t *) (va)) 60#define readl(va) (*(volatile uint32_t *) (va)) 61 62#define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 63#define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 64#define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 65 66#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 67 68static __inline void 69breakpoint(void) 70{ 71 __asm __volatile("int $3"); 72} 73 74static __inline u_int 75bsfl(u_int mask) 76{ 77 u_int result; 78 79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81} 82 83static __inline u_int 84bsrl(u_int mask) 85{ 86 u_int result; 87 88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 89 return (result); 90} 91 92static __inline void 93clflush(u_long addr) 94{ 95 96 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 97} 98 99static __inline void 100clflushopt(u_long addr) 101{ 102 103 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 104} 105 106static __inline void 107clts(void) 108{ 109 110 __asm __volatile("clts"); 111} 112 113static __inline void 114disable_intr(void) 115{ 116#ifdef XEN 117 xen_cli(); 118#else 119 __asm __volatile("cli" : : : "memory"); 120#endif 121} 122 123static __inline void 124do_cpuid(u_int ax, u_int *p) 125{ 126 __asm __volatile("cpuid" 127 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 128 : "0" (ax)); 129} 130 131static __inline void 132cpuid_count(u_int ax, u_int cx, u_int *p) 133{ 134 __asm __volatile("cpuid" 135 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 136 : "0" (ax), "c" (cx)); 137} 138 139static __inline void 140enable_intr(void) 141{ 142#ifdef XEN 143 xen_sti(); 144#else 145 __asm __volatile("sti"); 146#endif 147} 148 149static __inline void 150cpu_monitor(const void *addr, u_long extensions, u_int hints) 151{ 152 153 __asm __volatile("monitor" 154 : : "a" (addr), "c" (extensions), "d" (hints)); 155} 156 157static __inline void 158cpu_mwait(u_long extensions, u_int hints) 159{ 160 161 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 162} 163 164static __inline void 165lfence(void) 166{ 167 168 __asm __volatile("lfence" : : : "memory"); 169} 170 171static __inline void 172mfence(void) 173{ 174 175 __asm __volatile("mfence" : : : "memory"); 176} 177 178#ifdef _KERNEL 179 180#define HAVE_INLINE_FFS 181 182static __inline int 183ffs(int mask) 184{ 185 /* 186 * Note that gcc-2's builtin ffs would be used if we didn't declare 187 * this inline or turn off the builtin. The builtin is faster but 188 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 189 * versions. 190 */ 191 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 192} 193 194#define HAVE_INLINE_FFSL 195 196static __inline int 197ffsl(long mask) 198{ 199 return (ffs((int)mask)); 200} 201 202#define HAVE_INLINE_FLS 203 204static __inline int 205fls(int mask) 206{ 207 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 208} 209 210#define HAVE_INLINE_FLSL 211 212static __inline int 213flsl(long mask) 214{ 215 return (fls((int)mask)); 216} 217 218#endif /* _KERNEL */ 219 220static __inline void 221halt(void) 222{ 223 __asm __volatile("hlt"); 224} 225 226static __inline u_char 227inb(u_int port) 228{ 229 u_char data; 230 231 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 232 return (data); 233} 234 235static __inline u_int 236inl(u_int port) 237{ 238 u_int data; 239 240 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 241 return (data); 242} 243 244static __inline void 245insb(u_int port, void *addr, size_t count) 246{ 247 __asm __volatile("cld; rep; insb" 248 : "+D" (addr), "+c" (count) 249 : "d" (port) 250 : "memory"); 251} 252 253static __inline void 254insw(u_int port, void *addr, size_t count) 255{ 256 __asm __volatile("cld; rep; insw" 257 : "+D" (addr), "+c" (count) 258 : "d" (port) 259 : "memory"); 260} 261 262static __inline void 263insl(u_int port, void *addr, size_t count) 264{ 265 __asm __volatile("cld; rep; insl" 266 : "+D" (addr), "+c" (count) 267 : "d" (port) 268 : "memory"); 269} 270 271static __inline void 272invd(void) 273{ 274 __asm __volatile("invd"); 275} 276 277static __inline u_short 278inw(u_int port) 279{ 280 u_short data; 281 282 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 283 return (data); 284} 285 286static __inline void 287outb(u_int port, u_char data) 288{ 289 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 290} 291 292static __inline void 293outl(u_int port, u_int data) 294{ 295 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 296} 297 298static __inline void 299outsb(u_int port, const void *addr, size_t count) 300{ 301 __asm __volatile("cld; rep; outsb" 302 : "+S" (addr), "+c" (count) 303 : "d" (port)); 304} 305 306static __inline void 307outsw(u_int port, const void *addr, size_t count) 308{ 309 __asm __volatile("cld; rep; outsw" 310 : "+S" (addr), "+c" (count) 311 : "d" (port)); 312} 313 314static __inline void 315outsl(u_int port, const void *addr, size_t count) 316{ 317 __asm __volatile("cld; rep; outsl" 318 : "+S" (addr), "+c" (count) 319 : "d" (port)); 320} 321 322static __inline void 323outw(u_int port, u_short data) 324{ 325 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 326} 327 328static __inline void 329ia32_pause(void) 330{ 331 __asm __volatile("pause"); 332} 333 334static __inline u_int 335#ifdef XEN 336_read_eflags(void) 337#else 338read_eflags(void) 339#endif 340{ 341 u_int ef; 342 343 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 344 return (ef); 345} 346 347static __inline uint64_t 348rdmsr(u_int msr) 349{ 350 uint64_t rv; 351 352 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 353 return (rv); 354} 355 356static __inline uint64_t 357rdpmc(u_int pmc) 358{ 359 uint64_t rv; 360 361 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 362 return (rv); 363} 364 365static __inline uint64_t 366rdtsc(void) 367{ 368 uint64_t rv; 369 370 __asm __volatile("rdtsc" : "=A" (rv)); 371 return (rv); 372} 373 374static __inline uint32_t 375rdtsc32(void) 376{ 377 uint32_t rv; 378 379 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 380 return (rv); 381} 382 383static __inline void 384wbinvd(void) 385{ 386 __asm __volatile("wbinvd"); 387} 388 389static __inline void 390#ifdef XEN 391_write_eflags(u_int ef) 392#else 393write_eflags(u_int ef) 394#endif 395{ 396 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 397} 398 399static __inline void 400wrmsr(u_int msr, uint64_t newval) 401{ 402 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 403} 404 405static __inline void 406load_cr0(u_int data) 407{ 408 409 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 410} 411 412static __inline u_int 413rcr0(void) 414{ 415 u_int data; 416 417 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 418 return (data); 419} 420 421static __inline u_int 422rcr2(void) 423{ 424 u_int data; 425 426#ifdef XEN 427 return (xen_rcr2()); 428#endif 429 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 430 return (data); 431} 432 433static __inline void 434load_cr3(u_int data) 435{ 436#ifdef XEN 437 xen_load_cr3(data); 438#else 439 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 440#endif 441} 442 443static __inline u_int 444rcr3(void) 445{ 446 u_int data; 447 448 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 449 return (data); 450} 451 452static __inline void 453load_cr4(u_int data) 454{ 455 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 456} 457 458static __inline u_int 459rcr4(void) 460{ 461 u_int data; 462 463 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 464 return (data); 465} 466 467static __inline uint64_t 468rxcr(u_int reg) 469{ 470 u_int low, high; 471 472 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 473 return (low | ((uint64_t)high << 32)); 474} 475 476static __inline void 477load_xcr(u_int reg, uint64_t val) 478{ 479 u_int low, high; 480 481 low = val; 482 high = val >> 32; 483 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 484} 485 486/* 487 * Global TLB flush (except for thise for pages marked PG_G) 488 */ 489static __inline void 490invltlb(void) 491{ 492#ifdef XEN 493 xen_tlb_flush(); 494#else 495 load_cr3(rcr3()); 496#endif 497} 498 499/* 500 * TLB flush for an individual page (even if it has PG_G). 501 * Only works on 486+ CPUs (i386 does not have PG_G). 502 */ 503static __inline void 504invlpg(u_int addr) 505{ 506 507#ifdef XEN 508 xen_invlpg(addr); 509#else 510 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 511#endif 512} 513 514static __inline u_short 515rfs(void) 516{ 517 u_short sel; 518 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 519 return (sel); 520} 521 522static __inline uint64_t 523rgdt(void) 524{ 525 uint64_t gdtr; 526 __asm __volatile("sgdt %0" : "=m" (gdtr)); 527 return (gdtr); 528} 529 530static __inline u_short 531rgs(void) 532{ 533 u_short sel; 534 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 535 return (sel); 536} 537 538static __inline uint64_t 539ridt(void) 540{ 541 uint64_t idtr; 542 __asm __volatile("sidt %0" : "=m" (idtr)); 543 return (idtr); 544} 545 546static __inline u_short 547rldt(void) 548{ 549 u_short ldtr; 550 __asm __volatile("sldt %0" : "=g" (ldtr)); 551 return (ldtr); 552} 553 554static __inline u_short 555rss(void) 556{ 557 u_short sel; 558 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 559 return (sel); 560} 561 562static __inline u_short 563rtr(void) 564{ 565 u_short tr; 566 __asm __volatile("str %0" : "=g" (tr)); 567 return (tr); 568} 569 570static __inline void 571load_fs(u_short sel) 572{ 573 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 574} 575 576static __inline void 577load_gs(u_short sel) 578{ 579 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 580} 581 582static __inline void 583lidt(struct region_descriptor *addr) 584{ 585 __asm __volatile("lidt (%0)" : : "r" (addr)); 586} 587 588static __inline void 589lldt(u_short sel) 590{ 591 __asm __volatile("lldt %0" : : "r" (sel)); 592} 593 594static __inline void 595ltr(u_short sel) 596{ 597 __asm __volatile("ltr %0" : : "r" (sel)); 598} 599 600static __inline u_int 601rdr0(void) 602{ 603 u_int data; 604 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 605 return (data); 606} 607 608static __inline void 609load_dr0(u_int dr0) 610{ 611 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 612} 613 614static __inline u_int 615rdr1(void) 616{ 617 u_int data; 618 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 619 return (data); 620} 621 622static __inline void 623load_dr1(u_int dr1) 624{ 625 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 626} 627 628static __inline u_int 629rdr2(void) 630{ 631 u_int data; 632 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 633 return (data); 634} 635 636static __inline void 637load_dr2(u_int dr2) 638{ 639 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 640} 641 642static __inline u_int 643rdr3(void) 644{ 645 u_int data; 646 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 647 return (data); 648} 649 650static __inline void 651load_dr3(u_int dr3) 652{ 653 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 654} 655 656static __inline u_int 657rdr4(void) 658{ 659 u_int data; 660 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 661 return (data); 662} 663 664static __inline void 665load_dr4(u_int dr4) 666{ 667 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 668} 669 670static __inline u_int 671rdr5(void) 672{ 673 u_int data; 674 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 675 return (data); 676} 677 678static __inline void 679load_dr5(u_int dr5) 680{ 681 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 682} 683 684static __inline u_int 685rdr6(void) 686{ 687 u_int data; 688 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 689 return (data); 690} 691 692static __inline void 693load_dr6(u_int dr6) 694{ 695 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 696} 697 698static __inline u_int 699rdr7(void) 700{ 701 u_int data; 702 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 703 return (data); 704} 705 706static __inline void 707load_dr7(u_int dr7) 708{ 709 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 710} 711 712static __inline u_char 713read_cyrix_reg(u_char reg) 714{ 715 outb(0x22, reg); 716 return inb(0x23); 717} 718 719static __inline void 720write_cyrix_reg(u_char reg, u_char data) 721{ 722 outb(0x22, reg); 723 outb(0x23, data); 724} 725 726static __inline register_t 727intr_disable(void) 728{ 729 register_t eflags; 730 731 eflags = read_eflags(); 732 disable_intr(); 733 return (eflags); 734} 735 736static __inline void 737intr_restore(register_t eflags) 738{ 739 write_eflags(eflags); 740} 741 742#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 743 744int breakpoint(void); 745u_int bsfl(u_int mask); 746u_int bsrl(u_int mask); 747void clflush(u_long addr); 748void clts(void); 749void cpuid_count(u_int ax, u_int cx, u_int *p); 750void disable_intr(void); 751void do_cpuid(u_int ax, u_int *p); 752void enable_intr(void); 753void halt(void); 754void ia32_pause(void); 755u_char inb(u_int port); 756u_int inl(u_int port); 757void insb(u_int port, void *addr, size_t count); 758void insl(u_int port, void *addr, size_t count); 759void insw(u_int port, void *addr, size_t count); 760register_t intr_disable(void); 761void intr_restore(register_t ef); 762void invd(void); 763void invlpg(u_int addr); 764void invltlb(void); 765u_short inw(u_int port); 766void lidt(struct region_descriptor *addr); 767void lldt(u_short sel); 768void load_cr0(u_int cr0); 769void load_cr3(u_int cr3); 770void load_cr4(u_int cr4); 771void load_dr0(u_int dr0); 772void load_dr1(u_int dr1); 773void load_dr2(u_int dr2); 774void load_dr3(u_int dr3); 775void load_dr4(u_int dr4); 776void load_dr5(u_int dr5); 777void load_dr6(u_int dr6); 778void load_dr7(u_int dr7); 779void load_fs(u_short sel); 780void load_gs(u_short sel); 781void ltr(u_short sel); 782void outb(u_int port, u_char data); 783void outl(u_int port, u_int data); 784void outsb(u_int port, const void *addr, size_t count); 785void outsl(u_int port, const void *addr, size_t count); 786void outsw(u_int port, const void *addr, size_t count); 787void outw(u_int port, u_short data); 788u_int rcr0(void); 789u_int rcr2(void); 790u_int rcr3(void); 791u_int rcr4(void); 792uint64_t rdmsr(u_int msr); 793uint64_t rdpmc(u_int pmc); 794u_int rdr0(void); 795u_int rdr1(void); 796u_int rdr2(void); 797u_int rdr3(void); 798u_int rdr4(void); 799u_int rdr5(void); 800u_int rdr6(void); 801u_int rdr7(void); 802uint64_t rdtsc(void); 803u_char read_cyrix_reg(u_char reg); 804u_int read_eflags(void); 805u_int rfs(void); 806uint64_t rgdt(void); 807u_int rgs(void); 808uint64_t ridt(void); 809u_short rldt(void); 810u_short rtr(void); 811void wbinvd(void); 812void write_cyrix_reg(u_char reg, u_char data); 813void write_eflags(u_int ef); 814void wrmsr(u_int msr, uint64_t newval); 815 816#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 817 818void reset_dbregs(void); 819 820#ifdef _KERNEL 821int rdmsr_safe(u_int msr, uint64_t *val); 822int wrmsr_safe(u_int msr, uint64_t newval); 823#endif 824 825#endif /* !_MACHINE_CPUFUNC_H_ */ 826