1/*- 2 * Copyright (c) 1993 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32/* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38#ifndef _MACHINE_CPUFUNC_H_ 39#define _MACHINE_CPUFUNC_H_ 40 41#ifndef _SYS_CDEFS_H_ 42#error this file needs sys/cdefs.h as a prerequisite 43#endif 44 45#ifdef XEN 46extern void xen_cli(void); 47extern void xen_sti(void); 48extern u_int xen_rcr2(void); 49extern void xen_load_cr3(u_int data); 50extern void xen_tlb_flush(void); 51extern void xen_invlpg(u_int addr); 52extern void write_eflags(u_int eflags); 53extern u_int read_eflags(void); 54#endif 55 56struct region_descriptor; 57 58#define readb(va) (*(volatile uint8_t *) (va)) 59#define readw(va) (*(volatile uint16_t *) (va)) 60#define readl(va) (*(volatile uint32_t *) (va)) 61 62#define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 63#define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 64#define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 65 66#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) 67 68static __inline void 69breakpoint(void) 70{ 71 __asm __volatile("int $3"); 72} 73 74static __inline u_int 75bsfl(u_int mask) 76{ 77 u_int result; 78 79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 80 return (result); 81} 82 83static __inline u_int 84bsrl(u_int mask) 85{ 86 u_int result; 87 88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 89 return (result); 90} 91 92static __inline void 93clflush(u_long addr) 94{ 95 96 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 97} 98 99static __inline void 100clts(void) 101{ 102 103 __asm __volatile("clts"); 104} 105 106static __inline void 107disable_intr(void) 108{ 109#ifdef XEN 110 xen_cli(); 111#else 112 __asm __volatile("cli" : : : "memory"); 113#endif 114} 115 116static __inline void 117do_cpuid(u_int ax, u_int *p) 118{ 119 __asm __volatile("cpuid" 120 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 121 : "0" (ax)); 122} 123 124static __inline void 125cpuid_count(u_int ax, u_int cx, u_int *p) 126{ 127 __asm __volatile("cpuid" 128 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 129 : "0" (ax), "c" (cx)); 130} 131 132static __inline void 133enable_intr(void) 134{ 135#ifdef XEN 136 xen_sti(); 137#else 138 __asm __volatile("sti"); 139#endif 140} 141 142static __inline void 143cpu_monitor(const void *addr, u_long extensions, u_int hints) 144{ 145 146 __asm __volatile("monitor" 147 : : "a" (addr), "c" (extensions), "d" (hints)); 148} 149 150static __inline void 151cpu_mwait(u_long extensions, u_int hints) 152{ 153 154 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 155} 156 157static __inline void 158lfence(void) 159{ 160 161 __asm __volatile("lfence" : : : "memory"); 162} 163 164static __inline void 165mfence(void) 166{ 167 168 __asm __volatile("mfence" : : : "memory"); 169} 170 171#ifdef _KERNEL 172 173#define HAVE_INLINE_FFS 174 175static __inline int 176ffs(int mask) 177{ 178 /* 179 * Note that gcc-2's builtin ffs would be used if we didn't declare 180 * this inline or turn off the builtin. The builtin is faster but 181 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later 182 * versions. 183 */ 184 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1); 185} 186 187#define HAVE_INLINE_FLS 188 189static __inline int 190fls(int mask) 191{ 192 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1); 193} 194 195#endif /* _KERNEL */ 196 197static __inline void 198halt(void) 199{ 200 __asm __volatile("hlt"); 201} 202 203static __inline u_char 204inb(u_int port) 205{ 206 u_char data; 207 208 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 209 return (data); 210} 211 212static __inline u_int 213inl(u_int port) 214{ 215 u_int data; 216 217 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 218 return (data); 219} 220 221static __inline void 222insb(u_int port, void *addr, size_t count) 223{ 224 __asm __volatile("cld; rep; insb" 225 : "+D" (addr), "+c" (count) 226 : "d" (port) 227 : "memory"); 228} 229 230static __inline void 231insw(u_int port, void *addr, size_t count) 232{ 233 __asm __volatile("cld; rep; insw" 234 : "+D" (addr), "+c" (count) 235 : "d" (port) 236 : "memory"); 237} 238 239static __inline void 240insl(u_int port, void *addr, size_t count) 241{ 242 __asm __volatile("cld; rep; insl" 243 : "+D" (addr), "+c" (count) 244 : "d" (port) 245 : "memory"); 246} 247 248static __inline void 249invd(void) 250{ 251 __asm __volatile("invd"); 252} 253 254static __inline u_short 255inw(u_int port) 256{ 257 u_short data; 258 259 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 260 return (data); 261} 262 263static __inline void 264outb(u_int port, u_char data) 265{ 266 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 267} 268 269static __inline void 270outl(u_int port, u_int data) 271{ 272 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 273} 274 275static __inline void 276outsb(u_int port, const void *addr, size_t count) 277{ 278 __asm __volatile("cld; rep; outsb" 279 : "+S" (addr), "+c" (count) 280 : "d" (port)); 281} 282 283static __inline void 284outsw(u_int port, const void *addr, size_t count) 285{ 286 __asm __volatile("cld; rep; outsw" 287 : "+S" (addr), "+c" (count) 288 : "d" (port)); 289} 290 291static __inline void 292outsl(u_int port, const void *addr, size_t count) 293{ 294 __asm __volatile("cld; rep; outsl" 295 : "+S" (addr), "+c" (count) 296 : "d" (port)); 297} 298 299static __inline void 300outw(u_int port, u_short data) 301{ 302 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 303} 304 305static __inline void 306ia32_pause(void) 307{ 308 __asm __volatile("pause"); 309} 310 311static __inline u_int 312#ifdef XEN 313_read_eflags(void) 314#else 315read_eflags(void) 316#endif 317{ 318 u_int ef; 319 320 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 321 return (ef); 322} 323 324static __inline uint64_t 325rdmsr(u_int msr) 326{ 327 uint64_t rv; 328 329 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 330 return (rv); 331} 332 333static __inline uint64_t 334rdpmc(u_int pmc) 335{ 336 uint64_t rv; 337 338 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 339 return (rv); 340} 341 342static __inline uint64_t 343rdtsc(void) 344{ 345 uint64_t rv; 346 347 __asm __volatile("rdtsc" : "=A" (rv)); 348 return (rv); 349} 350 351static __inline uint32_t 352rdtsc32(void) 353{ 354 uint32_t rv; 355 356 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 357 return (rv); 358} 359 360static __inline void 361wbinvd(void) 362{ 363 __asm __volatile("wbinvd"); 364} 365 366static __inline void 367#ifdef XEN 368_write_eflags(u_int ef) 369#else 370write_eflags(u_int ef) 371#endif 372{ 373 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 374} 375 376static __inline void 377wrmsr(u_int msr, uint64_t newval) 378{ 379 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 380} 381 382static __inline void 383load_cr0(u_int data) 384{ 385 386 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 387} 388 389static __inline u_int 390rcr0(void) 391{ 392 u_int data; 393 394 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 395 return (data); 396} 397 398static __inline u_int 399rcr2(void) 400{ 401 u_int data; 402 403#ifdef XEN 404 return (xen_rcr2()); 405#endif 406 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 407 return (data); 408} 409 410static __inline void 411load_cr3(u_int data) 412{ 413#ifdef XEN 414 xen_load_cr3(data); 415#else 416 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 417#endif 418} 419 420static __inline u_int 421rcr3(void) 422{ 423 u_int data; 424 425 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 426 return (data); 427} 428 429static __inline void 430load_cr4(u_int data) 431{ 432 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 433} 434 435static __inline u_int 436rcr4(void) 437{ 438 u_int data; 439 440 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 441 return (data); 442} 443 444/* 445 * Global TLB flush (except for thise for pages marked PG_G) 446 */ 447static __inline void 448invltlb(void) 449{ 450#ifdef XEN 451 xen_tlb_flush(); 452#else 453 load_cr3(rcr3()); 454#endif 455} 456 457/* 458 * TLB flush for an individual page (even if it has PG_G). 459 * Only works on 486+ CPUs (i386 does not have PG_G). 460 */ 461static __inline void 462invlpg(u_int addr) 463{ 464 465#ifdef XEN 466 xen_invlpg(addr); 467#else 468 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 469#endif 470} 471 472static __inline u_short 473rfs(void) 474{ 475 u_short sel; 476 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 477 return (sel); 478} 479 480static __inline uint64_t 481rgdt(void) 482{ 483 uint64_t gdtr; 484 __asm __volatile("sgdt %0" : "=m" (gdtr)); 485 return (gdtr); 486} 487 488static __inline u_short 489rgs(void) 490{ 491 u_short sel; 492 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 493 return (sel); 494} 495 496static __inline uint64_t 497ridt(void) 498{ 499 uint64_t idtr; 500 __asm __volatile("sidt %0" : "=m" (idtr)); 501 return (idtr); 502} 503 504static __inline u_short 505rldt(void) 506{ 507 u_short ldtr; 508 __asm __volatile("sldt %0" : "=g" (ldtr)); 509 return (ldtr); 510} 511 512static __inline u_short 513rss(void) 514{ 515 u_short sel; 516 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 517 return (sel); 518} 519 520static __inline u_short 521rtr(void) 522{ 523 u_short tr; 524 __asm __volatile("str %0" : "=g" (tr)); 525 return (tr); 526} 527 528static __inline void 529load_fs(u_short sel) 530{ 531 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 532} 533 534static __inline void 535load_gs(u_short sel) 536{ 537 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 538} 539 540static __inline void 541lidt(struct region_descriptor *addr) 542{ 543 __asm __volatile("lidt (%0)" : : "r" (addr)); 544} 545 546static __inline void 547lldt(u_short sel) 548{ 549 __asm __volatile("lldt %0" : : "r" (sel)); 550} 551 552static __inline void 553ltr(u_short sel) 554{ 555 __asm __volatile("ltr %0" : : "r" (sel)); 556} 557 558static __inline u_int 559rdr0(void) 560{ 561 u_int data; 562 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 563 return (data); 564} 565 566static __inline void 567load_dr0(u_int dr0) 568{ 569 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 570} 571 572static __inline u_int 573rdr1(void) 574{ 575 u_int data; 576 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 577 return (data); 578} 579 580static __inline void 581load_dr1(u_int dr1) 582{ 583 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 584} 585 586static __inline u_int 587rdr2(void) 588{ 589 u_int data; 590 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 591 return (data); 592} 593 594static __inline void 595load_dr2(u_int dr2) 596{ 597 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 598} 599 600static __inline u_int 601rdr3(void) 602{ 603 u_int data; 604 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 605 return (data); 606} 607 608static __inline void 609load_dr3(u_int dr3) 610{ 611 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 612} 613 614static __inline u_int 615rdr4(void) 616{ 617 u_int data; 618 __asm __volatile("movl %%dr4,%0" : "=r" (data)); 619 return (data); 620} 621 622static __inline void 623load_dr4(u_int dr4) 624{ 625 __asm __volatile("movl %0,%%dr4" : : "r" (dr4)); 626} 627 628static __inline u_int 629rdr5(void) 630{ 631 u_int data; 632 __asm __volatile("movl %%dr5,%0" : "=r" (data)); 633 return (data); 634} 635 636static __inline void 637load_dr5(u_int dr5) 638{ 639 __asm __volatile("movl %0,%%dr5" : : "r" (dr5)); 640} 641 642static __inline u_int 643rdr6(void) 644{ 645 u_int data; 646 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 647 return (data); 648} 649 650static __inline void 651load_dr6(u_int dr6) 652{ 653 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 654} 655 656static __inline u_int 657rdr7(void) 658{ 659 u_int data; 660 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 661 return (data); 662} 663 664static __inline void 665load_dr7(u_int dr7) 666{ 667 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 668} 669 670static __inline u_char 671read_cyrix_reg(u_char reg) 672{ 673 outb(0x22, reg); 674 return inb(0x23); 675} 676 677static __inline void 678write_cyrix_reg(u_char reg, u_char data) 679{ 680 outb(0x22, reg); 681 outb(0x23, data); 682} 683 684static __inline register_t 685intr_disable(void) 686{ 687 register_t eflags; 688 689 eflags = read_eflags(); 690 disable_intr(); 691 return (eflags); 692} 693 694static __inline void 695intr_restore(register_t eflags) 696{ 697 write_eflags(eflags); 698} 699 700#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ 701 702int breakpoint(void); 703u_int bsfl(u_int mask); 704u_int bsrl(u_int mask); 705void clflush(u_long addr); 706void clts(void); 707void cpuid_count(u_int ax, u_int cx, u_int *p); 708void disable_intr(void); 709void do_cpuid(u_int ax, u_int *p); 710void enable_intr(void); 711void halt(void); 712void ia32_pause(void); 713u_char inb(u_int port); 714u_int inl(u_int port); 715void insb(u_int port, void *addr, size_t count); 716void insl(u_int port, void *addr, size_t count); 717void insw(u_int port, void *addr, size_t count); 718register_t intr_disable(void); 719void intr_restore(register_t ef); 720void invd(void); 721void invlpg(u_int addr); 722void invltlb(void); 723u_short inw(u_int port); 724void lidt(struct region_descriptor *addr); 725void lldt(u_short sel); 726void load_cr0(u_int cr0); 727void load_cr3(u_int cr3); 728void load_cr4(u_int cr4); 729void load_dr0(u_int dr0); 730void load_dr1(u_int dr1); 731void load_dr2(u_int dr2); 732void load_dr3(u_int dr3); 733void load_dr4(u_int dr4); 734void load_dr5(u_int dr5); 735void load_dr6(u_int dr6); 736void load_dr7(u_int dr7); 737void load_fs(u_short sel); 738void load_gs(u_short sel); 739void ltr(u_short sel); 740void outb(u_int port, u_char data); 741void outl(u_int port, u_int data); 742void outsb(u_int port, const void *addr, size_t count); 743void outsl(u_int port, const void *addr, size_t count); 744void outsw(u_int port, const void *addr, size_t count); 745void outw(u_int port, u_short data); 746u_int rcr0(void); 747u_int rcr2(void); 748u_int rcr3(void); 749u_int rcr4(void); 750uint64_t rdmsr(u_int msr); 751uint64_t rdpmc(u_int pmc); 752u_int rdr0(void); 753u_int rdr1(void); 754u_int rdr2(void); 755u_int rdr3(void); 756u_int rdr4(void); 757u_int rdr5(void); 758u_int rdr6(void); 759u_int rdr7(void); 760uint64_t rdtsc(void); 761u_char read_cyrix_reg(u_char reg); 762u_int read_eflags(void); 763u_int rfs(void); 764uint64_t rgdt(void); 765u_int rgs(void); 766uint64_t ridt(void); 767u_short rldt(void); 768u_short rtr(void); 769void wbinvd(void); 770void write_cyrix_reg(u_char reg, u_char data); 771void write_eflags(u_int ef); 772void wrmsr(u_int msr, uint64_t newval); 773 774#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ 775 776void reset_dbregs(void); 777 778#ifdef _KERNEL 779int rdmsr_safe(u_int msr, uint64_t *val); 780int wrmsr_safe(u_int msr, uint64_t newval); 781#endif 782 783#endif /* !_MACHINE_CPUFUNC_H_ */ 784