1/*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1993 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32/* 33 * Functions to provide access to special i386 instructions. 34 * This in included in sys/systm.h, and that file should be 35 * used in preference to this. 36 */ 37 38#ifndef _MACHINE_CPUFUNC_H_ 39#define _MACHINE_CPUFUNC_H_ 40 41struct region_descriptor; 42 43#define readb(va) (*(volatile uint8_t *) (va)) 44#define readw(va) (*(volatile uint16_t *) (va)) 45#define readl(va) (*(volatile uint32_t *) (va)) 46 47#define writeb(va, d) (*(volatile uint8_t *) (va) = (d)) 48#define writew(va, d) (*(volatile uint16_t *) (va) = (d)) 49#define writel(va, d) (*(volatile uint32_t *) (va) = (d)) 50 51static __inline void 52breakpoint(void) 53{ 54 __asm __volatile("int $3"); 55} 56 57static __inline __pure2 u_int 58bsfl(u_int mask) 59{ 60 u_int result; 61 62 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc"); 63 return (result); 64} 65 66static __inline void 67clflush(u_long addr) 68{ 69 70 __asm __volatile("clflush %0" : : "m" (*(char *)addr)); 71} 72 73static __inline void 74clflushopt(u_long addr) 75{ 76 77 __asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr)); 78} 79 80static __inline void 81clts(void) 82{ 83 84 __asm __volatile("clts"); 85} 86 87static __inline void 88disable_intr(void) 89{ 90 __asm __volatile("cli" : : : "memory"); 91} 92 93#ifdef _KERNEL 94static __inline void 95do_cpuid(u_int ax, u_int *p) 96{ 97 __asm __volatile("cpuid" 98 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 99 : "0" (ax)); 100} 101 102static __inline void 103cpuid_count(u_int ax, u_int cx, u_int *p) 104{ 105 __asm __volatile("cpuid" 106 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3]) 107 : "0" (ax), "c" (cx)); 108} 109#else 110static __inline void 111do_cpuid(u_int ax, u_int *p) 112{ 113 __asm __volatile( 114 "pushl\t%%ebx\n\t" 115 "cpuid\n\t" 116 "movl\t%%ebx,%1\n\t" 117 "popl\t%%ebx" 118 : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3]) 119 : "0" (ax)); 120} 121 122static __inline void 123cpuid_count(u_int ax, u_int cx, u_int *p) 124{ 125 __asm __volatile( 126 "pushl\t%%ebx\n\t" 127 "cpuid\n\t" 128 "movl\t%%ebx,%1\n\t" 129 "popl\t%%ebx" 130 : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3]) 131 : "0" (ax), "c" (cx)); 132} 133#endif 134 135static __inline void 136enable_intr(void) 137{ 138 __asm __volatile("sti"); 139} 140 141static __inline void 142cpu_monitor(const void *addr, u_long extensions, u_int hints) 143{ 144 __asm __volatile("monitor" 145 : : "a" (addr), "c" (extensions), "d" (hints)); 146} 147 148static __inline void 149cpu_mwait(u_long extensions, u_int hints) 150{ 151 __asm __volatile("mwait" : : "a" (hints), "c" (extensions)); 152} 153 154static __inline void 155lfence(void) 156{ 157 __asm __volatile("lfence" : : : "memory"); 158} 159 160static __inline void 161mfence(void) 162{ 163 __asm __volatile("mfence" : : : "memory"); 164} 165 166static __inline void 167sfence(void) 168{ 169 __asm __volatile("sfence" : : : "memory"); 170} 171 172static __inline void 173halt(void) 174{ 175 __asm __volatile("hlt"); 176} 177 178static __inline u_char 179inb(u_int port) 180{ 181 u_char data; 182 183 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port)); 184 return (data); 185} 186 187static __inline u_int 188inl(u_int port) 189{ 190 u_int data; 191 192 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port)); 193 return (data); 194} 195 196static __inline void 197insb(u_int port, void *addr, size_t count) 198{ 199 __asm __volatile("cld; rep; insb" 200 : "+D" (addr), "+c" (count) 201 : "d" (port) 202 : "memory"); 203} 204 205static __inline void 206insw(u_int port, void *addr, size_t count) 207{ 208 __asm __volatile("cld; rep; insw" 209 : "+D" (addr), "+c" (count) 210 : "d" (port) 211 : "memory"); 212} 213 214static __inline void 215insl(u_int port, void *addr, size_t count) 216{ 217 __asm __volatile("cld; rep; insl" 218 : "+D" (addr), "+c" (count) 219 : "d" (port) 220 : "memory"); 221} 222 223static __inline void 224invd(void) 225{ 226 __asm __volatile("invd"); 227} 228 229static __inline u_short 230inw(u_int port) 231{ 232 u_short data; 233 234 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port)); 235 return (data); 236} 237 238static __inline void 239outb(u_int port, u_char data) 240{ 241 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port)); 242} 243 244static __inline void 245outl(u_int port, u_int data) 246{ 247 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port)); 248} 249 250static __inline void 251outsb(u_int port, const void *addr, size_t count) 252{ 253 __asm __volatile("cld; rep; outsb" 254 : "+S" (addr), "+c" (count) 255 : "d" (port)); 256} 257 258static __inline void 259outsw(u_int port, const void *addr, size_t count) 260{ 261 __asm __volatile("cld; rep; outsw" 262 : "+S" (addr), "+c" (count) 263 : "d" (port)); 264} 265 266static __inline void 267outsl(u_int port, const void *addr, size_t count) 268{ 269 __asm __volatile("cld; rep; outsl" 270 : "+S" (addr), "+c" (count) 271 : "d" (port)); 272} 273 274static __inline void 275outw(u_int port, u_short data) 276{ 277 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port)); 278} 279 280static __inline void 281ia32_pause(void) 282{ 283 __asm __volatile("pause"); 284} 285 286static __inline u_int 287read_eflags(void) 288{ 289 u_int ef; 290 291 __asm __volatile("pushfl; popl %0" : "=r" (ef)); 292 return (ef); 293} 294 295static __inline uint64_t 296rdmsr(u_int msr) 297{ 298 uint64_t rv; 299 300 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr)); 301 return (rv); 302} 303 304static __inline uint32_t 305rdmsr32(u_int msr) 306{ 307 uint32_t low; 308 309 __asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx"); 310 return (low); 311} 312 313static __inline uint64_t 314rdpmc(u_int pmc) 315{ 316 uint64_t rv; 317 318 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc)); 319 return (rv); 320} 321 322static __inline uint64_t 323rdtsc(void) 324{ 325 uint64_t rv; 326 327 __asm __volatile("rdtsc" : "=A" (rv)); 328 return (rv); 329} 330 331static __inline uint64_t 332rdtsc_ordered_lfence(void) 333{ 334 lfence(); 335 return (rdtsc()); 336} 337 338static __inline uint64_t 339rdtsc_ordered_mfence(void) 340{ 341 mfence(); 342 return (rdtsc()); 343} 344 345static __inline uint64_t 346rdtscp(void) 347{ 348 uint64_t rv; 349 350 __asm __volatile("rdtscp" : "=A" (rv) : : "ecx"); 351 return (rv); 352} 353 354static __inline uint64_t 355rdtscp_aux(uint32_t *aux) 356{ 357 uint64_t rv; 358 359 __asm __volatile("rdtscp" : "=A" (rv), "=c" (*aux)); 360 return (rv); 361} 362 363static __inline uint32_t 364rdtsc32(void) 365{ 366 uint32_t rv; 367 368 __asm __volatile("rdtsc" : "=a" (rv) : : "edx"); 369 return (rv); 370} 371 372static __inline uint32_t 373rdtscp32(void) 374{ 375 uint32_t rv; 376 377 __asm __volatile("rdtscp" : "=a" (rv) : : "ecx", "edx"); 378 return (rv); 379} 380 381static __inline void 382wbinvd(void) 383{ 384 __asm __volatile("wbinvd"); 385} 386 387static __inline void 388write_eflags(u_int ef) 389{ 390 __asm __volatile("pushl %0; popfl" : : "r" (ef)); 391} 392 393static __inline void 394wrmsr(u_int msr, uint64_t newval) 395{ 396 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr)); 397} 398 399static __inline void 400load_cr0(u_int data) 401{ 402 403 __asm __volatile("movl %0,%%cr0" : : "r" (data)); 404} 405 406static __inline u_int 407rcr0(void) 408{ 409 u_int data; 410 411 __asm __volatile("movl %%cr0,%0" : "=r" (data)); 412 return (data); 413} 414 415static __inline u_int 416rcr2(void) 417{ 418 u_int data; 419 420 __asm __volatile("movl %%cr2,%0" : "=r" (data)); 421 return (data); 422} 423 424static __inline void 425load_cr3(u_int data) 426{ 427 428 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory"); 429} 430 431static __inline u_int 432rcr3(void) 433{ 434 u_int data; 435 436 __asm __volatile("movl %%cr3,%0" : "=r" (data)); 437 return (data); 438} 439 440static __inline void 441load_cr4(u_int data) 442{ 443 __asm __volatile("movl %0,%%cr4" : : "r" (data)); 444} 445 446static __inline u_int 447rcr4(void) 448{ 449 u_int data; 450 451 __asm __volatile("movl %%cr4,%0" : "=r" (data)); 452 return (data); 453} 454 455static __inline uint64_t 456rxcr(u_int reg) 457{ 458 u_int low, high; 459 460 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg)); 461 return (low | ((uint64_t)high << 32)); 462} 463 464static __inline void 465load_xcr(u_int reg, uint64_t val) 466{ 467 u_int low, high; 468 469 low = val; 470 high = val >> 32; 471 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high)); 472} 473 474/* 475 * Global TLB flush (except for thise for pages marked PG_G) 476 */ 477static __inline void 478invltlb(void) 479{ 480 481 load_cr3(rcr3()); 482} 483 484/* 485 * TLB flush for an individual page (even if it has PG_G). 486 * Only works on 486+ CPUs (i386 does not have PG_G). 487 */ 488static __inline void 489invlpg(u_int addr) 490{ 491 492 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory"); 493} 494 495static __inline u_short 496rfs(void) 497{ 498 u_short sel; 499 __asm __volatile("movw %%fs,%0" : "=rm" (sel)); 500 return (sel); 501} 502 503static __inline uint64_t 504rgdt(void) 505{ 506 uint64_t gdtr; 507 __asm __volatile("sgdt %0" : "=m" (gdtr)); 508 return (gdtr); 509} 510 511static __inline u_short 512rgs(void) 513{ 514 u_short sel; 515 __asm __volatile("movw %%gs,%0" : "=rm" (sel)); 516 return (sel); 517} 518 519static __inline uint64_t 520ridt(void) 521{ 522 uint64_t idtr; 523 __asm __volatile("sidt %0" : "=m" (idtr)); 524 return (idtr); 525} 526 527static __inline u_short 528rldt(void) 529{ 530 u_short ldtr; 531 __asm __volatile("sldt %0" : "=g" (ldtr)); 532 return (ldtr); 533} 534 535static __inline u_short 536rss(void) 537{ 538 u_short sel; 539 __asm __volatile("movw %%ss,%0" : "=rm" (sel)); 540 return (sel); 541} 542 543static __inline u_short 544rtr(void) 545{ 546 u_short tr; 547 __asm __volatile("str %0" : "=g" (tr)); 548 return (tr); 549} 550 551static __inline void 552load_fs(u_short sel) 553{ 554 __asm __volatile("movw %0,%%fs" : : "rm" (sel)); 555} 556 557static __inline void 558load_gs(u_short sel) 559{ 560 __asm __volatile("movw %0,%%gs" : : "rm" (sel)); 561} 562 563static __inline void 564lidt(struct region_descriptor *addr) 565{ 566 __asm __volatile("lidt (%0)" : : "r" (addr)); 567} 568 569static __inline void 570lldt(u_short sel) 571{ 572 __asm __volatile("lldt %0" : : "r" (sel)); 573} 574 575static __inline void 576ltr(u_short sel) 577{ 578 __asm __volatile("ltr %0" : : "r" (sel)); 579} 580 581static __inline u_int 582rdr0(void) 583{ 584 u_int data; 585 __asm __volatile("movl %%dr0,%0" : "=r" (data)); 586 return (data); 587} 588 589static __inline void 590load_dr0(u_int dr0) 591{ 592 __asm __volatile("movl %0,%%dr0" : : "r" (dr0)); 593} 594 595static __inline u_int 596rdr1(void) 597{ 598 u_int data; 599 __asm __volatile("movl %%dr1,%0" : "=r" (data)); 600 return (data); 601} 602 603static __inline void 604load_dr1(u_int dr1) 605{ 606 __asm __volatile("movl %0,%%dr1" : : "r" (dr1)); 607} 608 609static __inline u_int 610rdr2(void) 611{ 612 u_int data; 613 __asm __volatile("movl %%dr2,%0" : "=r" (data)); 614 return (data); 615} 616 617static __inline void 618load_dr2(u_int dr2) 619{ 620 __asm __volatile("movl %0,%%dr2" : : "r" (dr2)); 621} 622 623static __inline u_int 624rdr3(void) 625{ 626 u_int data; 627 __asm __volatile("movl %%dr3,%0" : "=r" (data)); 628 return (data); 629} 630 631static __inline void 632load_dr3(u_int dr3) 633{ 634 __asm __volatile("movl %0,%%dr3" : : "r" (dr3)); 635} 636 637static __inline u_int 638rdr6(void) 639{ 640 u_int data; 641 __asm __volatile("movl %%dr6,%0" : "=r" (data)); 642 return (data); 643} 644 645static __inline void 646load_dr6(u_int dr6) 647{ 648 __asm __volatile("movl %0,%%dr6" : : "r" (dr6)); 649} 650 651static __inline u_int 652rdr7(void) 653{ 654 u_int data; 655 __asm __volatile("movl %%dr7,%0" : "=r" (data)); 656 return (data); 657} 658 659static __inline void 660load_dr7(u_int dr7) 661{ 662 __asm __volatile("movl %0,%%dr7" : : "r" (dr7)); 663} 664 665static __inline u_char 666read_cyrix_reg(u_char reg) 667{ 668 outb(0x22, reg); 669 return inb(0x23); 670} 671 672static __inline void 673write_cyrix_reg(u_char reg, u_char data) 674{ 675 outb(0x22, reg); 676 outb(0x23, data); 677} 678 679static __inline register_t 680intr_disable(void) 681{ 682 register_t eflags; 683 684 eflags = read_eflags(); 685 disable_intr(); 686 return (eflags); 687} 688 689static __inline void 690intr_restore(register_t eflags) 691{ 692 write_eflags(eflags); 693} 694 695static __inline uint32_t 696rdpkru(void) 697{ 698 uint32_t res; 699 700 __asm __volatile("rdpkru" : "=a" (res) : "c" (0) : "edx"); 701 return (res); 702} 703 704static __inline void 705wrpkru(uint32_t mask) 706{ 707 708 __asm __volatile("wrpkru" : : "a" (mask), "c" (0), "d" (0)); 709} 710 711void reset_dbregs(void); 712 713#ifdef _KERNEL 714int rdmsr_safe(u_int msr, uint64_t *val); 715int wrmsr_safe(u_int msr, uint64_t newval); 716#endif 717 718#endif /* !_MACHINE_CPUFUNC_H_ */ 719