1/* $NetBSD: cpufunc.S,v 1.20 2011/09/24 10:32:52 jym Exp $ */ 2 3/*- 4 * Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Charles M. Hannum, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * Functions to provide access to i386-specific instructions. 34 */ 35 36#include <sys/errno.h> 37 38#include <machine/asm.h> 39#include <machine/frameasm.h> 40#include <machine/specialreg.h> 41#include <machine/segments.h> 42 43#include "opt_xen.h" 44 45#include "assym.h" 46 47/* Small and slow, so align less. */ 48#undef _ALIGN_TEXT 49#define _ALIGN_TEXT .align 8 50 51ENTRY(x86_lfence) 52 lfence 53 ret 54 55ENTRY(x86_sfence) 56 sfence 57 ret 58 59ENTRY(x86_mfence) 60 mfence 61 ret 62 63#ifndef XEN 64ENTRY(invlpg) 65 invlpg (%rdi) 66 ret 67 68ENTRY(lidt) 69 lidt (%rdi) 70 ret 71 72ENTRY(lldt) 73 cmpl %edi, CPUVAR(CURLDT) 74 jne 1f 75 ret 761: 77 movl %edi, CPUVAR(CURLDT) 78 lldt %di 79 ret 80 81ENTRY(ltr) 82 ltr %di 83 ret 84 85ENTRY(lcr0) 86 movq %rdi, %cr0 87 ret 88 89ENTRY(rcr0) 90 movq %cr0, %rax 91 ret 92 93ENTRY(lcr2) 94 movq %rdi, %cr2 95 ret 96 97ENTRY(rcr2) 98 movq %cr2, %rax 99 ret 100 101ENTRY(lcr3) 102 movq %rdi, %cr3 103 ret 104 105ENTRY(rcr3) 106 movq %cr3, %rax 107 ret 108#endif 109 110ENTRY(lcr4) 111 movq %rdi, %cr4 112 ret 113 114ENTRY(rcr4) 115 movq %cr4, %rax 116 ret 117 118ENTRY(lcr8) 119 movq %rdi, %cr8 120 ret 121 122ENTRY(rcr8) 123 movq %cr8, %rax 124 ret 125 126/* 127 * Big hammer: flush all TLB entries, including ones from PTE's 128 * with the G bit set. This should only be necessary if TLB 129 * shootdown falls far behind. 130 * 131 * Intel Architecture Software Developer's Manual, Volume 3, 132 * System Programming, section 9.10, "Invalidating the 133 * Translation Lookaside Buffers (TLBS)": 134 * "The following operations invalidate all TLB entries, irrespective 135 * of the setting of the G flag: 136 * ... 137 * "(P6 family processors only): Writing to control register CR4 to 138 * modify the PSE, PGE, or PAE flag." 139 * 140 * (the alternatives not quoted above are not an option here.) 141 * 142 * If PGE is not in use, we reload CR3. 143 */ 144#ifndef XEN 145ENTRY(tlbflushg) 146 movq %cr4, %rax 147 testq $CR4_PGE, %rax 148 jz 1f 149 movq %rax, %rdx 150 andq $~CR4_PGE, %rdx 151 movq %rdx, %cr4 152 movq %rax, %cr4 153 ret 154 155ENTRY(tlbflush) 1561: 157 movq %cr3, %rax 158 movq %rax, %cr3 159 ret 160 161ENTRY(ldr6) 162 movq %rdi, %dr6 163 ret 164 165ENTRY(rdr6) 166 movq %dr6, %rdi 167 ret 168 169ENTRY(x86_disable_intr) 170 cli 171 ret 172 173ENTRY(x86_enable_intr) 174 sti 175 ret 176 177ENTRY(x86_read_flags) 178 pushfq 179 popq %rax 180 ret 181 182STRONG_ALIAS(x86_read_psl,x86_read_flags) 183 184ENTRY(x86_write_flags) 185 pushq %rdi 186 popfq 187 ret 188 189STRONG_ALIAS(x86_write_psl,x86_write_flags) 190#endif /* XEN */ 191 192ENTRY(rdmsr) 193 movq %rdi, %rcx 194 xorq %rax, %rax 195 rdmsr 196 shlq $32, %rdx 197 orq %rdx, %rax 198 ret 199 200ENTRY(wrmsr) 201 movq %rdi, %rcx 202 movq %rsi, %rax 203 movq %rsi, %rdx 204 shrq $32, %rdx 205 wrmsr 206 ret 207 208ENTRY(rdmsr_locked) 209 movq %rdi, %rcx 210 xorq %rax, %rax 211 movl $OPTERON_MSR_PASSCODE, %edi 212 rdmsr 213 shlq $32, %rdx 214 orq %rdx, %rax 215 ret 216 217ENTRY(wrmsr_locked) 218 movq %rdi, %rcx 219 movq %rsi, %rax 220 movq %rsi, %rdx 221 shrq $32, %rdx 222 movl $OPTERON_MSR_PASSCODE, %edi 223 wrmsr 224 ret 225 226/* 227 * Support for reading MSRs in the safe manner (returns EFAULT on fault) 228 */ 229/* int rdmsr_safe(u_int msr, uint64_t *data) */ 230ENTRY(rdmsr_safe) 231 movq CPUVAR(CURLWP), %r8 232 movq L_PCB(%r8), %r8 233 movq $_C_LABEL(msr_onfault), PCB_ONFAULT(%r8) 234 235 movl %edi, %ecx /* u_int msr */ 236 rdmsr /* Read MSR pointed by %ecx. Returns 237 hi byte in edx, lo in %eax */ 238 salq $32, %rdx /* sign-shift %rdx left */ 239 movl %eax, %eax /* zero-extend %eax -> %rax */ 240 orq %rdx, %rax 241 movq %rax, (%rsi) /* *data */ 242 xorq %rax, %rax /* "no error" */ 243 244 movq %rax, PCB_ONFAULT(%r8) 245 ret 246 247/* 248 * MSR operations fault handler 249 */ 250NENTRY(msr_onfault) 251 movq CPUVAR(CURLWP), %r8 252 movq L_PCB(%r8), %r8 253 movq $0, PCB_ONFAULT(%r8) 254 movl $EFAULT, %eax 255 ret 256 257#ifndef XEN 258ENTRY(wbinvd) 259 wbinvd 260 ret 261#endif 262 263ENTRY(cpu_counter) 264 xorq %rax, %rax 265 rdtsc 266 shlq $32, %rdx 267 orq %rdx, %rax 268 addq CPUVAR(CC_SKEW), %rax 269 ret 270 271ENTRY(cpu_counter32) 272 rdtsc 273 addl CPUVAR(CC_SKEW), %eax 274 ret 275 276ENTRY(rdpmc) 277 movq %rdi, %rcx 278 xorq %rax, %rax 279 rdpmc 280 shlq $32, %rdx 281 orq %rdx, %rax 282 ret 283 284ENTRY(breakpoint) 285 pushq %rbp 286 movq %rsp, %rbp 287 int $0x03 /* paranoid, not 'int3' */ 288 leave 289 ret 290 291ENTRY(x86_curcpu) 292 movq %gs:(CPU_INFO_SELF), %rax 293 ret 294 295ENTRY(x86_curlwp) 296 movq %gs:(CPU_INFO_CURLWP), %rax 297 ret 298 299ENTRY(cpu_set_curpri) 300 movl %edi, %gs:(CPU_INFO_CURPRIORITY) 301 ret 302 303ENTRY(__byte_swap_u32_variable) 304 movl %edi, %eax 305 bswapl %eax 306 ret 307 308ENTRY(__byte_swap_u16_variable) 309 movl %edi, %eax 310 xchgb %al, %ah 311 ret 312 313/* 314 * void lgdt(struct region_descriptor *rdp); 315 * 316 * Load a new GDT pointer (and do any necessary cleanup). 317 * XXX It's somewhat questionable whether reloading all the segment registers 318 * is necessary, since the actual descriptor data is not changed except by 319 * process creation and exit, both of which clean up via task switches. OTOH, 320 * this only happens at run time when the GDT is resized. 321 */ 322#ifndef XEN 323ENTRY(lgdt) 324 /* Reload the descriptor table. */ 325 movq %rdi,%rax 326 lgdt (%rax) 327 /* Flush the prefetch q. */ 328 jmp 1f 329 nop 3301: /* Reload "stale" selectors. */ 331#else /* XEN */ 332/* 333 * void lgdt_finish(void); 334 * Reload segments after a GDT change 335 */ 336ENTRY(lgdt_finish) 337#endif /* XEN */ 338 movl $GSEL(GDATA_SEL, SEL_KPL),%eax 339 movl %eax,%ds 340 movl %eax,%es 341 movl %eax,%ss 342 /* FALLTHROUGH */ 343 344/* 345 * void x86_flush() 346 * 347 * Flush instruction pipelines by doing an intersegment (far) return. 348 */ 349ENTRY(x86_flush) 350 popq %rax 351 pushq $GSEL(GCODE_SEL, SEL_KPL) 352 pushq %rax 353 lretq 354 355/* Waits - set up stack frame. */ 356ENTRY(x86_hlt) 357 pushq %rbp 358 movq %rsp, %rbp 359 hlt 360 leave 361 ret 362 363/* Waits - set up stack frame. */ 364ENTRY(x86_stihlt) 365 pushq %rbp 366 movq %rsp, %rbp 367 sti 368 hlt 369 leave 370 ret 371 372ENTRY(x86_monitor) 373 movq %rdi, %rax 374 movq %rsi, %rcx 375 monitor %rax, %rcx, %rdx 376 ret 377 378/* Waits - set up stack frame. */ 379ENTRY(x86_mwait) 380 pushq %rbp 381 movq %rsp, %rbp 382 movq %rdi, %rax 383 movq %rsi, %rcx 384 mwait %rax, %rcx 385 leave 386 ret 387 388NENTRY(x86_pause) 389 pause 390 ret 391 392ENTRY(x86_cpuid2) 393 movq %rbx, %r8 394 movq %rdi, %rax 395 movq %rsi, %rcx 396 movq %rdx, %rsi 397 cpuid 398 movl %eax, 0(%rsi) 399 movl %ebx, 4(%rsi) 400 movl %ecx, 8(%rsi) 401 movl %edx, 12(%rsi) 402 movq %r8, %rbx 403 ret 404 405ENTRY(x86_getss) 406 movl %ss, %eax 407 ret 408 409ENTRY(fldcw) 410 fldcw (%rdi) 411 ret 412 413ENTRY(fnclex) 414 fnclex 415 ret 416 417ENTRY(fninit) 418 fninit 419 ret 420 421ENTRY(fnsave) 422 fnsave (%rdi) 423 ret 424 425ENTRY(fnstcw) 426 fnstcw (%rdi) 427 ret 428 429ENTRY(fnstsw) 430 fnstsw (%rdi) 431 ret 432 433ENTRY(fp_divide_by_0) 434 fldz 435 fld1 436 fdiv %st, %st(1) 437 fwait 438 ret 439 440ENTRY(frstor) 441 frstor (%rdi) 442 ret 443 444ENTRY(fwait) 445 fwait 446 ret 447 448ENTRY(clts) 449 clts 450 ret 451 452ENTRY(stts) 453 movq %cr0, %rax 454 orq $CR0_TS, %rax 455 movq %rax, %cr0 456 ret 457 458ENTRY(fxsave) 459 fxsave (%rdi) 460 ret 461 462ENTRY(fxrstor) 463 fxrstor (%rdi) 464 ret 465 466ENTRY(fldummy) 467 ffree %st(7) 468 flds (%rdi) 469 ret 470 471ENTRY(x86_ldmxcsr) 472 ldmxcsr (%rdi) 473 ret 474 475ENTRY(inb) 476 movq %rdi, %rdx 477 xorq %rax, %rax 478 inb %dx, %al 479 ret 480 481ENTRY(insb) 482 movl %edx, %ecx 483 movl %edi, %edx 484 movq %rsi, %rdi 485 rep 486 insb 487 ret 488 489ENTRY(inw) 490 movq %rdi, %rdx 491 xorq %rax, %rax 492 inw %dx, %ax 493 ret 494 495ENTRY(insw) 496 movl %edx, %ecx 497 movl %edi, %edx 498 movq %rsi, %rdi 499 rep 500 insw 501 ret 502 503ENTRY(inl) 504 movq %rdi, %rdx 505 xorq %rax, %rax 506 inl %dx, %eax 507 ret 508 509ENTRY(insl) 510 movl %edx, %ecx 511 movl %edi, %edx 512 movq %rsi, %rdi 513 rep 514 insl 515 ret 516 517ENTRY(outb) 518 movq %rdi, %rdx 519 movq %rsi, %rax 520 outb %al, %dx 521 ret 522 523ENTRY(outsb) 524 movl %edx, %ecx 525 movl %edi, %edx 526 rep 527 outsb 528 ret 529 530ENTRY(outw) 531 movq %rdi, %rdx 532 movq %rsi, %rax 533 outw %ax, %dx 534 ret 535 536ENTRY(outsw) 537 movl %edx, %ecx 538 movl %edi, %edx 539 rep 540 outsw 541 ret 542 543ENTRY(outl) 544 movq %rdi, %rdx 545 movq %rsi, %rax 546 outl %eax, %dx 547 ret 548 549ENTRY(outsl) 550 movl %edx, %ecx 551 movl %edi, %edx 552 rep 553 outsl 554 ret 555 556ENTRY(setfs) 557 movw %di, %fs 558 ret 559 560ENTRY(setusergs) 561 CLI(ax) 562 swapgs 563 movw %di, %gs 564 swapgs 565 STI(ax) 566 ret 567