31 */
32
33#include "opt_ddb.h"
34
35#include <machine/asmacros.h>
36#include <machine/specialreg.h>
37#include <machine/pmap.h>
38
39#include "assym.s"
40
41 .text
42
43/*
44 * bcopy family
45 * void bzero(void *buf, u_int len)
46 */
47
48/* done */
49ENTRY(bzero)
50 PUSH_FRAME_POINTER
51 movq %rsi,%rcx
52 xorl %eax,%eax
53 shrq $3,%rcx
54 rep
55 stosq
56 movq %rsi,%rcx
57 andq $7,%rcx
58 rep
59 stosb
60 POP_FRAME_POINTER
61 ret
62END(bzero)
63
64/* Address: %rdi */
65ENTRY(pagezero)
66 PUSH_FRAME_POINTER
67 movq $PAGE_SIZE/8,%rcx
68 xorl %eax,%eax
69 rep
70 stosq
71 POP_FRAME_POINTER
72 ret
73END(pagezero)
74
75ENTRY(bcmp)
76 PUSH_FRAME_POINTER
77 movq %rdx,%rcx
78 shrq $3,%rcx
79 repe
80 cmpsq
81 jne 1f
82
83 movq %rdx,%rcx
84 andq $7,%rcx
85 repe
86 cmpsb
871:
88 setne %al
89 movsbl %al,%eax
90 POP_FRAME_POINTER
91 ret
92END(bcmp)
93
94/*
95 * bcopy(src, dst, cnt)
96 * rdi, rsi, rdx
97 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800
98 */
99ENTRY(bcopy)
100 PUSH_FRAME_POINTER
101 xchgq %rsi,%rdi
102 movq %rdx,%rcx
103
104 movq %rdi,%rax
105 subq %rsi,%rax
106 cmpq %rcx,%rax /* overlapping && src < dst? */
107 jb 1f
108
109 shrq $3,%rcx /* copy by 64-bit words */
110 rep
111 movsq
112 movq %rdx,%rcx
113 andq $7,%rcx /* any bytes left? */
114 rep
115 movsb
116 POP_FRAME_POINTER
117 ret
118
119 /* ALIGN_TEXT */
1201:
121 addq %rcx,%rdi /* copy backwards */
122 addq %rcx,%rsi
123 decq %rdi
124 decq %rsi
125 andq $7,%rcx /* any fractional bytes? */
126 std
127 rep
128 movsb
129 movq %rdx,%rcx /* copy remainder by 32-bit words */
130 shrq $3,%rcx
131 subq $7,%rsi
132 subq $7,%rdi
133 rep
134 movsq
135 cld
136 POP_FRAME_POINTER
137 ret
138END(bcopy)
139
140/*
141 * Note: memcpy does not support overlapping copies
142 */
143ENTRY(memcpy)
144 PUSH_FRAME_POINTER
145 movq %rdi,%rax
146 movq %rdx,%rcx
147 shrq $3,%rcx /* copy by 64-bit words */
148 rep
149 movsq
150 movq %rdx,%rcx
151 andq $7,%rcx /* any bytes left? */
152 rep
153 movsb
154 POP_FRAME_POINTER
155 ret
156END(memcpy)
157
158/*
159 * pagecopy(%rdi=from, %rsi=to)
160 */
161ENTRY(pagecopy)
162 PUSH_FRAME_POINTER
163 movq $-PAGE_SIZE,%rax
164 movq %rax,%rdx
165 subq %rax,%rdi
166 subq %rax,%rsi
1671:
168 prefetchnta (%rdi,%rax)
169 addq $64,%rax
170 jne 1b
1712:
172 movq (%rdi,%rdx),%rax
173 movnti %rax,(%rsi,%rdx)
174 movq 8(%rdi,%rdx),%rax
175 movnti %rax,8(%rsi,%rdx)
176 movq 16(%rdi,%rdx),%rax
177 movnti %rax,16(%rsi,%rdx)
178 movq 24(%rdi,%rdx),%rax
179 movnti %rax,24(%rsi,%rdx)
180 addq $32,%rdx
181 jne 2b
182 sfence
183 POP_FRAME_POINTER
184 ret
185END(pagecopy)
186
187/* fillw(pat, base, cnt) */
188/* %rdi,%rsi, %rdx */
189ENTRY(fillw)
190 PUSH_FRAME_POINTER
191 movq %rdi,%rax
192 movq %rsi,%rdi
193 movq %rdx,%rcx
194 rep
195 stosw
196 POP_FRAME_POINTER
197 ret
198END(fillw)
199
200/*****************************************************************************/
201/* copyout and fubyte family */
202/*****************************************************************************/
203/*
204 * Access user memory from inside the kernel. These routines should be
205 * the only places that do this.
206 *
207 * These routines set curpcb->pcb_onfault for the time they execute. When a
208 * protection violation occurs inside the functions, the trap handler
209 * returns to *curpcb->pcb_onfault instead of the function.
210 */
211
212/*
213 * copyout(from_kernel, to_user, len)
214 * %rdi, %rsi, %rdx
215 */
216ENTRY(copyout)
217 PUSH_FRAME_POINTER
218 movq PCPU(CURPCB),%rax
219 movq $copyout_fault,PCB_ONFAULT(%rax)
220 testq %rdx,%rdx /* anything to do? */
221 jz done_copyout
222
223 /*
224 * Check explicitly for non-user addresses. This check is essential
225 * because it prevents usermode from writing into the kernel. We do
226 * not verify anywhere else that the user did not specify a rogue
227 * address.
228 */
229 /*
230 * First, prevent address wrapping.
231 */
232 movq %rsi,%rax
233 addq %rdx,%rax
234 jc copyout_fault
235/*
236 * XXX STOP USING VM_MAXUSER_ADDRESS.
237 * It is an end address, not a max, so every time it is used correctly it
238 * looks like there is an off by one error, and of course it caused an off
239 * by one error in several places.
240 */
241 movq $VM_MAXUSER_ADDRESS,%rcx
242 cmpq %rcx,%rax
243 ja copyout_fault
244
245 xchgq %rdi,%rsi
246 /* bcopy(%rsi, %rdi, %rdx) */
247 movq %rdx,%rcx
248
249 shrq $3,%rcx
250 rep
251 movsq
252 movb %dl,%cl
253 andb $7,%cl
254 rep
255 movsb
256
257done_copyout:
258 xorl %eax,%eax
259 movq PCPU(CURPCB),%rdx
260 movq %rax,PCB_ONFAULT(%rdx)
261 POP_FRAME_POINTER
262 ret
263
264 ALIGN_TEXT
265copyout_fault:
266 movq PCPU(CURPCB),%rdx
267 movq $0,PCB_ONFAULT(%rdx)
268 movq $EFAULT,%rax
269 POP_FRAME_POINTER
270 ret
271END(copyout)
272
273/*
274 * copyin(from_user, to_kernel, len)
275 * %rdi, %rsi, %rdx
276 */
277ENTRY(copyin)
278 PUSH_FRAME_POINTER
279 movq PCPU(CURPCB),%rax
280 movq $copyin_fault,PCB_ONFAULT(%rax)
281 testq %rdx,%rdx /* anything to do? */
282 jz done_copyin
283
284 /*
285 * make sure address is valid
286 */
287 movq %rdi,%rax
288 addq %rdx,%rax
289 jc copyin_fault
290 movq $VM_MAXUSER_ADDRESS,%rcx
291 cmpq %rcx,%rax
292 ja copyin_fault
293
294 xchgq %rdi,%rsi
295 movq %rdx,%rcx
296 movb %cl,%al
297 shrq $3,%rcx /* copy longword-wise */
298 rep
299 movsq
300 movb %al,%cl
301 andb $7,%cl /* copy remaining bytes */
302 rep
303 movsb
304
305done_copyin:
306 xorl %eax,%eax
307 movq PCPU(CURPCB),%rdx
308 movq %rax,PCB_ONFAULT(%rdx)
309 POP_FRAME_POINTER
310 ret
311
312 ALIGN_TEXT
313copyin_fault:
314 movq PCPU(CURPCB),%rdx
315 movq $0,PCB_ONFAULT(%rdx)
316 movq $EFAULT,%rax
317 POP_FRAME_POINTER
318 ret
319END(copyin)
320
321/*
322 * casueword32. Compare and set user integer. Returns -1 on fault,
323 * 0 if access was successful. Old value is written to *oldp.
324 * dst = %rdi, old = %esi, oldp = %rdx, new = %ecx
325 */
326ENTRY(casueword32)
327 PUSH_FRAME_POINTER
328 movq PCPU(CURPCB),%r8
329 movq $fusufault,PCB_ONFAULT(%r8)
330
331 movq $VM_MAXUSER_ADDRESS-4,%rax
332 cmpq %rax,%rdi /* verify address is valid */
333 ja fusufault
334
335 movl %esi,%eax /* old */
336#ifdef SMP
337 lock
338#endif
339 cmpxchgl %ecx,(%rdi) /* new = %ecx */
340
341 /*
342 * The old value is in %eax. If the store succeeded it will be the
343 * value we expected (old) from before the store, otherwise it will
344 * be the current value. Save %eax into %esi to prepare the return
345 * value.
346 */
347 movl %eax,%esi
348 xorl %eax,%eax
349 movq %rax,PCB_ONFAULT(%r8)
350
351 /*
352 * Access the oldp after the pcb_onfault is cleared, to correctly
353 * catch corrupted pointer.
354 */
355 movl %esi,(%rdx) /* oldp = %rdx */
356 POP_FRAME_POINTER
357 ret
358END(casueword32)
359
360/*
361 * casueword. Compare and set user long. Returns -1 on fault,
362 * 0 if access was successful. Old value is written to *oldp.
363 * dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx
364 */
365ENTRY(casueword)
366 PUSH_FRAME_POINTER
367 movq PCPU(CURPCB),%r8
368 movq $fusufault,PCB_ONFAULT(%r8)
369
370 movq $VM_MAXUSER_ADDRESS-4,%rax
371 cmpq %rax,%rdi /* verify address is valid */
372 ja fusufault
373
374 movq %rsi,%rax /* old */
375#ifdef SMP
376 lock
377#endif
378 cmpxchgq %rcx,(%rdi) /* new = %rcx */
379
380 /*
381 * The old value is in %rax. If the store succeeded it will be the
382 * value we expected (old) from before the store, otherwise it will
383 * be the current value.
384 */
385 movq %rax,%rsi
386 xorl %eax,%eax
387 movq %rax,PCB_ONFAULT(%r8)
388 movq %rsi,(%rdx)
389 POP_FRAME_POINTER
390 ret
391END(casueword)
392
393/*
394 * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit
395 * byte from user memory.
396 * addr = %rdi, valp = %rsi
397 */
398
399ALTENTRY(fueword64)
400ENTRY(fueword)
401 PUSH_FRAME_POINTER
402 movq PCPU(CURPCB),%rcx
403 movq $fusufault,PCB_ONFAULT(%rcx)
404
405 movq $VM_MAXUSER_ADDRESS-8,%rax
406 cmpq %rax,%rdi /* verify address is valid */
407 ja fusufault
408
409 xorl %eax,%eax
410 movq (%rdi),%r11
411 movq %rax,PCB_ONFAULT(%rcx)
412 movq %r11,(%rsi)
413 POP_FRAME_POINTER
414 ret
415END(fueword64)
416END(fueword)
417
418ENTRY(fueword32)
419 PUSH_FRAME_POINTER
420 movq PCPU(CURPCB),%rcx
421 movq $fusufault,PCB_ONFAULT(%rcx)
422
423 movq $VM_MAXUSER_ADDRESS-4,%rax
424 cmpq %rax,%rdi /* verify address is valid */
425 ja fusufault
426
427 xorl %eax,%eax
428 movl (%rdi),%r11d
429 movq %rax,PCB_ONFAULT(%rcx)
430 movl %r11d,(%rsi)
431 POP_FRAME_POINTER
432 ret
433END(fueword32)
434
435/*
436 * fuswintr() and suswintr() are specialized variants of fuword16() and
437 * suword16(), respectively. They are called from the profiling code,
438 * potentially at interrupt time. If they fail, that's okay; good things
439 * will happen later. They always fail for now, until the trap code is
440 * able to deal with this.
441 */
442ALTENTRY(suswintr)
443ENTRY(fuswintr)
444 movq $-1,%rax
445 ret
446END(suswintr)
447END(fuswintr)
448
449ENTRY(fuword16)
450 PUSH_FRAME_POINTER
451 movq PCPU(CURPCB),%rcx
452 movq $fusufault,PCB_ONFAULT(%rcx)
453
454 movq $VM_MAXUSER_ADDRESS-2,%rax
455 cmpq %rax,%rdi
456 ja fusufault
457
458 movzwl (%rdi),%eax
459 movq $0,PCB_ONFAULT(%rcx)
460 POP_FRAME_POINTER
461 ret
462END(fuword16)
463
464ENTRY(fubyte)
465 PUSH_FRAME_POINTER
466 movq PCPU(CURPCB),%rcx
467 movq $fusufault,PCB_ONFAULT(%rcx)
468
469 movq $VM_MAXUSER_ADDRESS-1,%rax
470 cmpq %rax,%rdi
471 ja fusufault
472
473 movzbl (%rdi),%eax
474 movq $0,PCB_ONFAULT(%rcx)
475 POP_FRAME_POINTER
476 ret
477END(fubyte)
478
479 ALIGN_TEXT
480fusufault:
481 movq PCPU(CURPCB),%rcx
482 xorl %eax,%eax
483 movq %rax,PCB_ONFAULT(%rcx)
484 decq %rax
485 POP_FRAME_POINTER
486 ret
487
488/*
489 * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to
490 * user memory.
491 * addr = %rdi, value = %rsi
492 */
493ALTENTRY(suword64)
494ENTRY(suword)
495 PUSH_FRAME_POINTER
496 movq PCPU(CURPCB),%rcx
497 movq $fusufault,PCB_ONFAULT(%rcx)
498
499 movq $VM_MAXUSER_ADDRESS-8,%rax
500 cmpq %rax,%rdi /* verify address validity */
501 ja fusufault
502
503 movq %rsi,(%rdi)
504 xorl %eax,%eax
505 movq PCPU(CURPCB),%rcx
506 movq %rax,PCB_ONFAULT(%rcx)
507 POP_FRAME_POINTER
508 ret
509END(suword64)
510END(suword)
511
512ENTRY(suword32)
513 PUSH_FRAME_POINTER
514 movq PCPU(CURPCB),%rcx
515 movq $fusufault,PCB_ONFAULT(%rcx)
516
517 movq $VM_MAXUSER_ADDRESS-4,%rax
518 cmpq %rax,%rdi /* verify address validity */
519 ja fusufault
520
521 movl %esi,(%rdi)
522 xorl %eax,%eax
523 movq PCPU(CURPCB),%rcx
524 movq %rax,PCB_ONFAULT(%rcx)
525 POP_FRAME_POINTER
526 ret
527END(suword32)
528
529ENTRY(suword16)
530 PUSH_FRAME_POINTER
531 movq PCPU(CURPCB),%rcx
532 movq $fusufault,PCB_ONFAULT(%rcx)
533
534 movq $VM_MAXUSER_ADDRESS-2,%rax
535 cmpq %rax,%rdi /* verify address validity */
536 ja fusufault
537
538 movw %si,(%rdi)
539 xorl %eax,%eax
540 movq PCPU(CURPCB),%rcx /* restore trashed register */
541 movq %rax,PCB_ONFAULT(%rcx)
542 POP_FRAME_POINTER
543 ret
544END(suword16)
545
546ENTRY(subyte)
547 PUSH_FRAME_POINTER
548 movq PCPU(CURPCB),%rcx
549 movq $fusufault,PCB_ONFAULT(%rcx)
550
551 movq $VM_MAXUSER_ADDRESS-1,%rax
552 cmpq %rax,%rdi /* verify address validity */
553 ja fusufault
554
555 movl %esi,%eax
556 movb %al,(%rdi)
557 xorl %eax,%eax
558 movq PCPU(CURPCB),%rcx /* restore trashed register */
559 movq %rax,PCB_ONFAULT(%rcx)
560 POP_FRAME_POINTER
561 ret
562END(subyte)
563
564/*
565 * copyinstr(from, to, maxlen, int *lencopied)
566 * %rdi, %rsi, %rdx, %rcx
567 *
568 * copy a string from 'from' to 'to', stop when a 0 character is reached.
569 * return ENAMETOOLONG if string is longer than maxlen, and
570 * EFAULT on protection violations. If lencopied is non-zero,
571 * return the actual length in *lencopied.
572 */
573ENTRY(copyinstr)
574 PUSH_FRAME_POINTER
575 movq %rdx,%r8 /* %r8 = maxlen */
576 movq %rcx,%r9 /* %r9 = *len */
577 xchgq %rdi,%rsi /* %rdi = from, %rsi = to */
578 movq PCPU(CURPCB),%rcx
579 movq $cpystrflt,PCB_ONFAULT(%rcx)
580
581 movq $VM_MAXUSER_ADDRESS,%rax
582
583 /* make sure 'from' is within bounds */
584 subq %rsi,%rax
585 jbe cpystrflt
586
587 /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
588 cmpq %rdx,%rax
589 jae 1f
590 movq %rax,%rdx
591 movq %rax,%r8
5921:
593 incq %rdx
594
5952:
596 decq %rdx
597 jz 3f
598
599 lodsb
600 stosb
601 orb %al,%al
602 jnz 2b
603
604 /* Success -- 0 byte reached */
605 decq %rdx
606 xorl %eax,%eax
607 jmp cpystrflt_x
6083:
609 /* rdx is zero - return ENAMETOOLONG or EFAULT */
610 movq $VM_MAXUSER_ADDRESS,%rax
611 cmpq %rax,%rsi
612 jae cpystrflt
6134:
614 movq $ENAMETOOLONG,%rax
615 jmp cpystrflt_x
616
617cpystrflt:
618 movq $EFAULT,%rax
619
620cpystrflt_x:
621 /* set *lencopied and return %eax */
622 movq PCPU(CURPCB),%rcx
623 movq $0,PCB_ONFAULT(%rcx)
624
625 testq %r9,%r9
626 jz 1f
627 subq %rdx,%r8
628 movq %r8,(%r9)
6291:
630 POP_FRAME_POINTER
631 ret
632END(copyinstr)
633
634/*
635 * copystr(from, to, maxlen, int *lencopied)
636 * %rdi, %rsi, %rdx, %rcx
637 */
638ENTRY(copystr)
639 PUSH_FRAME_POINTER
640 movq %rdx,%r8 /* %r8 = maxlen */
641
642 xchgq %rdi,%rsi
643 incq %rdx
6441:
645 decq %rdx
646 jz 4f
647 lodsb
648 stosb
649 orb %al,%al
650 jnz 1b
651
652 /* Success -- 0 byte reached */
653 decq %rdx
654 xorl %eax,%eax
655 jmp 6f
6564:
657 /* rdx is zero -- return ENAMETOOLONG */
658 movq $ENAMETOOLONG,%rax
659
6606:
661
662 testq %rcx,%rcx
663 jz 7f
664 /* set *lencopied and return %rax */
665 subq %rdx,%r8
666 movq %r8,(%rcx)
6677:
668 POP_FRAME_POINTER
669 ret
670END(copystr)
671
672/*
673 * Handling of special amd64 registers and descriptor tables etc
674 */
675/* void lgdt(struct region_descriptor *rdp); */
676ENTRY(lgdt)
677 /* reload the descriptor table */
678 lgdt (%rdi)
679
680 /* flush the prefetch q */
681 jmp 1f
682 nop
6831:
684 movl $KDSEL,%eax
685 movl %eax,%ds
686 movl %eax,%es
687 movl %eax,%fs /* Beware, use wrmsr to set 64 bit base */
688 movl %eax,%gs
689 movl %eax,%ss
690
691 /* reload code selector by turning return into intersegmental return */
692 popq %rax
693 pushq $KCSEL
694 pushq %rax
695 MEXITCOUNT
696 lretq
697END(lgdt)
698
699/*****************************************************************************/
700/* setjump, longjump */
701/*****************************************************************************/
702
703ENTRY(setjmp)
704 movq %rbx,0(%rdi) /* save rbx */
705 movq %rsp,8(%rdi) /* save rsp */
706 movq %rbp,16(%rdi) /* save rbp */
707 movq %r12,24(%rdi) /* save r12 */
708 movq %r13,32(%rdi) /* save r13 */
709 movq %r14,40(%rdi) /* save r14 */
710 movq %r15,48(%rdi) /* save r15 */
711 movq 0(%rsp),%rdx /* get rta */
712 movq %rdx,56(%rdi) /* save rip */
713 xorl %eax,%eax /* return(0); */
714 ret
715END(setjmp)
716
717ENTRY(longjmp)
718 movq 0(%rdi),%rbx /* restore rbx */
719 movq 8(%rdi),%rsp /* restore rsp */
720 movq 16(%rdi),%rbp /* restore rbp */
721 movq 24(%rdi),%r12 /* restore r12 */
722 movq 32(%rdi),%r13 /* restore r13 */
723 movq 40(%rdi),%r14 /* restore r14 */
724 movq 48(%rdi),%r15 /* restore r15 */
725 movq 56(%rdi),%rdx /* get rta */
726 movq %rdx,0(%rsp) /* put in return frame */
727 xorl %eax,%eax /* return(1); */
728 incl %eax
729 ret
730END(longjmp)
731
732/*
733 * Support for reading MSRs in the safe manner.
734 */
735ENTRY(rdmsr_safe)
736/* int rdmsr_safe(u_int msr, uint64_t *data) */
737 PUSH_FRAME_POINTER
738 movq PCPU(CURPCB),%r8
739 movq $msr_onfault,PCB_ONFAULT(%r8)
740 movl %edi,%ecx
741 rdmsr /* Read MSR pointed by %ecx. Returns
742 hi byte in edx, lo in %eax */
743 salq $32,%rdx /* sign-shift %rdx left */
744 movl %eax,%eax /* zero-extend %eax -> %rax */
745 orq %rdx,%rax
746 movq %rax,(%rsi)
747 xorq %rax,%rax
748 movq %rax,PCB_ONFAULT(%r8)
749 POP_FRAME_POINTER
750 ret
751
752/*
753 * Support for writing MSRs in the safe manner.
754 */
755ENTRY(wrmsr_safe)
756/* int wrmsr_safe(u_int msr, uint64_t data) */
757 PUSH_FRAME_POINTER
758 movq PCPU(CURPCB),%r8
759 movq $msr_onfault,PCB_ONFAULT(%r8)
760 movl %edi,%ecx
761 movl %esi,%eax
762 sarq $32,%rsi
763 movl %esi,%edx
764 wrmsr /* Write MSR pointed by %ecx. Accepts
765 hi byte in edx, lo in %eax. */
766 xorq %rax,%rax
767 movq %rax,PCB_ONFAULT(%r8)
768 POP_FRAME_POINTER
769 ret
770
771/*
772 * MSR operations fault handler
773 */
774 ALIGN_TEXT
775msr_onfault:
776 movq $0,PCB_ONFAULT(%r8)
777 movl $EFAULT,%eax
778 POP_FRAME_POINTER
779 ret
780
781/*
782 * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3);
783 * Invalidates address space addressed by ucr3, then returns to kcr3.
784 * Done in assembler to ensure no other memory accesses happen while
785 * on ucr3.
786 */
787 ALIGN_TEXT
788ENTRY(pmap_pti_pcid_invalidate)
789 pushfq
790 cli
791 movq %rdi,%cr3 /* to user page table */
792 movq %rsi,%cr3 /* back to kernel */
793 popfq
794 retq
795
796/*
797 * void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va);
798 * Invalidates virtual address va in address space ucr3, then returns to kcr3.
799 */
800 ALIGN_TEXT
801ENTRY(pmap_pti_pcid_invlpg)
802 pushfq
803 cli
804 movq %rdi,%cr3 /* to user page table */
805 invlpg (%rdx)
806 movq %rsi,%cr3 /* back to kernel */
807 popfq
808 retq
809
810/*
811 * void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva,
812 * vm_offset_t eva);
813 * Invalidates virtual addresses between sva and eva in address space ucr3,
814 * then returns to kcr3.
815 */
816 ALIGN_TEXT
817ENTRY(pmap_pti_pcid_invlrng)
818 pushfq
819 cli
820 movq %rdi,%cr3 /* to user page table */
8211: invlpg (%rdx)
822 addq $PAGE_SIZE,%rdx
823 cmpq %rdx,%rcx
824 ja 1b
825 movq %rsi,%cr3 /* back to kernel */
826 popfq
827 retq
828
829 .altmacro
830 .macro ibrs_seq_label l
831handle_ibrs_\l:
832 .endm
833 .macro ibrs_call_label l
834 call handle_ibrs_\l
835 .endm
836 .macro ibrs_seq count
837 ll=1
838 .rept \count
839 ibrs_call_label %(ll)
840 nop
841 ibrs_seq_label %(ll)
842 addq $8,%rsp
843 ll=ll+1
844 .endr
845 .endm
846
847/* all callers already saved %rax, %rdx, and %rcx */
848ENTRY(handle_ibrs_entry)
849 cmpb $0,hw_ibrs_active(%rip)
850 je 1f
851 movl $MSR_IA32_SPEC_CTRL,%ecx
852 rdmsr
853 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
854 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx
855 wrmsr
856 movb $1,PCPU(IBPB_SET)
857 testl $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip)
858 jne 1f
859 ibrs_seq 32
8601: ret
861END(handle_ibrs_entry)
862
863ENTRY(handle_ibrs_exit)
864 cmpb $0,PCPU(IBPB_SET)
865 je 1f
866 movl $MSR_IA32_SPEC_CTRL,%ecx
867 rdmsr
868 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
869 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
870 wrmsr
871 movb $0,PCPU(IBPB_SET)
8721: ret
873END(handle_ibrs_exit)
874
875/* registers-neutral version, but needs stack */
876ENTRY(handle_ibrs_exit_rs)
877 cmpb $0,PCPU(IBPB_SET)
878 je 1f
879 pushq %rax
880 pushq %rdx
881 pushq %rcx
882 movl $MSR_IA32_SPEC_CTRL,%ecx
883 rdmsr
884 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
885 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
886 wrmsr
887 popq %rcx
888 popq %rdx
889 popq %rax
890 movb $0,PCPU(IBPB_SET)
8911: ret
892END(handle_ibrs_exit_rs)
893
894 .noaltmacro
895
896/*
897 * Flush L1D cache. Load enough of the data from the kernel text
898 * to flush existing L1D content.
899 *
900 * N.B. The function does not follow ABI calling conventions, it corrupts %rbx.
901 * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags
902 * registers are clobbered. The NMI handler caller only needs %r13 preserved.
903 */
904ENTRY(flush_l1d_sw)
905#define L1D_FLUSH_SIZE (64 * 1024)
906 movq $KERNBASE, %r9
907 movq $-L1D_FLUSH_SIZE, %rcx
908 /*
909 * pass 1: Preload TLB.
910 * Kernel text is mapped using superpages. TLB preload is
911 * done for the benefit of older CPUs which split 2M page
912 * into 4k TLB entries.
913 */
9141: movb L1D_FLUSH_SIZE(%r9, %rcx), %al
915 addq $PAGE_SIZE, %rcx
916 jne 1b
917 xorl %eax, %eax
918 cpuid
919 movq $-L1D_FLUSH_SIZE, %rcx
920 /* pass 2: Read each cache line. */
9212: movb L1D_FLUSH_SIZE(%r9, %rcx), %al
922 addq $64, %rcx
923 jne 2b
924 lfence
925 ret
926#undef L1D_FLUSH_SIZE
927END(flush_l1d_sw)
| 36 */
37
38#include "opt_ddb.h"
39
40#include <machine/asmacros.h>
41#include <machine/specialreg.h>
42#include <machine/pmap.h>
43
44#include "assym.s"
45
46 .text
47
48/*
49 * bcopy family
50 * void bzero(void *buf, u_int len)
51 */
52
53/* done */
54ENTRY(bzero)
55 PUSH_FRAME_POINTER
56 movq %rsi,%rcx
57 xorl %eax,%eax
58 shrq $3,%rcx
59 rep
60 stosq
61 movq %rsi,%rcx
62 andq $7,%rcx
63 rep
64 stosb
65 POP_FRAME_POINTER
66 ret
67END(bzero)
68
69/* Address: %rdi */
70ENTRY(pagezero)
71 PUSH_FRAME_POINTER
72 movq $PAGE_SIZE/8,%rcx
73 xorl %eax,%eax
74 rep
75 stosq
76 POP_FRAME_POINTER
77 ret
78END(pagezero)
79
80ENTRY(bcmp)
81 PUSH_FRAME_POINTER
82 movq %rdx,%rcx
83 shrq $3,%rcx
84 repe
85 cmpsq
86 jne 1f
87
88 movq %rdx,%rcx
89 andq $7,%rcx
90 repe
91 cmpsb
921:
93 setne %al
94 movsbl %al,%eax
95 POP_FRAME_POINTER
96 ret
97END(bcmp)
98
99/*
100 * bcopy(src, dst, cnt)
101 * rdi, rsi, rdx
102 * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800
103 */
104ENTRY(bcopy)
105 PUSH_FRAME_POINTER
106 xchgq %rsi,%rdi
107 movq %rdx,%rcx
108
109 movq %rdi,%rax
110 subq %rsi,%rax
111 cmpq %rcx,%rax /* overlapping && src < dst? */
112 jb 1f
113
114 shrq $3,%rcx /* copy by 64-bit words */
115 rep
116 movsq
117 movq %rdx,%rcx
118 andq $7,%rcx /* any bytes left? */
119 rep
120 movsb
121 POP_FRAME_POINTER
122 ret
123
124 /* ALIGN_TEXT */
1251:
126 addq %rcx,%rdi /* copy backwards */
127 addq %rcx,%rsi
128 decq %rdi
129 decq %rsi
130 andq $7,%rcx /* any fractional bytes? */
131 std
132 rep
133 movsb
134 movq %rdx,%rcx /* copy remainder by 32-bit words */
135 shrq $3,%rcx
136 subq $7,%rsi
137 subq $7,%rdi
138 rep
139 movsq
140 cld
141 POP_FRAME_POINTER
142 ret
143END(bcopy)
144
145/*
146 * Note: memcpy does not support overlapping copies
147 */
148ENTRY(memcpy)
149 PUSH_FRAME_POINTER
150 movq %rdi,%rax
151 movq %rdx,%rcx
152 shrq $3,%rcx /* copy by 64-bit words */
153 rep
154 movsq
155 movq %rdx,%rcx
156 andq $7,%rcx /* any bytes left? */
157 rep
158 movsb
159 POP_FRAME_POINTER
160 ret
161END(memcpy)
162
163/*
164 * pagecopy(%rdi=from, %rsi=to)
165 */
166ENTRY(pagecopy)
167 PUSH_FRAME_POINTER
168 movq $-PAGE_SIZE,%rax
169 movq %rax,%rdx
170 subq %rax,%rdi
171 subq %rax,%rsi
1721:
173 prefetchnta (%rdi,%rax)
174 addq $64,%rax
175 jne 1b
1762:
177 movq (%rdi,%rdx),%rax
178 movnti %rax,(%rsi,%rdx)
179 movq 8(%rdi,%rdx),%rax
180 movnti %rax,8(%rsi,%rdx)
181 movq 16(%rdi,%rdx),%rax
182 movnti %rax,16(%rsi,%rdx)
183 movq 24(%rdi,%rdx),%rax
184 movnti %rax,24(%rsi,%rdx)
185 addq $32,%rdx
186 jne 2b
187 sfence
188 POP_FRAME_POINTER
189 ret
190END(pagecopy)
191
192/* fillw(pat, base, cnt) */
193/* %rdi,%rsi, %rdx */
194ENTRY(fillw)
195 PUSH_FRAME_POINTER
196 movq %rdi,%rax
197 movq %rsi,%rdi
198 movq %rdx,%rcx
199 rep
200 stosw
201 POP_FRAME_POINTER
202 ret
203END(fillw)
204
205/*****************************************************************************/
206/* copyout and fubyte family */
207/*****************************************************************************/
208/*
209 * Access user memory from inside the kernel. These routines should be
210 * the only places that do this.
211 *
212 * These routines set curpcb->pcb_onfault for the time they execute. When a
213 * protection violation occurs inside the functions, the trap handler
214 * returns to *curpcb->pcb_onfault instead of the function.
215 */
216
217/*
218 * copyout(from_kernel, to_user, len)
219 * %rdi, %rsi, %rdx
220 */
221ENTRY(copyout)
222 PUSH_FRAME_POINTER
223 movq PCPU(CURPCB),%rax
224 movq $copyout_fault,PCB_ONFAULT(%rax)
225 testq %rdx,%rdx /* anything to do? */
226 jz done_copyout
227
228 /*
229 * Check explicitly for non-user addresses. This check is essential
230 * because it prevents usermode from writing into the kernel. We do
231 * not verify anywhere else that the user did not specify a rogue
232 * address.
233 */
234 /*
235 * First, prevent address wrapping.
236 */
237 movq %rsi,%rax
238 addq %rdx,%rax
239 jc copyout_fault
240/*
241 * XXX STOP USING VM_MAXUSER_ADDRESS.
242 * It is an end address, not a max, so every time it is used correctly it
243 * looks like there is an off by one error, and of course it caused an off
244 * by one error in several places.
245 */
246 movq $VM_MAXUSER_ADDRESS,%rcx
247 cmpq %rcx,%rax
248 ja copyout_fault
249
250 xchgq %rdi,%rsi
251 /* bcopy(%rsi, %rdi, %rdx) */
252 movq %rdx,%rcx
253
254 shrq $3,%rcx
255 rep
256 movsq
257 movb %dl,%cl
258 andb $7,%cl
259 rep
260 movsb
261
262done_copyout:
263 xorl %eax,%eax
264 movq PCPU(CURPCB),%rdx
265 movq %rax,PCB_ONFAULT(%rdx)
266 POP_FRAME_POINTER
267 ret
268
269 ALIGN_TEXT
270copyout_fault:
271 movq PCPU(CURPCB),%rdx
272 movq $0,PCB_ONFAULT(%rdx)
273 movq $EFAULT,%rax
274 POP_FRAME_POINTER
275 ret
276END(copyout)
277
278/*
279 * copyin(from_user, to_kernel, len)
280 * %rdi, %rsi, %rdx
281 */
282ENTRY(copyin)
283 PUSH_FRAME_POINTER
284 movq PCPU(CURPCB),%rax
285 movq $copyin_fault,PCB_ONFAULT(%rax)
286 testq %rdx,%rdx /* anything to do? */
287 jz done_copyin
288
289 /*
290 * make sure address is valid
291 */
292 movq %rdi,%rax
293 addq %rdx,%rax
294 jc copyin_fault
295 movq $VM_MAXUSER_ADDRESS,%rcx
296 cmpq %rcx,%rax
297 ja copyin_fault
298
299 xchgq %rdi,%rsi
300 movq %rdx,%rcx
301 movb %cl,%al
302 shrq $3,%rcx /* copy longword-wise */
303 rep
304 movsq
305 movb %al,%cl
306 andb $7,%cl /* copy remaining bytes */
307 rep
308 movsb
309
310done_copyin:
311 xorl %eax,%eax
312 movq PCPU(CURPCB),%rdx
313 movq %rax,PCB_ONFAULT(%rdx)
314 POP_FRAME_POINTER
315 ret
316
317 ALIGN_TEXT
318copyin_fault:
319 movq PCPU(CURPCB),%rdx
320 movq $0,PCB_ONFAULT(%rdx)
321 movq $EFAULT,%rax
322 POP_FRAME_POINTER
323 ret
324END(copyin)
325
326/*
327 * casueword32. Compare and set user integer. Returns -1 on fault,
328 * 0 if access was successful. Old value is written to *oldp.
329 * dst = %rdi, old = %esi, oldp = %rdx, new = %ecx
330 */
331ENTRY(casueword32)
332 PUSH_FRAME_POINTER
333 movq PCPU(CURPCB),%r8
334 movq $fusufault,PCB_ONFAULT(%r8)
335
336 movq $VM_MAXUSER_ADDRESS-4,%rax
337 cmpq %rax,%rdi /* verify address is valid */
338 ja fusufault
339
340 movl %esi,%eax /* old */
341#ifdef SMP
342 lock
343#endif
344 cmpxchgl %ecx,(%rdi) /* new = %ecx */
345
346 /*
347 * The old value is in %eax. If the store succeeded it will be the
348 * value we expected (old) from before the store, otherwise it will
349 * be the current value. Save %eax into %esi to prepare the return
350 * value.
351 */
352 movl %eax,%esi
353 xorl %eax,%eax
354 movq %rax,PCB_ONFAULT(%r8)
355
356 /*
357 * Access the oldp after the pcb_onfault is cleared, to correctly
358 * catch corrupted pointer.
359 */
360 movl %esi,(%rdx) /* oldp = %rdx */
361 POP_FRAME_POINTER
362 ret
363END(casueword32)
364
365/*
366 * casueword. Compare and set user long. Returns -1 on fault,
367 * 0 if access was successful. Old value is written to *oldp.
368 * dst = %rdi, old = %rsi, oldp = %rdx, new = %rcx
369 */
370ENTRY(casueword)
371 PUSH_FRAME_POINTER
372 movq PCPU(CURPCB),%r8
373 movq $fusufault,PCB_ONFAULT(%r8)
374
375 movq $VM_MAXUSER_ADDRESS-4,%rax
376 cmpq %rax,%rdi /* verify address is valid */
377 ja fusufault
378
379 movq %rsi,%rax /* old */
380#ifdef SMP
381 lock
382#endif
383 cmpxchgq %rcx,(%rdi) /* new = %rcx */
384
385 /*
386 * The old value is in %rax. If the store succeeded it will be the
387 * value we expected (old) from before the store, otherwise it will
388 * be the current value.
389 */
390 movq %rax,%rsi
391 xorl %eax,%eax
392 movq %rax,PCB_ONFAULT(%r8)
393 movq %rsi,(%rdx)
394 POP_FRAME_POINTER
395 ret
396END(casueword)
397
398/*
399 * Fetch (load) a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit
400 * byte from user memory.
401 * addr = %rdi, valp = %rsi
402 */
403
404ALTENTRY(fueword64)
405ENTRY(fueword)
406 PUSH_FRAME_POINTER
407 movq PCPU(CURPCB),%rcx
408 movq $fusufault,PCB_ONFAULT(%rcx)
409
410 movq $VM_MAXUSER_ADDRESS-8,%rax
411 cmpq %rax,%rdi /* verify address is valid */
412 ja fusufault
413
414 xorl %eax,%eax
415 movq (%rdi),%r11
416 movq %rax,PCB_ONFAULT(%rcx)
417 movq %r11,(%rsi)
418 POP_FRAME_POINTER
419 ret
420END(fueword64)
421END(fueword)
422
423ENTRY(fueword32)
424 PUSH_FRAME_POINTER
425 movq PCPU(CURPCB),%rcx
426 movq $fusufault,PCB_ONFAULT(%rcx)
427
428 movq $VM_MAXUSER_ADDRESS-4,%rax
429 cmpq %rax,%rdi /* verify address is valid */
430 ja fusufault
431
432 xorl %eax,%eax
433 movl (%rdi),%r11d
434 movq %rax,PCB_ONFAULT(%rcx)
435 movl %r11d,(%rsi)
436 POP_FRAME_POINTER
437 ret
438END(fueword32)
439
440/*
441 * fuswintr() and suswintr() are specialized variants of fuword16() and
442 * suword16(), respectively. They are called from the profiling code,
443 * potentially at interrupt time. If they fail, that's okay; good things
444 * will happen later. They always fail for now, until the trap code is
445 * able to deal with this.
446 */
447ALTENTRY(suswintr)
448ENTRY(fuswintr)
449 movq $-1,%rax
450 ret
451END(suswintr)
452END(fuswintr)
453
454ENTRY(fuword16)
455 PUSH_FRAME_POINTER
456 movq PCPU(CURPCB),%rcx
457 movq $fusufault,PCB_ONFAULT(%rcx)
458
459 movq $VM_MAXUSER_ADDRESS-2,%rax
460 cmpq %rax,%rdi
461 ja fusufault
462
463 movzwl (%rdi),%eax
464 movq $0,PCB_ONFAULT(%rcx)
465 POP_FRAME_POINTER
466 ret
467END(fuword16)
468
469ENTRY(fubyte)
470 PUSH_FRAME_POINTER
471 movq PCPU(CURPCB),%rcx
472 movq $fusufault,PCB_ONFAULT(%rcx)
473
474 movq $VM_MAXUSER_ADDRESS-1,%rax
475 cmpq %rax,%rdi
476 ja fusufault
477
478 movzbl (%rdi),%eax
479 movq $0,PCB_ONFAULT(%rcx)
480 POP_FRAME_POINTER
481 ret
482END(fubyte)
483
484 ALIGN_TEXT
485fusufault:
486 movq PCPU(CURPCB),%rcx
487 xorl %eax,%eax
488 movq %rax,PCB_ONFAULT(%rcx)
489 decq %rax
490 POP_FRAME_POINTER
491 ret
492
493/*
494 * Store a 64-bit word, a 32-bit word, a 16-bit word, or an 8-bit byte to
495 * user memory.
496 * addr = %rdi, value = %rsi
497 */
498ALTENTRY(suword64)
499ENTRY(suword)
500 PUSH_FRAME_POINTER
501 movq PCPU(CURPCB),%rcx
502 movq $fusufault,PCB_ONFAULT(%rcx)
503
504 movq $VM_MAXUSER_ADDRESS-8,%rax
505 cmpq %rax,%rdi /* verify address validity */
506 ja fusufault
507
508 movq %rsi,(%rdi)
509 xorl %eax,%eax
510 movq PCPU(CURPCB),%rcx
511 movq %rax,PCB_ONFAULT(%rcx)
512 POP_FRAME_POINTER
513 ret
514END(suword64)
515END(suword)
516
517ENTRY(suword32)
518 PUSH_FRAME_POINTER
519 movq PCPU(CURPCB),%rcx
520 movq $fusufault,PCB_ONFAULT(%rcx)
521
522 movq $VM_MAXUSER_ADDRESS-4,%rax
523 cmpq %rax,%rdi /* verify address validity */
524 ja fusufault
525
526 movl %esi,(%rdi)
527 xorl %eax,%eax
528 movq PCPU(CURPCB),%rcx
529 movq %rax,PCB_ONFAULT(%rcx)
530 POP_FRAME_POINTER
531 ret
532END(suword32)
533
534ENTRY(suword16)
535 PUSH_FRAME_POINTER
536 movq PCPU(CURPCB),%rcx
537 movq $fusufault,PCB_ONFAULT(%rcx)
538
539 movq $VM_MAXUSER_ADDRESS-2,%rax
540 cmpq %rax,%rdi /* verify address validity */
541 ja fusufault
542
543 movw %si,(%rdi)
544 xorl %eax,%eax
545 movq PCPU(CURPCB),%rcx /* restore trashed register */
546 movq %rax,PCB_ONFAULT(%rcx)
547 POP_FRAME_POINTER
548 ret
549END(suword16)
550
551ENTRY(subyte)
552 PUSH_FRAME_POINTER
553 movq PCPU(CURPCB),%rcx
554 movq $fusufault,PCB_ONFAULT(%rcx)
555
556 movq $VM_MAXUSER_ADDRESS-1,%rax
557 cmpq %rax,%rdi /* verify address validity */
558 ja fusufault
559
560 movl %esi,%eax
561 movb %al,(%rdi)
562 xorl %eax,%eax
563 movq PCPU(CURPCB),%rcx /* restore trashed register */
564 movq %rax,PCB_ONFAULT(%rcx)
565 POP_FRAME_POINTER
566 ret
567END(subyte)
568
569/*
570 * copyinstr(from, to, maxlen, int *lencopied)
571 * %rdi, %rsi, %rdx, %rcx
572 *
573 * copy a string from 'from' to 'to', stop when a 0 character is reached.
574 * return ENAMETOOLONG if string is longer than maxlen, and
575 * EFAULT on protection violations. If lencopied is non-zero,
576 * return the actual length in *lencopied.
577 */
578ENTRY(copyinstr)
579 PUSH_FRAME_POINTER
580 movq %rdx,%r8 /* %r8 = maxlen */
581 movq %rcx,%r9 /* %r9 = *len */
582 xchgq %rdi,%rsi /* %rdi = from, %rsi = to */
583 movq PCPU(CURPCB),%rcx
584 movq $cpystrflt,PCB_ONFAULT(%rcx)
585
586 movq $VM_MAXUSER_ADDRESS,%rax
587
588 /* make sure 'from' is within bounds */
589 subq %rsi,%rax
590 jbe cpystrflt
591
592 /* restrict maxlen to <= VM_MAXUSER_ADDRESS-from */
593 cmpq %rdx,%rax
594 jae 1f
595 movq %rax,%rdx
596 movq %rax,%r8
5971:
598 incq %rdx
599
6002:
601 decq %rdx
602 jz 3f
603
604 lodsb
605 stosb
606 orb %al,%al
607 jnz 2b
608
609 /* Success -- 0 byte reached */
610 decq %rdx
611 xorl %eax,%eax
612 jmp cpystrflt_x
6133:
614 /* rdx is zero - return ENAMETOOLONG or EFAULT */
615 movq $VM_MAXUSER_ADDRESS,%rax
616 cmpq %rax,%rsi
617 jae cpystrflt
6184:
619 movq $ENAMETOOLONG,%rax
620 jmp cpystrflt_x
621
622cpystrflt:
623 movq $EFAULT,%rax
624
625cpystrflt_x:
626 /* set *lencopied and return %eax */
627 movq PCPU(CURPCB),%rcx
628 movq $0,PCB_ONFAULT(%rcx)
629
630 testq %r9,%r9
631 jz 1f
632 subq %rdx,%r8
633 movq %r8,(%r9)
6341:
635 POP_FRAME_POINTER
636 ret
637END(copyinstr)
638
639/*
640 * copystr(from, to, maxlen, int *lencopied)
641 * %rdi, %rsi, %rdx, %rcx
642 */
643ENTRY(copystr)
644 PUSH_FRAME_POINTER
645 movq %rdx,%r8 /* %r8 = maxlen */
646
647 xchgq %rdi,%rsi
648 incq %rdx
6491:
650 decq %rdx
651 jz 4f
652 lodsb
653 stosb
654 orb %al,%al
655 jnz 1b
656
657 /* Success -- 0 byte reached */
658 decq %rdx
659 xorl %eax,%eax
660 jmp 6f
6614:
662 /* rdx is zero -- return ENAMETOOLONG */
663 movq $ENAMETOOLONG,%rax
664
6656:
666
667 testq %rcx,%rcx
668 jz 7f
669 /* set *lencopied and return %rax */
670 subq %rdx,%r8
671 movq %r8,(%rcx)
6727:
673 POP_FRAME_POINTER
674 ret
675END(copystr)
676
677/*
678 * Handling of special amd64 registers and descriptor tables etc
679 */
680/* void lgdt(struct region_descriptor *rdp); */
681ENTRY(lgdt)
682 /* reload the descriptor table */
683 lgdt (%rdi)
684
685 /* flush the prefetch q */
686 jmp 1f
687 nop
6881:
689 movl $KDSEL,%eax
690 movl %eax,%ds
691 movl %eax,%es
692 movl %eax,%fs /* Beware, use wrmsr to set 64 bit base */
693 movl %eax,%gs
694 movl %eax,%ss
695
696 /* reload code selector by turning return into intersegmental return */
697 popq %rax
698 pushq $KCSEL
699 pushq %rax
700 MEXITCOUNT
701 lretq
702END(lgdt)
703
704/*****************************************************************************/
705/* setjump, longjump */
706/*****************************************************************************/
707
708ENTRY(setjmp)
709 movq %rbx,0(%rdi) /* save rbx */
710 movq %rsp,8(%rdi) /* save rsp */
711 movq %rbp,16(%rdi) /* save rbp */
712 movq %r12,24(%rdi) /* save r12 */
713 movq %r13,32(%rdi) /* save r13 */
714 movq %r14,40(%rdi) /* save r14 */
715 movq %r15,48(%rdi) /* save r15 */
716 movq 0(%rsp),%rdx /* get rta */
717 movq %rdx,56(%rdi) /* save rip */
718 xorl %eax,%eax /* return(0); */
719 ret
720END(setjmp)
721
722ENTRY(longjmp)
723 movq 0(%rdi),%rbx /* restore rbx */
724 movq 8(%rdi),%rsp /* restore rsp */
725 movq 16(%rdi),%rbp /* restore rbp */
726 movq 24(%rdi),%r12 /* restore r12 */
727 movq 32(%rdi),%r13 /* restore r13 */
728 movq 40(%rdi),%r14 /* restore r14 */
729 movq 48(%rdi),%r15 /* restore r15 */
730 movq 56(%rdi),%rdx /* get rta */
731 movq %rdx,0(%rsp) /* put in return frame */
732 xorl %eax,%eax /* return(1); */
733 incl %eax
734 ret
735END(longjmp)
736
737/*
738 * Support for reading MSRs in the safe manner.
739 */
740ENTRY(rdmsr_safe)
741/* int rdmsr_safe(u_int msr, uint64_t *data) */
742 PUSH_FRAME_POINTER
743 movq PCPU(CURPCB),%r8
744 movq $msr_onfault,PCB_ONFAULT(%r8)
745 movl %edi,%ecx
746 rdmsr /* Read MSR pointed by %ecx. Returns
747 hi byte in edx, lo in %eax */
748 salq $32,%rdx /* sign-shift %rdx left */
749 movl %eax,%eax /* zero-extend %eax -> %rax */
750 orq %rdx,%rax
751 movq %rax,(%rsi)
752 xorq %rax,%rax
753 movq %rax,PCB_ONFAULT(%r8)
754 POP_FRAME_POINTER
755 ret
756
757/*
758 * Support for writing MSRs in the safe manner.
759 */
760ENTRY(wrmsr_safe)
761/* int wrmsr_safe(u_int msr, uint64_t data) */
762 PUSH_FRAME_POINTER
763 movq PCPU(CURPCB),%r8
764 movq $msr_onfault,PCB_ONFAULT(%r8)
765 movl %edi,%ecx
766 movl %esi,%eax
767 sarq $32,%rsi
768 movl %esi,%edx
769 wrmsr /* Write MSR pointed by %ecx. Accepts
770 hi byte in edx, lo in %eax. */
771 xorq %rax,%rax
772 movq %rax,PCB_ONFAULT(%r8)
773 POP_FRAME_POINTER
774 ret
775
776/*
777 * MSR operations fault handler
778 */
779 ALIGN_TEXT
780msr_onfault:
781 movq $0,PCB_ONFAULT(%r8)
782 movl $EFAULT,%eax
783 POP_FRAME_POINTER
784 ret
785
786/*
787 * void pmap_pti_pcid_invalidate(uint64_t ucr3, uint64_t kcr3);
788 * Invalidates address space addressed by ucr3, then returns to kcr3.
789 * Done in assembler to ensure no other memory accesses happen while
790 * on ucr3.
791 */
792 ALIGN_TEXT
793ENTRY(pmap_pti_pcid_invalidate)
794 pushfq
795 cli
796 movq %rdi,%cr3 /* to user page table */
797 movq %rsi,%cr3 /* back to kernel */
798 popfq
799 retq
800
801/*
802 * void pmap_pti_pcid_invlpg(uint64_t ucr3, uint64_t kcr3, vm_offset_t va);
803 * Invalidates virtual address va in address space ucr3, then returns to kcr3.
804 */
805 ALIGN_TEXT
806ENTRY(pmap_pti_pcid_invlpg)
807 pushfq
808 cli
809 movq %rdi,%cr3 /* to user page table */
810 invlpg (%rdx)
811 movq %rsi,%cr3 /* back to kernel */
812 popfq
813 retq
814
815/*
816 * void pmap_pti_pcid_invlrng(uint64_t ucr3, uint64_t kcr3, vm_offset_t sva,
817 * vm_offset_t eva);
818 * Invalidates virtual addresses between sva and eva in address space ucr3,
819 * then returns to kcr3.
820 */
821 ALIGN_TEXT
822ENTRY(pmap_pti_pcid_invlrng)
823 pushfq
824 cli
825 movq %rdi,%cr3 /* to user page table */
8261: invlpg (%rdx)
827 addq $PAGE_SIZE,%rdx
828 cmpq %rdx,%rcx
829 ja 1b
830 movq %rsi,%cr3 /* back to kernel */
831 popfq
832 retq
833
834 .altmacro
835 .macro ibrs_seq_label l
836handle_ibrs_\l:
837 .endm
838 .macro ibrs_call_label l
839 call handle_ibrs_\l
840 .endm
841 .macro ibrs_seq count
842 ll=1
843 .rept \count
844 ibrs_call_label %(ll)
845 nop
846 ibrs_seq_label %(ll)
847 addq $8,%rsp
848 ll=ll+1
849 .endr
850 .endm
851
852/* all callers already saved %rax, %rdx, and %rcx */
853ENTRY(handle_ibrs_entry)
854 cmpb $0,hw_ibrs_active(%rip)
855 je 1f
856 movl $MSR_IA32_SPEC_CTRL,%ecx
857 rdmsr
858 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
859 orl $(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32,%edx
860 wrmsr
861 movb $1,PCPU(IBPB_SET)
862 testl $CPUID_STDEXT_SMEP,cpu_stdext_feature(%rip)
863 jne 1f
864 ibrs_seq 32
8651: ret
866END(handle_ibrs_entry)
867
868ENTRY(handle_ibrs_exit)
869 cmpb $0,PCPU(IBPB_SET)
870 je 1f
871 movl $MSR_IA32_SPEC_CTRL,%ecx
872 rdmsr
873 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
874 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
875 wrmsr
876 movb $0,PCPU(IBPB_SET)
8771: ret
878END(handle_ibrs_exit)
879
880/* registers-neutral version, but needs stack */
881ENTRY(handle_ibrs_exit_rs)
882 cmpb $0,PCPU(IBPB_SET)
883 je 1f
884 pushq %rax
885 pushq %rdx
886 pushq %rcx
887 movl $MSR_IA32_SPEC_CTRL,%ecx
888 rdmsr
889 andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax
890 andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx
891 wrmsr
892 popq %rcx
893 popq %rdx
894 popq %rax
895 movb $0,PCPU(IBPB_SET)
8961: ret
897END(handle_ibrs_exit_rs)
898
899 .noaltmacro
900
901/*
902 * Flush L1D cache. Load enough of the data from the kernel text
903 * to flush existing L1D content.
904 *
905 * N.B. The function does not follow ABI calling conventions, it corrupts %rbx.
906 * The vmm.ko caller expects that only %rax, %rdx, %rbx, %rcx, %r9, and %rflags
907 * registers are clobbered. The NMI handler caller only needs %r13 preserved.
908 */
909ENTRY(flush_l1d_sw)
910#define L1D_FLUSH_SIZE (64 * 1024)
911 movq $KERNBASE, %r9
912 movq $-L1D_FLUSH_SIZE, %rcx
913 /*
914 * pass 1: Preload TLB.
915 * Kernel text is mapped using superpages. TLB preload is
916 * done for the benefit of older CPUs which split 2M page
917 * into 4k TLB entries.
918 */
9191: movb L1D_FLUSH_SIZE(%r9, %rcx), %al
920 addq $PAGE_SIZE, %rcx
921 jne 1b
922 xorl %eax, %eax
923 cpuid
924 movq $-L1D_FLUSH_SIZE, %rcx
925 /* pass 2: Read each cache line. */
9262: movb L1D_FLUSH_SIZE(%r9, %rcx), %al
927 addq $64, %rcx
928 jne 2b
929 lfence
930 ret
931#undef L1D_FLUSH_SIZE
932END(flush_l1d_sw)
|