1/*	$NetBSD: cpufunc.S,v 1.20 2011/09/24 10:32:52 jym Exp $	*/
2
3/*-
4 * Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Charles M. Hannum, and by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * Functions to provide access to i386-specific instructions.
34 */
35
36#include <sys/errno.h>
37
38#include <machine/asm.h>
39#include <machine/frameasm.h>
40#include <machine/specialreg.h>
41#include <machine/segments.h>
42
43#include "opt_xen.h"
44
45#include "assym.h"
46
47/* Small and slow, so align less. */
48#undef _ALIGN_TEXT
49#define	_ALIGN_TEXT	.align 8
50
51ENTRY(x86_lfence)
52	lfence
53	ret
54
55ENTRY(x86_sfence)
56	sfence
57	ret
58
59ENTRY(x86_mfence)
60	mfence
61	ret
62
63#ifndef XEN
64ENTRY(invlpg)
65	invlpg	(%rdi)
66	ret
67
68ENTRY(lidt)
69	lidt	(%rdi)
70	ret
71
72ENTRY(lldt)
73	cmpl	%edi, CPUVAR(CURLDT)
74	jne	1f
75	ret
761:
77	movl	%edi, CPUVAR(CURLDT)
78	lldt	%di
79	ret
80
81ENTRY(ltr)
82	ltr	%di
83	ret
84
85ENTRY(lcr0)
86	movq	%rdi, %cr0
87	ret
88
89ENTRY(rcr0)
90	movq	%cr0, %rax
91	ret
92
93ENTRY(lcr2)
94	movq	%rdi, %cr2
95	ret
96
97ENTRY(rcr2)
98	movq	%cr2, %rax
99	ret
100
101ENTRY(lcr3)
102	movq	%rdi, %cr3
103	ret
104
105ENTRY(rcr3)
106	movq	%cr3, %rax
107	ret
108#endif
109
110ENTRY(lcr4)
111	movq	%rdi, %cr4
112	ret
113
114ENTRY(rcr4)
115	movq	%cr4, %rax
116	ret
117
118ENTRY(lcr8)
119	movq	%rdi, %cr8
120	ret
121
122ENTRY(rcr8)
123	movq	%cr8, %rax
124	ret
125
126/*
127 * Big hammer: flush all TLB entries, including ones from PTE's
128 * with the G bit set.  This should only be necessary if TLB
129 * shootdown falls far behind.
130 *
131 * Intel Architecture Software Developer's Manual, Volume 3,
132 *	System Programming, section 9.10, "Invalidating the
133 * Translation Lookaside Buffers (TLBS)":
134 * "The following operations invalidate all TLB entries, irrespective
135 * of the setting of the G flag:
136 * ...
137 * "(P6 family processors only): Writing to control register CR4 to
138 * modify the PSE, PGE, or PAE flag."
139 *
140 * (the alternatives not quoted above are not an option here.)
141 *
142 * If PGE is not in use, we reload CR3.
143 */
144#ifndef XEN
145ENTRY(tlbflushg)
146	movq	%cr4, %rax
147	testq	$CR4_PGE, %rax
148	jz	1f
149	movq	%rax, %rdx
150	andq	$~CR4_PGE, %rdx
151	movq	%rdx, %cr4
152	movq	%rax, %cr4
153	ret
154
155ENTRY(tlbflush)
1561:
157	movq	%cr3, %rax
158	movq	%rax, %cr3
159	ret
160
161ENTRY(ldr6)
162	movq	%rdi, %dr6
163	ret
164
165ENTRY(rdr6)
166	movq	%dr6, %rdi
167	ret
168
169ENTRY(x86_disable_intr)
170	cli
171	ret
172
173ENTRY(x86_enable_intr)
174	sti
175	ret
176
177ENTRY(x86_read_flags)
178	pushfq
179	popq	%rax
180	ret
181
182STRONG_ALIAS(x86_read_psl,x86_read_flags)
183
184ENTRY(x86_write_flags)
185	pushq	%rdi
186	popfq
187	ret
188
189STRONG_ALIAS(x86_write_psl,x86_write_flags)
190#endif /* XEN */
191
192ENTRY(rdmsr)
193	movq	%rdi, %rcx
194	xorq	%rax, %rax
195	rdmsr
196	shlq	$32, %rdx
197	orq	%rdx, %rax
198	ret
199
200ENTRY(wrmsr)
201	movq	%rdi, %rcx
202	movq	%rsi, %rax
203	movq	%rsi, %rdx
204	shrq	$32, %rdx
205	wrmsr
206	ret
207
208ENTRY(rdmsr_locked)
209	movq	%rdi, %rcx
210	xorq	%rax, %rax
211	movl	$OPTERON_MSR_PASSCODE, %edi
212	rdmsr
213	shlq	$32, %rdx
214	orq	%rdx, %rax
215	ret
216
217ENTRY(wrmsr_locked)
218	movq	%rdi, %rcx
219	movq	%rsi, %rax
220	movq	%rsi, %rdx
221	shrq	$32, %rdx
222	movl	$OPTERON_MSR_PASSCODE, %edi
223	wrmsr
224	ret
225
226/*
227 * Support for reading MSRs in the safe manner (returns EFAULT on fault)
228 */
229/* int rdmsr_safe(u_int msr, uint64_t *data) */
230ENTRY(rdmsr_safe)
231	movq	CPUVAR(CURLWP), %r8
232	movq	L_PCB(%r8), %r8
233	movq	$_C_LABEL(msr_onfault), PCB_ONFAULT(%r8)
234
235	movl	%edi, %ecx /* u_int msr */
236	rdmsr			/* Read MSR pointed by %ecx. Returns
237				   hi byte in edx, lo in %eax */
238	salq	$32, %rdx	/* sign-shift %rdx left */
239	movl	%eax, %eax	/* zero-extend %eax -> %rax */
240	orq	%rdx, %rax
241	movq	%rax, (%rsi)  /* *data */
242	xorq	%rax, %rax    /* "no error" */
243
244	movq	%rax, PCB_ONFAULT(%r8)
245	ret
246
247/*
248 * MSR operations fault handler
249 */
250NENTRY(msr_onfault)
251	movq	CPUVAR(CURLWP), %r8
252	movq	L_PCB(%r8), %r8
253	movq	$0, PCB_ONFAULT(%r8)
254	movl	$EFAULT, %eax
255	ret
256
257#ifndef XEN
258ENTRY(wbinvd)
259	wbinvd
260	ret
261#endif
262
263ENTRY(cpu_counter)
264	xorq	%rax, %rax
265	rdtsc
266	shlq	$32, %rdx
267	orq	%rdx, %rax
268	addq	CPUVAR(CC_SKEW), %rax
269	ret
270
271ENTRY(cpu_counter32)
272	rdtsc
273	addl	CPUVAR(CC_SKEW), %eax
274	ret
275
276ENTRY(rdpmc)
277	movq	%rdi, %rcx
278	xorq	%rax, %rax
279	rdpmc
280	shlq	$32, %rdx
281	orq	%rdx, %rax
282	ret
283
284ENTRY(breakpoint)
285	pushq	%rbp
286	movq	%rsp, %rbp
287	int	$0x03		/* paranoid, not 'int3' */
288	leave
289	ret
290
291ENTRY(x86_curcpu)
292	movq	%gs:(CPU_INFO_SELF), %rax
293	ret
294
295ENTRY(x86_curlwp)
296	movq	%gs:(CPU_INFO_CURLWP), %rax
297	ret
298
299ENTRY(cpu_set_curpri)
300	movl	%edi, %gs:(CPU_INFO_CURPRIORITY)
301	ret
302
303ENTRY(__byte_swap_u32_variable)
304	movl	%edi, %eax
305	bswapl	%eax
306	ret
307
308ENTRY(__byte_swap_u16_variable)
309	movl	%edi, %eax
310	xchgb	%al, %ah
311	ret
312
313/*
314 * void lgdt(struct region_descriptor *rdp);
315 *
316 * Load a new GDT pointer (and do any necessary cleanup).
317 * XXX It's somewhat questionable whether reloading all the segment registers
318 * is necessary, since the actual descriptor data is not changed except by
319 * process creation and exit, both of which clean up via task switches.  OTOH,
320 * this only happens at run time when the GDT is resized.
321 */
322#ifndef XEN
323ENTRY(lgdt)
324	/* Reload the descriptor table. */
325	movq	%rdi,%rax
326	lgdt	(%rax)
327	/* Flush the prefetch q. */
328	jmp	1f
329	nop
3301:	/* Reload "stale" selectors. */
331#else /* XEN */
332/*
333 * void lgdt_finish(void);
334 * Reload segments after a GDT change
335 */
336ENTRY(lgdt_finish)
337#endif /* XEN */
338	movl	$GSEL(GDATA_SEL, SEL_KPL),%eax
339	movl	%eax,%ds
340	movl	%eax,%es
341	movl	%eax,%ss
342	/* FALLTHROUGH */
343
344/*
345 * void x86_flush()
346 *
347 * Flush instruction pipelines by doing an intersegment (far) return.
348 */
349ENTRY(x86_flush)
350	popq	%rax
351	pushq	$GSEL(GCODE_SEL, SEL_KPL)
352	pushq	%rax
353	lretq
354
355/* Waits - set up stack frame. */
356ENTRY(x86_hlt)
357	pushq	%rbp
358	movq	%rsp, %rbp
359	hlt
360	leave
361	ret
362
363/* Waits - set up stack frame. */
364ENTRY(x86_stihlt)
365	pushq	%rbp
366	movq	%rsp, %rbp
367	sti
368	hlt
369	leave
370	ret
371
372ENTRY(x86_monitor)
373	movq	%rdi, %rax
374	movq	%rsi, %rcx
375	monitor	%rax, %rcx, %rdx
376	ret
377
378/* Waits - set up stack frame. */
379ENTRY(x86_mwait)
380	pushq	%rbp
381	movq	%rsp, %rbp
382	movq	%rdi, %rax
383	movq	%rsi, %rcx
384	mwait	%rax, %rcx
385	leave
386	ret
387
388NENTRY(x86_pause)
389	pause
390	ret
391
392ENTRY(x86_cpuid2)
393	movq	%rbx, %r8
394	movq	%rdi, %rax
395	movq	%rsi, %rcx
396	movq	%rdx, %rsi
397	cpuid
398	movl	%eax, 0(%rsi)
399	movl	%ebx, 4(%rsi)
400	movl	%ecx, 8(%rsi)
401	movl	%edx, 12(%rsi)
402	movq	%r8, %rbx
403	ret
404
405ENTRY(x86_getss)
406	movl	%ss, %eax
407	ret
408
409ENTRY(fldcw)
410	fldcw	(%rdi)
411	ret
412
413ENTRY(fnclex)
414	fnclex
415	ret
416
417ENTRY(fninit)
418	fninit
419	ret
420
421ENTRY(fnsave)
422	fnsave	(%rdi)
423	ret
424
425ENTRY(fnstcw)
426	fnstcw	(%rdi)
427	ret
428
429ENTRY(fnstsw)
430	fnstsw	(%rdi)
431	ret
432
433ENTRY(fp_divide_by_0)
434	fldz
435	fld1
436	fdiv	%st, %st(1)
437	fwait
438	ret
439
440ENTRY(frstor)
441	frstor	(%rdi)
442	ret
443
444ENTRY(fwait)
445	fwait
446	ret
447
448ENTRY(clts)
449	clts
450	ret
451
452ENTRY(stts)
453	movq	%cr0, %rax
454	orq	$CR0_TS, %rax
455	movq	%rax, %cr0
456	ret
457
458ENTRY(fxsave)
459	fxsave	(%rdi)
460	ret
461
462ENTRY(fxrstor)
463	fxrstor	(%rdi)
464	ret
465
466ENTRY(fldummy)
467	ffree	%st(7)
468	flds	(%rdi)
469	ret
470
471ENTRY(x86_ldmxcsr)
472	ldmxcsr	(%rdi)
473	ret
474
475ENTRY(inb)
476	movq	%rdi, %rdx
477	xorq	%rax, %rax
478	inb	%dx, %al
479	ret
480
481ENTRY(insb)
482	movl	%edx, %ecx
483	movl	%edi, %edx
484	movq	%rsi, %rdi
485	rep
486	insb
487	ret
488
489ENTRY(inw)
490	movq	%rdi, %rdx
491	xorq	%rax, %rax
492	inw	%dx, %ax
493	ret
494
495ENTRY(insw)
496	movl	%edx, %ecx
497	movl	%edi, %edx
498	movq	%rsi, %rdi
499	rep
500	insw
501	ret
502
503ENTRY(inl)
504	movq	%rdi, %rdx
505	xorq	%rax, %rax
506	inl	%dx, %eax
507	ret
508
509ENTRY(insl)
510	movl	%edx, %ecx
511	movl	%edi, %edx
512	movq	%rsi, %rdi
513	rep
514	insl
515	ret
516
517ENTRY(outb)
518	movq	%rdi, %rdx
519	movq	%rsi, %rax
520	outb	%al, %dx
521	ret
522
523ENTRY(outsb)
524	movl	%edx, %ecx
525	movl	%edi, %edx
526	rep
527	outsb
528	ret
529
530ENTRY(outw)
531	movq	%rdi, %rdx
532	movq	%rsi, %rax
533	outw	%ax, %dx
534	ret
535
536ENTRY(outsw)
537	movl	%edx, %ecx
538	movl	%edi, %edx
539	rep
540	outsw
541	ret
542
543ENTRY(outl)
544	movq	%rdi, %rdx
545	movq	%rsi, %rax
546	outl	%eax, %dx
547	ret
548
549ENTRY(outsl)
550	movl	%edx, %ecx
551	movl	%edi, %edx
552	rep
553	outsl
554	ret
555
556ENTRY(setfs)
557	movw	%di, %fs
558	ret
559
560ENTRY(setusergs)
561	CLI(ax)
562	swapgs
563	movw	%di, %gs
564	swapgs
565	STI(ax)
566	ret
567