1/*-
2 * Copyright (c) 2003 Peter Wemm.
3 * Copyright (c) 1993 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 *    may be used to endorse or promote products derived from this software
16 *    without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * $FreeBSD: stable/10/sys/amd64/include/cpufunc.h 313150 2017-02-03 12:20:44Z kib $
31 */
32
33/*
34 * Functions to provide access to special i386 instructions.
35 * This in included in sys/systm.h, and that file should be
36 * used in preference to this.
37 */
38
39#ifndef _MACHINE_CPUFUNC_H_
40#define	_MACHINE_CPUFUNC_H_
41
42#ifndef _SYS_CDEFS_H_
43#error this file needs sys/cdefs.h as a prerequisite
44#endif
45
46struct region_descriptor;
47
48#define readb(va)	(*(volatile uint8_t *) (va))
49#define readw(va)	(*(volatile uint16_t *) (va))
50#define readl(va)	(*(volatile uint32_t *) (va))
51#define readq(va)	(*(volatile uint64_t *) (va))
52
53#define writeb(va, d)	(*(volatile uint8_t *) (va) = (d))
54#define writew(va, d)	(*(volatile uint16_t *) (va) = (d))
55#define writel(va, d)	(*(volatile uint32_t *) (va) = (d))
56#define writeq(va, d)	(*(volatile uint64_t *) (va) = (d))
57
58#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
59
60static __inline void
61breakpoint(void)
62{
63	__asm __volatile("int $3");
64}
65
66static __inline u_int
67bsfl(u_int mask)
68{
69	u_int	result;
70
71	__asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask));
72	return (result);
73}
74
75static __inline u_long
76bsfq(u_long mask)
77{
78	u_long	result;
79
80	__asm __volatile("bsfq %1,%0" : "=r" (result) : "rm" (mask));
81	return (result);
82}
83
84static __inline u_int
85bsrl(u_int mask)
86{
87	u_int	result;
88
89	__asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask));
90	return (result);
91}
92
93static __inline u_long
94bsrq(u_long mask)
95{
96	u_long	result;
97
98	__asm __volatile("bsrq %1,%0" : "=r" (result) : "rm" (mask));
99	return (result);
100}
101
102static __inline void
103clflush(u_long addr)
104{
105
106	__asm __volatile("clflush %0" : : "m" (*(char *)addr));
107}
108
109static __inline void
110clflushopt(u_long addr)
111{
112
113	__asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
114}
115
116static __inline void
117clts(void)
118{
119
120	__asm __volatile("clts");
121}
122
123static __inline void
124disable_intr(void)
125{
126	__asm __volatile("cli" : : : "memory");
127}
128
129static __inline void
130do_cpuid(u_int ax, u_int *p)
131{
132	__asm __volatile("cpuid"
133			 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
134			 :  "0" (ax));
135}
136
137static __inline void
138cpuid_count(u_int ax, u_int cx, u_int *p)
139{
140	__asm __volatile("cpuid"
141			 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
142			 :  "0" (ax), "c" (cx));
143}
144
145static __inline void
146enable_intr(void)
147{
148	__asm __volatile("sti");
149}
150
151#ifdef _KERNEL
152
153#define	HAVE_INLINE_FFS
154#define        ffs(x)  __builtin_ffs(x)
155
156#define	HAVE_INLINE_FFSL
157
158static __inline int
159ffsl(long mask)
160{
161	return (mask == 0 ? mask : (int)bsfq((u_long)mask) + 1);
162}
163
164#define	HAVE_INLINE_FFSLL
165
166static __inline int
167ffsll(long long mask)
168{
169	return (ffsl((long)mask));
170}
171
172#define	HAVE_INLINE_FLS
173
174static __inline int
175fls(int mask)
176{
177	return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
178}
179
180#define	HAVE_INLINE_FLSL
181
182static __inline int
183flsl(long mask)
184{
185	return (mask == 0 ? mask : (int)bsrq((u_long)mask) + 1);
186}
187
188#define	HAVE_INLINE_FLSLL
189
190static __inline int
191flsll(long long mask)
192{
193	return (flsl((long)mask));
194}
195
196#endif /* _KERNEL */
197
198static __inline void
199halt(void)
200{
201	__asm __volatile("hlt");
202}
203
204static __inline u_char
205inb(u_int port)
206{
207	u_char	data;
208
209	__asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
210	return (data);
211}
212
213static __inline u_int
214inl(u_int port)
215{
216	u_int	data;
217
218	__asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
219	return (data);
220}
221
222static __inline void
223insb(u_int port, void *addr, size_t count)
224{
225	__asm __volatile("cld; rep; insb"
226			 : "+D" (addr), "+c" (count)
227			 : "d" (port)
228			 : "memory");
229}
230
231static __inline void
232insw(u_int port, void *addr, size_t count)
233{
234	__asm __volatile("cld; rep; insw"
235			 : "+D" (addr), "+c" (count)
236			 : "d" (port)
237			 : "memory");
238}
239
240static __inline void
241insl(u_int port, void *addr, size_t count)
242{
243	__asm __volatile("cld; rep; insl"
244			 : "+D" (addr), "+c" (count)
245			 : "d" (port)
246			 : "memory");
247}
248
249static __inline void
250invd(void)
251{
252	__asm __volatile("invd");
253}
254
255static __inline u_short
256inw(u_int port)
257{
258	u_short	data;
259
260	__asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
261	return (data);
262}
263
264static __inline void
265outb(u_int port, u_char data)
266{
267	__asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
268}
269
270static __inline void
271outl(u_int port, u_int data)
272{
273	__asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
274}
275
276static __inline void
277outsb(u_int port, const void *addr, size_t count)
278{
279	__asm __volatile("cld; rep; outsb"
280			 : "+S" (addr), "+c" (count)
281			 : "d" (port));
282}
283
284static __inline void
285outsw(u_int port, const void *addr, size_t count)
286{
287	__asm __volatile("cld; rep; outsw"
288			 : "+S" (addr), "+c" (count)
289			 : "d" (port));
290}
291
292static __inline void
293outsl(u_int port, const void *addr, size_t count)
294{
295	__asm __volatile("cld; rep; outsl"
296			 : "+S" (addr), "+c" (count)
297			 : "d" (port));
298}
299
300static __inline void
301outw(u_int port, u_short data)
302{
303	__asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
304}
305
306static __inline u_long
307popcntq(u_long mask)
308{
309	u_long result;
310
311	__asm __volatile("popcntq %1,%0" : "=r" (result) : "rm" (mask));
312	return (result);
313}
314
315static __inline void
316lfence(void)
317{
318
319	__asm __volatile("lfence" : : : "memory");
320}
321
322static __inline void
323mfence(void)
324{
325
326	__asm __volatile("mfence" : : : "memory");
327}
328
329static __inline void
330sfence(void)
331{
332
333	__asm __volatile("sfence" : : : "memory");
334}
335
336static __inline void
337ia32_pause(void)
338{
339	__asm __volatile("pause");
340}
341
342static __inline u_long
343read_rflags(void)
344{
345	u_long	rf;
346
347	__asm __volatile("pushfq; popq %0" : "=r" (rf));
348	return (rf);
349}
350
351static __inline uint64_t
352rdmsr(u_int msr)
353{
354	uint32_t low, high;
355
356	__asm __volatile("rdmsr" : "=a" (low), "=d" (high) : "c" (msr));
357	return (low | ((uint64_t)high << 32));
358}
359
360static __inline uint64_t
361rdpmc(u_int pmc)
362{
363	uint32_t low, high;
364
365	__asm __volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (pmc));
366	return (low | ((uint64_t)high << 32));
367}
368
369static __inline uint64_t
370rdtsc(void)
371{
372	uint32_t low, high;
373
374	__asm __volatile("rdtsc" : "=a" (low), "=d" (high));
375	return (low | ((uint64_t)high << 32));
376}
377
378static __inline uint32_t
379rdtsc32(void)
380{
381	uint32_t rv;
382
383	__asm __volatile("rdtsc" : "=a" (rv) : : "edx");
384	return (rv);
385}
386
387static __inline void
388wbinvd(void)
389{
390	__asm __volatile("wbinvd");
391}
392
393static __inline void
394write_rflags(u_long rf)
395{
396	__asm __volatile("pushq %0;  popfq" : : "r" (rf));
397}
398
399static __inline void
400wrmsr(u_int msr, uint64_t newval)
401{
402	uint32_t low, high;
403
404	low = newval;
405	high = newval >> 32;
406	__asm __volatile("wrmsr" : : "a" (low), "d" (high), "c" (msr));
407}
408
409static __inline void
410load_cr0(u_long data)
411{
412
413	__asm __volatile("movq %0,%%cr0" : : "r" (data));
414}
415
416static __inline u_long
417rcr0(void)
418{
419	u_long	data;
420
421	__asm __volatile("movq %%cr0,%0" : "=r" (data));
422	return (data);
423}
424
425static __inline u_long
426rcr2(void)
427{
428	u_long	data;
429
430	__asm __volatile("movq %%cr2,%0" : "=r" (data));
431	return (data);
432}
433
434static __inline void
435load_cr3(u_long data)
436{
437
438	__asm __volatile("movq %0,%%cr3" : : "r" (data) : "memory");
439}
440
441static __inline u_long
442rcr3(void)
443{
444	u_long	data;
445
446	__asm __volatile("movq %%cr3,%0" : "=r" (data));
447	return (data);
448}
449
450static __inline void
451load_cr4(u_long data)
452{
453	__asm __volatile("movq %0,%%cr4" : : "r" (data));
454}
455
456static __inline u_long
457rcr4(void)
458{
459	u_long	data;
460
461	__asm __volatile("movq %%cr4,%0" : "=r" (data));
462	return (data);
463}
464
465static __inline u_long
466rxcr(u_int reg)
467{
468	u_int low, high;
469
470	__asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
471	return (low | ((uint64_t)high << 32));
472}
473
474static __inline void
475load_xcr(u_int reg, u_long val)
476{
477	u_int low, high;
478
479	low = val;
480	high = val >> 32;
481	__asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
482}
483
484/*
485 * Global TLB flush (except for thise for pages marked PG_G)
486 */
487static __inline void
488invltlb(void)
489{
490
491	load_cr3(rcr3());
492}
493
494#ifndef CR4_PGE
495#define	CR4_PGE	0x00000080	/* Page global enable */
496#endif
497
498/*
499 * Perform the guaranteed invalidation of all TLB entries.  This
500 * includes the global entries, and entries in all PCIDs, not only the
501 * current context.  The function works both on non-PCID CPUs and CPUs
502 * with the PCID turned off or on.  See IA-32 SDM Vol. 3a 4.10.4.1
503 * Operations that Invalidate TLBs and Paging-Structure Caches.
504 */
505static __inline void
506invltlb_globpcid(void)
507{
508	uint64_t cr4;
509
510	cr4 = rcr4();
511	load_cr4(cr4 & ~CR4_PGE);
512	/*
513	 * Although preemption at this point could be detrimental to
514	 * performance, it would not lead to an error.  PG_G is simply
515	 * ignored if CR4.PGE is clear.  Moreover, in case this block
516	 * is re-entered, the load_cr4() either above or below will
517	 * modify CR4.PGE flushing the TLB.
518	 */
519	load_cr4(cr4 | CR4_PGE);
520}
521
522/*
523 * TLB flush for an individual page (even if it has PG_G).
524 * Only works on 486+ CPUs (i386 does not have PG_G).
525 */
526static __inline void
527invlpg(u_long addr)
528{
529
530	__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
531}
532
533#define	INVPCID_ADDR	0
534#define	INVPCID_CTX	1
535#define	INVPCID_CTXGLOB	2
536#define	INVPCID_ALLCTX	3
537
538struct invpcid_descr {
539	uint64_t	pcid:12 __packed;
540	uint64_t	pad:52 __packed;
541	uint64_t	addr;
542} __packed;
543
544static __inline void
545invpcid(struct invpcid_descr *d, int type)
546{
547
548	/* invpcid (%rdx),%rax */
549	__asm __volatile(".byte 0x66,0x0f,0x38,0x82,0x02"
550	    : : "d" (d), "a" ((u_long)type) : "memory");
551}
552
553static __inline u_short
554rfs(void)
555{
556	u_short sel;
557	__asm __volatile("movw %%fs,%0" : "=rm" (sel));
558	return (sel);
559}
560
561static __inline u_short
562rgs(void)
563{
564	u_short sel;
565	__asm __volatile("movw %%gs,%0" : "=rm" (sel));
566	return (sel);
567}
568
569static __inline u_short
570rss(void)
571{
572	u_short sel;
573	__asm __volatile("movw %%ss,%0" : "=rm" (sel));
574	return (sel);
575}
576
577static __inline void
578load_ds(u_short sel)
579{
580	__asm __volatile("movw %0,%%ds" : : "rm" (sel));
581}
582
583static __inline void
584load_es(u_short sel)
585{
586	__asm __volatile("movw %0,%%es" : : "rm" (sel));
587}
588
589static __inline void
590cpu_monitor(const void *addr, u_long extensions, u_int hints)
591{
592
593	__asm __volatile("monitor"
594	    : : "a" (addr), "c" (extensions), "d" (hints));
595}
596
597static __inline void
598cpu_mwait(u_long extensions, u_int hints)
599{
600
601	__asm __volatile("mwait" : : "a" (hints), "c" (extensions));
602}
603
604#ifdef _KERNEL
605/* This is defined in <machine/specialreg.h> but is too painful to get to */
606#ifndef	MSR_FSBASE
607#define	MSR_FSBASE	0xc0000100
608#endif
609static __inline void
610load_fs(u_short sel)
611{
612	/* Preserve the fsbase value across the selector load */
613	__asm __volatile("rdmsr; movw %0,%%fs; wrmsr"
614	    : : "rm" (sel), "c" (MSR_FSBASE) : "eax", "edx");
615}
616
617#ifndef	MSR_GSBASE
618#define	MSR_GSBASE	0xc0000101
619#endif
620static __inline void
621load_gs(u_short sel)
622{
623	/*
624	 * Preserve the gsbase value across the selector load.
625	 * Note that we have to disable interrupts because the gsbase
626	 * being trashed happens to be the kernel gsbase at the time.
627	 */
628	__asm __volatile("pushfq; cli; rdmsr; movw %0,%%gs; wrmsr; popfq"
629	    : : "rm" (sel), "c" (MSR_GSBASE) : "eax", "edx");
630}
631#else
632/* Usable by userland */
633static __inline void
634load_fs(u_short sel)
635{
636	__asm __volatile("movw %0,%%fs" : : "rm" (sel));
637}
638
639static __inline void
640load_gs(u_short sel)
641{
642	__asm __volatile("movw %0,%%gs" : : "rm" (sel));
643}
644#endif
645
646static __inline void
647lidt(struct region_descriptor *addr)
648{
649	__asm __volatile("lidt (%0)" : : "r" (addr));
650}
651
652static __inline void
653lldt(u_short sel)
654{
655	__asm __volatile("lldt %0" : : "r" (sel));
656}
657
658static __inline void
659ltr(u_short sel)
660{
661	__asm __volatile("ltr %0" : : "r" (sel));
662}
663
664static __inline uint64_t
665rdr0(void)
666{
667	uint64_t data;
668	__asm __volatile("movq %%dr0,%0" : "=r" (data));
669	return (data);
670}
671
672static __inline void
673load_dr0(uint64_t dr0)
674{
675	__asm __volatile("movq %0,%%dr0" : : "r" (dr0));
676}
677
678static __inline uint64_t
679rdr1(void)
680{
681	uint64_t data;
682	__asm __volatile("movq %%dr1,%0" : "=r" (data));
683	return (data);
684}
685
686static __inline void
687load_dr1(uint64_t dr1)
688{
689	__asm __volatile("movq %0,%%dr1" : : "r" (dr1));
690}
691
692static __inline uint64_t
693rdr2(void)
694{
695	uint64_t data;
696	__asm __volatile("movq %%dr2,%0" : "=r" (data));
697	return (data);
698}
699
700static __inline void
701load_dr2(uint64_t dr2)
702{
703	__asm __volatile("movq %0,%%dr2" : : "r" (dr2));
704}
705
706static __inline uint64_t
707rdr3(void)
708{
709	uint64_t data;
710	__asm __volatile("movq %%dr3,%0" : "=r" (data));
711	return (data);
712}
713
714static __inline void
715load_dr3(uint64_t dr3)
716{
717	__asm __volatile("movq %0,%%dr3" : : "r" (dr3));
718}
719
720static __inline uint64_t
721rdr4(void)
722{
723	uint64_t data;
724	__asm __volatile("movq %%dr4,%0" : "=r" (data));
725	return (data);
726}
727
728static __inline void
729load_dr4(uint64_t dr4)
730{
731	__asm __volatile("movq %0,%%dr4" : : "r" (dr4));
732}
733
734static __inline uint64_t
735rdr5(void)
736{
737	uint64_t data;
738	__asm __volatile("movq %%dr5,%0" : "=r" (data));
739	return (data);
740}
741
742static __inline void
743load_dr5(uint64_t dr5)
744{
745	__asm __volatile("movq %0,%%dr5" : : "r" (dr5));
746}
747
748static __inline uint64_t
749rdr6(void)
750{
751	uint64_t data;
752	__asm __volatile("movq %%dr6,%0" : "=r" (data));
753	return (data);
754}
755
756static __inline void
757load_dr6(uint64_t dr6)
758{
759	__asm __volatile("movq %0,%%dr6" : : "r" (dr6));
760}
761
762static __inline uint64_t
763rdr7(void)
764{
765	uint64_t data;
766	__asm __volatile("movq %%dr7,%0" : "=r" (data));
767	return (data);
768}
769
770static __inline void
771load_dr7(uint64_t dr7)
772{
773	__asm __volatile("movq %0,%%dr7" : : "r" (dr7));
774}
775
776static __inline register_t
777intr_disable(void)
778{
779	register_t rflags;
780
781	rflags = read_rflags();
782	disable_intr();
783	return (rflags);
784}
785
786static __inline void
787intr_restore(register_t rflags)
788{
789	write_rflags(rflags);
790}
791
792#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
793
794int	breakpoint(void);
795u_int	bsfl(u_int mask);
796u_int	bsrl(u_int mask);
797void	clflush(u_long addr);
798void	clts(void);
799void	cpuid_count(u_int ax, u_int cx, u_int *p);
800void	disable_intr(void);
801void	do_cpuid(u_int ax, u_int *p);
802void	enable_intr(void);
803void	halt(void);
804void	ia32_pause(void);
805u_char	inb(u_int port);
806u_int	inl(u_int port);
807void	insb(u_int port, void *addr, size_t count);
808void	insl(u_int port, void *addr, size_t count);
809void	insw(u_int port, void *addr, size_t count);
810register_t	intr_disable(void);
811void	intr_restore(register_t rf);
812void	invd(void);
813void	invlpg(u_int addr);
814void	invltlb(void);
815u_short	inw(u_int port);
816void	lidt(struct region_descriptor *addr);
817void	lldt(u_short sel);
818void	load_cr0(u_long cr0);
819void	load_cr3(u_long cr3);
820void	load_cr4(u_long cr4);
821void	load_dr0(uint64_t dr0);
822void	load_dr1(uint64_t dr1);
823void	load_dr2(uint64_t dr2);
824void	load_dr3(uint64_t dr3);
825void	load_dr4(uint64_t dr4);
826void	load_dr5(uint64_t dr5);
827void	load_dr6(uint64_t dr6);
828void	load_dr7(uint64_t dr7);
829void	load_fs(u_short sel);
830void	load_gs(u_short sel);
831void	ltr(u_short sel);
832void	outb(u_int port, u_char data);
833void	outl(u_int port, u_int data);
834void	outsb(u_int port, const void *addr, size_t count);
835void	outsl(u_int port, const void *addr, size_t count);
836void	outsw(u_int port, const void *addr, size_t count);
837void	outw(u_int port, u_short data);
838u_long	rcr0(void);
839u_long	rcr2(void);
840u_long	rcr3(void);
841u_long	rcr4(void);
842uint64_t rdmsr(u_int msr);
843uint64_t rdpmc(u_int pmc);
844uint64_t rdr0(void);
845uint64_t rdr1(void);
846uint64_t rdr2(void);
847uint64_t rdr3(void);
848uint64_t rdr4(void);
849uint64_t rdr5(void);
850uint64_t rdr6(void);
851uint64_t rdr7(void);
852uint64_t rdtsc(void);
853u_long	read_rflags(void);
854u_int	rfs(void);
855u_int	rgs(void);
856void	wbinvd(void);
857void	write_rflags(u_int rf);
858void	wrmsr(u_int msr, uint64_t newval);
859
860#endif	/* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
861
862void	reset_dbregs(void);
863
864#ifdef _KERNEL
865int	rdmsr_safe(u_int msr, uint64_t *val);
866int	wrmsr_safe(u_int msr, uint64_t newval);
867#endif
868
869#endif /* !_MACHINE_CPUFUNC_H_ */
870