1/*-
2 * Copyright (c) 2003 Peter Wemm.
3 * Copyright (c) 1993 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 4. Neither the name of the University nor the names of its contributors
15 *    may be used to endorse or promote products derived from this software
16 *    without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * $FreeBSD: releng/10.3/sys/amd64/include/cpufunc.h 290189 2015-10-30 10:02:57Z kib $
31 */
32
33/*
34 * Functions to provide access to special i386 instructions.
35 * This in included in sys/systm.h, and that file should be
36 * used in preference to this.
37 */
38
39#ifndef _MACHINE_CPUFUNC_H_
40#define	_MACHINE_CPUFUNC_H_
41
42#ifndef _SYS_CDEFS_H_
43#error this file needs sys/cdefs.h as a prerequisite
44#endif
45
46struct region_descriptor;
47
48#define readb(va)	(*(volatile uint8_t *) (va))
49#define readw(va)	(*(volatile uint16_t *) (va))
50#define readl(va)	(*(volatile uint32_t *) (va))
51#define readq(va)	(*(volatile uint64_t *) (va))
52
53#define writeb(va, d)	(*(volatile uint8_t *) (va) = (d))
54#define writew(va, d)	(*(volatile uint16_t *) (va) = (d))
55#define writel(va, d)	(*(volatile uint32_t *) (va) = (d))
56#define writeq(va, d)	(*(volatile uint64_t *) (va) = (d))
57
58#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
59
60static __inline void
61breakpoint(void)
62{
63	__asm __volatile("int $3");
64}
65
66static __inline u_int
67bsfl(u_int mask)
68{
69	u_int	result;
70
71	__asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask));
72	return (result);
73}
74
75static __inline u_long
76bsfq(u_long mask)
77{
78	u_long	result;
79
80	__asm __volatile("bsfq %1,%0" : "=r" (result) : "rm" (mask));
81	return (result);
82}
83
84static __inline u_int
85bsrl(u_int mask)
86{
87	u_int	result;
88
89	__asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask));
90	return (result);
91}
92
93static __inline u_long
94bsrq(u_long mask)
95{
96	u_long	result;
97
98	__asm __volatile("bsrq %1,%0" : "=r" (result) : "rm" (mask));
99	return (result);
100}
101
102static __inline void
103clflush(u_long addr)
104{
105
106	__asm __volatile("clflush %0" : : "m" (*(char *)addr));
107}
108
109static __inline void
110clflushopt(u_long addr)
111{
112
113	__asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
114}
115
116static __inline void
117clts(void)
118{
119
120	__asm __volatile("clts");
121}
122
123static __inline void
124disable_intr(void)
125{
126	__asm __volatile("cli" : : : "memory");
127}
128
129static __inline void
130do_cpuid(u_int ax, u_int *p)
131{
132	__asm __volatile("cpuid"
133			 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
134			 :  "0" (ax));
135}
136
137static __inline void
138cpuid_count(u_int ax, u_int cx, u_int *p)
139{
140	__asm __volatile("cpuid"
141			 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
142			 :  "0" (ax), "c" (cx));
143}
144
145static __inline void
146enable_intr(void)
147{
148	__asm __volatile("sti");
149}
150
151#ifdef _KERNEL
152
153#define	HAVE_INLINE_FFS
154#define        ffs(x)  __builtin_ffs(x)
155
156#define	HAVE_INLINE_FFSL
157
158static __inline int
159ffsl(long mask)
160{
161	return (mask == 0 ? mask : (int)bsfq((u_long)mask) + 1);
162}
163
164#define	HAVE_INLINE_FFSLL
165
166static __inline int
167ffsll(long long mask)
168{
169	return (ffsl((long)mask));
170}
171
172#define	HAVE_INLINE_FLS
173
174static __inline int
175fls(int mask)
176{
177	return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
178}
179
180#define	HAVE_INLINE_FLSL
181
182static __inline int
183flsl(long mask)
184{
185	return (mask == 0 ? mask : (int)bsrq((u_long)mask) + 1);
186}
187
188#define	HAVE_INLINE_FLSLL
189
190static __inline int
191flsll(long long mask)
192{
193	return (flsl((long)mask));
194}
195
196#endif /* _KERNEL */
197
198static __inline void
199halt(void)
200{
201	__asm __volatile("hlt");
202}
203
204static __inline u_char
205inb(u_int port)
206{
207	u_char	data;
208
209	__asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
210	return (data);
211}
212
213static __inline u_int
214inl(u_int port)
215{
216	u_int	data;
217
218	__asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
219	return (data);
220}
221
222static __inline void
223insb(u_int port, void *addr, size_t count)
224{
225	__asm __volatile("cld; rep; insb"
226			 : "+D" (addr), "+c" (count)
227			 : "d" (port)
228			 : "memory");
229}
230
231static __inline void
232insw(u_int port, void *addr, size_t count)
233{
234	__asm __volatile("cld; rep; insw"
235			 : "+D" (addr), "+c" (count)
236			 : "d" (port)
237			 : "memory");
238}
239
240static __inline void
241insl(u_int port, void *addr, size_t count)
242{
243	__asm __volatile("cld; rep; insl"
244			 : "+D" (addr), "+c" (count)
245			 : "d" (port)
246			 : "memory");
247}
248
249static __inline void
250invd(void)
251{
252	__asm __volatile("invd");
253}
254
255static __inline u_short
256inw(u_int port)
257{
258	u_short	data;
259
260	__asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
261	return (data);
262}
263
264static __inline void
265outb(u_int port, u_char data)
266{
267	__asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
268}
269
270static __inline void
271outl(u_int port, u_int data)
272{
273	__asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
274}
275
276static __inline void
277outsb(u_int port, const void *addr, size_t count)
278{
279	__asm __volatile("cld; rep; outsb"
280			 : "+S" (addr), "+c" (count)
281			 : "d" (port));
282}
283
284static __inline void
285outsw(u_int port, const void *addr, size_t count)
286{
287	__asm __volatile("cld; rep; outsw"
288			 : "+S" (addr), "+c" (count)
289			 : "d" (port));
290}
291
292static __inline void
293outsl(u_int port, const void *addr, size_t count)
294{
295	__asm __volatile("cld; rep; outsl"
296			 : "+S" (addr), "+c" (count)
297			 : "d" (port));
298}
299
300static __inline void
301outw(u_int port, u_short data)
302{
303	__asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
304}
305
306static __inline u_long
307popcntq(u_long mask)
308{
309	u_long result;
310
311	__asm __volatile("popcntq %1,%0" : "=r" (result) : "rm" (mask));
312	return (result);
313}
314
315static __inline void
316lfence(void)
317{
318
319	__asm __volatile("lfence" : : : "memory");
320}
321
322static __inline void
323mfence(void)
324{
325
326	__asm __volatile("mfence" : : : "memory");
327}
328
329static __inline void
330ia32_pause(void)
331{
332	__asm __volatile("pause");
333}
334
335static __inline u_long
336read_rflags(void)
337{
338	u_long	rf;
339
340	__asm __volatile("pushfq; popq %0" : "=r" (rf));
341	return (rf);
342}
343
344static __inline uint64_t
345rdmsr(u_int msr)
346{
347	uint32_t low, high;
348
349	__asm __volatile("rdmsr" : "=a" (low), "=d" (high) : "c" (msr));
350	return (low | ((uint64_t)high << 32));
351}
352
353static __inline uint64_t
354rdpmc(u_int pmc)
355{
356	uint32_t low, high;
357
358	__asm __volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (pmc));
359	return (low | ((uint64_t)high << 32));
360}
361
362static __inline uint64_t
363rdtsc(void)
364{
365	uint32_t low, high;
366
367	__asm __volatile("rdtsc" : "=a" (low), "=d" (high));
368	return (low | ((uint64_t)high << 32));
369}
370
371static __inline uint32_t
372rdtsc32(void)
373{
374	uint32_t rv;
375
376	__asm __volatile("rdtsc" : "=a" (rv) : : "edx");
377	return (rv);
378}
379
380static __inline void
381wbinvd(void)
382{
383	__asm __volatile("wbinvd");
384}
385
386static __inline void
387write_rflags(u_long rf)
388{
389	__asm __volatile("pushq %0;  popfq" : : "r" (rf));
390}
391
392static __inline void
393wrmsr(u_int msr, uint64_t newval)
394{
395	uint32_t low, high;
396
397	low = newval;
398	high = newval >> 32;
399	__asm __volatile("wrmsr" : : "a" (low), "d" (high), "c" (msr));
400}
401
402static __inline void
403load_cr0(u_long data)
404{
405
406	__asm __volatile("movq %0,%%cr0" : : "r" (data));
407}
408
409static __inline u_long
410rcr0(void)
411{
412	u_long	data;
413
414	__asm __volatile("movq %%cr0,%0" : "=r" (data));
415	return (data);
416}
417
418static __inline u_long
419rcr2(void)
420{
421	u_long	data;
422
423	__asm __volatile("movq %%cr2,%0" : "=r" (data));
424	return (data);
425}
426
427static __inline void
428load_cr3(u_long data)
429{
430
431	__asm __volatile("movq %0,%%cr3" : : "r" (data) : "memory");
432}
433
434static __inline u_long
435rcr3(void)
436{
437	u_long	data;
438
439	__asm __volatile("movq %%cr3,%0" : "=r" (data));
440	return (data);
441}
442
443static __inline void
444load_cr4(u_long data)
445{
446	__asm __volatile("movq %0,%%cr4" : : "r" (data));
447}
448
449static __inline u_long
450rcr4(void)
451{
452	u_long	data;
453
454	__asm __volatile("movq %%cr4,%0" : "=r" (data));
455	return (data);
456}
457
458static __inline u_long
459rxcr(u_int reg)
460{
461	u_int low, high;
462
463	__asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
464	return (low | ((uint64_t)high << 32));
465}
466
467static __inline void
468load_xcr(u_int reg, u_long val)
469{
470	u_int low, high;
471
472	low = val;
473	high = val >> 32;
474	__asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
475}
476
477/*
478 * Global TLB flush (except for thise for pages marked PG_G)
479 */
480static __inline void
481invltlb(void)
482{
483
484	load_cr3(rcr3());
485}
486
487#ifndef CR4_PGE
488#define	CR4_PGE	0x00000080	/* Page global enable */
489#endif
490
491/*
492 * Perform the guaranteed invalidation of all TLB entries.  This
493 * includes the global entries, and entries in all PCIDs, not only the
494 * current context.  The function works both on non-PCID CPUs and CPUs
495 * with the PCID turned off or on.  See IA-32 SDM Vol. 3a 4.10.4.1
496 * Operations that Invalidate TLBs and Paging-Structure Caches.
497 */
498static __inline void
499invltlb_globpcid(void)
500{
501	uint64_t cr4;
502
503	cr4 = rcr4();
504	load_cr4(cr4 & ~CR4_PGE);
505	/*
506	 * Although preemption at this point could be detrimental to
507	 * performance, it would not lead to an error.  PG_G is simply
508	 * ignored if CR4.PGE is clear.  Moreover, in case this block
509	 * is re-entered, the load_cr4() either above or below will
510	 * modify CR4.PGE flushing the TLB.
511	 */
512	load_cr4(cr4 | CR4_PGE);
513}
514
515/*
516 * TLB flush for an individual page (even if it has PG_G).
517 * Only works on 486+ CPUs (i386 does not have PG_G).
518 */
519static __inline void
520invlpg(u_long addr)
521{
522
523	__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
524}
525
526#define	INVPCID_ADDR	0
527#define	INVPCID_CTX	1
528#define	INVPCID_CTXGLOB	2
529#define	INVPCID_ALLCTX	3
530
531struct invpcid_descr {
532	uint64_t	pcid:12 __packed;
533	uint64_t	pad:52 __packed;
534	uint64_t	addr;
535} __packed;
536
537static __inline void
538invpcid(struct invpcid_descr *d, int type)
539{
540
541	/* invpcid (%rdx),%rax */
542	__asm __volatile(".byte 0x66,0x0f,0x38,0x82,0x02"
543	    : : "d" (d), "a" ((u_long)type) : "memory");
544}
545
546static __inline u_short
547rfs(void)
548{
549	u_short sel;
550	__asm __volatile("movw %%fs,%0" : "=rm" (sel));
551	return (sel);
552}
553
554static __inline u_short
555rgs(void)
556{
557	u_short sel;
558	__asm __volatile("movw %%gs,%0" : "=rm" (sel));
559	return (sel);
560}
561
562static __inline u_short
563rss(void)
564{
565	u_short sel;
566	__asm __volatile("movw %%ss,%0" : "=rm" (sel));
567	return (sel);
568}
569
570static __inline void
571load_ds(u_short sel)
572{
573	__asm __volatile("movw %0,%%ds" : : "rm" (sel));
574}
575
576static __inline void
577load_es(u_short sel)
578{
579	__asm __volatile("movw %0,%%es" : : "rm" (sel));
580}
581
582static __inline void
583cpu_monitor(const void *addr, u_long extensions, u_int hints)
584{
585
586	__asm __volatile("monitor"
587	    : : "a" (addr), "c" (extensions), "d" (hints));
588}
589
590static __inline void
591cpu_mwait(u_long extensions, u_int hints)
592{
593
594	__asm __volatile("mwait" : : "a" (hints), "c" (extensions));
595}
596
597#ifdef _KERNEL
598/* This is defined in <machine/specialreg.h> but is too painful to get to */
599#ifndef	MSR_FSBASE
600#define	MSR_FSBASE	0xc0000100
601#endif
602static __inline void
603load_fs(u_short sel)
604{
605	/* Preserve the fsbase value across the selector load */
606	__asm __volatile("rdmsr; movw %0,%%fs; wrmsr"
607	    : : "rm" (sel), "c" (MSR_FSBASE) : "eax", "edx");
608}
609
610#ifndef	MSR_GSBASE
611#define	MSR_GSBASE	0xc0000101
612#endif
613static __inline void
614load_gs(u_short sel)
615{
616	/*
617	 * Preserve the gsbase value across the selector load.
618	 * Note that we have to disable interrupts because the gsbase
619	 * being trashed happens to be the kernel gsbase at the time.
620	 */
621	__asm __volatile("pushfq; cli; rdmsr; movw %0,%%gs; wrmsr; popfq"
622	    : : "rm" (sel), "c" (MSR_GSBASE) : "eax", "edx");
623}
624#else
625/* Usable by userland */
626static __inline void
627load_fs(u_short sel)
628{
629	__asm __volatile("movw %0,%%fs" : : "rm" (sel));
630}
631
632static __inline void
633load_gs(u_short sel)
634{
635	__asm __volatile("movw %0,%%gs" : : "rm" (sel));
636}
637#endif
638
639static __inline void
640lidt(struct region_descriptor *addr)
641{
642	__asm __volatile("lidt (%0)" : : "r" (addr));
643}
644
645static __inline void
646lldt(u_short sel)
647{
648	__asm __volatile("lldt %0" : : "r" (sel));
649}
650
651static __inline void
652ltr(u_short sel)
653{
654	__asm __volatile("ltr %0" : : "r" (sel));
655}
656
657static __inline uint64_t
658rdr0(void)
659{
660	uint64_t data;
661	__asm __volatile("movq %%dr0,%0" : "=r" (data));
662	return (data);
663}
664
665static __inline void
666load_dr0(uint64_t dr0)
667{
668	__asm __volatile("movq %0,%%dr0" : : "r" (dr0));
669}
670
671static __inline uint64_t
672rdr1(void)
673{
674	uint64_t data;
675	__asm __volatile("movq %%dr1,%0" : "=r" (data));
676	return (data);
677}
678
679static __inline void
680load_dr1(uint64_t dr1)
681{
682	__asm __volatile("movq %0,%%dr1" : : "r" (dr1));
683}
684
685static __inline uint64_t
686rdr2(void)
687{
688	uint64_t data;
689	__asm __volatile("movq %%dr2,%0" : "=r" (data));
690	return (data);
691}
692
693static __inline void
694load_dr2(uint64_t dr2)
695{
696	__asm __volatile("movq %0,%%dr2" : : "r" (dr2));
697}
698
699static __inline uint64_t
700rdr3(void)
701{
702	uint64_t data;
703	__asm __volatile("movq %%dr3,%0" : "=r" (data));
704	return (data);
705}
706
707static __inline void
708load_dr3(uint64_t dr3)
709{
710	__asm __volatile("movq %0,%%dr3" : : "r" (dr3));
711}
712
713static __inline uint64_t
714rdr4(void)
715{
716	uint64_t data;
717	__asm __volatile("movq %%dr4,%0" : "=r" (data));
718	return (data);
719}
720
721static __inline void
722load_dr4(uint64_t dr4)
723{
724	__asm __volatile("movq %0,%%dr4" : : "r" (dr4));
725}
726
727static __inline uint64_t
728rdr5(void)
729{
730	uint64_t data;
731	__asm __volatile("movq %%dr5,%0" : "=r" (data));
732	return (data);
733}
734
735static __inline void
736load_dr5(uint64_t dr5)
737{
738	__asm __volatile("movq %0,%%dr5" : : "r" (dr5));
739}
740
741static __inline uint64_t
742rdr6(void)
743{
744	uint64_t data;
745	__asm __volatile("movq %%dr6,%0" : "=r" (data));
746	return (data);
747}
748
749static __inline void
750load_dr6(uint64_t dr6)
751{
752	__asm __volatile("movq %0,%%dr6" : : "r" (dr6));
753}
754
755static __inline uint64_t
756rdr7(void)
757{
758	uint64_t data;
759	__asm __volatile("movq %%dr7,%0" : "=r" (data));
760	return (data);
761}
762
763static __inline void
764load_dr7(uint64_t dr7)
765{
766	__asm __volatile("movq %0,%%dr7" : : "r" (dr7));
767}
768
769static __inline register_t
770intr_disable(void)
771{
772	register_t rflags;
773
774	rflags = read_rflags();
775	disable_intr();
776	return (rflags);
777}
778
779static __inline void
780intr_restore(register_t rflags)
781{
782	write_rflags(rflags);
783}
784
785#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
786
787int	breakpoint(void);
788u_int	bsfl(u_int mask);
789u_int	bsrl(u_int mask);
790void	clflush(u_long addr);
791void	clts(void);
792void	cpuid_count(u_int ax, u_int cx, u_int *p);
793void	disable_intr(void);
794void	do_cpuid(u_int ax, u_int *p);
795void	enable_intr(void);
796void	halt(void);
797void	ia32_pause(void);
798u_char	inb(u_int port);
799u_int	inl(u_int port);
800void	insb(u_int port, void *addr, size_t count);
801void	insl(u_int port, void *addr, size_t count);
802void	insw(u_int port, void *addr, size_t count);
803register_t	intr_disable(void);
804void	intr_restore(register_t rf);
805void	invd(void);
806void	invlpg(u_int addr);
807void	invltlb(void);
808u_short	inw(u_int port);
809void	lidt(struct region_descriptor *addr);
810void	lldt(u_short sel);
811void	load_cr0(u_long cr0);
812void	load_cr3(u_long cr3);
813void	load_cr4(u_long cr4);
814void	load_dr0(uint64_t dr0);
815void	load_dr1(uint64_t dr1);
816void	load_dr2(uint64_t dr2);
817void	load_dr3(uint64_t dr3);
818void	load_dr4(uint64_t dr4);
819void	load_dr5(uint64_t dr5);
820void	load_dr6(uint64_t dr6);
821void	load_dr7(uint64_t dr7);
822void	load_fs(u_short sel);
823void	load_gs(u_short sel);
824void	ltr(u_short sel);
825void	outb(u_int port, u_char data);
826void	outl(u_int port, u_int data);
827void	outsb(u_int port, const void *addr, size_t count);
828void	outsl(u_int port, const void *addr, size_t count);
829void	outsw(u_int port, const void *addr, size_t count);
830void	outw(u_int port, u_short data);
831u_long	rcr0(void);
832u_long	rcr2(void);
833u_long	rcr3(void);
834u_long	rcr4(void);
835uint64_t rdmsr(u_int msr);
836uint64_t rdpmc(u_int pmc);
837uint64_t rdr0(void);
838uint64_t rdr1(void);
839uint64_t rdr2(void);
840uint64_t rdr3(void);
841uint64_t rdr4(void);
842uint64_t rdr5(void);
843uint64_t rdr6(void);
844uint64_t rdr7(void);
845uint64_t rdtsc(void);
846u_long	read_rflags(void);
847u_int	rfs(void);
848u_int	rgs(void);
849void	wbinvd(void);
850void	write_rflags(u_int rf);
851void	wrmsr(u_int msr, uint64_t newval);
852
853#endif	/* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
854
855void	reset_dbregs(void);
856
857#ifdef _KERNEL
858int	rdmsr_safe(u_int msr, uint64_t *val);
859int	wrmsr_safe(u_int msr, uint64_t newval);
860#endif
861
862#endif /* !_MACHINE_CPUFUNC_H_ */
863