1/*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1993 The Regents of the University of California.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the University nor the names of its contributors
16 *    may be used to endorse or promote products derived from this software
17 *    without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32/*
33 * Functions to provide access to special i386 instructions.
34 * This in included in sys/systm.h, and that file should be
35 * used in preference to this.
36 */
37
38#ifndef _MACHINE_CPUFUNC_H_
39#define	_MACHINE_CPUFUNC_H_
40
41struct region_descriptor;
42
43#define readb(va)	(*(volatile uint8_t *) (va))
44#define readw(va)	(*(volatile uint16_t *) (va))
45#define readl(va)	(*(volatile uint32_t *) (va))
46
47#define writeb(va, d)	(*(volatile uint8_t *) (va) = (d))
48#define writew(va, d)	(*(volatile uint16_t *) (va) = (d))
49#define writel(va, d)	(*(volatile uint32_t *) (va) = (d))
50
51static __inline void
52breakpoint(void)
53{
54	__asm __volatile("int $3");
55}
56
57static __inline __pure2 u_int
58bsfl(u_int mask)
59{
60	u_int	result;
61
62	__asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
63	return (result);
64}
65
66static __inline void
67clflush(u_long addr)
68{
69
70	__asm __volatile("clflush %0" : : "m" (*(char *)addr));
71}
72
73static __inline void
74clflushopt(u_long addr)
75{
76
77	__asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
78}
79
80static __inline void
81clts(void)
82{
83
84	__asm __volatile("clts");
85}
86
87static __inline void
88disable_intr(void)
89{
90	__asm __volatile("cli" : : : "memory");
91}
92
93#ifdef _KERNEL
94static __inline void
95do_cpuid(u_int ax, u_int *p)
96{
97	__asm __volatile("cpuid"
98	    : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
99	    :  "0" (ax));
100}
101
102static __inline void
103cpuid_count(u_int ax, u_int cx, u_int *p)
104{
105	__asm __volatile("cpuid"
106	    : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
107	    :  "0" (ax), "c" (cx));
108}
109#else
110static __inline void
111do_cpuid(u_int ax, u_int *p)
112{
113	__asm __volatile(
114	    "pushl\t%%ebx\n\t"
115	    "cpuid\n\t"
116	    "movl\t%%ebx,%1\n\t"
117	    "popl\t%%ebx"
118	    : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3])
119	    :  "0" (ax));
120}
121
122static __inline void
123cpuid_count(u_int ax, u_int cx, u_int *p)
124{
125	__asm __volatile(
126	    "pushl\t%%ebx\n\t"
127	    "cpuid\n\t"
128	    "movl\t%%ebx,%1\n\t"
129	    "popl\t%%ebx"
130	    : "=a" (p[0]), "=DS" (p[1]), "=c" (p[2]), "=d" (p[3])
131	    :  "0" (ax), "c" (cx));
132}
133#endif
134
135static __inline void
136enable_intr(void)
137{
138	__asm __volatile("sti");
139}
140
141static __inline void
142cpu_monitor(const void *addr, u_long extensions, u_int hints)
143{
144	__asm __volatile("monitor"
145	    : : "a" (addr), "c" (extensions), "d" (hints));
146}
147
148static __inline void
149cpu_mwait(u_long extensions, u_int hints)
150{
151	__asm __volatile("mwait" : : "a" (hints), "c" (extensions));
152}
153
154static __inline void
155lfence(void)
156{
157	__asm __volatile("lfence" : : : "memory");
158}
159
160static __inline void
161mfence(void)
162{
163	__asm __volatile("mfence" : : : "memory");
164}
165
166static __inline void
167sfence(void)
168{
169	__asm __volatile("sfence" : : : "memory");
170}
171
172static __inline void
173halt(void)
174{
175	__asm __volatile("hlt");
176}
177
178static __inline u_char
179inb(u_int port)
180{
181	u_char	data;
182
183	__asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
184	return (data);
185}
186
187static __inline u_int
188inl(u_int port)
189{
190	u_int	data;
191
192	__asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
193	return (data);
194}
195
196static __inline void
197insb(u_int port, void *addr, size_t count)
198{
199	__asm __volatile("cld; rep; insb"
200			 : "+D" (addr), "+c" (count)
201			 : "d" (port)
202			 : "memory");
203}
204
205static __inline void
206insw(u_int port, void *addr, size_t count)
207{
208	__asm __volatile("cld; rep; insw"
209			 : "+D" (addr), "+c" (count)
210			 : "d" (port)
211			 : "memory");
212}
213
214static __inline void
215insl(u_int port, void *addr, size_t count)
216{
217	__asm __volatile("cld; rep; insl"
218			 : "+D" (addr), "+c" (count)
219			 : "d" (port)
220			 : "memory");
221}
222
223static __inline void
224invd(void)
225{
226	__asm __volatile("invd");
227}
228
229static __inline u_short
230inw(u_int port)
231{
232	u_short	data;
233
234	__asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
235	return (data);
236}
237
238static __inline void
239outb(u_int port, u_char data)
240{
241	__asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
242}
243
244static __inline void
245outl(u_int port, u_int data)
246{
247	__asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
248}
249
250static __inline void
251outsb(u_int port, const void *addr, size_t count)
252{
253	__asm __volatile("cld; rep; outsb"
254			 : "+S" (addr), "+c" (count)
255			 : "d" (port));
256}
257
258static __inline void
259outsw(u_int port, const void *addr, size_t count)
260{
261	__asm __volatile("cld; rep; outsw"
262			 : "+S" (addr), "+c" (count)
263			 : "d" (port));
264}
265
266static __inline void
267outsl(u_int port, const void *addr, size_t count)
268{
269	__asm __volatile("cld; rep; outsl"
270			 : "+S" (addr), "+c" (count)
271			 : "d" (port));
272}
273
274static __inline void
275outw(u_int port, u_short data)
276{
277	__asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
278}
279
280static __inline void
281ia32_pause(void)
282{
283	__asm __volatile("pause");
284}
285
286static __inline u_int
287read_eflags(void)
288{
289	u_int	ef;
290
291	__asm __volatile("pushfl; popl %0" : "=r" (ef));
292	return (ef);
293}
294
295static __inline uint64_t
296rdmsr(u_int msr)
297{
298	uint64_t rv;
299
300	__asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
301	return (rv);
302}
303
304static __inline uint32_t
305rdmsr32(u_int msr)
306{
307	uint32_t low;
308
309	__asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "edx");
310	return (low);
311}
312
313static __inline uint64_t
314rdpmc(u_int pmc)
315{
316	uint64_t rv;
317
318	__asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
319	return (rv);
320}
321
322static __inline uint64_t
323rdtsc(void)
324{
325	uint64_t rv;
326
327	__asm __volatile("rdtsc" : "=A" (rv));
328	return (rv);
329}
330
331static __inline uint64_t
332rdtsc_ordered_lfence(void)
333{
334	lfence();
335	return (rdtsc());
336}
337
338static __inline uint64_t
339rdtsc_ordered_mfence(void)
340{
341	mfence();
342	return (rdtsc());
343}
344
345static __inline uint64_t
346rdtscp(void)
347{
348	uint64_t rv;
349
350	__asm __volatile("rdtscp" : "=A" (rv) : : "ecx");
351	return (rv);
352}
353
354static __inline uint64_t
355rdtscp_aux(uint32_t *aux)
356{
357	uint64_t rv;
358
359	__asm __volatile("rdtscp" : "=A" (rv), "=c" (*aux));
360	return (rv);
361}
362
363static __inline uint32_t
364rdtsc32(void)
365{
366	uint32_t rv;
367
368	__asm __volatile("rdtsc" : "=a" (rv) : : "edx");
369	return (rv);
370}
371
372static __inline uint32_t
373rdtscp32(void)
374{
375	uint32_t rv;
376
377	__asm __volatile("rdtscp" : "=a" (rv) : : "ecx", "edx");
378	return (rv);
379}
380
381static __inline void
382wbinvd(void)
383{
384	__asm __volatile("wbinvd");
385}
386
387static __inline void
388write_eflags(u_int ef)
389{
390	__asm __volatile("pushl %0; popfl" : : "r" (ef));
391}
392
393static __inline void
394wrmsr(u_int msr, uint64_t newval)
395{
396	__asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
397}
398
399static __inline void
400load_cr0(u_int data)
401{
402
403	__asm __volatile("movl %0,%%cr0" : : "r" (data));
404}
405
406static __inline u_int
407rcr0(void)
408{
409	u_int	data;
410
411	__asm __volatile("movl %%cr0,%0" : "=r" (data));
412	return (data);
413}
414
415static __inline u_int
416rcr2(void)
417{
418	u_int	data;
419
420	__asm __volatile("movl %%cr2,%0" : "=r" (data));
421	return (data);
422}
423
424static __inline void
425load_cr3(u_int data)
426{
427
428	__asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
429}
430
431static __inline u_int
432rcr3(void)
433{
434	u_int	data;
435
436	__asm __volatile("movl %%cr3,%0" : "=r" (data));
437	return (data);
438}
439
440static __inline void
441load_cr4(u_int data)
442{
443	__asm __volatile("movl %0,%%cr4" : : "r" (data));
444}
445
446static __inline u_int
447rcr4(void)
448{
449	u_int	data;
450
451	__asm __volatile("movl %%cr4,%0" : "=r" (data));
452	return (data);
453}
454
455static __inline uint64_t
456rxcr(u_int reg)
457{
458	u_int low, high;
459
460	__asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
461	return (low | ((uint64_t)high << 32));
462}
463
464static __inline void
465load_xcr(u_int reg, uint64_t val)
466{
467	u_int low, high;
468
469	low = val;
470	high = val >> 32;
471	__asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
472}
473
474/*
475 * Global TLB flush (except for thise for pages marked PG_G)
476 */
477static __inline void
478invltlb(void)
479{
480
481	load_cr3(rcr3());
482}
483
484/*
485 * TLB flush for an individual page (even if it has PG_G).
486 * Only works on 486+ CPUs (i386 does not have PG_G).
487 */
488static __inline void
489invlpg(u_int addr)
490{
491
492	__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
493}
494
495static __inline u_short
496rfs(void)
497{
498	u_short sel;
499	__asm __volatile("movw %%fs,%0" : "=rm" (sel));
500	return (sel);
501}
502
503static __inline uint64_t
504rgdt(void)
505{
506	uint64_t gdtr;
507	__asm __volatile("sgdt %0" : "=m" (gdtr));
508	return (gdtr);
509}
510
511static __inline u_short
512rgs(void)
513{
514	u_short sel;
515	__asm __volatile("movw %%gs,%0" : "=rm" (sel));
516	return (sel);
517}
518
519static __inline uint64_t
520ridt(void)
521{
522	uint64_t idtr;
523	__asm __volatile("sidt %0" : "=m" (idtr));
524	return (idtr);
525}
526
527static __inline u_short
528rldt(void)
529{
530	u_short ldtr;
531	__asm __volatile("sldt %0" : "=g" (ldtr));
532	return (ldtr);
533}
534
535static __inline u_short
536rss(void)
537{
538	u_short sel;
539	__asm __volatile("movw %%ss,%0" : "=rm" (sel));
540	return (sel);
541}
542
543static __inline u_short
544rtr(void)
545{
546	u_short tr;
547	__asm __volatile("str %0" : "=g" (tr));
548	return (tr);
549}
550
551static __inline void
552load_fs(u_short sel)
553{
554	__asm __volatile("movw %0,%%fs" : : "rm" (sel));
555}
556
557static __inline void
558load_gs(u_short sel)
559{
560	__asm __volatile("movw %0,%%gs" : : "rm" (sel));
561}
562
563static __inline void
564lidt(struct region_descriptor *addr)
565{
566	__asm __volatile("lidt (%0)" : : "r" (addr));
567}
568
569static __inline void
570lldt(u_short sel)
571{
572	__asm __volatile("lldt %0" : : "r" (sel));
573}
574
575static __inline void
576ltr(u_short sel)
577{
578	__asm __volatile("ltr %0" : : "r" (sel));
579}
580
581static __inline u_int
582rdr0(void)
583{
584	u_int	data;
585	__asm __volatile("movl %%dr0,%0" : "=r" (data));
586	return (data);
587}
588
589static __inline void
590load_dr0(u_int dr0)
591{
592	__asm __volatile("movl %0,%%dr0" : : "r" (dr0));
593}
594
595static __inline u_int
596rdr1(void)
597{
598	u_int	data;
599	__asm __volatile("movl %%dr1,%0" : "=r" (data));
600	return (data);
601}
602
603static __inline void
604load_dr1(u_int dr1)
605{
606	__asm __volatile("movl %0,%%dr1" : : "r" (dr1));
607}
608
609static __inline u_int
610rdr2(void)
611{
612	u_int	data;
613	__asm __volatile("movl %%dr2,%0" : "=r" (data));
614	return (data);
615}
616
617static __inline void
618load_dr2(u_int dr2)
619{
620	__asm __volatile("movl %0,%%dr2" : : "r" (dr2));
621}
622
623static __inline u_int
624rdr3(void)
625{
626	u_int	data;
627	__asm __volatile("movl %%dr3,%0" : "=r" (data));
628	return (data);
629}
630
631static __inline void
632load_dr3(u_int dr3)
633{
634	__asm __volatile("movl %0,%%dr3" : : "r" (dr3));
635}
636
637static __inline u_int
638rdr6(void)
639{
640	u_int	data;
641	__asm __volatile("movl %%dr6,%0" : "=r" (data));
642	return (data);
643}
644
645static __inline void
646load_dr6(u_int dr6)
647{
648	__asm __volatile("movl %0,%%dr6" : : "r" (dr6));
649}
650
651static __inline u_int
652rdr7(void)
653{
654	u_int	data;
655	__asm __volatile("movl %%dr7,%0" : "=r" (data));
656	return (data);
657}
658
659static __inline void
660load_dr7(u_int dr7)
661{
662	__asm __volatile("movl %0,%%dr7" : : "r" (dr7));
663}
664
665static __inline u_char
666read_cyrix_reg(u_char reg)
667{
668	outb(0x22, reg);
669	return inb(0x23);
670}
671
672static __inline void
673write_cyrix_reg(u_char reg, u_char data)
674{
675	outb(0x22, reg);
676	outb(0x23, data);
677}
678
679static __inline register_t
680intr_disable(void)
681{
682	register_t eflags;
683
684	eflags = read_eflags();
685	disable_intr();
686	return (eflags);
687}
688
689static __inline void
690intr_restore(register_t eflags)
691{
692	write_eflags(eflags);
693}
694
695static __inline uint32_t
696rdpkru(void)
697{
698	uint32_t res;
699
700	__asm __volatile("rdpkru" :  "=a" (res) : "c" (0) : "edx");
701	return (res);
702}
703
704static __inline void
705wrpkru(uint32_t mask)
706{
707
708	__asm __volatile("wrpkru" :  : "a" (mask),  "c" (0), "d" (0));
709}
710
711void    reset_dbregs(void);
712
713#ifdef _KERNEL
714int	rdmsr_safe(u_int msr, uint64_t *val);
715int	wrmsr_safe(u_int msr, uint64_t newval);
716#endif
717
718#endif /* !_MACHINE_CPUFUNC_H_ */
719