1/*-
2 * Copyright (c) 1993 The Regents of the University of California.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 4. Neither the name of the University nor the names of its contributors
14 * may be used to endorse or promote products derived from this software
15 * without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD: stable/10/sys/i386/include/cpufunc.h 289818 2015-10-23 10:05:43Z avg $
30 */
31
32/*
33 * Functions to provide access to special i386 instructions.
34 * This in included in sys/systm.h, and that file should be
35 * used in preference to this.
36 */
37
38#ifndef _MACHINE_CPUFUNC_H_
39#define _MACHINE_CPUFUNC_H_
40
41#ifndef _SYS_CDEFS_H_
42#error this file needs sys/cdefs.h as a prerequisite
43#endif
44
45#ifdef XEN
46extern void xen_cli(void);
47extern void xen_sti(void);
48extern u_int xen_rcr2(void);
49extern void xen_load_cr3(u_int data);
50extern void xen_tlb_flush(void);
51extern void xen_invlpg(u_int addr);
52extern void write_eflags(u_int eflags);
53extern u_int read_eflags(void);
54#endif
55
56struct region_descriptor;
57
58#define readb(va) (*(volatile uint8_t *) (va))
59#define readw(va) (*(volatile uint16_t *) (va))
60#define readl(va) (*(volatile uint32_t *) (va))
61
62#define writeb(va, d) (*(volatile uint8_t *) (va) = (d))
63#define writew(va, d) (*(volatile uint16_t *) (va) = (d))
64#define writel(va, d) (*(volatile uint32_t *) (va) = (d))
65
66#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
67
68static __inline void
69breakpoint(void)
70{
71 __asm __volatile("int $3");
72}
73
74static __inline u_int
75bsfl(u_int mask)
76{
77 u_int result;
78
79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
80 return (result);
81}
82
83static __inline u_int
84bsrl(u_int mask)
85{
86 u_int result;
87
88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
89 return (result);
90}
91
92static __inline void
93clflush(u_long addr)
94{
95
96 __asm __volatile("clflush %0" : : "m" (*(char *)addr));
97}
98
99static __inline void
100clts(void)
101{
102
103 __asm __volatile("clts");
104}
105
106static __inline void
107disable_intr(void)
108{
109#ifdef XEN
110 xen_cli();
111#else
112 __asm __volatile("cli" : : : "memory");
113#endif
114}
115
116static __inline void
117do_cpuid(u_int ax, u_int *p)
118{
119 __asm __volatile("cpuid"
120 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
121 : "0" (ax));
122}
123
124static __inline void
125cpuid_count(u_int ax, u_int cx, u_int *p)
126{
127 __asm __volatile("cpuid"
128 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
129 : "0" (ax), "c" (cx));
130}
131
132static __inline void
133enable_intr(void)
134{
135#ifdef XEN
136 xen_sti();
137#else
138 __asm __volatile("sti");
139#endif
140}
141
142static __inline void
143cpu_monitor(const void *addr, u_long extensions, u_int hints)
144{
145
146 __asm __volatile("monitor"
147 : : "a" (addr), "c" (extensions), "d" (hints));
148}
149
150static __inline void
151cpu_mwait(u_long extensions, u_int hints)
152{
153
154 __asm __volatile("mwait" : : "a" (hints), "c" (extensions));
155}
156
157static __inline void
158lfence(void)
159{
160
161 __asm __volatile("lfence" : : : "memory");
162}
163
164static __inline void
165mfence(void)
166{
167
168 __asm __volatile("mfence" : : : "memory");
169}
170
171#ifdef _KERNEL
172
173#define HAVE_INLINE_FFS
174
175static __inline int
176ffs(int mask)
177{
178 /*
179 * Note that gcc-2's builtin ffs would be used if we didn't declare
180 * this inline or turn off the builtin. The builtin is faster but
181 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later
182 * versions.
183 */
184 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1);
185}
186
187#define HAVE_INLINE_FFSL
188
189static __inline int
190ffsl(long mask)
191{
192 return (ffs((int)mask));
193}
194
195#define HAVE_INLINE_FLS
196
197static __inline int
198fls(int mask)
199{
200 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
201}
202
203#define HAVE_INLINE_FLSL
204
205static __inline int
206flsl(long mask)
207{
208 return (fls((int)mask));
209}
210
211#endif /* _KERNEL */
212
213static __inline void
214halt(void)
215{
216 __asm __volatile("hlt");
217}
218
219static __inline u_char
220inb(u_int port)
221{
222 u_char data;
223
224 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
225 return (data);
226}
227
228static __inline u_int
229inl(u_int port)
230{
231 u_int data;
232
233 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
234 return (data);
235}
236
237static __inline void
238insb(u_int port, void *addr, size_t count)
239{
240 __asm __volatile("cld; rep; insb"
241 : "+D" (addr), "+c" (count)
242 : "d" (port)
243 : "memory");
244}
245
246static __inline void
247insw(u_int port, void *addr, size_t count)
248{
249 __asm __volatile("cld; rep; insw"
250 : "+D" (addr), "+c" (count)
251 : "d" (port)
252 : "memory");
253}
254
255static __inline void
256insl(u_int port, void *addr, size_t count)
257{
258 __asm __volatile("cld; rep; insl"
259 : "+D" (addr), "+c" (count)
260 : "d" (port)
261 : "memory");
262}
263
264static __inline void
265invd(void)
266{
267 __asm __volatile("invd");
268}
269
270static __inline u_short
271inw(u_int port)
272{
273 u_short data;
274
275 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
276 return (data);
277}
278
279static __inline void
280outb(u_int port, u_char data)
281{
282 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
283}
284
285static __inline void
286outl(u_int port, u_int data)
287{
288 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
289}
290
291static __inline void
292outsb(u_int port, const void *addr, size_t count)
293{
294 __asm __volatile("cld; rep; outsb"
295 : "+S" (addr), "+c" (count)
296 : "d" (port));
297}
298
299static __inline void
300outsw(u_int port, const void *addr, size_t count)
301{
302 __asm __volatile("cld; rep; outsw"
303 : "+S" (addr), "+c" (count)
304 : "d" (port));
305}
306
307static __inline void
308outsl(u_int port, const void *addr, size_t count)
309{
310 __asm __volatile("cld; rep; outsl"
311 : "+S" (addr), "+c" (count)
312 : "d" (port));
313}
314
315static __inline void
316outw(u_int port, u_short data)
317{
318 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
319}
320
321static __inline void
322ia32_pause(void)
323{
324 __asm __volatile("pause");
325}
326
327static __inline u_int
328#ifdef XEN
329_read_eflags(void)
330#else
331read_eflags(void)
332#endif
333{
334 u_int ef;
335
336 __asm __volatile("pushfl; popl %0" : "=r" (ef));
337 return (ef);
338}
339
340static __inline uint64_t
341rdmsr(u_int msr)
342{
343 uint64_t rv;
344
345 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
346 return (rv);
347}
348
349static __inline uint64_t
350rdpmc(u_int pmc)
351{
352 uint64_t rv;
353
354 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
355 return (rv);
356}
357
358static __inline uint64_t
359rdtsc(void)
360{
361 uint64_t rv;
362
363 __asm __volatile("rdtsc" : "=A" (rv));
364 return (rv);
365}
366
367static __inline uint32_t
368rdtsc32(void)
369{
370 uint32_t rv;
371
372 __asm __volatile("rdtsc" : "=a" (rv) : : "edx");
373 return (rv);
374}
375
376static __inline void
377wbinvd(void)
378{
379 __asm __volatile("wbinvd");
380}
381
382static __inline void
383#ifdef XEN
384_write_eflags(u_int ef)
385#else
386write_eflags(u_int ef)
387#endif
388{
389 __asm __volatile("pushl %0; popfl" : : "r" (ef));
390}
391
392static __inline void
393wrmsr(u_int msr, uint64_t newval)
394{
395 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
396}
397
398static __inline void
399load_cr0(u_int data)
400{
401
402 __asm __volatile("movl %0,%%cr0" : : "r" (data));
403}
404
405static __inline u_int
406rcr0(void)
407{
408 u_int data;
409
410 __asm __volatile("movl %%cr0,%0" : "=r" (data));
411 return (data);
412}
413
414static __inline u_int
415rcr2(void)
416{
417 u_int data;
418
419#ifdef XEN
420 return (xen_rcr2());
421#endif
422 __asm __volatile("movl %%cr2,%0" : "=r" (data));
423 return (data);
424}
425
426static __inline void
427load_cr3(u_int data)
428{
429#ifdef XEN
430 xen_load_cr3(data);
431#else
432 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
433#endif
434}
435
436static __inline u_int
437rcr3(void)
438{
439 u_int data;
440
441 __asm __volatile("movl %%cr3,%0" : "=r" (data));
442 return (data);
443}
444
445static __inline void
446load_cr4(u_int data)
447{
448 __asm __volatile("movl %0,%%cr4" : : "r" (data));
449}
450
451static __inline u_int
452rcr4(void)
453{
454 u_int data;
455
456 __asm __volatile("movl %%cr4,%0" : "=r" (data));
457 return (data);
458}
459
460static __inline uint64_t
461rxcr(u_int reg)
462{
463 u_int low, high;
464
465 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
466 return (low | ((uint64_t)high << 32));
467}
468
469static __inline void
470load_xcr(u_int reg, uint64_t val)
471{
472 u_int low, high;
473
474 low = val;
475 high = val >> 32;
476 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
477}
478
479/*
480 * Global TLB flush (except for thise for pages marked PG_G)
481 */
482static __inline void
483invltlb(void)
484{
485#ifdef XEN
486 xen_tlb_flush();
487#else
488 load_cr3(rcr3());
489#endif
490}
491
492/*
493 * TLB flush for an individual page (even if it has PG_G).
494 * Only works on 486+ CPUs (i386 does not have PG_G).
495 */
496static __inline void
497invlpg(u_int addr)
498{
499
500#ifdef XEN
501 xen_invlpg(addr);
502#else
503 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
504#endif
505}
506
507static __inline u_short
508rfs(void)
509{
510 u_short sel;
511 __asm __volatile("movw %%fs,%0" : "=rm" (sel));
512 return (sel);
513}
514
515static __inline uint64_t
516rgdt(void)
517{
518 uint64_t gdtr;
519 __asm __volatile("sgdt %0" : "=m" (gdtr));
520 return (gdtr);
521}
522
523static __inline u_short
524rgs(void)
525{
526 u_short sel;
527 __asm __volatile("movw %%gs,%0" : "=rm" (sel));
528 return (sel);
529}
530
531static __inline uint64_t
532ridt(void)
533{
534 uint64_t idtr;
535 __asm __volatile("sidt %0" : "=m" (idtr));
536 return (idtr);
537}
538
539static __inline u_short
540rldt(void)
541{
542 u_short ldtr;
543 __asm __volatile("sldt %0" : "=g" (ldtr));
544 return (ldtr);
545}
546
547static __inline u_short
548rss(void)
549{
550 u_short sel;
551 __asm __volatile("movw %%ss,%0" : "=rm" (sel));
552 return (sel);
553}
554
555static __inline u_short
556rtr(void)
557{
558 u_short tr;
559 __asm __volatile("str %0" : "=g" (tr));
560 return (tr);
561}
562
563static __inline void
564load_fs(u_short sel)
565{
566 __asm __volatile("movw %0,%%fs" : : "rm" (sel));
567}
568
569static __inline void
570load_gs(u_short sel)
571{
572 __asm __volatile("movw %0,%%gs" : : "rm" (sel));
573}
574
575static __inline void
576lidt(struct region_descriptor *addr)
577{
578 __asm __volatile("lidt (%0)" : : "r" (addr));
579}
580
581static __inline void
582lldt(u_short sel)
583{
584 __asm __volatile("lldt %0" : : "r" (sel));
585}
586
587static __inline void
588ltr(u_short sel)
589{
590 __asm __volatile("ltr %0" : : "r" (sel));
591}
592
593static __inline u_int
594rdr0(void)
595{
596 u_int data;
597 __asm __volatile("movl %%dr0,%0" : "=r" (data));
598 return (data);
599}
600
601static __inline void
602load_dr0(u_int dr0)
603{
604 __asm __volatile("movl %0,%%dr0" : : "r" (dr0));
605}
606
607static __inline u_int
608rdr1(void)
609{
610 u_int data;
611 __asm __volatile("movl %%dr1,%0" : "=r" (data));
612 return (data);
613}
614
615static __inline void
616load_dr1(u_int dr1)
617{
618 __asm __volatile("movl %0,%%dr1" : : "r" (dr1));
619}
620
621static __inline u_int
622rdr2(void)
623{
624 u_int data;
625 __asm __volatile("movl %%dr2,%0" : "=r" (data));
626 return (data);
627}
628
629static __inline void
630load_dr2(u_int dr2)
631{
632 __asm __volatile("movl %0,%%dr2" : : "r" (dr2));
633}
634
635static __inline u_int
636rdr3(void)
637{
638 u_int data;
639 __asm __volatile("movl %%dr3,%0" : "=r" (data));
640 return (data);
641}
642
643static __inline void
644load_dr3(u_int dr3)
645{
646 __asm __volatile("movl %0,%%dr3" : : "r" (dr3));
647}
648
649static __inline u_int
650rdr4(void)
651{
652 u_int data;
653 __asm __volatile("movl %%dr4,%0" : "=r" (data));
654 return (data);
655}
656
657static __inline void
658load_dr4(u_int dr4)
659{
660 __asm __volatile("movl %0,%%dr4" : : "r" (dr4));
661}
662
663static __inline u_int
664rdr5(void)
665{
666 u_int data;
667 __asm __volatile("movl %%dr5,%0" : "=r" (data));
668 return (data);
669}
670
671static __inline void
672load_dr5(u_int dr5)
673{
674 __asm __volatile("movl %0,%%dr5" : : "r" (dr5));
675}
676
677static __inline u_int
678rdr6(void)
679{
680 u_int data;
681 __asm __volatile("movl %%dr6,%0" : "=r" (data));
682 return (data);
683}
684
685static __inline void
686load_dr6(u_int dr6)
687{
688 __asm __volatile("movl %0,%%dr6" : : "r" (dr6));
689}
690
691static __inline u_int
692rdr7(void)
693{
694 u_int data;
695 __asm __volatile("movl %%dr7,%0" : "=r" (data));
696 return (data);
697}
698
699static __inline void
700load_dr7(u_int dr7)
701{
702 __asm __volatile("movl %0,%%dr7" : : "r" (dr7));
703}
704
705static __inline u_char
706read_cyrix_reg(u_char reg)
707{
708 outb(0x22, reg);
709 return inb(0x23);
710}
711
712static __inline void
713write_cyrix_reg(u_char reg, u_char data)
714{
715 outb(0x22, reg);
716 outb(0x23, data);
717}
718
719static __inline register_t
720intr_disable(void)
721{
722 register_t eflags;
723
724 eflags = read_eflags();
725 disable_intr();
726 return (eflags);
727}
728
729static __inline void
730intr_restore(register_t eflags)
731{
732 write_eflags(eflags);
733}
734
735#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
736
737int breakpoint(void);
738u_int bsfl(u_int mask);
739u_int bsrl(u_int mask);
740void clflush(u_long addr);
741void clts(void);
742void cpuid_count(u_int ax, u_int cx, u_int *p);
743void disable_intr(void);
744void do_cpuid(u_int ax, u_int *p);
745void enable_intr(void);
746void halt(void);
747void ia32_pause(void);
748u_char inb(u_int port);
749u_int inl(u_int port);
750void insb(u_int port, void *addr, size_t count);
751void insl(u_int port, void *addr, size_t count);
752void insw(u_int port, void *addr, size_t count);
753register_t intr_disable(void);
754void intr_restore(register_t ef);
755void invd(void);
756void invlpg(u_int addr);
757void invltlb(void);
758u_short inw(u_int port);
759void lidt(struct region_descriptor *addr);
760void lldt(u_short sel);
761void load_cr0(u_int cr0);
762void load_cr3(u_int cr3);
763void load_cr4(u_int cr4);
764void load_dr0(u_int dr0);
765void load_dr1(u_int dr1);
766void load_dr2(u_int dr2);
767void load_dr3(u_int dr3);
768void load_dr4(u_int dr4);
769void load_dr5(u_int dr5);
770void load_dr6(u_int dr6);
771void load_dr7(u_int dr7);
772void load_fs(u_short sel);
773void load_gs(u_short sel);
774void ltr(u_short sel);
775void outb(u_int port, u_char data);
776void outl(u_int port, u_int data);
777void outsb(u_int port, const void *addr, size_t count);
778void outsl(u_int port, const void *addr, size_t count);
779void outsw(u_int port, const void *addr, size_t count);
780void outw(u_int port, u_short data);
781u_int rcr0(void);
782u_int rcr2(void);
783u_int rcr3(void);
784u_int rcr4(void);
785uint64_t rdmsr(u_int msr);
786uint64_t rdpmc(u_int pmc);
787u_int rdr0(void);
788u_int rdr1(void);
789u_int rdr2(void);
790u_int rdr3(void);
791u_int rdr4(void);
792u_int rdr5(void);
793u_int rdr6(void);
794u_int rdr7(void);
795uint64_t rdtsc(void);
796u_char read_cyrix_reg(u_char reg);
797u_int read_eflags(void);
798u_int rfs(void);
799uint64_t rgdt(void);
800u_int rgs(void);
801uint64_t ridt(void);
802u_short rldt(void);
803u_short rtr(void);
804void wbinvd(void);
805void write_cyrix_reg(u_char reg, u_char data);
806void write_eflags(u_int ef);
807void wrmsr(u_int msr, uint64_t newval);
808
809#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
810
811void reset_dbregs(void);
812
813#ifdef _KERNEL
814int rdmsr_safe(u_int msr, uint64_t *val);
815int wrmsr_safe(u_int msr, uint64_t newval);
816#endif
817
818#endif /* !_MACHINE_CPUFUNC_H_ */
2 * Copyright (c) 1993 The Regents of the University of California.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 4. Neither the name of the University nor the names of its contributors
14 * may be used to endorse or promote products derived from this software
15 * without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD: stable/10/sys/i386/include/cpufunc.h 289818 2015-10-23 10:05:43Z avg $
30 */
31
32/*
33 * Functions to provide access to special i386 instructions.
34 * This in included in sys/systm.h, and that file should be
35 * used in preference to this.
36 */
37
38#ifndef _MACHINE_CPUFUNC_H_
39#define _MACHINE_CPUFUNC_H_
40
41#ifndef _SYS_CDEFS_H_
42#error this file needs sys/cdefs.h as a prerequisite
43#endif
44
45#ifdef XEN
46extern void xen_cli(void);
47extern void xen_sti(void);
48extern u_int xen_rcr2(void);
49extern void xen_load_cr3(u_int data);
50extern void xen_tlb_flush(void);
51extern void xen_invlpg(u_int addr);
52extern void write_eflags(u_int eflags);
53extern u_int read_eflags(void);
54#endif
55
56struct region_descriptor;
57
58#define readb(va) (*(volatile uint8_t *) (va))
59#define readw(va) (*(volatile uint16_t *) (va))
60#define readl(va) (*(volatile uint32_t *) (va))
61
62#define writeb(va, d) (*(volatile uint8_t *) (va) = (d))
63#define writew(va, d) (*(volatile uint16_t *) (va) = (d))
64#define writel(va, d) (*(volatile uint32_t *) (va) = (d))
65
66#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
67
68static __inline void
69breakpoint(void)
70{
71 __asm __volatile("int $3");
72}
73
74static __inline u_int
75bsfl(u_int mask)
76{
77 u_int result;
78
79 __asm("bsfl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
80 return (result);
81}
82
83static __inline u_int
84bsrl(u_int mask)
85{
86 u_int result;
87
88 __asm("bsrl %1,%0" : "=r" (result) : "rm" (mask) : "cc");
89 return (result);
90}
91
92static __inline void
93clflush(u_long addr)
94{
95
96 __asm __volatile("clflush %0" : : "m" (*(char *)addr));
97}
98
99static __inline void
100clts(void)
101{
102
103 __asm __volatile("clts");
104}
105
106static __inline void
107disable_intr(void)
108{
109#ifdef XEN
110 xen_cli();
111#else
112 __asm __volatile("cli" : : : "memory");
113#endif
114}
115
116static __inline void
117do_cpuid(u_int ax, u_int *p)
118{
119 __asm __volatile("cpuid"
120 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
121 : "0" (ax));
122}
123
124static __inline void
125cpuid_count(u_int ax, u_int cx, u_int *p)
126{
127 __asm __volatile("cpuid"
128 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
129 : "0" (ax), "c" (cx));
130}
131
132static __inline void
133enable_intr(void)
134{
135#ifdef XEN
136 xen_sti();
137#else
138 __asm __volatile("sti");
139#endif
140}
141
142static __inline void
143cpu_monitor(const void *addr, u_long extensions, u_int hints)
144{
145
146 __asm __volatile("monitor"
147 : : "a" (addr), "c" (extensions), "d" (hints));
148}
149
150static __inline void
151cpu_mwait(u_long extensions, u_int hints)
152{
153
154 __asm __volatile("mwait" : : "a" (hints), "c" (extensions));
155}
156
157static __inline void
158lfence(void)
159{
160
161 __asm __volatile("lfence" : : : "memory");
162}
163
164static __inline void
165mfence(void)
166{
167
168 __asm __volatile("mfence" : : : "memory");
169}
170
171#ifdef _KERNEL
172
173#define HAVE_INLINE_FFS
174
175static __inline int
176ffs(int mask)
177{
178 /*
179 * Note that gcc-2's builtin ffs would be used if we didn't declare
180 * this inline or turn off the builtin. The builtin is faster but
181 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later
182 * versions.
183 */
184 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1);
185}
186
187#define HAVE_INLINE_FFSL
188
189static __inline int
190ffsl(long mask)
191{
192 return (ffs((int)mask));
193}
194
195#define HAVE_INLINE_FLS
196
197static __inline int
198fls(int mask)
199{
200 return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
201}
202
203#define HAVE_INLINE_FLSL
204
205static __inline int
206flsl(long mask)
207{
208 return (fls((int)mask));
209}
210
211#endif /* _KERNEL */
212
213static __inline void
214halt(void)
215{
216 __asm __volatile("hlt");
217}
218
219static __inline u_char
220inb(u_int port)
221{
222 u_char data;
223
224 __asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
225 return (data);
226}
227
228static __inline u_int
229inl(u_int port)
230{
231 u_int data;
232
233 __asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
234 return (data);
235}
236
237static __inline void
238insb(u_int port, void *addr, size_t count)
239{
240 __asm __volatile("cld; rep; insb"
241 : "+D" (addr), "+c" (count)
242 : "d" (port)
243 : "memory");
244}
245
246static __inline void
247insw(u_int port, void *addr, size_t count)
248{
249 __asm __volatile("cld; rep; insw"
250 : "+D" (addr), "+c" (count)
251 : "d" (port)
252 : "memory");
253}
254
255static __inline void
256insl(u_int port, void *addr, size_t count)
257{
258 __asm __volatile("cld; rep; insl"
259 : "+D" (addr), "+c" (count)
260 : "d" (port)
261 : "memory");
262}
263
264static __inline void
265invd(void)
266{
267 __asm __volatile("invd");
268}
269
270static __inline u_short
271inw(u_int port)
272{
273 u_short data;
274
275 __asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
276 return (data);
277}
278
279static __inline void
280outb(u_int port, u_char data)
281{
282 __asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
283}
284
285static __inline void
286outl(u_int port, u_int data)
287{
288 __asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
289}
290
291static __inline void
292outsb(u_int port, const void *addr, size_t count)
293{
294 __asm __volatile("cld; rep; outsb"
295 : "+S" (addr), "+c" (count)
296 : "d" (port));
297}
298
299static __inline void
300outsw(u_int port, const void *addr, size_t count)
301{
302 __asm __volatile("cld; rep; outsw"
303 : "+S" (addr), "+c" (count)
304 : "d" (port));
305}
306
307static __inline void
308outsl(u_int port, const void *addr, size_t count)
309{
310 __asm __volatile("cld; rep; outsl"
311 : "+S" (addr), "+c" (count)
312 : "d" (port));
313}
314
315static __inline void
316outw(u_int port, u_short data)
317{
318 __asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
319}
320
321static __inline void
322ia32_pause(void)
323{
324 __asm __volatile("pause");
325}
326
327static __inline u_int
328#ifdef XEN
329_read_eflags(void)
330#else
331read_eflags(void)
332#endif
333{
334 u_int ef;
335
336 __asm __volatile("pushfl; popl %0" : "=r" (ef));
337 return (ef);
338}
339
340static __inline uint64_t
341rdmsr(u_int msr)
342{
343 uint64_t rv;
344
345 __asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
346 return (rv);
347}
348
349static __inline uint64_t
350rdpmc(u_int pmc)
351{
352 uint64_t rv;
353
354 __asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
355 return (rv);
356}
357
358static __inline uint64_t
359rdtsc(void)
360{
361 uint64_t rv;
362
363 __asm __volatile("rdtsc" : "=A" (rv));
364 return (rv);
365}
366
367static __inline uint32_t
368rdtsc32(void)
369{
370 uint32_t rv;
371
372 __asm __volatile("rdtsc" : "=a" (rv) : : "edx");
373 return (rv);
374}
375
376static __inline void
377wbinvd(void)
378{
379 __asm __volatile("wbinvd");
380}
381
382static __inline void
383#ifdef XEN
384_write_eflags(u_int ef)
385#else
386write_eflags(u_int ef)
387#endif
388{
389 __asm __volatile("pushl %0; popfl" : : "r" (ef));
390}
391
392static __inline void
393wrmsr(u_int msr, uint64_t newval)
394{
395 __asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
396}
397
398static __inline void
399load_cr0(u_int data)
400{
401
402 __asm __volatile("movl %0,%%cr0" : : "r" (data));
403}
404
405static __inline u_int
406rcr0(void)
407{
408 u_int data;
409
410 __asm __volatile("movl %%cr0,%0" : "=r" (data));
411 return (data);
412}
413
414static __inline u_int
415rcr2(void)
416{
417 u_int data;
418
419#ifdef XEN
420 return (xen_rcr2());
421#endif
422 __asm __volatile("movl %%cr2,%0" : "=r" (data));
423 return (data);
424}
425
426static __inline void
427load_cr3(u_int data)
428{
429#ifdef XEN
430 xen_load_cr3(data);
431#else
432 __asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
433#endif
434}
435
436static __inline u_int
437rcr3(void)
438{
439 u_int data;
440
441 __asm __volatile("movl %%cr3,%0" : "=r" (data));
442 return (data);
443}
444
445static __inline void
446load_cr4(u_int data)
447{
448 __asm __volatile("movl %0,%%cr4" : : "r" (data));
449}
450
451static __inline u_int
452rcr4(void)
453{
454 u_int data;
455
456 __asm __volatile("movl %%cr4,%0" : "=r" (data));
457 return (data);
458}
459
460static __inline uint64_t
461rxcr(u_int reg)
462{
463 u_int low, high;
464
465 __asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
466 return (low | ((uint64_t)high << 32));
467}
468
469static __inline void
470load_xcr(u_int reg, uint64_t val)
471{
472 u_int low, high;
473
474 low = val;
475 high = val >> 32;
476 __asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
477}
478
479/*
480 * Global TLB flush (except for thise for pages marked PG_G)
481 */
482static __inline void
483invltlb(void)
484{
485#ifdef XEN
486 xen_tlb_flush();
487#else
488 load_cr3(rcr3());
489#endif
490}
491
492/*
493 * TLB flush for an individual page (even if it has PG_G).
494 * Only works on 486+ CPUs (i386 does not have PG_G).
495 */
496static __inline void
497invlpg(u_int addr)
498{
499
500#ifdef XEN
501 xen_invlpg(addr);
502#else
503 __asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
504#endif
505}
506
507static __inline u_short
508rfs(void)
509{
510 u_short sel;
511 __asm __volatile("movw %%fs,%0" : "=rm" (sel));
512 return (sel);
513}
514
515static __inline uint64_t
516rgdt(void)
517{
518 uint64_t gdtr;
519 __asm __volatile("sgdt %0" : "=m" (gdtr));
520 return (gdtr);
521}
522
523static __inline u_short
524rgs(void)
525{
526 u_short sel;
527 __asm __volatile("movw %%gs,%0" : "=rm" (sel));
528 return (sel);
529}
530
531static __inline uint64_t
532ridt(void)
533{
534 uint64_t idtr;
535 __asm __volatile("sidt %0" : "=m" (idtr));
536 return (idtr);
537}
538
539static __inline u_short
540rldt(void)
541{
542 u_short ldtr;
543 __asm __volatile("sldt %0" : "=g" (ldtr));
544 return (ldtr);
545}
546
547static __inline u_short
548rss(void)
549{
550 u_short sel;
551 __asm __volatile("movw %%ss,%0" : "=rm" (sel));
552 return (sel);
553}
554
555static __inline u_short
556rtr(void)
557{
558 u_short tr;
559 __asm __volatile("str %0" : "=g" (tr));
560 return (tr);
561}
562
563static __inline void
564load_fs(u_short sel)
565{
566 __asm __volatile("movw %0,%%fs" : : "rm" (sel));
567}
568
569static __inline void
570load_gs(u_short sel)
571{
572 __asm __volatile("movw %0,%%gs" : : "rm" (sel));
573}
574
575static __inline void
576lidt(struct region_descriptor *addr)
577{
578 __asm __volatile("lidt (%0)" : : "r" (addr));
579}
580
581static __inline void
582lldt(u_short sel)
583{
584 __asm __volatile("lldt %0" : : "r" (sel));
585}
586
587static __inline void
588ltr(u_short sel)
589{
590 __asm __volatile("ltr %0" : : "r" (sel));
591}
592
593static __inline u_int
594rdr0(void)
595{
596 u_int data;
597 __asm __volatile("movl %%dr0,%0" : "=r" (data));
598 return (data);
599}
600
601static __inline void
602load_dr0(u_int dr0)
603{
604 __asm __volatile("movl %0,%%dr0" : : "r" (dr0));
605}
606
607static __inline u_int
608rdr1(void)
609{
610 u_int data;
611 __asm __volatile("movl %%dr1,%0" : "=r" (data));
612 return (data);
613}
614
615static __inline void
616load_dr1(u_int dr1)
617{
618 __asm __volatile("movl %0,%%dr1" : : "r" (dr1));
619}
620
621static __inline u_int
622rdr2(void)
623{
624 u_int data;
625 __asm __volatile("movl %%dr2,%0" : "=r" (data));
626 return (data);
627}
628
629static __inline void
630load_dr2(u_int dr2)
631{
632 __asm __volatile("movl %0,%%dr2" : : "r" (dr2));
633}
634
635static __inline u_int
636rdr3(void)
637{
638 u_int data;
639 __asm __volatile("movl %%dr3,%0" : "=r" (data));
640 return (data);
641}
642
643static __inline void
644load_dr3(u_int dr3)
645{
646 __asm __volatile("movl %0,%%dr3" : : "r" (dr3));
647}
648
649static __inline u_int
650rdr4(void)
651{
652 u_int data;
653 __asm __volatile("movl %%dr4,%0" : "=r" (data));
654 return (data);
655}
656
657static __inline void
658load_dr4(u_int dr4)
659{
660 __asm __volatile("movl %0,%%dr4" : : "r" (dr4));
661}
662
663static __inline u_int
664rdr5(void)
665{
666 u_int data;
667 __asm __volatile("movl %%dr5,%0" : "=r" (data));
668 return (data);
669}
670
671static __inline void
672load_dr5(u_int dr5)
673{
674 __asm __volatile("movl %0,%%dr5" : : "r" (dr5));
675}
676
677static __inline u_int
678rdr6(void)
679{
680 u_int data;
681 __asm __volatile("movl %%dr6,%0" : "=r" (data));
682 return (data);
683}
684
685static __inline void
686load_dr6(u_int dr6)
687{
688 __asm __volatile("movl %0,%%dr6" : : "r" (dr6));
689}
690
691static __inline u_int
692rdr7(void)
693{
694 u_int data;
695 __asm __volatile("movl %%dr7,%0" : "=r" (data));
696 return (data);
697}
698
699static __inline void
700load_dr7(u_int dr7)
701{
702 __asm __volatile("movl %0,%%dr7" : : "r" (dr7));
703}
704
705static __inline u_char
706read_cyrix_reg(u_char reg)
707{
708 outb(0x22, reg);
709 return inb(0x23);
710}
711
712static __inline void
713write_cyrix_reg(u_char reg, u_char data)
714{
715 outb(0x22, reg);
716 outb(0x23, data);
717}
718
719static __inline register_t
720intr_disable(void)
721{
722 register_t eflags;
723
724 eflags = read_eflags();
725 disable_intr();
726 return (eflags);
727}
728
729static __inline void
730intr_restore(register_t eflags)
731{
732 write_eflags(eflags);
733}
734
735#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
736
737int breakpoint(void);
738u_int bsfl(u_int mask);
739u_int bsrl(u_int mask);
740void clflush(u_long addr);
741void clts(void);
742void cpuid_count(u_int ax, u_int cx, u_int *p);
743void disable_intr(void);
744void do_cpuid(u_int ax, u_int *p);
745void enable_intr(void);
746void halt(void);
747void ia32_pause(void);
748u_char inb(u_int port);
749u_int inl(u_int port);
750void insb(u_int port, void *addr, size_t count);
751void insl(u_int port, void *addr, size_t count);
752void insw(u_int port, void *addr, size_t count);
753register_t intr_disable(void);
754void intr_restore(register_t ef);
755void invd(void);
756void invlpg(u_int addr);
757void invltlb(void);
758u_short inw(u_int port);
759void lidt(struct region_descriptor *addr);
760void lldt(u_short sel);
761void load_cr0(u_int cr0);
762void load_cr3(u_int cr3);
763void load_cr4(u_int cr4);
764void load_dr0(u_int dr0);
765void load_dr1(u_int dr1);
766void load_dr2(u_int dr2);
767void load_dr3(u_int dr3);
768void load_dr4(u_int dr4);
769void load_dr5(u_int dr5);
770void load_dr6(u_int dr6);
771void load_dr7(u_int dr7);
772void load_fs(u_short sel);
773void load_gs(u_short sel);
774void ltr(u_short sel);
775void outb(u_int port, u_char data);
776void outl(u_int port, u_int data);
777void outsb(u_int port, const void *addr, size_t count);
778void outsl(u_int port, const void *addr, size_t count);
779void outsw(u_int port, const void *addr, size_t count);
780void outw(u_int port, u_short data);
781u_int rcr0(void);
782u_int rcr2(void);
783u_int rcr3(void);
784u_int rcr4(void);
785uint64_t rdmsr(u_int msr);
786uint64_t rdpmc(u_int pmc);
787u_int rdr0(void);
788u_int rdr1(void);
789u_int rdr2(void);
790u_int rdr3(void);
791u_int rdr4(void);
792u_int rdr5(void);
793u_int rdr6(void);
794u_int rdr7(void);
795uint64_t rdtsc(void);
796u_char read_cyrix_reg(u_char reg);
797u_int read_eflags(void);
798u_int rfs(void);
799uint64_t rgdt(void);
800u_int rgs(void);
801uint64_t ridt(void);
802u_short rldt(void);
803u_short rtr(void);
804void wbinvd(void);
805void write_cyrix_reg(u_char reg, u_char data);
806void write_eflags(u_int ef);
807void wrmsr(u_int msr, uint64_t newval);
808
809#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
810
811void reset_dbregs(void);
812
813#ifdef _KERNEL
814int rdmsr_safe(u_int msr, uint64_t *val);
815int wrmsr_safe(u_int msr, uint64_t newval);
816#endif
817
818#endif /* !_MACHINE_CPUFUNC_H_ */