cpufunc.h revision 187948 
1/*-
2 * Copyright (c) 1993 The Regents of the University of California.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 * 4. Neither the name of the University nor the names of its contributors
14 *    may be used to endorse or promote products derived from this software
15 *    without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD: head/sys/i386/include/cpufunc.h 187948 2009-01-31 11:37:21Z obrien $
30 */
31
32/*
33 * Functions to provide access to special i386 instructions.
34 * This in included in sys/systm.h, and that file should be
35 * used in preference to this.
36 */
37
38#ifndef _MACHINE_CPUFUNC_H_
39#define	_MACHINE_CPUFUNC_H_
40
41#ifndef _SYS_CDEFS_H_
42#error this file needs sys/cdefs.h as a prerequisite
43#endif
44
45#ifdef XEN
46extern void xen_cli(void);
47extern void xen_sti(void);
48extern u_int xen_rcr2(void);
49extern void xen_load_cr3(u_int data);
50extern void xen_tlb_flush(void);
51extern void xen_invlpg(u_int addr);
52extern int xen_save_and_cli(void);
53extern void xen_restore_flags(u_int eflags);
54#endif
55
56struct region_descriptor;
57
58#define readb(va)	(*(volatile u_int8_t *) (va))
59#define readw(va)	(*(volatile u_int16_t *) (va))
60#define readl(va)	(*(volatile u_int32_t *) (va))
61
62#define writeb(va, d)	(*(volatile u_int8_t *) (va) = (d))
63#define writew(va, d)	(*(volatile u_int16_t *) (va) = (d))
64#define writel(va, d)	(*(volatile u_int32_t *) (va) = (d))
65
66#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
67
68static __inline void
69breakpoint(void)
70{
71	__asm __volatile("int $3");
72}
73
74static __inline u_int
75bsfl(u_int mask)
76{
77	u_int	result;
78
79	__asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask));
80	return (result);
81}
82
83static __inline u_int
84bsrl(u_int mask)
85{
86	u_int	result;
87
88	__asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask));
89	return (result);
90}
91
92static __inline void
93disable_intr(void)
94{
95#ifdef XEN
96	xen_cli();
97#else
98	__asm __volatile("cli" : : : "memory");
99#endif
100}
101
102static __inline void
103do_cpuid(u_int ax, u_int *p)
104{
105	__asm __volatile("cpuid"
106			 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
107			 :  "0" (ax));
108}
109
110static __inline void
111cpuid_count(u_int ax, u_int cx, u_int *p)
112{
113	__asm __volatile("cpuid"
114			 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
115			 :  "0" (ax), "c" (cx));
116}
117
118static __inline void
119enable_intr(void)
120{
121#ifdef XEN
122	xen_sti();
123#else
124	__asm __volatile("sti");
125#endif
126}
127
128static inline void
129cpu_monitor(const void *addr, int extensions, int hints)
130{
131	__asm __volatile("monitor;"
132	    : :"a" (addr), "c" (extensions), "d"(hints));
133}
134
135static inline void
136cpu_mwait(int extensions, int hints)
137{
138	__asm __volatile("mwait;" : :"a" (hints), "c" (extensions));
139}
140
141#ifdef _KERNEL
142
143#define	HAVE_INLINE_FFS
144
145static __inline int
146ffs(int mask)
147{
148	/*
149	 * Note that gcc-2's builtin ffs would be used if we didn't declare
150	 * this inline or turn off the builtin.  The builtin is faster but
151	 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later
152	 * versions.
153	 */
154	 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1);
155}
156
157#define	HAVE_INLINE_FLS
158
159static __inline int
160fls(int mask)
161{
162	return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
163}
164
165#endif /* _KERNEL */
166
167static __inline void
168halt(void)
169{
170	__asm __volatile("hlt");
171}
172
173#if !defined(__GNUCLIKE_BUILTIN_CONSTANT_P) || __GNUCLIKE_ASM < 3
174
175#define	inb(port)		inbv(port)
176#define	outb(port, data)	outbv(port, data)
177
178#else /* __GNUCLIKE_BUILTIN_CONSTANT_P && __GNUCLIKE_ASM >= 3 */
179
180/*
181 * The following complications are to get around gcc not having a
182 * constraint letter for the range 0..255.  We still put "d" in the
183 * constraint because "i" isn't a valid constraint when the port
184 * isn't constant.  This only matters for -O0 because otherwise
185 * the non-working version gets optimized away.
186 *
187 * Use an expression-statement instead of a conditional expression
188 * because gcc-2.6.0 would promote the operands of the conditional
189 * and produce poor code for "if ((inb(var) & const1) == const2)".
190 *
191 * The unnecessary test `(port) < 0x10000' is to generate a warning if
192 * the `port' has type u_short or smaller.  Such types are pessimal.
193 * This actually only works for signed types.  The range check is
194 * careful to avoid generating warnings.
195 */
196#define	inb(port) __extension__ ({					\
197	u_char	_data;							\
198	if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100	\
199	    && (port) < 0x10000)					\
200		_data = inbc(port);					\
201	else								\
202		_data = inbv(port);					\
203	_data; })
204
205#define	outb(port, data) (						\
206	__builtin_constant_p(port) && ((port) & 0xffff) < 0x100		\
207	&& (port) < 0x10000						\
208	? outbc(port, data) : outbv(port, data))
209
210static __inline u_char
211inbc(u_int port)
212{
213	u_char	data;
214
215	__asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port)));
216	return (data);
217}
218
219static __inline void
220outbc(u_int port, u_char data)
221{
222	__asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port)));
223}
224
225#endif /* __GNUCLIKE_BUILTIN_CONSTANT_P  && __GNUCLIKE_ASM >= 3*/
226
227static __inline u_char
228inbv(u_int port)
229{
230	u_char	data;
231	/*
232	 * We use %%dx and not %1 here because i/o is done at %dx and not at
233	 * %edx, while gcc generates inferior code (movw instead of movl)
234	 * if we tell it to load (u_short) port.
235	 */
236	__asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port));
237	return (data);
238}
239
240static __inline u_int
241inl(u_int port)
242{
243	u_int	data;
244
245	__asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port));
246	return (data);
247}
248
249static __inline void
250insb(u_int port, void *addr, size_t cnt)
251{
252	__asm __volatile("cld; rep; insb"
253			 : "+D" (addr), "+c" (cnt)
254			 : "d" (port)
255			 : "memory");
256}
257
258static __inline void
259insw(u_int port, void *addr, size_t cnt)
260{
261	__asm __volatile("cld; rep; insw"
262			 : "+D" (addr), "+c" (cnt)
263			 : "d" (port)
264			 : "memory");
265}
266
267static __inline void
268insl(u_int port, void *addr, size_t cnt)
269{
270	__asm __volatile("cld; rep; insl"
271			 : "+D" (addr), "+c" (cnt)
272			 : "d" (port)
273			 : "memory");
274}
275
276static __inline void
277invd(void)
278{
279	__asm __volatile("invd");
280}
281
282static __inline u_short
283inw(u_int port)
284{
285	u_short	data;
286
287	__asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port));
288	return (data);
289}
290
291static __inline void
292outbv(u_int port, u_char data)
293{
294	u_char	al;
295	/*
296	 * Use an unnecessary assignment to help gcc's register allocator.
297	 * This make a large difference for gcc-1.40 and a tiny difference
298	 * for gcc-2.6.0.  For gcc-1.40, al had to be ``asm("ax")'' for
299	 * best results.  gcc-2.6.0 can't handle this.
300	 */
301	al = data;
302	__asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port));
303}
304
305static __inline void
306outl(u_int port, u_int data)
307{
308	/*
309	 * outl() and outw() aren't used much so we haven't looked at
310	 * possible micro-optimizations such as the unnecessary
311	 * assignment for them.
312	 */
313	__asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port));
314}
315
316static __inline void
317outsb(u_int port, const void *addr, size_t cnt)
318{
319	__asm __volatile("cld; rep; outsb"
320			 : "+S" (addr), "+c" (cnt)
321			 : "d" (port));
322}
323
324static __inline void
325outsw(u_int port, const void *addr, size_t cnt)
326{
327	__asm __volatile("cld; rep; outsw"
328			 : "+S" (addr), "+c" (cnt)
329			 : "d" (port));
330}
331
332static __inline void
333outsl(u_int port, const void *addr, size_t cnt)
334{
335	__asm __volatile("cld; rep; outsl"
336			 : "+S" (addr), "+c" (cnt)
337			 : "d" (port));
338}
339
340static __inline void
341outw(u_int port, u_short data)
342{
343	__asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port));
344}
345
346static __inline void
347ia32_pause(void)
348{
349	__asm __volatile("pause");
350}
351
352static __inline u_int
353read_eflags(void)
354{
355	u_int	ef;
356
357	__asm __volatile("pushfl; popl %0" : "=r" (ef));
358	return (ef);
359}
360
361static __inline uint64_t
362rdmsr(u_int msr)
363{
364	uint64_t rv;
365
366	__asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
367	return (rv);
368}
369
370static __inline uint64_t
371rdpmc(u_int pmc)
372{
373	uint64_t rv;
374
375	__asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
376	return (rv);
377}
378
379static __inline uint64_t
380rdtsc(void)
381{
382	uint64_t rv;
383
384	__asm __volatile("rdtsc" : "=A" (rv));
385	return (rv);
386}
387
388static __inline void
389wbinvd(void)
390{
391	__asm __volatile("wbinvd");
392}
393
394static __inline void
395write_eflags(u_int ef)
396{
397	__asm __volatile("pushl %0; popfl" : : "r" (ef));
398}
399
400static __inline void
401wrmsr(u_int msr, uint64_t newval)
402{
403	__asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
404}
405
406static __inline void
407load_cr0(u_int data)
408{
409
410	__asm __volatile("movl %0,%%cr0" : : "r" (data));
411}
412
413static __inline u_int
414rcr0(void)
415{
416	u_int	data;
417
418	__asm __volatile("movl %%cr0,%0" : "=r" (data));
419	return (data);
420}
421
422static __inline u_int
423rcr2(void)
424{
425	u_int	data;
426
427#ifdef XEN
428	return (xen_rcr2());
429#endif
430	__asm __volatile("movl %%cr2,%0" : "=r" (data));
431	return (data);
432}
433
434static __inline void
435load_cr3(u_int data)
436{
437#ifdef XEN
438	xen_load_cr3(data);
439#else
440	__asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
441#endif
442}
443
444static __inline u_int
445rcr3(void)
446{
447	u_int	data;
448
449	__asm __volatile("movl %%cr3,%0" : "=r" (data));
450	return (data);
451}
452
453static __inline void
454load_cr4(u_int data)
455{
456	__asm __volatile("movl %0,%%cr4" : : "r" (data));
457}
458
459static __inline u_int
460rcr4(void)
461{
462	u_int	data;
463
464	__asm __volatile("movl %%cr4,%0" : "=r" (data));
465	return (data);
466}
467
468/*
469 * Global TLB flush (except for thise for pages marked PG_G)
470 */
471static __inline void
472invltlb(void)
473{
474#ifdef XEN
475	xen_tlb_flush();
476#else
477	load_cr3(rcr3());
478#endif
479}
480
481/*
482 * TLB flush for an individual page (even if it has PG_G).
483 * Only works on 486+ CPUs (i386 does not have PG_G).
484 */
485static __inline void
486invlpg(u_int addr)
487{
488
489#ifdef XEN
490	xen_invlpg(addr);
491#else
492	__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
493#endif
494}
495
496static __inline u_int
497rfs(void)
498{
499	u_int sel;
500	__asm __volatile("mov %%fs,%0" : "=rm" (sel));
501	return (sel);
502}
503
504static __inline uint64_t
505rgdt(void)
506{
507	uint64_t gdtr;
508	__asm __volatile("sgdt %0" : "=m" (gdtr));
509	return (gdtr);
510}
511
512static __inline u_int
513rgs(void)
514{
515	u_int sel;
516	__asm __volatile("mov %%gs,%0" : "=rm" (sel));
517	return (sel);
518}
519
520static __inline uint64_t
521ridt(void)
522{
523	uint64_t idtr;
524	__asm __volatile("sidt %0" : "=m" (idtr));
525	return (idtr);
526}
527
528static __inline u_short
529rldt(void)
530{
531	u_short ldtr;
532	__asm __volatile("sldt %0" : "=g" (ldtr));
533	return (ldtr);
534}
535
536static __inline u_int
537rss(void)
538{
539	u_int sel;
540	__asm __volatile("mov %%ss,%0" : "=rm" (sel));
541	return (sel);
542}
543
544static __inline u_short
545rtr(void)
546{
547	u_short tr;
548	__asm __volatile("str %0" : "=g" (tr));
549	return (tr);
550}
551
552static __inline void
553load_fs(u_int sel)
554{
555	__asm __volatile("mov %0,%%fs" : : "rm" (sel));
556}
557
558static __inline void
559load_gs(u_int sel)
560{
561	__asm __volatile("mov %0,%%gs" : : "rm" (sel));
562}
563
564static __inline void
565lidt(struct region_descriptor *addr)
566{
567	__asm __volatile("lidt (%0)" : : "r" (addr));
568}
569
570static __inline void
571lldt(u_short sel)
572{
573	__asm __volatile("lldt %0" : : "r" (sel));
574}
575
576static __inline void
577ltr(u_short sel)
578{
579	__asm __volatile("ltr %0" : : "r" (sel));
580}
581
582static __inline u_int
583rdr0(void)
584{
585	u_int	data;
586	__asm __volatile("movl %%dr0,%0" : "=r" (data));
587	return (data);
588}
589
590static __inline void
591load_dr0(u_int dr0)
592{
593	__asm __volatile("movl %0,%%dr0" : : "r" (dr0));
594}
595
596static __inline u_int
597rdr1(void)
598{
599	u_int	data;
600	__asm __volatile("movl %%dr1,%0" : "=r" (data));
601	return (data);
602}
603
604static __inline void
605load_dr1(u_int dr1)
606{
607	__asm __volatile("movl %0,%%dr1" : : "r" (dr1));
608}
609
610static __inline u_int
611rdr2(void)
612{
613	u_int	data;
614	__asm __volatile("movl %%dr2,%0" : "=r" (data));
615	return (data);
616}
617
618static __inline void
619load_dr2(u_int dr2)
620{
621	__asm __volatile("movl %0,%%dr2" : : "r" (dr2));
622}
623
624static __inline u_int
625rdr3(void)
626{
627	u_int	data;
628	__asm __volatile("movl %%dr3,%0" : "=r" (data));
629	return (data);
630}
631
632static __inline void
633load_dr3(u_int dr3)
634{
635	__asm __volatile("movl %0,%%dr3" : : "r" (dr3));
636}
637
638static __inline u_int
639rdr4(void)
640{
641	u_int	data;
642	__asm __volatile("movl %%dr4,%0" : "=r" (data));
643	return (data);
644}
645
646static __inline void
647load_dr4(u_int dr4)
648{
649	__asm __volatile("movl %0,%%dr4" : : "r" (dr4));
650}
651
652static __inline u_int
653rdr5(void)
654{
655	u_int	data;
656	__asm __volatile("movl %%dr5,%0" : "=r" (data));
657	return (data);
658}
659
660static __inline void
661load_dr5(u_int dr5)
662{
663	__asm __volatile("movl %0,%%dr5" : : "r" (dr5));
664}
665
666static __inline u_int
667rdr6(void)
668{
669	u_int	data;
670	__asm __volatile("movl %%dr6,%0" : "=r" (data));
671	return (data);
672}
673
674static __inline void
675load_dr6(u_int dr6)
676{
677	__asm __volatile("movl %0,%%dr6" : : "r" (dr6));
678}
679
680static __inline u_int
681rdr7(void)
682{
683	u_int	data;
684	__asm __volatile("movl %%dr7,%0" : "=r" (data));
685	return (data);
686}
687
688static __inline void
689load_dr7(u_int dr7)
690{
691	__asm __volatile("movl %0,%%dr7" : : "r" (dr7));
692}
693
694static __inline register_t
695intr_disable(void)
696{
697	register_t eflags;
698
699#ifdef XEN
700	eflags = xen_save_and_cli();
701#else
702	eflags = read_eflags();
703	disable_intr();
704#endif
705	return (eflags);
706}
707
708static __inline void
709intr_restore(register_t eflags)
710{
711#ifdef XEN
712	xen_restore_flags(eflags);
713#else
714	write_eflags(eflags);
715#endif
716}
717
718#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
719
720int	breakpoint(void);
721u_int	bsfl(u_int mask);
722u_int	bsrl(u_int mask);
723void	disable_intr(void);
724void	do_cpuid(u_int ax, u_int *p);
725void	enable_intr(void);
726void	halt(void);
727void	ia32_pause(void);
728u_char	inb(u_int port);
729u_int	inl(u_int port);
730void	insb(u_int port, void *addr, size_t cnt);
731void	insl(u_int port, void *addr, size_t cnt);
732void	insw(u_int port, void *addr, size_t cnt);
733register_t	intr_disable(void);
734void	intr_restore(register_t ef);
735void	invd(void);
736void	invlpg(u_int addr);
737void	invltlb(void);
738u_short	inw(u_int port);
739void	lidt(struct region_descriptor *addr);
740void	lldt(u_short sel);
741void	load_cr0(u_int cr0);
742void	load_cr3(u_int cr3);
743void	load_cr4(u_int cr4);
744void	load_dr0(u_int dr0);
745void	load_dr1(u_int dr1);
746void	load_dr2(u_int dr2);
747void	load_dr3(u_int dr3);
748void	load_dr4(u_int dr4);
749void	load_dr5(u_int dr5);
750void	load_dr6(u_int dr6);
751void	load_dr7(u_int dr7);
752void	load_fs(u_int sel);
753void	load_gs(u_int sel);
754void	ltr(u_short sel);
755void	outb(u_int port, u_char data);
756void	outl(u_int port, u_int data);
757void	outsb(u_int port, const void *addr, size_t cnt);
758void	outsl(u_int port, const void *addr, size_t cnt);
759void	outsw(u_int port, const void *addr, size_t cnt);
760void	outw(u_int port, u_short data);
761u_int	rcr0(void);
762u_int	rcr2(void);
763u_int	rcr3(void);
764u_int	rcr4(void);
765uint64_t rdmsr(u_int msr);
766uint64_t rdpmc(u_int pmc);
767u_int	rdr0(void);
768u_int	rdr1(void);
769u_int	rdr2(void);
770u_int	rdr3(void);
771u_int	rdr4(void);
772u_int	rdr5(void);
773u_int	rdr6(void);
774u_int	rdr7(void);
775uint64_t rdtsc(void);
776u_int	read_eflags(void);
777u_int	rfs(void);
778uint64_t rgdt(void);
779u_int	rgs(void);
780uint64_t ridt(void);
781u_short	rldt(void);
782u_short	rtr(void);
783void	wbinvd(void);
784void	write_eflags(u_int ef);
785void	wrmsr(u_int msr, uint64_t newval);
786
787#endif	/* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
788
789void    reset_dbregs(void);
790
791#ifdef _KERNEL
792int	rdmsr_safe(u_int msr, uint64_t *val);
793int	wrmsr_safe(u_int msr, uint64_t newval);
794#endif
795
796#endif /* !_MACHINE_CPUFUNC_H_ */
797