cpufunc.h revision 143063 
1/*-
2 * Copyright (c) 1993 The Regents of the University of California.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 * 4. Neither the name of the University nor the names of its contributors
14 *    may be used to endorse or promote products derived from this software
15 *    without specific prior written permission.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 * $FreeBSD: head/sys/i386/include/cpufunc.h 143063 2005-03-02 21:33:29Z joerg $
30 */
31
32/*
33 * Functions to provide access to special i386 instructions.
34 * This in included in sys/systm.h, and that file should be
35 * used in preference to this.
36 */
37
38#ifndef _MACHINE_CPUFUNC_H_
39#define	_MACHINE_CPUFUNC_H_
40
41#ifndef _SYS_CDEFS_H_
42#error this file needs sys/cdefs.h as a prerequisite
43#endif
44
45struct region_descriptor;
46
47#define readb(va)	(*(volatile u_int8_t *) (va))
48#define readw(va)	(*(volatile u_int16_t *) (va))
49#define readl(va)	(*(volatile u_int32_t *) (va))
50
51#define writeb(va, d)	(*(volatile u_int8_t *) (va) = (d))
52#define writew(va, d)	(*(volatile u_int16_t *) (va) = (d))
53#define writel(va, d)	(*(volatile u_int32_t *) (va) = (d))
54
55#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
56
57static __inline void
58breakpoint(void)
59{
60	__asm __volatile("int $3");
61}
62
63static __inline u_int
64bsfl(u_int mask)
65{
66	u_int	result;
67
68	__asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask));
69	return (result);
70}
71
72static __inline u_int
73bsrl(u_int mask)
74{
75	u_int	result;
76
77	__asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask));
78	return (result);
79}
80
81static __inline void
82disable_intr(void)
83{
84	__asm __volatile("cli" : : : "memory");
85}
86
87static __inline void
88do_cpuid(u_int ax, u_int *p)
89{
90	__asm __volatile("cpuid"
91			 : "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
92			 :  "0" (ax));
93}
94
95static __inline void
96enable_intr(void)
97{
98	__asm __volatile("sti");
99}
100
101#ifdef _KERNEL
102
103#define	HAVE_INLINE_FFS
104
105static __inline int
106ffs(int mask)
107{
108	/*
109	 * Note that gcc-2's builtin ffs would be used if we didn't declare
110	 * this inline or turn off the builtin.  The builtin is faster but
111	 * broken in gcc-2.4.5 and slower but working in gcc-2.5 and later
112	 * versions.
113	 */
114	 return (mask == 0 ? mask : (int)bsfl((u_int)mask) + 1);
115}
116
117#define	HAVE_INLINE_FLS
118
119static __inline int
120fls(int mask)
121{
122	return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
123}
124
125#endif /* _KERNEL */
126
127static __inline void
128halt(void)
129{
130	__asm __volatile("hlt");
131}
132
133#if !defined(__GNUCLIKE_BUILTIN_CONSTANT_P) || __GNUCLIKE_ASM < 3
134
135#define	inb(port)		inbv(port)
136#define	outb(port, data)	outbv(port, data)
137
138#else /* __GNUCLIKE_BUILTIN_CONSTANT_P && __GNUCLIKE_ASM >= 3 */
139
140/*
141 * The following complications are to get around gcc not having a
142 * constraint letter for the range 0..255.  We still put "d" in the
143 * constraint because "i" isn't a valid constraint when the port
144 * isn't constant.  This only matters for -O0 because otherwise
145 * the non-working version gets optimized away.
146 *
147 * Use an expression-statement instead of a conditional expression
148 * because gcc-2.6.0 would promote the operands of the conditional
149 * and produce poor code for "if ((inb(var) & const1) == const2)".
150 *
151 * The unnecessary test `(port) < 0x10000' is to generate a warning if
152 * the `port' has type u_short or smaller.  Such types are pessimal.
153 * This actually only works for signed types.  The range check is
154 * careful to avoid generating warnings.
155 */
156#define	inb(port) __extension__ ({					\
157	u_char	_data;							\
158	if (__builtin_constant_p(port) && ((port) & 0xffff) < 0x100	\
159	    && (port) < 0x10000)					\
160		_data = inbc(port);					\
161	else								\
162		_data = inbv(port);					\
163	_data; })
164
165#define	outb(port, data) (						\
166	__builtin_constant_p(port) && ((port) & 0xffff) < 0x100		\
167	&& (port) < 0x10000						\
168	? outbc(port, data) : outbv(port, data))
169
170static __inline u_char
171inbc(u_int port)
172{
173	u_char	data;
174
175	__asm __volatile("inb %1,%0" : "=a" (data) : "id" ((u_short)(port)));
176	return (data);
177}
178
179static __inline void
180outbc(u_int port, u_char data)
181{
182	__asm __volatile("outb %0,%1" : : "a" (data), "id" ((u_short)(port)));
183}
184
185#endif /* __GNUCLIKE_BUILTIN_CONSTANT_P  && __GNUCLIKE_ASM >= 3*/
186
187static __inline u_char
188inbv(u_int port)
189{
190	u_char	data;
191	/*
192	 * We use %%dx and not %1 here because i/o is done at %dx and not at
193	 * %edx, while gcc generates inferior code (movw instead of movl)
194	 * if we tell it to load (u_short) port.
195	 */
196	__asm __volatile("inb %%dx,%0" : "=a" (data) : "d" (port));
197	return (data);
198}
199
200static __inline u_int
201inl(u_int port)
202{
203	u_int	data;
204
205	__asm __volatile("inl %%dx,%0" : "=a" (data) : "d" (port));
206	return (data);
207}
208
209static __inline void
210insb(u_int port, void *addr, size_t cnt)
211{
212	__asm __volatile("cld; rep; insb"
213			 : "+D" (addr), "+c" (cnt)
214			 : "d" (port)
215			 : "memory");
216}
217
218static __inline void
219insw(u_int port, void *addr, size_t cnt)
220{
221	__asm __volatile("cld; rep; insw"
222			 : "+D" (addr), "+c" (cnt)
223			 : "d" (port)
224			 : "memory");
225}
226
227static __inline void
228insl(u_int port, void *addr, size_t cnt)
229{
230	__asm __volatile("cld; rep; insl"
231			 : "+D" (addr), "+c" (cnt)
232			 : "d" (port)
233			 : "memory");
234}
235
236static __inline void
237invd(void)
238{
239	__asm __volatile("invd");
240}
241
242static __inline u_short
243inw(u_int port)
244{
245	u_short	data;
246
247	__asm __volatile("inw %%dx,%0" : "=a" (data) : "d" (port));
248	return (data);
249}
250
251static __inline void
252outbv(u_int port, u_char data)
253{
254	u_char	al;
255	/*
256	 * Use an unnecessary assignment to help gcc's register allocator.
257	 * This make a large difference for gcc-1.40 and a tiny difference
258	 * for gcc-2.6.0.  For gcc-1.40, al had to be ``asm("ax")'' for
259	 * best results.  gcc-2.6.0 can't handle this.
260	 */
261	al = data;
262	__asm __volatile("outb %0,%%dx" : : "a" (al), "d" (port));
263}
264
265static __inline void
266outl(u_int port, u_int data)
267{
268	/*
269	 * outl() and outw() aren't used much so we haven't looked at
270	 * possible micro-optimizations such as the unnecessary
271	 * assignment for them.
272	 */
273	__asm __volatile("outl %0,%%dx" : : "a" (data), "d" (port));
274}
275
276static __inline void
277outsb(u_int port, const void *addr, size_t cnt)
278{
279	__asm __volatile("cld; rep; outsb"
280			 : "+S" (addr), "+c" (cnt)
281			 : "d" (port));
282}
283
284static __inline void
285outsw(u_int port, const void *addr, size_t cnt)
286{
287	__asm __volatile("cld; rep; outsw"
288			 : "+S" (addr), "+c" (cnt)
289			 : "d" (port));
290}
291
292static __inline void
293outsl(u_int port, const void *addr, size_t cnt)
294{
295	__asm __volatile("cld; rep; outsl"
296			 : "+S" (addr), "+c" (cnt)
297			 : "d" (port));
298}
299
300static __inline void
301outw(u_int port, u_short data)
302{
303	__asm __volatile("outw %0,%%dx" : : "a" (data), "d" (port));
304}
305
306static __inline void
307ia32_pause(void)
308{
309	__asm __volatile("pause");
310}
311
312static __inline u_int
313read_eflags(void)
314{
315	u_int	ef;
316
317	__asm __volatile("pushfl; popl %0" : "=r" (ef));
318	return (ef);
319}
320
321static __inline u_int64_t
322rdmsr(u_int msr)
323{
324	u_int64_t rv;
325
326	__asm __volatile("rdmsr" : "=A" (rv) : "c" (msr));
327	return (rv);
328}
329
330static __inline u_int64_t
331rdpmc(u_int pmc)
332{
333	u_int64_t rv;
334
335	__asm __volatile("rdpmc" : "=A" (rv) : "c" (pmc));
336	return (rv);
337}
338
339static __inline u_int64_t
340rdtsc(void)
341{
342	u_int64_t rv;
343
344	__asm __volatile("rdtsc" : "=A" (rv));
345	return (rv);
346}
347
348static __inline void
349wbinvd(void)
350{
351	__asm __volatile("wbinvd");
352}
353
354static __inline void
355write_eflags(u_int ef)
356{
357	__asm __volatile("pushl %0; popfl" : : "r" (ef));
358}
359
360static __inline void
361wrmsr(u_int msr, u_int64_t newval)
362{
363	__asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
364}
365
366static __inline void
367load_cr0(u_int data)
368{
369
370	__asm __volatile("movl %0,%%cr0" : : "r" (data));
371}
372
373static __inline u_int
374rcr0(void)
375{
376	u_int	data;
377
378	__asm __volatile("movl %%cr0,%0" : "=r" (data));
379	return (data);
380}
381
382static __inline u_int
383rcr2(void)
384{
385	u_int	data;
386
387	__asm __volatile("movl %%cr2,%0" : "=r" (data));
388	return (data);
389}
390
391static __inline void
392load_cr3(u_int data)
393{
394
395	__asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
396}
397
398static __inline u_int
399rcr3(void)
400{
401	u_int	data;
402
403	__asm __volatile("movl %%cr3,%0" : "=r" (data));
404	return (data);
405}
406
407static __inline void
408load_cr4(u_int data)
409{
410	__asm __volatile("movl %0,%%cr4" : : "r" (data));
411}
412
413static __inline u_int
414rcr4(void)
415{
416	u_int	data;
417
418	__asm __volatile("movl %%cr4,%0" : "=r" (data));
419	return (data);
420}
421
422/*
423 * Global TLB flush (except for thise for pages marked PG_G)
424 */
425static __inline void
426invltlb(void)
427{
428
429	load_cr3(rcr3());
430}
431
432/*
433 * TLB flush for an individual page (even if it has PG_G).
434 * Only works on 486+ CPUs (i386 does not have PG_G).
435 */
436static __inline void
437invlpg(u_int addr)
438{
439
440	__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
441}
442
443static __inline u_int
444rfs(void)
445{
446	u_int sel;
447	__asm __volatile("movl %%fs,%0" : "=rm" (sel));
448	return (sel);
449}
450
451static __inline u_int
452rgs(void)
453{
454	u_int sel;
455	__asm __volatile("movl %%gs,%0" : "=rm" (sel));
456	return (sel);
457}
458
459static __inline u_int
460rss(void)
461{
462	u_int sel;
463	__asm __volatile("movl %%ss,%0" : "=rm" (sel));
464	return (sel);
465}
466
467static __inline void
468load_fs(u_int sel)
469{
470	__asm __volatile("movl %0,%%fs" : : "rm" (sel));
471}
472
473static __inline void
474load_gs(u_int sel)
475{
476	__asm __volatile("movl %0,%%gs" : : "rm" (sel));
477}
478
479static __inline void
480lidt(struct region_descriptor *addr)
481{
482	__asm __volatile("lidt (%0)" : : "r" (addr));
483}
484
485static __inline void
486lldt(u_short sel)
487{
488	__asm __volatile("lldt %0" : : "r" (sel));
489}
490
491static __inline void
492ltr(u_short sel)
493{
494	__asm __volatile("ltr %0" : : "r" (sel));
495}
496
497static __inline u_int
498rdr0(void)
499{
500	u_int	data;
501	__asm __volatile("movl %%dr0,%0" : "=r" (data));
502	return (data);
503}
504
505static __inline void
506load_dr0(u_int dr0)
507{
508	__asm __volatile("movl %0,%%dr0" : : "r" (dr0));
509}
510
511static __inline u_int
512rdr1(void)
513{
514	u_int	data;
515	__asm __volatile("movl %%dr1,%0" : "=r" (data));
516	return (data);
517}
518
519static __inline void
520load_dr1(u_int dr1)
521{
522	__asm __volatile("movl %0,%%dr1" : : "r" (dr1));
523}
524
525static __inline u_int
526rdr2(void)
527{
528	u_int	data;
529	__asm __volatile("movl %%dr2,%0" : "=r" (data));
530	return (data);
531}
532
533static __inline void
534load_dr2(u_int dr2)
535{
536	__asm __volatile("movl %0,%%dr2" : : "r" (dr2));
537}
538
539static __inline u_int
540rdr3(void)
541{
542	u_int	data;
543	__asm __volatile("movl %%dr3,%0" : "=r" (data));
544	return (data);
545}
546
547static __inline void
548load_dr3(u_int dr3)
549{
550	__asm __volatile("movl %0,%%dr3" : : "r" (dr3));
551}
552
553static __inline u_int
554rdr4(void)
555{
556	u_int	data;
557	__asm __volatile("movl %%dr4,%0" : "=r" (data));
558	return (data);
559}
560
561static __inline void
562load_dr4(u_int dr4)
563{
564	__asm __volatile("movl %0,%%dr4" : : "r" (dr4));
565}
566
567static __inline u_int
568rdr5(void)
569{
570	u_int	data;
571	__asm __volatile("movl %%dr5,%0" : "=r" (data));
572	return (data);
573}
574
575static __inline void
576load_dr5(u_int dr5)
577{
578	__asm __volatile("movl %0,%%dr5" : : "r" (dr5));
579}
580
581static __inline u_int
582rdr6(void)
583{
584	u_int	data;
585	__asm __volatile("movl %%dr6,%0" : "=r" (data));
586	return (data);
587}
588
589static __inline void
590load_dr6(u_int dr6)
591{
592	__asm __volatile("movl %0,%%dr6" : : "r" (dr6));
593}
594
595static __inline u_int
596rdr7(void)
597{
598	u_int	data;
599	__asm __volatile("movl %%dr7,%0" : "=r" (data));
600	return (data);
601}
602
603static __inline void
604load_dr7(u_int dr7)
605{
606	__asm __volatile("movl %0,%%dr7" : : "r" (dr7));
607}
608
609static __inline register_t
610intr_disable(void)
611{
612	register_t eflags;
613
614	eflags = read_eflags();
615	disable_intr();
616	return (eflags);
617}
618
619static __inline void
620intr_restore(register_t eflags)
621{
622	write_eflags(eflags);
623}
624
625#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
626
627int	breakpoint(void);
628u_int	bsfl(u_int mask);
629u_int	bsrl(u_int mask);
630void	disable_intr(void);
631void	do_cpuid(u_int ax, u_int *p);
632void	enable_intr(void);
633void	halt(void);
634void	ia32_pause(void);
635u_char	inb(u_int port);
636u_int	inl(u_int port);
637void	insb(u_int port, void *addr, size_t cnt);
638void	insl(u_int port, void *addr, size_t cnt);
639void	insw(u_int port, void *addr, size_t cnt);
640register_t	intr_disable(void);
641void	intr_restore(register_t ef);
642void	invd(void);
643void	invlpg(u_int addr);
644void	invltlb(void);
645u_short	inw(u_int port);
646void	lidt(struct region_descriptor *addr);
647void	lldt(u_short sel);
648void	load_cr0(u_int cr0);
649void	load_cr3(u_int cr3);
650void	load_cr4(u_int cr4);
651void	load_dr0(u_int dr0);
652void	load_dr1(u_int dr1);
653void	load_dr2(u_int dr2);
654void	load_dr3(u_int dr3);
655void	load_dr4(u_int dr4);
656void	load_dr5(u_int dr5);
657void	load_dr6(u_int dr6);
658void	load_dr7(u_int dr7);
659void	load_fs(u_int sel);
660void	load_gs(u_int sel);
661void	ltr(u_short sel);
662void	outb(u_int port, u_char data);
663void	outl(u_int port, u_int data);
664void	outsb(u_int port, const void *addr, size_t cnt);
665void	outsl(u_int port, const void *addr, size_t cnt);
666void	outsw(u_int port, const void *addr, size_t cnt);
667void	outw(u_int port, u_short data);
668u_int	rcr0(void);
669u_int	rcr2(void);
670u_int	rcr3(void);
671u_int	rcr4(void);
672u_int64_t rdmsr(u_int msr);
673u_int64_t rdpmc(u_int pmc);
674u_int	rdr0(void);
675u_int	rdr1(void);
676u_int	rdr2(void);
677u_int	rdr3(void);
678u_int	rdr4(void);
679u_int	rdr5(void);
680u_int	rdr6(void);
681u_int	rdr7(void);
682u_int64_t rdtsc(void);
683u_int	read_eflags(void);
684u_int	rfs(void);
685u_int	rgs(void);
686void	wbinvd(void);
687void	write_eflags(u_int ef);
688void	wrmsr(u_int msr, u_int64_t newval);
689
690#endif	/* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
691
692void    reset_dbregs(void);
693
694#endif /* !_MACHINE_CPUFUNC_H_ */
695