altivec.c revision 1.24 
1/*	$NetBSD: altivec.c,v 1.24 2011/05/25 05:42:37 matt Exp $	*/
2
3/*
4 * Copyright (C) 1996 Wolfgang Solfrank.
5 * Copyright (C) 1996 TooLs GmbH.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 *    must display the following acknowledgement:
18 *	This product includes software developed by TooLs GmbH.
19 * 4. The name of TooLs GmbH may not be used to endorse or promote products
20 *    derived from this software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
28 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
29 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
30 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
31 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <sys/cdefs.h>
35__KERNEL_RCSID(0, "$NetBSD: altivec.c,v 1.24 2011/05/25 05:42:37 matt Exp $");
36
37#include "opt_multiprocessor.h"
38
39#include <sys/param.h>
40#include <sys/proc.h>
41#include <sys/systm.h>
42#include <sys/atomic.h>
43
44#include <uvm/uvm_extern.h>		/*  for vcopypage/vzeropage */
45
46#include <powerpc/pcb.h>
47#include <powerpc/altivec.h>
48#include <powerpc/spr.h>
49#include <powerpc/oea/spr.h>
50#include <powerpc/psl.h>
51
52static void vec_state_load(lwp_t *, bool);
53static void vec_state_save(lwp_t *);
54static void vec_state_release(lwp_t *);
55
56const pcu_ops_t vec_ops = {
57	.pcu_id = PCU_VEC,
58	.pcu_state_load = vec_state_load,
59	.pcu_state_save = vec_state_save,
60	.pcu_state_release = vec_state_release,
61};
62
63bool
64vec_used_p(lwp_t *l)
65{
66	return (l->l_md.md_flags & MDLWP_USEDVEC) != 0;
67}
68
69void
70vec_mark_used(lwp_t *l)
71{
72	l->l_md.md_flags |= MDLWP_USEDVEC;
73}
74
75void
76vec_state_load(lwp_t *l, bool used)
77{
78	struct pcb * const pcb = lwp_getpcb(l);
79
80	/*
81	 * Enable AltiVec temporarily (and disable interrupts).
82	 */
83	const register_t msr = mfmsr();
84	mtmsr((msr & ~PSL_EE) | PSL_VEC);
85	__asm volatile ("isync");
86
87	/*
88	 * Load the vector unit from vreg which is best done in
89	 * assembly.
90	 */
91	vec_load_from_vreg(&pcb->pcb_vr);
92
93	/*
94	 * VRSAVE will be restored when trap frame returns
95	 */
96	l->l_md.md_utf->tf_vrsave = pcb->pcb_vr.vrsave;
97
98	/*
99	 * Restore MSR (turn off AltiVec)
100	 */
101	mtmsr(msr);
102	__asm volatile ("isync");
103
104	/*
105	 * Mark vector registers as modified.
106	 */
107	l->l_md.md_flags |= MDLWP_USEDVEC|PSL_VEC;
108	l->l_md.md_utf->tf_srr1 |= PSL_VEC;
109}
110
111void
112vec_state_save(lwp_t *l)
113{
114	struct pcb * const pcb = lwp_getpcb(l);
115
116	/*
117	 * Turn on AltiVEC, turn off interrupts.
118	 */
119	const register_t msr = mfmsr();
120	mtmsr((msr & ~PSL_EE) | PSL_VEC);
121	__asm volatile ("isync");
122
123	/*
124	 * Grab contents of vector unit.
125	 */
126	vec_unload_to_vreg(&pcb->pcb_vr);
127
128	/*
129	 * Save VRSAVE
130	 */
131	pcb->pcb_vr.vrsave = l->l_md.md_utf->tf_vrsave;
132
133	/*
134	 * Note that we aren't using any CPU resources and stop any
135	 * data streams.
136	 */
137	__asm volatile ("dssall; sync");
138
139	/*
140	 * Restore MSR (turn off AltiVec)
141	 */
142	mtmsr(msr);
143	__asm volatile ("isync");
144}
145
146void
147vec_state_release(lwp_t *l)
148{
149	__asm volatile("dssall;sync");
150	l->l_md.md_utf->tf_srr1 &= ~PSL_VEC;
151	l->l_md.md_flags &= ~PSL_VEC;
152}
153
154void
155vec_restore_from_mcontext(struct lwp *l, const mcontext_t *mcp)
156{
157	struct pcb * const pcb = lwp_getpcb(l);
158
159	KASSERT(l == curlwp);
160
161	/* we don't need to save the state, just drop it */
162	pcu_discard(&vec_ops);
163	memcpy(pcb->pcb_vr.vreg, &mcp->__vrf.__vrs, sizeof (pcb->pcb_vr.vreg));
164	pcb->pcb_vr.vscr = mcp->__vrf.__vscr;
165	pcb->pcb_vr.vrsave = mcp->__vrf.__vrsave;
166	l->l_md.md_utf->tf_vrsave = pcb->pcb_vr.vrsave;
167}
168
169bool
170vec_save_to_mcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flagp)
171{
172	struct pcb * const pcb = lwp_getpcb(l);
173
174	KASSERT(l == curlwp);
175
176	/* Save AltiVec context, if any. */
177	if (!vec_used_p(l))
178		return false;
179
180	/*
181	 * If we're the AltiVec owner, dump its context to the PCB first.
182	 */
183	pcu_save(&vec_ops);
184
185	mcp->__gregs[_REG_MSR] |= PSL_VEC;
186	mcp->__vrf.__vscr = pcb->pcb_vr.vscr;
187	mcp->__vrf.__vrsave = l->l_md.md_utf->tf_vrsave;
188	memcpy(mcp->__vrf.__vrs, pcb->pcb_vr.vreg, sizeof (mcp->__vrf.__vrs));
189	*flagp |= _UC_POWERPC_VEC;
190	return true;
191}
192
193#define ZERO_VEC	19
194
195void
196vzeropage(paddr_t pa)
197{
198	const paddr_t ea = pa + PAGE_SIZE;
199	uint32_t vec[7], *vp = (void *) roundup((uintptr_t) vec, 16);
200	register_t omsr, msr;
201
202	__asm volatile("mfmsr %0" : "=r"(omsr) :);
203
204	/*
205	 * Turn on AltiVec, turn off interrupts.
206	 */
207	msr = (omsr & ~PSL_EE) | PSL_VEC;
208	__asm volatile("sync; mtmsr %0; isync" :: "r"(msr));
209
210	/*
211	 * Save the VEC register we are going to use before we disable
212	 * relocation.
213	 */
214	__asm("stvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC));
215	__asm("vxor %0,%0,%0" :: "n"(ZERO_VEC));
216
217	/*
218	 * Zero the page using a single cache line.
219	 */
220	__asm volatile(
221	    "   sync ;"
222	    "   mfmsr  %[msr];"
223	    "   rlwinm %[msr],%[msr],0,28,26;"	/* Clear PSL_DR */
224	    "   mtmsr  %[msr];"			/* Turn off DMMU */
225	    "   isync;"
226	    "1: stvx   %[zv], %[pa], %[off0];"
227	    "   stvxl  %[zv], %[pa], %[off16];"
228	    "   stvx   %[zv], %[pa], %[off32];"
229	    "   stvxl  %[zv], %[pa], %[off48];"
230	    "   addi   %[pa], %[pa], 64;"
231	    "   cmplw  %[pa], %[ea];"
232	    "	blt+   1b;"
233	    "   ori    %[msr], %[msr], 0x10;"	/* Set PSL_DR */
234	    "   sync;"
235	    "	mtmsr  %[msr];"			/* Turn on DMMU */
236	    "   isync;"
237	    :: [msr] "r"(msr), [pa] "b"(pa), [ea] "b"(ea),
238	    [off0] "r"(0), [off16] "r"(16), [off32] "r"(32), [off48] "r"(48),
239	    [zv] "n"(ZERO_VEC));
240
241	/*
242	 * Restore VEC register (now that we can access the stack again).
243	 */
244	__asm("lvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC));
245
246	/*
247	 * Restore old MSR (AltiVec OFF).
248	 */
249	__asm volatile("sync; mtmsr %0; isync" :: "r"(omsr));
250}
251
252#define LO_VEC	16
253#define HI_VEC	17
254
255void
256vcopypage(paddr_t dst, paddr_t src)
257{
258	const paddr_t edst = dst + PAGE_SIZE;
259	uint32_t vec[11], *vp = (void *) roundup((uintptr_t) vec, 16);
260	register_t omsr, msr;
261
262	__asm volatile("mfmsr %0" : "=r"(omsr) :);
263
264	/*
265	 * Turn on AltiVec, turn off interrupts.
266	 */
267	msr = (omsr & ~PSL_EE) | PSL_VEC;
268	__asm volatile("sync; mtmsr %0; isync" :: "r"(msr));
269
270	/*
271	 * Save the VEC registers we will be using before we disable
272	 * relocation.
273	 */
274	__asm("stvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC));
275	__asm("stvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC));
276
277	/*
278	 * Copy the page using a single cache line, with DMMU
279	 * disabled.  On most PPCs, two vector registers occupy one
280	 * cache line.
281	 */
282	__asm volatile(
283	    "   sync ;"
284	    "   mfmsr  %[msr];"
285	    "   rlwinm %[msr],%[msr],0,28,26;"	/* Clear PSL_DR */
286	    "   mtmsr  %[msr];"			/* Turn off DMMU */
287	    "   isync;"
288	    "1: lvx    %[lv], %[src], %[off0];"
289	    "   stvx   %[lv], %[dst], %[off0];"
290	    "   lvxl   %[hv], %[src], %[off16];"
291	    "   stvxl  %[hv], %[dst], %[off16];"
292	    "   addi   %[src], %[src], 32;"
293	    "   addi   %[dst], %[dst], 32;"
294	    "   cmplw  %[dst], %[edst];"
295	    "	blt+   1b;"
296	    "   ori    %[msr], %[msr], 0x10;"	/* Set PSL_DR */
297	    "   sync;"
298	    "	mtmsr  %[msr];"			/* Turn on DMMU */
299	    "   isync;"
300	    :: [msr] "r"(msr), [src] "b"(src), [dst] "b"(dst),
301	    [edst] "b"(edst), [off0] "r"(0), [off16] "r"(16),
302	    [lv] "n"(LO_VEC), [hv] "n"(HI_VEC));
303
304	/*
305	 * Restore VEC registers (now that we can access the stack again).
306	 */
307	__asm("lvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC));
308	__asm("lvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC));
309
310	/*
311	 * Restore old MSR (AltiVec OFF).
312	 */
313	__asm volatile("sync; mtmsr %0; isync" :: "r"(omsr));
314}
315