altivec.c revision 1.23
1/* $NetBSD: altivec.c,v 1.23 2011/05/02 06:43:16 matt Exp $ */ 2 3/* 4 * Copyright (C) 1996 Wolfgang Solfrank. 5 * Copyright (C) 1996 TooLs GmbH. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by TooLs GmbH. 19 * 4. The name of TooLs GmbH may not be used to endorse or promote products 20 * derived from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34#include <sys/cdefs.h> 35__KERNEL_RCSID(0, "$NetBSD: altivec.c,v 1.23 2011/05/02 06:43:16 matt Exp $"); 36 37#include "opt_multiprocessor.h" 38 39#include <sys/param.h> 40#include <sys/proc.h> 41#include <sys/systm.h> 42#include <sys/atomic.h> 43 44#include <uvm/uvm_extern.h> /* for vcopypage/vzeropage */ 45 46#include <powerpc/pcb.h> 47#include <powerpc/altivec.h> 48#include <powerpc/spr.h> 49#include <powerpc/oea/spr.h> 50#include <powerpc/psl.h> 51 52static void vec_state_load(lwp_t *, bool); 53static void vec_state_save(lwp_t *); 54static void vec_state_release(lwp_t *); 55 56const pcu_ops_t vec_ops = { 57 .pcu_id = PCU_VEC, 58 .pcu_state_load = vec_state_load, 59 .pcu_state_save = vec_state_save, 60 .pcu_state_release = vec_state_release, 61}; 62 63bool 64vec_used_p(lwp_t *l) 65{ 66 return (l->l_md.md_flags & MDLWP_USEDVEC) != 0; 67} 68 69void 70vec_mark_used(lwp_t *l) 71{ 72 l->l_md.md_flags |= MDLWP_USEDVEC; 73} 74 75void 76vec_state_load(lwp_t *l, bool used) 77{ 78 struct pcb * const pcb = lwp_getpcb(l); 79 80 /* 81 * Enable AltiVec temporarily (and disable interrupts). 82 */ 83 const register_t msr = mfmsr(); 84 mtmsr((msr & ~PSL_EE) | PSL_VEC); 85 __asm volatile ("isync"); 86 87 /* 88 * Load the vector unit from vreg which is best done in 89 * assembly. 90 */ 91 vec_load_from_vreg(&pcb->pcb_vr); 92 93 /* 94 * VRSAVE will be restored when trap frame returns 95 */ 96 l->l_md.md_utf->tf_vrsave = pcb->pcb_vr.vrsave; 97 98 /* 99 * Restore MSR (turn off AltiVec) 100 */ 101 mtmsr(msr); 102 __asm volatile ("isync"); 103 104 /* 105 * Mark vector registers as modified. 106 */ 107 l->l_md.md_flags |= MDLWP_USEDVEC; 108 l->l_md.md_utf->tf_srr1 |= PSL_VEC; 109} 110 111void 112vec_state_save(lwp_t *l) 113{ 114 struct pcb * const pcb = lwp_getpcb(l); 115 116 /* 117 * Turn on AltiVEC, turn off interrupts. 118 */ 119 const register_t msr = mfmsr(); 120 mtmsr((msr & ~PSL_EE) | PSL_VEC); 121 __asm volatile ("isync"); 122 123 /* 124 * Grab contents of vector unit. 125 */ 126 vec_unload_to_vreg(&pcb->pcb_vr); 127 128 /* 129 * Save VRSAVE 130 */ 131 pcb->pcb_vr.vrsave = l->l_md.md_utf->tf_vrsave; 132 133 /* 134 * Note that we aren't using any CPU resources and stop any 135 * data streams. 136 */ 137 __asm volatile ("dssall; sync"); 138 139 /* 140 * Restore MSR (turn off AltiVec) 141 */ 142 mtmsr(msr); 143 __asm volatile ("isync"); 144} 145 146void 147vec_state_release(lwp_t *l) 148{ 149 __asm volatile("dssall;sync"); 150 l->l_md.md_utf->tf_srr1 &= ~PSL_VEC; 151 l->l_md.md_flags &= ~PSL_VEC; 152} 153 154void 155vec_restore_from_mcontext(struct lwp *l, const mcontext_t *mcp) 156{ 157 struct pcb * const pcb = lwp_getpcb(l); 158 159 KASSERT(l == curlwp); 160 161 /* we don't need to save the state, just drop it */ 162 pcu_discard(&vec_ops); 163 memcpy(pcb->pcb_vr.vreg, &mcp->__vrf.__vrs, sizeof (pcb->pcb_vr.vreg)); 164 pcb->pcb_vr.vscr = mcp->__vrf.__vscr; 165 pcb->pcb_vr.vrsave = mcp->__vrf.__vrsave; 166 l->l_md.md_utf->tf_vrsave = pcb->pcb_vr.vrsave; 167} 168 169bool 170vec_save_to_mcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flagp) 171{ 172 struct pcb * const pcb = lwp_getpcb(l); 173 174 KASSERT(l == curlwp); 175 176 /* Save AltiVec context, if any. */ 177 if (!vec_used_p(l)) 178 return false; 179 180 /* 181 * If we're the AltiVec owner, dump its context to the PCB first. 182 */ 183 pcu_save(&vec_ops); 184 185 mcp->__gregs[_REG_MSR] |= PSL_VEC; 186 mcp->__vrf.__vscr = pcb->pcb_vr.vscr; 187 mcp->__vrf.__vrsave = l->l_md.md_utf->tf_vrsave; 188 memcpy(mcp->__vrf.__vrs, pcb->pcb_vr.vreg, sizeof (mcp->__vrf.__vrs)); 189 *flagp |= _UC_POWERPC_VEC; 190 return true; 191} 192 193#define ZERO_VEC 19 194 195void 196vzeropage(paddr_t pa) 197{ 198 const paddr_t ea = pa + PAGE_SIZE; 199 uint32_t vec[7], *vp = (void *) roundup((uintptr_t) vec, 16); 200 register_t omsr, msr; 201 202 __asm volatile("mfmsr %0" : "=r"(omsr) :); 203 204 /* 205 * Turn on AltiVec, turn off interrupts. 206 */ 207 msr = (omsr & ~PSL_EE) | PSL_VEC; 208 __asm volatile("sync; mtmsr %0; isync" :: "r"(msr)); 209 210 /* 211 * Save the VEC register we are going to use before we disable 212 * relocation. 213 */ 214 __asm("stvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC)); 215 __asm("vxor %0,%0,%0" :: "n"(ZERO_VEC)); 216 217 /* 218 * Zero the page using a single cache line. 219 */ 220 __asm volatile( 221 " sync ;" 222 " mfmsr %[msr];" 223 " rlwinm %[msr],%[msr],0,28,26;" /* Clear PSL_DR */ 224 " mtmsr %[msr];" /* Turn off DMMU */ 225 " isync;" 226 "1: stvx %[zv], %[pa], %[off0];" 227 " stvxl %[zv], %[pa], %[off16];" 228 " stvx %[zv], %[pa], %[off32];" 229 " stvxl %[zv], %[pa], %[off48];" 230 " addi %[pa], %[pa], 64;" 231 " cmplw %[pa], %[ea];" 232 " blt+ 1b;" 233 " ori %[msr], %[msr], 0x10;" /* Set PSL_DR */ 234 " sync;" 235 " mtmsr %[msr];" /* Turn on DMMU */ 236 " isync;" 237 :: [msr] "r"(msr), [pa] "b"(pa), [ea] "b"(ea), 238 [off0] "r"(0), [off16] "r"(16), [off32] "r"(32), [off48] "r"(48), 239 [zv] "n"(ZERO_VEC)); 240 241 /* 242 * Restore VEC register (now that we can access the stack again). 243 */ 244 __asm("lvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC)); 245 246 /* 247 * Restore old MSR (AltiVec OFF). 248 */ 249 __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr)); 250} 251 252#define LO_VEC 16 253#define HI_VEC 17 254 255void 256vcopypage(paddr_t dst, paddr_t src) 257{ 258 const paddr_t edst = dst + PAGE_SIZE; 259 uint32_t vec[11], *vp = (void *) roundup((uintptr_t) vec, 16); 260 register_t omsr, msr; 261 262 __asm volatile("mfmsr %0" : "=r"(omsr) :); 263 264 /* 265 * Turn on AltiVec, turn off interrupts. 266 */ 267 msr = (omsr & ~PSL_EE) | PSL_VEC; 268 __asm volatile("sync; mtmsr %0; isync" :: "r"(msr)); 269 270 /* 271 * Save the VEC registers we will be using before we disable 272 * relocation. 273 */ 274 __asm("stvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC)); 275 __asm("stvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC)); 276 277 /* 278 * Copy the page using a single cache line, with DMMU 279 * disabled. On most PPCs, two vector registers occupy one 280 * cache line. 281 */ 282 __asm volatile( 283 " sync ;" 284 " mfmsr %[msr];" 285 " rlwinm %[msr],%[msr],0,28,26;" /* Clear PSL_DR */ 286 " mtmsr %[msr];" /* Turn off DMMU */ 287 " isync;" 288 "1: lvx %[lv], %[src], %[off0];" 289 " stvx %[lv], %[dst], %[off0];" 290 " lvxl %[hv], %[src], %[off16];" 291 " stvxl %[hv], %[dst], %[off16];" 292 " addi %[src], %[src], 32;" 293 " addi %[dst], %[dst], 32;" 294 " cmplw %[dst], %[edst];" 295 " blt+ 1b;" 296 " ori %[msr], %[msr], 0x10;" /* Set PSL_DR */ 297 " sync;" 298 " mtmsr %[msr];" /* Turn on DMMU */ 299 " isync;" 300 :: [msr] "r"(msr), [src] "b"(src), [dst] "b"(dst), 301 [edst] "b"(edst), [off0] "r"(0), [off16] "r"(16), 302 [lv] "n"(LO_VEC), [hv] "n"(HI_VEC)); 303 304 /* 305 * Restore VEC registers (now that we can access the stack again). 306 */ 307 __asm("lvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC)); 308 __asm("lvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC)); 309 310 /* 311 * Restore old MSR (AltiVec OFF). 312 */ 313 __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr)); 314} 315