1226584Sdim/* SPDX-License-Identifier: GPL-2.0-only */ 2226584Sdim/* 3226584Sdim * linux/arch/arm/vfp/vfp.h 4226584Sdim * 5226584Sdim * Copyright (C) 2004 ARM Limited. 6226584Sdim * Written by Deep Blue Solutions Limited. 7226584Sdim */ 8226584Sdim 9226584Sdimstatic inline u32 vfp_shiftright32jamming(u32 val, unsigned int shift) 10226584Sdim{ 11226584Sdim if (shift) { 12226584Sdim if (shift < 32) 13226584Sdim val = val >> shift | ((val << (32 - shift)) != 0); 14226584Sdim else 15226584Sdim val = val != 0; 16226584Sdim } 17226584Sdim return val; 18226584Sdim} 19226584Sdim 20226584Sdimstatic inline u64 vfp_shiftright64jamming(u64 val, unsigned int shift) 21252723Sdim{ 22252723Sdim if (shift) { 23226584Sdim if (shift < 64) 24226584Sdim val = val >> shift | ((val << (64 - shift)) != 0); 25252723Sdim else 26226584Sdim val = val != 0; 27226584Sdim } 28226584Sdim return val; 29226584Sdim} 30226584Sdim 31226584Sdimstatic inline u32 vfp_hi64to32jamming(u64 val) 32226584Sdim{ 33226584Sdim u32 v; 34226584Sdim 35226584Sdim asm( 36226584Sdim "cmp %Q1, #1 @ vfp_hi64to32jamming\n\t" 37226584Sdim "movcc %0, %R1\n\t" 38226584Sdim "orrcs %0, %R1, #1" 39226584Sdim : "=r" (v) : "r" (val) : "cc"); 40226584Sdim 41226584Sdim return v; 42226584Sdim} 43226584Sdim 44226584Sdimstatic inline void add128(u64 *resh, u64 *resl, u64 nh, u64 nl, u64 mh, u64 ml) 45226584Sdim{ 46226584Sdim asm( "adds %Q0, %Q2, %Q4\n\t" 47226584Sdim "adcs %R0, %R2, %R4\n\t" 48226584Sdim "adcs %Q1, %Q3, %Q5\n\t" 49226584Sdim "adc %R1, %R3, %R5" 50226584Sdim : "=r" (nl), "=r" (nh) 51226584Sdim : "0" (nl), "1" (nh), "r" (ml), "r" (mh) 52226584Sdim : "cc"); 53226584Sdim *resh = nh; 54226584Sdim *resl = nl; 55226584Sdim} 56226584Sdim 57235633Sdimstatic inline void sub128(u64 *resh, u64 *resl, u64 nh, u64 nl, u64 mh, u64 ml) 58226584Sdim{ 59235633Sdim asm( "subs %Q0, %Q2, %Q4\n\t" 60235633Sdim "sbcs %R0, %R2, %R4\n\t" 61226584Sdim "sbcs %Q1, %Q3, %Q5\n\t" 62226584Sdim "sbc %R1, %R3, %R5\n\t" 63226584Sdim : "=r" (nl), "=r" (nh) 64226584Sdim : "0" (nl), "1" (nh), "r" (ml), "r" (mh) 65226584Sdim : "cc"); 66226584Sdim *resh = nh; 67226584Sdim *resl = nl; 68226584Sdim} 69226584Sdim 70226584Sdimstatic inline void mul64to128(u64 *resh, u64 *resl, u64 n, u64 m) 71226584Sdim{ 72226584Sdim u32 nh, nl, mh, ml; 73226584Sdim u64 rh, rma, rmb, rl; 74226584Sdim 75226584Sdim nl = n; 76226584Sdim ml = m; 77226584Sdim rl = (u64)nl * ml; 78226584Sdim 79226584Sdim nh = n >> 32; 80226584Sdim rma = (u64)nh * ml; 81226584Sdim 82226584Sdim mh = m >> 32; 83226584Sdim rmb = (u64)nl * mh; 84226584Sdim rma += rmb; 85226584Sdim 86226584Sdim rh = (u64)nh * mh; 87226584Sdim rh += ((u64)(rma < rmb) << 32) + (rma >> 32); 88226584Sdim 89226584Sdim rma <<= 32; 90226584Sdim rl += rma; 91226584Sdim rh += (rl < rma); 92226584Sdim 93226584Sdim *resl = rl; 94226584Sdim *resh = rh; 95226584Sdim} 96226584Sdim 97226584Sdimstatic inline void shift64left(u64 *resh, u64 *resl, u64 n) 98226584Sdim{ 99226584Sdim *resh = n >> 63; 100252723Sdim *resl = n << 1; 101252723Sdim} 102252723Sdim 103252723Sdimstatic inline u64 vfp_hi64multiply64(u64 n, u64 m) 104252723Sdim{ 105252723Sdim u64 rh, rl; 106226584Sdim mul64to128(&rh, &rl, n, m); 107226584Sdim return rh | (rl != 0); 108226584Sdim} 109226584Sdim 110226584Sdimstatic inline u64 vfp_estimate_div128to64(u64 nh, u64 nl, u64 m) 111226584Sdim{ 112226584Sdim u64 mh, ml, remh, reml, termh, terml, z; 113226584Sdim 114226584Sdim if (nh >= m) 115226584Sdim return ~0ULL; 116226584Sdim mh = m >> 32; 117226584Sdim if (mh << 32 <= nh) { 118226584Sdim z = 0xffffffff00000000ULL; 119226584Sdim } else { 120226584Sdim z = nh; 121226584Sdim do_div(z, mh); 122245431Sdim z <<= 32; 123245431Sdim } 124245431Sdim mul64to128(&termh, &terml, m, z); 125245431Sdim sub128(&remh, &reml, nh, nl, termh, terml); 126245431Sdim ml = m << 32; 127245431Sdim while ((s64)remh < 0) { 128226584Sdim z -= 0x100000000ULL; 129226584Sdim add128(&remh, &reml, remh, reml, mh, ml); 130226584Sdim } 131226584Sdim remh = (remh << 32) | (reml >> 32); 132226584Sdim if (mh << 32 <= remh) { 133226584Sdim z |= 0xffffffff; 134226584Sdim } else { 135226584Sdim do_div(remh, mh); 136252723Sdim z |= remh; 137252723Sdim } 138252723Sdim return z; 139252723Sdim} 140252723Sdim 141252723Sdim/* 142252723Sdim * Operations on unpacked elements 143252723Sdim */ 144226584Sdim#define vfp_sign_negate(sign) (sign ^ 0x8000) 145226584Sdim 146226584Sdim/* 147226584Sdim * Single-precision 148226584Sdim */ 149226584Sdimstruct vfp_single { 150226584Sdim s16 exponent; 151252723Sdim u16 sign; 152226584Sdim u32 significand; 153226584Sdim}; 154226584Sdim 155226584Sdimasmlinkage s32 vfp_get_float(unsigned int reg); 156226584Sdimasmlinkage void vfp_put_float(s32 val, unsigned int reg); 157226584Sdim 158226584Sdim/* 159226584Sdim * VFP_SINGLE_MANTISSA_BITS - number of bits in the mantissa 160226584Sdim * VFP_SINGLE_EXPONENT_BITS - number of bits in the exponent 161226584Sdim * VFP_SINGLE_LOW_BITS - number of low bits in the unpacked significand 162226584Sdim * which are not propagated to the float upon packing. 163226584Sdim */ 164226584Sdim#define VFP_SINGLE_MANTISSA_BITS (23) 165226584Sdim#define VFP_SINGLE_EXPONENT_BITS (8) 166226584Sdim#define VFP_SINGLE_LOW_BITS (32 - VFP_SINGLE_MANTISSA_BITS - 2) 167226584Sdim#define VFP_SINGLE_LOW_BITS_MASK ((1 << VFP_SINGLE_LOW_BITS) - 1) 168226584Sdim 169226584Sdim/* 170226584Sdim * The bit in an unpacked float which indicates that it is a quiet NaN 171226584Sdim */ 172226584Sdim#define VFP_SINGLE_SIGNIFICAND_QNAN (1 << (VFP_SINGLE_MANTISSA_BITS - 1 + VFP_SINGLE_LOW_BITS)) 173226584Sdim 174226584Sdim/* 175226584Sdim * Operations on packed single-precision numbers 176226584Sdim */ 177226584Sdim#define vfp_single_packed_sign(v) ((v) & 0x80000000) 178226584Sdim#define vfp_single_packed_negate(v) ((v) ^ 0x80000000) 179226584Sdim#define vfp_single_packed_abs(v) ((v) & ~0x80000000) 180226584Sdim#define vfp_single_packed_exponent(v) (((v) >> VFP_SINGLE_MANTISSA_BITS) & ((1 << VFP_SINGLE_EXPONENT_BITS) - 1)) 181226584Sdim#define vfp_single_packed_mantissa(v) ((v) & ((1 << VFP_SINGLE_MANTISSA_BITS) - 1)) 182226584Sdim 183245431Sdim/* 184226584Sdim * Unpack a single-precision float. Note that this returns the magnitude 185226584Sdim * of the single-precision float mantissa with the 1. if necessary, 186226584Sdim * aligned to bit 30. 187226584Sdim */ 188226584Sdimstatic inline void vfp_single_unpack(struct vfp_single *s, s32 val) 189226584Sdim{ 190226584Sdim u32 significand; 191226584Sdim 192226584Sdim s->sign = vfp_single_packed_sign(val) >> 16, 193226584Sdim s->exponent = vfp_single_packed_exponent(val); 194226584Sdim 195226584Sdim significand = (u32) val; 196226584Sdim significand = (significand << (32 - VFP_SINGLE_MANTISSA_BITS)) >> 2; 197226584Sdim if (s->exponent && s->exponent != 255) 198235633Sdim significand |= 0x40000000; 199226584Sdim s->significand = significand; 200226584Sdim} 201226584Sdim 202226584Sdim/* 203226584Sdim * Re-pack a single-precision float. This assumes that the float is 204226584Sdim * already normalised such that the MSB is bit 30, _not_ bit 31. 205226584Sdim */ 206226584Sdimstatic inline s32 vfp_single_pack(struct vfp_single *s) 207226584Sdim{ 208226584Sdim u32 val; 209226584Sdim val = (s->sign << 16) + 210226584Sdim (s->exponent << VFP_SINGLE_MANTISSA_BITS) + 211226584Sdim (s->significand >> VFP_SINGLE_LOW_BITS); 212226584Sdim return (s32)val; 213226584Sdim} 214226584Sdim 215226584Sdim#define VFP_NUMBER (1<<0) 216226584Sdim#define VFP_ZERO (1<<1) 217226584Sdim#define VFP_DENORMAL (1<<2) 218226584Sdim#define VFP_INFINITY (1<<3) 219226584Sdim#define VFP_NAN (1<<4) 220226584Sdim#define VFP_NAN_SIGNAL (1<<5) 221226584Sdim 222226584Sdim#define VFP_QNAN (VFP_NAN) 223226584Sdim#define VFP_SNAN (VFP_NAN|VFP_NAN_SIGNAL) 224226584Sdim 225226584Sdimstatic inline int vfp_single_type(struct vfp_single *s) 226{ 227 int type = VFP_NUMBER; 228 if (s->exponent == 255) { 229 if (s->significand == 0) 230 type = VFP_INFINITY; 231 else if (s->significand & VFP_SINGLE_SIGNIFICAND_QNAN) 232 type = VFP_QNAN; 233 else 234 type = VFP_SNAN; 235 } else if (s->exponent == 0) { 236 if (s->significand == 0) 237 type |= VFP_ZERO; 238 else 239 type |= VFP_DENORMAL; 240 } 241 return type; 242} 243 244#ifndef DEBUG 245#define vfp_single_normaliseround(sd,vsd,fpscr,except,func) __vfp_single_normaliseround(sd,vsd,fpscr,except) 246u32 __vfp_single_normaliseround(int sd, struct vfp_single *vs, u32 fpscr, u32 exceptions); 247#else 248u32 vfp_single_normaliseround(int sd, struct vfp_single *vs, u32 fpscr, u32 exceptions, const char *func); 249#endif 250 251/* 252 * Double-precision 253 */ 254struct vfp_double { 255 s16 exponent; 256 u16 sign; 257 u64 significand; 258}; 259 260/* 261 * VFP_REG_ZERO is a special register number for vfp_get_double 262 * which returns (double)0.0. This is useful for the compare with 263 * zero instructions. 264 */ 265#ifdef CONFIG_VFPv3 266#define VFP_REG_ZERO 32 267#else 268#define VFP_REG_ZERO 16 269#endif 270asmlinkage u64 vfp_get_double(unsigned int reg); 271asmlinkage void vfp_put_double(u64 val, unsigned int reg); 272 273#define VFP_DOUBLE_MANTISSA_BITS (52) 274#define VFP_DOUBLE_EXPONENT_BITS (11) 275#define VFP_DOUBLE_LOW_BITS (64 - VFP_DOUBLE_MANTISSA_BITS - 2) 276#define VFP_DOUBLE_LOW_BITS_MASK ((1 << VFP_DOUBLE_LOW_BITS) - 1) 277 278/* 279 * The bit in an unpacked double which indicates that it is a quiet NaN 280 */ 281#define VFP_DOUBLE_SIGNIFICAND_QNAN (1ULL << (VFP_DOUBLE_MANTISSA_BITS - 1 + VFP_DOUBLE_LOW_BITS)) 282 283/* 284 * Operations on packed single-precision numbers 285 */ 286#define vfp_double_packed_sign(v) ((v) & (1ULL << 63)) 287#define vfp_double_packed_negate(v) ((v) ^ (1ULL << 63)) 288#define vfp_double_packed_abs(v) ((v) & ~(1ULL << 63)) 289#define vfp_double_packed_exponent(v) (((v) >> VFP_DOUBLE_MANTISSA_BITS) & ((1 << VFP_DOUBLE_EXPONENT_BITS) - 1)) 290#define vfp_double_packed_mantissa(v) ((v) & ((1ULL << VFP_DOUBLE_MANTISSA_BITS) - 1)) 291 292/* 293 * Unpack a double-precision float. Note that this returns the magnitude 294 * of the double-precision float mantissa with the 1. if necessary, 295 * aligned to bit 62. 296 */ 297static inline void vfp_double_unpack(struct vfp_double *s, s64 val) 298{ 299 u64 significand; 300 301 s->sign = vfp_double_packed_sign(val) >> 48; 302 s->exponent = vfp_double_packed_exponent(val); 303 304 significand = (u64) val; 305 significand = (significand << (64 - VFP_DOUBLE_MANTISSA_BITS)) >> 2; 306 if (s->exponent && s->exponent != 2047) 307 significand |= (1ULL << 62); 308 s->significand = significand; 309} 310 311/* 312 * Re-pack a double-precision float. This assumes that the float is 313 * already normalised such that the MSB is bit 30, _not_ bit 31. 314 */ 315static inline s64 vfp_double_pack(struct vfp_double *s) 316{ 317 u64 val; 318 val = ((u64)s->sign << 48) + 319 ((u64)s->exponent << VFP_DOUBLE_MANTISSA_BITS) + 320 (s->significand >> VFP_DOUBLE_LOW_BITS); 321 return (s64)val; 322} 323 324static inline int vfp_double_type(struct vfp_double *s) 325{ 326 int type = VFP_NUMBER; 327 if (s->exponent == 2047) { 328 if (s->significand == 0) 329 type = VFP_INFINITY; 330 else if (s->significand & VFP_DOUBLE_SIGNIFICAND_QNAN) 331 type = VFP_QNAN; 332 else 333 type = VFP_SNAN; 334 } else if (s->exponent == 0) { 335 if (s->significand == 0) 336 type |= VFP_ZERO; 337 else 338 type |= VFP_DENORMAL; 339 } 340 return type; 341} 342 343u32 vfp_double_normaliseround(int dd, struct vfp_double *vd, u32 fpscr, u32 exceptions, const char *func); 344 345u32 vfp_estimate_sqrt_significand(u32 exponent, u32 significand); 346 347/* 348 * A special flag to tell the normalisation code not to normalise. 349 */ 350#define VFP_NAN_FLAG 0x100 351 352/* 353 * A bit pattern used to indicate the initial (unset) value of the 354 * exception mask, in case nothing handles an instruction. This 355 * doesn't include the NAN flag, which get masked out before 356 * we check for an error. 357 */ 358#define VFP_EXCEPTION_ERROR ((u32)-1 & ~VFP_NAN_FLAG) 359 360/* 361 * A flag to tell vfp instruction type. 362 * OP_SCALAR - this operation always operates in scalar mode 363 * OP_SD - the instruction exceptionally writes to a single precision result. 364 * OP_DD - the instruction exceptionally writes to a double precision result. 365 * OP_SM - the instruction exceptionally reads from a single precision operand. 366 */ 367#define OP_SCALAR (1 << 0) 368#define OP_SD (1 << 1) 369#define OP_DD (1 << 1) 370#define OP_SM (1 << 2) 371 372struct op { 373 u32 (* const fn)(int dd, int dn, int dm, u32 fpscr); 374 u32 flags; 375}; 376 377asmlinkage void vfp_save_state(void *location, u32 fpexc); 378asmlinkage u32 vfp_load_state(const void *location); 379