1/* 2 * unaligned.c: Unaligned load/store trap handling with special 3 * cases for the kernel to do them more quickly. 4 * 5 * Copyright (C) 1996,2008 David S. Miller (davem@davemloft.net) 6 * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 7 */ 8 9 10#include <linux/jiffies.h> 11#include <linux/kernel.h> 12#include <linux/sched.h> 13#include <linux/mm.h> 14#include <linux/module.h> 15#include <asm/asi.h> 16#include <asm/ptrace.h> 17#include <asm/pstate.h> 18#include <asm/processor.h> 19#include <asm/system.h> 20#include <asm/uaccess.h> 21#include <linux/smp.h> 22#include <linux/bitops.h> 23#include <linux/perf_event.h> 24#include <linux/ratelimit.h> 25#include <asm/fpumacro.h> 26 27enum direction { 28 load, /* ld, ldd, ldh, ldsh */ 29 store, /* st, std, sth, stsh */ 30 both, /* Swap, ldstub, cas, ... */ 31 fpld, 32 fpst, 33 invalid, 34}; 35 36static inline enum direction decode_direction(unsigned int insn) 37{ 38 unsigned long tmp = (insn >> 21) & 1; 39 40 if (!tmp) 41 return load; 42 else { 43 switch ((insn>>19)&0xf) { 44 case 15: /* swap* */ 45 return both; 46 default: 47 return store; 48 } 49 } 50} 51 52/* 16 = double-word, 8 = extra-word, 4 = word, 2 = half-word */ 53static inline int decode_access_size(struct pt_regs *regs, unsigned int insn) 54{ 55 unsigned int tmp; 56 57 tmp = ((insn >> 19) & 0xf); 58 if (tmp == 11 || tmp == 14) /* ldx/stx */ 59 return 8; 60 tmp &= 3; 61 if (!tmp) 62 return 4; 63 else if (tmp == 3) 64 return 16; /* ldd/std - Although it is actually 8 */ 65 else if (tmp == 2) 66 return 2; 67 else { 68 printk("Impossible unaligned trap. insn=%08x\n", insn); 69 die_if_kernel("Byte sized unaligned access?!?!", regs); 70 71 /* GCC should never warn that control reaches the end 72 * of this function without returning a value because 73 * die_if_kernel() is marked with attribute 'noreturn'. 74 * Alas, some versions do... 75 */ 76 77 return 0; 78 } 79} 80 81static inline int decode_asi(unsigned int insn, struct pt_regs *regs) 82{ 83 if (insn & 0x800000) { 84 if (insn & 0x2000) 85 return (unsigned char)(regs->tstate >> 24); /* %asi */ 86 else 87 return (unsigned char)(insn >> 5); /* imm_asi */ 88 } else 89 return ASI_P; 90} 91 92/* 0x400000 = signed, 0 = unsigned */ 93static inline int decode_signedness(unsigned int insn) 94{ 95 return (insn & 0x400000); 96} 97 98static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2, 99 unsigned int rd, int from_kernel) 100{ 101 if (rs2 >= 16 || rs1 >= 16 || rd >= 16) { 102 if (from_kernel != 0) 103 __asm__ __volatile__("flushw"); 104 else 105 flushw_user(); 106 } 107} 108 109static inline long sign_extend_imm13(long imm) 110{ 111 return imm << 51 >> 51; 112} 113 114static unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs) 115{ 116 unsigned long value; 117 118 if (reg < 16) 119 return (!reg ? 0 : regs->u_regs[reg]); 120 if (regs->tstate & TSTATE_PRIV) { 121 struct reg_window *win; 122 win = (struct reg_window *)(regs->u_regs[UREG_FP] + STACK_BIAS); 123 value = win->locals[reg - 16]; 124 } else if (test_thread_flag(TIF_32BIT)) { 125 struct reg_window32 __user *win32; 126 win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP])); 127 get_user(value, &win32->locals[reg - 16]); 128 } else { 129 struct reg_window __user *win; 130 win = (struct reg_window __user *)(regs->u_regs[UREG_FP] + STACK_BIAS); 131 get_user(value, &win->locals[reg - 16]); 132 } 133 return value; 134} 135 136static unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs) 137{ 138 if (reg < 16) 139 return ®s->u_regs[reg]; 140 if (regs->tstate & TSTATE_PRIV) { 141 struct reg_window *win; 142 win = (struct reg_window *)(regs->u_regs[UREG_FP] + STACK_BIAS); 143 return &win->locals[reg - 16]; 144 } else if (test_thread_flag(TIF_32BIT)) { 145 struct reg_window32 *win32; 146 win32 = (struct reg_window32 *)((unsigned long)((u32)regs->u_regs[UREG_FP])); 147 return (unsigned long *)&win32->locals[reg - 16]; 148 } else { 149 struct reg_window *win; 150 win = (struct reg_window *)(regs->u_regs[UREG_FP] + STACK_BIAS); 151 return &win->locals[reg - 16]; 152 } 153} 154 155unsigned long compute_effective_address(struct pt_regs *regs, 156 unsigned int insn, unsigned int rd) 157{ 158 unsigned int rs1 = (insn >> 14) & 0x1f; 159 unsigned int rs2 = insn & 0x1f; 160 int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; 161 162 if (insn & 0x2000) { 163 maybe_flush_windows(rs1, 0, rd, from_kernel); 164 return (fetch_reg(rs1, regs) + sign_extend_imm13(insn)); 165 } else { 166 maybe_flush_windows(rs1, rs2, rd, from_kernel); 167 return (fetch_reg(rs1, regs) + fetch_reg(rs2, regs)); 168 } 169} 170 171/* This is just to make gcc think die_if_kernel does return... */ 172static void __used unaligned_panic(char *str, struct pt_regs *regs) 173{ 174 die_if_kernel(str, regs); 175} 176 177extern int do_int_load(unsigned long *dest_reg, int size, 178 unsigned long *saddr, int is_signed, int asi); 179 180extern int __do_int_store(unsigned long *dst_addr, int size, 181 unsigned long src_val, int asi); 182 183static inline int do_int_store(int reg_num, int size, unsigned long *dst_addr, 184 struct pt_regs *regs, int asi, int orig_asi) 185{ 186 unsigned long zero = 0; 187 unsigned long *src_val_p = &zero; 188 unsigned long src_val; 189 190 if (size == 16) { 191 size = 8; 192 zero = (((long)(reg_num ? 193 (unsigned)fetch_reg(reg_num, regs) : 0)) << 32) | 194 (unsigned)fetch_reg(reg_num + 1, regs); 195 } else if (reg_num) { 196 src_val_p = fetch_reg_addr(reg_num, regs); 197 } 198 src_val = *src_val_p; 199 if (unlikely(asi != orig_asi)) { 200 switch (size) { 201 case 2: 202 src_val = swab16(src_val); 203 break; 204 case 4: 205 src_val = swab32(src_val); 206 break; 207 case 8: 208 src_val = swab64(src_val); 209 break; 210 case 16: 211 default: 212 BUG(); 213 break; 214 }; 215 } 216 return __do_int_store(dst_addr, size, src_val, asi); 217} 218 219static inline void advance(struct pt_regs *regs) 220{ 221 regs->tpc = regs->tnpc; 222 regs->tnpc += 4; 223 if (test_thread_flag(TIF_32BIT)) { 224 regs->tpc &= 0xffffffff; 225 regs->tnpc &= 0xffffffff; 226 } 227} 228 229static inline int floating_point_load_or_store_p(unsigned int insn) 230{ 231 return (insn >> 24) & 1; 232} 233 234static inline int ok_for_kernel(unsigned int insn) 235{ 236 return !floating_point_load_or_store_p(insn); 237} 238 239static void kernel_mna_trap_fault(int fixup_tstate_asi) 240{ 241 struct pt_regs *regs = current_thread_info()->kern_una_regs; 242 unsigned int insn = current_thread_info()->kern_una_insn; 243 const struct exception_table_entry *entry; 244 245 entry = search_exception_tables(regs->tpc); 246 if (!entry) { 247 unsigned long address; 248 249 address = compute_effective_address(regs, insn, 250 ((insn >> 25) & 0x1f)); 251 if (address < PAGE_SIZE) { 252 printk(KERN_ALERT "Unable to handle kernel NULL " 253 "pointer dereference in mna handler"); 254 } else 255 printk(KERN_ALERT "Unable to handle kernel paging " 256 "request in mna handler"); 257 printk(KERN_ALERT " at virtual address %016lx\n",address); 258 printk(KERN_ALERT "current->{active_,}mm->context = %016lx\n", 259 (current->mm ? CTX_HWBITS(current->mm->context) : 260 CTX_HWBITS(current->active_mm->context))); 261 printk(KERN_ALERT "current->{active_,}mm->pgd = %016lx\n", 262 (current->mm ? (unsigned long) current->mm->pgd : 263 (unsigned long) current->active_mm->pgd)); 264 die_if_kernel("Oops", regs); 265 /* Not reached */ 266 } 267 regs->tpc = entry->fixup; 268 regs->tnpc = regs->tpc + 4; 269 270 if (fixup_tstate_asi) { 271 regs->tstate &= ~TSTATE_ASI; 272 regs->tstate |= (ASI_AIUS << 24UL); 273 } 274} 275 276static void log_unaligned(struct pt_regs *regs) 277{ 278 static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5); 279 280 if (__ratelimit(&ratelimit)) { 281 printk("Kernel unaligned access at TPC[%lx] %pS\n", 282 regs->tpc, (void *) regs->tpc); 283 } 284} 285 286asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) 287{ 288 enum direction dir = decode_direction(insn); 289 int size = decode_access_size(regs, insn); 290 int orig_asi, asi; 291 292 current_thread_info()->kern_una_regs = regs; 293 current_thread_info()->kern_una_insn = insn; 294 295 orig_asi = asi = decode_asi(insn, regs); 296 297 /* If this is a {get,put}_user() on an unaligned userspace pointer, 298 * just signal a fault and do not log the event. 299 */ 300 if (asi == ASI_AIUS) { 301 kernel_mna_trap_fault(0); 302 return; 303 } 304 305 log_unaligned(regs); 306 307 if (!ok_for_kernel(insn) || dir == both) { 308 printk("Unsupported unaligned load/store trap for kernel " 309 "at <%016lx>.\n", regs->tpc); 310 unaligned_panic("Kernel does fpu/atomic " 311 "unaligned load/store.", regs); 312 313 kernel_mna_trap_fault(0); 314 } else { 315 unsigned long addr, *reg_addr; 316 int err; 317 318 addr = compute_effective_address(regs, insn, 319 ((insn >> 25) & 0x1f)); 320 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, addr); 321 switch (asi) { 322 case ASI_NL: 323 case ASI_AIUPL: 324 case ASI_AIUSL: 325 case ASI_PL: 326 case ASI_SL: 327 case ASI_PNFL: 328 case ASI_SNFL: 329 asi &= ~0x08; 330 break; 331 }; 332 switch (dir) { 333 case load: 334 reg_addr = fetch_reg_addr(((insn>>25)&0x1f), regs); 335 err = do_int_load(reg_addr, size, 336 (unsigned long *) addr, 337 decode_signedness(insn), asi); 338 if (likely(!err) && unlikely(asi != orig_asi)) { 339 unsigned long val_in = *reg_addr; 340 switch (size) { 341 case 2: 342 val_in = swab16(val_in); 343 break; 344 case 4: 345 val_in = swab32(val_in); 346 break; 347 case 8: 348 val_in = swab64(val_in); 349 break; 350 case 16: 351 default: 352 BUG(); 353 break; 354 }; 355 *reg_addr = val_in; 356 } 357 break; 358 359 case store: 360 err = do_int_store(((insn>>25)&0x1f), size, 361 (unsigned long *) addr, regs, 362 asi, orig_asi); 363 break; 364 365 default: 366 panic("Impossible kernel unaligned trap."); 367 /* Not reached... */ 368 } 369 if (unlikely(err)) 370 kernel_mna_trap_fault(1); 371 else 372 advance(regs); 373 } 374} 375 376static char popc_helper[] = { 3770, 1, 1, 2, 1, 2, 2, 3, 3781, 2, 2, 3, 2, 3, 3, 4, 379}; 380 381int handle_popc(u32 insn, struct pt_regs *regs) 382{ 383 u64 value; 384 int ret, i, rd = ((insn >> 25) & 0x1f); 385 int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; 386 387 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); 388 if (insn & 0x2000) { 389 maybe_flush_windows(0, 0, rd, from_kernel); 390 value = sign_extend_imm13(insn); 391 } else { 392 maybe_flush_windows(0, insn & 0x1f, rd, from_kernel); 393 value = fetch_reg(insn & 0x1f, regs); 394 } 395 for (ret = 0, i = 0; i < 16; i++) { 396 ret += popc_helper[value & 0xf]; 397 value >>= 4; 398 } 399 if (rd < 16) { 400 if (rd) 401 regs->u_regs[rd] = ret; 402 } else { 403 if (test_thread_flag(TIF_32BIT)) { 404 struct reg_window32 __user *win32; 405 win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP])); 406 put_user(ret, &win32->locals[rd - 16]); 407 } else { 408 struct reg_window __user *win; 409 win = (struct reg_window __user *)(regs->u_regs[UREG_FP] + STACK_BIAS); 410 put_user(ret, &win->locals[rd - 16]); 411 } 412 } 413 advance(regs); 414 return 1; 415} 416 417extern void do_fpother(struct pt_regs *regs); 418extern void do_privact(struct pt_regs *regs); 419extern void spitfire_data_access_exception(struct pt_regs *regs, 420 unsigned long sfsr, 421 unsigned long sfar); 422extern void sun4v_data_access_exception(struct pt_regs *regs, 423 unsigned long addr, 424 unsigned long type_ctx); 425 426int handle_ldf_stq(u32 insn, struct pt_regs *regs) 427{ 428 unsigned long addr = compute_effective_address(regs, insn, 0); 429 int freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20); 430 struct fpustate *f = FPUSTATE; 431 int asi = decode_asi(insn, regs); 432 int flag = (freg < 32) ? FPRS_DL : FPRS_DU; 433 434 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); 435 436 save_and_clear_fpu(); 437 current_thread_info()->xfsr[0] &= ~0x1c000; 438 if (freg & 3) { 439 current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */; 440 do_fpother(regs); 441 return 0; 442 } 443 if (insn & 0x200000) { 444 /* STQ */ 445 u64 first = 0, second = 0; 446 447 if (current_thread_info()->fpsaved[0] & flag) { 448 first = *(u64 *)&f->regs[freg]; 449 second = *(u64 *)&f->regs[freg+2]; 450 } 451 if (asi < 0x80) { 452 do_privact(regs); 453 return 1; 454 } 455 switch (asi) { 456 case ASI_P: 457 case ASI_S: break; 458 case ASI_PL: 459 case ASI_SL: 460 { 461 /* Need to convert endians */ 462 u64 tmp = __swab64p(&first); 463 464 first = __swab64p(&second); 465 second = tmp; 466 break; 467 } 468 default: 469 if (tlb_type == hypervisor) 470 sun4v_data_access_exception(regs, addr, 0); 471 else 472 spitfire_data_access_exception(regs, 0, addr); 473 return 1; 474 } 475 if (put_user (first >> 32, (u32 __user *)addr) || 476 __put_user ((u32)first, (u32 __user *)(addr + 4)) || 477 __put_user (second >> 32, (u32 __user *)(addr + 8)) || 478 __put_user ((u32)second, (u32 __user *)(addr + 12))) { 479 if (tlb_type == hypervisor) 480 sun4v_data_access_exception(regs, addr, 0); 481 else 482 spitfire_data_access_exception(regs, 0, addr); 483 return 1; 484 } 485 } else { 486 /* LDF, LDDF, LDQF */ 487 u32 data[4] __attribute__ ((aligned(8))); 488 int size, i; 489 int err; 490 491 if (asi < 0x80) { 492 do_privact(regs); 493 return 1; 494 } else if (asi > ASI_SNFL) { 495 if (tlb_type == hypervisor) 496 sun4v_data_access_exception(regs, addr, 0); 497 else 498 spitfire_data_access_exception(regs, 0, addr); 499 return 1; 500 } 501 switch (insn & 0x180000) { 502 case 0x000000: size = 1; break; 503 case 0x100000: size = 4; break; 504 default: size = 2; break; 505 } 506 for (i = 0; i < size; i++) 507 data[i] = 0; 508 509 err = get_user (data[0], (u32 __user *) addr); 510 if (!err) { 511 for (i = 1; i < size; i++) 512 err |= __get_user (data[i], (u32 __user *)(addr + 4*i)); 513 } 514 if (err && !(asi & 0x2 /* NF */)) { 515 if (tlb_type == hypervisor) 516 sun4v_data_access_exception(regs, addr, 0); 517 else 518 spitfire_data_access_exception(regs, 0, addr); 519 return 1; 520 } 521 if (asi & 0x8) /* Little */ { 522 u64 tmp; 523 524 switch (size) { 525 case 1: data[0] = le32_to_cpup(data + 0); break; 526 default:*(u64 *)(data + 0) = le64_to_cpup((u64 *)(data + 0)); 527 break; 528 case 4: tmp = le64_to_cpup((u64 *)(data + 0)); 529 *(u64 *)(data + 0) = le64_to_cpup((u64 *)(data + 2)); 530 *(u64 *)(data + 2) = tmp; 531 break; 532 } 533 } 534 if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) { 535 current_thread_info()->fpsaved[0] = FPRS_FEF; 536 current_thread_info()->gsr[0] = 0; 537 } 538 if (!(current_thread_info()->fpsaved[0] & flag)) { 539 if (freg < 32) 540 memset(f->regs, 0, 32*sizeof(u32)); 541 else 542 memset(f->regs+32, 0, 32*sizeof(u32)); 543 } 544 memcpy(f->regs + freg, data, size * 4); 545 current_thread_info()->fpsaved[0] |= flag; 546 } 547 advance(regs); 548 return 1; 549} 550 551void handle_ld_nf(u32 insn, struct pt_regs *regs) 552{ 553 int rd = ((insn >> 25) & 0x1f); 554 int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; 555 unsigned long *reg; 556 557 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); 558 559 maybe_flush_windows(0, 0, rd, from_kernel); 560 reg = fetch_reg_addr(rd, regs); 561 if (from_kernel || rd < 16) { 562 reg[0] = 0; 563 if ((insn & 0x780000) == 0x180000) 564 reg[1] = 0; 565 } else if (test_thread_flag(TIF_32BIT)) { 566 put_user(0, (int __user *) reg); 567 if ((insn & 0x780000) == 0x180000) 568 put_user(0, ((int __user *) reg) + 1); 569 } else { 570 put_user(0, (unsigned long __user *) reg); 571 if ((insn & 0x780000) == 0x180000) 572 put_user(0, (unsigned long __user *) reg + 1); 573 } 574 advance(regs); 575} 576 577void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) 578{ 579 unsigned long pc = regs->tpc; 580 unsigned long tstate = regs->tstate; 581 u32 insn; 582 u64 value; 583 u8 freg; 584 int flag; 585 struct fpustate *f = FPUSTATE; 586 587 if (tstate & TSTATE_PRIV) 588 die_if_kernel("lddfmna from kernel", regs); 589 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, sfar); 590 if (test_thread_flag(TIF_32BIT)) 591 pc = (u32)pc; 592 if (get_user(insn, (u32 __user *) pc) != -EFAULT) { 593 int asi = decode_asi(insn, regs); 594 u32 first, second; 595 int err; 596 597 if ((asi > ASI_SNFL) || 598 (asi < ASI_P)) 599 goto daex; 600 first = second = 0; 601 err = get_user(first, (u32 __user *)sfar); 602 if (!err) 603 err = get_user(second, (u32 __user *)(sfar + 4)); 604 if (err) { 605 if (!(asi & 0x2)) 606 goto daex; 607 first = second = 0; 608 } 609 save_and_clear_fpu(); 610 freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20); 611 value = (((u64)first) << 32) | second; 612 if (asi & 0x8) /* Little */ 613 value = __swab64p(&value); 614 flag = (freg < 32) ? FPRS_DL : FPRS_DU; 615 if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) { 616 current_thread_info()->fpsaved[0] = FPRS_FEF; 617 current_thread_info()->gsr[0] = 0; 618 } 619 if (!(current_thread_info()->fpsaved[0] & flag)) { 620 if (freg < 32) 621 memset(f->regs, 0, 32*sizeof(u32)); 622 else 623 memset(f->regs+32, 0, 32*sizeof(u32)); 624 } 625 *(u64 *)(f->regs + freg) = value; 626 current_thread_info()->fpsaved[0] |= flag; 627 } else { 628daex: 629 if (tlb_type == hypervisor) 630 sun4v_data_access_exception(regs, sfar, sfsr); 631 else 632 spitfire_data_access_exception(regs, sfsr, sfar); 633 return; 634 } 635 advance(regs); 636} 637 638void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) 639{ 640 unsigned long pc = regs->tpc; 641 unsigned long tstate = regs->tstate; 642 u32 insn; 643 u64 value; 644 u8 freg; 645 int flag; 646 struct fpustate *f = FPUSTATE; 647 648 if (tstate & TSTATE_PRIV) 649 die_if_kernel("stdfmna from kernel", regs); 650 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, sfar); 651 if (test_thread_flag(TIF_32BIT)) 652 pc = (u32)pc; 653 if (get_user(insn, (u32 __user *) pc) != -EFAULT) { 654 int asi = decode_asi(insn, regs); 655 freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20); 656 value = 0; 657 flag = (freg < 32) ? FPRS_DL : FPRS_DU; 658 if ((asi > ASI_SNFL) || 659 (asi < ASI_P)) 660 goto daex; 661 save_and_clear_fpu(); 662 if (current_thread_info()->fpsaved[0] & flag) 663 value = *(u64 *)&f->regs[freg]; 664 switch (asi) { 665 case ASI_P: 666 case ASI_S: break; 667 case ASI_PL: 668 case ASI_SL: 669 value = __swab64p(&value); break; 670 default: goto daex; 671 } 672 if (put_user (value >> 32, (u32 __user *) sfar) || 673 __put_user ((u32)value, (u32 __user *)(sfar + 4))) 674 goto daex; 675 } else { 676daex: 677 if (tlb_type == hypervisor) 678 sun4v_data_access_exception(regs, sfar, sfsr); 679 else 680 spitfire_data_access_exception(regs, sfsr, sfar); 681 return; 682 } 683 advance(regs); 684} 685