1#ifndef __ALPHA_SYSTEM_H 2#define __ALPHA_SYSTEM_H 3 4#include <asm/pal.h> 5#include <asm/page.h> 6#include <asm/barrier.h> 7 8/* 9 * System defines.. Note that this is included both from .c and .S 10 * files, so it does only defines, not any C code. 11 */ 12 13/* 14 * We leave one page for the initial stack page, and one page for 15 * the initial process structure. Also, the console eats 3 MB for 16 * the initial bootloader (one of which we can reclaim later). 17 */ 18#define BOOT_PCB 0x20000000 19#define BOOT_ADDR 0x20000000 20/* Remove when official MILO sources have ELF support: */ 21#define BOOT_SIZE (16*1024) 22 23#ifdef CONFIG_ALPHA_LEGACY_START_ADDRESS 24#define KERNEL_START_PHYS 0x300000 /* Old bootloaders hardcoded this. */ 25#else 26#define KERNEL_START_PHYS 0x1000000 /* required: Wildfire/Titan/Marvel */ 27#endif 28 29#define KERNEL_START (PAGE_OFFSET+KERNEL_START_PHYS) 30#define SWAPPER_PGD KERNEL_START 31#define INIT_STACK (PAGE_OFFSET+KERNEL_START_PHYS+0x02000) 32#define EMPTY_PGT (PAGE_OFFSET+KERNEL_START_PHYS+0x04000) 33#define EMPTY_PGE (PAGE_OFFSET+KERNEL_START_PHYS+0x08000) 34#define ZERO_PGE (PAGE_OFFSET+KERNEL_START_PHYS+0x0A000) 35 36#define START_ADDR (PAGE_OFFSET+KERNEL_START_PHYS+0x10000) 37 38/* 39 * This is setup by the secondary bootstrap loader. Because 40 * the zero page is zeroed out as soon as the vm system is 41 * initialized, we need to copy things out into a more permanent 42 * place. 43 */ 44#define PARAM ZERO_PGE 45#define COMMAND_LINE ((char*)(PARAM + 0x0000)) 46#define INITRD_START (*(unsigned long *) (PARAM+0x100)) 47#define INITRD_SIZE (*(unsigned long *) (PARAM+0x108)) 48 49#ifndef __ASSEMBLY__ 50#include <linux/kernel.h> 51 52/* 53 * This is the logout header that should be common to all platforms 54 * (assuming they are running OSF/1 PALcode, I guess). 55 */ 56struct el_common { 57 unsigned int size; /* size in bytes of logout area */ 58 unsigned int sbz1 : 30; /* should be zero */ 59 unsigned int err2 : 1; /* second error */ 60 unsigned int retry : 1; /* retry flag */ 61 unsigned int proc_offset; /* processor-specific offset */ 62 unsigned int sys_offset; /* system-specific offset */ 63 unsigned int code; /* machine check code */ 64 unsigned int frame_rev; /* frame revision */ 65}; 66 67/* Machine Check Frame for uncorrectable errors (Large format) 68 * --- This is used to log uncorrectable errors such as 69 * double bit ECC errors. 70 * --- These errors are detected by both processor and systems. 71 */ 72struct el_common_EV5_uncorrectable_mcheck { 73 unsigned long shadow[8]; /* Shadow reg. 8-14, 25 */ 74 unsigned long paltemp[24]; /* PAL TEMP REGS. */ 75 unsigned long exc_addr; /* Address of excepting instruction*/ 76 unsigned long exc_sum; /* Summary of arithmetic traps. */ 77 unsigned long exc_mask; /* Exception mask (from exc_sum). */ 78 unsigned long pal_base; /* Base address for PALcode. */ 79 unsigned long isr; /* Interrupt Status Reg. */ 80 unsigned long icsr; /* CURRENT SETUP OF EV5 IBOX */ 81 unsigned long ic_perr_stat; /* I-CACHE Reg. <11> set Data parity 82 <12> set TAG parity*/ 83 unsigned long dc_perr_stat; /* D-CACHE error Reg. Bits set to 1: 84 <2> Data error in bank 0 85 <3> Data error in bank 1 86 <4> Tag error in bank 0 87 <5> Tag error in bank 1 */ 88 unsigned long va; /* Effective VA of fault or miss. */ 89 unsigned long mm_stat; /* Holds the reason for D-stream 90 fault or D-cache parity errors */ 91 unsigned long sc_addr; /* Address that was being accessed 92 when EV5 detected Secondary cache 93 failure. */ 94 unsigned long sc_stat; /* Helps determine if the error was 95 TAG/Data parity(Secondary Cache)*/ 96 unsigned long bc_tag_addr; /* Contents of EV5 BC_TAG_ADDR */ 97 unsigned long ei_addr; /* Physical address of any transfer 98 that is logged in EV5 EI_STAT */ 99 unsigned long fill_syndrome; /* For correcting ECC errors. */ 100 unsigned long ei_stat; /* Helps identify reason of any 101 processor uncorrectable error 102 at its external interface. */ 103 unsigned long ld_lock; /* Contents of EV5 LD_LOCK register*/ 104}; 105 106struct el_common_EV6_mcheck { 107 unsigned int FrameSize; /* Bytes, including this field */ 108 unsigned int FrameFlags; /* <31> = Retry, <30> = Second Error */ 109 unsigned int CpuOffset; /* Offset to CPU-specific info */ 110 unsigned int SystemOffset; /* Offset to system-specific info */ 111 unsigned int MCHK_Code; 112 unsigned int MCHK_Frame_Rev; 113 unsigned long I_STAT; /* EV6 Internal Processor Registers */ 114 unsigned long DC_STAT; /* (See the 21264 Spec) */ 115 unsigned long C_ADDR; 116 unsigned long DC1_SYNDROME; 117 unsigned long DC0_SYNDROME; 118 unsigned long C_STAT; 119 unsigned long C_STS; 120 unsigned long MM_STAT; 121 unsigned long EXC_ADDR; 122 unsigned long IER_CM; 123 unsigned long ISUM; 124 unsigned long RESERVED0; 125 unsigned long PAL_BASE; 126 unsigned long I_CTL; 127 unsigned long PCTX; 128}; 129 130extern void halt(void) __attribute__((noreturn)); 131#define __halt() __asm__ __volatile__ ("call_pal %0 #halt" : : "i" (PAL_halt)) 132 133#define switch_to(P,N,L) \ 134 do { \ 135 (L) = alpha_switch_to(virt_to_phys(&task_thread_info(N)->pcb), (P)); \ 136 check_mmu_context(); \ 137 } while (0) 138 139struct task_struct; 140extern struct task_struct *alpha_switch_to(unsigned long, struct task_struct*); 141 142/* 143 * On SMP systems, when the scheduler does migration-cost autodetection, 144 * it needs a way to flush as much of the CPU's caches as possible. 145 * 146 * TODO: fill this in! 147 */ 148static inline void sched_cacheflush(void) 149{ 150} 151 152#define imb() \ 153__asm__ __volatile__ ("call_pal %0 #imb" : : "i" (PAL_imb) : "memory") 154 155#define draina() \ 156__asm__ __volatile__ ("call_pal %0 #draina" : : "i" (PAL_draina) : "memory") 157 158enum implver_enum { 159 IMPLVER_EV4, 160 IMPLVER_EV5, 161 IMPLVER_EV6 162}; 163 164#ifdef CONFIG_ALPHA_GENERIC 165#define implver() \ 166({ unsigned long __implver; \ 167 __asm__ ("implver %0" : "=r"(__implver)); \ 168 (enum implver_enum) __implver; }) 169#else 170/* Try to eliminate some dead code. */ 171#ifdef CONFIG_ALPHA_EV4 172#define implver() IMPLVER_EV4 173#endif 174#ifdef CONFIG_ALPHA_EV5 175#define implver() IMPLVER_EV5 176#endif 177#if defined(CONFIG_ALPHA_EV6) 178#define implver() IMPLVER_EV6 179#endif 180#endif 181 182enum amask_enum { 183 AMASK_BWX = (1UL << 0), 184 AMASK_FIX = (1UL << 1), 185 AMASK_CIX = (1UL << 2), 186 AMASK_MAX = (1UL << 8), 187 AMASK_PRECISE_TRAP = (1UL << 9), 188}; 189 190#define amask(mask) \ 191({ unsigned long __amask, __input = (mask); \ 192 __asm__ ("amask %1,%0" : "=r"(__amask) : "rI"(__input)); \ 193 __amask; }) 194 195#define __CALL_PAL_R0(NAME, TYPE) \ 196static inline TYPE NAME(void) \ 197{ \ 198 register TYPE __r0 __asm__("$0"); \ 199 __asm__ __volatile__( \ 200 "call_pal %1 # " #NAME \ 201 :"=r" (__r0) \ 202 :"i" (PAL_ ## NAME) \ 203 :"$1", "$16", "$22", "$23", "$24", "$25"); \ 204 return __r0; \ 205} 206 207#define __CALL_PAL_W1(NAME, TYPE0) \ 208static inline void NAME(TYPE0 arg0) \ 209{ \ 210 register TYPE0 __r16 __asm__("$16") = arg0; \ 211 __asm__ __volatile__( \ 212 "call_pal %1 # "#NAME \ 213 : "=r"(__r16) \ 214 : "i"(PAL_ ## NAME), "0"(__r16) \ 215 : "$1", "$22", "$23", "$24", "$25"); \ 216} 217 218#define __CALL_PAL_W2(NAME, TYPE0, TYPE1) \ 219static inline void NAME(TYPE0 arg0, TYPE1 arg1) \ 220{ \ 221 register TYPE0 __r16 __asm__("$16") = arg0; \ 222 register TYPE1 __r17 __asm__("$17") = arg1; \ 223 __asm__ __volatile__( \ 224 "call_pal %2 # "#NAME \ 225 : "=r"(__r16), "=r"(__r17) \ 226 : "i"(PAL_ ## NAME), "0"(__r16), "1"(__r17) \ 227 : "$1", "$22", "$23", "$24", "$25"); \ 228} 229 230#define __CALL_PAL_RW1(NAME, RTYPE, TYPE0) \ 231static inline RTYPE NAME(TYPE0 arg0) \ 232{ \ 233 register RTYPE __r0 __asm__("$0"); \ 234 register TYPE0 __r16 __asm__("$16") = arg0; \ 235 __asm__ __volatile__( \ 236 "call_pal %2 # "#NAME \ 237 : "=r"(__r16), "=r"(__r0) \ 238 : "i"(PAL_ ## NAME), "0"(__r16) \ 239 : "$1", "$22", "$23", "$24", "$25"); \ 240 return __r0; \ 241} 242 243#define __CALL_PAL_RW2(NAME, RTYPE, TYPE0, TYPE1) \ 244static inline RTYPE NAME(TYPE0 arg0, TYPE1 arg1) \ 245{ \ 246 register RTYPE __r0 __asm__("$0"); \ 247 register TYPE0 __r16 __asm__("$16") = arg0; \ 248 register TYPE1 __r17 __asm__("$17") = arg1; \ 249 __asm__ __volatile__( \ 250 "call_pal %3 # "#NAME \ 251 : "=r"(__r16), "=r"(__r17), "=r"(__r0) \ 252 : "i"(PAL_ ## NAME), "0"(__r16), "1"(__r17) \ 253 : "$1", "$22", "$23", "$24", "$25"); \ 254 return __r0; \ 255} 256 257__CALL_PAL_W1(cflush, unsigned long); 258__CALL_PAL_R0(rdmces, unsigned long); 259__CALL_PAL_R0(rdps, unsigned long); 260__CALL_PAL_R0(rdusp, unsigned long); 261__CALL_PAL_RW1(swpipl, unsigned long, unsigned long); 262__CALL_PAL_R0(whami, unsigned long); 263__CALL_PAL_W2(wrent, void*, unsigned long); 264__CALL_PAL_W1(wripir, unsigned long); 265__CALL_PAL_W1(wrkgp, unsigned long); 266__CALL_PAL_W1(wrmces, unsigned long); 267__CALL_PAL_RW2(wrperfmon, unsigned long, unsigned long, unsigned long); 268__CALL_PAL_W1(wrusp, unsigned long); 269__CALL_PAL_W1(wrvptptr, unsigned long); 270 271#define IPL_MIN 0 272#define IPL_SW0 1 273#define IPL_SW1 2 274#define IPL_DEV0 3 275#define IPL_DEV1 4 276#define IPL_TIMER 5 277#define IPL_PERF 6 278#define IPL_POWERFAIL 6 279#define IPL_MCHECK 7 280#define IPL_MAX 7 281 282#ifdef CONFIG_ALPHA_BROKEN_IRQ_MASK 283#undef IPL_MIN 284#define IPL_MIN __min_ipl 285extern int __min_ipl; 286#endif 287 288#define getipl() (rdps() & 7) 289#define setipl(ipl) ((void) swpipl(ipl)) 290 291#define local_irq_disable() do { setipl(IPL_MAX); barrier(); } while(0) 292#define local_irq_enable() do { barrier(); setipl(IPL_MIN); } while(0) 293#define local_save_flags(flags) ((flags) = rdps()) 294#define local_irq_save(flags) do { (flags) = swpipl(IPL_MAX); barrier(); } while(0) 295#define local_irq_restore(flags) do { barrier(); setipl(flags); barrier(); } while(0) 296 297#define irqs_disabled() (getipl() == IPL_MAX) 298 299/* 300 * TB routines.. 301 */ 302#define __tbi(nr,arg,arg1...) \ 303({ \ 304 register unsigned long __r16 __asm__("$16") = (nr); \ 305 register unsigned long __r17 __asm__("$17"); arg; \ 306 __asm__ __volatile__( \ 307 "call_pal %3 #__tbi" \ 308 :"=r" (__r16),"=r" (__r17) \ 309 :"0" (__r16),"i" (PAL_tbi) ,##arg1 \ 310 :"$0", "$1", "$22", "$23", "$24", "$25"); \ 311}) 312 313#define tbi(x,y) __tbi(x,__r17=(y),"1" (__r17)) 314#define tbisi(x) __tbi(1,__r17=(x),"1" (__r17)) 315#define tbisd(x) __tbi(2,__r17=(x),"1" (__r17)) 316#define tbis(x) __tbi(3,__r17=(x),"1" (__r17)) 317#define tbiap() __tbi(-1, /* no second argument */) 318#define tbia() __tbi(-2, /* no second argument */) 319 320/* 321 * Atomic exchange. 322 * Since it can be used to implement critical sections 323 * it must clobber "memory" (also for interrupts in UP). 324 */ 325 326static inline unsigned long 327__xchg_u8(volatile char *m, unsigned long val) 328{ 329 unsigned long ret, tmp, addr64; 330 331 __asm__ __volatile__( 332 " andnot %4,7,%3\n" 333 " insbl %1,%4,%1\n" 334 "1: ldq_l %2,0(%3)\n" 335 " extbl %2,%4,%0\n" 336 " mskbl %2,%4,%2\n" 337 " or %1,%2,%2\n" 338 " stq_c %2,0(%3)\n" 339 " beq %2,2f\n" 340#ifdef CONFIG_SMP 341 " mb\n" 342#endif 343 ".subsection 2\n" 344 "2: br 1b\n" 345 ".previous" 346 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) 347 : "r" ((long)m), "1" (val) : "memory"); 348 349 return ret; 350} 351 352static inline unsigned long 353__xchg_u16(volatile short *m, unsigned long val) 354{ 355 unsigned long ret, tmp, addr64; 356 357 __asm__ __volatile__( 358 " andnot %4,7,%3\n" 359 " inswl %1,%4,%1\n" 360 "1: ldq_l %2,0(%3)\n" 361 " extwl %2,%4,%0\n" 362 " mskwl %2,%4,%2\n" 363 " or %1,%2,%2\n" 364 " stq_c %2,0(%3)\n" 365 " beq %2,2f\n" 366#ifdef CONFIG_SMP 367 " mb\n" 368#endif 369 ".subsection 2\n" 370 "2: br 1b\n" 371 ".previous" 372 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) 373 : "r" ((long)m), "1" (val) : "memory"); 374 375 return ret; 376} 377 378static inline unsigned long 379__xchg_u32(volatile int *m, unsigned long val) 380{ 381 unsigned long dummy; 382 383 __asm__ __volatile__( 384 "1: ldl_l %0,%4\n" 385 " bis $31,%3,%1\n" 386 " stl_c %1,%2\n" 387 " beq %1,2f\n" 388#ifdef CONFIG_SMP 389 " mb\n" 390#endif 391 ".subsection 2\n" 392 "2: br 1b\n" 393 ".previous" 394 : "=&r" (val), "=&r" (dummy), "=m" (*m) 395 : "rI" (val), "m" (*m) : "memory"); 396 397 return val; 398} 399 400static inline unsigned long 401__xchg_u64(volatile long *m, unsigned long val) 402{ 403 unsigned long dummy; 404 405 __asm__ __volatile__( 406 "1: ldq_l %0,%4\n" 407 " bis $31,%3,%1\n" 408 " stq_c %1,%2\n" 409 " beq %1,2f\n" 410#ifdef CONFIG_SMP 411 " mb\n" 412#endif 413 ".subsection 2\n" 414 "2: br 1b\n" 415 ".previous" 416 : "=&r" (val), "=&r" (dummy), "=m" (*m) 417 : "rI" (val), "m" (*m) : "memory"); 418 419 return val; 420} 421 422/* This function doesn't exist, so you'll get a linker error 423 if something tries to do an invalid xchg(). */ 424extern void __xchg_called_with_bad_pointer(void); 425 426#define __xchg(ptr, x, size) \ 427({ \ 428 unsigned long __xchg__res; \ 429 volatile void *__xchg__ptr = (ptr); \ 430 switch (size) { \ 431 case 1: __xchg__res = __xchg_u8(__xchg__ptr, x); break; \ 432 case 2: __xchg__res = __xchg_u16(__xchg__ptr, x); break; \ 433 case 4: __xchg__res = __xchg_u32(__xchg__ptr, x); break; \ 434 case 8: __xchg__res = __xchg_u64(__xchg__ptr, x); break; \ 435 default: __xchg_called_with_bad_pointer(); __xchg__res = x; \ 436 } \ 437 __xchg__res; \ 438}) 439 440#define xchg(ptr,x) \ 441 ({ \ 442 __typeof__(*(ptr)) _x_ = (x); \ 443 (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \ 444 }) 445 446static inline unsigned long 447__xchg_u8_local(volatile char *m, unsigned long val) 448{ 449 unsigned long ret, tmp, addr64; 450 451 __asm__ __volatile__( 452 " andnot %4,7,%3\n" 453 " insbl %1,%4,%1\n" 454 "1: ldq_l %2,0(%3)\n" 455 " extbl %2,%4,%0\n" 456 " mskbl %2,%4,%2\n" 457 " or %1,%2,%2\n" 458 " stq_c %2,0(%3)\n" 459 " beq %2,2f\n" 460 ".subsection 2\n" 461 "2: br 1b\n" 462 ".previous" 463 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) 464 : "r" ((long)m), "1" (val) : "memory"); 465 466 return ret; 467} 468 469static inline unsigned long 470__xchg_u16_local(volatile short *m, unsigned long val) 471{ 472 unsigned long ret, tmp, addr64; 473 474 __asm__ __volatile__( 475 " andnot %4,7,%3\n" 476 " inswl %1,%4,%1\n" 477 "1: ldq_l %2,0(%3)\n" 478 " extwl %2,%4,%0\n" 479 " mskwl %2,%4,%2\n" 480 " or %1,%2,%2\n" 481 " stq_c %2,0(%3)\n" 482 " beq %2,2f\n" 483 ".subsection 2\n" 484 "2: br 1b\n" 485 ".previous" 486 : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) 487 : "r" ((long)m), "1" (val) : "memory"); 488 489 return ret; 490} 491 492static inline unsigned long 493__xchg_u32_local(volatile int *m, unsigned long val) 494{ 495 unsigned long dummy; 496 497 __asm__ __volatile__( 498 "1: ldl_l %0,%4\n" 499 " bis $31,%3,%1\n" 500 " stl_c %1,%2\n" 501 " beq %1,2f\n" 502 ".subsection 2\n" 503 "2: br 1b\n" 504 ".previous" 505 : "=&r" (val), "=&r" (dummy), "=m" (*m) 506 : "rI" (val), "m" (*m) : "memory"); 507 508 return val; 509} 510 511static inline unsigned long 512__xchg_u64_local(volatile long *m, unsigned long val) 513{ 514 unsigned long dummy; 515 516 __asm__ __volatile__( 517 "1: ldq_l %0,%4\n" 518 " bis $31,%3,%1\n" 519 " stq_c %1,%2\n" 520 " beq %1,2f\n" 521 ".subsection 2\n" 522 "2: br 1b\n" 523 ".previous" 524 : "=&r" (val), "=&r" (dummy), "=m" (*m) 525 : "rI" (val), "m" (*m) : "memory"); 526 527 return val; 528} 529 530#define __xchg_local(ptr, x, size) \ 531({ \ 532 unsigned long __xchg__res; \ 533 volatile void *__xchg__ptr = (ptr); \ 534 switch (size) { \ 535 case 1: __xchg__res = __xchg_u8_local(__xchg__ptr, x); break; \ 536 case 2: __xchg__res = __xchg_u16_local(__xchg__ptr, x); break; \ 537 case 4: __xchg__res = __xchg_u32_local(__xchg__ptr, x); break; \ 538 case 8: __xchg__res = __xchg_u64_local(__xchg__ptr, x); break; \ 539 default: __xchg_called_with_bad_pointer(); __xchg__res = x; \ 540 } \ 541 __xchg__res; \ 542}) 543 544#define xchg_local(ptr,x) \ 545 ({ \ 546 __typeof__(*(ptr)) _x_ = (x); \ 547 (__typeof__(*(ptr))) __xchg_local((ptr), (unsigned long)_x_, \ 548 sizeof(*(ptr))); \ 549 }) 550 551/* 552 * Atomic compare and exchange. Compare OLD with MEM, if identical, 553 * store NEW in MEM. Return the initial value in MEM. Success is 554 * indicated by comparing RETURN with OLD. 555 * 556 * The memory barrier should be placed in SMP only when we actually 557 * make the change. If we don't change anything (so if the returned 558 * prev is equal to old) then we aren't acquiring anything new and 559 * we don't need any memory barrier as far I can tell. 560 */ 561 562#define __HAVE_ARCH_CMPXCHG 1 563 564static inline unsigned long 565__cmpxchg_u8(volatile char *m, long old, long new) 566{ 567 unsigned long prev, tmp, cmp, addr64; 568 569 __asm__ __volatile__( 570 " andnot %5,7,%4\n" 571 " insbl %1,%5,%1\n" 572 "1: ldq_l %2,0(%4)\n" 573 " extbl %2,%5,%0\n" 574 " cmpeq %0,%6,%3\n" 575 " beq %3,2f\n" 576 " mskbl %2,%5,%2\n" 577 " or %1,%2,%2\n" 578 " stq_c %2,0(%4)\n" 579 " beq %2,3f\n" 580#ifdef CONFIG_SMP 581 " mb\n" 582#endif 583 "2:\n" 584 ".subsection 2\n" 585 "3: br 1b\n" 586 ".previous" 587 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) 588 : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); 589 590 return prev; 591} 592 593static inline unsigned long 594__cmpxchg_u16(volatile short *m, long old, long new) 595{ 596 unsigned long prev, tmp, cmp, addr64; 597 598 __asm__ __volatile__( 599 " andnot %5,7,%4\n" 600 " inswl %1,%5,%1\n" 601 "1: ldq_l %2,0(%4)\n" 602 " extwl %2,%5,%0\n" 603 " cmpeq %0,%6,%3\n" 604 " beq %3,2f\n" 605 " mskwl %2,%5,%2\n" 606 " or %1,%2,%2\n" 607 " stq_c %2,0(%4)\n" 608 " beq %2,3f\n" 609#ifdef CONFIG_SMP 610 " mb\n" 611#endif 612 "2:\n" 613 ".subsection 2\n" 614 "3: br 1b\n" 615 ".previous" 616 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) 617 : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); 618 619 return prev; 620} 621 622static inline unsigned long 623__cmpxchg_u32(volatile int *m, int old, int new) 624{ 625 unsigned long prev, cmp; 626 627 __asm__ __volatile__( 628 "1: ldl_l %0,%5\n" 629 " cmpeq %0,%3,%1\n" 630 " beq %1,2f\n" 631 " mov %4,%1\n" 632 " stl_c %1,%2\n" 633 " beq %1,3f\n" 634#ifdef CONFIG_SMP 635 " mb\n" 636#endif 637 "2:\n" 638 ".subsection 2\n" 639 "3: br 1b\n" 640 ".previous" 641 : "=&r"(prev), "=&r"(cmp), "=m"(*m) 642 : "r"((long) old), "r"(new), "m"(*m) : "memory"); 643 644 return prev; 645} 646 647static inline unsigned long 648__cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new) 649{ 650 unsigned long prev, cmp; 651 652 __asm__ __volatile__( 653 "1: ldq_l %0,%5\n" 654 " cmpeq %0,%3,%1\n" 655 " beq %1,2f\n" 656 " mov %4,%1\n" 657 " stq_c %1,%2\n" 658 " beq %1,3f\n" 659#ifdef CONFIG_SMP 660 " mb\n" 661#endif 662 "2:\n" 663 ".subsection 2\n" 664 "3: br 1b\n" 665 ".previous" 666 : "=&r"(prev), "=&r"(cmp), "=m"(*m) 667 : "r"((long) old), "r"(new), "m"(*m) : "memory"); 668 669 return prev; 670} 671 672/* This function doesn't exist, so you'll get a linker error 673 if something tries to do an invalid cmpxchg(). */ 674extern void __cmpxchg_called_with_bad_pointer(void); 675 676static __always_inline unsigned long 677__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) 678{ 679 switch (size) { 680 case 1: 681 return __cmpxchg_u8(ptr, old, new); 682 case 2: 683 return __cmpxchg_u16(ptr, old, new); 684 case 4: 685 return __cmpxchg_u32(ptr, old, new); 686 case 8: 687 return __cmpxchg_u64(ptr, old, new); 688 } 689 __cmpxchg_called_with_bad_pointer(); 690 return old; 691} 692 693#define cmpxchg(ptr,o,n) \ 694 ({ \ 695 __typeof__(*(ptr)) _o_ = (o); \ 696 __typeof__(*(ptr)) _n_ = (n); \ 697 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ 698 (unsigned long)_n_, sizeof(*(ptr))); \ 699 }) 700 701static inline unsigned long 702__cmpxchg_u8_local(volatile char *m, long old, long new) 703{ 704 unsigned long prev, tmp, cmp, addr64; 705 706 __asm__ __volatile__( 707 " andnot %5,7,%4\n" 708 " insbl %1,%5,%1\n" 709 "1: ldq_l %2,0(%4)\n" 710 " extbl %2,%5,%0\n" 711 " cmpeq %0,%6,%3\n" 712 " beq %3,2f\n" 713 " mskbl %2,%5,%2\n" 714 " or %1,%2,%2\n" 715 " stq_c %2,0(%4)\n" 716 " beq %2,3f\n" 717 "2:\n" 718 ".subsection 2\n" 719 "3: br 1b\n" 720 ".previous" 721 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) 722 : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); 723 724 return prev; 725} 726 727static inline unsigned long 728__cmpxchg_u16_local(volatile short *m, long old, long new) 729{ 730 unsigned long prev, tmp, cmp, addr64; 731 732 __asm__ __volatile__( 733 " andnot %5,7,%4\n" 734 " inswl %1,%5,%1\n" 735 "1: ldq_l %2,0(%4)\n" 736 " extwl %2,%5,%0\n" 737 " cmpeq %0,%6,%3\n" 738 " beq %3,2f\n" 739 " mskwl %2,%5,%2\n" 740 " or %1,%2,%2\n" 741 " stq_c %2,0(%4)\n" 742 " beq %2,3f\n" 743 "2:\n" 744 ".subsection 2\n" 745 "3: br 1b\n" 746 ".previous" 747 : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) 748 : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); 749 750 return prev; 751} 752 753static inline unsigned long 754__cmpxchg_u32_local(volatile int *m, int old, int new) 755{ 756 unsigned long prev, cmp; 757 758 __asm__ __volatile__( 759 "1: ldl_l %0,%5\n" 760 " cmpeq %0,%3,%1\n" 761 " beq %1,2f\n" 762 " mov %4,%1\n" 763 " stl_c %1,%2\n" 764 " beq %1,3f\n" 765 "2:\n" 766 ".subsection 2\n" 767 "3: br 1b\n" 768 ".previous" 769 : "=&r"(prev), "=&r"(cmp), "=m"(*m) 770 : "r"((long) old), "r"(new), "m"(*m) : "memory"); 771 772 return prev; 773} 774 775static inline unsigned long 776__cmpxchg_u64_local(volatile long *m, unsigned long old, unsigned long new) 777{ 778 unsigned long prev, cmp; 779 780 __asm__ __volatile__( 781 "1: ldq_l %0,%5\n" 782 " cmpeq %0,%3,%1\n" 783 " beq %1,2f\n" 784 " mov %4,%1\n" 785 " stq_c %1,%2\n" 786 " beq %1,3f\n" 787 "2:\n" 788 ".subsection 2\n" 789 "3: br 1b\n" 790 ".previous" 791 : "=&r"(prev), "=&r"(cmp), "=m"(*m) 792 : "r"((long) old), "r"(new), "m"(*m) : "memory"); 793 794 return prev; 795} 796 797static __always_inline unsigned long 798__cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new, 799 int size) 800{ 801 switch (size) { 802 case 1: 803 return __cmpxchg_u8_local(ptr, old, new); 804 case 2: 805 return __cmpxchg_u16_local(ptr, old, new); 806 case 4: 807 return __cmpxchg_u32_local(ptr, old, new); 808 case 8: 809 return __cmpxchg_u64_local(ptr, old, new); 810 } 811 __cmpxchg_called_with_bad_pointer(); 812 return old; 813} 814 815#define cmpxchg_local(ptr,o,n) \ 816 ({ \ 817 __typeof__(*(ptr)) _o_ = (o); \ 818 __typeof__(*(ptr)) _n_ = (n); \ 819 (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \ 820 (unsigned long)_n_, sizeof(*(ptr))); \ 821 }) 822 823#endif /* __ASSEMBLY__ */ 824 825#define arch_align_stack(x) (x) 826 827#endif 828