1210284Sjmallett/***********************license start*************** 2232812Sjmallett * Copyright (c) 2003-2010 Cavium Inc. (support@cavium.com). All rights 3215990Sjmallett * reserved. 4210284Sjmallett * 5210284Sjmallett * 6215990Sjmallett * Redistribution and use in source and binary forms, with or without 7215990Sjmallett * modification, are permitted provided that the following conditions are 8215990Sjmallett * met: 9210284Sjmallett * 10215990Sjmallett * * Redistributions of source code must retain the above copyright 11215990Sjmallett * notice, this list of conditions and the following disclaimer. 12210284Sjmallett * 13215990Sjmallett * * Redistributions in binary form must reproduce the above 14215990Sjmallett * copyright notice, this list of conditions and the following 15215990Sjmallett * disclaimer in the documentation and/or other materials provided 16215990Sjmallett * with the distribution. 17215990Sjmallett 18232812Sjmallett * * Neither the name of Cavium Inc. nor the names of 19215990Sjmallett * its contributors may be used to endorse or promote products 20215990Sjmallett * derived from this software without specific prior written 21215990Sjmallett * permission. 22215990Sjmallett 23215990Sjmallett * This Software, including technical data, may be subject to U.S. export control 24215990Sjmallett * laws, including the U.S. Export Administration Act and its associated 25215990Sjmallett * regulations, and may be subject to export or import regulations in other 26215990Sjmallett * countries. 27215990Sjmallett 28215990Sjmallett * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" 29232812Sjmallett * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR 30215990Sjmallett * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO 31215990Sjmallett * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR 32215990Sjmallett * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM 33215990Sjmallett * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, 34215990Sjmallett * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF 35215990Sjmallett * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR 36215990Sjmallett * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR 37215990Sjmallett * PERFORMANCE OF THE SOFTWARE LIES WITH YOU. 38210284Sjmallett ***********************license end**************************************/ 39210284Sjmallett 40210284Sjmallett 41210284Sjmallett 42210284Sjmallett 43210284Sjmallett 44210284Sjmallett 45215990Sjmallett 46210284Sjmallett/** 47210284Sjmallett * @file 48210284Sjmallett * 49210284Sjmallett * This file provides atomic operations 50210284Sjmallett * 51232812Sjmallett * <hr>$Revision: 70030 $<hr> 52210284Sjmallett * 53210284Sjmallett * 54210284Sjmallett */ 55210284Sjmallett 56210284Sjmallett 57210284Sjmallett#ifndef __CVMX_ATOMIC_H__ 58210284Sjmallett#define __CVMX_ATOMIC_H__ 59210284Sjmallett 60210284Sjmallett#ifdef __cplusplus 61210284Sjmallettextern "C" { 62210284Sjmallett#endif 63210284Sjmallett 64210284Sjmallett 65210284Sjmallett/** 66210284Sjmallett * Atomically adds a signed value to a 32 bit (aligned) memory location. 67210284Sjmallett * 68210284Sjmallett * This version does not perform 'sync' operations to enforce memory 69210284Sjmallett * operations. This should only be used when there are no memory operation 70210284Sjmallett * ordering constraints. (This should NOT be used for reference counting - 71210284Sjmallett * use the standard version instead.) 72210284Sjmallett * 73210284Sjmallett * @param ptr address in memory to add incr to 74210284Sjmallett * @param incr amount to increment memory location by (signed) 75210284Sjmallett */ 76210284Sjmallettstatic inline void cvmx_atomic_add32_nosync(int32_t *ptr, int32_t incr) 77210284Sjmallett{ 78210284Sjmallett if (OCTEON_IS_MODEL(OCTEON_CN3XXX)) 79210284Sjmallett { 80210284Sjmallett uint32_t tmp; 81210284Sjmallett 82210284Sjmallett __asm__ __volatile__( 83210284Sjmallett ".set noreorder \n" 84210284Sjmallett "1: ll %[tmp], %[val] \n" 85210284Sjmallett " addu %[tmp], %[inc] \n" 86210284Sjmallett " sc %[tmp], %[val] \n" 87210284Sjmallett " beqz %[tmp], 1b \n" 88210284Sjmallett " nop \n" 89210284Sjmallett ".set reorder \n" 90210284Sjmallett : [val] "+m" (*ptr), [tmp] "=&r" (tmp) 91210284Sjmallett : [inc] "r" (incr) 92210284Sjmallett : "memory"); 93210284Sjmallett } 94210284Sjmallett else 95210284Sjmallett { 96210284Sjmallett __asm__ __volatile__( 97210284Sjmallett " saa %[inc], (%[base]) \n" 98210284Sjmallett : "+m" (*ptr) 99210284Sjmallett : [inc] "r" (incr), [base] "r" (ptr) 100210284Sjmallett : "memory"); 101210284Sjmallett } 102210284Sjmallett} 103210284Sjmallett 104210284Sjmallett/** 105210284Sjmallett * Atomically adds a signed value to a 32 bit (aligned) memory location. 106210284Sjmallett * 107210284Sjmallett * Memory access ordering is enforced before/after the atomic operation, 108210284Sjmallett * so no additional 'sync' instructions are required. 109210284Sjmallett * 110210284Sjmallett * 111210284Sjmallett * @param ptr address in memory to add incr to 112210284Sjmallett * @param incr amount to increment memory location by (signed) 113210284Sjmallett */ 114210284Sjmallettstatic inline void cvmx_atomic_add32(int32_t *ptr, int32_t incr) 115210284Sjmallett{ 116210284Sjmallett CVMX_SYNCWS; 117210284Sjmallett cvmx_atomic_add32_nosync(ptr, incr); 118210284Sjmallett CVMX_SYNCWS; 119210284Sjmallett} 120210284Sjmallett 121210284Sjmallett/** 122210284Sjmallett * Atomically sets a 32 bit (aligned) memory location to a value 123210284Sjmallett * 124210284Sjmallett * @param ptr address of memory to set 125210284Sjmallett * @param value value to set memory location to. 126210284Sjmallett */ 127210284Sjmallettstatic inline void cvmx_atomic_set32(int32_t *ptr, int32_t value) 128210284Sjmallett{ 129210284Sjmallett CVMX_SYNCWS; 130210284Sjmallett *ptr = value; 131210284Sjmallett CVMX_SYNCWS; 132210284Sjmallett} 133210284Sjmallett 134210284Sjmallett/** 135210284Sjmallett * Returns the current value of a 32 bit (aligned) memory 136210284Sjmallett * location. 137210284Sjmallett * 138210284Sjmallett * @param ptr Address of memory to get 139210284Sjmallett * @return Value of the memory 140210284Sjmallett */ 141210284Sjmallettstatic inline int32_t cvmx_atomic_get32(int32_t *ptr) 142210284Sjmallett{ 143210284Sjmallett return *(volatile int32_t *)ptr; 144210284Sjmallett} 145210284Sjmallett 146210284Sjmallett/** 147210284Sjmallett * Atomically adds a signed value to a 64 bit (aligned) memory location. 148210284Sjmallett * 149210284Sjmallett * This version does not perform 'sync' operations to enforce memory 150210284Sjmallett * operations. This should only be used when there are no memory operation 151210284Sjmallett * ordering constraints. (This should NOT be used for reference counting - 152210284Sjmallett * use the standard version instead.) 153210284Sjmallett * 154210284Sjmallett * @param ptr address in memory to add incr to 155210284Sjmallett * @param incr amount to increment memory location by (signed) 156210284Sjmallett */ 157210284Sjmallettstatic inline void cvmx_atomic_add64_nosync(int64_t *ptr, int64_t incr) 158210284Sjmallett{ 159210284Sjmallett if (OCTEON_IS_MODEL(OCTEON_CN3XXX)) 160210284Sjmallett { 161210284Sjmallett uint64_t tmp; 162210284Sjmallett __asm__ __volatile__( 163210284Sjmallett ".set noreorder \n" 164210284Sjmallett "1: lld %[tmp], %[val] \n" 165210284Sjmallett " daddu %[tmp], %[inc] \n" 166210284Sjmallett " scd %[tmp], %[val] \n" 167210284Sjmallett " beqz %[tmp], 1b \n" 168210284Sjmallett " nop \n" 169210284Sjmallett ".set reorder \n" 170210284Sjmallett : [val] "+m" (*ptr), [tmp] "=&r" (tmp) 171210284Sjmallett : [inc] "r" (incr) 172210284Sjmallett : "memory"); 173210284Sjmallett } 174210284Sjmallett else 175210284Sjmallett { 176210284Sjmallett __asm__ __volatile__( 177210284Sjmallett " saad %[inc], (%[base]) \n" 178210284Sjmallett : "+m" (*ptr) 179210284Sjmallett : [inc] "r" (incr), [base] "r" (ptr) 180210284Sjmallett : "memory"); 181210284Sjmallett } 182210284Sjmallett} 183210284Sjmallett 184210284Sjmallett/** 185210284Sjmallett * Atomically adds a signed value to a 64 bit (aligned) memory location. 186210284Sjmallett * 187210284Sjmallett * Memory access ordering is enforced before/after the atomic operation, 188210284Sjmallett * so no additional 'sync' instructions are required. 189210284Sjmallett * 190210284Sjmallett * 191210284Sjmallett * @param ptr address in memory to add incr to 192210284Sjmallett * @param incr amount to increment memory location by (signed) 193210284Sjmallett */ 194210284Sjmallettstatic inline void cvmx_atomic_add64(int64_t *ptr, int64_t incr) 195210284Sjmallett{ 196210284Sjmallett CVMX_SYNCWS; 197210284Sjmallett cvmx_atomic_add64_nosync(ptr, incr); 198210284Sjmallett CVMX_SYNCWS; 199210284Sjmallett} 200210284Sjmallett 201210284Sjmallett/** 202210284Sjmallett * Atomically sets a 64 bit (aligned) memory location to a value 203210284Sjmallett * 204210284Sjmallett * @param ptr address of memory to set 205210284Sjmallett * @param value value to set memory location to. 206210284Sjmallett */ 207210284Sjmallettstatic inline void cvmx_atomic_set64(int64_t *ptr, int64_t value) 208210284Sjmallett{ 209210284Sjmallett CVMX_SYNCWS; 210210284Sjmallett *ptr = value; 211210284Sjmallett CVMX_SYNCWS; 212210284Sjmallett} 213210284Sjmallett 214210284Sjmallett/** 215210284Sjmallett * Returns the current value of a 64 bit (aligned) memory 216210284Sjmallett * location. 217210284Sjmallett * 218210284Sjmallett * @param ptr Address of memory to get 219210284Sjmallett * @return Value of the memory 220210284Sjmallett */ 221210284Sjmallettstatic inline int64_t cvmx_atomic_get64(int64_t *ptr) 222210284Sjmallett{ 223210284Sjmallett return *(volatile int64_t *)ptr; 224210284Sjmallett} 225210284Sjmallett 226210284Sjmallett/** 227210284Sjmallett * Atomically compares the old value with the value at ptr, and if they match, 228210284Sjmallett * stores new_val to ptr. 229210284Sjmallett * If *ptr and old don't match, function returns failure immediately. 230210284Sjmallett * If *ptr and old match, function spins until *ptr updated to new atomically, or 231210284Sjmallett * until *ptr and old no longer match 232210284Sjmallett * 233210284Sjmallett * Does no memory synchronization. 234210284Sjmallett * 235210284Sjmallett * @return 1 on success (match and store) 236210284Sjmallett * 0 on no match 237210284Sjmallett */ 238210284Sjmallettstatic inline uint32_t cvmx_atomic_compare_and_store32_nosync(uint32_t *ptr, uint32_t old_val, uint32_t new_val) 239210284Sjmallett{ 240210284Sjmallett uint32_t tmp, ret; 241210284Sjmallett 242210284Sjmallett __asm__ __volatile__( 243210284Sjmallett ".set noreorder \n" 244210284Sjmallett "1: ll %[tmp], %[val] \n" 245210284Sjmallett " li %[ret], 0 \n" 246210284Sjmallett " bne %[tmp], %[old], 2f \n" 247210284Sjmallett " move %[tmp], %[new_val] \n" 248210284Sjmallett " sc %[tmp], %[val] \n" 249210284Sjmallett " beqz %[tmp], 1b \n" 250210284Sjmallett " li %[ret], 1 \n" 251210284Sjmallett "2: nop \n" 252210284Sjmallett ".set reorder \n" 253210284Sjmallett : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret) 254210284Sjmallett : [old] "r" (old_val), [new_val] "r" (new_val) 255210284Sjmallett : "memory"); 256210284Sjmallett 257210284Sjmallett return(ret); 258210284Sjmallett 259210284Sjmallett} 260210284Sjmallett 261210284Sjmallett/** 262210284Sjmallett * Atomically compares the old value with the value at ptr, and if they match, 263210284Sjmallett * stores new_val to ptr. 264210284Sjmallett * If *ptr and old don't match, function returns failure immediately. 265210284Sjmallett * If *ptr and old match, function spins until *ptr updated to new atomically, or 266210284Sjmallett * until *ptr and old no longer match 267210284Sjmallett * 268210284Sjmallett * Does memory synchronization that is required to use this as a locking primitive. 269210284Sjmallett * 270210284Sjmallett * @return 1 on success (match and store) 271210284Sjmallett * 0 on no match 272210284Sjmallett */ 273210284Sjmallettstatic inline uint32_t cvmx_atomic_compare_and_store32(uint32_t *ptr, uint32_t old_val, uint32_t new_val) 274210284Sjmallett{ 275210284Sjmallett uint32_t ret; 276210284Sjmallett CVMX_SYNCWS; 277210284Sjmallett ret = cvmx_atomic_compare_and_store32_nosync(ptr, old_val, new_val); 278210284Sjmallett CVMX_SYNCWS; 279210284Sjmallett return ret; 280210284Sjmallett 281210284Sjmallett 282210284Sjmallett} 283210284Sjmallett 284210284Sjmallett/** 285210284Sjmallett * Atomically compares the old value with the value at ptr, and if they match, 286210284Sjmallett * stores new_val to ptr. 287210284Sjmallett * If *ptr and old don't match, function returns failure immediately. 288210284Sjmallett * If *ptr and old match, function spins until *ptr updated to new atomically, or 289210284Sjmallett * until *ptr and old no longer match 290210284Sjmallett * 291210284Sjmallett * Does no memory synchronization. 292210284Sjmallett * 293210284Sjmallett * @return 1 on success (match and store) 294210284Sjmallett * 0 on no match 295210284Sjmallett */ 296210284Sjmallettstatic inline uint64_t cvmx_atomic_compare_and_store64_nosync(uint64_t *ptr, uint64_t old_val, uint64_t new_val) 297210284Sjmallett{ 298210284Sjmallett uint64_t tmp, ret; 299210284Sjmallett 300210284Sjmallett __asm__ __volatile__( 301210284Sjmallett ".set noreorder \n" 302210284Sjmallett "1: lld %[tmp], %[val] \n" 303210284Sjmallett " li %[ret], 0 \n" 304210284Sjmallett " bne %[tmp], %[old], 2f \n" 305210284Sjmallett " move %[tmp], %[new_val] \n" 306210284Sjmallett " scd %[tmp], %[val] \n" 307210284Sjmallett " beqz %[tmp], 1b \n" 308210284Sjmallett " li %[ret], 1 \n" 309210284Sjmallett "2: nop \n" 310210284Sjmallett ".set reorder \n" 311210284Sjmallett : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret) 312210284Sjmallett : [old] "r" (old_val), [new_val] "r" (new_val) 313210284Sjmallett : "memory"); 314210284Sjmallett 315210284Sjmallett return(ret); 316210284Sjmallett 317210284Sjmallett} 318210284Sjmallett 319210284Sjmallett/** 320210284Sjmallett * Atomically compares the old value with the value at ptr, and if they match, 321210284Sjmallett * stores new_val to ptr. 322210284Sjmallett * If *ptr and old don't match, function returns failure immediately. 323210284Sjmallett * If *ptr and old match, function spins until *ptr updated to new atomically, or 324210284Sjmallett * until *ptr and old no longer match 325210284Sjmallett * 326210284Sjmallett * Does memory synchronization that is required to use this as a locking primitive. 327210284Sjmallett * 328210284Sjmallett * @return 1 on success (match and store) 329210284Sjmallett * 0 on no match 330210284Sjmallett */ 331210284Sjmallettstatic inline uint64_t cvmx_atomic_compare_and_store64(uint64_t *ptr, uint64_t old_val, uint64_t new_val) 332210284Sjmallett{ 333210284Sjmallett uint64_t ret; 334210284Sjmallett CVMX_SYNCWS; 335210284Sjmallett ret = cvmx_atomic_compare_and_store64_nosync(ptr, old_val, new_val); 336210284Sjmallett CVMX_SYNCWS; 337210284Sjmallett return ret; 338210284Sjmallett} 339210284Sjmallett 340210284Sjmallett/** 341210284Sjmallett * Atomically adds a signed value to a 64 bit (aligned) memory location, 342210284Sjmallett * and returns previous value. 343210284Sjmallett * 344210284Sjmallett * This version does not perform 'sync' operations to enforce memory 345210284Sjmallett * operations. This should only be used when there are no memory operation 346210284Sjmallett * ordering constraints. (This should NOT be used for reference counting - 347210284Sjmallett * use the standard version instead.) 348210284Sjmallett * 349210284Sjmallett * @param ptr address in memory to add incr to 350210284Sjmallett * @param incr amount to increment memory location by (signed) 351210284Sjmallett * 352210284Sjmallett * @return Value of memory location before increment 353210284Sjmallett */ 354210284Sjmallettstatic inline int64_t cvmx_atomic_fetch_and_add64_nosync(int64_t *ptr, int64_t incr) 355210284Sjmallett{ 356210284Sjmallett uint64_t tmp, ret; 357210284Sjmallett 358215990Sjmallett#if !defined(__FreeBSD__) || !defined(_KERNEL) 359232812Sjmallett if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX)) 360215990Sjmallett { 361215990Sjmallett CVMX_PUSH_OCTEON2; 362215990Sjmallett if (__builtin_constant_p(incr) && incr == 1) 363215990Sjmallett { 364215990Sjmallett __asm__ __volatile__( 365215990Sjmallett "laid %0,(%2)" 366215990Sjmallett : "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory"); 367215990Sjmallett } 368215990Sjmallett else if (__builtin_constant_p(incr) && incr == -1) 369215990Sjmallett { 370215990Sjmallett __asm__ __volatile__( 371215990Sjmallett "ladd %0,(%2)" 372215990Sjmallett : "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory"); 373215990Sjmallett } 374215990Sjmallett else 375215990Sjmallett { 376215990Sjmallett __asm__ __volatile__( 377215990Sjmallett "laad %0,(%2),%3" 378215990Sjmallett : "=r" (ret), "+m" (ptr) : "r" (ptr), "r" (incr) : "memory"); 379215990Sjmallett } 380215990Sjmallett CVMX_POP_OCTEON2; 381215990Sjmallett } 382215990Sjmallett else 383215990Sjmallett { 384215990Sjmallett#endif 385215990Sjmallett __asm__ __volatile__( 386215990Sjmallett ".set noreorder \n" 387215990Sjmallett "1: lld %[tmp], %[val] \n" 388215990Sjmallett " move %[ret], %[tmp] \n" 389215990Sjmallett " daddu %[tmp], %[inc] \n" 390215990Sjmallett " scd %[tmp], %[val] \n" 391215990Sjmallett " beqz %[tmp], 1b \n" 392215990Sjmallett " nop \n" 393215990Sjmallett ".set reorder \n" 394215990Sjmallett : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret) 395215990Sjmallett : [inc] "r" (incr) 396215990Sjmallett : "memory"); 397215990Sjmallett#if !defined(__FreeBSD__) || !defined(_KERNEL) 398215990Sjmallett } 399215990Sjmallett#endif 400210284Sjmallett 401210284Sjmallett return (ret); 402210284Sjmallett} 403210284Sjmallett 404210284Sjmallett/** 405210284Sjmallett * Atomically adds a signed value to a 64 bit (aligned) memory location, 406210284Sjmallett * and returns previous value. 407210284Sjmallett * 408210284Sjmallett * Memory access ordering is enforced before/after the atomic operation, 409210284Sjmallett * so no additional 'sync' instructions are required. 410210284Sjmallett * 411210284Sjmallett * @param ptr address in memory to add incr to 412210284Sjmallett * @param incr amount to increment memory location by (signed) 413210284Sjmallett * 414210284Sjmallett * @return Value of memory location before increment 415210284Sjmallett */ 416210284Sjmallettstatic inline int64_t cvmx_atomic_fetch_and_add64(int64_t *ptr, int64_t incr) 417210284Sjmallett{ 418210284Sjmallett uint64_t ret; 419210284Sjmallett CVMX_SYNCWS; 420210284Sjmallett ret = cvmx_atomic_fetch_and_add64_nosync(ptr, incr); 421210284Sjmallett CVMX_SYNCWS; 422210284Sjmallett return ret; 423210284Sjmallett} 424210284Sjmallett 425210284Sjmallett/** 426210284Sjmallett * Atomically adds a signed value to a 32 bit (aligned) memory location, 427210284Sjmallett * and returns previous value. 428210284Sjmallett * 429210284Sjmallett * This version does not perform 'sync' operations to enforce memory 430210284Sjmallett * operations. This should only be used when there are no memory operation 431210284Sjmallett * ordering constraints. (This should NOT be used for reference counting - 432210284Sjmallett * use the standard version instead.) 433210284Sjmallett * 434210284Sjmallett * @param ptr address in memory to add incr to 435210284Sjmallett * @param incr amount to increment memory location by (signed) 436210284Sjmallett * 437210284Sjmallett * @return Value of memory location before increment 438210284Sjmallett */ 439210284Sjmallettstatic inline int32_t cvmx_atomic_fetch_and_add32_nosync(int32_t *ptr, int32_t incr) 440210284Sjmallett{ 441210284Sjmallett uint32_t tmp, ret; 442210284Sjmallett 443215990Sjmallett#if !defined(__FreeBSD__) || !defined(_KERNEL) 444232812Sjmallett if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX)) 445215990Sjmallett { 446215990Sjmallett CVMX_PUSH_OCTEON2; 447215990Sjmallett if (__builtin_constant_p(incr) && incr == 1) 448215990Sjmallett { 449215990Sjmallett __asm__ __volatile__( 450215990Sjmallett "lai %0,(%2)" 451215990Sjmallett : "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory"); 452215990Sjmallett } 453215990Sjmallett else if (__builtin_constant_p(incr) && incr == -1) 454215990Sjmallett { 455215990Sjmallett __asm__ __volatile__( 456215990Sjmallett "lad %0,(%2)" 457215990Sjmallett : "=r" (ret), "+m" (ptr) : "r" (ptr) : "memory"); 458215990Sjmallett } 459215990Sjmallett else 460215990Sjmallett { 461215990Sjmallett __asm__ __volatile__( 462215990Sjmallett "laa %0,(%2),%3" 463215990Sjmallett : "=r" (ret), "+m" (ptr) : "r" (ptr), "r" (incr) : "memory"); 464215990Sjmallett } 465215990Sjmallett CVMX_POP_OCTEON2; 466215990Sjmallett } 467215990Sjmallett else 468215990Sjmallett { 469215990Sjmallett#endif 470215990Sjmallett __asm__ __volatile__( 471215990Sjmallett ".set noreorder \n" 472215990Sjmallett "1: ll %[tmp], %[val] \n" 473215990Sjmallett " move %[ret], %[tmp] \n" 474215990Sjmallett " addu %[tmp], %[inc] \n" 475215990Sjmallett " sc %[tmp], %[val] \n" 476215990Sjmallett " beqz %[tmp], 1b \n" 477215990Sjmallett " nop \n" 478215990Sjmallett ".set reorder \n" 479215990Sjmallett : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret) 480215990Sjmallett : [inc] "r" (incr) 481215990Sjmallett : "memory"); 482215990Sjmallett#if !defined(__FreeBSD__) || !defined(_KERNEL) 483215990Sjmallett } 484215990Sjmallett#endif 485210284Sjmallett 486210284Sjmallett return (ret); 487210284Sjmallett} 488210284Sjmallett 489210284Sjmallett/** 490210284Sjmallett * Atomically adds a signed value to a 32 bit (aligned) memory location, 491210284Sjmallett * and returns previous value. 492210284Sjmallett * 493210284Sjmallett * Memory access ordering is enforced before/after the atomic operation, 494210284Sjmallett * so no additional 'sync' instructions are required. 495210284Sjmallett * 496210284Sjmallett * @param ptr address in memory to add incr to 497210284Sjmallett * @param incr amount to increment memory location by (signed) 498210284Sjmallett * 499210284Sjmallett * @return Value of memory location before increment 500210284Sjmallett */ 501210284Sjmallettstatic inline int32_t cvmx_atomic_fetch_and_add32(int32_t *ptr, int32_t incr) 502210284Sjmallett{ 503210284Sjmallett uint32_t ret; 504210284Sjmallett CVMX_SYNCWS; 505210284Sjmallett ret = cvmx_atomic_fetch_and_add32_nosync(ptr, incr); 506210284Sjmallett CVMX_SYNCWS; 507210284Sjmallett return ret; 508210284Sjmallett} 509210284Sjmallett 510210284Sjmallett/** 511210284Sjmallett * Atomically set bits in a 64 bit (aligned) memory location, 512210284Sjmallett * and returns previous value. 513210284Sjmallett * 514210284Sjmallett * This version does not perform 'sync' operations to enforce memory 515210284Sjmallett * operations. This should only be used when there are no memory operation 516210284Sjmallett * ordering constraints. 517210284Sjmallett * 518210284Sjmallett * @param ptr address in memory 519210284Sjmallett * @param mask mask of bits to set 520210284Sjmallett * 521210284Sjmallett * @return Value of memory location before setting bits 522210284Sjmallett */ 523210284Sjmallettstatic inline uint64_t cvmx_atomic_fetch_and_bset64_nosync(uint64_t *ptr, uint64_t mask) 524210284Sjmallett{ 525210284Sjmallett uint64_t tmp, ret; 526210284Sjmallett 527210284Sjmallett __asm__ __volatile__( 528210284Sjmallett ".set noreorder \n" 529210284Sjmallett "1: lld %[tmp], %[val] \n" 530210284Sjmallett " move %[ret], %[tmp] \n" 531210284Sjmallett " or %[tmp], %[msk] \n" 532210284Sjmallett " scd %[tmp], %[val] \n" 533210284Sjmallett " beqz %[tmp], 1b \n" 534210284Sjmallett " nop \n" 535210284Sjmallett ".set reorder \n" 536210284Sjmallett : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret) 537210284Sjmallett : [msk] "r" (mask) 538210284Sjmallett : "memory"); 539210284Sjmallett 540210284Sjmallett return (ret); 541210284Sjmallett} 542210284Sjmallett 543210284Sjmallett/** 544210284Sjmallett * Atomically set bits in a 32 bit (aligned) memory location, 545210284Sjmallett * and returns previous value. 546210284Sjmallett * 547210284Sjmallett * This version does not perform 'sync' operations to enforce memory 548210284Sjmallett * operations. This should only be used when there are no memory operation 549210284Sjmallett * ordering constraints. 550210284Sjmallett * 551210284Sjmallett * @param ptr address in memory 552210284Sjmallett * @param mask mask of bits to set 553210284Sjmallett * 554210284Sjmallett * @return Value of memory location before setting bits 555210284Sjmallett */ 556210284Sjmallettstatic inline uint32_t cvmx_atomic_fetch_and_bset32_nosync(uint32_t *ptr, uint32_t mask) 557210284Sjmallett{ 558210284Sjmallett uint32_t tmp, ret; 559210284Sjmallett 560210284Sjmallett __asm__ __volatile__( 561210284Sjmallett ".set noreorder \n" 562210284Sjmallett "1: ll %[tmp], %[val] \n" 563210284Sjmallett " move %[ret], %[tmp] \n" 564210284Sjmallett " or %[tmp], %[msk] \n" 565210284Sjmallett " sc %[tmp], %[val] \n" 566210284Sjmallett " beqz %[tmp], 1b \n" 567210284Sjmallett " nop \n" 568210284Sjmallett ".set reorder \n" 569210284Sjmallett : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret) 570210284Sjmallett : [msk] "r" (mask) 571210284Sjmallett : "memory"); 572210284Sjmallett 573210284Sjmallett return (ret); 574210284Sjmallett} 575210284Sjmallett 576210284Sjmallett/** 577210284Sjmallett * Atomically clear bits in a 64 bit (aligned) memory location, 578210284Sjmallett * and returns previous value. 579210284Sjmallett * 580210284Sjmallett * This version does not perform 'sync' operations to enforce memory 581210284Sjmallett * operations. This should only be used when there are no memory operation 582210284Sjmallett * ordering constraints. 583210284Sjmallett * 584210284Sjmallett * @param ptr address in memory 585210284Sjmallett * @param mask mask of bits to clear 586210284Sjmallett * 587210284Sjmallett * @return Value of memory location before clearing bits 588210284Sjmallett */ 589210284Sjmallettstatic inline uint64_t cvmx_atomic_fetch_and_bclr64_nosync(uint64_t *ptr, uint64_t mask) 590210284Sjmallett{ 591210284Sjmallett uint64_t tmp, ret; 592210284Sjmallett 593210284Sjmallett __asm__ __volatile__( 594210284Sjmallett ".set noreorder \n" 595210284Sjmallett " nor %[msk], 0 \n" 596210284Sjmallett "1: lld %[tmp], %[val] \n" 597210284Sjmallett " move %[ret], %[tmp] \n" 598210284Sjmallett " and %[tmp], %[msk] \n" 599210284Sjmallett " scd %[tmp], %[val] \n" 600210284Sjmallett " beqz %[tmp], 1b \n" 601210284Sjmallett " nop \n" 602210284Sjmallett ".set reorder \n" 603215990Sjmallett : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret), [msk] "+r" (mask) 604215990Sjmallett : : "memory"); 605210284Sjmallett 606210284Sjmallett return (ret); 607210284Sjmallett} 608210284Sjmallett 609210284Sjmallett/** 610210284Sjmallett * Atomically clear bits in a 32 bit (aligned) memory location, 611210284Sjmallett * and returns previous value. 612210284Sjmallett * 613210284Sjmallett * This version does not perform 'sync' operations to enforce memory 614210284Sjmallett * operations. This should only be used when there are no memory operation 615210284Sjmallett * ordering constraints. 616210284Sjmallett * 617210284Sjmallett * @param ptr address in memory 618210284Sjmallett * @param mask mask of bits to clear 619210284Sjmallett * 620210284Sjmallett * @return Value of memory location before clearing bits 621210284Sjmallett */ 622210284Sjmallettstatic inline uint32_t cvmx_atomic_fetch_and_bclr32_nosync(uint32_t *ptr, uint32_t mask) 623210284Sjmallett{ 624210284Sjmallett uint32_t tmp, ret; 625210284Sjmallett 626210284Sjmallett __asm__ __volatile__( 627210284Sjmallett ".set noreorder \n" 628210284Sjmallett " nor %[msk], 0 \n" 629210284Sjmallett "1: ll %[tmp], %[val] \n" 630210284Sjmallett " move %[ret], %[tmp] \n" 631210284Sjmallett " and %[tmp], %[msk] \n" 632210284Sjmallett " sc %[tmp], %[val] \n" 633210284Sjmallett " beqz %[tmp], 1b \n" 634210284Sjmallett " nop \n" 635210284Sjmallett ".set reorder \n" 636215990Sjmallett : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret), [msk] "+r" (mask) 637215990Sjmallett : : "memory"); 638210284Sjmallett 639210284Sjmallett return (ret); 640210284Sjmallett} 641210284Sjmallett 642210284Sjmallett/** 643210284Sjmallett * Atomically swaps value in 64 bit (aligned) memory location, 644210284Sjmallett * and returns previous value. 645210284Sjmallett * 646210284Sjmallett * This version does not perform 'sync' operations to enforce memory 647210284Sjmallett * operations. This should only be used when there are no memory operation 648210284Sjmallett * ordering constraints. 649210284Sjmallett * 650210284Sjmallett * @param ptr address in memory 651210284Sjmallett * @param new_val new value to write 652210284Sjmallett * 653210284Sjmallett * @return Value of memory location before swap operation 654210284Sjmallett */ 655210284Sjmallettstatic inline uint64_t cvmx_atomic_swap64_nosync(uint64_t *ptr, uint64_t new_val) 656210284Sjmallett{ 657210284Sjmallett uint64_t tmp, ret; 658210284Sjmallett 659215990Sjmallett#if !defined(__FreeBSD__) || !defined(_KERNEL) 660232812Sjmallett if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX)) 661215990Sjmallett { 662215990Sjmallett CVMX_PUSH_OCTEON2; 663215990Sjmallett if (__builtin_constant_p(new_val) && new_val == 0) 664215990Sjmallett { 665215990Sjmallett __asm__ __volatile__( 666215990Sjmallett "lacd %0,(%1)" 667215990Sjmallett : "=r" (ret) : "r" (ptr) : "memory"); 668215990Sjmallett } 669215990Sjmallett else if (__builtin_constant_p(new_val) && new_val == ~0ull) 670215990Sjmallett { 671215990Sjmallett __asm__ __volatile__( 672215990Sjmallett "lasd %0,(%1)" 673215990Sjmallett : "=r" (ret) : "r" (ptr) : "memory"); 674215990Sjmallett } 675215990Sjmallett else 676215990Sjmallett { 677215990Sjmallett __asm__ __volatile__( 678215990Sjmallett "lawd %0,(%1),%2" 679215990Sjmallett : "=r" (ret) : "r" (ptr), "r" (new_val) : "memory"); 680215990Sjmallett } 681215990Sjmallett CVMX_POP_OCTEON2; 682215990Sjmallett } 683215990Sjmallett else 684215990Sjmallett { 685215990Sjmallett#endif 686215990Sjmallett __asm__ __volatile__( 687215990Sjmallett ".set noreorder \n" 688215990Sjmallett "1: lld %[ret], %[val] \n" 689215990Sjmallett " move %[tmp], %[new_val] \n" 690215990Sjmallett " scd %[tmp], %[val] \n" 691215990Sjmallett " beqz %[tmp], 1b \n" 692215990Sjmallett " nop \n" 693215990Sjmallett ".set reorder \n" 694215990Sjmallett : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret) 695215990Sjmallett : [new_val] "r" (new_val) 696215990Sjmallett : "memory"); 697215990Sjmallett#if !defined(__FreeBSD__) || !defined(_KERNEL) 698215990Sjmallett } 699215990Sjmallett#endif 700210284Sjmallett 701210284Sjmallett return (ret); 702210284Sjmallett} 703210284Sjmallett 704210284Sjmallett/** 705210284Sjmallett * Atomically swaps value in 32 bit (aligned) memory location, 706210284Sjmallett * and returns previous value. 707210284Sjmallett * 708210284Sjmallett * This version does not perform 'sync' operations to enforce memory 709210284Sjmallett * operations. This should only be used when there are no memory operation 710210284Sjmallett * ordering constraints. 711210284Sjmallett * 712210284Sjmallett * @param ptr address in memory 713210284Sjmallett * @param new_val new value to write 714210284Sjmallett * 715210284Sjmallett * @return Value of memory location before swap operation 716210284Sjmallett */ 717210284Sjmallettstatic inline uint32_t cvmx_atomic_swap32_nosync(uint32_t *ptr, uint32_t new_val) 718210284Sjmallett{ 719210284Sjmallett uint32_t tmp, ret; 720210284Sjmallett 721215990Sjmallett#if !defined(__FreeBSD__) || !defined(_KERNEL) 722232812Sjmallett if (OCTEON_IS_MODEL(OCTEON_CN6XXX) || OCTEON_IS_MODEL(OCTEON_CNF7XXX)) 723215990Sjmallett { 724215990Sjmallett CVMX_PUSH_OCTEON2; 725215990Sjmallett if (__builtin_constant_p(new_val) && new_val == 0) 726215990Sjmallett { 727215990Sjmallett __asm__ __volatile__( 728215990Sjmallett "lac %0,(%1)" 729215990Sjmallett : "=r" (ret) : "r" (ptr) : "memory"); 730215990Sjmallett } 731215990Sjmallett else if (__builtin_constant_p(new_val) && new_val == ~0u) 732215990Sjmallett { 733215990Sjmallett __asm__ __volatile__( 734215990Sjmallett "las %0,(%1)" 735215990Sjmallett : "=r" (ret) : "r" (ptr) : "memory"); 736215990Sjmallett } 737215990Sjmallett else 738215990Sjmallett { 739215990Sjmallett __asm__ __volatile__( 740215990Sjmallett "law %0,(%1),%2" 741215990Sjmallett : "=r" (ret) : "r" (ptr), "r" (new_val) : "memory"); 742215990Sjmallett } 743215990Sjmallett CVMX_POP_OCTEON2; 744215990Sjmallett } 745215990Sjmallett else 746215990Sjmallett { 747215990Sjmallett#endif 748215990Sjmallett __asm__ __volatile__( 749215990Sjmallett ".set noreorder \n" 750215990Sjmallett "1: ll %[ret], %[val] \n" 751215990Sjmallett " move %[tmp], %[new_val] \n" 752215990Sjmallett " sc %[tmp], %[val] \n" 753215990Sjmallett " beqz %[tmp], 1b \n" 754215990Sjmallett " nop \n" 755215990Sjmallett ".set reorder \n" 756215990Sjmallett : [val] "+m" (*ptr), [tmp] "=&r" (tmp), [ret] "=&r" (ret) 757215990Sjmallett : [new_val] "r" (new_val) 758215990Sjmallett : "memory"); 759215990Sjmallett#if !defined(__FreeBSD__) || !defined(_KERNEL) 760215990Sjmallett } 761215990Sjmallett#endif 762210284Sjmallett 763210284Sjmallett return (ret); 764210284Sjmallett} 765210284Sjmallett 766210284Sjmallett#ifdef __cplusplus 767210284Sjmallett} 768210284Sjmallett#endif 769210284Sjmallett 770210284Sjmallett#endif /* __CVMX_ATOMIC_H__ */ 771