opensolaris_atomic.S revision 168675
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22/* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 .ident "%Z%%M% %I% %E% SMI" 28 29 .file "%M%" 30 31#define _ASM 32#include <sys/asm_linkage.h> 33 34#if defined(_KERNEL) 35 /* 36 * Legacy kernel interfaces; they will go away (eventually). 37 */ 38 ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function) 39 ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function) 40 ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function) 41 ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function) 42 ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function) 43 ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function) 44 ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function) 45#endif 46 47 ENTRY(atomic_inc_8) 48 ALTENTRY(atomic_inc_uchar) 49 lock 50 incb (%rdi) 51 ret 52 SET_SIZE(atomic_inc_uchar) 53 SET_SIZE(atomic_inc_8) 54 55 ENTRY(atomic_inc_16) 56 ALTENTRY(atomic_inc_ushort) 57 lock 58 incw (%rdi) 59 ret 60 SET_SIZE(atomic_inc_ushort) 61 SET_SIZE(atomic_inc_16) 62 63 ENTRY(atomic_inc_32) 64 ALTENTRY(atomic_inc_uint) 65 lock 66 incl (%rdi) 67 ret 68 SET_SIZE(atomic_inc_uint) 69 SET_SIZE(atomic_inc_32) 70 71 ENTRY(atomic_inc_64) 72 ALTENTRY(atomic_inc_ulong) 73 lock 74 incq (%rdi) 75 ret 76 SET_SIZE(atomic_inc_ulong) 77 SET_SIZE(atomic_inc_64) 78 79 ENTRY(atomic_inc_8_nv) 80 ALTENTRY(atomic_inc_uchar_nv) 81 movb (%rdi), %al // %al = old value 821: 83 leaq 1(%rax), %rcx // %cl = new value 84 lock 85 cmpxchgb %cl, (%rdi) // try to stick it in 86 jne 1b 87 movzbl %cl, %eax // return new value 88 ret 89 SET_SIZE(atomic_inc_uchar_nv) 90 SET_SIZE(atomic_inc_8_nv) 91 92 ENTRY(atomic_inc_16_nv) 93 ALTENTRY(atomic_inc_ushort_nv) 94 movw (%rdi), %ax // %ax = old value 951: 96 leaq 1(%rax), %rcx // %cx = new value 97 lock 98 cmpxchgw %cx, (%rdi) // try to stick it in 99 jne 1b 100 movzwl %cx, %eax // return new value 101 ret 102 SET_SIZE(atomic_inc_ushort_nv) 103 SET_SIZE(atomic_inc_16_nv) 104 105 ENTRY(atomic_inc_32_nv) 106 ALTENTRY(atomic_inc_uint_nv) 107 movl (%rdi), %eax // %eax = old value 1081: 109 leaq 1(%rax), %rcx // %ecx = new value 110 lock 111 cmpxchgl %ecx, (%rdi) // try to stick it in 112 jne 1b 113 movl %ecx, %eax // return new value 114 ret 115 SET_SIZE(atomic_inc_uint_nv) 116 SET_SIZE(atomic_inc_32_nv) 117 118 ENTRY(atomic_inc_64_nv) 119 ALTENTRY(atomic_inc_ulong_nv) 120 movq (%rdi), %rax // %rax = old value 1211: 122 leaq 1(%rax), %rcx // %rcx = new value 123 lock 124 cmpxchgq %rcx, (%rdi) // try to stick it in 125 jne 1b 126 movq %rcx, %rax // return new value 127 ret 128 SET_SIZE(atomic_inc_ulong_nv) 129 SET_SIZE(atomic_inc_64_nv) 130 131 ENTRY(atomic_dec_8) 132 ALTENTRY(atomic_dec_uchar) 133 lock 134 decb (%rdi) 135 ret 136 SET_SIZE(atomic_dec_uchar) 137 SET_SIZE(atomic_dec_8) 138 139 ENTRY(atomic_dec_16) 140 ALTENTRY(atomic_dec_ushort) 141 lock 142 decw (%rdi) 143 ret 144 SET_SIZE(atomic_dec_ushort) 145 SET_SIZE(atomic_dec_16) 146 147 ENTRY(atomic_dec_32) 148 ALTENTRY(atomic_dec_uint) 149 lock 150 decl (%rdi) 151 ret 152 SET_SIZE(atomic_dec_uint) 153 SET_SIZE(atomic_dec_32) 154 155 ENTRY(atomic_dec_64) 156 ALTENTRY(atomic_dec_ulong) 157 lock 158 decq (%rdi) 159 ret 160 SET_SIZE(atomic_dec_ulong) 161 SET_SIZE(atomic_dec_64) 162 163 ENTRY(atomic_dec_8_nv) 164 ALTENTRY(atomic_dec_uchar_nv) 165 movb (%rdi), %al // %al = old value 1661: 167 leaq -1(%rax), %rcx // %cl = new value 168 lock 169 cmpxchgb %cl, (%rdi) // try to stick it in 170 jne 1b 171 movzbl %cl, %eax // return new value 172 ret 173 SET_SIZE(atomic_dec_uchar_nv) 174 SET_SIZE(atomic_dec_8_nv) 175 176 ENTRY(atomic_dec_16_nv) 177 ALTENTRY(atomic_dec_ushort_nv) 178 movw (%rdi), %ax // %ax = old value 1791: 180 leaq -1(%rax), %rcx // %cx = new value 181 lock 182 cmpxchgw %cx, (%rdi) // try to stick it in 183 jne 1b 184 movzwl %cx, %eax // return new value 185 ret 186 SET_SIZE(atomic_dec_ushort_nv) 187 SET_SIZE(atomic_dec_16_nv) 188 189 ENTRY(atomic_dec_32_nv) 190 ALTENTRY(atomic_dec_uint_nv) 191 movl (%rdi), %eax // %eax = old value 1921: 193 leaq -1(%rax), %rcx // %ecx = new value 194 lock 195 cmpxchgl %ecx, (%rdi) // try to stick it in 196 jne 1b 197 movl %ecx, %eax // return new value 198 ret 199 SET_SIZE(atomic_dec_uint_nv) 200 SET_SIZE(atomic_dec_32_nv) 201 202 ENTRY(atomic_dec_64_nv) 203 ALTENTRY(atomic_dec_ulong_nv) 204 movq (%rdi), %rax // %rax = old value 2051: 206 leaq -1(%rax), %rcx // %rcx = new value 207 lock 208 cmpxchgq %rcx, (%rdi) // try to stick it in 209 jne 1b 210 movq %rcx, %rax // return new value 211 ret 212 SET_SIZE(atomic_dec_ulong_nv) 213 SET_SIZE(atomic_dec_64_nv) 214 215 ENTRY(atomic_or_8) 216 ALTENTRY(atomic_or_uchar) 217 lock 218 orb %sil, (%rdi) 219 ret 220 SET_SIZE(atomic_or_uchar) 221 SET_SIZE(atomic_or_8) 222 223 ENTRY(atomic_or_16) 224 ALTENTRY(atomic_or_ushort) 225 lock 226 orw %si, (%rdi) 227 ret 228 SET_SIZE(atomic_or_ushort) 229 SET_SIZE(atomic_or_16) 230 231 ENTRY(atomic_or_32) 232 ALTENTRY(atomic_or_uint) 233 lock 234 orl %esi, (%rdi) 235 ret 236 SET_SIZE(atomic_or_uint) 237 SET_SIZE(atomic_or_32) 238 239 ENTRY(atomic_or_64) 240 ALTENTRY(atomic_or_ulong) 241 lock 242 orq %rsi, (%rdi) 243 ret 244 SET_SIZE(atomic_or_ulong) 245 SET_SIZE(atomic_or_64) 246 247 ENTRY(atomic_and_8) 248 ALTENTRY(atomic_and_uchar) 249 lock 250 andb %sil, (%rdi) 251 ret 252 SET_SIZE(atomic_and_uchar) 253 SET_SIZE(atomic_and_8) 254 255 ENTRY(atomic_and_16) 256 ALTENTRY(atomic_and_ushort) 257 lock 258 andw %si, (%rdi) 259 ret 260 SET_SIZE(atomic_and_ushort) 261 SET_SIZE(atomic_and_16) 262 263 ENTRY(atomic_and_32) 264 ALTENTRY(atomic_and_uint) 265 lock 266 andl %esi, (%rdi) 267 ret 268 SET_SIZE(atomic_and_uint) 269 SET_SIZE(atomic_and_32) 270 271 ENTRY(atomic_and_64) 272 ALTENTRY(atomic_and_ulong) 273 lock 274 andq %rsi, (%rdi) 275 ret 276 SET_SIZE(atomic_and_ulong) 277 SET_SIZE(atomic_and_64) 278 279 ENTRY(atomic_add_8_nv) 280 ALTENTRY(atomic_add_char_nv) 281 movb (%rdi), %al // %al = old value 2821: 283 movb %sil, %cl 284 addb %al, %cl // %cl = new value 285 lock 286 cmpxchgb %cl, (%rdi) // try to stick it in 287 jne 1b 288 movzbl %cl, %eax // return new value 289 ret 290 SET_SIZE(atomic_add_char_nv) 291 SET_SIZE(atomic_add_8_nv) 292 293 ENTRY(atomic_add_16_nv) 294 ALTENTRY(atomic_add_short_nv) 295 movw (%rdi), %ax // %ax = old value 2961: 297 movw %si, %cx 298 addw %ax, %cx // %cx = new value 299 lock 300 cmpxchgw %cx, (%rdi) // try to stick it in 301 jne 1b 302 movzwl %cx, %eax // return new value 303 ret 304 SET_SIZE(atomic_add_short_nv) 305 SET_SIZE(atomic_add_16_nv) 306 307 ENTRY(atomic_add_32_nv) 308 ALTENTRY(atomic_add_int_nv) 309 movl (%rdi), %eax 3101: 311 movl %esi, %ecx 312 addl %eax, %ecx 313 lock 314 cmpxchgl %ecx, (%rdi) 315 jne 1b 316 movl %ecx, %eax 317 ret 318 SET_SIZE(atomic_add_int_nv) 319 SET_SIZE(atomic_add_32_nv) 320 321 ENTRY(atomic_add_64_nv) 322 ALTENTRY(atomic_add_ptr_nv) 323 ALTENTRY(atomic_add_long_nv) 324 movq (%rdi), %rax 3251: 326 movq %rsi, %rcx 327 addq %rax, %rcx 328 lock 329 cmpxchgq %rcx, (%rdi) 330 jne 1b 331 movq %rcx, %rax 332 ret 333 SET_SIZE(atomic_add_long_nv) 334 SET_SIZE(atomic_add_ptr_nv) 335 SET_SIZE(atomic_add_64_nv) 336 337 ENTRY(atomic_and_8_nv) 338 ALTENTRY(atomic_and_uchar_nv) 339 movb (%rdi), %al // %al = old value 3401: 341 movb %sil, %cl 342 andb %al, %cl // %cl = new value 343 lock 344 cmpxchgb %cl, (%rdi) // try to stick it in 345 jne 1b 346 movzbl %cl, %eax // return new value 347 ret 348 SET_SIZE(atomic_and_uchar_nv) 349 SET_SIZE(atomic_and_8_nv) 350 351 ENTRY(atomic_and_16_nv) 352 ALTENTRY(atomic_and_ushort_nv) 353 movw (%rdi), %ax // %ax = old value 3541: 355 movw %si, %cx 356 andw %ax, %cx // %cx = new value 357 lock 358 cmpxchgw %cx, (%rdi) // try to stick it in 359 jne 1b 360 movzwl %cx, %eax // return new value 361 ret 362 SET_SIZE(atomic_and_ushort_nv) 363 SET_SIZE(atomic_and_16_nv) 364 365 ENTRY(atomic_and_32_nv) 366 ALTENTRY(atomic_and_uint_nv) 367 movl (%rdi), %eax 3681: 369 movl %esi, %ecx 370 andl %eax, %ecx 371 lock 372 cmpxchgl %ecx, (%rdi) 373 jne 1b 374 movl %ecx, %eax 375 ret 376 SET_SIZE(atomic_and_uint_nv) 377 SET_SIZE(atomic_and_32_nv) 378 379 ENTRY(atomic_and_64_nv) 380 ALTENTRY(atomic_and_ulong_nv) 381 movq (%rdi), %rax 3821: 383 movq %rsi, %rcx 384 andq %rax, %rcx 385 lock 386 cmpxchgq %rcx, (%rdi) 387 jne 1b 388 movq %rcx, %rax 389 ret 390 SET_SIZE(atomic_and_ulong_nv) 391 SET_SIZE(atomic_and_64_nv) 392 393 ENTRY(atomic_or_8_nv) 394 ALTENTRY(atomic_or_uchar_nv) 395 movb (%rdi), %al // %al = old value 3961: 397 movb %sil, %cl 398 orb %al, %cl // %cl = new value 399 lock 400 cmpxchgb %cl, (%rdi) // try to stick it in 401 jne 1b 402 movzbl %cl, %eax // return new value 403 ret 404 SET_SIZE(atomic_and_uchar_nv) 405 SET_SIZE(atomic_and_8_nv) 406 407 ENTRY(atomic_or_16_nv) 408 ALTENTRY(atomic_or_ushort_nv) 409 movw (%rdi), %ax // %ax = old value 4101: 411 movw %si, %cx 412 orw %ax, %cx // %cx = new value 413 lock 414 cmpxchgw %cx, (%rdi) // try to stick it in 415 jne 1b 416 movzwl %cx, %eax // return new value 417 ret 418 SET_SIZE(atomic_or_ushort_nv) 419 SET_SIZE(atomic_or_16_nv) 420 421 ENTRY(atomic_or_32_nv) 422 ALTENTRY(atomic_or_uint_nv) 423 movl (%rdi), %eax 4241: 425 movl %esi, %ecx 426 orl %eax, %ecx 427 lock 428 cmpxchgl %ecx, (%rdi) 429 jne 1b 430 movl %ecx, %eax 431 ret 432 SET_SIZE(atomic_or_uint_nv) 433 SET_SIZE(atomic_or_32_nv) 434 435 ENTRY(atomic_or_64_nv) 436 ALTENTRY(atomic_or_ulong_nv) 437 movq (%rdi), %rax 4381: 439 movq %rsi, %rcx 440 orq %rax, %rcx 441 lock 442 cmpxchgq %rcx, (%rdi) 443 jne 1b 444 movq %rcx, %rax 445 ret 446 SET_SIZE(atomic_or_ulong_nv) 447 SET_SIZE(atomic_or_64_nv) 448 449 ENTRY(atomic_cas_8) 450 ALTENTRY(atomic_cas_uchar) 451 movzbl %sil, %eax 452 lock 453 cmpxchgb %dl, (%rdi) 454 ret 455 SET_SIZE(atomic_cas_uchar) 456 SET_SIZE(atomic_cas_8) 457 458 ENTRY(atomic_cas_16) 459 ALTENTRY(atomic_cas_ushort) 460 movzwl %si, %eax 461 lock 462 cmpxchgw %dx, (%rdi) 463 ret 464 SET_SIZE(atomic_cas_ushort) 465 SET_SIZE(atomic_cas_16) 466 467 ENTRY(atomic_cas_32) 468 ALTENTRY(atomic_cas_uint) 469 movl %esi, %eax 470 lock 471 cmpxchgl %edx, (%rdi) 472 ret 473 SET_SIZE(atomic_cas_uint) 474 SET_SIZE(atomic_cas_32) 475 476 ENTRY(atomic_cas_64) 477 ALTENTRY(atomic_cas_ulong) 478 ALTENTRY(atomic_cas_ptr) 479 movq %rsi, %rax 480 lock 481 cmpxchgq %rdx, (%rdi) 482 ret 483 SET_SIZE(atomic_cas_ptr) 484 SET_SIZE(atomic_cas_ulong) 485 SET_SIZE(atomic_cas_64) 486 487 ENTRY(atomic_swap_8) 488 ALTENTRY(atomic_swap_uchar) 489 movzbl %sil, %eax 490 lock 491 xchgb %al, (%rdi) 492 ret 493 SET_SIZE(atomic_swap_uchar) 494 SET_SIZE(atomic_swap_8) 495 496 ENTRY(atomic_swap_16) 497 ALTENTRY(atomic_swap_ushort) 498 movzwl %si, %eax 499 lock 500 xchgw %ax, (%rdi) 501 ret 502 SET_SIZE(atomic_swap_ushort) 503 SET_SIZE(atomic_swap_16) 504 505 ENTRY(atomic_swap_32) 506 ALTENTRY(atomic_swap_uint) 507 movl %esi, %eax 508 lock 509 xchgl %eax, (%rdi) 510 ret 511 SET_SIZE(atomic_swap_uint) 512 SET_SIZE(atomic_swap_32) 513 514 ENTRY(atomic_swap_64) 515 ALTENTRY(atomic_swap_ulong) 516 ALTENTRY(atomic_swap_ptr) 517 movq %rsi, %rax 518 lock 519 xchgq %rax, (%rdi) 520 ret 521 SET_SIZE(atomic_swap_ptr) 522 SET_SIZE(atomic_swap_ulong) 523 SET_SIZE(atomic_swap_64) 524 525 ENTRY(atomic_set_long_excl) 526 xorl %eax, %eax 527 lock 528 btsq %rsi, (%rdi) 529 jnc 1f 530 decl %eax // return -1 5311: 532 ret 533 SET_SIZE(atomic_set_long_excl) 534 535 ENTRY(atomic_clear_long_excl) 536 xorl %eax, %eax 537 lock 538 btrq %rsi, (%rdi) 539 jc 1f 540 decl %eax // return -1 5411: 542 ret 543 SET_SIZE(atomic_clear_long_excl) 544 545 ENTRY(membar_enter) 546 ALTENTRY(membar_exit) 547 mfence 548 ret 549 SET_SIZE(membar_exit) 550 SET_SIZE(membar_enter) 551 552 ENTRY(membar_producer) 553 sfence 554 ret 555 SET_SIZE(membar_producer) 556 557 ENTRY(membar_consumer) 558 lfence 559 ret 560 SET_SIZE(membar_consumer) 561