opensolaris_atomic.S revision 168675
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22/* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 .ident "%Z%%M% %I% %E% SMI" 28 29 .file "%M%" 30 31#define _ASM 32#include <sys/asm_linkage.h> 33 34#if defined(_KERNEL) 35 /* 36 * Legacy kernel interfaces; they will go away (eventually). 37 */ 38 ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function) 39 ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function) 40 ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function) 41 ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function) 42 ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function) 43 ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function) 44 ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function) 45#endif 46 47 ENTRY(atomic_inc_8) 48 ALTENTRY(atomic_inc_uchar) 49 movl 4(%esp), %eax 50 lock 51 incb (%eax) 52 ret 53 SET_SIZE(atomic_inc_uchar) 54 SET_SIZE(atomic_inc_8) 55 56 ENTRY(atomic_inc_16) 57 ALTENTRY(atomic_inc_ushort) 58 movl 4(%esp), %eax 59 lock 60 incw (%eax) 61 ret 62 SET_SIZE(atomic_inc_ushort) 63 SET_SIZE(atomic_inc_16) 64 65 ENTRY(atomic_inc_32) 66 ALTENTRY(atomic_inc_uint) 67 ALTENTRY(atomic_inc_ulong) 68 movl 4(%esp), %eax 69 lock 70 incl (%eax) 71 ret 72 SET_SIZE(atomic_inc_ulong) 73 SET_SIZE(atomic_inc_uint) 74 SET_SIZE(atomic_inc_32) 75 76 ENTRY(atomic_inc_8_nv) 77 ALTENTRY(atomic_inc_uchar_nv) 78 movl 4(%esp), %edx // %edx = target address 79 movb (%edx), %al // %al = old value 801: 81 leal 1(%eax), %ecx // %cl = new value 82 lock 83 cmpxchgb %cl, (%edx) // try to stick it in 84 jne 1b 85 movzbl %cl, %eax // return new value 86 ret 87 SET_SIZE(atomic_inc_uchar_nv) 88 SET_SIZE(atomic_inc_8_nv) 89 90 ENTRY(atomic_inc_16_nv) 91 ALTENTRY(atomic_inc_ushort_nv) 92 movl 4(%esp), %edx // %edx = target address 93 movw (%edx), %ax // %ax = old value 941: 95 leal 1(%eax), %ecx // %cx = new value 96 lock 97 cmpxchgw %cx, (%edx) // try to stick it in 98 jne 1b 99 movzwl %cx, %eax // return new value 100 ret 101 SET_SIZE(atomic_inc_ushort_nv) 102 SET_SIZE(atomic_inc_16_nv) 103 104 ENTRY(atomic_inc_32_nv) 105 ALTENTRY(atomic_inc_uint_nv) 106 ALTENTRY(atomic_inc_ulong_nv) 107 movl 4(%esp), %edx // %edx = target address 108 movl (%edx), %eax // %eax = old value 1091: 110 leal 1(%eax), %ecx // %ecx = new value 111 lock 112 cmpxchgl %ecx, (%edx) // try to stick it in 113 jne 1b 114 movl %ecx, %eax // return new value 115 ret 116 SET_SIZE(atomic_inc_ulong_nv) 117 SET_SIZE(atomic_inc_uint_nv) 118 SET_SIZE(atomic_inc_32_nv) 119 120 ENTRY(atomic_inc_64) 121 ALTENTRY(atomic_inc_64_nv) 122 pushl %edi 123 pushl %ebx 124 movl 12(%esp), %edi // %edi = target address 125 movl (%edi), %eax 126 movl 4(%edi), %edx // %edx:%eax = old value 1271: 128 xorl %ebx, %ebx 129 xorl %ecx, %ecx 130 incl %ebx // %ecx:%ebx = 1 131 addl %eax, %ebx 132 adcl %edx, %ecx // add in the carry from inc 133 lock 134 cmpxchg8b (%edi) // try to stick it in 135 jne 1b 136 movl %ebx, %eax 137 movl %ecx, %edx // return new value 138 popl %ebx 139 popl %edi 140 ret 141 SET_SIZE(atomic_inc_64_nv) 142 SET_SIZE(atomic_inc_64) 143 144 ENTRY(atomic_dec_8) 145 ALTENTRY(atomic_dec_uchar) 146 movl 4(%esp), %eax 147 lock 148 decb (%eax) 149 ret 150 SET_SIZE(atomic_dec_uchar) 151 SET_SIZE(atomic_dec_8) 152 153 ENTRY(atomic_dec_16) 154 ALTENTRY(atomic_dec_ushort) 155 movl 4(%esp), %eax 156 lock 157 decw (%eax) 158 ret 159 SET_SIZE(atomic_dec_ushort) 160 SET_SIZE(atomic_dec_16) 161 162 ENTRY(atomic_dec_32) 163 ALTENTRY(atomic_dec_uint) 164 ALTENTRY(atomic_dec_ulong) 165 movl 4(%esp), %eax 166 lock 167 decl (%eax) 168 ret 169 SET_SIZE(atomic_dec_ulong) 170 SET_SIZE(atomic_dec_uint) 171 SET_SIZE(atomic_dec_32) 172 173 ENTRY(atomic_dec_8_nv) 174 ALTENTRY(atomic_dec_uchar_nv) 175 movl 4(%esp), %edx // %edx = target address 176 movb (%edx), %al // %al = old value 1771: 178 leal -1(%eax), %ecx // %cl = new value 179 lock 180 cmpxchgb %cl, (%edx) // try to stick it in 181 jne 1b 182 movzbl %cl, %eax // return new value 183 ret 184 SET_SIZE(atomic_dec_uchar_nv) 185 SET_SIZE(atomic_dec_8_nv) 186 187 ENTRY(atomic_dec_16_nv) 188 ALTENTRY(atomic_dec_ushort_nv) 189 movl 4(%esp), %edx // %edx = target address 190 movw (%edx), %ax // %ax = old value 1911: 192 leal -1(%eax), %ecx // %cx = new value 193 lock 194 cmpxchgw %cx, (%edx) // try to stick it in 195 jne 1b 196 movzwl %cx, %eax // return new value 197 ret 198 SET_SIZE(atomic_dec_ushort_nv) 199 SET_SIZE(atomic_dec_16_nv) 200 201 ENTRY(atomic_dec_32_nv) 202 ALTENTRY(atomic_dec_uint_nv) 203 ALTENTRY(atomic_dec_ulong_nv) 204 movl 4(%esp), %edx // %edx = target address 205 movl (%edx), %eax // %eax = old value 2061: 207 leal -1(%eax), %ecx // %ecx = new value 208 lock 209 cmpxchgl %ecx, (%edx) // try to stick it in 210 jne 1b 211 movl %ecx, %eax // return new value 212 ret 213 SET_SIZE(atomic_dec_ulong_nv) 214 SET_SIZE(atomic_dec_uint_nv) 215 SET_SIZE(atomic_dec_32_nv) 216 217 ENTRY(atomic_dec_64) 218 ALTENTRY(atomic_dec_64_nv) 219 pushl %edi 220 pushl %ebx 221 movl 12(%esp), %edi // %edi = target address 222 movl (%edi), %eax 223 movl 4(%edi), %edx // %edx:%eax = old value 2241: 225 xorl %ebx, %ebx 226 xorl %ecx, %ecx 227 not %ecx 228 not %ebx // %ecx:%ebx = -1 229 addl %eax, %ebx 230 adcl %edx, %ecx // add in the carry from inc 231 lock 232 cmpxchg8b (%edi) // try to stick it in 233 jne 1b 234 movl %ebx, %eax 235 movl %ecx, %edx // return new value 236 popl %ebx 237 popl %edi 238 ret 239 SET_SIZE(atomic_dec_64_nv) 240 SET_SIZE(atomic_dec_64) 241 242 ENTRY(atomic_or_8) 243 ALTENTRY(atomic_or_uchar) 244 movl 4(%esp), %eax 245 movb 8(%esp), %cl 246 lock 247 orb %cl, (%eax) 248 ret 249 SET_SIZE(atomic_or_uchar) 250 SET_SIZE(atomic_or_8) 251 252 ENTRY(atomic_or_16) 253 ALTENTRY(atomic_or_ushort) 254 movl 4(%esp), %eax 255 movw 8(%esp), %cx 256 lock 257 orw %cx, (%eax) 258 ret 259 SET_SIZE(atomic_or_ushort) 260 SET_SIZE(atomic_or_16) 261 262 ENTRY(atomic_or_32) 263 ALTENTRY(atomic_or_uint) 264 ALTENTRY(atomic_or_ulong) 265 movl 4(%esp), %eax 266 movl 8(%esp), %ecx 267 lock 268 orl %ecx, (%eax) 269 ret 270 SET_SIZE(atomic_or_ulong) 271 SET_SIZE(atomic_or_uint) 272 SET_SIZE(atomic_or_32) 273 274 ENTRY(atomic_and_8) 275 ALTENTRY(atomic_and_uchar) 276 movl 4(%esp), %eax 277 movb 8(%esp), %cl 278 lock 279 andb %cl, (%eax) 280 ret 281 SET_SIZE(atomic_and_uchar) 282 SET_SIZE(atomic_and_8) 283 284 ENTRY(atomic_and_16) 285 ALTENTRY(atomic_and_ushort) 286 movl 4(%esp), %eax 287 movw 8(%esp), %cx 288 lock 289 andw %cx, (%eax) 290 ret 291 SET_SIZE(atomic_and_ushort) 292 SET_SIZE(atomic_and_16) 293 294 ENTRY(atomic_and_32) 295 ALTENTRY(atomic_and_uint) 296 ALTENTRY(atomic_and_ulong) 297 movl 4(%esp), %eax 298 movl 8(%esp), %ecx 299 lock 300 andl %ecx, (%eax) 301 ret 302 SET_SIZE(atomic_and_ulong) 303 SET_SIZE(atomic_and_uint) 304 SET_SIZE(atomic_and_32) 305 306 ENTRY(atomic_add_8_nv) 307 ALTENTRY(atomic_add_char_nv) 308 movl 4(%esp), %edx // %edx = target address 309 movb (%edx), %al // %al = old value 3101: 311 movl 8(%esp), %ecx // %ecx = delta 312 addb %al, %cl // %cl = new value 313 lock 314 cmpxchgb %cl, (%edx) // try to stick it in 315 jne 1b 316 movzbl %cl, %eax // return new value 317 ret 318 SET_SIZE(atomic_add_char_nv) 319 SET_SIZE(atomic_add_8_nv) 320 321 ENTRY(atomic_add_16_nv) 322 ALTENTRY(atomic_add_short_nv) 323 movl 4(%esp), %edx // %edx = target address 324 movw (%edx), %ax // %ax = old value 3251: 326 movl 8(%esp), %ecx // %ecx = delta 327 addw %ax, %cx // %cx = new value 328 lock 329 cmpxchgw %cx, (%edx) // try to stick it in 330 jne 1b 331 movzwl %cx, %eax // return new value 332 ret 333 SET_SIZE(atomic_add_short_nv) 334 SET_SIZE(atomic_add_16_nv) 335 336 ENTRY(atomic_add_32_nv) 337 ALTENTRY(atomic_add_int_nv) 338 ALTENTRY(atomic_add_ptr_nv) 339 ALTENTRY(atomic_add_long_nv) 340 movl 4(%esp), %edx // %edx = target address 341 movl (%edx), %eax // %eax = old value 3421: 343 movl 8(%esp), %ecx // %ecx = delta 344 addl %eax, %ecx // %ecx = new value 345 lock 346 cmpxchgl %ecx, (%edx) // try to stick it in 347 jne 1b 348 movl %ecx, %eax // return new value 349 ret 350 SET_SIZE(atomic_add_long_nv) 351 SET_SIZE(atomic_add_ptr_nv) 352 SET_SIZE(atomic_add_int_nv) 353 SET_SIZE(atomic_add_32_nv) 354 355 ENTRY(atomic_add_64) 356 ALTENTRY(atomic_add_64_nv) 357 pushl %edi 358 pushl %ebx 359 movl 12(%esp), %edi // %edi = target address 360 movl (%edi), %eax 361 movl 4(%edi), %edx // %edx:%eax = old value 3621: 363 movl 16(%esp), %ebx 364 movl 20(%esp), %ecx // %ecx:%ebx = delta 365 addl %eax, %ebx 366 adcl %edx, %ecx // %ecx:%ebx = new value 367 lock 368 cmpxchg8b (%edi) // try to stick it in 369 jne 1b 370 movl %ebx, %eax 371 movl %ecx, %edx // return new value 372 popl %ebx 373 popl %edi 374 ret 375 SET_SIZE(atomic_add_64_nv) 376 SET_SIZE(atomic_add_64) 377 378 ENTRY(atomic_or_8_nv) 379 ALTENTRY(atomic_or_uchar_nv) 380 movl 4(%esp), %edx // %edx = target address 381 movb (%edx), %al // %al = old value 3821: 383 movl 8(%esp), %ecx // %ecx = delta 384 orb %al, %cl // %cl = new value 385 lock 386 cmpxchgb %cl, (%edx) // try to stick it in 387 jne 1b 388 movzbl %cl, %eax // return new value 389 ret 390 SET_SIZE(atomic_or_uchar_nv) 391 SET_SIZE(atomic_or_8_nv) 392 393 ENTRY(atomic_or_16_nv) 394 ALTENTRY(atomic_or_ushort_nv) 395 movl 4(%esp), %edx // %edx = target address 396 movw (%edx), %ax // %ax = old value 3971: 398 movl 8(%esp), %ecx // %ecx = delta 399 orw %ax, %cx // %cx = new value 400 lock 401 cmpxchgw %cx, (%edx) // try to stick it in 402 jne 1b 403 movzwl %cx, %eax // return new value 404 ret 405 SET_SIZE(atomic_or_ushort_nv) 406 SET_SIZE(atomic_or_16_nv) 407 408 ENTRY(atomic_or_32_nv) 409 ALTENTRY(atomic_or_uint_nv) 410 ALTENTRY(atomic_or_ulong_nv) 411 movl 4(%esp), %edx // %edx = target address 412 movl (%edx), %eax // %eax = old value 4131: 414 movl 8(%esp), %ecx // %ecx = delta 415 orl %eax, %ecx // %ecx = new value 416 lock 417 cmpxchgl %ecx, (%edx) // try to stick it in 418 jne 1b 419 movl %ecx, %eax // return new value 420 ret 421 SET_SIZE(atomic_or_ulong_nv) 422 SET_SIZE(atomic_or_uint_nv) 423 SET_SIZE(atomic_or_32_nv) 424 425 ENTRY(atomic_or_64) 426 ALTENTRY(atomic_or_64_nv) 427 pushl %edi 428 pushl %ebx 429 movl 12(%esp), %edi // %edi = target address 430 movl (%edi), %eax 431 movl 4(%edi), %edx // %edx:%eax = old value 4321: 433 movl 16(%esp), %ebx 434 movl 20(%esp), %ecx // %ecx:%ebx = delta 435 orl %eax, %ebx 436 orl %edx, %ecx // %ecx:%ebx = new value 437 lock 438 cmpxchg8b (%edi) // try to stick it in 439 jne 1b 440 movl %ebx, %eax 441 movl %ecx, %edx // return new value 442 popl %ebx 443 popl %edi 444 ret 445 SET_SIZE(atomic_or_64_nv) 446 SET_SIZE(atomic_or_64) 447 448 ENTRY(atomic_and_8_nv) 449 ALTENTRY(atomic_and_uchar_nv) 450 movl 4(%esp), %edx // %edx = target address 451 movb (%edx), %al // %al = old value 4521: 453 movl 8(%esp), %ecx // %ecx = delta 454 andb %al, %cl // %cl = new value 455 lock 456 cmpxchgb %cl, (%edx) // try to stick it in 457 jne 1b 458 movzbl %cl, %eax // return new value 459 ret 460 SET_SIZE(atomic_and_uchar_nv) 461 SET_SIZE(atomic_and_8_nv) 462 463 ENTRY(atomic_and_16_nv) 464 ALTENTRY(atomic_and_ushort_nv) 465 movl 4(%esp), %edx // %edx = target address 466 movw (%edx), %ax // %ax = old value 4671: 468 movl 8(%esp), %ecx // %ecx = delta 469 andw %ax, %cx // %cx = new value 470 lock 471 cmpxchgw %cx, (%edx) // try to stick it in 472 jne 1b 473 movzwl %cx, %eax // return new value 474 ret 475 SET_SIZE(atomic_and_ushort_nv) 476 SET_SIZE(atomic_and_16_nv) 477 478 ENTRY(atomic_and_32_nv) 479 ALTENTRY(atomic_and_uint_nv) 480 ALTENTRY(atomic_and_ulong_nv) 481 movl 4(%esp), %edx // %edx = target address 482 movl (%edx), %eax // %eax = old value 4831: 484 movl 8(%esp), %ecx // %ecx = delta 485 andl %eax, %ecx // %ecx = new value 486 lock 487 cmpxchgl %ecx, (%edx) // try to stick it in 488 jne 1b 489 movl %ecx, %eax // return new value 490 ret 491 SET_SIZE(atomic_and_ulong_nv) 492 SET_SIZE(atomic_and_uint_nv) 493 SET_SIZE(atomic_and_32_nv) 494 495 ENTRY(atomic_and_64) 496 ALTENTRY(atomic_and_64_nv) 497 pushl %edi 498 pushl %ebx 499 movl 12(%esp), %edi // %edi = target address 500 movl (%edi), %eax 501 movl 4(%edi), %edx // %edx:%eax = old value 5021: 503 movl 16(%esp), %ebx 504 movl 20(%esp), %ecx // %ecx:%ebx = delta 505 andl %eax, %ebx 506 andl %edx, %ecx // %ecx:%ebx = new value 507 lock 508 cmpxchg8b (%edi) // try to stick it in 509 jne 1b 510 movl %ebx, %eax 511 movl %ecx, %edx // return new value 512 popl %ebx 513 popl %edi 514 ret 515 SET_SIZE(atomic_and_64_nv) 516 SET_SIZE(atomic_and_64) 517 518 ENTRY(atomic_cas_8) 519 ALTENTRY(atomic_cas_uchar) 520 movl 4(%esp), %edx 521 movzbl 8(%esp), %eax 522 movb 12(%esp), %cl 523 lock 524 cmpxchgb %cl, (%edx) 525 ret 526 SET_SIZE(atomic_cas_uchar) 527 SET_SIZE(atomic_cas_8) 528 529 ENTRY(atomic_cas_16) 530 ALTENTRY(atomic_cas_ushort) 531 movl 4(%esp), %edx 532 movzwl 8(%esp), %eax 533 movw 12(%esp), %cx 534 lock 535 cmpxchgw %cx, (%edx) 536 ret 537 SET_SIZE(atomic_cas_ushort) 538 SET_SIZE(atomic_cas_16) 539 540 ENTRY(atomic_cas_32) 541 ALTENTRY(atomic_cas_uint) 542 ALTENTRY(atomic_cas_ulong) 543 ALTENTRY(atomic_cas_ptr) 544 movl 4(%esp), %edx 545 movl 8(%esp), %eax 546 movl 12(%esp), %ecx 547 lock 548 cmpxchgl %ecx, (%edx) 549 ret 550 SET_SIZE(atomic_cas_ptr) 551 SET_SIZE(atomic_cas_ulong) 552 SET_SIZE(atomic_cas_uint) 553 SET_SIZE(atomic_cas_32) 554 555 ENTRY(atomic_cas_64) 556 pushl %ebx 557 pushl %esi 558 movl 12(%esp), %esi 559 movl 16(%esp), %eax 560 movl 20(%esp), %edx 561 movl 24(%esp), %ebx 562 movl 28(%esp), %ecx 563 lock 564 cmpxchg8b (%esi) 565 popl %esi 566 popl %ebx 567 ret 568 SET_SIZE(atomic_cas_64) 569 570 ENTRY(atomic_swap_8) 571 ALTENTRY(atomic_swap_uchar) 572 movl 4(%esp), %edx 573 movzbl 8(%esp), %eax 574 lock 575 xchgb %al, (%edx) 576 ret 577 SET_SIZE(atomic_swap_uchar) 578 SET_SIZE(atomic_swap_8) 579 580 ENTRY(atomic_swap_16) 581 ALTENTRY(atomic_swap_ushort) 582 movl 4(%esp), %edx 583 movzwl 8(%esp), %eax 584 lock 585 xchgw %ax, (%edx) 586 ret 587 SET_SIZE(atomic_swap_ushort) 588 SET_SIZE(atomic_swap_16) 589 590 ENTRY(atomic_swap_32) 591 ALTENTRY(atomic_swap_uint) 592 ALTENTRY(atomic_swap_ptr) 593 ALTENTRY(atomic_swap_ulong) 594 movl 4(%esp), %edx 595 movl 8(%esp), %eax 596 lock 597 xchgl %eax, (%edx) 598 ret 599 SET_SIZE(atomic_swap_ulong) 600 SET_SIZE(atomic_swap_ptr) 601 SET_SIZE(atomic_swap_uint) 602 SET_SIZE(atomic_swap_32) 603 604 ENTRY(atomic_swap_64) 605 pushl %esi 606 pushl %ebx 607 movl 12(%esp), %esi 608 movl 16(%esp), %ebx 609 movl 20(%esp), %ecx 610 movl (%esi), %eax 611 movl 4(%esi), %edx // %edx:%eax = old value 6121: 613 lock 614 cmpxchg8b (%esi) 615 jne 1b 616 popl %ebx 617 popl %esi 618 ret 619 SET_SIZE(atomic_swap_64) 620 621 ENTRY(atomic_set_long_excl) 622 movl 4(%esp), %edx // %edx = target address 623 movl 8(%esp), %ecx // %ecx = bit id 624 xorl %eax, %eax 625 lock 626 btsl %ecx, (%edx) 627 jnc 1f 628 decl %eax // return -1 6291: 630 ret 631 SET_SIZE(atomic_set_long_excl) 632 633 ENTRY(atomic_clear_long_excl) 634 movl 4(%esp), %edx // %edx = target address 635 movl 8(%esp), %ecx // %ecx = bit id 636 xorl %eax, %eax 637 lock 638 btrl %ecx, (%edx) 639 jc 1f 640 decl %eax // return -1 6411: 642 ret 643 SET_SIZE(atomic_clear_long_excl) 644 645 ENTRY(membar_enter) 646 ALTENTRY(membar_exit) 647 ALTENTRY(membar_producer) 648 ALTENTRY(membar_consumer) 649 lock 650 xorl $0, (%esp) 651 ret 652 SET_SIZE(membar_consumer) 653 SET_SIZE(membar_producer) 654 SET_SIZE(membar_exit) 655 SET_SIZE(membar_enter) 656