1;; Machine description for ARM processor synchronization primitives. 2;; Copyright (C) 2010-2015 Free Software Foundation, Inc. 3;; Written by Marcus Shawcroft (marcus.shawcroft@arm.com) 4;; 64bit Atomics by Dave Gilbert (david.gilbert@linaro.org) 5;; 6;; This file is part of GCC. 7;; 8;; GCC is free software; you can redistribute it and/or modify it 9;; under the terms of the GNU General Public License as published by 10;; the Free Software Foundation; either version 3, or (at your option) 11;; any later version. 12;; 13;; GCC is distributed in the hope that it will be useful, but 14;; WITHOUT ANY WARRANTY; without even the implied warranty of 15;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16;; General Public License for more details. 17;; 18;; You should have received a copy of the GNU General Public License 19;; along with GCC; see the file COPYING3. If not see 20;; <http://www.gnu.org/licenses/>. */ 21 22(define_mode_attr sync_predtab 23 [(QI "TARGET_HAVE_LDREXBH && TARGET_HAVE_MEMORY_BARRIER") 24 (HI "TARGET_HAVE_LDREXBH && TARGET_HAVE_MEMORY_BARRIER") 25 (SI "TARGET_HAVE_LDREX && TARGET_HAVE_MEMORY_BARRIER") 26 (DI "TARGET_HAVE_LDREXD && ARM_DOUBLEWORD_ALIGN 27 && TARGET_HAVE_MEMORY_BARRIER")]) 28 29(define_code_iterator syncop [plus minus ior xor and]) 30 31(define_code_attr sync_optab 32 [(ior "or") (xor "xor") (and "and") (plus "add") (minus "sub")]) 33 34(define_mode_attr sync_sfx 35 [(QI "b") (HI "h") (SI "") (DI "d")]) 36 37(define_expand "memory_barrier" 38 [(set (match_dup 0) 39 (unspec:BLK [(match_dup 0)] UNSPEC_MEMORY_BARRIER))] 40 "TARGET_HAVE_MEMORY_BARRIER" 41{ 42 operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode)); 43 MEM_VOLATILE_P (operands[0]) = 1; 44}) 45 46(define_insn "*memory_barrier" 47 [(set (match_operand:BLK 0 "" "") 48 (unspec:BLK [(match_dup 0)] UNSPEC_MEMORY_BARRIER))] 49 "TARGET_HAVE_MEMORY_BARRIER" 50 { 51 if (TARGET_HAVE_DMB) 52 { 53 /* Note we issue a system level barrier. We should consider issuing 54 a inner shareabilty zone barrier here instead, ie. "DMB ISH". */ 55 /* ??? Differentiate based on SEQ_CST vs less strict? */ 56 return "dmb\tsy"; 57 } 58 59 if (TARGET_HAVE_DMB_MCR) 60 return "mcr\tp15, 0, r0, c7, c10, 5"; 61 62 gcc_unreachable (); 63 } 64 [(set_attr "length" "4") 65 (set_attr "conds" "unconditional") 66 (set_attr "predicable" "no")]) 67 68(define_insn "atomic_load<mode>" 69 [(set (match_operand:QHSI 0 "register_operand" "=r") 70 (unspec_volatile:QHSI 71 [(match_operand:QHSI 1 "arm_sync_memory_operand" "Q") 72 (match_operand:SI 2 "const_int_operand")] ;; model 73 VUNSPEC_LDA))] 74 "TARGET_HAVE_LDACQ" 75 { 76 enum memmodel model = memmodel_from_int (INTVAL (operands[2])); 77 if (is_mm_relaxed (model) || is_mm_consume (model) || is_mm_release (model)) 78 return \"ldr%(<sync_sfx>%)\\t%0, %1\"; 79 else 80 return \"lda<sync_sfx>%?\\t%0, %1\"; 81 } 82 [(set_attr "predicable" "yes") 83 (set_attr "predicable_short_it" "no")]) 84 85(define_insn "atomic_store<mode>" 86 [(set (match_operand:QHSI 0 "memory_operand" "=Q") 87 (unspec_volatile:QHSI 88 [(match_operand:QHSI 1 "general_operand" "r") 89 (match_operand:SI 2 "const_int_operand")] ;; model 90 VUNSPEC_STL))] 91 "TARGET_HAVE_LDACQ" 92 { 93 enum memmodel model = memmodel_from_int (INTVAL (operands[2])); 94 if (is_mm_relaxed (model) || is_mm_consume (model) || is_mm_acquire (model)) 95 return \"str%(<sync_sfx>%)\t%1, %0\"; 96 else 97 return \"stl<sync_sfx>%?\t%1, %0\"; 98 } 99 [(set_attr "predicable" "yes") 100 (set_attr "predicable_short_it" "no")]) 101 102;; An LDRD instruction usable by the atomic_loaddi expander on LPAE targets 103 104(define_insn "arm_atomic_loaddi2_ldrd" 105 [(set (match_operand:DI 0 "register_operand" "=r") 106 (unspec_volatile:DI 107 [(match_operand:DI 1 "arm_sync_memory_operand" "Q")] 108 VUNSPEC_LDRD_ATOMIC))] 109 "ARM_DOUBLEWORD_ALIGN && TARGET_HAVE_LPAE" 110 "ldr%(d%)\t%0, %H0, %C1" 111 [(set_attr "predicable" "yes") 112 (set_attr "predicable_short_it" "no")]) 113 114;; There are three ways to expand this depending on the architecture 115;; features available. As for the barriers, a load needs a barrier 116;; after it on all non-relaxed memory models except when the load 117;; has acquire semantics (for ARMv8-A). 118 119(define_expand "atomic_loaddi" 120 [(match_operand:DI 0 "s_register_operand") ;; val out 121 (match_operand:DI 1 "mem_noofs_operand") ;; memory 122 (match_operand:SI 2 "const_int_operand")] ;; model 123 "(TARGET_HAVE_LDREXD || TARGET_HAVE_LPAE || TARGET_HAVE_LDACQ) 124 && ARM_DOUBLEWORD_ALIGN" 125{ 126 memmodel model = memmodel_from_int (INTVAL (operands[2])); 127 128 /* For ARMv8-A we can use an LDAEXD to atomically load two 32-bit registers 129 when acquire or stronger semantics are needed. When the relaxed model is 130 used this can be relaxed to a normal LDRD. */ 131 if (TARGET_HAVE_LDACQ) 132 { 133 if (is_mm_relaxed (model)) 134 emit_insn (gen_arm_atomic_loaddi2_ldrd (operands[0], operands[1])); 135 else 136 emit_insn (gen_arm_load_acquire_exclusivedi (operands[0], operands[1])); 137 138 DONE; 139 } 140 141 /* On LPAE targets LDRD and STRD accesses to 64-bit aligned 142 locations are 64-bit single-copy atomic. We still need barriers in the 143 appropriate places to implement the ordering constraints. */ 144 if (TARGET_HAVE_LPAE) 145 emit_insn (gen_arm_atomic_loaddi2_ldrd (operands[0], operands[1])); 146 else 147 emit_insn (gen_arm_load_exclusivedi (operands[0], operands[1])); 148 149 150 /* All non-relaxed models need a barrier after the load when load-acquire 151 instructions are not available. */ 152 if (!is_mm_relaxed (model)) 153 expand_mem_thread_fence (model); 154 155 DONE; 156}) 157 158(define_expand "atomic_compare_and_swap<mode>" 159 [(match_operand:SI 0 "s_register_operand" "") ;; bool out 160 (match_operand:QHSD 1 "s_register_operand" "") ;; val out 161 (match_operand:QHSD 2 "mem_noofs_operand" "") ;; memory 162 (match_operand:QHSD 3 "general_operand" "") ;; expected 163 (match_operand:QHSD 4 "s_register_operand" "") ;; desired 164 (match_operand:SI 5 "const_int_operand") ;; is_weak 165 (match_operand:SI 6 "const_int_operand") ;; mod_s 166 (match_operand:SI 7 "const_int_operand")] ;; mod_f 167 "<sync_predtab>" 168{ 169 arm_expand_compare_and_swap (operands); 170 DONE; 171}) 172 173(define_insn_and_split "atomic_compare_and_swap<mode>_1" 174 [(set (reg:CC_Z CC_REGNUM) ;; bool out 175 (unspec_volatile:CC_Z [(const_int 0)] VUNSPEC_ATOMIC_CAS)) 176 (set (match_operand:SI 0 "s_register_operand" "=&r") ;; val out 177 (zero_extend:SI 178 (match_operand:NARROW 1 "mem_noofs_operand" "+Ua"))) ;; memory 179 (set (match_dup 1) 180 (unspec_volatile:NARROW 181 [(match_operand:SI 2 "arm_add_operand" "rIL") ;; expected 182 (match_operand:NARROW 3 "s_register_operand" "r") ;; desired 183 (match_operand:SI 4 "const_int_operand") ;; is_weak 184 (match_operand:SI 5 "const_int_operand") ;; mod_s 185 (match_operand:SI 6 "const_int_operand")] ;; mod_f 186 VUNSPEC_ATOMIC_CAS)) 187 (clobber (match_scratch:SI 7 "=&r"))] 188 "<sync_predtab>" 189 "#" 190 "&& reload_completed" 191 [(const_int 0)] 192 { 193 arm_split_compare_and_swap (operands); 194 DONE; 195 }) 196 197(define_mode_attr cas_cmp_operand 198 [(SI "arm_add_operand") (DI "cmpdi_operand")]) 199(define_mode_attr cas_cmp_str 200 [(SI "rIL") (DI "rDi")]) 201 202(define_insn_and_split "atomic_compare_and_swap<mode>_1" 203 [(set (reg:CC_Z CC_REGNUM) ;; bool out 204 (unspec_volatile:CC_Z [(const_int 0)] VUNSPEC_ATOMIC_CAS)) 205 (set (match_operand:SIDI 0 "s_register_operand" "=&r") ;; val out 206 (match_operand:SIDI 1 "mem_noofs_operand" "+Ua")) ;; memory 207 (set (match_dup 1) 208 (unspec_volatile:SIDI 209 [(match_operand:SIDI 2 "<cas_cmp_operand>" "<cas_cmp_str>") ;; expect 210 (match_operand:SIDI 3 "s_register_operand" "r") ;; desired 211 (match_operand:SI 4 "const_int_operand") ;; is_weak 212 (match_operand:SI 5 "const_int_operand") ;; mod_s 213 (match_operand:SI 6 "const_int_operand")] ;; mod_f 214 VUNSPEC_ATOMIC_CAS)) 215 (clobber (match_scratch:SI 7 "=&r"))] 216 "<sync_predtab>" 217 "#" 218 "&& reload_completed" 219 [(const_int 0)] 220 { 221 arm_split_compare_and_swap (operands); 222 DONE; 223 }) 224 225(define_insn_and_split "atomic_exchange<mode>" 226 [(set (match_operand:QHSD 0 "s_register_operand" "=&r") ;; output 227 (match_operand:QHSD 1 "mem_noofs_operand" "+Ua")) ;; memory 228 (set (match_dup 1) 229 (unspec_volatile:QHSD 230 [(match_operand:QHSD 2 "s_register_operand" "r") ;; input 231 (match_operand:SI 3 "const_int_operand" "")] ;; model 232 VUNSPEC_ATOMIC_XCHG)) 233 (clobber (reg:CC CC_REGNUM)) 234 (clobber (match_scratch:SI 4 "=&r"))] 235 "<sync_predtab>" 236 "#" 237 "&& reload_completed" 238 [(const_int 0)] 239 { 240 arm_split_atomic_op (SET, operands[0], NULL, operands[1], 241 operands[2], operands[3], operands[4]); 242 DONE; 243 }) 244 245(define_mode_attr atomic_op_operand 246 [(QI "reg_or_int_operand") 247 (HI "reg_or_int_operand") 248 (SI "reg_or_int_operand") 249 (DI "s_register_operand")]) 250 251(define_mode_attr atomic_op_str 252 [(QI "rn") (HI "rn") (SI "rn") (DI "r")]) 253 254(define_insn_and_split "atomic_<sync_optab><mode>" 255 [(set (match_operand:QHSD 0 "mem_noofs_operand" "+Ua") 256 (unspec_volatile:QHSD 257 [(syncop:QHSD (match_dup 0) 258 (match_operand:QHSD 1 "<atomic_op_operand>" "<atomic_op_str>")) 259 (match_operand:SI 2 "const_int_operand")] ;; model 260 VUNSPEC_ATOMIC_OP)) 261 (clobber (reg:CC CC_REGNUM)) 262 (clobber (match_scratch:QHSD 3 "=&r")) 263 (clobber (match_scratch:SI 4 "=&r"))] 264 "<sync_predtab>" 265 "#" 266 "&& reload_completed" 267 [(const_int 0)] 268 { 269 arm_split_atomic_op (<CODE>, NULL, operands[3], operands[0], 270 operands[1], operands[2], operands[4]); 271 DONE; 272 }) 273 274(define_insn_and_split "atomic_nand<mode>" 275 [(set (match_operand:QHSD 0 "mem_noofs_operand" "+Ua") 276 (unspec_volatile:QHSD 277 [(not:QHSD 278 (and:QHSD (match_dup 0) 279 (match_operand:QHSD 1 "<atomic_op_operand>" "<atomic_op_str>"))) 280 (match_operand:SI 2 "const_int_operand")] ;; model 281 VUNSPEC_ATOMIC_OP)) 282 (clobber (reg:CC CC_REGNUM)) 283 (clobber (match_scratch:QHSD 3 "=&r")) 284 (clobber (match_scratch:SI 4 "=&r"))] 285 "<sync_predtab>" 286 "#" 287 "&& reload_completed" 288 [(const_int 0)] 289 { 290 arm_split_atomic_op (NOT, NULL, operands[3], operands[0], 291 operands[1], operands[2], operands[4]); 292 DONE; 293 }) 294 295(define_insn_and_split "atomic_fetch_<sync_optab><mode>" 296 [(set (match_operand:QHSD 0 "s_register_operand" "=&r") 297 (match_operand:QHSD 1 "mem_noofs_operand" "+Ua")) 298 (set (match_dup 1) 299 (unspec_volatile:QHSD 300 [(syncop:QHSD (match_dup 1) 301 (match_operand:QHSD 2 "<atomic_op_operand>" "<atomic_op_str>")) 302 (match_operand:SI 3 "const_int_operand")] ;; model 303 VUNSPEC_ATOMIC_OP)) 304 (clobber (reg:CC CC_REGNUM)) 305 (clobber (match_scratch:QHSD 4 "=&r")) 306 (clobber (match_scratch:SI 5 "=&r"))] 307 "<sync_predtab>" 308 "#" 309 "&& reload_completed" 310 [(const_int 0)] 311 { 312 arm_split_atomic_op (<CODE>, operands[0], operands[4], operands[1], 313 operands[2], operands[3], operands[5]); 314 DONE; 315 }) 316 317(define_insn_and_split "atomic_fetch_nand<mode>" 318 [(set (match_operand:QHSD 0 "s_register_operand" "=&r") 319 (match_operand:QHSD 1 "mem_noofs_operand" "+Ua")) 320 (set (match_dup 1) 321 (unspec_volatile:QHSD 322 [(not:QHSD 323 (and:QHSD (match_dup 1) 324 (match_operand:QHSD 2 "<atomic_op_operand>" "<atomic_op_str>"))) 325 (match_operand:SI 3 "const_int_operand")] ;; model 326 VUNSPEC_ATOMIC_OP)) 327 (clobber (reg:CC CC_REGNUM)) 328 (clobber (match_scratch:QHSD 4 "=&r")) 329 (clobber (match_scratch:SI 5 "=&r"))] 330 "<sync_predtab>" 331 "#" 332 "&& reload_completed" 333 [(const_int 0)] 334 { 335 arm_split_atomic_op (NOT, operands[0], operands[4], operands[1], 336 operands[2], operands[3], operands[5]); 337 DONE; 338 }) 339 340(define_insn_and_split "atomic_<sync_optab>_fetch<mode>" 341 [(set (match_operand:QHSD 0 "s_register_operand" "=&r") 342 (syncop:QHSD 343 (match_operand:QHSD 1 "mem_noofs_operand" "+Ua") 344 (match_operand:QHSD 2 "<atomic_op_operand>" "<atomic_op_str>"))) 345 (set (match_dup 1) 346 (unspec_volatile:QHSD 347 [(match_dup 1) (match_dup 2) 348 (match_operand:SI 3 "const_int_operand")] ;; model 349 VUNSPEC_ATOMIC_OP)) 350 (clobber (reg:CC CC_REGNUM)) 351 (clobber (match_scratch:SI 4 "=&r"))] 352 "<sync_predtab>" 353 "#" 354 "&& reload_completed" 355 [(const_int 0)] 356 { 357 arm_split_atomic_op (<CODE>, NULL, operands[0], operands[1], 358 operands[2], operands[3], operands[4]); 359 DONE; 360 }) 361 362(define_insn_and_split "atomic_nand_fetch<mode>" 363 [(set (match_operand:QHSD 0 "s_register_operand" "=&r") 364 (not:QHSD 365 (and:QHSD 366 (match_operand:QHSD 1 "mem_noofs_operand" "+Ua") 367 (match_operand:QHSD 2 "<atomic_op_operand>" "<atomic_op_str>")))) 368 (set (match_dup 1) 369 (unspec_volatile:QHSD 370 [(match_dup 1) (match_dup 2) 371 (match_operand:SI 3 "const_int_operand")] ;; model 372 VUNSPEC_ATOMIC_OP)) 373 (clobber (reg:CC CC_REGNUM)) 374 (clobber (match_scratch:SI 4 "=&r"))] 375 "<sync_predtab>" 376 "#" 377 "&& reload_completed" 378 [(const_int 0)] 379 { 380 arm_split_atomic_op (NOT, NULL, operands[0], operands[1], 381 operands[2], operands[3], operands[4]); 382 DONE; 383 }) 384 385(define_insn "arm_load_exclusive<mode>" 386 [(set (match_operand:SI 0 "s_register_operand" "=r") 387 (zero_extend:SI 388 (unspec_volatile:NARROW 389 [(match_operand:NARROW 1 "mem_noofs_operand" "Ua")] 390 VUNSPEC_LL)))] 391 "TARGET_HAVE_LDREXBH" 392 "ldrex<sync_sfx>%?\t%0, %C1" 393 [(set_attr "predicable" "yes") 394 (set_attr "predicable_short_it" "no")]) 395 396(define_insn "arm_load_acquire_exclusive<mode>" 397 [(set (match_operand:SI 0 "s_register_operand" "=r") 398 (zero_extend:SI 399 (unspec_volatile:NARROW 400 [(match_operand:NARROW 1 "mem_noofs_operand" "Ua")] 401 VUNSPEC_LAX)))] 402 "TARGET_HAVE_LDACQ" 403 "ldaex<sync_sfx>%?\\t%0, %C1" 404 [(set_attr "predicable" "yes") 405 (set_attr "predicable_short_it" "no")]) 406 407(define_insn "arm_load_exclusivesi" 408 [(set (match_operand:SI 0 "s_register_operand" "=r") 409 (unspec_volatile:SI 410 [(match_operand:SI 1 "mem_noofs_operand" "Ua")] 411 VUNSPEC_LL))] 412 "TARGET_HAVE_LDREX" 413 "ldrex%?\t%0, %C1" 414 [(set_attr "predicable" "yes") 415 (set_attr "predicable_short_it" "no")]) 416 417(define_insn "arm_load_acquire_exclusivesi" 418 [(set (match_operand:SI 0 "s_register_operand" "=r") 419 (unspec_volatile:SI 420 [(match_operand:SI 1 "mem_noofs_operand" "Ua")] 421 VUNSPEC_LAX))] 422 "TARGET_HAVE_LDACQ" 423 "ldaex%?\t%0, %C1" 424 [(set_attr "predicable" "yes") 425 (set_attr "predicable_short_it" "no")]) 426 427(define_insn "arm_load_exclusivedi" 428 [(set (match_operand:DI 0 "s_register_operand" "=r") 429 (unspec_volatile:DI 430 [(match_operand:DI 1 "mem_noofs_operand" "Ua")] 431 VUNSPEC_LL))] 432 "TARGET_HAVE_LDREXD" 433 "ldrexd%?\t%0, %H0, %C1" 434 [(set_attr "predicable" "yes") 435 (set_attr "predicable_short_it" "no")]) 436 437(define_insn "arm_load_acquire_exclusivedi" 438 [(set (match_operand:DI 0 "s_register_operand" "=r") 439 (unspec_volatile:DI 440 [(match_operand:DI 1 "mem_noofs_operand" "Ua")] 441 VUNSPEC_LAX))] 442 "TARGET_HAVE_LDACQ && ARM_DOUBLEWORD_ALIGN" 443 "ldaexd%?\t%0, %H0, %C1" 444 [(set_attr "predicable" "yes") 445 (set_attr "predicable_short_it" "no")]) 446 447(define_insn "arm_store_exclusive<mode>" 448 [(set (match_operand:SI 0 "s_register_operand" "=&r") 449 (unspec_volatile:SI [(const_int 0)] VUNSPEC_SC)) 450 (set (match_operand:QHSD 1 "mem_noofs_operand" "=Ua") 451 (unspec_volatile:QHSD 452 [(match_operand:QHSD 2 "s_register_operand" "r")] 453 VUNSPEC_SC))] 454 "<sync_predtab>" 455 { 456 if (<MODE>mode == DImode) 457 { 458 rtx value = operands[2]; 459 /* The restrictions on target registers in ARM mode are that the two 460 registers are consecutive and the first one is even; Thumb is 461 actually more flexible, but DI should give us this anyway. 462 Note that the 1st register always gets the lowest word in memory. */ 463 gcc_assert ((REGNO (value) & 1) == 0 || TARGET_THUMB2); 464 operands[3] = gen_rtx_REG (SImode, REGNO (value) + 1); 465 return "strexd%?\t%0, %2, %3, %C1"; 466 } 467 return "strex<sync_sfx>%?\t%0, %2, %C1"; 468 } 469 [(set_attr "predicable" "yes") 470 (set_attr "predicable_short_it" "no")]) 471 472(define_insn "arm_store_release_exclusivedi" 473 [(set (match_operand:SI 0 "s_register_operand" "=&r") 474 (unspec_volatile:SI [(const_int 0)] VUNSPEC_SLX)) 475 (set (match_operand:DI 1 "mem_noofs_operand" "=Ua") 476 (unspec_volatile:DI 477 [(match_operand:DI 2 "s_register_operand" "r")] 478 VUNSPEC_SLX))] 479 "TARGET_HAVE_LDACQ && ARM_DOUBLEWORD_ALIGN" 480 { 481 rtx value = operands[2]; 482 /* See comment in arm_store_exclusive<mode> above. */ 483 gcc_assert ((REGNO (value) & 1) == 0 || TARGET_THUMB2); 484 operands[3] = gen_rtx_REG (SImode, REGNO (value) + 1); 485 return "stlexd%?\t%0, %2, %3, %C1"; 486 } 487 [(set_attr "predicable" "yes") 488 (set_attr "predicable_short_it" "no")]) 489 490(define_insn "arm_store_release_exclusive<mode>" 491 [(set (match_operand:SI 0 "s_register_operand" "=&r") 492 (unspec_volatile:SI [(const_int 0)] VUNSPEC_SLX)) 493 (set (match_operand:QHSI 1 "mem_noofs_operand" "=Ua") 494 (unspec_volatile:QHSI 495 [(match_operand:QHSI 2 "s_register_operand" "r")] 496 VUNSPEC_SLX))] 497 "TARGET_HAVE_LDACQ" 498 "stlex<sync_sfx>%?\t%0, %2, %C1" 499 [(set_attr "predicable" "yes") 500 (set_attr "predicable_short_it" "no")]) 501