1285163Sdim// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 2285163Sdim// 3353358Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4353358Sdim// See https://llvm.org/LICENSE.txt for license information. 5353358Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6285163Sdim// 7285163Sdim//===----------------------------------------------------------------------===// 8286684Sdim/// 9286684Sdim/// \file 10341825Sdim/// WebAssembly Atomic operand code-gen constructs. 11286684Sdim/// 12285163Sdim//===----------------------------------------------------------------------===// 13285163Sdim 14353358Sdimlet UseNamedOperandTable = 1 in 15353358Sdimmulticlass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 16353358Sdim list<dag> pattern_r, string asmstr_r = "", 17353358Sdim string asmstr_s = "", bits<32> atomic_op = -1> { 18353358Sdim defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 19353358Sdim !or(0xfe00, !and(0xff, atomic_op))>, 20353358Sdim Requires<[HasAtomics]>; 21353358Sdim} 22353358Sdim 23353358Sdimmulticlass ATOMIC_NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "", 24353358Sdim bits<32> atomic_op = -1> { 25353358Sdim defm "" : NRI<oops, iops, pattern, asmstr, 26353358Sdim !or(0xfe00, !and(0xff, atomic_op))>, 27353358Sdim Requires<[HasAtomics]>; 28353358Sdim} 29353358Sdim 30285163Sdim//===----------------------------------------------------------------------===// 31353358Sdim// Atomic wait / notify 32353358Sdim//===----------------------------------------------------------------------===// 33353358Sdim 34353358Sdimlet hasSideEffects = 1 in { 35353358Sdimdefm ATOMIC_NOTIFY : 36353358Sdim ATOMIC_I<(outs I32:$dst), 37353358Sdim (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), 38353358Sdim (outs), (ins P2Align:$p2align, offset32_op:$off), [], 39353358Sdim "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 40353358Sdim "atomic.notify \t${off}${p2align}", 0x00>; 41353358Sdimlet mayLoad = 1 in { 42353358Sdimdefm ATOMIC_WAIT_I32 : 43353358Sdim ATOMIC_I<(outs I32:$dst), 44353358Sdim (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, 45353358Sdim I64:$timeout), 46353358Sdim (outs), (ins P2Align:$p2align, offset32_op:$off), [], 47353358Sdim "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 48353358Sdim "i32.atomic.wait \t${off}${p2align}", 0x01>; 49353358Sdimdefm ATOMIC_WAIT_I64 : 50353358Sdim ATOMIC_I<(outs I32:$dst), 51353358Sdim (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, 52353358Sdim I64:$timeout), 53353358Sdim (outs), (ins P2Align:$p2align, offset32_op:$off), [], 54353358Sdim "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 55353358Sdim "i64.atomic.wait \t${off}${p2align}", 0x02>; 56353358Sdim} // mayLoad = 1 57353358Sdim} // hasSideEffects = 1 58353358Sdim 59353358Sdimlet Predicates = [HasAtomics] in { 60353358Sdim// Select notifys with no constant offset. 61353358Sdimdef NotifyPatNoOffset : 62353358Sdim Pat<(i32 (int_wasm_atomic_notify I32:$addr, I32:$count)), 63353358Sdim (ATOMIC_NOTIFY 0, 0, I32:$addr, I32:$count)>; 64353358Sdim 65353358Sdim// Select notifys with a constant offset. 66353358Sdim 67353358Sdim// Pattern with address + immediate offset 68353358Sdimclass NotifyPatImmOff<PatFrag operand> : 69353358Sdim Pat<(i32 (int_wasm_atomic_notify (operand I32:$addr, imm:$off), I32:$count)), 70353358Sdim (ATOMIC_NOTIFY 0, imm:$off, I32:$addr, I32:$count)>; 71353358Sdimdef : NotifyPatImmOff<regPlusImm>; 72353358Sdimdef : NotifyPatImmOff<or_is_add>; 73353358Sdim 74353358Sdim// Select notifys with just a constant offset. 75353358Sdimdef NotifyPatOffsetOnly : 76353358Sdim Pat<(i32 (int_wasm_atomic_notify imm:$off, I32:$count)), 77353358Sdim (ATOMIC_NOTIFY 0, imm:$off, (CONST_I32 0), I32:$count)>; 78353358Sdim 79353358Sdimdef NotifyPatGlobalAddrOffOnly : 80353358Sdim Pat<(i32 (int_wasm_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 81353358Sdim I32:$count)), 82353358Sdim (ATOMIC_NOTIFY 0, tglobaladdr:$off, (CONST_I32 0), I32:$count)>; 83353358Sdim 84353358Sdim// Select waits with no constant offset. 85353358Sdimclass WaitPatNoOffset<ValueType ty, Intrinsic kind, NI inst> : 86353358Sdim Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), 87353358Sdim (inst 0, 0, I32:$addr, ty:$exp, I64:$timeout)>; 88353358Sdimdef : WaitPatNoOffset<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 89353358Sdimdef : WaitPatNoOffset<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 90353358Sdim 91353358Sdim// Select waits with a constant offset. 92353358Sdim 93353358Sdim// Pattern with address + immediate offset 94353358Sdimclass WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, NI inst> : 95353358Sdim Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), 96353358Sdim (inst 0, imm:$off, I32:$addr, ty:$exp, I64:$timeout)>; 97353358Sdimdef : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, regPlusImm, ATOMIC_WAIT_I32>; 98353358Sdimdef : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, or_is_add, ATOMIC_WAIT_I32>; 99353358Sdimdef : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, regPlusImm, ATOMIC_WAIT_I64>; 100353358Sdimdef : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, or_is_add, ATOMIC_WAIT_I64>; 101353358Sdim 102353358Sdim// Select wait_i32, ATOMIC_WAIT_I32s with just a constant offset. 103353358Sdimclass WaitPatOffsetOnly<ValueType ty, Intrinsic kind, NI inst> : 104353358Sdim Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 105353358Sdim (inst 0, imm:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 106353358Sdimdef : WaitPatOffsetOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 107353358Sdimdef : WaitPatOffsetOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 108353358Sdim 109353358Sdimclass WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, NI inst> : 110353358Sdim Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, I64:$timeout)), 111353358Sdim (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, I64:$timeout)>; 112353358Sdimdef : WaitPatGlobalAddrOffOnly<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>; 113353358Sdimdef : WaitPatGlobalAddrOffOnly<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>; 114353358Sdim} // Predicates = [HasAtomics] 115353358Sdim 116353358Sdim//===----------------------------------------------------------------------===// 117360784Sdim// Atomic fences 118360784Sdim//===----------------------------------------------------------------------===// 119360784Sdim 120360784Sdim// A compiler fence instruction that prevents reordering of instructions. 121360784Sdimlet Defs = [ARGUMENTS] in { 122360784Sdimlet isPseudo = 1, hasSideEffects = 1 in 123360784Sdimdefm COMPILER_FENCE : ATOMIC_NRI<(outs), (ins), [], "compiler_fence">; 124360784Sdimlet hasSideEffects = 1 in 125360784Sdimdefm ATOMIC_FENCE : ATOMIC_NRI<(outs), (ins i8imm:$flags), [], "atomic.fence", 126360784Sdim 0x03>; 127360784Sdim} // Defs = [ARGUMENTS] 128360784Sdim 129360784Sdim//===----------------------------------------------------------------------===// 130327952Sdim// Atomic loads 131285163Sdim//===----------------------------------------------------------------------===// 132285163Sdim 133353358Sdimmulticlass AtomicLoad<WebAssemblyRegClass rc, string name, int atomic_op> { 134353358Sdim defm "" : WebAssemblyLoad<rc, name, !or(0xfe00, !and(0xff, atomic_op))>, 135344779Sdim Requires<[HasAtomics]>; 136344779Sdim} 137344779Sdim 138353358Sdimdefm ATOMIC_LOAD_I32 : AtomicLoad<I32, "i32.atomic.load", 0x10>; 139353358Sdimdefm ATOMIC_LOAD_I64 : AtomicLoad<I64, "i64.atomic.load", 0x11>; 140285163Sdim 141327952Sdim// Select loads with no constant offset. 142327952Sdimlet Predicates = [HasAtomics] in { 143327952Sdimdef : LoadPatNoOffset<i32, atomic_load_32, ATOMIC_LOAD_I32>; 144327952Sdimdef : LoadPatNoOffset<i64, atomic_load_64, ATOMIC_LOAD_I64>; 145285163Sdim 146327952Sdim// Select loads with a constant offset. 147285163Sdim 148327952Sdim// Pattern with address + immediate offset 149327952Sdimdef : LoadPatImmOff<i32, atomic_load_32, regPlusImm, ATOMIC_LOAD_I32>; 150327952Sdimdef : LoadPatImmOff<i64, atomic_load_64, regPlusImm, ATOMIC_LOAD_I64>; 151327952Sdimdef : LoadPatImmOff<i32, atomic_load_32, or_is_add, ATOMIC_LOAD_I32>; 152327952Sdimdef : LoadPatImmOff<i64, atomic_load_64, or_is_add, ATOMIC_LOAD_I64>; 153327952Sdim 154327952Sdim// Select loads with just a constant offset. 155327952Sdimdef : LoadPatOffsetOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 156327952Sdimdef : LoadPatOffsetOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 157327952Sdim 158327952Sdimdef : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>; 159327952Sdimdef : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>; 160327952Sdim 161327952Sdim} // Predicates = [HasAtomics] 162327952Sdim 163327952Sdim// Extending loads. Note that there are only zero-extending atomic loads, no 164327952Sdim// sign-extending loads. 165353358Sdimdefm ATOMIC_LOAD8_U_I32 : AtomicLoad<I32, "i32.atomic.load8_u", 0x12>; 166353358Sdimdefm ATOMIC_LOAD16_U_I32 : AtomicLoad<I32, "i32.atomic.load16_u", 0x13>; 167353358Sdimdefm ATOMIC_LOAD8_U_I64 : AtomicLoad<I64, "i64.atomic.load8_u", 0x14>; 168353358Sdimdefm ATOMIC_LOAD16_U_I64 : AtomicLoad<I64, "i64.atomic.load16_u", 0x15>; 169353358Sdimdefm ATOMIC_LOAD32_U_I64 : AtomicLoad<I64, "i64.atomic.load32_u", 0x16>; 170327952Sdim 171341825Sdim// Fragments for extending loads. These are different from regular loads because 172327952Sdim// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 173327952Sdim// therefore don't have the extension type field. So instead of matching that, 174327952Sdim// we match the patterns that the type legalizer expands them to. 175327952Sdim 176327952Sdim// We directly match zext patterns and select the zext atomic loads. 177327952Sdim// i32 (zext (i8 (atomic_load_8))) gets legalized to 178327952Sdim// i32 (and (i32 (atomic_load_8)), 255) 179327952Sdim// These can be selected to a single zero-extending atomic load instruction. 180341825Sdimdef zext_aload_8_32 : 181341825Sdim PatFrag<(ops node:$addr), (and (i32 (atomic_load_8 node:$addr)), 255)>; 182341825Sdimdef zext_aload_16_32 : 183341825Sdim PatFrag<(ops node:$addr), (and (i32 (atomic_load_16 node:$addr)), 65535)>; 184327952Sdim// Unlike regular loads, extension to i64 is handled differently than i32. 185327952Sdim// i64 (zext (i8 (atomic_load_8))) gets legalized to 186327952Sdim// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 187327952Sdimdef zext_aload_8_64 : 188327952Sdim PatFrag<(ops node:$addr), 189327952Sdim (and (i64 (anyext (i32 (atomic_load_8 node:$addr)))), 255)>; 190327952Sdimdef zext_aload_16_64 : 191327952Sdim PatFrag<(ops node:$addr), 192327952Sdim (and (i64 (anyext (i32 (atomic_load_16 node:$addr)))), 65535)>; 193327952Sdimdef zext_aload_32_64 : 194327952Sdim PatFrag<(ops node:$addr), 195327952Sdim (zext (i32 (atomic_load node:$addr)))>; 196327952Sdim 197327952Sdim// We don't have single sext atomic load instructions. So for sext loads, we 198327952Sdim// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 199327952Sdim// results) and select a zext load; the next instruction will be sext_inreg 200327952Sdim// which is selected by itself. 201341825Sdimdef sext_aload_8_64 : 202327952Sdim PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 203341825Sdimdef sext_aload_16_64 : 204327952Sdim PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 205327952Sdim 206327952Sdimlet Predicates = [HasAtomics] in { 207327952Sdim// Select zero-extending loads with no constant offset. 208341825Sdimdef : LoadPatNoOffset<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 209341825Sdimdef : LoadPatNoOffset<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 210327952Sdimdef : LoadPatNoOffset<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 211327952Sdimdef : LoadPatNoOffset<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 212327952Sdimdef : LoadPatNoOffset<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 213327952Sdim 214327952Sdim// Select sign-extending loads with no constant offset 215327952Sdimdef : LoadPatNoOffset<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 216327952Sdimdef : LoadPatNoOffset<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 217341825Sdimdef : LoadPatNoOffset<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 218341825Sdimdef : LoadPatNoOffset<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 219344779Sdim// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s 220327952Sdim 221327952Sdim// Zero-extending loads with constant offset 222341825Sdimdef : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, ATOMIC_LOAD8_U_I32>; 223341825Sdimdef : LoadPatImmOff<i32, zext_aload_16_32, regPlusImm, ATOMIC_LOAD16_U_I32>; 224341825Sdimdef : LoadPatImmOff<i32, zext_aload_8_32, or_is_add, ATOMIC_LOAD8_U_I32>; 225341825Sdimdef : LoadPatImmOff<i32, zext_aload_16_32, or_is_add, ATOMIC_LOAD16_U_I32>; 226327952Sdimdef : LoadPatImmOff<i64, zext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 227327952Sdimdef : LoadPatImmOff<i64, zext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 228327952Sdimdef : LoadPatImmOff<i64, zext_aload_32_64, regPlusImm, ATOMIC_LOAD32_U_I64>; 229327952Sdimdef : LoadPatImmOff<i64, zext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 230327952Sdimdef : LoadPatImmOff<i64, zext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 231327952Sdimdef : LoadPatImmOff<i64, zext_aload_32_64, or_is_add, ATOMIC_LOAD32_U_I64>; 232327952Sdim 233327952Sdim// Sign-extending loads with constant offset 234327952Sdimdef : LoadPatImmOff<i32, atomic_load_8, regPlusImm, ATOMIC_LOAD8_U_I32>; 235327952Sdimdef : LoadPatImmOff<i32, atomic_load_16, regPlusImm, ATOMIC_LOAD16_U_I32>; 236327952Sdimdef : LoadPatImmOff<i32, atomic_load_8, or_is_add, ATOMIC_LOAD8_U_I32>; 237327952Sdimdef : LoadPatImmOff<i32, atomic_load_16, or_is_add, ATOMIC_LOAD16_U_I32>; 238341825Sdimdef : LoadPatImmOff<i64, sext_aload_8_64, regPlusImm, ATOMIC_LOAD8_U_I64>; 239341825Sdimdef : LoadPatImmOff<i64, sext_aload_16_64, regPlusImm, ATOMIC_LOAD16_U_I64>; 240341825Sdimdef : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>; 241341825Sdimdef : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>; 242327952Sdim// No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64 243327952Sdim 244327952Sdim// Extending loads with just a constant offset 245341825Sdimdef : LoadPatOffsetOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 246341825Sdimdef : LoadPatOffsetOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 247327952Sdimdef : LoadPatOffsetOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 248327952Sdimdef : LoadPatOffsetOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 249327952Sdimdef : LoadPatOffsetOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 250327952Sdimdef : LoadPatOffsetOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 251327952Sdimdef : LoadPatOffsetOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 252341825Sdimdef : LoadPatOffsetOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 253341825Sdimdef : LoadPatOffsetOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 254327952Sdim 255341825Sdimdef : LoadPatGlobalAddrOffOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>; 256341825Sdimdef : LoadPatGlobalAddrOffOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>; 257327952Sdimdef : LoadPatGlobalAddrOffOnly<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>; 258327952Sdimdef : LoadPatGlobalAddrOffOnly<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>; 259327952Sdimdef : LoadPatGlobalAddrOffOnly<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>; 260327952Sdimdef : LoadPatGlobalAddrOffOnly<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>; 261327952Sdimdef : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>; 262341825Sdimdef : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>; 263341825Sdimdef : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>; 264327952Sdim 265327952Sdim} // Predicates = [HasAtomics] 266327952Sdim 267285163Sdim//===----------------------------------------------------------------------===// 268285163Sdim// Atomic stores 269285163Sdim//===----------------------------------------------------------------------===// 270285163Sdim 271353358Sdimmulticlass AtomicStore<WebAssemblyRegClass rc, string name, int atomic_op> { 272353358Sdim defm "" : WebAssemblyStore<rc, name, !or(0xfe00, !and(0xff, atomic_op))>, 273353358Sdim Requires<[HasAtomics]>; 274353358Sdim} 275285163Sdim 276353358Sdimdefm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>; 277353358Sdimdefm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>; 278353358Sdim 279341825Sdim// We need an 'atomic' version of store patterns because store and atomic_store 280341825Sdim// nodes have different operand orders: 281341825Sdim// store: (store $val, $ptr) 282341825Sdim// atomic_store: (store $ptr, $val) 283341825Sdim 284341825Sdimlet Predicates = [HasAtomics] in { 285341825Sdim 286341825Sdim// Select stores with no constant offset. 287341825Sdimclass AStorePatNoOffset<ValueType ty, PatFrag kind, NI inst> : 288341825Sdim Pat<(kind I32:$addr, ty:$val), (inst 0, 0, I32:$addr, ty:$val)>; 289341825Sdimdef : AStorePatNoOffset<i32, atomic_store_32, ATOMIC_STORE_I32>; 290341825Sdimdef : AStorePatNoOffset<i64, atomic_store_64, ATOMIC_STORE_I64>; 291341825Sdim 292341825Sdim// Select stores with a constant offset. 293341825Sdim 294341825Sdim// Pattern with address + immediate offset 295341825Sdimclass AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 296341825Sdim Pat<(kind (operand I32:$addr, imm:$off), ty:$val), 297341825Sdim (inst 0, imm:$off, I32:$addr, ty:$val)>; 298341825Sdimdef : AStorePatImmOff<i32, atomic_store_32, regPlusImm, ATOMIC_STORE_I32>; 299341825Sdimdef : AStorePatImmOff<i64, atomic_store_64, regPlusImm, ATOMIC_STORE_I64>; 300341825Sdimdef : AStorePatImmOff<i32, atomic_store_32, or_is_add, ATOMIC_STORE_I32>; 301341825Sdimdef : AStorePatImmOff<i64, atomic_store_64, or_is_add, ATOMIC_STORE_I64>; 302341825Sdim 303341825Sdim// Select stores with just a constant offset. 304341825Sdimclass AStorePatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 305341825Sdim Pat<(kind imm:$off, ty:$val), (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 306341825Sdimdef : AStorePatOffsetOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 307341825Sdimdef : AStorePatOffsetOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 308341825Sdim 309341825Sdimclass AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 310341825Sdim Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 311341825Sdim (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 312341825Sdimdef : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, ATOMIC_STORE_I32>; 313341825Sdimdef : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, ATOMIC_STORE_I64>; 314341825Sdim 315341825Sdim} // Predicates = [HasAtomics] 316341825Sdim 317341825Sdim// Truncating stores. 318353358Sdimdefm ATOMIC_STORE8_I32 : AtomicStore<I32, "i32.atomic.store8", 0x19>; 319353358Sdimdefm ATOMIC_STORE16_I32 : AtomicStore<I32, "i32.atomic.store16", 0x1a>; 320353358Sdimdefm ATOMIC_STORE8_I64 : AtomicStore<I64, "i64.atomic.store8", 0x1b>; 321353358Sdimdefm ATOMIC_STORE16_I64 : AtomicStore<I64, "i64.atomic.store16", 0x1c>; 322353358Sdimdefm ATOMIC_STORE32_I64 : AtomicStore<I64, "i64.atomic.store32", 0x1d>; 323341825Sdim 324341825Sdim// Fragments for truncating stores. 325341825Sdim 326341825Sdim// We don't have single truncating atomic store instructions. For 32-bit 327341825Sdim// instructions, we just need to match bare atomic stores. On the other hand, 328341825Sdim// truncating stores from i64 values are once truncated to i32 first. 329341825Sdimclass trunc_astore_64<PatFrag kind> : 330341825Sdim PatFrag<(ops node:$addr, node:$val), 331341825Sdim (kind node:$addr, (i32 (trunc (i64 node:$val))))>; 332341825Sdimdef trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 333341825Sdimdef trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 334341825Sdimdef trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 335341825Sdim 336341825Sdimlet Predicates = [HasAtomics] in { 337341825Sdim 338341825Sdim// Truncating stores with no constant offset 339341825Sdimdef : AStorePatNoOffset<i32, atomic_store_8, ATOMIC_STORE8_I32>; 340341825Sdimdef : AStorePatNoOffset<i32, atomic_store_16, ATOMIC_STORE16_I32>; 341341825Sdimdef : AStorePatNoOffset<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 342341825Sdimdef : AStorePatNoOffset<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 343341825Sdimdef : AStorePatNoOffset<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 344341825Sdim 345341825Sdim// Truncating stores with a constant offset 346341825Sdimdef : AStorePatImmOff<i32, atomic_store_8, regPlusImm, ATOMIC_STORE8_I32>; 347341825Sdimdef : AStorePatImmOff<i32, atomic_store_16, regPlusImm, ATOMIC_STORE16_I32>; 348341825Sdimdef : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, ATOMIC_STORE8_I64>; 349341825Sdimdef : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm, ATOMIC_STORE16_I64>; 350341825Sdimdef : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm, ATOMIC_STORE32_I64>; 351341825Sdimdef : AStorePatImmOff<i32, atomic_store_8, or_is_add, ATOMIC_STORE8_I32>; 352341825Sdimdef : AStorePatImmOff<i32, atomic_store_16, or_is_add, ATOMIC_STORE16_I32>; 353341825Sdimdef : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, ATOMIC_STORE8_I64>; 354341825Sdimdef : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, ATOMIC_STORE16_I64>; 355341825Sdimdef : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, ATOMIC_STORE32_I64>; 356341825Sdim 357341825Sdim// Truncating stores with just a constant offset 358341825Sdimdef : AStorePatOffsetOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 359341825Sdimdef : AStorePatOffsetOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 360341825Sdimdef : AStorePatOffsetOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 361341825Sdimdef : AStorePatOffsetOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 362341825Sdimdef : AStorePatOffsetOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 363341825Sdim 364341825Sdimdef : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>; 365341825Sdimdef : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>; 366341825Sdimdef : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>; 367341825Sdimdef : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>; 368341825Sdimdef : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>; 369341825Sdim 370341825Sdim} // Predicates = [HasAtomics] 371341825Sdim 372285163Sdim//===----------------------------------------------------------------------===// 373341825Sdim// Atomic binary read-modify-writes 374285163Sdim//===----------------------------------------------------------------------===// 375285163Sdim 376353358Sdimmulticlass WebAssemblyBinRMW<WebAssemblyRegClass rc, string name, 377353358Sdim int atomic_op> { 378353358Sdim defm "" : 379353358Sdim ATOMIC_I<(outs rc:$dst), 380353358Sdim (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 381353358Sdim (outs), (ins P2Align:$p2align, offset32_op:$off), [], 382353358Sdim !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 383353358Sdim !strconcat(name, "\t${off}${p2align}"), atomic_op>; 384341825Sdim} 385285163Sdim 386353358Sdimdefm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0x1e>; 387353358Sdimdefm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0x1f>; 388341825Sdimdefm ATOMIC_RMW8_U_ADD_I32 : 389353358Sdim WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0x20>; 390341825Sdimdefm ATOMIC_RMW16_U_ADD_I32 : 391353358Sdim WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0x21>; 392341825Sdimdefm ATOMIC_RMW8_U_ADD_I64 : 393353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0x22>; 394341825Sdimdefm ATOMIC_RMW16_U_ADD_I64 : 395353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0x23>; 396341825Sdimdefm ATOMIC_RMW32_U_ADD_I64 : 397353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0x24>; 398285163Sdim 399353358Sdimdefm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0x25>; 400353358Sdimdefm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0x26>; 401341825Sdimdefm ATOMIC_RMW8_U_SUB_I32 : 402353358Sdim WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0x27>; 403341825Sdimdefm ATOMIC_RMW16_U_SUB_I32 : 404353358Sdim WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0x28>; 405341825Sdimdefm ATOMIC_RMW8_U_SUB_I64 : 406353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0x29>; 407341825Sdimdefm ATOMIC_RMW16_U_SUB_I64 : 408353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0x2a>; 409341825Sdimdefm ATOMIC_RMW32_U_SUB_I64 : 410353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0x2b>; 411285163Sdim 412353358Sdimdefm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0x2c>; 413353358Sdimdefm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0x2d>; 414341825Sdimdefm ATOMIC_RMW8_U_AND_I32 : 415353358Sdim WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0x2e>; 416341825Sdimdefm ATOMIC_RMW16_U_AND_I32 : 417353358Sdim WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0x2f>; 418341825Sdimdefm ATOMIC_RMW8_U_AND_I64 : 419353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0x30>; 420341825Sdimdefm ATOMIC_RMW16_U_AND_I64 : 421353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0x31>; 422341825Sdimdefm ATOMIC_RMW32_U_AND_I64 : 423353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0x32>; 424327952Sdim 425353358Sdimdefm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0x33>; 426353358Sdimdefm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0x34>; 427341825Sdimdefm ATOMIC_RMW8_U_OR_I32 : 428353358Sdim WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0x35>; 429341825Sdimdefm ATOMIC_RMW16_U_OR_I32 : 430353358Sdim WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0x36>; 431341825Sdimdefm ATOMIC_RMW8_U_OR_I64 : 432353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0x37>; 433341825Sdimdefm ATOMIC_RMW16_U_OR_I64 : 434353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0x38>; 435341825Sdimdefm ATOMIC_RMW32_U_OR_I64 : 436353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0x39>; 437341825Sdim 438353358Sdimdefm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0x3a>; 439353358Sdimdefm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0x3b>; 440341825Sdimdefm ATOMIC_RMW8_U_XOR_I32 : 441353358Sdim WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0x3c>; 442341825Sdimdefm ATOMIC_RMW16_U_XOR_I32 : 443353358Sdim WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0x3d>; 444341825Sdimdefm ATOMIC_RMW8_U_XOR_I64 : 445353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0x3e>; 446341825Sdimdefm ATOMIC_RMW16_U_XOR_I64 : 447353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0x3f>; 448341825Sdimdefm ATOMIC_RMW32_U_XOR_I64 : 449353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0x40>; 450341825Sdim 451341825Sdimdefm ATOMIC_RMW_XCHG_I32 : 452353358Sdim WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0x41>; 453341825Sdimdefm ATOMIC_RMW_XCHG_I64 : 454353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0x42>; 455341825Sdimdefm ATOMIC_RMW8_U_XCHG_I32 : 456353358Sdim WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0x43>; 457341825Sdimdefm ATOMIC_RMW16_U_XCHG_I32 : 458353358Sdim WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0x44>; 459341825Sdimdefm ATOMIC_RMW8_U_XCHG_I64 : 460353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0x45>; 461341825Sdimdefm ATOMIC_RMW16_U_XCHG_I64 : 462353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0x46>; 463341825Sdimdefm ATOMIC_RMW32_U_XCHG_I64 : 464353358Sdim WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0x47>; 465341825Sdim 466341825Sdim// Select binary RMWs with no constant offset. 467341825Sdimclass BinRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 468341825Sdim Pat<(ty (kind I32:$addr, ty:$val)), (inst 0, 0, I32:$addr, ty:$val)>; 469341825Sdim 470341825Sdim// Select binary RMWs with a constant offset. 471341825Sdim 472341825Sdim// Pattern with address + immediate offset 473341825Sdimclass BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 474341825Sdim Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), 475341825Sdim (inst 0, imm:$off, I32:$addr, ty:$val)>; 476341825Sdim 477341825Sdim// Select binary RMWs with just a constant offset. 478341825Sdimclass BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 479341825Sdim Pat<(ty (kind imm:$off, ty:$val)), 480341825Sdim (inst 0, imm:$off, (CONST_I32 0), ty:$val)>; 481341825Sdim 482341825Sdimclass BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 483341825Sdim Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 484341825Sdim (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>; 485341825Sdim 486341825Sdim// Patterns for various addressing modes. 487341825Sdimmulticlass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 488341825Sdim NI inst_64> { 489341825Sdim def : BinRMWPatNoOffset<i32, rmw_32, inst_32>; 490341825Sdim def : BinRMWPatNoOffset<i64, rmw_64, inst_64>; 491341825Sdim 492341825Sdim def : BinRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 493341825Sdim def : BinRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 494341825Sdim def : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 495341825Sdim def : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 496341825Sdim 497341825Sdim def : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>; 498341825Sdim def : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>; 499341825Sdim 500341825Sdim def : BinRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 501341825Sdim def : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 502341825Sdim} 503341825Sdim 504341825Sdimlet Predicates = [HasAtomics] in { 505341825Sdimdefm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, ATOMIC_RMW_ADD_I32, 506341825Sdim ATOMIC_RMW_ADD_I64>; 507341825Sdimdefm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, ATOMIC_RMW_SUB_I32, 508341825Sdim ATOMIC_RMW_SUB_I64>; 509341825Sdimdefm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, ATOMIC_RMW_AND_I32, 510341825Sdim ATOMIC_RMW_AND_I64>; 511341825Sdimdefm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, ATOMIC_RMW_OR_I32, 512341825Sdim ATOMIC_RMW_OR_I64>; 513341825Sdimdefm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, ATOMIC_RMW_XOR_I32, 514341825Sdim ATOMIC_RMW_XOR_I64>; 515341825Sdimdefm : BinRMWPattern<atomic_swap_32, atomic_swap_64, ATOMIC_RMW_XCHG_I32, 516341825Sdim ATOMIC_RMW_XCHG_I64>; 517341825Sdim} // Predicates = [HasAtomics] 518341825Sdim 519341825Sdim// Truncating & zero-extending binary RMW patterns. 520341825Sdim// These are combined patterns of truncating store patterns and zero-extending 521341825Sdim// load patterns above. 522341825Sdimclass zext_bin_rmw_8_32<PatFrag kind> : 523341825Sdim PatFrag<(ops node:$addr, node:$val), 524341825Sdim (and (i32 (kind node:$addr, node:$val)), 255)>; 525341825Sdimclass zext_bin_rmw_16_32<PatFrag kind> : 526341825Sdim PatFrag<(ops node:$addr, node:$val), 527341825Sdim (and (i32 (kind node:$addr, node:$val)), 65535)>; 528341825Sdimclass zext_bin_rmw_8_64<PatFrag kind> : 529341825Sdim PatFrag<(ops node:$addr, node:$val), 530341825Sdim (and (i64 (anyext (i32 (kind node:$addr, 531341825Sdim (i32 (trunc (i64 node:$val))))))), 255)>; 532341825Sdimclass zext_bin_rmw_16_64<PatFrag kind> : 533341825Sdim PatFrag<(ops node:$addr, node:$val), 534341825Sdim (and (i64 (anyext (i32 (kind node:$addr, 535341825Sdim (i32 (trunc (i64 node:$val))))))), 65535)>; 536341825Sdimclass zext_bin_rmw_32_64<PatFrag kind> : 537341825Sdim PatFrag<(ops node:$addr, node:$val), 538341825Sdim (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 539341825Sdim 540341825Sdim// Truncating & sign-extending binary RMW patterns. 541341825Sdim// These are combined patterns of truncating store patterns and sign-extending 542341825Sdim// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 543341825Sdim// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 544341825Sdim// is selected by itself. 545341825Sdimclass sext_bin_rmw_8_32<PatFrag kind> : 546341825Sdim PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 547341825Sdimclass sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 548341825Sdimclass sext_bin_rmw_8_64<PatFrag kind> : 549341825Sdim PatFrag<(ops node:$addr, node:$val), 550341825Sdim (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 551341825Sdimclass sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 552344779Sdim// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 553341825Sdim 554341825Sdim// Patterns for various addressing modes for truncating-extending binary RMWs. 555341825Sdimmulticlass BinRMWTruncExtPattern< 556341825Sdim PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 557341825Sdim NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 558341825Sdim // Truncating-extending binary RMWs with no constant offset 559341825Sdim def : BinRMWPatNoOffset<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 560341825Sdim def : BinRMWPatNoOffset<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 561341825Sdim def : BinRMWPatNoOffset<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 562341825Sdim def : BinRMWPatNoOffset<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 563341825Sdim def : BinRMWPatNoOffset<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 564341825Sdim 565341825Sdim def : BinRMWPatNoOffset<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 566341825Sdim def : BinRMWPatNoOffset<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 567341825Sdim def : BinRMWPatNoOffset<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 568341825Sdim def : BinRMWPatNoOffset<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 569341825Sdim 570341825Sdim // Truncating-extending binary RMWs with a constant offset 571341825Sdim def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 572341825Sdim def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 573341825Sdim def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 574341825Sdim def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 575341825Sdim def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 576341825Sdim def : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 577341825Sdim def : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 578341825Sdim def : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 579341825Sdim def : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 580341825Sdim def : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 581341825Sdim 582341825Sdim def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 583341825Sdim def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 584341825Sdim def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 585341825Sdim def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 586341825Sdim def : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 587341825Sdim def : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 588341825Sdim def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 589341825Sdim def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 590341825Sdim 591341825Sdim // Truncating-extending binary RMWs with just a constant offset 592341825Sdim def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 593341825Sdim def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 594341825Sdim def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 595341825Sdim def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 596341825Sdim def : BinRMWPatOffsetOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 597341825Sdim 598341825Sdim def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 599341825Sdim def : BinRMWPatOffsetOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 600341825Sdim def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 601341825Sdim def : BinRMWPatOffsetOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 602341825Sdim 603341825Sdim def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 604341825Sdim def : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 605341825Sdim def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 606341825Sdim def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 607341825Sdim def : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 608341825Sdim 609341825Sdim def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 610341825Sdim def : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 611341825Sdim def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 612341825Sdim def : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 613341825Sdim} 614341825Sdim 615341825Sdimlet Predicates = [HasAtomics] in { 616341825Sdimdefm : BinRMWTruncExtPattern< 617341825Sdim atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64, 618341825Sdim ATOMIC_RMW8_U_ADD_I32, ATOMIC_RMW16_U_ADD_I32, 619341825Sdim ATOMIC_RMW8_U_ADD_I64, ATOMIC_RMW16_U_ADD_I64, ATOMIC_RMW32_U_ADD_I64>; 620341825Sdimdefm : BinRMWTruncExtPattern< 621341825Sdim atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, atomic_load_sub_64, 622341825Sdim ATOMIC_RMW8_U_SUB_I32, ATOMIC_RMW16_U_SUB_I32, 623341825Sdim ATOMIC_RMW8_U_SUB_I64, ATOMIC_RMW16_U_SUB_I64, ATOMIC_RMW32_U_SUB_I64>; 624341825Sdimdefm : BinRMWTruncExtPattern< 625341825Sdim atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, atomic_load_and_64, 626341825Sdim ATOMIC_RMW8_U_AND_I32, ATOMIC_RMW16_U_AND_I32, 627341825Sdim ATOMIC_RMW8_U_AND_I64, ATOMIC_RMW16_U_AND_I64, ATOMIC_RMW32_U_AND_I64>; 628341825Sdimdefm : BinRMWTruncExtPattern< 629341825Sdim atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, atomic_load_or_64, 630341825Sdim ATOMIC_RMW8_U_OR_I32, ATOMIC_RMW16_U_OR_I32, 631341825Sdim ATOMIC_RMW8_U_OR_I64, ATOMIC_RMW16_U_OR_I64, ATOMIC_RMW32_U_OR_I64>; 632341825Sdimdefm : BinRMWTruncExtPattern< 633341825Sdim atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, atomic_load_xor_64, 634341825Sdim ATOMIC_RMW8_U_XOR_I32, ATOMIC_RMW16_U_XOR_I32, 635341825Sdim ATOMIC_RMW8_U_XOR_I64, ATOMIC_RMW16_U_XOR_I64, ATOMIC_RMW32_U_XOR_I64>; 636341825Sdimdefm : BinRMWTruncExtPattern< 637341825Sdim atomic_swap_8, atomic_swap_16, atomic_swap_32, atomic_swap_64, 638341825Sdim ATOMIC_RMW8_U_XCHG_I32, ATOMIC_RMW16_U_XCHG_I32, 639341825Sdim ATOMIC_RMW8_U_XCHG_I64, ATOMIC_RMW16_U_XCHG_I64, ATOMIC_RMW32_U_XCHG_I64>; 640341825Sdim} // Predicates = [HasAtomics] 641344779Sdim 642344779Sdim//===----------------------------------------------------------------------===// 643344779Sdim// Atomic ternary read-modify-writes 644344779Sdim//===----------------------------------------------------------------------===// 645344779Sdim 646344779Sdim// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 647344779Sdim// flag}. When we use the success flag or both values, we can't make use of i64 648344779Sdim// truncate/extend versions of instructions for now, which is suboptimal. 649344779Sdim// Consider adding a pass after instruction selection that optimizes this case 650344779Sdim// if it is frequent. 651344779Sdim 652353358Sdimmulticlass WebAssemblyTerRMW<WebAssemblyRegClass rc, string name, 653353358Sdim int atomic_op> { 654353358Sdim defm "" : 655353358Sdim ATOMIC_I<(outs rc:$dst), 656353358Sdim (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 657353358Sdim rc:$new_), 658353358Sdim (outs), (ins P2Align:$p2align, offset32_op:$off), [], 659353358Sdim !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 660353358Sdim !strconcat(name, "\t${off}${p2align}"), atomic_op>; 661344779Sdim} 662344779Sdim 663344779Sdimdefm ATOMIC_RMW_CMPXCHG_I32 : 664353358Sdim WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0x48>; 665344779Sdimdefm ATOMIC_RMW_CMPXCHG_I64 : 666353358Sdim WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0x49>; 667344779Sdimdefm ATOMIC_RMW8_U_CMPXCHG_I32 : 668353358Sdim WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0x4a>; 669344779Sdimdefm ATOMIC_RMW16_U_CMPXCHG_I32 : 670353358Sdim WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0x4b>; 671344779Sdimdefm ATOMIC_RMW8_U_CMPXCHG_I64 : 672353358Sdim WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0x4c>; 673344779Sdimdefm ATOMIC_RMW16_U_CMPXCHG_I64 : 674353358Sdim WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0x4d>; 675344779Sdimdefm ATOMIC_RMW32_U_CMPXCHG_I64 : 676353358Sdim WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0x4e>; 677344779Sdim 678344779Sdim// Select ternary RMWs with no constant offset. 679344779Sdimclass TerRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> : 680344779Sdim Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), 681344779Sdim (inst 0, 0, I32:$addr, ty:$exp, ty:$new)>; 682344779Sdim 683344779Sdim// Select ternary RMWs with a constant offset. 684344779Sdim 685344779Sdim// Pattern with address + immediate offset 686344779Sdimclass TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> : 687344779Sdim Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), 688344779Sdim (inst 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>; 689344779Sdim 690344779Sdim// Select ternary RMWs with just a constant offset. 691344779Sdimclass TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> : 692344779Sdim Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 693344779Sdim (inst 0, imm:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 694344779Sdim 695344779Sdimclass TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> : 696344779Sdim Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 697344779Sdim (inst 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, ty:$new)>; 698344779Sdim 699344779Sdim// Patterns for various addressing modes. 700344779Sdimmulticlass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32, 701344779Sdim NI inst_64> { 702344779Sdim def : TerRMWPatNoOffset<i32, rmw_32, inst_32>; 703344779Sdim def : TerRMWPatNoOffset<i64, rmw_64, inst_64>; 704344779Sdim 705344779Sdim def : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 706344779Sdim def : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 707344779Sdim def : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 708344779Sdim def : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 709344779Sdim 710344779Sdim def : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>; 711344779Sdim def : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>; 712344779Sdim 713344779Sdim def : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 714344779Sdim def : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 715344779Sdim} 716344779Sdim 717353358Sdimlet Predicates = [HasAtomics] in 718344779Sdimdefm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64, 719344779Sdim ATOMIC_RMW_CMPXCHG_I32, ATOMIC_RMW_CMPXCHG_I64>; 720344779Sdim 721344779Sdim// Truncating & zero-extending ternary RMW patterns. 722344779Sdim// DAG legalization & optimization before instruction selection may introduce 723344779Sdim// additional nodes such as anyext or assertzext depending on operand types. 724344779Sdimclass zext_ter_rmw_8_32<PatFrag kind> : 725344779Sdim PatFrag<(ops node:$addr, node:$exp, node:$new), 726344779Sdim (and (i32 (kind node:$addr, node:$exp, node:$new)), 255)>; 727344779Sdimclass zext_ter_rmw_16_32<PatFrag kind> : 728344779Sdim PatFrag<(ops node:$addr, node:$exp, node:$new), 729344779Sdim (and (i32 (kind node:$addr, node:$exp, node:$new)), 65535)>; 730344779Sdimclass zext_ter_rmw_8_64<PatFrag kind> : 731344779Sdim PatFrag<(ops node:$addr, node:$exp, node:$new), 732344779Sdim (zext (i32 (assertzext (i32 (kind node:$addr, 733344779Sdim (i32 (trunc (i64 node:$exp))), 734344779Sdim (i32 (trunc (i64 node:$new))))))))>; 735344779Sdimclass zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 736344779Sdimclass zext_ter_rmw_32_64<PatFrag kind> : 737344779Sdim PatFrag<(ops node:$addr, node:$exp, node:$new), 738344779Sdim (zext (i32 (kind node:$addr, 739344779Sdim (i32 (trunc (i64 node:$exp))), 740344779Sdim (i32 (trunc (i64 node:$new))))))>; 741344779Sdim 742344779Sdim// Truncating & sign-extending ternary RMW patterns. 743344779Sdim// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 744344779Sdim// zext RMW; the next instruction will be sext_inreg which is selected by 745344779Sdim// itself. 746344779Sdimclass sext_ter_rmw_8_32<PatFrag kind> : 747344779Sdim PatFrag<(ops node:$addr, node:$exp, node:$new), 748344779Sdim (kind node:$addr, node:$exp, node:$new)>; 749344779Sdimclass sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 750344779Sdimclass sext_ter_rmw_8_64<PatFrag kind> : 751344779Sdim PatFrag<(ops node:$addr, node:$exp, node:$new), 752344779Sdim (anyext (i32 (assertzext (i32 753344779Sdim (kind node:$addr, 754344779Sdim (i32 (trunc (i64 node:$exp))), 755344779Sdim (i32 (trunc (i64 node:$new))))))))>; 756344779Sdimclass sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 757344779Sdim// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 758344779Sdim 759344779Sdim// Patterns for various addressing modes for truncating-extending ternary RMWs. 760344779Sdimmulticlass TerRMWTruncExtPattern< 761344779Sdim PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 762344779Sdim NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 763344779Sdim // Truncating-extending ternary RMWs with no constant offset 764344779Sdim def : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 765344779Sdim def : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 766344779Sdim def : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 767344779Sdim def : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 768344779Sdim def : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 769344779Sdim 770344779Sdim def : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 771344779Sdim def : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 772344779Sdim def : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 773344779Sdim def : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 774344779Sdim 775344779Sdim // Truncating-extending ternary RMWs with a constant offset 776344779Sdim def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 777344779Sdim def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 778344779Sdim def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 779344779Sdim def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 780344779Sdim def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, inst32_64>; 781344779Sdim def : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 782344779Sdim def : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 783344779Sdim def : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 784344779Sdim def : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 785344779Sdim def : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 786344779Sdim 787344779Sdim def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 788344779Sdim def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, inst16_32>; 789344779Sdim def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 790344779Sdim def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, inst16_64>; 791344779Sdim def : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 792344779Sdim def : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 793344779Sdim def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 794344779Sdim def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 795344779Sdim 796344779Sdim // Truncating-extending ternary RMWs with just a constant offset 797344779Sdim def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 798344779Sdim def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 799344779Sdim def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 800344779Sdim def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 801344779Sdim def : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 802344779Sdim 803344779Sdim def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 804344779Sdim def : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 805344779Sdim def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 806344779Sdim def : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 807344779Sdim 808344779Sdim def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 809344779Sdim def : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 810344779Sdim def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 811344779Sdim def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 812344779Sdim def : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 813344779Sdim 814344779Sdim def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 815344779Sdim def : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 816344779Sdim def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 817344779Sdim def : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 818344779Sdim} 819344779Sdim 820353358Sdimlet Predicates = [HasAtomics] in 821344779Sdimdefm : TerRMWTruncExtPattern< 822344779Sdim atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, 823344779Sdim ATOMIC_RMW8_U_CMPXCHG_I32, ATOMIC_RMW16_U_CMPXCHG_I32, 824344779Sdim ATOMIC_RMW8_U_CMPXCHG_I64, ATOMIC_RMW16_U_CMPXCHG_I64, 825344779Sdim ATOMIC_RMW32_U_CMPXCHG_I64>; 826