1// WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*- 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8/// 9/// \file 10/// WebAssembly Atomic operand code-gen constructs. 11/// 12//===----------------------------------------------------------------------===// 13 14let UseNamedOperandTable = 1 in 15multiclass ATOMIC_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 16 list<dag> pattern_r, string asmstr_r, 17 string asmstr_s, bits<32> atomic_op, 18 string is64 = "false"> { 19 defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 20 !or(0xfe00, !and(0xff, atomic_op)), is64>, 21 Requires<[HasAtomics]>; 22} 23 24multiclass ATOMIC_NRI<dag oops, dag iops, list<dag> pattern, string asmstr = "", 25 bits<32> atomic_op = -1> { 26 defm "" : NRI<oops, iops, pattern, asmstr, 27 !or(0xfe00, !and(0xff, atomic_op))>, 28 Requires<[HasAtomics]>; 29} 30 31//===----------------------------------------------------------------------===// 32// Atomic wait / notify 33//===----------------------------------------------------------------------===// 34 35let hasSideEffects = 1 in { 36defm ATOMIC_NOTIFY_A32 : 37 ATOMIC_I<(outs I32:$dst), 38 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$count), 39 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 40 "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 41 "atomic.notify \t${off}${p2align}", 0x00, "false">; 42defm ATOMIC_NOTIFY_A64 : 43 ATOMIC_I<(outs I32:$dst), 44 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$count), 45 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 46 "atomic.notify \t$dst, ${off}(${addr})${p2align}, $count", 47 "atomic.notify \t${off}${p2align}", 0x00, "true">; 48let mayLoad = 1 in { 49defm ATOMIC_WAIT_I32_A32 : 50 ATOMIC_I<(outs I32:$dst), 51 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I32:$exp, 52 I64:$timeout), 53 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 54 "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 55 "i32.atomic.wait \t${off}${p2align}", 0x01, "false">; 56defm ATOMIC_WAIT_I32_A64 : 57 ATOMIC_I<(outs I32:$dst), 58 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I32:$exp, 59 I64:$timeout), 60 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 61 "i32.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 62 "i32.atomic.wait \t${off}${p2align}", 0x01, "true">; 63defm ATOMIC_WAIT_I64_A32 : 64 ATOMIC_I<(outs I32:$dst), 65 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, I64:$exp, 66 I64:$timeout), 67 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 68 "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 69 "i64.atomic.wait \t${off}${p2align}", 0x02, "false">; 70defm ATOMIC_WAIT_I64_A64 : 71 ATOMIC_I<(outs I32:$dst), 72 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, I64:$exp, 73 I64:$timeout), 74 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 75 "i64.atomic.wait \t$dst, ${off}(${addr})${p2align}, $exp, $timeout", 76 "i64.atomic.wait \t${off}${p2align}", 0x02, "true">; 77} // mayLoad = 1 78} // hasSideEffects = 1 79 80let Predicates = [HasAtomics] in { 81// Select notifys with no constant offset. 82def NotifyPatNoOffset_A32 : 83 Pat<(i32 (int_wasm_atomic_notify I32:$addr, I32:$count)), 84 (ATOMIC_NOTIFY_A32 0, 0, I32:$addr, I32:$count)>, 85 Requires<[HasAddr32]>; 86def NotifyPatNoOffset_A64 : 87 Pat<(i32 (int_wasm_atomic_notify I64:$addr, I32:$count)), 88 (ATOMIC_NOTIFY_A64 0, 0, I64:$addr, I32:$count)>, 89 Requires<[HasAddr64]>; 90 91// Select notifys with a constant offset. 92 93// Pattern with address + immediate offset 94multiclass NotifyPatImmOff<PatFrag operand, string inst> { 95 def : Pat<(i32 (int_wasm_atomic_notify (operand I32:$addr, imm:$off), 96 I32:$count)), 97 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, I32:$count)>, 98 Requires<[HasAddr32]>; 99 def : Pat<(i32 (int_wasm_atomic_notify (operand I64:$addr, imm:$off), 100 I32:$count)), 101 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, I32:$count)>, 102 Requires<[HasAddr64]>; 103} 104defm : NotifyPatImmOff<regPlusImm, "ATOMIC_NOTIFY">; 105defm : NotifyPatImmOff<or_is_add, "ATOMIC_NOTIFY">; 106 107// Select notifys with just a constant offset. 108def NotifyPatOffsetOnly_A32 : 109 Pat<(i32 (int_wasm_atomic_notify imm:$off, I32:$count)), 110 (ATOMIC_NOTIFY_A32 0, imm:$off, (CONST_I32 0), I32:$count)>, 111 Requires<[HasAddr32]>; 112def NotifyPatOffsetOnly_A64 : 113 Pat<(i32 (int_wasm_atomic_notify imm:$off, I32:$count)), 114 (ATOMIC_NOTIFY_A64 0, imm:$off, (CONST_I64 0), I32:$count)>, 115 Requires<[HasAddr64]>; 116 117def NotifyPatGlobalAddrOffOnly_A32 : 118 Pat<(i32 (int_wasm_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 119 I32:$count)), 120 (ATOMIC_NOTIFY_A32 0, tglobaladdr:$off, (CONST_I32 0), I32:$count)>, 121 Requires<[HasAddr32]>; 122def NotifyPatGlobalAddrOffOnly_A64 : 123 Pat<(i32 (int_wasm_atomic_notify (WebAssemblywrapper tglobaladdr:$off), 124 I32:$count)), 125 (ATOMIC_NOTIFY_A64 0, tglobaladdr:$off, (CONST_I64 0), I32:$count)>, 126 Requires<[HasAddr64]>; 127 128// Select waits with no constant offset. 129multiclass WaitPatNoOffset<ValueType ty, Intrinsic kind, 130 string inst> { 131 def : Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)), 132 (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, I64:$timeout)>, 133 Requires<[HasAddr32]>; 134 def : Pat<(i32 (kind I64:$addr, ty:$exp, I64:$timeout)), 135 (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, I64:$timeout)>, 136 Requires<[HasAddr64]>; 137} 138defm : WaitPatNoOffset<i32, int_wasm_atomic_wait_i32, "ATOMIC_WAIT_I32">; 139defm : WaitPatNoOffset<i64, int_wasm_atomic_wait_i64, "ATOMIC_WAIT_I64">; 140defm : WaitPatNoOffset<i32, int_wasm_atomic_wait_i32, "ATOMIC_WAIT_I32">; 141defm : WaitPatNoOffset<i64, int_wasm_atomic_wait_i64, "ATOMIC_WAIT_I64">; 142 143// Select waits with a constant offset. 144 145// Pattern with address + immediate offset 146multiclass WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand, 147 string inst> { 148 def : Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)), 149 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, 150 I64:$timeout)>, 151 Requires<[HasAddr32]>; 152 def : Pat<(i32 (kind (operand I64:$addr, imm:$off), ty:$exp, I64:$timeout)), 153 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, 154 I64:$timeout)>, 155 Requires<[HasAddr64]>; 156} 157defm : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, regPlusImm, 158 "ATOMIC_WAIT_I32">; 159defm : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, or_is_add, 160 "ATOMIC_WAIT_I32">; 161defm : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, regPlusImm, 162 "ATOMIC_WAIT_I64">; 163defm : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, or_is_add, 164 "ATOMIC_WAIT_I64">; 165 166// Select wait_i32, "ATOMIC_WAIT_I32s with just a constant offset. 167multiclass WaitPatOffsetOnly<ValueType ty, Intrinsic kind, string inst> { 168 def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 169 (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp, 170 I64:$timeout)>, 171 Requires<[HasAddr32]>; 172 def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)), 173 (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp, 174 I64:$timeout)>, 175 Requires<[HasAddr64]>; 176} 177defm : WaitPatOffsetOnly<i32, int_wasm_atomic_wait_i32, "ATOMIC_WAIT_I32">; 178defm : WaitPatOffsetOnly<i64, int_wasm_atomic_wait_i64, "ATOMIC_WAIT_I64">; 179 180multiclass WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, string inst> { 181 def : Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, 182 I64:$timeout)), 183 (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, 184 I64:$timeout)>, 185 Requires<[HasAddr32]>; 186 def : Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, 187 I64:$timeout)), 188 (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp, 189 I64:$timeout)>, 190 Requires<[HasAddr64]>; 191} 192defm : WaitPatGlobalAddrOffOnly<i32, int_wasm_atomic_wait_i32, 193 "ATOMIC_WAIT_I32">; 194defm : WaitPatGlobalAddrOffOnly<i64, int_wasm_atomic_wait_i64, 195 "ATOMIC_WAIT_I64">; 196} // Predicates = [HasAtomics] 197 198//===----------------------------------------------------------------------===// 199// Atomic fences 200//===----------------------------------------------------------------------===// 201 202// A compiler fence instruction that prevents reordering of instructions. 203let Defs = [ARGUMENTS] in { 204let isPseudo = 1, hasSideEffects = 1 in 205defm COMPILER_FENCE : ATOMIC_NRI<(outs), (ins), [], "compiler_fence">; 206let hasSideEffects = 1 in 207defm ATOMIC_FENCE : ATOMIC_NRI<(outs), (ins i8imm:$flags), [], "atomic.fence", 208 0x03>; 209} // Defs = [ARGUMENTS] 210 211//===----------------------------------------------------------------------===// 212// Atomic loads 213//===----------------------------------------------------------------------===// 214 215multiclass AtomicLoad<WebAssemblyRegClass rc, string name, int atomic_op> { 216 defm "" : WebAssemblyLoad<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 217 [HasAtomics]>; 218} 219 220defm ATOMIC_LOAD_I32 : AtomicLoad<I32, "i32.atomic.load", 0x10>; 221defm ATOMIC_LOAD_I64 : AtomicLoad<I64, "i64.atomic.load", 0x11>; 222 223// Select loads with no constant offset. 224let Predicates = [HasAtomics] in { 225defm : LoadPatNoOffset<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 226defm : LoadPatNoOffset<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 227 228// Select loads with a constant offset. 229 230// Pattern with address + immediate offset 231defm : LoadPatImmOff<i32, atomic_load_32, regPlusImm, "ATOMIC_LOAD_I32">; 232defm : LoadPatImmOff<i64, atomic_load_64, regPlusImm, "ATOMIC_LOAD_I64">; 233defm : LoadPatImmOff<i32, atomic_load_32, or_is_add, "ATOMIC_LOAD_I32">; 234defm : LoadPatImmOff<i64, atomic_load_64, or_is_add, "ATOMIC_LOAD_I64">; 235 236// Select loads with just a constant offset. 237defm : LoadPatOffsetOnly<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 238defm : LoadPatOffsetOnly<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 239 240defm : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, "ATOMIC_LOAD_I32">; 241defm : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, "ATOMIC_LOAD_I64">; 242 243} // Predicates = [HasAtomics] 244 245// Extending loads. Note that there are only zero-extending atomic loads, no 246// sign-extending loads. 247defm ATOMIC_LOAD8_U_I32 : AtomicLoad<I32, "i32.atomic.load8_u", 0x12>; 248defm ATOMIC_LOAD16_U_I32 : AtomicLoad<I32, "i32.atomic.load16_u", 0x13>; 249defm ATOMIC_LOAD8_U_I64 : AtomicLoad<I64, "i64.atomic.load8_u", 0x14>; 250defm ATOMIC_LOAD16_U_I64 : AtomicLoad<I64, "i64.atomic.load16_u", 0x15>; 251defm ATOMIC_LOAD32_U_I64 : AtomicLoad<I64, "i64.atomic.load32_u", 0x16>; 252 253// Fragments for extending loads. These are different from regular loads because 254// the SDNodes are derived from AtomicSDNode rather than LoadSDNode and 255// therefore don't have the extension type field. So instead of matching that, 256// we match the patterns that the type legalizer expands them to. 257 258// We directly match zext patterns and select the zext atomic loads. 259// i32 (zext (i8 (atomic_load_8))) gets legalized to 260// i32 (and (i32 (atomic_load_8)), 255) 261// These can be selected to a single zero-extending atomic load instruction. 262def zext_aload_8_32 : 263 PatFrag<(ops node:$addr), (and (i32 (atomic_load_8 node:$addr)), 255)>; 264def zext_aload_16_32 : 265 PatFrag<(ops node:$addr), (and (i32 (atomic_load_16 node:$addr)), 65535)>; 266// Unlike regular loads, extension to i64 is handled differently than i32. 267// i64 (zext (i8 (atomic_load_8))) gets legalized to 268// i64 (and (i64 (anyext (i32 (atomic_load_8)))), 255) 269def zext_aload_8_64 : 270 PatFrag<(ops node:$addr), 271 (and (i64 (anyext (i32 (atomic_load_8 node:$addr)))), 255)>; 272def zext_aload_16_64 : 273 PatFrag<(ops node:$addr), 274 (and (i64 (anyext (i32 (atomic_load_16 node:$addr)))), 65535)>; 275def zext_aload_32_64 : 276 PatFrag<(ops node:$addr), 277 (zext (i32 (atomic_load node:$addr)))>; 278 279// We don't have single sext atomic load instructions. So for sext loads, we 280// match bare subword loads (for 32-bit results) and anyext loads (for 64-bit 281// results) and select a zext load; the next instruction will be sext_inreg 282// which is selected by itself. 283def sext_aload_8_64 : 284 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_8 node:$addr)))>; 285def sext_aload_16_64 : 286 PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>; 287 288let Predicates = [HasAtomics] in { 289// Select zero-extending loads with no constant offset. 290defm : LoadPatNoOffset<i32, zext_aload_8_32, "ATOMIC_LOAD8_U_I32">; 291defm : LoadPatNoOffset<i32, zext_aload_16_32, "ATOMIC_LOAD16_U_I32">; 292defm : LoadPatNoOffset<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 293defm : LoadPatNoOffset<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 294defm : LoadPatNoOffset<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 295 296// Select sign-extending loads with no constant offset 297defm : LoadPatNoOffset<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 298defm : LoadPatNoOffset<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 299defm : LoadPatNoOffset<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 300defm : LoadPatNoOffset<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 301// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s 302 303// Zero-extending loads with constant offset 304defm : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, "ATOMIC_LOAD8_U_I32">; 305defm : LoadPatImmOff<i32, zext_aload_16_32, regPlusImm, "ATOMIC_LOAD16_U_I32">; 306defm : LoadPatImmOff<i32, zext_aload_8_32, or_is_add, "ATOMIC_LOAD8_U_I32">; 307defm : LoadPatImmOff<i32, zext_aload_16_32, or_is_add, "ATOMIC_LOAD16_U_I32">; 308defm : LoadPatImmOff<i64, zext_aload_8_64, regPlusImm, "ATOMIC_LOAD8_U_I64">; 309defm : LoadPatImmOff<i64, zext_aload_16_64, regPlusImm, "ATOMIC_LOAD16_U_I64">; 310defm : LoadPatImmOff<i64, zext_aload_32_64, regPlusImm, "ATOMIC_LOAD32_U_I64">; 311defm : LoadPatImmOff<i64, zext_aload_8_64, or_is_add, "ATOMIC_LOAD8_U_I64">; 312defm : LoadPatImmOff<i64, zext_aload_16_64, or_is_add, "ATOMIC_LOAD16_U_I64">; 313defm : LoadPatImmOff<i64, zext_aload_32_64, or_is_add, "ATOMIC_LOAD32_U_I64">; 314 315// Sign-extending loads with constant offset 316defm : LoadPatImmOff<i32, atomic_load_8, regPlusImm, "ATOMIC_LOAD8_U_I32">; 317defm : LoadPatImmOff<i32, atomic_load_16, regPlusImm, "ATOMIC_LOAD16_U_I32">; 318defm : LoadPatImmOff<i32, atomic_load_8, or_is_add, "ATOMIC_LOAD8_U_I32">; 319defm : LoadPatImmOff<i32, atomic_load_16, or_is_add, "ATOMIC_LOAD16_U_I32">; 320defm : LoadPatImmOff<i64, sext_aload_8_64, regPlusImm, "ATOMIC_LOAD8_U_I64">; 321defm : LoadPatImmOff<i64, sext_aload_16_64, regPlusImm, "ATOMIC_LOAD16_U_I64">; 322defm : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, "ATOMIC_LOAD8_U_I64">; 323defm : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, "ATOMIC_LOAD16_U_I64">; 324// No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64 325 326// Extending loads with just a constant offset 327defm : LoadPatOffsetOnly<i32, zext_aload_8_32, "ATOMIC_LOAD8_U_I32">; 328defm : LoadPatOffsetOnly<i32, zext_aload_16_32, "ATOMIC_LOAD16_U_I32">; 329defm : LoadPatOffsetOnly<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 330defm : LoadPatOffsetOnly<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 331defm : LoadPatOffsetOnly<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 332defm : LoadPatOffsetOnly<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 333defm : LoadPatOffsetOnly<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 334defm : LoadPatOffsetOnly<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 335defm : LoadPatOffsetOnly<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 336 337defm : LoadPatGlobalAddrOffOnly<i32, zext_aload_8_32, "ATOMIC_LOAD8_U_I32">; 338defm : LoadPatGlobalAddrOffOnly<i32, zext_aload_16_32, "ATOMIC_LOAD16_U_I32">; 339defm : LoadPatGlobalAddrOffOnly<i64, zext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 340defm : LoadPatGlobalAddrOffOnly<i64, zext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 341defm : LoadPatGlobalAddrOffOnly<i64, zext_aload_32_64, "ATOMIC_LOAD32_U_I64">; 342defm : LoadPatGlobalAddrOffOnly<i32, atomic_load_8, "ATOMIC_LOAD8_U_I32">; 343defm : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">; 344defm : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">; 345defm : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">; 346 347} // Predicates = [HasAtomics] 348 349//===----------------------------------------------------------------------===// 350// Atomic stores 351//===----------------------------------------------------------------------===// 352 353multiclass AtomicStore<WebAssemblyRegClass rc, string name, int atomic_op> { 354 defm "" : WebAssemblyStore<rc, name, !or(0xfe00, !and(0xff, atomic_op)), 355 [HasAtomics]>; 356} 357 358defm ATOMIC_STORE_I32 : AtomicStore<I32, "i32.atomic.store", 0x17>; 359defm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>; 360 361// We need an 'atomic' version of store patterns because store and atomic_store 362// nodes have different operand orders: 363// store: (store $val, $ptr) 364// atomic_store: (store $ptr, $val) 365 366let Predicates = [HasAtomics] in { 367 368// Select stores with no constant offset. 369multiclass AStorePatNoOffset<ValueType ty, PatFrag kind, string inst> { 370 def : Pat<(kind I32:$addr, ty:$val), 371 (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>, 372 Requires<[HasAddr32]>; 373 def : Pat<(kind I64:$addr, ty:$val), 374 (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>, 375 Requires<[HasAddr64]>; 376} 377defm : AStorePatNoOffset<i32, atomic_store_32, "ATOMIC_STORE_I32">; 378defm : AStorePatNoOffset<i64, atomic_store_64, "ATOMIC_STORE_I64">; 379 380// Select stores with a constant offset. 381 382// Pattern with address + immediate offset 383multiclass AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 384 string inst> { 385 def : Pat<(kind (operand I32:$addr, imm:$off), ty:$val), 386 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>, 387 Requires<[HasAddr32]>; 388 def : Pat<(kind (operand I64:$addr, imm:$off), ty:$val), 389 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>, 390 Requires<[HasAddr64]>; 391} 392defm : AStorePatImmOff<i32, atomic_store_32, regPlusImm, "ATOMIC_STORE_I32">; 393defm : AStorePatImmOff<i64, atomic_store_64, regPlusImm, "ATOMIC_STORE_I64">; 394 395// Select stores with just a constant offset. 396multiclass AStorePatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 397 def : Pat<(kind imm:$off, ty:$val), 398 (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>, 399 Requires<[HasAddr32]>; 400 def : Pat<(kind imm:$off, ty:$val), 401 (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>, 402 Requires<[HasAddr64]>; 403} 404defm : AStorePatOffsetOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">; 405defm : AStorePatOffsetOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">; 406 407multiclass AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> { 408 def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 409 (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>, 410 Requires<[HasAddr32]>; 411 def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val), 412 (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>, 413 Requires<[HasAddr64]>; 414} 415defm : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">; 416defm : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">; 417 418} // Predicates = [HasAtomics] 419 420// Truncating stores. 421defm ATOMIC_STORE8_I32 : AtomicStore<I32, "i32.atomic.store8", 0x19>; 422defm ATOMIC_STORE16_I32 : AtomicStore<I32, "i32.atomic.store16", 0x1a>; 423defm ATOMIC_STORE8_I64 : AtomicStore<I64, "i64.atomic.store8", 0x1b>; 424defm ATOMIC_STORE16_I64 : AtomicStore<I64, "i64.atomic.store16", 0x1c>; 425defm ATOMIC_STORE32_I64 : AtomicStore<I64, "i64.atomic.store32", 0x1d>; 426 427// Fragments for truncating stores. 428 429// We don't have single truncating atomic store instructions. For 32-bit 430// instructions, we just need to match bare atomic stores. On the other hand, 431// truncating stores from i64 values are once truncated to i32 first. 432class trunc_astore_64<PatFrag kind> : 433 PatFrag<(ops node:$addr, node:$val), 434 (kind node:$addr, (i32 (trunc (i64 node:$val))))>; 435def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>; 436def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>; 437def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>; 438 439let Predicates = [HasAtomics] in { 440 441// Truncating stores with no constant offset 442defm : AStorePatNoOffset<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 443defm : AStorePatNoOffset<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 444defm : AStorePatNoOffset<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 445defm : AStorePatNoOffset<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 446defm : AStorePatNoOffset<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 447 448// Truncating stores with a constant offset 449defm : AStorePatImmOff<i32, atomic_store_8, regPlusImm, "ATOMIC_STORE8_I32">; 450defm : AStorePatImmOff<i32, atomic_store_16, regPlusImm, "ATOMIC_STORE16_I32">; 451defm : AStorePatImmOff<i64, trunc_astore_8_64, regPlusImm, "ATOMIC_STORE8_I64">; 452defm : AStorePatImmOff<i64, trunc_astore_16_64, regPlusImm, 453 "ATOMIC_STORE16_I64">; 454defm : AStorePatImmOff<i64, trunc_astore_32_64, regPlusImm, 455 "ATOMIC_STORE32_I64">; 456defm : AStorePatImmOff<i32, atomic_store_8, or_is_add, "ATOMIC_STORE8_I32">; 457defm : AStorePatImmOff<i32, atomic_store_16, or_is_add, "ATOMIC_STORE16_I32">; 458defm : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, "ATOMIC_STORE8_I64">; 459defm : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, 460 "ATOMIC_STORE16_I64">; 461defm : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, 462 "ATOMIC_STORE32_I64">; 463 464// Truncating stores with just a constant offset 465defm : AStorePatOffsetOnly<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 466defm : AStorePatOffsetOnly<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 467defm : AStorePatOffsetOnly<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 468defm : AStorePatOffsetOnly<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 469defm : AStorePatOffsetOnly<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 470 471defm : AStorePatGlobalAddrOffOnly<i32, atomic_store_8, "ATOMIC_STORE8_I32">; 472defm : AStorePatGlobalAddrOffOnly<i32, atomic_store_16, "ATOMIC_STORE16_I32">; 473defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">; 474defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">; 475defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">; 476 477} // Predicates = [HasAtomics] 478 479//===----------------------------------------------------------------------===// 480// Atomic binary read-modify-writes 481//===----------------------------------------------------------------------===// 482 483multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string name, 484 int atomic_op> { 485 defm "_A32" : 486 ATOMIC_I<(outs rc:$dst), 487 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$val), 488 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 489 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 490 !strconcat(name, "\t${off}${p2align}"), atomic_op, "false">; 491 defm "_A64" : 492 ATOMIC_I<(outs rc:$dst), 493 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$val), 494 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 495 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $val"), 496 !strconcat(name, "\t${off}${p2align}"), atomic_op, "true">; 497} 498 499defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0x1e>; 500defm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0x1f>; 501defm ATOMIC_RMW8_U_ADD_I32 : 502 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0x20>; 503defm ATOMIC_RMW16_U_ADD_I32 : 504 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0x21>; 505defm ATOMIC_RMW8_U_ADD_I64 : 506 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0x22>; 507defm ATOMIC_RMW16_U_ADD_I64 : 508 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0x23>; 509defm ATOMIC_RMW32_U_ADD_I64 : 510 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0x24>; 511 512defm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0x25>; 513defm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0x26>; 514defm ATOMIC_RMW8_U_SUB_I32 : 515 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0x27>; 516defm ATOMIC_RMW16_U_SUB_I32 : 517 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0x28>; 518defm ATOMIC_RMW8_U_SUB_I64 : 519 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0x29>; 520defm ATOMIC_RMW16_U_SUB_I64 : 521 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0x2a>; 522defm ATOMIC_RMW32_U_SUB_I64 : 523 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0x2b>; 524 525defm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0x2c>; 526defm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0x2d>; 527defm ATOMIC_RMW8_U_AND_I32 : 528 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0x2e>; 529defm ATOMIC_RMW16_U_AND_I32 : 530 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0x2f>; 531defm ATOMIC_RMW8_U_AND_I64 : 532 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0x30>; 533defm ATOMIC_RMW16_U_AND_I64 : 534 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0x31>; 535defm ATOMIC_RMW32_U_AND_I64 : 536 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0x32>; 537 538defm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0x33>; 539defm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0x34>; 540defm ATOMIC_RMW8_U_OR_I32 : 541 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0x35>; 542defm ATOMIC_RMW16_U_OR_I32 : 543 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0x36>; 544defm ATOMIC_RMW8_U_OR_I64 : 545 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0x37>; 546defm ATOMIC_RMW16_U_OR_I64 : 547 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0x38>; 548defm ATOMIC_RMW32_U_OR_I64 : 549 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0x39>; 550 551defm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0x3a>; 552defm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0x3b>; 553defm ATOMIC_RMW8_U_XOR_I32 : 554 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0x3c>; 555defm ATOMIC_RMW16_U_XOR_I32 : 556 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0x3d>; 557defm ATOMIC_RMW8_U_XOR_I64 : 558 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0x3e>; 559defm ATOMIC_RMW16_U_XOR_I64 : 560 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0x3f>; 561defm ATOMIC_RMW32_U_XOR_I64 : 562 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0x40>; 563 564defm ATOMIC_RMW_XCHG_I32 : 565 WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0x41>; 566defm ATOMIC_RMW_XCHG_I64 : 567 WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0x42>; 568defm ATOMIC_RMW8_U_XCHG_I32 : 569 WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0x43>; 570defm ATOMIC_RMW16_U_XCHG_I32 : 571 WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0x44>; 572defm ATOMIC_RMW8_U_XCHG_I64 : 573 WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0x45>; 574defm ATOMIC_RMW16_U_XCHG_I64 : 575 WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0x46>; 576defm ATOMIC_RMW32_U_XCHG_I64 : 577 WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0x47>; 578 579// Select binary RMWs with no constant offset. 580multiclass BinRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> { 581 def : Pat<(ty (kind I32:$addr, ty:$val)), 582 (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>, 583 Requires<[HasAddr32]>; 584 def : Pat<(ty (kind I64:$addr, ty:$val)), 585 (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>, 586 Requires<[HasAddr64]>; 587} 588 589// Select binary RMWs with a constant offset. 590 591// Pattern with address + immediate offset 592multiclass BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 593 string inst> { 594 def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)), 595 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>, 596 Requires<[HasAddr32]>; 597 def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$val)), 598 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>, 599 Requires<[HasAddr64]>; 600} 601 602// Select binary RMWs with just a constant offset. 603multiclass BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 604 def : Pat<(ty (kind imm:$off, ty:$val)), 605 (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>, 606 Requires<[HasAddr32]>; 607 def : Pat<(ty (kind imm:$off, ty:$val)), 608 (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>, 609 Requires<[HasAddr64]>; 610} 611 612multiclass BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> { 613 def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 614 (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>, 615 Requires<[HasAddr32]>; 616 def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)), 617 (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>, 618 Requires<[HasAddr64]>; 619} 620 621// Patterns for various addressing modes. 622multiclass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32, 623 string inst_64> { 624 defm : BinRMWPatNoOffset<i32, rmw_32, inst_32>; 625 defm : BinRMWPatNoOffset<i64, rmw_64, inst_64>; 626 627 defm : BinRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 628 defm : BinRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 629 defm : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 630 defm : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 631 632 defm : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>; 633 defm : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>; 634 635 defm : BinRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 636 defm : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 637} 638 639let Predicates = [HasAtomics] in { 640defm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64, 641 "ATOMIC_RMW_ADD_I32", "ATOMIC_RMW_ADD_I64">; 642defm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64, 643 "ATOMIC_RMW_SUB_I32", "ATOMIC_RMW_SUB_I64">; 644defm : BinRMWPattern<atomic_load_and_32, atomic_load_and_64, 645 "ATOMIC_RMW_AND_I32", "ATOMIC_RMW_AND_I64">; 646defm : BinRMWPattern<atomic_load_or_32, atomic_load_or_64, 647 "ATOMIC_RMW_OR_I32", "ATOMIC_RMW_OR_I64">; 648defm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64, 649 "ATOMIC_RMW_XOR_I32", "ATOMIC_RMW_XOR_I64">; 650defm : BinRMWPattern<atomic_swap_32, atomic_swap_64, 651 "ATOMIC_RMW_XCHG_I32", "ATOMIC_RMW_XCHG_I64">; 652} // Predicates = [HasAtomics] 653 654// Truncating & zero-extending binary RMW patterns. 655// These are combined patterns of truncating store patterns and zero-extending 656// load patterns above. 657class zext_bin_rmw_8_32<PatFrag kind> : 658 PatFrag<(ops node:$addr, node:$val), 659 (and (i32 (kind node:$addr, node:$val)), 255)>; 660class zext_bin_rmw_16_32<PatFrag kind> : 661 PatFrag<(ops node:$addr, node:$val), 662 (and (i32 (kind node:$addr, node:$val)), 65535)>; 663class zext_bin_rmw_8_64<PatFrag kind> : 664 PatFrag<(ops node:$addr, node:$val), 665 (and (i64 (anyext (i32 (kind node:$addr, 666 (i32 (trunc (i64 node:$val))))))), 255)>; 667class zext_bin_rmw_16_64<PatFrag kind> : 668 PatFrag<(ops node:$addr, node:$val), 669 (and (i64 (anyext (i32 (kind node:$addr, 670 (i32 (trunc (i64 node:$val))))))), 65535)>; 671class zext_bin_rmw_32_64<PatFrag kind> : 672 PatFrag<(ops node:$addr, node:$val), 673 (zext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 674 675// Truncating & sign-extending binary RMW patterns. 676// These are combined patterns of truncating store patterns and sign-extending 677// load patterns above. We match subword RMWs (for 32-bit) and anyext RMWs (for 678// 64-bit) and select a zext RMW; the next instruction will be sext_inreg which 679// is selected by itself. 680class sext_bin_rmw_8_32<PatFrag kind> : 681 PatFrag<(ops node:$addr, node:$val), (kind node:$addr, node:$val)>; 682class sext_bin_rmw_16_32<PatFrag kind> : sext_bin_rmw_8_32<kind>; 683class sext_bin_rmw_8_64<PatFrag kind> : 684 PatFrag<(ops node:$addr, node:$val), 685 (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>; 686class sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>; 687// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 688 689// Patterns for various addressing modes for truncating-extending binary RMWs. 690multiclass BinRMWTruncExtPattern< 691 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 692 NI inst8_32, NI inst16_32, NI inst8_64, NI inst16_64, NI inst32_64> { 693 // Truncating-extending binary RMWs with no constant offset 694 defm : BinRMWPatNoOffset<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 695 defm : BinRMWPatNoOffset<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 696 defm : BinRMWPatNoOffset<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 697 defm : BinRMWPatNoOffset<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 698 defm : BinRMWPatNoOffset<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 699 700 defm : BinRMWPatNoOffset<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 701 defm : BinRMWPatNoOffset<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 702 defm : BinRMWPatNoOffset<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 703 defm : BinRMWPatNoOffset<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 704 705 // Truncating-extending binary RMWs with a constant offset 706 defm : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 707 defm : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, regPlusImm, 708 inst16_32>; 709 defm : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 710 defm : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, regPlusImm, 711 inst16_64>; 712 defm : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, regPlusImm, 713 inst32_64>; 714 defm : BinRMWPatImmOff<i32, zext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 715 defm : BinRMWPatImmOff<i32, zext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 716 defm : BinRMWPatImmOff<i64, zext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 717 defm : BinRMWPatImmOff<i64, zext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 718 defm : BinRMWPatImmOff<i64, zext_bin_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 719 720 defm : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 721 defm : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, regPlusImm, 722 inst16_32>; 723 defm : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 724 defm : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, regPlusImm, 725 inst16_64>; 726 defm : BinRMWPatImmOff<i32, sext_bin_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 727 defm : BinRMWPatImmOff<i32, sext_bin_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 728 defm : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 729 defm : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 730 731 // Truncating-extending binary RMWs with just a constant offset 732 defm : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 733 defm : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 734 defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 735 defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 736 defm : BinRMWPatOffsetOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 737 738 defm : BinRMWPatOffsetOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 739 defm : BinRMWPatOffsetOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 740 defm : BinRMWPatOffsetOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 741 defm : BinRMWPatOffsetOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 742 743 defm : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>; 744 defm : BinRMWPatGlobalAddrOffOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>; 745 defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>; 746 defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>; 747 defm : BinRMWPatGlobalAddrOffOnly<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>; 748 749 defm : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>; 750 defm : BinRMWPatGlobalAddrOffOnly<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>; 751 defm : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>; 752 defm : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>; 753} 754 755let Predicates = [HasAtomics] in { 756defm : BinRMWTruncExtPattern< 757 atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64, 758 "ATOMIC_RMW8_U_ADD_I32", "ATOMIC_RMW16_U_ADD_I32", 759 "ATOMIC_RMW8_U_ADD_I64", "ATOMIC_RMW16_U_ADD_I64", "ATOMIC_RMW32_U_ADD_I64">; 760defm : BinRMWTruncExtPattern< 761 atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, atomic_load_sub_64, 762 "ATOMIC_RMW8_U_SUB_I32", "ATOMIC_RMW16_U_SUB_I32", 763 "ATOMIC_RMW8_U_SUB_I64", "ATOMIC_RMW16_U_SUB_I64", "ATOMIC_RMW32_U_SUB_I64">; 764defm : BinRMWTruncExtPattern< 765 atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, atomic_load_and_64, 766 "ATOMIC_RMW8_U_AND_I32", "ATOMIC_RMW16_U_AND_I32", 767 "ATOMIC_RMW8_U_AND_I64", "ATOMIC_RMW16_U_AND_I64", "ATOMIC_RMW32_U_AND_I64">; 768defm : BinRMWTruncExtPattern< 769 atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, atomic_load_or_64, 770 "ATOMIC_RMW8_U_OR_I32", "ATOMIC_RMW16_U_OR_I32", 771 "ATOMIC_RMW8_U_OR_I64", "ATOMIC_RMW16_U_OR_I64", "ATOMIC_RMW32_U_OR_I64">; 772defm : BinRMWTruncExtPattern< 773 atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, atomic_load_xor_64, 774 "ATOMIC_RMW8_U_XOR_I32", "ATOMIC_RMW16_U_XOR_I32", 775 "ATOMIC_RMW8_U_XOR_I64", "ATOMIC_RMW16_U_XOR_I64", "ATOMIC_RMW32_U_XOR_I64">; 776defm : BinRMWTruncExtPattern< 777 atomic_swap_8, atomic_swap_16, atomic_swap_32, atomic_swap_64, 778 "ATOMIC_RMW8_U_XCHG_I32", "ATOMIC_RMW16_U_XCHG_I32", 779 "ATOMIC_RMW8_U_XCHG_I64", "ATOMIC_RMW16_U_XCHG_I64", 780 "ATOMIC_RMW32_U_XCHG_I64">; 781} // Predicates = [HasAtomics] 782 783//===----------------------------------------------------------------------===// 784// Atomic ternary read-modify-writes 785//===----------------------------------------------------------------------===// 786 787// TODO LLVM IR's cmpxchg instruction returns a pair of {loaded value, success 788// flag}. When we use the success flag or both values, we can't make use of i64 789// truncate/extend versions of instructions for now, which is suboptimal. 790// Consider adding a pass after instruction selection that optimizes this case 791// if it is frequent. 792 793multiclass WebAssemblyTerRMW<WebAssemblyRegClass rc, string name, 794 int atomic_op> { 795 defm "_A32" : 796 ATOMIC_I<(outs rc:$dst), 797 (ins P2Align:$p2align, offset32_op:$off, I32:$addr, rc:$exp, 798 rc:$new_), 799 (outs), (ins P2Align:$p2align, offset32_op:$off), [], 800 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 801 !strconcat(name, "\t${off}${p2align}"), atomic_op, "false">; 802 defm "_A64" : 803 ATOMIC_I<(outs rc:$dst), 804 (ins P2Align:$p2align, offset64_op:$off, I64:$addr, rc:$exp, 805 rc:$new_), 806 (outs), (ins P2Align:$p2align, offset64_op:$off), [], 807 !strconcat(name, "\t$dst, ${off}(${addr})${p2align}, $exp, $new_"), 808 !strconcat(name, "\t${off}${p2align}"), atomic_op, "true">; 809} 810 811defm ATOMIC_RMW_CMPXCHG_I32 : 812 WebAssemblyTerRMW<I32, "i32.atomic.rmw.cmpxchg", 0x48>; 813defm ATOMIC_RMW_CMPXCHG_I64 : 814 WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0x49>; 815defm ATOMIC_RMW8_U_CMPXCHG_I32 : 816 WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0x4a>; 817defm ATOMIC_RMW16_U_CMPXCHG_I32 : 818 WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0x4b>; 819defm ATOMIC_RMW8_U_CMPXCHG_I64 : 820 WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0x4c>; 821defm ATOMIC_RMW16_U_CMPXCHG_I64 : 822 WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0x4d>; 823defm ATOMIC_RMW32_U_CMPXCHG_I64 : 824 WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0x4e>; 825 826// Select ternary RMWs with no constant offset. 827multiclass TerRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> { 828 def : Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)), 829 (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, ty:$new)>, 830 Requires<[HasAddr32]>; 831 def : Pat<(ty (kind I64:$addr, ty:$exp, ty:$new)), 832 (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, ty:$new)>, 833 Requires<[HasAddr64]>; 834} 835 836// Select ternary RMWs with a constant offset. 837 838// Pattern with address + immediate offset 839multiclass TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, 840 string inst> { 841 def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)), 842 (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>, 843 Requires<[HasAddr32]>; 844 def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$exp, ty:$new)), 845 (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, ty:$new)>, 846 Requires<[HasAddr64]>; 847} 848 849// Select ternary RMWs with just a constant offset. 850multiclass TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, string inst> { 851 def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 852 (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp, 853 ty:$new)>; 854 def : Pat<(ty (kind imm:$off, ty:$exp, ty:$new)), 855 (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp, 856 ty:$new)>; 857} 858 859multiclass TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> { 860 def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 861 (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp, 862 ty:$new)>, 863 Requires<[HasAddr32]>; 864 def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)), 865 (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp, 866 ty:$new)>, 867 Requires<[HasAddr64]>; 868} 869 870// Patterns for various addressing modes. 871multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32, 872 string inst_64> { 873 defm : TerRMWPatNoOffset<i32, rmw_32, inst_32>; 874 defm : TerRMWPatNoOffset<i64, rmw_64, inst_64>; 875 876 defm : TerRMWPatImmOff<i32, rmw_32, regPlusImm, inst_32>; 877 defm : TerRMWPatImmOff<i64, rmw_64, regPlusImm, inst_64>; 878 defm : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>; 879 defm : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>; 880 881 defm : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>; 882 defm : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>; 883 884 defm : TerRMWPatGlobalAddrOffOnly<i32, rmw_32, inst_32>; 885 defm : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>; 886} 887 888let Predicates = [HasAtomics] in 889defm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64, 890 "ATOMIC_RMW_CMPXCHG_I32", "ATOMIC_RMW_CMPXCHG_I64">; 891 892// Truncating & zero-extending ternary RMW patterns. 893// DAG legalization & optimization before instruction selection may introduce 894// additional nodes such as anyext or assertzext depending on operand types. 895class zext_ter_rmw_8_32<PatFrag kind> : 896 PatFrag<(ops node:$addr, node:$exp, node:$new), 897 (and (i32 (kind node:$addr, node:$exp, node:$new)), 255)>; 898class zext_ter_rmw_16_32<PatFrag kind> : 899 PatFrag<(ops node:$addr, node:$exp, node:$new), 900 (and (i32 (kind node:$addr, node:$exp, node:$new)), 65535)>; 901class zext_ter_rmw_8_64<PatFrag kind> : 902 PatFrag<(ops node:$addr, node:$exp, node:$new), 903 (zext (i32 (assertzext (i32 (kind node:$addr, 904 (i32 (trunc (i64 node:$exp))), 905 (i32 (trunc (i64 node:$new))))))))>; 906class zext_ter_rmw_16_64<PatFrag kind> : zext_ter_rmw_8_64<kind>; 907class zext_ter_rmw_32_64<PatFrag kind> : 908 PatFrag<(ops node:$addr, node:$exp, node:$new), 909 (zext (i32 (kind node:$addr, 910 (i32 (trunc (i64 node:$exp))), 911 (i32 (trunc (i64 node:$new))))))>; 912 913// Truncating & sign-extending ternary RMW patterns. 914// We match subword RMWs (for 32-bit) and anyext RMWs (for 64-bit) and select a 915// zext RMW; the next instruction will be sext_inreg which is selected by 916// itself. 917class sext_ter_rmw_8_32<PatFrag kind> : 918 PatFrag<(ops node:$addr, node:$exp, node:$new), 919 (kind node:$addr, node:$exp, node:$new)>; 920class sext_ter_rmw_16_32<PatFrag kind> : sext_ter_rmw_8_32<kind>; 921class sext_ter_rmw_8_64<PatFrag kind> : 922 PatFrag<(ops node:$addr, node:$exp, node:$new), 923 (anyext (i32 (assertzext (i32 924 (kind node:$addr, 925 (i32 (trunc (i64 node:$exp))), 926 (i32 (trunc (i64 node:$new))))))))>; 927class sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>; 928// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s 929 930// Patterns for various addressing modes for truncating-extending ternary RMWs. 931multiclass TerRMWTruncExtPattern< 932 PatFrag rmw_8, PatFrag rmw_16, PatFrag rmw_32, PatFrag rmw_64, 933 string inst8_32, string inst16_32, string inst8_64, string inst16_64, 934 string inst32_64> { 935 // Truncating-extending ternary RMWs with no constant offset 936 defm : TerRMWPatNoOffset<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 937 defm : TerRMWPatNoOffset<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 938 defm : TerRMWPatNoOffset<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 939 defm : TerRMWPatNoOffset<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 940 defm : TerRMWPatNoOffset<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 941 942 defm : TerRMWPatNoOffset<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 943 defm : TerRMWPatNoOffset<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 944 defm : TerRMWPatNoOffset<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 945 defm : TerRMWPatNoOffset<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 946 947 // Truncating-extending ternary RMWs with a constant offset 948 defm : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 949 defm : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, regPlusImm, 950 inst16_32>; 951 defm : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 952 defm : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, regPlusImm, 953 inst16_64>; 954 defm : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, regPlusImm, 955 inst32_64>; 956 defm : TerRMWPatImmOff<i32, zext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 957 defm : TerRMWPatImmOff<i32, zext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 958 defm : TerRMWPatImmOff<i64, zext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 959 defm : TerRMWPatImmOff<i64, zext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 960 defm : TerRMWPatImmOff<i64, zext_ter_rmw_32_64<rmw_32>, or_is_add, inst32_64>; 961 962 defm : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, regPlusImm, inst8_32>; 963 defm : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, regPlusImm, 964 inst16_32>; 965 defm : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, regPlusImm, inst8_64>; 966 defm : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, regPlusImm, 967 inst16_64>; 968 defm : TerRMWPatImmOff<i32, sext_ter_rmw_8_32<rmw_8>, or_is_add, inst8_32>; 969 defm : TerRMWPatImmOff<i32, sext_ter_rmw_16_32<rmw_16>, or_is_add, inst16_32>; 970 defm : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>; 971 defm : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>; 972 973 // Truncating-extending ternary RMWs with just a constant offset 974 defm : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 975 defm : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 976 defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 977 defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 978 defm : TerRMWPatOffsetOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 979 980 defm : TerRMWPatOffsetOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 981 defm : TerRMWPatOffsetOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 982 defm : TerRMWPatOffsetOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 983 defm : TerRMWPatOffsetOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 984 985 defm : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>; 986 defm : TerRMWPatGlobalAddrOffOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>; 987 defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>; 988 defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>; 989 defm : TerRMWPatGlobalAddrOffOnly<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>; 990 991 defm : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>; 992 defm : TerRMWPatGlobalAddrOffOnly<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>; 993 defm : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>; 994 defm : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>; 995} 996 997let Predicates = [HasAtomics] in 998defm : TerRMWTruncExtPattern< 999 atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64, 1000 "ATOMIC_RMW8_U_CMPXCHG_I32", "ATOMIC_RMW16_U_CMPXCHG_I32", 1001 "ATOMIC_RMW8_U_CMPXCHG_I64", "ATOMIC_RMW16_U_CMPXCHG_I64", 1002 "ATOMIC_RMW32_U_CMPXCHG_I64">; 1003