1285163Sdim// WebAssemblyInstrSIMD.td - WebAssembly SIMD codegen support -*- tablegen -*-// 2285163Sdim// 3353358Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4353358Sdim// See https://llvm.org/LICENSE.txt for license information. 5353358Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6285163Sdim// 7285163Sdim//===----------------------------------------------------------------------===// 8286684Sdim/// 9286684Sdim/// \file 10341825Sdim/// WebAssembly SIMD operand code-gen constructs. 11286684Sdim/// 12285163Sdim//===----------------------------------------------------------------------===// 13285163Sdim 14344779Sdim// Instructions requiring HasSIMD128 and the simd128 prefix byte 15344779Sdimmulticlass SIMD_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s, 16344779Sdim list<dag> pattern_r, string asmstr_r = "", 17344779Sdim string asmstr_s = "", bits<32> simdop = -1> { 18344779Sdim defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s, 19344779Sdim !or(0xfd00, !and(0xff, simdop))>, 20344779Sdim Requires<[HasSIMD128]>; 21344779Sdim} 22344779Sdim 23344779Sdimdefm "" : ARGUMENT<V128, v16i8>; 24344779Sdimdefm "" : ARGUMENT<V128, v8i16>; 25344779Sdimdefm "" : ARGUMENT<V128, v4i32>; 26344779Sdimdefm "" : ARGUMENT<V128, v2i64>; 27344779Sdimdefm "" : ARGUMENT<V128, v4f32>; 28344779Sdimdefm "" : ARGUMENT<V128, v2f64>; 29344779Sdim 30344779Sdim// Constrained immediate argument types 31344779Sdimforeach SIZE = [8, 16] in 32344779Sdimdef ImmI#SIZE : ImmLeaf<i32, 33353358Sdim "return -(1 << ("#SIZE#" - 1)) <= Imm && Imm < (1 << ("#SIZE#" - 1));" 34344779Sdim>; 35344779Sdimforeach SIZE = [2, 4, 8, 16, 32] in 36344779Sdimdef LaneIdx#SIZE : ImmLeaf<i32, "return 0 <= Imm && Imm < "#SIZE#";">; 37344779Sdim 38344779Sdim//===----------------------------------------------------------------------===// 39344779Sdim// Load and store 40344779Sdim//===----------------------------------------------------------------------===// 41344779Sdim 42344779Sdim// Load: v128.load 43360784Sdimlet mayLoad = 1, UseNamedOperandTable = 1 in 44360784Sdimdefm LOAD_V128 : 45360784Sdim SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr), 46360784Sdim (outs), (ins P2Align:$p2align, offset32_op:$off), [], 47360784Sdim "v128.load\t$dst, ${off}(${addr})$p2align", 48360784Sdim "v128.load\t$off$p2align", 0>; 49360784Sdim 50360784Sdim// Def load and store patterns from WebAssemblyInstrMemory.td for vector types 51360784Sdimforeach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in { 52360784Sdimdef : LoadPatNoOffset<vec_t, load, LOAD_V128>; 53360784Sdimdef : LoadPatImmOff<vec_t, load, regPlusImm, LOAD_V128>; 54360784Sdimdef : LoadPatImmOff<vec_t, load, or_is_add, LOAD_V128>; 55360784Sdimdef : LoadPatOffsetOnly<vec_t, load, LOAD_V128>; 56360784Sdimdef : LoadPatGlobalAddrOffOnly<vec_t, load, LOAD_V128>; 57360784Sdim} 58360784Sdim 59360784Sdim// vNxM.load_splat 60360784Sdimmulticlass SIMDLoadSplat<string vec, bits<32> simdop> { 61360784Sdim let mayLoad = 1, UseNamedOperandTable = 1, 62360784Sdim Predicates = [HasUnimplementedSIMD128] in 63360784Sdim defm LOAD_SPLAT_#vec : 64353358Sdim SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr), 65353358Sdim (outs), (ins P2Align:$p2align, offset32_op:$off), [], 66360784Sdim vec#".load_splat\t$dst, ${off}(${addr})$p2align", 67360784Sdim vec#".load_splat\t$off$p2align", simdop>; 68344779Sdim} 69344779Sdim 70360784Sdimdefm "" : SIMDLoadSplat<"v8x16", 194>; 71360784Sdimdefm "" : SIMDLoadSplat<"v16x8", 195>; 72360784Sdimdefm "" : SIMDLoadSplat<"v32x4", 196>; 73360784Sdimdefm "" : SIMDLoadSplat<"v64x2", 197>; 74344779Sdim 75360784Sdimdef wasm_load_splat_t : SDTypeProfile<1, 1, [SDTCisPtrTy<1>]>; 76360784Sdimdef wasm_load_splat : SDNode<"WebAssemblyISD::LOAD_SPLAT", wasm_load_splat_t, 77360784Sdim [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>; 78360784Sdimdef load_splat : PatFrag<(ops node:$addr), (wasm_load_splat node:$addr)>; 79360784Sdim 80360784Sdimlet Predicates = [HasUnimplementedSIMD128] in 81360784Sdimforeach args = [["v16i8", "v8x16"], ["v8i16", "v16x8"], ["v4i32", "v32x4"], 82360784Sdim ["v2i64", "v64x2"], ["v4f32", "v32x4"], ["v2f64", "v64x2"]] in { 83360784Sdimdef : LoadPatNoOffset<!cast<ValueType>(args[0]), 84360784Sdim load_splat, 85360784Sdim !cast<NI>("LOAD_SPLAT_"#args[1])>; 86360784Sdimdef : LoadPatImmOff<!cast<ValueType>(args[0]), 87360784Sdim load_splat, 88360784Sdim regPlusImm, 89360784Sdim !cast<NI>("LOAD_SPLAT_"#args[1])>; 90360784Sdimdef : LoadPatImmOff<!cast<ValueType>(args[0]), 91360784Sdim load_splat, 92360784Sdim or_is_add, 93360784Sdim !cast<NI>("LOAD_SPLAT_"#args[1])>; 94360784Sdimdef : LoadPatOffsetOnly<!cast<ValueType>(args[0]), 95360784Sdim load_splat, 96360784Sdim !cast<NI>("LOAD_SPLAT_"#args[1])>; 97360784Sdimdef : LoadPatGlobalAddrOffOnly<!cast<ValueType>(args[0]), 98360784Sdim load_splat, 99360784Sdim !cast<NI>("LOAD_SPLAT_"#args[1])>; 100344779Sdim} 101344779Sdim 102360784Sdim// Load and extend 103360784Sdimmulticlass SIMDLoadExtend<ValueType vec_t, string name, bits<32> simdop> { 104360784Sdim let mayLoad = 1, UseNamedOperandTable = 1, 105360784Sdim Predicates = [HasUnimplementedSIMD128] in { 106360784Sdim defm LOAD_EXTEND_S_#vec_t : 107360784Sdim SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr), 108353358Sdim (outs), (ins P2Align:$p2align, offset32_op:$off), [], 109360784Sdim name#"_s\t$dst, ${off}(${addr})$p2align", 110360784Sdim name#"_s\t$off$p2align", simdop>; 111360784Sdim defm LOAD_EXTEND_U_#vec_t : 112360784Sdim SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr), 113360784Sdim (outs), (ins P2Align:$p2align, offset32_op:$off), [], 114360784Sdim name#"_u\t$dst, ${off}(${addr})$p2align", 115360784Sdim name#"_u\t$off$p2align", !add(simdop, 1)>; 116360784Sdim } 117344779Sdim} 118344779Sdim 119360784Sdimdefm "" : SIMDLoadExtend<v8i16, "i16x8.load8x8", 210>; 120360784Sdimdefm "" : SIMDLoadExtend<v4i32, "i32x4.load16x4", 212>; 121360784Sdimdefm "" : SIMDLoadExtend<v2i64, "i64x2.load32x2", 214>; 122360784Sdim 123360784Sdimlet Predicates = [HasUnimplementedSIMD128] in 124360784Sdimforeach types = [[v8i16, i8], [v4i32, i16], [v2i64, i32]] in 125360784Sdimforeach exts = [["sextloadv", "_S"], 126360784Sdim ["zextloadv", "_U"], 127360784Sdim ["extloadv", "_U"]] in { 128360784Sdimdef : LoadPatNoOffset<types[0], !cast<PatFrag>(exts[0]#types[1]), 129360784Sdim !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>; 130360784Sdimdef : LoadPatImmOff<types[0], !cast<PatFrag>(exts[0]#types[1]), regPlusImm, 131360784Sdim !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>; 132360784Sdimdef : LoadPatImmOff<types[0], !cast<PatFrag>(exts[0]#types[1]), or_is_add, 133360784Sdim !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>; 134360784Sdimdef : LoadPatOffsetOnly<types[0], !cast<PatFrag>(exts[0]#types[1]), 135360784Sdim !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>; 136360784Sdimdef : LoadPatGlobalAddrOffOnly<types[0], !cast<PatFrag>(exts[0]#types[1]), 137360784Sdim !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>; 138360784Sdim} 139360784Sdim 140360784Sdim 141360784Sdim// Store: v128.store 142360784Sdimlet mayStore = 1, UseNamedOperandTable = 1 in 143360784Sdimdefm STORE_V128 : 144360784Sdim SIMD_I<(outs), (ins P2Align:$p2align, offset32_op:$off, I32:$addr, V128:$vec), 145360784Sdim (outs), (ins P2Align:$p2align, offset32_op:$off), [], 146360784Sdim "v128.store\t${off}(${addr})$p2align, $vec", 147360784Sdim "v128.store\t$off$p2align", 1>; 148360784Sdim 149344779Sdimforeach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in { 150344779Sdim// Def load and store patterns from WebAssemblyInstrMemory.td for vector types 151360784Sdimdef : StorePatNoOffset<vec_t, store, STORE_V128>; 152360784Sdimdef : StorePatImmOff<vec_t, store, regPlusImm, STORE_V128>; 153360784Sdimdef : StorePatImmOff<vec_t, store, or_is_add, STORE_V128>; 154360784Sdimdef : StorePatOffsetOnly<vec_t, store, STORE_V128>; 155360784Sdimdef : StorePatGlobalAddrOffOnly<vec_t, store, STORE_V128>; 156344779Sdim} 157344779Sdim 158344779Sdim//===----------------------------------------------------------------------===// 159344779Sdim// Constructing SIMD values 160344779Sdim//===----------------------------------------------------------------------===// 161344779Sdim 162344779Sdim// Constant: v128.const 163344779Sdimmulticlass ConstVec<ValueType vec_t, dag ops, dag pat, string args> { 164344779Sdim let isMoveImm = 1, isReMaterializable = 1, 165360784Sdim Predicates = [HasUnimplementedSIMD128] in 166344779Sdim defm CONST_V128_#vec_t : SIMD_I<(outs V128:$dst), ops, (outs), ops, 167344779Sdim [(set V128:$dst, (vec_t pat))], 168344779Sdim "v128.const\t$dst, "#args, 169344779Sdim "v128.const\t"#args, 2>; 170344779Sdim} 171344779Sdim 172344779Sdimdefm "" : ConstVec<v16i8, 173344779Sdim (ins vec_i8imm_op:$i0, vec_i8imm_op:$i1, 174344779Sdim vec_i8imm_op:$i2, vec_i8imm_op:$i3, 175344779Sdim vec_i8imm_op:$i4, vec_i8imm_op:$i5, 176344779Sdim vec_i8imm_op:$i6, vec_i8imm_op:$i7, 177344779Sdim vec_i8imm_op:$i8, vec_i8imm_op:$i9, 178344779Sdim vec_i8imm_op:$iA, vec_i8imm_op:$iB, 179344779Sdim vec_i8imm_op:$iC, vec_i8imm_op:$iD, 180344779Sdim vec_i8imm_op:$iE, vec_i8imm_op:$iF), 181344779Sdim (build_vector ImmI8:$i0, ImmI8:$i1, ImmI8:$i2, ImmI8:$i3, 182344779Sdim ImmI8:$i4, ImmI8:$i5, ImmI8:$i6, ImmI8:$i7, 183344779Sdim ImmI8:$i8, ImmI8:$i9, ImmI8:$iA, ImmI8:$iB, 184344779Sdim ImmI8:$iC, ImmI8:$iD, ImmI8:$iE, ImmI8:$iF), 185344779Sdim !strconcat("$i0, $i1, $i2, $i3, $i4, $i5, $i6, $i7, ", 186344779Sdim "$i8, $i9, $iA, $iB, $iC, $iD, $iE, $iF")>; 187344779Sdimdefm "" : ConstVec<v8i16, 188344779Sdim (ins vec_i16imm_op:$i0, vec_i16imm_op:$i1, 189344779Sdim vec_i16imm_op:$i2, vec_i16imm_op:$i3, 190344779Sdim vec_i16imm_op:$i4, vec_i16imm_op:$i5, 191344779Sdim vec_i16imm_op:$i6, vec_i16imm_op:$i7), 192344779Sdim (build_vector 193344779Sdim ImmI16:$i0, ImmI16:$i1, ImmI16:$i2, ImmI16:$i3, 194344779Sdim ImmI16:$i4, ImmI16:$i5, ImmI16:$i6, ImmI16:$i7), 195344779Sdim "$i0, $i1, $i2, $i3, $i4, $i5, $i6, $i7">; 196353358Sdimlet IsCanonical = 1 in 197344779Sdimdefm "" : ConstVec<v4i32, 198344779Sdim (ins vec_i32imm_op:$i0, vec_i32imm_op:$i1, 199344779Sdim vec_i32imm_op:$i2, vec_i32imm_op:$i3), 200344779Sdim (build_vector (i32 imm:$i0), (i32 imm:$i1), 201344779Sdim (i32 imm:$i2), (i32 imm:$i3)), 202344779Sdim "$i0, $i1, $i2, $i3">; 203344779Sdimdefm "" : ConstVec<v2i64, 204344779Sdim (ins vec_i64imm_op:$i0, vec_i64imm_op:$i1), 205344779Sdim (build_vector (i64 imm:$i0), (i64 imm:$i1)), 206344779Sdim "$i0, $i1">; 207344779Sdimdefm "" : ConstVec<v4f32, 208344779Sdim (ins f32imm_op:$i0, f32imm_op:$i1, 209344779Sdim f32imm_op:$i2, f32imm_op:$i3), 210344779Sdim (build_vector (f32 fpimm:$i0), (f32 fpimm:$i1), 211344779Sdim (f32 fpimm:$i2), (f32 fpimm:$i3)), 212344779Sdim "$i0, $i1, $i2, $i3">; 213344779Sdimdefm "" : ConstVec<v2f64, 214344779Sdim (ins f64imm_op:$i0, f64imm_op:$i1), 215344779Sdim (build_vector (f64 fpimm:$i0), (f64 fpimm:$i1)), 216344779Sdim "$i0, $i1">; 217344779Sdim 218344779Sdim// Shuffle lanes: shuffle 219344779Sdimdefm SHUFFLE : 220344779Sdim SIMD_I<(outs V128:$dst), 221344779Sdim (ins V128:$x, V128:$y, 222344779Sdim vec_i8imm_op:$m0, vec_i8imm_op:$m1, 223344779Sdim vec_i8imm_op:$m2, vec_i8imm_op:$m3, 224344779Sdim vec_i8imm_op:$m4, vec_i8imm_op:$m5, 225344779Sdim vec_i8imm_op:$m6, vec_i8imm_op:$m7, 226344779Sdim vec_i8imm_op:$m8, vec_i8imm_op:$m9, 227344779Sdim vec_i8imm_op:$mA, vec_i8imm_op:$mB, 228344779Sdim vec_i8imm_op:$mC, vec_i8imm_op:$mD, 229344779Sdim vec_i8imm_op:$mE, vec_i8imm_op:$mF), 230344779Sdim (outs), 231344779Sdim (ins 232344779Sdim vec_i8imm_op:$m0, vec_i8imm_op:$m1, 233344779Sdim vec_i8imm_op:$m2, vec_i8imm_op:$m3, 234344779Sdim vec_i8imm_op:$m4, vec_i8imm_op:$m5, 235344779Sdim vec_i8imm_op:$m6, vec_i8imm_op:$m7, 236344779Sdim vec_i8imm_op:$m8, vec_i8imm_op:$m9, 237344779Sdim vec_i8imm_op:$mA, vec_i8imm_op:$mB, 238344779Sdim vec_i8imm_op:$mC, vec_i8imm_op:$mD, 239344779Sdim vec_i8imm_op:$mE, vec_i8imm_op:$mF), 240344779Sdim [], 241344779Sdim "v8x16.shuffle\t$dst, $x, $y, "# 242344779Sdim "$m0, $m1, $m2, $m3, $m4, $m5, $m6, $m7, "# 243344779Sdim "$m8, $m9, $mA, $mB, $mC, $mD, $mE, $mF", 244344779Sdim "v8x16.shuffle\t"# 245344779Sdim "$m0, $m1, $m2, $m3, $m4, $m5, $m6, $m7, "# 246344779Sdim "$m8, $m9, $mA, $mB, $mC, $mD, $mE, $mF", 247344779Sdim 3>; 248344779Sdim 249344779Sdim// Shuffles after custom lowering 250344779Sdimdef wasm_shuffle_t : SDTypeProfile<1, 18, []>; 251344779Sdimdef wasm_shuffle : SDNode<"WebAssemblyISD::SHUFFLE", wasm_shuffle_t>; 252344779Sdimforeach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in { 253344779Sdimdef : Pat<(vec_t (wasm_shuffle (vec_t V128:$x), (vec_t V128:$y), 254344779Sdim (i32 LaneIdx32:$m0), (i32 LaneIdx32:$m1), 255344779Sdim (i32 LaneIdx32:$m2), (i32 LaneIdx32:$m3), 256344779Sdim (i32 LaneIdx32:$m4), (i32 LaneIdx32:$m5), 257344779Sdim (i32 LaneIdx32:$m6), (i32 LaneIdx32:$m7), 258344779Sdim (i32 LaneIdx32:$m8), (i32 LaneIdx32:$m9), 259344779Sdim (i32 LaneIdx32:$mA), (i32 LaneIdx32:$mB), 260344779Sdim (i32 LaneIdx32:$mC), (i32 LaneIdx32:$mD), 261344779Sdim (i32 LaneIdx32:$mE), (i32 LaneIdx32:$mF))), 262344779Sdim (vec_t (SHUFFLE (vec_t V128:$x), (vec_t V128:$y), 263344779Sdim (i32 LaneIdx32:$m0), (i32 LaneIdx32:$m1), 264344779Sdim (i32 LaneIdx32:$m2), (i32 LaneIdx32:$m3), 265344779Sdim (i32 LaneIdx32:$m4), (i32 LaneIdx32:$m5), 266344779Sdim (i32 LaneIdx32:$m6), (i32 LaneIdx32:$m7), 267344779Sdim (i32 LaneIdx32:$m8), (i32 LaneIdx32:$m9), 268344779Sdim (i32 LaneIdx32:$mA), (i32 LaneIdx32:$mB), 269344779Sdim (i32 LaneIdx32:$mC), (i32 LaneIdx32:$mD), 270344779Sdim (i32 LaneIdx32:$mE), (i32 LaneIdx32:$mF)))>; 271344779Sdim} 272344779Sdim 273360784Sdim// Swizzle lanes: v8x16.swizzle 274360784Sdimdef wasm_swizzle_t : SDTypeProfile<1, 2, []>; 275360784Sdimdef wasm_swizzle : SDNode<"WebAssemblyISD::SWIZZLE", wasm_swizzle_t>; 276360784Sdimlet Predicates = [HasUnimplementedSIMD128] in 277360784Sdimdefm SWIZZLE : 278360784Sdim SIMD_I<(outs V128:$dst), (ins V128:$src, V128:$mask), (outs), (ins), 279360784Sdim [(set (v16i8 V128:$dst), 280360784Sdim (wasm_swizzle (v16i8 V128:$src), (v16i8 V128:$mask)))], 281360784Sdim "v8x16.swizzle\t$dst, $src, $mask", "v8x16.swizzle", 192>; 282360784Sdim 283360784Sdimdef : Pat<(int_wasm_swizzle (v16i8 V128:$src), (v16i8 V128:$mask)), 284360784Sdim (SWIZZLE V128:$src, V128:$mask)>; 285360784Sdim 286344779Sdim// Create vector with identical lanes: splat 287344779Sdimdef splat2 : PatFrag<(ops node:$x), (build_vector node:$x, node:$x)>; 288344779Sdimdef splat4 : PatFrag<(ops node:$x), (build_vector 289344779Sdim node:$x, node:$x, node:$x, node:$x)>; 290344779Sdimdef splat8 : PatFrag<(ops node:$x), (build_vector 291344779Sdim node:$x, node:$x, node:$x, node:$x, 292344779Sdim node:$x, node:$x, node:$x, node:$x)>; 293344779Sdimdef splat16 : PatFrag<(ops node:$x), (build_vector 294344779Sdim node:$x, node:$x, node:$x, node:$x, 295344779Sdim node:$x, node:$x, node:$x, node:$x, 296344779Sdim node:$x, node:$x, node:$x, node:$x, 297344779Sdim node:$x, node:$x, node:$x, node:$x)>; 298344779Sdim 299344779Sdimmulticlass Splat<ValueType vec_t, string vec, WebAssemblyRegClass reg_t, 300344779Sdim PatFrag splat_pat, bits<32> simdop> { 301344779Sdim // Prefer splats over v128.const for const splats (65 is lowest that works) 302344779Sdim let AddedComplexity = 65 in 303344779Sdim defm SPLAT_#vec_t : SIMD_I<(outs V128:$dst), (ins reg_t:$x), (outs), (ins), 304344779Sdim [(set (vec_t V128:$dst), (splat_pat reg_t:$x))], 305344779Sdim vec#".splat\t$dst, $x", vec#".splat", simdop>; 306344779Sdim} 307344779Sdim 308344779Sdimdefm "" : Splat<v16i8, "i8x16", I32, splat16, 4>; 309344779Sdimdefm "" : Splat<v8i16, "i16x8", I32, splat8, 8>; 310344779Sdimdefm "" : Splat<v4i32, "i32x4", I32, splat4, 12>; 311344779Sdimdefm "" : Splat<v2i64, "i64x2", I64, splat2, 15>; 312344779Sdimdefm "" : Splat<v4f32, "f32x4", F32, splat4, 18>; 313344779Sdimdefm "" : Splat<v2f64, "f64x2", F64, splat2, 21>; 314344779Sdim 315353358Sdim// scalar_to_vector leaves high lanes undefined, so can be a splat 316353358Sdimclass ScalarSplatPat<ValueType vec_t, ValueType lane_t, 317353358Sdim WebAssemblyRegClass reg_t> : 318353358Sdim Pat<(vec_t (scalar_to_vector (lane_t reg_t:$x))), 319353358Sdim (!cast<Instruction>("SPLAT_"#vec_t) reg_t:$x)>; 320353358Sdim 321353358Sdimdef : ScalarSplatPat<v16i8, i32, I32>; 322353358Sdimdef : ScalarSplatPat<v8i16, i32, I32>; 323353358Sdimdef : ScalarSplatPat<v4i32, i32, I32>; 324353358Sdimdef : ScalarSplatPat<v2i64, i64, I64>; 325353358Sdimdef : ScalarSplatPat<v4f32, f32, F32>; 326353358Sdimdef : ScalarSplatPat<v2f64, f64, F64>; 327353358Sdim 328344779Sdim//===----------------------------------------------------------------------===// 329344779Sdim// Accessing lanes 330344779Sdim//===----------------------------------------------------------------------===// 331344779Sdim 332344779Sdim// Extract lane as a scalar: extract_lane / extract_lane_s / extract_lane_u 333344779Sdimmulticlass ExtractLane<ValueType vec_t, string vec, ImmLeaf imm_t, 334344779Sdim WebAssemblyRegClass reg_t, bits<32> simdop, 335344779Sdim string suffix = "", SDNode extract = vector_extract> { 336344779Sdim defm EXTRACT_LANE_#vec_t#suffix : 337344779Sdim SIMD_I<(outs reg_t:$dst), (ins V128:$vec, vec_i8imm_op:$idx), 338344779Sdim (outs), (ins vec_i8imm_op:$idx), 339344779Sdim [(set reg_t:$dst, (extract (vec_t V128:$vec), (i32 imm_t:$idx)))], 340344779Sdim vec#".extract_lane"#suffix#"\t$dst, $vec, $idx", 341344779Sdim vec#".extract_lane"#suffix#"\t$idx", simdop>; 342344779Sdim} 343344779Sdim 344344779Sdimmulticlass ExtractPat<ValueType lane_t, int mask> { 345344779Sdim def _s : PatFrag<(ops node:$vec, node:$idx), 346344779Sdim (i32 (sext_inreg 347344779Sdim (i32 (vector_extract 348344779Sdim node:$vec, 349344779Sdim node:$idx 350344779Sdim )), 351344779Sdim lane_t 352344779Sdim ))>; 353344779Sdim def _u : PatFrag<(ops node:$vec, node:$idx), 354344779Sdim (i32 (and 355344779Sdim (i32 (vector_extract 356344779Sdim node:$vec, 357344779Sdim node:$idx 358344779Sdim )), 359344779Sdim (i32 mask) 360344779Sdim ))>; 361344779Sdim} 362344779Sdim 363344779Sdimdefm extract_i8x16 : ExtractPat<i8, 0xff>; 364344779Sdimdefm extract_i16x8 : ExtractPat<i16, 0xffff>; 365344779Sdim 366344779Sdimmulticlass ExtractLaneExtended<string sign, bits<32> baseInst> { 367344779Sdim defm "" : ExtractLane<v16i8, "i8x16", LaneIdx16, I32, baseInst, sign, 368344779Sdim !cast<PatFrag>("extract_i8x16"#sign)>; 369344779Sdim defm "" : ExtractLane<v8i16, "i16x8", LaneIdx8, I32, !add(baseInst, 4), sign, 370344779Sdim !cast<PatFrag>("extract_i16x8"#sign)>; 371344779Sdim} 372344779Sdim 373344779Sdimdefm "" : ExtractLaneExtended<"_s", 5>; 374360784Sdimlet Predicates = [HasUnimplementedSIMD128] in 375344779Sdimdefm "" : ExtractLaneExtended<"_u", 6>; 376344779Sdimdefm "" : ExtractLane<v4i32, "i32x4", LaneIdx4, I32, 13>; 377344779Sdimdefm "" : ExtractLane<v2i64, "i64x2", LaneIdx2, I64, 16>; 378344779Sdimdefm "" : ExtractLane<v4f32, "f32x4", LaneIdx4, F32, 19>; 379344779Sdimdefm "" : ExtractLane<v2f64, "f64x2", LaneIdx2, F64, 22>; 380344779Sdim 381344779Sdim// It would be more conventional to use unsigned extracts, but v8 382344779Sdim// doesn't implement them yet 383344779Sdimdef : Pat<(i32 (vector_extract (v16i8 V128:$vec), (i32 LaneIdx16:$idx))), 384344779Sdim (EXTRACT_LANE_v16i8_s V128:$vec, (i32 LaneIdx16:$idx))>; 385344779Sdimdef : Pat<(i32 (vector_extract (v8i16 V128:$vec), (i32 LaneIdx8:$idx))), 386344779Sdim (EXTRACT_LANE_v8i16_s V128:$vec, (i32 LaneIdx8:$idx))>; 387344779Sdim 388344779Sdim// Lower undef lane indices to zero 389344779Sdimdef : Pat<(and (i32 (vector_extract (v16i8 V128:$vec), undef)), (i32 0xff)), 390344779Sdim (EXTRACT_LANE_v16i8_u V128:$vec, 0)>; 391344779Sdimdef : Pat<(and (i32 (vector_extract (v8i16 V128:$vec), undef)), (i32 0xffff)), 392344779Sdim (EXTRACT_LANE_v8i16_u V128:$vec, 0)>; 393344779Sdimdef : Pat<(i32 (vector_extract (v16i8 V128:$vec), undef)), 394344779Sdim (EXTRACT_LANE_v16i8_u V128:$vec, 0)>; 395344779Sdimdef : Pat<(i32 (vector_extract (v8i16 V128:$vec), undef)), 396344779Sdim (EXTRACT_LANE_v8i16_u V128:$vec, 0)>; 397344779Sdimdef : Pat<(sext_inreg (i32 (vector_extract (v16i8 V128:$vec), undef)), i8), 398344779Sdim (EXTRACT_LANE_v16i8_s V128:$vec, 0)>; 399344779Sdimdef : Pat<(sext_inreg (i32 (vector_extract (v8i16 V128:$vec), undef)), i16), 400344779Sdim (EXTRACT_LANE_v8i16_s V128:$vec, 0)>; 401344779Sdimdef : Pat<(vector_extract (v4i32 V128:$vec), undef), 402344779Sdim (EXTRACT_LANE_v4i32 V128:$vec, 0)>; 403344779Sdimdef : Pat<(vector_extract (v2i64 V128:$vec), undef), 404344779Sdim (EXTRACT_LANE_v2i64 V128:$vec, 0)>; 405344779Sdimdef : Pat<(vector_extract (v4f32 V128:$vec), undef), 406344779Sdim (EXTRACT_LANE_v4f32 V128:$vec, 0)>; 407344779Sdimdef : Pat<(vector_extract (v2f64 V128:$vec), undef), 408344779Sdim (EXTRACT_LANE_v2f64 V128:$vec, 0)>; 409344779Sdim 410344779Sdim// Replace lane value: replace_lane 411344779Sdimmulticlass ReplaceLane<ValueType vec_t, string vec, ImmLeaf imm_t, 412344779Sdim WebAssemblyRegClass reg_t, ValueType lane_t, 413344779Sdim bits<32> simdop> { 414344779Sdim defm REPLACE_LANE_#vec_t : 415344779Sdim SIMD_I<(outs V128:$dst), (ins V128:$vec, vec_i8imm_op:$idx, reg_t:$x), 416344779Sdim (outs), (ins vec_i8imm_op:$idx), 417344779Sdim [(set V128:$dst, (vector_insert 418344779Sdim (vec_t V128:$vec), (lane_t reg_t:$x), (i32 imm_t:$idx)))], 419344779Sdim vec#".replace_lane\t$dst, $vec, $idx, $x", 420344779Sdim vec#".replace_lane\t$idx", simdop>; 421344779Sdim} 422344779Sdim 423344779Sdimdefm "" : ReplaceLane<v16i8, "i8x16", LaneIdx16, I32, i32, 7>; 424344779Sdimdefm "" : ReplaceLane<v8i16, "i16x8", LaneIdx8, I32, i32, 11>; 425344779Sdimdefm "" : ReplaceLane<v4i32, "i32x4", LaneIdx4, I32, i32, 14>; 426344779Sdimdefm "" : ReplaceLane<v2i64, "i64x2", LaneIdx2, I64, i64, 17>; 427344779Sdimdefm "" : ReplaceLane<v4f32, "f32x4", LaneIdx4, F32, f32, 20>; 428344779Sdimdefm "" : ReplaceLane<v2f64, "f64x2", LaneIdx2, F64, f64, 23>; 429344779Sdim 430344779Sdim// Lower undef lane indices to zero 431344779Sdimdef : Pat<(vector_insert (v16i8 V128:$vec), I32:$x, undef), 432344779Sdim (REPLACE_LANE_v16i8 V128:$vec, 0, I32:$x)>; 433344779Sdimdef : Pat<(vector_insert (v8i16 V128:$vec), I32:$x, undef), 434344779Sdim (REPLACE_LANE_v8i16 V128:$vec, 0, I32:$x)>; 435344779Sdimdef : Pat<(vector_insert (v4i32 V128:$vec), I32:$x, undef), 436344779Sdim (REPLACE_LANE_v4i32 V128:$vec, 0, I32:$x)>; 437344779Sdimdef : Pat<(vector_insert (v2i64 V128:$vec), I64:$x, undef), 438344779Sdim (REPLACE_LANE_v2i64 V128:$vec, 0, I64:$x)>; 439344779Sdimdef : Pat<(vector_insert (v4f32 V128:$vec), F32:$x, undef), 440344779Sdim (REPLACE_LANE_v4f32 V128:$vec, 0, F32:$x)>; 441344779Sdimdef : Pat<(vector_insert (v2f64 V128:$vec), F64:$x, undef), 442344779Sdim (REPLACE_LANE_v2f64 V128:$vec, 0, F64:$x)>; 443344779Sdim 444344779Sdim//===----------------------------------------------------------------------===// 445344779Sdim// Comparisons 446344779Sdim//===----------------------------------------------------------------------===// 447344779Sdim 448344779Sdimmulticlass SIMDCondition<ValueType vec_t, ValueType out_t, string vec, 449344779Sdim string name, CondCode cond, bits<32> simdop> { 450344779Sdim defm _#vec_t : 451344779Sdim SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs), (outs), (ins), 452344779Sdim [(set (out_t V128:$dst), 453344779Sdim (setcc (vec_t V128:$lhs), (vec_t V128:$rhs), cond) 454344779Sdim )], 455344779Sdim vec#"."#name#"\t$dst, $lhs, $rhs", vec#"."#name, simdop>; 456344779Sdim} 457344779Sdim 458344779Sdimmulticlass SIMDConditionInt<string name, CondCode cond, bits<32> baseInst> { 459344779Sdim defm "" : SIMDCondition<v16i8, v16i8, "i8x16", name, cond, baseInst>; 460344779Sdim defm "" : SIMDCondition<v8i16, v8i16, "i16x8", name, cond, 461344779Sdim !add(baseInst, 10)>; 462344779Sdim defm "" : SIMDCondition<v4i32, v4i32, "i32x4", name, cond, 463344779Sdim !add(baseInst, 20)>; 464344779Sdim} 465344779Sdim 466344779Sdimmulticlass SIMDConditionFP<string name, CondCode cond, bits<32> baseInst> { 467344779Sdim defm "" : SIMDCondition<v4f32, v4i32, "f32x4", name, cond, baseInst>; 468344779Sdim defm "" : SIMDCondition<v2f64, v2i64, "f64x2", name, cond, 469344779Sdim !add(baseInst, 6)>; 470344779Sdim} 471344779Sdim 472344779Sdim// Equality: eq 473314564Sdimlet isCommutable = 1 in { 474344779Sdimdefm EQ : SIMDConditionInt<"eq", SETEQ, 24>; 475344779Sdimdefm EQ : SIMDConditionFP<"eq", SETOEQ, 64>; 476314564Sdim} // isCommutable = 1 477344779Sdim 478344779Sdim// Non-equality: ne 479344779Sdimlet isCommutable = 1 in { 480344779Sdimdefm NE : SIMDConditionInt<"ne", SETNE, 25>; 481344779Sdimdefm NE : SIMDConditionFP<"ne", SETUNE, 65>; 482344779Sdim} // isCommutable = 1 483344779Sdim 484344779Sdim// Less than: lt_s / lt_u / lt 485344779Sdimdefm LT_S : SIMDConditionInt<"lt_s", SETLT, 26>; 486344779Sdimdefm LT_U : SIMDConditionInt<"lt_u", SETULT, 27>; 487344779Sdimdefm LT : SIMDConditionFP<"lt", SETOLT, 66>; 488344779Sdim 489344779Sdim// Greater than: gt_s / gt_u / gt 490344779Sdimdefm GT_S : SIMDConditionInt<"gt_s", SETGT, 28>; 491344779Sdimdefm GT_U : SIMDConditionInt<"gt_u", SETUGT, 29>; 492344779Sdimdefm GT : SIMDConditionFP<"gt", SETOGT, 67>; 493344779Sdim 494344779Sdim// Less than or equal: le_s / le_u / le 495344779Sdimdefm LE_S : SIMDConditionInt<"le_s", SETLE, 30>; 496344779Sdimdefm LE_U : SIMDConditionInt<"le_u", SETULE, 31>; 497344779Sdimdefm LE : SIMDConditionFP<"le", SETOLE, 68>; 498344779Sdim 499344779Sdim// Greater than or equal: ge_s / ge_u / ge 500344779Sdimdefm GE_S : SIMDConditionInt<"ge_s", SETGE, 32>; 501344779Sdimdefm GE_U : SIMDConditionInt<"ge_u", SETUGE, 33>; 502344779Sdimdefm GE : SIMDConditionFP<"ge", SETOGE, 69>; 503344779Sdim 504344779Sdim// Lower float comparisons that don't care about NaN to standard WebAssembly 505353358Sdim// float comparisons. These instructions are generated with nnan and in the 506353358Sdim// target-independent expansion of unordered comparisons and ordered ne. 507353358Sdimforeach nodes = [[seteq, EQ_v4f32], [setne, NE_v4f32], [setlt, LT_v4f32], 508353358Sdim [setgt, GT_v4f32], [setle, LE_v4f32], [setge, GE_v4f32]] in 509353358Sdimdef : Pat<(v4i32 (nodes[0] (v4f32 V128:$lhs), (v4f32 V128:$rhs))), 510353358Sdim (v4i32 (nodes[1] (v4f32 V128:$lhs), (v4f32 V128:$rhs)))>; 511344779Sdim 512353358Sdimforeach nodes = [[seteq, EQ_v2f64], [setne, NE_v2f64], [setlt, LT_v2f64], 513353358Sdim [setgt, GT_v2f64], [setle, LE_v2f64], [setge, GE_v2f64]] in 514353358Sdimdef : Pat<(v2i64 (nodes[0] (v2f64 V128:$lhs), (v2f64 V128:$rhs))), 515353358Sdim (v2i64 (nodes[1] (v2f64 V128:$lhs), (v2f64 V128:$rhs)))>; 516353358Sdim 517353358Sdim 518344779Sdim//===----------------------------------------------------------------------===// 519344779Sdim// Bitwise operations 520344779Sdim//===----------------------------------------------------------------------===// 521344779Sdim 522344779Sdimmulticlass SIMDBinary<ValueType vec_t, string vec, SDNode node, string name, 523344779Sdim bits<32> simdop> { 524344779Sdim defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs), 525344779Sdim (outs), (ins), 526344779Sdim [(set (vec_t V128:$dst), 527344779Sdim (node (vec_t V128:$lhs), (vec_t V128:$rhs)) 528344779Sdim )], 529344779Sdim vec#"."#name#"\t$dst, $lhs, $rhs", vec#"."#name, 530344779Sdim simdop>; 531344779Sdim} 532344779Sdim 533344779Sdimmulticlass SIMDBitwise<SDNode node, string name, bits<32> simdop> { 534344779Sdim defm "" : SIMDBinary<v16i8, "v128", node, name, simdop>; 535344779Sdim defm "" : SIMDBinary<v8i16, "v128", node, name, simdop>; 536344779Sdim defm "" : SIMDBinary<v4i32, "v128", node, name, simdop>; 537344779Sdim defm "" : SIMDBinary<v2i64, "v128", node, name, simdop>; 538344779Sdim} 539344779Sdim 540344779Sdimmulticlass SIMDUnary<ValueType vec_t, string vec, SDNode node, string name, 541344779Sdim bits<32> simdop> { 542344779Sdim defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$vec), (outs), (ins), 543344779Sdim [(set (vec_t V128:$dst), 544344779Sdim (vec_t (node (vec_t V128:$vec))) 545344779Sdim )], 546344779Sdim vec#"."#name#"\t$dst, $vec", vec#"."#name, simdop>; 547344779Sdim} 548344779Sdim 549344779Sdim// Bitwise logic: v128.not 550344779Sdimforeach vec_t = [v16i8, v8i16, v4i32, v2i64] in 551344779Sdimdefm NOT: SIMDUnary<vec_t, "v128", vnot, "not", 76>; 552344779Sdim 553344779Sdim// Bitwise logic: v128.and / v128.or / v128.xor 554344779Sdimlet isCommutable = 1 in { 555344779Sdimdefm AND : SIMDBitwise<and, "and", 77>; 556344779Sdimdefm OR : SIMDBitwise<or, "or", 78>; 557344779Sdimdefm XOR : SIMDBitwise<xor, "xor", 79>; 558344779Sdim} // isCommutable = 1 559344779Sdim 560360784Sdim// Bitwise logic: v128.andnot 561360784Sdimdef andnot : PatFrag<(ops node:$left, node:$right), (and $left, (vnot $right))>; 562360784Sdimlet Predicates = [HasUnimplementedSIMD128] in 563360784Sdimdefm ANDNOT : SIMDBitwise<andnot, "andnot", 216>; 564360784Sdim 565344779Sdim// Bitwise select: v128.bitselect 566344779Sdimforeach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in 567344779Sdim defm BITSELECT_#vec_t : 568344779Sdim SIMD_I<(outs V128:$dst), (ins V128:$v1, V128:$v2, V128:$c), (outs), (ins), 569344779Sdim [(set (vec_t V128:$dst), 570344779Sdim (vec_t (int_wasm_bitselect 571344779Sdim (vec_t V128:$v1), (vec_t V128:$v2), (vec_t V128:$c) 572344779Sdim )) 573344779Sdim )], 574344779Sdim "v128.bitselect\t$dst, $v1, $v2, $c", "v128.bitselect", 80>; 575344779Sdim 576344779Sdim// Bitselect is equivalent to (c & v1) | (~c & v2) 577344779Sdimforeach vec_t = [v16i8, v8i16, v4i32, v2i64] in 578344779Sdim def : Pat<(vec_t (or (and (vec_t V128:$c), (vec_t V128:$v1)), 579344779Sdim (and (vnot V128:$c), (vec_t V128:$v2)))), 580344779Sdim (!cast<Instruction>("BITSELECT_"#vec_t) 581344779Sdim V128:$v1, V128:$v2, V128:$c)>; 582344779Sdim 583344779Sdim//===----------------------------------------------------------------------===// 584344779Sdim// Integer unary arithmetic 585344779Sdim//===----------------------------------------------------------------------===// 586344779Sdim 587344779Sdimmulticlass SIMDUnaryInt<SDNode node, string name, bits<32> baseInst> { 588344779Sdim defm "" : SIMDUnary<v16i8, "i8x16", node, name, baseInst>; 589344779Sdim defm "" : SIMDUnary<v8i16, "i16x8", node, name, !add(baseInst, 17)>; 590344779Sdim defm "" : SIMDUnary<v4i32, "i32x4", node, name, !add(baseInst, 34)>; 591344779Sdim defm "" : SIMDUnary<v2i64, "i64x2", node, name, !add(baseInst, 51)>; 592344779Sdim} 593344779Sdim 594344779Sdimmulticlass SIMDReduceVec<ValueType vec_t, string vec, SDNode op, string name, 595344779Sdim bits<32> simdop> { 596344779Sdim defm _#vec_t : SIMD_I<(outs I32:$dst), (ins V128:$vec), (outs), (ins), 597344779Sdim [(set I32:$dst, (i32 (op (vec_t V128:$vec))))], 598344779Sdim vec#"."#name#"\t$dst, $vec", vec#"."#name, simdop>; 599344779Sdim} 600344779Sdim 601344779Sdimmulticlass SIMDReduce<SDNode op, string name, bits<32> baseInst> { 602344779Sdim defm "" : SIMDReduceVec<v16i8, "i8x16", op, name, baseInst>; 603344779Sdim defm "" : SIMDReduceVec<v8i16, "i16x8", op, name, !add(baseInst, 17)>; 604344779Sdim defm "" : SIMDReduceVec<v4i32, "i32x4", op, name, !add(baseInst, 34)>; 605344779Sdim defm "" : SIMDReduceVec<v2i64, "i64x2", op, name, !add(baseInst, 51)>; 606344779Sdim} 607344779Sdim 608344779Sdim// Integer vector negation 609344779Sdimdef ivneg : PatFrag<(ops node:$in), (sub immAllZerosV, node:$in)>; 610344779Sdim 611344779Sdim// Integer negation: neg 612344779Sdimdefm NEG : SIMDUnaryInt<ivneg, "neg", 81>; 613344779Sdim 614344779Sdim// Any lane true: any_true 615344779Sdimdefm ANYTRUE : SIMDReduce<int_wasm_anytrue, "any_true", 82>; 616344779Sdim 617344779Sdim// All lanes true: all_true 618344779Sdimdefm ALLTRUE : SIMDReduce<int_wasm_alltrue, "all_true", 83>; 619344779Sdim 620353358Sdim// Reductions already return 0 or 1, so and 1, setne 0, and seteq 1 621353358Sdim// can be folded out 622353358Sdimforeach reduction = 623353358Sdim [["int_wasm_anytrue", "ANYTRUE"], ["int_wasm_alltrue", "ALLTRUE"]] in 624353358Sdimforeach ty = [v16i8, v8i16, v4i32, v2i64] in { 625353358Sdimdef : Pat<(i32 (and 626353358Sdim (i32 (!cast<Intrinsic>(reduction[0]) (ty V128:$x))), 627353358Sdim (i32 1) 628353358Sdim )), 629353358Sdim (i32 (!cast<NI>(reduction[1]#"_"#ty) (ty V128:$x)))>; 630353358Sdimdef : Pat<(i32 (setne 631353358Sdim (i32 (!cast<Intrinsic>(reduction[0]) (ty V128:$x))), 632353358Sdim (i32 0) 633353358Sdim )), 634353358Sdim (i32 (!cast<NI>(reduction[1]#"_"#ty) (ty V128:$x)))>; 635353358Sdimdef : Pat<(i32 (seteq 636353358Sdim (i32 (!cast<Intrinsic>(reduction[0]) (ty V128:$x))), 637353358Sdim (i32 1) 638353358Sdim )), 639353358Sdim (i32 (!cast<NI>(reduction[1]#"_"#ty) (ty V128:$x)))>; 640353358Sdim} 641353358Sdim 642344779Sdim//===----------------------------------------------------------------------===// 643344779Sdim// Bit shifts 644344779Sdim//===----------------------------------------------------------------------===// 645344779Sdim 646344779Sdimmulticlass SIMDShift<ValueType vec_t, string vec, SDNode node, dag shift_vec, 647344779Sdim string name, bits<32> simdop> { 648344779Sdim defm _#vec_t : SIMD_I<(outs V128:$dst), (ins V128:$vec, I32:$x), 649344779Sdim (outs), (ins), 650344779Sdim [(set (vec_t V128:$dst), 651344779Sdim (node V128:$vec, (vec_t shift_vec)))], 652344779Sdim vec#"."#name#"\t$dst, $vec, $x", vec#"."#name, simdop>; 653344779Sdim} 654344779Sdim 655344779Sdimmulticlass SIMDShiftInt<SDNode node, string name, bits<32> baseInst> { 656344779Sdim defm "" : SIMDShift<v16i8, "i8x16", node, (splat16 I32:$x), name, baseInst>; 657344779Sdim defm "" : SIMDShift<v8i16, "i16x8", node, (splat8 I32:$x), name, 658344779Sdim !add(baseInst, 17)>; 659344779Sdim defm "" : SIMDShift<v4i32, "i32x4", node, (splat4 I32:$x), name, 660344779Sdim !add(baseInst, 34)>; 661344779Sdim defm "" : SIMDShift<v2i64, "i64x2", node, (splat2 (i64 (zext I32:$x))), 662344779Sdim name, !add(baseInst, 51)>; 663344779Sdim} 664344779Sdim 665344779Sdim// Left shift by scalar: shl 666344779Sdimdefm SHL : SIMDShiftInt<shl, "shl", 84>; 667344779Sdim 668344779Sdim// Right shift by scalar: shr_s / shr_u 669344779Sdimdefm SHR_S : SIMDShiftInt<sra, "shr_s", 85>; 670344779Sdimdefm SHR_U : SIMDShiftInt<srl, "shr_u", 86>; 671344779Sdim 672353358Sdim// Truncate i64 shift operands to i32s, except if they are already i32s 673353358Sdimforeach shifts = [[shl, SHL_v2i64], [sra, SHR_S_v2i64], [srl, SHR_U_v2i64]] in { 674353358Sdimdef : Pat<(v2i64 (shifts[0] 675353358Sdim (v2i64 V128:$vec), 676353358Sdim (v2i64 (splat2 (i64 (sext I32:$x)))) 677353358Sdim )), 678353358Sdim (v2i64 (shifts[1] (v2i64 V128:$vec), (i32 I32:$x)))>; 679344779Sdimdef : Pat<(v2i64 (shifts[0] (v2i64 V128:$vec), (v2i64 (splat2 I64:$x)))), 680344779Sdim (v2i64 (shifts[1] (v2i64 V128:$vec), (I32_WRAP_I64 I64:$x)))>; 681353358Sdim} 682344779Sdim 683344779Sdim// 2xi64 shifts with constant shift amounts are custom lowered to avoid wrapping 684344779Sdimdef wasm_shift_t : SDTypeProfile<1, 2, 685344779Sdim [SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisVT<2, i32>] 686344779Sdim>; 687344779Sdimdef wasm_shl : SDNode<"WebAssemblyISD::VEC_SHL", wasm_shift_t>; 688344779Sdimdef wasm_shr_s : SDNode<"WebAssemblyISD::VEC_SHR_S", wasm_shift_t>; 689344779Sdimdef wasm_shr_u : SDNode<"WebAssemblyISD::VEC_SHR_U", wasm_shift_t>; 690344779Sdimforeach shifts = [[wasm_shl, SHL_v2i64], 691344779Sdim [wasm_shr_s, SHR_S_v2i64], 692344779Sdim [wasm_shr_u, SHR_U_v2i64]] in 693344779Sdimdef : Pat<(v2i64 (shifts[0] (v2i64 V128:$vec), I32:$x)), 694344779Sdim (v2i64 (shifts[1] (v2i64 V128:$vec), I32:$x))>; 695344779Sdim 696344779Sdim//===----------------------------------------------------------------------===// 697344779Sdim// Integer binary arithmetic 698344779Sdim//===----------------------------------------------------------------------===// 699344779Sdim 700344779Sdimmulticlass SIMDBinaryIntSmall<SDNode node, string name, bits<32> baseInst> { 701344779Sdim defm "" : SIMDBinary<v16i8, "i8x16", node, name, baseInst>; 702344779Sdim defm "" : SIMDBinary<v8i16, "i16x8", node, name, !add(baseInst, 17)>; 703344779Sdim} 704344779Sdim 705344779Sdimmulticlass SIMDBinaryIntNoI64x2<SDNode node, string name, bits<32> baseInst> { 706344779Sdim defm "" : SIMDBinaryIntSmall<node, name, baseInst>; 707344779Sdim defm "" : SIMDBinary<v4i32, "i32x4", node, name, !add(baseInst, 34)>; 708344779Sdim} 709344779Sdim 710344779Sdimmulticlass SIMDBinaryInt<SDNode node, string name, bits<32> baseInst> { 711344779Sdim defm "" : SIMDBinaryIntNoI64x2<node, name, baseInst>; 712344779Sdim defm "" : SIMDBinary<v2i64, "i64x2", node, name, !add(baseInst, 51)>; 713344779Sdim} 714344779Sdim 715344779Sdim// Integer addition: add / add_saturate_s / add_saturate_u 716344779Sdimlet isCommutable = 1 in { 717344779Sdimdefm ADD : SIMDBinaryInt<add, "add", 87>; 718344779Sdimdefm ADD_SAT_S : SIMDBinaryIntSmall<saddsat, "add_saturate_s", 88>; 719344779Sdimdefm ADD_SAT_U : SIMDBinaryIntSmall<uaddsat, "add_saturate_u", 89>; 720344779Sdim} // isCommutable = 1 721344779Sdim 722344779Sdim// Integer subtraction: sub / sub_saturate_s / sub_saturate_u 723344779Sdimdefm SUB : SIMDBinaryInt<sub, "sub", 90>; 724344779Sdimdefm SUB_SAT_S : 725344779Sdim SIMDBinaryIntSmall<int_wasm_sub_saturate_signed, "sub_saturate_s", 91>; 726344779Sdimdefm SUB_SAT_U : 727344779Sdim SIMDBinaryIntSmall<int_wasm_sub_saturate_unsigned, "sub_saturate_u", 92>; 728344779Sdim 729344779Sdim// Integer multiplication: mul 730360784Sdimlet isCommutable = 1 in 731344779Sdimdefm MUL : SIMDBinaryIntNoI64x2<mul, "mul", 93>; 732344779Sdim 733360784Sdim// Integer min_s / min_u / max_s / max_u 734360784Sdimlet isCommutable = 1 in { 735360784Sdimdefm MIN_S : SIMDBinaryIntNoI64x2<smin, "min_s", 94>; 736360784Sdimdefm MIN_U : SIMDBinaryIntNoI64x2<umin, "min_u", 95>; 737360784Sdimdefm MAX_S : SIMDBinaryIntNoI64x2<smax, "max_s", 96>; 738360784Sdimdefm MAX_U : SIMDBinaryIntNoI64x2<umax, "max_u", 97>; 739360784Sdim} // isCommutable = 1 740360784Sdim 741360784Sdim// Integer unsigned rounding average: avgr_u 742360784Sdimlet isCommutable = 1, Predicates = [HasUnimplementedSIMD128] in { 743360784Sdimdefm AVGR_U : SIMDBinary<v16i8, "i8x16", int_wasm_avgr_unsigned, "avgr_u", 217>; 744360784Sdimdefm AVGR_U : SIMDBinary<v8i16, "i16x8", int_wasm_avgr_unsigned, "avgr_u", 218>; 745360784Sdim} 746360784Sdim 747360784Sdimdef add_nuw : PatFrag<(ops node:$lhs, node:$rhs), 748360784Sdim (add node:$lhs, node:$rhs), 749360784Sdim "return N->getFlags().hasNoUnsignedWrap();">; 750360784Sdim 751360784Sdimforeach nodes = [[v16i8, splat16], [v8i16, splat8]] in 752360784Sdimdef : Pat<(srl 753360784Sdim (add_nuw 754360784Sdim (add_nuw (nodes[0] V128:$lhs), (nodes[0] V128:$rhs)), 755360784Sdim (nodes[1] (i32 1)) 756360784Sdim ), 757360784Sdim (nodes[0] (nodes[1] (i32 1))) 758360784Sdim ), 759360784Sdim (!cast<NI>("AVGR_U_"#nodes[0]) V128:$lhs, V128:$rhs)>; 760360784Sdim 761360784Sdim// Widening dot product: i32x4.dot_i16x8_s 762360784Sdimlet isCommutable = 1 in 763360784Sdimdefm DOT : SIMD_I<(outs V128:$dst), (ins V128:$lhs, V128:$rhs), (outs), (ins), 764360784Sdim [(set V128:$dst, (int_wasm_dot V128:$lhs, V128:$rhs))], 765360784Sdim "i32x4.dot_i16x8_s\t$dst, $lhs, $rhs", "i32x4.dot_i16x8_s", 766360784Sdim 219>; 767360784Sdim 768344779Sdim//===----------------------------------------------------------------------===// 769344779Sdim// Floating-point unary arithmetic 770344779Sdim//===----------------------------------------------------------------------===// 771344779Sdim 772344779Sdimmulticlass SIMDUnaryFP<SDNode node, string name, bits<32> baseInst> { 773344779Sdim defm "" : SIMDUnary<v4f32, "f32x4", node, name, baseInst>; 774344779Sdim defm "" : SIMDUnary<v2f64, "f64x2", node, name, !add(baseInst, 11)>; 775344779Sdim} 776344779Sdim 777344779Sdim// Absolute value: abs 778344779Sdimdefm ABS : SIMDUnaryFP<fabs, "abs", 149>; 779344779Sdim 780344779Sdim// Negation: neg 781344779Sdimdefm NEG : SIMDUnaryFP<fneg, "neg", 150>; 782344779Sdim 783344779Sdim// Square root: sqrt 784360784Sdimlet Predicates = [HasUnimplementedSIMD128] in 785344779Sdimdefm SQRT : SIMDUnaryFP<fsqrt, "sqrt", 151>; 786344779Sdim 787344779Sdim//===----------------------------------------------------------------------===// 788344779Sdim// Floating-point binary arithmetic 789344779Sdim//===----------------------------------------------------------------------===// 790344779Sdim 791344779Sdimmulticlass SIMDBinaryFP<SDNode node, string name, bits<32> baseInst> { 792344779Sdim defm "" : SIMDBinary<v4f32, "f32x4", node, name, baseInst>; 793344779Sdim defm "" : SIMDBinary<v2f64, "f64x2", node, name, !add(baseInst, 11)>; 794344779Sdim} 795344779Sdim 796344779Sdim// Addition: add 797344779Sdimlet isCommutable = 1 in 798344779Sdimdefm ADD : SIMDBinaryFP<fadd, "add", 154>; 799344779Sdim 800344779Sdim// Subtraction: sub 801344779Sdimdefm SUB : SIMDBinaryFP<fsub, "sub", 155>; 802344779Sdim 803344779Sdim// Multiplication: mul 804344779Sdimlet isCommutable = 1 in 805344779Sdimdefm MUL : SIMDBinaryFP<fmul, "mul", 156>; 806344779Sdim 807344779Sdim// Division: div 808360784Sdimlet Predicates = [HasUnimplementedSIMD128] in 809344779Sdimdefm DIV : SIMDBinaryFP<fdiv, "div", 157>; 810344779Sdim 811344779Sdim// NaN-propagating minimum: min 812344779Sdimdefm MIN : SIMDBinaryFP<fminimum, "min", 158>; 813344779Sdim 814344779Sdim// NaN-propagating maximum: max 815344779Sdimdefm MAX : SIMDBinaryFP<fmaximum, "max", 159>; 816344779Sdim 817344779Sdim//===----------------------------------------------------------------------===// 818344779Sdim// Conversions 819344779Sdim//===----------------------------------------------------------------------===// 820344779Sdim 821344779Sdimmulticlass SIMDConvert<ValueType vec_t, ValueType arg_t, SDNode op, 822344779Sdim string name, bits<32> simdop> { 823344779Sdim defm op#_#vec_t#_#arg_t : 824344779Sdim SIMD_I<(outs V128:$dst), (ins V128:$vec), (outs), (ins), 825344779Sdim [(set (vec_t V128:$dst), (vec_t (op (arg_t V128:$vec))))], 826344779Sdim name#"\t$dst, $vec", name, simdop>; 827344779Sdim} 828344779Sdim 829344779Sdim// Integer to floating point: convert 830344779Sdimdefm "" : SIMDConvert<v4f32, v4i32, sint_to_fp, "f32x4.convert_i32x4_s", 175>; 831344779Sdimdefm "" : SIMDConvert<v4f32, v4i32, uint_to_fp, "f32x4.convert_i32x4_u", 176>; 832344779Sdimdefm "" : SIMDConvert<v2f64, v2i64, sint_to_fp, "f64x2.convert_i64x2_s", 177>; 833344779Sdimdefm "" : SIMDConvert<v2f64, v2i64, uint_to_fp, "f64x2.convert_i64x2_u", 178>; 834344779Sdim 835344779Sdim// Floating point to integer with saturation: trunc_sat 836344779Sdimdefm "" : SIMDConvert<v4i32, v4f32, fp_to_sint, "i32x4.trunc_sat_f32x4_s", 171>; 837344779Sdimdefm "" : SIMDConvert<v4i32, v4f32, fp_to_uint, "i32x4.trunc_sat_f32x4_u", 172>; 838344779Sdimdefm "" : SIMDConvert<v2i64, v2f64, fp_to_sint, "i64x2.trunc_sat_f64x2_s", 173>; 839344779Sdimdefm "" : SIMDConvert<v2i64, v2f64, fp_to_uint, "i64x2.trunc_sat_f64x2_u", 174>; 840344779Sdim 841360784Sdim// Widening operations 842360784Sdimmulticlass SIMDWiden<ValueType vec_t, string vec, ValueType arg_t, string arg, 843360784Sdim bits<32> baseInst> { 844360784Sdim defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_low_signed, 845360784Sdim vec#".widen_low_"#arg#"_s", baseInst>; 846360784Sdim defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_high_signed, 847360784Sdim vec#".widen_high_"#arg#"_s", !add(baseInst, 1)>; 848360784Sdim defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_low_unsigned, 849360784Sdim vec#".widen_low_"#arg#"_u", !add(baseInst, 2)>; 850360784Sdim defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_high_unsigned, 851360784Sdim vec#".widen_high_"#arg#"_u", !add(baseInst, 3)>; 852360784Sdim} 853360784Sdim 854360784Sdimdefm "" : SIMDWiden<v8i16, "i16x8", v16i8, "i8x16", 202>; 855360784Sdimdefm "" : SIMDWiden<v4i32, "i32x4", v8i16, "i16x8", 206>; 856360784Sdim 857360784Sdim// Narrowing operations 858360784Sdimmulticlass SIMDNarrow<ValueType vec_t, string vec, ValueType arg_t, string arg, 859360784Sdim bits<32> baseInst> { 860360784Sdim defm NARROW_S_#vec_t : 861360784Sdim SIMD_I<(outs V128:$dst), (ins V128:$low, V128:$high), (outs), (ins), 862360784Sdim [(set (vec_t V128:$dst), (vec_t (int_wasm_narrow_signed 863360784Sdim (arg_t V128:$low), (arg_t V128:$high))))], 864360784Sdim vec#".narrow_"#arg#"_s\t$dst, $low, $high", vec#".narrow_"#arg#"_s", 865360784Sdim baseInst>; 866360784Sdim defm NARROW_U_#vec_t : 867360784Sdim SIMD_I<(outs V128:$dst), (ins V128:$low, V128:$high), (outs), (ins), 868360784Sdim [(set (vec_t V128:$dst), (vec_t (int_wasm_narrow_unsigned 869360784Sdim (arg_t V128:$low), (arg_t V128:$high))))], 870360784Sdim vec#".narrow_"#arg#"_u\t$dst, $low, $high", vec#".narrow_"#arg#"_u", 871360784Sdim !add(baseInst, 1)>; 872360784Sdim} 873360784Sdim 874360784Sdimdefm "" : SIMDNarrow<v16i8, "i8x16", v8i16, "i16x8", 198>; 875360784Sdimdefm "" : SIMDNarrow<v8i16, "i16x8", v4i32, "i32x4", 200>; 876360784Sdim 877344779Sdim// Lower llvm.wasm.trunc.saturate.* to saturating instructions 878344779Sdimdef : Pat<(v4i32 (int_wasm_trunc_saturate_signed (v4f32 V128:$src))), 879344779Sdim (fp_to_sint_v4i32_v4f32 (v4f32 V128:$src))>; 880344779Sdimdef : Pat<(v4i32 (int_wasm_trunc_saturate_unsigned (v4f32 V128:$src))), 881344779Sdim (fp_to_uint_v4i32_v4f32 (v4f32 V128:$src))>; 882344779Sdimdef : Pat<(v2i64 (int_wasm_trunc_saturate_signed (v2f64 V128:$src))), 883344779Sdim (fp_to_sint_v2i64_v2f64 (v2f64 V128:$src))>; 884344779Sdimdef : Pat<(v2i64 (int_wasm_trunc_saturate_unsigned (v2f64 V128:$src))), 885344779Sdim (fp_to_uint_v2i64_v2f64 (v2f64 V128:$src))>; 886344779Sdim 887344779Sdim// Bitcasts are nops 888344779Sdim// Matching bitcast t1 to t1 causes strange errors, so avoid repeating types 889344779Sdimforeach t1 = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in 890344779Sdimforeach t2 = !foldl( 891344779Sdim []<ValueType>, [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64], 892344779Sdim acc, cur, !if(!eq(!cast<string>(t1), !cast<string>(cur)), 893344779Sdim acc, !listconcat(acc, [cur]) 894344779Sdim ) 895344779Sdim) in 896344779Sdimdef : Pat<(t1 (bitconvert (t2 V128:$v))), (t1 V128:$v)>; 897360784Sdim 898360784Sdim//===----------------------------------------------------------------------===// 899360784Sdim// Quasi-Fused Multiply- Add and Subtract (QFMA/QFMS) 900360784Sdim//===----------------------------------------------------------------------===// 901360784Sdim 902360784Sdimmulticlass SIMDQFM<ValueType vec_t, string vec, bits<32> baseInst> { 903360784Sdim defm QFMA_#vec_t : 904360784Sdim SIMD_I<(outs V128:$dst), (ins V128:$a, V128:$b, V128:$c), 905360784Sdim (outs), (ins), 906360784Sdim [(set (vec_t V128:$dst), 907360784Sdim (int_wasm_qfma (vec_t V128:$a), (vec_t V128:$b), (vec_t V128:$c)))], 908360784Sdim vec#".qfma\t$dst, $a, $b, $c", vec#".qfma", baseInst>; 909360784Sdim defm QFMS_#vec_t : 910360784Sdim SIMD_I<(outs V128:$dst), (ins V128:$a, V128:$b, V128:$c), 911360784Sdim (outs), (ins), 912360784Sdim [(set (vec_t V128:$dst), 913360784Sdim (int_wasm_qfms (vec_t V128:$a), (vec_t V128:$b), (vec_t V128:$c)))], 914360784Sdim vec#".qfms\t$dst, $a, $b, $c", vec#".qfms", !add(baseInst, 1)>; 915360784Sdim} 916360784Sdim 917360784Sdimdefm "" : SIMDQFM<v4f32, "f32x4", 0x98>; 918360784Sdimdefm "" : SIMDQFM<v2f64, "f64x2", 0xa3>; 919