1263320SdimPull in r198286 from upstream llvm trunk (by Venkatraman Govindaraju): 2263320Sdim 3263320Sdim [Sparc] Handle atomic loads/stores in sparc backend. 4263320Sdim 5263320SdimIntroduced here: http://svn.freebsd.org/changeset/base/262261 6263320Sdim 7263320SdimIndex: lib/Target/Sparc/SparcInstrInfo.td 8263320Sdim=================================================================== 9263320Sdim--- lib/Target/Sparc/SparcInstrInfo.td 10263320Sdim+++ lib/Target/Sparc/SparcInstrInfo.td 11263320Sdim@@ -975,6 +975,33 @@ let rs1 = 0 in 12263320Sdim def : Pat<(ctpop i32:$src), 13263320Sdim (POPCrr (SRLri $src, 0))>; 14263320Sdim 15263320Sdim+// Atomic swap. 16263320Sdim+let hasSideEffects =1, rd = 0, rs1 = 0b01111, rs2 = 0 in 17263320Sdim+ def STBAR : F3_1<2, 0b101000, (outs), (ins), "stbar", []>; 18263320Sdim+ 19263320Sdim+let Predicates = [HasV9], hasSideEffects = 1, rd = 0, rs1 = 0b01111 in 20263320Sdim+ def MEMBARi : F3_2<2, 0b101000, (outs), (ins i32imm:$simm13), 21263320Sdim+ "membar $simm13", []>; 22263320Sdim+ 23263320Sdim+let Constraints = "$val = $rd" in { 24263320Sdim+ def SWAPrr : F3_1<3, 0b001111, 25263320Sdim+ (outs IntRegs:$rd), (ins IntRegs:$val, MEMrr:$addr), 26263320Sdim+ "swap [$addr], $rd", 27263320Sdim+ [(set i32:$rd, (atomic_swap_32 ADDRrr:$addr, i32:$val))]>; 28263320Sdim+ def SWAPri : F3_2<3, 0b001111, 29263320Sdim+ (outs IntRegs:$rd), (ins IntRegs:$val, MEMri:$addr), 30263320Sdim+ "swap [$addr], $rd", 31263320Sdim+ [(set i32:$rd, (atomic_swap_32 ADDRri:$addr, i32:$val))]>; 32263320Sdim+} 33263320Sdim+ 34263320Sdim+let Predicates = [HasV9], Constraints = "$swap = $rd" in 35263320Sdim+ def CASrr: F3_1<3, 0b111100, 36263320Sdim+ (outs IntRegs:$rd), (ins IntRegs:$rs1, IntRegs:$rs2, 37263320Sdim+ IntRegs:$swap), 38263320Sdim+ "cas [$rs1], $rs2, $rd", 39263320Sdim+ [(set i32:$rd, 40263320Sdim+ (atomic_cmp_swap iPTR:$rs1, i32:$rs2, i32:$swap))]>; 41263320Sdim+ 42263320Sdim //===----------------------------------------------------------------------===// 43263320Sdim // Non-Instruction Patterns 44263320Sdim //===----------------------------------------------------------------------===// 45263320Sdim@@ -1036,4 +1063,17 @@ def : Pat<(i32 (zextloadi1 ADDRri:$src)), (LDUBri 46263320Sdim def : Pat<(store (i32 0), ADDRrr:$dst), (STrr ADDRrr:$dst, (i32 G0))>; 47263320Sdim def : Pat<(store (i32 0), ADDRri:$dst), (STri ADDRri:$dst, (i32 G0))>; 48263320Sdim 49263320Sdim+// store bar for all atomic_fence in V8. 50263320Sdim+let Predicates = [HasNoV9] in 51263320Sdim+ def : Pat<(atomic_fence imm, imm), (STBAR)>; 52263320Sdim+ 53263320Sdim+// atomic_load_32 addr -> load addr 54263320Sdim+def : Pat<(i32 (atomic_load ADDRrr:$src)), (LDrr ADDRrr:$src)>; 55263320Sdim+def : Pat<(i32 (atomic_load ADDRri:$src)), (LDri ADDRri:$src)>; 56263320Sdim+ 57263320Sdim+// atomic_store_32 val, addr -> store val, addr 58263320Sdim+def : Pat<(atomic_store ADDRrr:$dst, i32:$val), (STrr ADDRrr:$dst, $val)>; 59263320Sdim+def : Pat<(atomic_store ADDRri:$dst, i32:$val), (STri ADDRri:$dst, $val)>; 60263320Sdim+ 61263320Sdim+ 62263320Sdim include "SparcInstr64Bit.td" 63263320SdimIndex: lib/Target/Sparc/SparcISelLowering.cpp 64263320Sdim=================================================================== 65263320Sdim--- lib/Target/Sparc/SparcISelLowering.cpp 66263320Sdim+++ lib/Target/Sparc/SparcISelLowering.cpp 67263320Sdim@@ -1472,10 +1472,30 @@ SparcTargetLowering::SparcTargetLowering(TargetMac 68263320Sdim setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom); 69263320Sdim } 70263320Sdim 71263320Sdim- // FIXME: There are instructions available for ATOMIC_FENCE 72263320Sdim- // on SparcV8 and later. 73263320Sdim- setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Expand); 74263320Sdim+ // ATOMICs. 75263320Sdim+ // FIXME: We insert fences for each atomics and generate sub-optimal code 76263320Sdim+ // for PSO/TSO. Also, implement other atomicrmw operations. 77263320Sdim 78263320Sdim+ setInsertFencesForAtomic(true); 79263320Sdim+ 80263320Sdim+ setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Legal); 81263320Sdim+ setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, 82263320Sdim+ (Subtarget->isV9() ? Legal: Expand)); 83263320Sdim+ 84263320Sdim+ 85263320Sdim+ setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Legal); 86263320Sdim+ 87263320Sdim+ // Custom Lower Atomic LOAD/STORE 88263320Sdim+ setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom); 89263320Sdim+ setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom); 90263320Sdim+ 91263320Sdim+ if (Subtarget->is64Bit()) { 92263320Sdim+ setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Legal); 93263320Sdim+ setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, Expand); 94263320Sdim+ setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Custom); 95263320Sdim+ setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Custom); 96263320Sdim+ } 97263320Sdim+ 98263320Sdim if (!Subtarget->isV9()) { 99263320Sdim // SparcV8 does not have FNEGD and FABSD. 100263320Sdim setOperationAction(ISD::FNEG, MVT::f64, Custom); 101263320Sdim@@ -2723,6 +2743,16 @@ static SDValue LowerUMULO_SMULO(SDValue Op, Select 102263320Sdim return DAG.getMergeValues(Ops, 2, dl); 103263320Sdim } 104263320Sdim 105263320Sdim+static SDValue LowerATOMIC_LOAD_STORE(SDValue Op, SelectionDAG &DAG) { 106263320Sdim+ // Monotonic load/stores are legal. 107263320Sdim+ if (cast<AtomicSDNode>(Op)->getOrdering() <= Monotonic) 108263320Sdim+ return Op; 109263320Sdim+ 110263320Sdim+ // Otherwise, expand with a fence. 111263320Sdim+ return SDValue(); 112263320Sdim+} 113263320Sdim+ 114263320Sdim+ 115263320Sdim SDValue SparcTargetLowering:: 116263320Sdim LowerOperation(SDValue Op, SelectionDAG &DAG) const { 117263320Sdim 118263320Sdim@@ -2778,6 +2808,8 @@ LowerOperation(SDValue Op, SelectionDAG &DAG) cons 119263320Sdim case ISD::SUBE: return LowerADDC_ADDE_SUBC_SUBE(Op, DAG); 120263320Sdim case ISD::UMULO: 121263320Sdim case ISD::SMULO: return LowerUMULO_SMULO(Op, DAG, *this); 122263320Sdim+ case ISD::ATOMIC_LOAD: 123263320Sdim+ case ISD::ATOMIC_STORE: return LowerATOMIC_LOAD_STORE(Op, DAG); 124263320Sdim } 125263320Sdim } 126263320Sdim 127263320SdimIndex: lib/Target/Sparc/SparcInstr64Bit.td 128263320Sdim=================================================================== 129263320Sdim--- lib/Target/Sparc/SparcInstr64Bit.td 130263320Sdim+++ lib/Target/Sparc/SparcInstr64Bit.td 131263320Sdim@@ -415,6 +415,32 @@ def SETHIXi : F2_1<0b100, 132263320Sdim "sethi $imm22, $rd", 133263320Sdim [(set i64:$rd, SETHIimm:$imm22)]>; 134263320Sdim } 135263320Sdim+ 136263320Sdim+// ATOMICS. 137263320Sdim+let Predicates = [Is64Bit], Constraints = "$swap = $rd" in { 138263320Sdim+ def CASXrr: F3_1<3, 0b111110, 139263320Sdim+ (outs I64Regs:$rd), (ins I64Regs:$rs1, I64Regs:$rs2, 140263320Sdim+ I64Regs:$swap), 141263320Sdim+ "casx [$rs1], $rs2, $rd", 142263320Sdim+ [(set i64:$rd, 143263320Sdim+ (atomic_cmp_swap i64:$rs1, i64:$rs2, i64:$swap))]>; 144263320Sdim+ 145263320Sdim+} // Predicates = [Is64Bit], Constraints = ... 146263320Sdim+ 147263320Sdim+let Predicates = [Is64Bit] in { 148263320Sdim+ 149263320Sdim+def : Pat<(atomic_fence imm, imm), (MEMBARi 0xf)>; 150263320Sdim+ 151263320Sdim+// atomic_load_64 addr -> load addr 152263320Sdim+def : Pat<(i64 (atomic_load ADDRrr:$src)), (LDXrr ADDRrr:$src)>; 153263320Sdim+def : Pat<(i64 (atomic_load ADDRri:$src)), (LDXri ADDRri:$src)>; 154263320Sdim+ 155263320Sdim+// atomic_store_64 val, addr -> store val, addr 156263320Sdim+def : Pat<(atomic_store ADDRrr:$dst, i64:$val), (STXrr ADDRrr:$dst, $val)>; 157263320Sdim+def : Pat<(atomic_store ADDRri:$dst, i64:$val), (STXri ADDRri:$dst, $val)>; 158263320Sdim+ 159263320Sdim+} // Predicates = [Is64Bit] 160263320Sdim+ 161263320Sdim // Global addresses, constant pool entries 162263320Sdim let Predicates = [Is64Bit] in { 163263320Sdim 164263320SdimIndex: test/CodeGen/SPARC/atomics.ll 165263320Sdim=================================================================== 166263320Sdim--- test/CodeGen/SPARC/atomics.ll 167263320Sdim+++ test/CodeGen/SPARC/atomics.ll 168263320Sdim@@ -0,0 +1,63 @@ 169263320Sdim+; RUN: llc < %s -march=sparcv9 | FileCheck %s 170263320Sdim+ 171263320Sdim+; CHECK-LABEL: test_atomic_i32 172263320Sdim+; CHECK: ld [%o0] 173263320Sdim+; CHECK: membar 174263320Sdim+; CHECK: ld [%o1] 175263320Sdim+; CHECK: membar 176263320Sdim+; CHECK: membar 177263320Sdim+; CHECK: st {{.+}}, [%o2] 178263320Sdim+define i32 @test_atomic_i32(i32* %ptr1, i32* %ptr2, i32* %ptr3) { 179263320Sdim+entry: 180263320Sdim+ %0 = load atomic i32* %ptr1 acquire, align 8 181263320Sdim+ %1 = load atomic i32* %ptr2 acquire, align 8 182263320Sdim+ %2 = add i32 %0, %1 183263320Sdim+ store atomic i32 %2, i32* %ptr3 release, align 8 184263320Sdim+ ret i32 %2 185263320Sdim+} 186263320Sdim+ 187263320Sdim+; CHECK-LABEL: test_atomic_i64 188263320Sdim+; CHECK: ldx [%o0] 189263320Sdim+; CHECK: membar 190263320Sdim+; CHECK: ldx [%o1] 191263320Sdim+; CHECK: membar 192263320Sdim+; CHECK: membar 193263320Sdim+; CHECK: stx {{.+}}, [%o2] 194263320Sdim+define i64 @test_atomic_i64(i64* %ptr1, i64* %ptr2, i64* %ptr3) { 195263320Sdim+entry: 196263320Sdim+ %0 = load atomic i64* %ptr1 acquire, align 8 197263320Sdim+ %1 = load atomic i64* %ptr2 acquire, align 8 198263320Sdim+ %2 = add i64 %0, %1 199263320Sdim+ store atomic i64 %2, i64* %ptr3 release, align 8 200263320Sdim+ ret i64 %2 201263320Sdim+} 202263320Sdim+ 203263320Sdim+; CHECK-LABEL: test_cmpxchg_i32 204263320Sdim+; CHECK: or %g0, 123, [[R:%[gilo][0-7]]] 205263320Sdim+; CHECK: cas [%o1], %o0, [[R]] 206263320Sdim+ 207263320Sdim+define i32 @test_cmpxchg_i32(i32 %a, i32* %ptr) { 208263320Sdim+entry: 209263320Sdim+ %b = cmpxchg i32* %ptr, i32 %a, i32 123 monotonic 210263320Sdim+ ret i32 %b 211263320Sdim+} 212263320Sdim+ 213263320Sdim+; CHECK-LABEL: test_cmpxchg_i64 214263320Sdim+; CHECK: or %g0, 123, [[R:%[gilo][0-7]]] 215263320Sdim+; CHECK: casx [%o1], %o0, [[R]] 216263320Sdim+ 217263320Sdim+define i64 @test_cmpxchg_i64(i64 %a, i64* %ptr) { 218263320Sdim+entry: 219263320Sdim+ %b = cmpxchg i64* %ptr, i64 %a, i64 123 monotonic 220263320Sdim+ ret i64 %b 221263320Sdim+} 222263320Sdim+ 223263320Sdim+; CHECK-LABEL: test_swap_i32 224263320Sdim+; CHECK: or %g0, 42, [[R:%[gilo][0-7]]] 225263320Sdim+; CHECK: swap [%o1], [[R]] 226263320Sdim+ 227263320Sdim+define i32 @test_swap_i32(i32 %a, i32* %ptr) { 228263320Sdim+entry: 229263320Sdim+ %b = atomicrmw xchg i32* %ptr, i32 42 monotonic 230263320Sdim+ ret i32 %b 231263320Sdim+} 232