64-bit atomic operations.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@49949 91177308-0d34-0410-b5e6-96231b3b80d8

6 files changed, 128 insertions, 56 deletions

diff --git a/lib/Target/PowerPC/PPCISelLowering.cpp b/lib/Target/PowerPC/PPCISelLowering.cpp
index 1d3bf22..09779be 100644
--- a/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -206,6 +206,11 @@ PPCTargetLowering::PPCTargetLowering(PPCTargetMachine &TM)
   setOperationAction(ISD::ATOMIC_LAS        , MVT::i32  , Custom);
   setOperationAction(ISD::ATOMIC_LCS        , MVT::i32  , Custom);
   setOperationAction(ISD::ATOMIC_SWAP       , MVT::i32  , Custom);
+  if (TM.getSubtarget<PPCSubtarget>().has64BitSupport()) {
+    setOperationAction(ISD::ATOMIC_LAS      , MVT::i64  , Custom);
+    setOperationAction(ISD::ATOMIC_LCS      , MVT::i64  , Custom);
+    setOperationAction(ISD::ATOMIC_SWAP     , MVT::i64  , Custom);
+  }
 
   // We want to custom lower some of our intrinsics.
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
@@ -398,8 +403,8 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::VCMPo:         return "PPCISD::VCMPo";
   case PPCISD::LBRX:          return "PPCISD::LBRX";
   case PPCISD::STBRX:         return "PPCISD::STBRX";
-  case PPCISD::LWARX:         return "PPCISD::LWARX";
-  case PPCISD::STWCX:         return "PPCISD::STWCX";
+  case PPCISD::LARX:          return "PPCISD::LARX";
+  case PPCISD::STCX:          return "PPCISD::STCX";
   case PPCISD::CMP_UNRESERVE: return "PPCISD::CMP_UNRESERVE";
   case PPCISD::COND_BRANCH:   return "PPCISD::COND_BRANCH";
   case PPCISD::MFFS:          return "PPCISD::MFFS";
@@ -2304,7 +2309,7 @@ SDOperand PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDOperand Op,
 }
 
 SDOperand PPCTargetLowering::LowerAtomicLAS(SDOperand Op, SelectionDAG &DAG) {
-  MVT::ValueType VT = Op.getValueType();
+  MVT::ValueType VT = Op.Val->getValueType(0);
   SDOperand Chain   = Op.getOperand(0);
   SDOperand Ptr     = Op.getOperand(1);
   SDOperand Incr    = Op.getOperand(2);
@@ -2316,11 +2321,11 @@ SDOperand PPCTargetLowering::LowerAtomicLAS(SDOperand Op, SelectionDAG &DAG) {
 
   SDOperand Label  = DAG.getConstant(PPCAtomicLabelIndex++, MVT::i32);
   SDOperand Ops[] = {
-    Chain,  // Chain
-    Ptr,    // Ptr
-    Label,  // Label
+    Chain,               // Chain
+    Ptr,                 // Ptr
+    Label,               // Label
   };
-  SDOperand Load = DAG.getNode(PPCISD::LWARX, VTs, Ops, 3);
+  SDOperand Load = DAG.getNode(PPCISD::LARX, VTs, Ops, 3);
   Chain = Load.getValue(1);
 
   // Compute new value.
@@ -2328,19 +2333,19 @@ SDOperand PPCTargetLowering::LowerAtomicLAS(SDOperand Op, SelectionDAG &DAG) {
 
   // Issue a "store and check".
   SDOperand Ops2[] = {
-    Chain,  // Chain
-    NewVal, // Value
-    Ptr,    // Ptr
-    Label,  // Label
+    Chain,               // Chain
+    NewVal,              // Value
+    Ptr,                 // Ptr
+    Label,               // Label
   };
-  SDOperand Store = DAG.getNode(PPCISD::STWCX, MVT::Other, Ops2, 4);
+  SDOperand Store = DAG.getNode(PPCISD::STCX, MVT::Other, Ops2, 4);
   SDOperand OutOps[] = { Load, Store };
   return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
                      OutOps, 2);
 }
 
 SDOperand PPCTargetLowering::LowerAtomicLCS(SDOperand Op, SelectionDAG &DAG) {
-  MVT::ValueType VT = Op.getValueType();
+  MVT::ValueType VT = Op.Val->getValueType(0);
   SDOperand Chain   = Op.getOperand(0);
   SDOperand Ptr     = Op.getOperand(1);
   SDOperand NewVal  = Op.getOperand(2);
@@ -2353,37 +2358,37 @@ SDOperand PPCTargetLowering::LowerAtomicLCS(SDOperand Op, SelectionDAG &DAG) {
 
   SDOperand Label  = DAG.getConstant(PPCAtomicLabelIndex++, MVT::i32);
   SDOperand Ops[] = {
-    Chain,  // Chain
-    Ptr,    // Ptr
-    Label,  // Label
+    Chain,               // Chain
+    Ptr,                 // Ptr
+    Label,               // Label
   };
-  SDOperand Load = DAG.getNode(PPCISD::LWARX, VTs, Ops, 3);
+  SDOperand Load = DAG.getNode(PPCISD::LARX, VTs, Ops, 3);
   Chain = Load.getValue(1);
 
   // Compare and unreserve if not equal.
   SDOperand Ops2[] = {
-    Chain,  // Chain
-    OldVal, // Old value
-    Load,   // Value in memory
-    Label,  // Label
+    Chain,               // Chain
+    OldVal,              // Old value
+    Load,                // Value in memory
+    Label,               // Label
   };
   Chain = DAG.getNode(PPCISD::CMP_UNRESERVE, MVT::Other, Ops2, 4);
 
   // Issue a "store and check".
   SDOperand Ops3[] = {
-    Chain,  // Chain
-    NewVal, // Value
-    Ptr,    // Ptr
-    Label,  // Label
+    Chain,               // Chain
+    NewVal,              // Value
+    Ptr,                 // Ptr
+    Label,               // Label
   };
-  SDOperand Store = DAG.getNode(PPCISD::STWCX, MVT::Other, Ops3, 4);
+  SDOperand Store = DAG.getNode(PPCISD::STCX, MVT::Other, Ops3, 4);
   SDOperand OutOps[] = { Load, Store };
   return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
                      OutOps, 2);
 }
 
 SDOperand PPCTargetLowering::LowerAtomicSWAP(SDOperand Op, SelectionDAG &DAG) {
-  MVT::ValueType VT = Op.getValueType();
+  MVT::ValueType VT = Op.Val->getValueType(0);
   SDOperand Chain   = Op.getOperand(0);
   SDOperand Ptr     = Op.getOperand(1);
   SDOperand NewVal  = Op.getOperand(2);
@@ -2395,21 +2400,21 @@ SDOperand PPCTargetLowering::LowerAtomicSWAP(SDOperand Op, SelectionDAG &DAG) {
 
   SDOperand Label  = DAG.getConstant(PPCAtomicLabelIndex++, MVT::i32);
   SDOperand Ops[] = {
-    Chain,  // Chain
-    Ptr,    // Ptr
-    Label,  // Label
+    Chain,               // Chain
+    Ptr,                 // Ptr
+    Label,               // Label
   };
-  SDOperand Load = DAG.getNode(PPCISD::LWARX, VTs, Ops, 3);
+  SDOperand Load = DAG.getNode(PPCISD::LARX, VTs, Ops, 3);
   Chain = Load.getValue(1);
 
   // Issue a "store and check".
   SDOperand Ops2[] = {
-    Chain,  // Chain
-    NewVal, // Value
-    Ptr,    // Ptr
-    Label,  // Label
+    Chain,               // Chain
+    NewVal,              // Value
+    Ptr,                 // Ptr
+    Label,               // Label
   };
-  SDOperand Store = DAG.getNode(PPCISD::STWCX, MVT::Other, Ops2, 4);
+  SDOperand Store = DAG.getNode(PPCISD::STCX, MVT::Other, Ops2, 4);
   SDOperand OutOps[] = { Load, Store };
   return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
                      OutOps, 2);
diff --git a/lib/Target/PowerPC/PPCISelLowering.h b/lib/Target/PowerPC/PPCISelLowering.h
index d9ced1b..bd13baf 100644
--- a/lib/Target/PowerPC/PPCISelLowering.h
+++ b/lib/Target/PowerPC/PPCISelLowering.h
@@ -152,13 +152,13 @@ namespace llvm {
       /// MTFSF = F8RC, INFLAG - This moves the register into the FPSCR.
       MTFSF,
 
-      /// LWARX = This corresponds to PPC lwarx instrcution: load word and
+      /// LARX = This corresponds to PPC l{w|d}arx instrcution: load and
       /// reserve indexed. This is used to implement atomic operations.
-      LWARX,
+      LARX,
 
-      /// STWCX = This corresponds to PPC stwcx. instrcution: store word
-      /// conditional indexed. This is used to implement atomic operations.
-      STWCX,
+      /// STCX = This corresponds to PPC stcx. instrcution: store conditional
+      /// indexed. This is used to implement atomic operations.
+      STCX,
 
       /// CMP_UNRESERVE = Test for equality and "unreserve" if not true. This
       /// is used to implement atomic operations.
diff --git a/lib/Target/PowerPC/PPCInstr64Bit.td b/lib/Target/PowerPC/PPCInstr64Bit.td
index 0b7d1cc..0b23c67 100644
--- a/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -116,6 +116,25 @@ def : Pat<(PPCcall_ELF (i64 tglobaladdr:$dst)),
 def : Pat<(PPCcall_ELF (i64 texternalsym:$dst)),
           (BL8_ELF texternalsym:$dst)>;
 
+
+// Atomic operations.
+def LDARX : Pseudo<(outs G8RC:$rD), (ins memrr:$ptr, i32imm:$label),
+                   "\nLa${label}_entry:\n\tldarx $rD, $ptr",
+                   [(set G8RC:$rD, (PPClarx xoaddr:$ptr, imm:$label))]>;
+
+let Defs = [CR0] in {
+def STDCX : Pseudo<(outs), (ins G8RC:$rS, memrr:$dst, i32imm:$label),
+                  "stdcx. $rS, $dst\n\tbne- La${label}_entry\nLa${label}_exit:",
+                   [(PPCstcx G8RC:$rS, xoaddr:$dst, imm:$label)]>;
+
+def CMP_UNRESd : Pseudo<(outs), (ins G8RC:$rA, G8RC:$rB, i32imm:$label),
+                         "cmpd $rA, $rB\n\tbne- La${label}_exit",
+                         [(PPCcmp_unres G8RC:$rA, G8RC:$rB, imm:$label)]>;
+def CMP_UNRESdi : Pseudo<(outs), (ins G8RC:$rA, s16imm64:$imm, i32imm:$label),
+                         "cmpdi $rA, $imm\n\tbne- La${label}_exit",
+                         [(PPCcmp_unres G8RC:$rA, immSExt16:$imm, imm:$label)]>;
+}
+
 //===----------------------------------------------------------------------===//
 // 64-bit SPR manipulation instrs.
 
diff --git a/lib/Target/PowerPC/PPCInstrInfo.td b/lib/Target/PowerPC/PPCInstrInfo.td
index 7cbdac3..a765494 100644
--- a/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/lib/Target/PowerPC/PPCInstrInfo.td
@@ -42,14 +42,14 @@ def SDT_PPCstbrx : SDTypeProfile<0, 4, [
   SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisVT<2, OtherVT>, SDTCisVT<3, OtherVT>
 ]>;
 
-def SDT_PPClwarx : SDTypeProfile<1, 2, [
-  SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisVT<2, i32>
+def SDT_PPClarx : SDTypeProfile<1, 2, [
+  SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32>
 ]>;
-def SDT_PPCstwcx : SDTypeProfile<0, 3, [
-  SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisVT<2, i32>
+def SDT_PPCstcx : SDTypeProfile<0, 3, [
+  SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32>
 ]>;
 def SDT_PPCcmp_unres : SDTypeProfile<0, 3, [
-  SDTCisVT<0, i32>, SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>
+  SDTCisSameAs<0, 1>, SDTCisInt<1>, SDTCisVT<2, i32>
 ]>;
 
 //===----------------------------------------------------------------------===//
@@ -132,10 +132,10 @@ def PPClbrx       : SDNode<"PPCISD::LBRX", SDT_PPClbrx,
 def PPCstbrx      : SDNode<"PPCISD::STBRX", SDT_PPCstbrx,
                            [SDNPHasChain, SDNPMayStore]>;
 
-def PPClwarx      : SDNode<"PPCISD::LWARX", SDT_PPClwarx,
-                           [SDNPHasChain, SDNPMayLoad]>;
-def PPCstwcx      : SDNode<"PPCISD::STWCX", SDT_PPCstwcx,
-                           [SDNPHasChain, SDNPMayStore]>;
+def PPClarx      : SDNode<"PPCISD::LARX", SDT_PPClarx,
+                          [SDNPHasChain, SDNPMayLoad]>;
+def PPCstcx      : SDNode<"PPCISD::STCX", SDT_PPCstcx,
+                          [SDNPHasChain, SDNPMayStore]>;
 def PPCcmp_unres  : SDNode<"PPCISD::CMP_UNRESERVE", SDT_PPCcmp_unres,
                            [SDNPHasChain]>;
 
@@ -482,19 +482,19 @@ def DCBZL  : DCB_Form<1014, 1, (outs), (ins memrr:$dst),
 // Atomic operations.
 def LWARX : Pseudo<(outs GPRC:$rD), (ins memrr:$ptr, i32imm:$label),
                    "\nLa${label}_entry:\n\tlwarx $rD, $ptr",
-                   [(set GPRC:$rD, (PPClwarx xoaddr:$ptr, imm:$label))]>;
+                   [(set GPRC:$rD, (PPClarx xoaddr:$ptr, imm:$label))]>;
 
 let Defs = [CR0] in {
 def STWCX : Pseudo<(outs), (ins GPRC:$rS, memrr:$dst, i32imm:$label),
-                   "stwcx. $rS, $dst\n\tbne- La${label}_entry\nLa${label}_exit:",
-                   [(PPCstwcx GPRC:$rS, xoaddr:$dst, imm:$label)]>;
+                  "stwcx. $rS, $dst\n\tbne- La${label}_entry\nLa${label}_exit:",
+                   [(PPCstcx GPRC:$rS, xoaddr:$dst, imm:$label)]>;
 
 def CMP_UNRESw : Pseudo<(outs), (ins GPRC:$rA, GPRC:$rB, i32imm:$label),
                          "cmpw $rA, $rB\n\tbne- La${label}_exit",
                          [(PPCcmp_unres GPRC:$rA, GPRC:$rB, imm:$label)]>;
 def CMP_UNRESwi : Pseudo<(outs), (ins GPRC:$rA, s16imm:$imm, i32imm:$label),
                          "cmpwi $rA, $imm\n\tbne- La${label}_exit",
-                         [(PPCcmp_unres GPRC:$rA, imm:$imm, imm:$label)]>;
+                         [(PPCcmp_unres GPRC:$rA, immSExt16:$imm, imm:$label)]>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1265,8 +1265,8 @@ def : Pat<(extloadf32 xaddr:$src),
           (FMRSD (LFSX xaddr:$src))>;
 
 // Atomic operations
-def : Pat<(PPCcmp_unres imm:$imm, GPRC:$rA, imm:$label),
-          (CMP_UNRESwi GPRC:$rA, imm:$imm, imm:$label)>;
+def : Pat<(PPCcmp_unres immSExt16:$imm, GPRC:$rA, imm:$label),
+          (CMP_UNRESwi GPRC:$rA, immSExt16:$imm, imm:$label)>;
 
 include "PPCInstrAltivec.td"
 include "PPCInstr64Bit.td"
diff --git a/test/CodeGen/PowerPC/atomic-1.ll b/test/CodeGen/PowerPC/atomic-1.ll
new file mode 100644
index 0000000..74b9c6a
--- /dev/null
+++ b/test/CodeGen/PowerPC/atomic-1.ll
@@ -0,0 +1,27 @@
+; RUN: llvm-as < %s | llc -march=ppc32 | grep lwarx  | count 4
+; RUN: llvm-as < %s | llc -march=ppc32 | grep stwcx. | count 4
+
+define i32 @exchange_and_add(i32* %mem, i32 %val) nounwind  {
+	%tmp = call i32 @llvm.atomic.las.i32( i32* %mem, i32 %val )
+	ret i32 %tmp
+}
+
+define i32 @exchange_and_cmp(i32* %mem) nounwind  {
+       	%tmp = call i32 @llvm.atomic.lcs.i32( i32* %mem, i32 0, i32 1 )
+	ret i32 %tmp
+}
+
+define i16 @exchange_and_cmp16(i16* %mem) nounwind  {
+	%tmp = call i16 @llvm.atomic.lcs.i16( i16* %mem, i16 0, i16 1 )
+	ret i16 %tmp
+}
+
+define i32 @exchange(i32* %mem, i32 %val) nounwind  {
+	%tmp = call i32 @llvm.atomic.swap.i32( i32* %mem, i32 1 )
+	ret i32 %tmp
+}
+
+declare i32 @llvm.atomic.las.i32(i32*, i32) nounwind 
+declare i32 @llvm.atomic.lcs.i32(i32*, i32, i32) nounwind 
+declare i16 @llvm.atomic.lcs.i16(i16*, i16, i16) nounwind 
+declare i32 @llvm.atomic.swap.i32(i32*, i32) nounwind 
diff --git a/test/CodeGen/PowerPC/atomic-2.ll b/test/CodeGen/PowerPC/atomic-2.ll
new file mode 100644
index 0000000..8e1ef1b
--- /dev/null
+++ b/test/CodeGen/PowerPC/atomic-2.ll
@@ -0,0 +1,21 @@
+; RUN: llvm-as < %s | llc -march=ppc64 | grep ldarx  | count 3
+; RUN: llvm-as < %s | llc -march=ppc64 | grep stdcx. | count 3
+
+define i64 @exchange_and_add(i64* %mem, i64 %val) nounwind  {
+	%tmp = call i64 @llvm.atomic.las.i64( i64* %mem, i64 %val )
+	ret i64 %tmp
+}
+
+define i64 @exchange_and_cmp(i64* %mem) nounwind  {
+       	%tmp = call i64 @llvm.atomic.lcs.i64( i64* %mem, i64 0, i64 1 )
+	ret i64 %tmp
+}
+
+define i64 @exchange(i64* %mem, i64 %val) nounwind  {
+	%tmp = call i64 @llvm.atomic.swap.i64( i64* %mem, i64 1 )
+	ret i64 %tmp
+}
+
+declare i64 @llvm.atomic.las.i64(i64*, i64) nounwind 
+declare i64 @llvm.atomic.lcs.i64(i64*, i64, i64) nounwind 
+declare i64 @llvm.atomic.swap.i64(i64*, i64) nounwind