aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/llvm/Analysis/AliasAnalysis.h30
-rw-r--r--include/llvm/CodeGen/SelectionDAG.h16
-rw-r--r--include/llvm/CodeGen/SelectionDAGNodes.h35
-rw-r--r--include/llvm/Support/IRBuilder.h10
-rw-r--r--lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp6
-rw-r--r--lib/CodeGen/SelectionDAG/SelectionDAG.cpp28
-rw-r--r--lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp43
-rw-r--r--lib/Target/X86/X86ISelLowering.cpp4
-rw-r--r--lib/Transforms/IPO/MergeFunctions.cpp32
-rw-r--r--lib/Transforms/Scalar/LowerAtomic.cpp74
-rw-r--r--lib/VMCore/Instruction.cpp26
11 files changed, 263 insertions, 41 deletions
diff --git a/include/llvm/Analysis/AliasAnalysis.h b/include/llvm/Analysis/AliasAnalysis.h
index c86aa03..9e9c586 100644
--- a/include/llvm/Analysis/AliasAnalysis.h
+++ b/include/llvm/Analysis/AliasAnalysis.h
@@ -342,6 +342,10 @@ public:
case Instruction::Load: return getModRefInfo((const LoadInst*)I, Loc);
case Instruction::Store: return getModRefInfo((const StoreInst*)I, Loc);
case Instruction::Fence: return getModRefInfo((const FenceInst*)I, Loc);
+ case Instruction::AtomicCmpXchg:
+ return getModRefInfo((const AtomicCmpXchgInst*)I, Loc);
+ case Instruction::AtomicRMW:
+ return getModRefInfo((const AtomicRMWInst*)I, Loc);
case Instruction::Call: return getModRefInfo((const CallInst*)I, Loc);
case Instruction::Invoke: return getModRefInfo((const InvokeInst*)I,Loc);
default: return NoModRef;
@@ -420,6 +424,32 @@ public:
return getModRefInfo(S, Location(P, Size));
}
+ /// getModRefInfo (for cmpxchges) - Return whether information about whether
+ /// a particular cmpxchg modifies or reads the specified memory location.
+ ModRefResult getModRefInfo(const AtomicCmpXchgInst *CX, const Location &Loc) {
+ // Conservatively correct. (But there are obvious ways to be smarter.)
+ return ModRef;
+ }
+
+ /// getModRefInfo (for cmpxchges) - A convenience wrapper.
+ ModRefResult getModRefInfo(const AtomicCmpXchgInst *CX,
+ const Value *P, unsigned Size) {
+ return getModRefInfo(CX, Location(P, Size));
+ }
+
+ /// getModRefInfo (for atomicrmws) - Return whether information about whether
+ /// a particular atomicrmw modifies or reads the specified memory location.
+ ModRefResult getModRefInfo(const AtomicRMWInst *RMW, const Location &Loc) {
+ // Conservatively correct. (But there are obvious ways to be smarter.)
+ return ModRef;
+ }
+
+ /// getModRefInfo (for atomicrmws) - A convenience wrapper.
+ ModRefResult getModRefInfo(const AtomicRMWInst *RMW,
+ const Value *P, unsigned Size) {
+ return getModRefInfo(RMW, Location(P, Size));
+ }
+
/// getModRefInfo (for va_args) - Return whether information about whether
/// a particular va_arg modifies or reads the specified memory location.
ModRefResult getModRefInfo(const VAArgInst* I, const Location &Loc);
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index 3ec5ada..4ea3983 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -589,19 +589,27 @@ public:
/// takes 3 operands
SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
SDValue Ptr, SDValue Cmp, SDValue Swp,
- MachinePointerInfo PtrInfo, unsigned Alignment=0);
+ MachinePointerInfo PtrInfo, unsigned Alignment,
+ AtomicOrdering Ordering,
+ SynchronizationScope SynchScope);
SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
SDValue Ptr, SDValue Cmp, SDValue Swp,
- MachineMemOperand *MMO);
+ MachineMemOperand *MMO,
+ AtomicOrdering Ordering,
+ SynchronizationScope SynchScope);
/// getAtomic - Gets a node for an atomic op, produces result and chain and
/// takes 2 operands.
SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
SDValue Ptr, SDValue Val, const Value* PtrVal,
- unsigned Alignment = 0);
+ unsigned Alignment,
+ AtomicOrdering Ordering,
+ SynchronizationScope SynchScope);
SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
SDValue Ptr, SDValue Val,
- MachineMemOperand *MMO);
+ MachineMemOperand *MMO,
+ AtomicOrdering Ordering,
+ SynchronizationScope SynchScope);
/// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a
/// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index d3ce793..c95fd9c 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -20,6 +20,7 @@
#define LLVM_CODEGEN_SELECTIONDAGNODES_H
#include "llvm/Constants.h"
+#include "llvm/Instructions.h"
#include "llvm/ADT/FoldingSet.h"
#include "llvm/ADT/GraphTraits.h"
#include "llvm/ADT/ilist_node.h"
@@ -917,6 +918,13 @@ public:
bool isVolatile() const { return (SubclassData >> 5) & 1; }
bool isNonTemporal() const { return (SubclassData >> 6) & 1; }
+ AtomicOrdering getOrdering() const {
+ return AtomicOrdering((SubclassData >> 7) & 15);
+ }
+ SynchronizationScope getSynchScope() const {
+ return SynchronizationScope((SubclassData >> 11) & 1);
+ }
+
/// Returns the SrcValue and offset that describes the location of the access
const Value *getSrcValue() const { return MMO->getValue(); }
int64_t getSrcValueOffset() const { return MMO->getOffset(); }
@@ -977,6 +985,21 @@ public:
class AtomicSDNode : public MemSDNode {
SDUse Ops[4];
+ void InitAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope) {
+ // This must match encodeMemSDNodeFlags() in SelectionDAG.cpp.
+ assert((Ordering & 15) == Ordering &&
+ "Ordering may not require more than 4 bits!");
+ assert((SynchScope & 1) == SynchScope &&
+ "SynchScope may not require more than 1 bit!");
+ SubclassData |= Ordering << 7;
+ SubclassData |= SynchScope << 11;
+ assert(getOrdering() == Ordering && "Ordering encoding error!");
+ assert(getSynchScope() == SynchScope && "Synch-scope encoding error!");
+
+ assert(readMem() && "Atomic MachineMemOperand is not a load!");
+ assert(writeMem() && "Atomic MachineMemOperand is not a store!");
+ }
+
public:
// Opc: opcode for atomic
// VTL: value type list
@@ -988,18 +1011,18 @@ public:
// Align: alignment of memory
AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
SDValue Chain, SDValue Ptr,
- SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
+ SDValue Cmp, SDValue Swp, MachineMemOperand *MMO,
+ AtomicOrdering Ordering, SynchronizationScope SynchScope)
: MemSDNode(Opc, dl, VTL, MemVT, MMO) {
- assert(readMem() && "Atomic MachineMemOperand is not a load!");
- assert(writeMem() && "Atomic MachineMemOperand is not a store!");
+ InitAtomic(Ordering, SynchScope);
InitOperands(Ops, Chain, Ptr, Cmp, Swp);
}
AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
SDValue Chain, SDValue Ptr,
- SDValue Val, MachineMemOperand *MMO)
+ SDValue Val, MachineMemOperand *MMO,
+ AtomicOrdering Ordering, SynchronizationScope SynchScope)
: MemSDNode(Opc, dl, VTL, MemVT, MMO) {
- assert(readMem() && "Atomic MachineMemOperand is not a load!");
- assert(writeMem() && "Atomic MachineMemOperand is not a store!");
+ InitAtomic(Ordering, SynchScope);
InitOperands(Ops, Chain, Ptr, Val);
}
diff --git a/include/llvm/Support/IRBuilder.h b/include/llvm/Support/IRBuilder.h
index 542ee10..22bec42 100644
--- a/include/llvm/Support/IRBuilder.h
+++ b/include/llvm/Support/IRBuilder.h
@@ -770,6 +770,16 @@ public:
SynchronizationScope SynchScope = CrossThread) {
return Insert(new FenceInst(Context, Ordering, SynchScope));
}
+ FenceInst *CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New,
+ AtomicOrdering Ordering,
+ SynchronizationScope SynchScope = CrossThread){
+ return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, Ordering, SynchScope));
+ }
+ FenceInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val,
+ AtomicOrdering Ordering,
+ SynchronizationScope SynchScope = CrossThread) {
+ return Insert(new AtomicRMWInst(Op, Ptr, Val, Ordering, SynchScope));
+ }
Value *CreateGEP(Value *Ptr, ArrayRef<Value *> IdxList,
const Twine &Name = "") {
if (Constant *PC = dyn_cast<Constant>(Ptr)) {
diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 8c2a3cd..6e9992e 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -155,7 +155,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
N->getMemoryVT(),
N->getChain(), N->getBasePtr(),
- Op2, N->getMemOperand());
+ Op2, N->getMemOperand(), N->getOrdering(),
+ N->getSynchScope());
// Legalized the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
@@ -167,7 +168,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) {
SDValue Op3 = GetPromotedInteger(N->getOperand(3));
SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
N->getMemoryVT(), N->getChain(), N->getBasePtr(),
- Op2, Op3, N->getMemOperand());
+ Op2, Op3, N->getMemOperand(), N->getOrdering(),
+ N->getSynchScope());
// Legalized the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 9785b4c..888d8b4 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -3815,7 +3815,9 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst,
SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
SDValue Chain, SDValue Ptr, SDValue Cmp,
SDValue Swp, MachinePointerInfo PtrInfo,
- unsigned Alignment) {
+ unsigned Alignment,
+ AtomicOrdering Ordering,
+ SynchronizationScope SynchScope) {
if (Alignment == 0) // Ensure that codegen never sees alignment 0
Alignment = getEVTAlignment(MemVT);
@@ -3828,13 +3830,16 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
MachineMemOperand *MMO =
MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
- return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO);
+ return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO,
+ Ordering, SynchScope);
}
SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
SDValue Chain,
SDValue Ptr, SDValue Cmp,
- SDValue Swp, MachineMemOperand *MMO) {
+ SDValue Swp, MachineMemOperand *MMO,
+ AtomicOrdering Ordering,
+ SynchronizationScope SynchScope) {
assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op");
assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
@@ -3851,7 +3856,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
return SDValue(E, 0);
}
SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
- Ptr, Cmp, Swp, MMO);
+ Ptr, Cmp, Swp, MMO, Ordering,
+ SynchScope);
CSEMap.InsertNode(N, IP);
AllNodes.push_back(N);
return SDValue(N, 0);
@@ -3861,7 +3867,9 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
SDValue Chain,
SDValue Ptr, SDValue Val,
const Value* PtrVal,
- unsigned Alignment) {
+ unsigned Alignment,
+ AtomicOrdering Ordering,
+ SynchronizationScope SynchScope) {
if (Alignment == 0) // Ensure that codegen never sees alignment 0
Alignment = getEVTAlignment(MemVT);
@@ -3875,13 +3883,16 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
MemVT.getStoreSize(), Alignment);
- return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO);
+ return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
+ Ordering, SynchScope);
}
SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
SDValue Chain,
SDValue Ptr, SDValue Val,
- MachineMemOperand *MMO) {
+ MachineMemOperand *MMO,
+ AtomicOrdering Ordering,
+ SynchronizationScope SynchScope) {
assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
Opcode == ISD::ATOMIC_LOAD_SUB ||
Opcode == ISD::ATOMIC_LOAD_AND ||
@@ -3908,7 +3919,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
return SDValue(E, 0);
}
SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
- Ptr, Val, MMO);
+ Ptr, Val, MMO,
+ Ordering, SynchScope);
CSEMap.InsertNode(N, IP);
AllNodes.push_back(N);
return SDValue(N, 0);
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 498e1d3..7d45d29 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3257,9 +3257,46 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
}
void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
+ SDValue Root = getRoot();
+ SDValue L =
+ DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, getCurDebugLoc(),
+ getValue(I.getCompareOperand()).getValueType().getSimpleVT(),
+ Root,
+ getValue(I.getPointerOperand()),
+ getValue(I.getCompareOperand()),
+ getValue(I.getNewValOperand()),
+ MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */,
+ I.getOrdering(), I.getSynchScope());
+ setValue(&I, L);
+ DAG.setRoot(L.getValue(1));
}
void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
+ ISD::NodeType NT;
+ switch (I.getOperation()) {
+ default: llvm_unreachable("Unknown atomicrmw operation"); return;
+ case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
+ case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
+ case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
+ case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
+ case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
+ case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
+ case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
+ case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
+ case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
+ case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
+ case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
+ }
+ SDValue L =
+ DAG.getAtomic(NT, getCurDebugLoc(),
+ getValue(I.getValOperand()).getValueType().getSimpleVT(),
+ getRoot(),
+ getValue(I.getPointerOperand()),
+ getValue(I.getValOperand()),
+ I.getPointerOperand(), 0 /* Alignment */,
+ I.getOrdering(), I.getSynchScope());
+ setValue(&I, L);
+ DAG.setRoot(L.getValue(1));
}
void SelectionDAGBuilder::visitFence(const FenceInst &I) {
@@ -3410,7 +3447,8 @@ SelectionDAGBuilder::implVisitBinaryAtomic(const CallInst& I,
Root,
getValue(I.getArgOperand(0)),
getValue(I.getArgOperand(1)),
- I.getArgOperand(0));
+ I.getArgOperand(0), 0 /* Alignment */,
+ Monotonic, CrossThread);
setValue(&I, L);
DAG.setRoot(L.getValue(1));
return 0;
@@ -4935,7 +4973,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
getValue(I.getArgOperand(0)),
getValue(I.getArgOperand(1)),
getValue(I.getArgOperand(2)),
- MachinePointerInfo(I.getArgOperand(0)));
+ MachinePointerInfo(I.getArgOperand(0)), 0 /* Alignment */,
+ Monotonic, CrossThread);
setValue(&I, L);
DAG.setRoot(L.getValue(1));
return 0;
diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp
index 5987e57..abfc6e2 100644
--- a/lib/Target/X86/X86ISelLowering.cpp
+++ b/lib/Target/X86/X86ISelLowering.cpp
@@ -9562,7 +9562,9 @@ SDValue X86TargetLowering::LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) const {
Node->getOperand(0),
Node->getOperand(1), negOp,
cast<AtomicSDNode>(Node)->getSrcValue(),
- cast<AtomicSDNode>(Node)->getAlignment());
+ cast<AtomicSDNode>(Node)->getAlignment(),
+ cast<AtomicSDNode>(Node)->getOrdering(),
+ cast<AtomicSDNode>(Node)->getSynchScope());
}
static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
diff --git a/lib/Transforms/IPO/MergeFunctions.cpp b/lib/Transforms/IPO/MergeFunctions.cpp
index f3d7e46..e46375b 100644
--- a/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/lib/Transforms/IPO/MergeFunctions.cpp
@@ -317,22 +317,22 @@ bool FunctionComparator::isEquivalentOperation(const Instruction *I1,
if (const InvokeInst *CI = dyn_cast<InvokeInst>(I1))
return CI->getCallingConv() == cast<InvokeInst>(I2)->getCallingConv() &&
CI->getAttributes() == cast<InvokeInst>(I2)->getAttributes();
- if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(I1)) {
- if (IVI->getNumIndices() != cast<InsertValueInst>(I2)->getNumIndices())
- return false;
- for (unsigned i = 0, e = IVI->getNumIndices(); i != e; ++i)
- if (IVI->idx_begin()[i] != cast<InsertValueInst>(I2)->idx_begin()[i])
- return false;
- return true;
- }
- if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(I1)) {
- if (EVI->getNumIndices() != cast<ExtractValueInst>(I2)->getNumIndices())
- return false;
- for (unsigned i = 0, e = EVI->getNumIndices(); i != e; ++i)
- if (EVI->idx_begin()[i] != cast<ExtractValueInst>(I2)->idx_begin()[i])
- return false;
- return true;
- }
+ if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(I1))
+ return IVI->getIndices() == cast<InsertValueInst>(I2)->getIndices();
+ if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(I1))
+ return EVI->getIndices() == cast<ExtractValueInst>(I2)->getIndices();
+ if (const FenceInst *FI = dyn_cast<FenceInst>(I1))
+ return FI->getOrdering() == cast<FenceInst>(I2)->getOrdering() &&
+ FI->getSynchScope() == cast<FenceInst>(I2)->getSynchScope();
+ if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(I1))
+ return CXI->isVolatile() == cast<AtomicCmpXchgInst>(I2)->isVolatile() &&
+ CXI->getOrdering() == cast<AtomicCmpXchgInst>(I2)->getOrdering() &&
+ CXI->getSynchScope() == cast<AtomicCmpXchgInst>(I2)->getSynchScope();
+ if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I1))
+ return RMWI->getOperation() == cast<AtomicRMWInst>(I2)->getOperation() &&
+ RMWI->isVolatile() == cast<AtomicRMWInst>(I2)->isVolatile() &&
+ RMWI->getOrdering() == cast<AtomicRMWInst>(I2)->getOrdering() &&
+ RMWI->getSynchScope() == cast<AtomicRMWInst>(I2)->getSynchScope();
return true;
}
diff --git a/lib/Transforms/Scalar/LowerAtomic.cpp b/lib/Transforms/Scalar/LowerAtomic.cpp
index 0954f77..5f5ed03 100644
--- a/lib/Transforms/Scalar/LowerAtomic.cpp
+++ b/lib/Transforms/Scalar/LowerAtomic.cpp
@@ -115,6 +115,76 @@ static bool LowerAtomicIntrinsic(IntrinsicInst *II) {
return true;
}
+bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
+ IRBuilder<> Builder(CXI->getParent(), CXI);
+ Value *Ptr = CXI->getPointerOperand();
+ Value *Cmp = CXI->getCompareOperand();
+ Value *Val = CXI->getNewValOperand();
+
+ LoadInst *Orig = Builder.CreateLoad(Ptr);
+ Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
+ Value *Res = Builder.CreateSelect(Equal, Val, Orig);
+ Builder.CreateStore(Res, Ptr);
+
+ CXI->replaceAllUsesWith(Orig);
+ CXI->eraseFromParent();
+ return true;
+}
+
+bool LowerAtomicRMWInst(AtomicRMWInst *RMWI) {
+ IRBuilder<> Builder(RMWI->getParent(), RMWI);
+ Value *Ptr = RMWI->getPointerOperand();
+ Value *Val = RMWI->getValOperand();
+
+ LoadInst *Orig = Builder.CreateLoad(Ptr);
+ Value *Res = NULL;
+
+ switch (RMWI->getOperation()) {
+ default: llvm_unreachable("Unexpected RMW operation");
+ case AtomicRMWInst::Xchg:
+ Res = Val;
+ break;
+ case AtomicRMWInst::Add:
+ Res = Builder.CreateAdd(Orig, Val);
+ break;
+ case AtomicRMWInst::Sub:
+ Res = Builder.CreateSub(Orig, Val);
+ break;
+ case AtomicRMWInst::And:
+ Res = Builder.CreateAnd(Orig, Val);
+ break;
+ case AtomicRMWInst::Nand:
+ Res = Builder.CreateNot(Builder.CreateAnd(Orig, Val));
+ break;
+ case AtomicRMWInst::Or:
+ Res = Builder.CreateOr(Orig, Val);
+ break;
+ case AtomicRMWInst::Xor:
+ Res = Builder.CreateXor(Orig, Val);
+ break;
+ case AtomicRMWInst::Max:
+ Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
+ Val, Orig);
+ break;
+ case AtomicRMWInst::Min:
+ Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
+ Orig, Val);
+ break;
+ case AtomicRMWInst::UMax:
+ Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
+ Val, Orig);
+ break;
+ case AtomicRMWInst::UMin:
+ Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
+ Orig, Val);
+ break;
+ }
+ Builder.CreateStore(Res, Ptr);
+ RMWI->replaceAllUsesWith(Orig);
+ RMWI->eraseFromParent();
+ return true;
+}
+
static bool LowerFenceInst(FenceInst *FI) {
FI->eraseFromParent();
return true;
@@ -134,6 +204,10 @@ namespace {
Changed |= LowerAtomicIntrinsic(II);
else if (FenceInst *FI = dyn_cast<FenceInst>(Inst))
Changed |= LowerFenceInst(FI);
+ else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(Inst))
+ Changed |= LowerAtomicCmpXchgInst(CXI);
+ else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Inst))
+ Changed |= LowerAtomicRMWInst(RMWI);
}
return Changed;
}
diff --git a/lib/VMCore/Instruction.cpp b/lib/VMCore/Instruction.cpp
index ad433ef..eadcd09 100644
--- a/lib/VMCore/Instruction.cpp
+++ b/lib/VMCore/Instruction.cpp
@@ -216,6 +216,15 @@ bool Instruction::isIdenticalToWhenDefined(const Instruction *I) const {
if (const FenceInst *FI = dyn_cast<FenceInst>(this))
return FI->getOrdering() == cast<FenceInst>(FI)->getOrdering() &&
FI->getSynchScope() == cast<FenceInst>(FI)->getSynchScope();
+ if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(this))
+ return CXI->isVolatile() == cast<AtomicCmpXchgInst>(I)->isVolatile() &&
+ CXI->getOrdering() == cast<AtomicCmpXchgInst>(I)->getOrdering() &&
+ CXI->getSynchScope() == cast<AtomicCmpXchgInst>(I)->getSynchScope();
+ if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(this))
+ return RMWI->getOperation() == cast<AtomicRMWInst>(I)->getOperation() &&
+ RMWI->isVolatile() == cast<AtomicRMWInst>(I)->isVolatile() &&
+ RMWI->getOrdering() == cast<AtomicRMWInst>(I)->getOrdering() &&
+ RMWI->getSynchScope() == cast<AtomicRMWInst>(I)->getSynchScope();
return true;
}
@@ -257,8 +266,17 @@ bool Instruction::isSameOperationAs(const Instruction *I) const {
if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(this))
return EVI->getIndices() == cast<ExtractValueInst>(I)->getIndices();
if (const FenceInst *FI = dyn_cast<FenceInst>(this))
- return FI->getOrdering() == cast<FenceInst>(FI)->getOrdering() &&
- FI->getSynchScope() == cast<FenceInst>(FI)->getSynchScope();
+ return FI->getOrdering() == cast<FenceInst>(I)->getOrdering() &&
+ FI->getSynchScope() == cast<FenceInst>(I)->getSynchScope();
+ if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(this))
+ return CXI->isVolatile() == cast<AtomicCmpXchgInst>(I)->isVolatile() &&
+ CXI->getOrdering() == cast<AtomicCmpXchgInst>(I)->getOrdering() &&
+ CXI->getSynchScope() == cast<AtomicCmpXchgInst>(I)->getSynchScope();
+ if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(this))
+ return RMWI->getOperation() == cast<AtomicRMWInst>(I)->getOperation() &&
+ RMWI->isVolatile() == cast<AtomicRMWInst>(I)->isVolatile() &&
+ RMWI->getOrdering() == cast<AtomicRMWInst>(I)->getOrdering() &&
+ RMWI->getSynchScope() == cast<AtomicRMWInst>(I)->getSynchScope();
return true;
}
@@ -292,6 +310,8 @@ bool Instruction::mayReadFromMemory() const {
case Instruction::VAArg:
case Instruction::Load:
case Instruction::Fence: // FIXME: refine definition of mayReadFromMemory
+ case Instruction::AtomicCmpXchg:
+ case Instruction::AtomicRMW:
return true;
case Instruction::Call:
return !cast<CallInst>(this)->doesNotAccessMemory();
@@ -310,6 +330,8 @@ bool Instruction::mayWriteToMemory() const {
case Instruction::Fence: // FIXME: refine definition of mayWriteToMemory
case Instruction::Store:
case Instruction::VAArg:
+ case Instruction::AtomicCmpXchg:
+ case Instruction::AtomicRMW:
return true;
case Instruction::Call:
return !cast<CallInst>(this)->onlyReadsMemory();
generated by cgit v1.1 at 2024-06-18 20:51:51 +0000