diff options
author | Stephen Hines <srhines@google.com> | 2014-12-01 14:51:49 -0800 |
---|---|---|
committer | Stephen Hines <srhines@google.com> | 2014-12-02 16:08:10 -0800 |
commit | 37ed9c199ca639565f6ce88105f9e39e898d82d0 (patch) | |
tree | 8fb36d3910e3ee4c4e1b7422f4f017108efc52f5 /lib/ExecutionEngine | |
parent | d2327b22152ced7bc46dc629fc908959e8a52d03 (diff) | |
download | external_llvm-37ed9c199ca639565f6ce88105f9e39e898d82d0.zip external_llvm-37ed9c199ca639565f6ce88105f9e39e898d82d0.tar.gz external_llvm-37ed9c199ca639565f6ce88105f9e39e898d82d0.tar.bz2 |
Update aosp/master LLVM for rebase to r222494.
Change-Id: Ic787f5e0124df789bd26f3f24680f45e678eef2d
Diffstat (limited to 'lib/ExecutionEngine')
40 files changed, 2862 insertions, 5229 deletions
diff --git a/lib/ExecutionEngine/CMakeLists.txt b/lib/ExecutionEngine/CMakeLists.txt index 3102c7b..fae5bb9 100644 --- a/lib/ExecutionEngine/CMakeLists.txt +++ b/lib/ExecutionEngine/CMakeLists.txt @@ -3,12 +3,12 @@ add_llvm_library(LLVMExecutionEngine ExecutionEngine.cpp ExecutionEngineBindings.cpp + JITEventListener.cpp RTDyldMemoryManager.cpp TargetSelect.cpp ) add_subdirectory(Interpreter) -add_subdirectory(JIT) add_subdirectory(MCJIT) add_subdirectory(RuntimeDyld) diff --git a/lib/ExecutionEngine/ExecutionEngine.cpp b/lib/ExecutionEngine/ExecutionEngine.cpp index b0e985d..5a6d656 100644 --- a/lib/ExecutionEngine/ExecutionEngine.cpp +++ b/lib/ExecutionEngine/ExecutionEngine.cpp @@ -16,7 +16,7 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/Statistic.h" #include "llvm/ExecutionEngine/GenericValue.h" -#include "llvm/ExecutionEngine/JITMemoryManager.h" +#include "llvm/ExecutionEngine/ObjectBuffer.h" #include "llvm/ExecutionEngine/ObjectCache.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" @@ -24,6 +24,7 @@ #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/Object/Archive.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Debug.h" #include "llvm/Support/DynamicLibrary.h" @@ -47,22 +48,13 @@ void ObjectCache::anchor() {} void ObjectBuffer::anchor() {} void ObjectBufferStream::anchor() {} -ExecutionEngine *(*ExecutionEngine::JITCtor)( - Module *M, - std::string *ErrorStr, - JITMemoryManager *JMM, - bool GVsWithCode, - TargetMachine *TM) = nullptr; ExecutionEngine *(*ExecutionEngine::MCJITCtor)( - Module *M, - std::string *ErrorStr, - RTDyldMemoryManager *MCJMM, - bool GVsWithCode, - TargetMachine *TM) = nullptr; -ExecutionEngine *(*ExecutionEngine::InterpCtor)(Module *M, + std::unique_ptr<Module> M, std::string *ErrorStr, + RTDyldMemoryManager *MCJMM, std::unique_ptr<TargetMachine> TM) = nullptr; +ExecutionEngine *(*ExecutionEngine::InterpCtor)(std::unique_ptr<Module> M, std::string *ErrorStr) =nullptr; -ExecutionEngine::ExecutionEngine(Module *M) +ExecutionEngine::ExecutionEngine(std::unique_ptr<Module> M) : EEState(*this), LazyFunctionCreator(nullptr) { CompilingLazily = false; @@ -77,14 +69,12 @@ ExecutionEngine::ExecutionEngine(Module *M) VerifyModules = false; #endif - Modules.push_back(M); assert(M && "Module is null?"); + Modules.push_back(std::move(M)); } ExecutionEngine::~ExecutionEngine() { clearAllGlobalMappings(); - for (unsigned i = 0, e = Modules.size(); i != e; ++i) - delete Modules[i]; } namespace { @@ -101,8 +91,8 @@ public: Type *ElTy = GV->getType()->getElementType(); size_t GVSize = (size_t)TD.getTypeAllocSize(ElTy); void *RawMemory = ::operator new( - DataLayout::RoundUpAlignment(sizeof(GVMemoryBlock), - TD.getPreferredAlignment(GV)) + RoundUpToAlignment(sizeof(GVMemoryBlock), + TD.getPreferredAlignment(GV)) + GVSize); new(RawMemory) GVMemoryBlock(GV); return static_cast<char*>(RawMemory) + sizeof(GVMemoryBlock); @@ -126,11 +116,20 @@ void ExecutionEngine::addObjectFile(std::unique_ptr<object::ObjectFile> O) { llvm_unreachable("ExecutionEngine subclass doesn't implement addObjectFile."); } +void +ExecutionEngine::addObjectFile(object::OwningBinary<object::ObjectFile> O) { + llvm_unreachable("ExecutionEngine subclass doesn't implement addObjectFile."); +} + +void ExecutionEngine::addArchive(object::OwningBinary<object::Archive> A) { + llvm_unreachable("ExecutionEngine subclass doesn't implement addArchive."); +} + bool ExecutionEngine::removeModule(Module *M) { - for(SmallVectorImpl<Module *>::iterator I = Modules.begin(), - E = Modules.end(); I != E; ++I) { - Module *Found = *I; + for (auto I = Modules.begin(), E = Modules.end(); I != E; ++I) { + Module *Found = I->get(); if (Found == M) { + I->release(); Modules.erase(I); clearGlobalMappingsFromModule(M); return true; @@ -254,19 +253,9 @@ const GlobalValue *ExecutionEngine::getGlobalValueAtAddress(void *Addr) { namespace { class ArgvArray { - char *Array; - std::vector<char*> Values; + std::unique_ptr<char[]> Array; + std::vector<std::unique_ptr<char[]>> Values; public: - ArgvArray() : Array(nullptr) {} - ~ArgvArray() { clear(); } - void clear() { - delete[] Array; - Array = nullptr; - for (size_t I = 0, E = Values.size(); I != E; ++I) { - delete[] Values[I]; - } - Values.clear(); - } /// Turn a vector of strings into a nice argv style array of pointers to null /// terminated strings. void *reset(LLVMContext &C, ExecutionEngine *EE, @@ -275,38 +264,39 @@ public: } // anonymous namespace void *ArgvArray::reset(LLVMContext &C, ExecutionEngine *EE, const std::vector<std::string> &InputArgv) { - clear(); // Free the old contents. + Values.clear(); // Free the old contents. + Values.reserve(InputArgv.size()); unsigned PtrSize = EE->getDataLayout()->getPointerSize(); - Array = new char[(InputArgv.size()+1)*PtrSize]; + Array = make_unique<char[]>((InputArgv.size()+1)*PtrSize); - DEBUG(dbgs() << "JIT: ARGV = " << (void*)Array << "\n"); + DEBUG(dbgs() << "JIT: ARGV = " << (void*)Array.get() << "\n"); Type *SBytePtr = Type::getInt8PtrTy(C); for (unsigned i = 0; i != InputArgv.size(); ++i) { unsigned Size = InputArgv[i].size()+1; - char *Dest = new char[Size]; - Values.push_back(Dest); - DEBUG(dbgs() << "JIT: ARGV[" << i << "] = " << (void*)Dest << "\n"); + auto Dest = make_unique<char[]>(Size); + DEBUG(dbgs() << "JIT: ARGV[" << i << "] = " << (void*)Dest.get() << "\n"); - std::copy(InputArgv[i].begin(), InputArgv[i].end(), Dest); + std::copy(InputArgv[i].begin(), InputArgv[i].end(), Dest.get()); Dest[Size-1] = 0; // Endian safe: Array[i] = (PointerTy)Dest; - EE->StoreValueToMemory(PTOGV(Dest), (GenericValue*)(Array+i*PtrSize), - SBytePtr); + EE->StoreValueToMemory(PTOGV(Dest.get()), + (GenericValue*)(&Array[i*PtrSize]), SBytePtr); + Values.push_back(std::move(Dest)); } // Null terminate it EE->StoreValueToMemory(PTOGV(nullptr), - (GenericValue*)(Array+InputArgv.size()*PtrSize), + (GenericValue*)(&Array[InputArgv.size()*PtrSize]), SBytePtr); - return Array; + return Array.get(); } -void ExecutionEngine::runStaticConstructorsDestructors(Module *module, +void ExecutionEngine::runStaticConstructorsDestructors(Module &module, bool isDtors) { const char *Name = isDtors ? "llvm.global_dtors" : "llvm.global_ctors"; - GlobalVariable *GV = module->getNamedGlobal(Name); + GlobalVariable *GV = module.getNamedGlobal(Name); // If this global has internal linkage, or if it has a use, then it must be // an old-style (llvmgcc3) static ctor with __main linked in and in use. If @@ -344,8 +334,8 @@ void ExecutionEngine::runStaticConstructorsDestructors(Module *module, void ExecutionEngine::runStaticConstructorsDestructors(bool isDtors) { // Execute global ctors/dtors for each module in the program. - for (unsigned i = 0, e = Modules.size(); i != e; ++i) - runStaticConstructorsDestructors(Modules[i], isDtors); + for (std::unique_ptr<Module> &M : Modules) + runStaticConstructorsDestructors(*M, isDtors); } #ifndef NDEBUG @@ -406,68 +396,14 @@ int ExecutionEngine::runFunctionAsMain(Function *Fn, return runFunction(Fn, GVArgs).IntVal.getZExtValue(); } -ExecutionEngine *ExecutionEngine::create(Module *M, - bool ForceInterpreter, - std::string *ErrorStr, - CodeGenOpt::Level OptLevel, - bool GVsWithCode) { - - EngineBuilder EB = - EngineBuilder(M) - .setEngineKind(ForceInterpreter ? EngineKind::Interpreter - : EngineKind::Either) - .setErrorStr(ErrorStr) - .setOptLevel(OptLevel) - .setAllocateGVsWithCode(GVsWithCode); - - return EB.create(); -} - -/// createJIT - This is the factory method for creating a JIT for the current -/// machine, it does not fall back to the interpreter. This takes ownership -/// of the module. -ExecutionEngine *ExecutionEngine::createJIT(Module *M, - std::string *ErrorStr, - JITMemoryManager *JMM, - CodeGenOpt::Level OL, - bool GVsWithCode, - Reloc::Model RM, - CodeModel::Model CMM) { - if (!ExecutionEngine::JITCtor) { - if (ErrorStr) - *ErrorStr = "JIT has not been linked in."; - return nullptr; - } - - // Use the defaults for extra parameters. Users can use EngineBuilder to - // set them. - EngineBuilder EB(M); - EB.setEngineKind(EngineKind::JIT); - EB.setErrorStr(ErrorStr); - EB.setRelocationModel(RM); - EB.setCodeModel(CMM); - EB.setAllocateGVsWithCode(GVsWithCode); - EB.setOptLevel(OL); - EB.setJITMemoryManager(JMM); - - // TODO: permit custom TargetOptions here - TargetMachine *TM = EB.selectTarget(); - if (!TM || (ErrorStr && ErrorStr->length() > 0)) return nullptr; - - return ExecutionEngine::JITCtor(M, ErrorStr, JMM, GVsWithCode, TM); -} - void EngineBuilder::InitEngine() { WhichEngine = EngineKind::Either; ErrorStr = nullptr; OptLevel = CodeGenOpt::Default; MCJMM = nullptr; - JMM = nullptr; Options = TargetOptions(); - AllocateGVsWithCode = false; RelocModel = Reloc::Default; CMModel = CodeModel::JITDefault; - UseMCJIT = false; // IR module verification is enabled by default in debug builds, and disabled // by default in release builds. @@ -485,13 +421,11 @@ ExecutionEngine *EngineBuilder::create(TargetMachine *TM) { // to the function tells DynamicLibrary to load the program, not a library. if (sys::DynamicLibrary::LoadLibraryPermanently(nullptr, ErrorStr)) return nullptr; - - assert(!(JMM && MCJMM)); // If the user specified a memory manager but didn't specify which engine to // create, we assume they only want the JIT, and we fail if they only want // the interpreter. - if (JMM || MCJMM) { + if (MCJMM) { if (WhichEngine & EngineKind::JIT) WhichEngine = EngineKind::JIT; else { @@ -500,14 +434,6 @@ ExecutionEngine *EngineBuilder::create(TargetMachine *TM) { return nullptr; } } - - if (MCJMM && ! UseMCJIT) { - if (ErrorStr) - *ErrorStr = - "Cannot create a legacy JIT with a runtime dyld memory " - "manager."; - return nullptr; - } // Unless the interpreter was explicitly selected or the JIT is not linked, // try making a JIT. @@ -520,13 +446,9 @@ ExecutionEngine *EngineBuilder::create(TargetMachine *TM) { } ExecutionEngine *EE = nullptr; - if (UseMCJIT && ExecutionEngine::MCJITCtor) - EE = ExecutionEngine::MCJITCtor(M, ErrorStr, MCJMM ? MCJMM : JMM, - AllocateGVsWithCode, TheTM.release()); - else if (ExecutionEngine::JITCtor) - EE = ExecutionEngine::JITCtor(M, ErrorStr, JMM, - AllocateGVsWithCode, TheTM.release()); - + if (ExecutionEngine::MCJITCtor) + EE = ExecutionEngine::MCJITCtor(std::move(M), ErrorStr, MCJMM, + std::move(TheTM)); if (EE) { EE->setVerifyModules(VerifyModules); return EE; @@ -537,14 +459,13 @@ ExecutionEngine *EngineBuilder::create(TargetMachine *TM) { // an interpreter instead. if (WhichEngine & EngineKind::Interpreter) { if (ExecutionEngine::InterpCtor) - return ExecutionEngine::InterpCtor(M, ErrorStr); + return ExecutionEngine::InterpCtor(std::move(M), ErrorStr); if (ErrorStr) *ErrorStr = "Interpreter has not been linked in."; return nullptr; } - if ((WhichEngine & EngineKind::JIT) && !ExecutionEngine::JITCtor && - !ExecutionEngine::MCJITCtor) { + if ((WhichEngine & EngineKind::JIT) && !ExecutionEngine::MCJITCtor) { if (ErrorStr) *ErrorStr = "JIT has not been linked in."; } @@ -890,9 +811,6 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { Result = PTOGV(getPointerToFunctionOrStub(const_cast<Function*>(F))); else if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(C)) Result = PTOGV(getOrEmitGlobalVariable(const_cast<GlobalVariable*>(GV))); - else if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) - Result = PTOGV(getPointerToBasicBlock(const_cast<BasicBlock*>( - BA->getBasicBlock()))); else llvm_unreachable("Unknown constant pointer type!"); break; diff --git a/lib/ExecutionEngine/ExecutionEngineBindings.cpp b/lib/ExecutionEngine/ExecutionEngineBindings.cpp index 6ff1e7a..58271df 100644 --- a/lib/ExecutionEngine/ExecutionEngineBindings.cpp +++ b/lib/ExecutionEngine/ExecutionEngineBindings.cpp @@ -27,14 +27,6 @@ using namespace llvm; // Wrapping the C bindings types. DEFINE_SIMPLE_CONVERSION_FUNCTIONS(GenericValue, LLVMGenericValueRef) -inline TargetLibraryInfo *unwrap(LLVMTargetLibraryInfoRef P) { - return reinterpret_cast<TargetLibraryInfo*>(P); -} - -inline LLVMTargetLibraryInfoRef wrap(const TargetLibraryInfo *P) { - TargetLibraryInfo *X = const_cast<TargetLibraryInfo*>(P); - return reinterpret_cast<LLVMTargetLibraryInfoRef>(X); -} inline LLVMTargetMachineRef wrap(const TargetMachine *P) { return @@ -110,7 +102,7 @@ LLVMBool LLVMCreateExecutionEngineForModule(LLVMExecutionEngineRef *OutEE, LLVMModuleRef M, char **OutError) { std::string Error; - EngineBuilder builder(unwrap(M)); + EngineBuilder builder(std::unique_ptr<Module>(unwrap(M))); builder.setEngineKind(EngineKind::Either) .setErrorStr(&Error); if (ExecutionEngine *EE = builder.create()){ @@ -125,7 +117,7 @@ LLVMBool LLVMCreateInterpreterForModule(LLVMExecutionEngineRef *OutInterp, LLVMModuleRef M, char **OutError) { std::string Error; - EngineBuilder builder(unwrap(M)); + EngineBuilder builder(std::unique_ptr<Module>(unwrap(M))); builder.setEngineKind(EngineKind::Interpreter) .setErrorStr(&Error); if (ExecutionEngine *Interp = builder.create()) { @@ -141,7 +133,7 @@ LLVMBool LLVMCreateJITCompilerForModule(LLVMExecutionEngineRef *OutJIT, unsigned OptLevel, char **OutError) { std::string Error; - EngineBuilder builder(unwrap(M)); + EngineBuilder builder(std::unique_ptr<Module>(unwrap(M))); builder.setEngineKind(EngineKind::JIT) .setErrorStr(&Error) .setOptLevel((CodeGenOpt::Level)OptLevel); @@ -189,10 +181,9 @@ LLVMBool LLVMCreateMCJITCompilerForModule( targetOptions.EnableFastISel = options.EnableFastISel; std::string Error; - EngineBuilder builder(unwrap(M)); + EngineBuilder builder(std::unique_ptr<Module>(unwrap(M))); builder.setEngineKind(EngineKind::JIT) .setErrorStr(&Error) - .setUseMCJIT(true) .setOptLevel((CodeGenOpt::Level)options.OptLevel) .setCodeModel(unwrap(options.CodeModel)) .setTargetOptions(targetOptions); @@ -275,11 +266,10 @@ LLVMGenericValueRef LLVMRunFunction(LLVMExecutionEngineRef EE, LLVMValueRef F, } void LLVMFreeMachineCodeForFunction(LLVMExecutionEngineRef EE, LLVMValueRef F) { - unwrap(EE)->freeMachineCodeForFunction(unwrap<Function>(F)); } void LLVMAddModule(LLVMExecutionEngineRef EE, LLVMModuleRef M){ - unwrap(EE)->addModule(unwrap(M)); + unwrap(EE)->addModule(std::unique_ptr<Module>(unwrap(M))); } void LLVMAddModuleProvider(LLVMExecutionEngineRef EE, LLVMModuleProviderRef MP){ @@ -314,7 +304,7 @@ LLVMBool LLVMFindFunction(LLVMExecutionEngineRef EE, const char *Name, void *LLVMRecompileAndRelinkFunction(LLVMExecutionEngineRef EE, LLVMValueRef Fn) { - return unwrap(EE)->recompileAndRelinkFunction(unwrap<Function>(Fn)); + return nullptr; } LLVMTargetDataRef LLVMGetExecutionEngineTargetData(LLVMExecutionEngineRef EE) { diff --git a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp index 4e22a8b..b23ca88 100644 --- a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp +++ b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp @@ -57,29 +57,11 @@ public: ~IntelJITEventListener() { } - virtual void NotifyFunctionEmitted(const Function &F, - void *FnStart, size_t FnSize, - const EmittedFunctionDetails &Details); - - virtual void NotifyFreeingMachineCode(void *OldPtr); - virtual void NotifyObjectEmitted(const ObjectImage &Obj); virtual void NotifyFreeingObject(const ObjectImage &Obj); }; -static LineNumberInfo LineStartToIntelJITFormat( - uintptr_t StartAddress, - uintptr_t Address, - DebugLoc Loc) { - LineNumberInfo Result; - - Result.Offset = Address - StartAddress; - Result.LineNumber = Loc.getLine(); - - return Result; -} - static LineNumberInfo DILineInfoToIntelJITFormat(uintptr_t StartAddress, uintptr_t Address, DILineInfo Line) { @@ -113,84 +95,10 @@ static iJIT_Method_Load FunctionDescToIntelJITFormat( return Result; } -// Adds the just-emitted function to the symbol table. -void IntelJITEventListener::NotifyFunctionEmitted( - const Function &F, void *FnStart, size_t FnSize, - const EmittedFunctionDetails &Details) { - iJIT_Method_Load FunctionMessage = FunctionDescToIntelJITFormat(*Wrapper, - F.getName().data(), - reinterpret_cast<uint64_t>(FnStart), - FnSize); - - std::vector<LineNumberInfo> LineInfo; - - if (!Details.LineStarts.empty()) { - // Now convert the line number information from the address/DebugLoc - // format in Details to the offset/lineno in Intel JIT API format. - - LineInfo.reserve(Details.LineStarts.size() + 1); - - DebugLoc FirstLoc = Details.LineStarts[0].Loc; - assert(!FirstLoc.isUnknown() - && "LineStarts should not contain unknown DebugLocs"); - - MDNode *FirstLocScope = FirstLoc.getScope(F.getContext()); - DISubprogram FunctionDI = getDISubprogram(FirstLocScope); - if (FunctionDI.Verify()) { - FunctionMessage.source_file_name = const_cast<char*>( - Filenames.getFullPath(FirstLocScope)); - - LineNumberInfo FirstLine; - FirstLine.Offset = 0; - FirstLine.LineNumber = FunctionDI.getLineNumber(); - LineInfo.push_back(FirstLine); - } - - for (std::vector<EmittedFunctionDetails::LineStart>::const_iterator I = - Details.LineStarts.begin(), E = Details.LineStarts.end(); - I != E; ++I) { - // This implementation ignores the DebugLoc filename because the Intel - // JIT API does not support multiple source files associated with a single - // JIT function - LineInfo.push_back(LineStartToIntelJITFormat( - reinterpret_cast<uintptr_t>(FnStart), - I->Address, - I->Loc)); - - // If we have no file name yet for the function, use the filename from - // the first instruction that has one - if (FunctionMessage.source_file_name == 0) { - MDNode *scope = I->Loc.getScope( - Details.MF->getFunction()->getContext()); - FunctionMessage.source_file_name = const_cast<char*>( - Filenames.getFullPath(scope)); - } - } - - FunctionMessage.line_number_size = LineInfo.size(); - FunctionMessage.line_number_table = &*LineInfo.begin(); - } else { - FunctionMessage.line_number_size = 0; - FunctionMessage.line_number_table = 0; - } - - Wrapper->iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, - &FunctionMessage); - MethodIDs[FnStart] = FunctionMessage.method_id; -} - -void IntelJITEventListener::NotifyFreeingMachineCode(void *FnStart) { - MethodIDMap::iterator I = MethodIDs.find(FnStart); - if (I != MethodIDs.end()) { - Wrapper->iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_UNLOAD_START, &I->second); - MethodIDs.erase(I); - } -} - void IntelJITEventListener::NotifyObjectEmitted(const ObjectImage &Obj) { // Get the address of the object image for use as a unique identifier const void* ObjData = Obj.getData().data(); - DIContext* Context = DIContext::getDWARFContext(Obj.getObjectFile()); + DIContext* Context = DIContext::getDWARFContext(*Obj.getObjectFile()); MethodAddressVector Functions; // Use symbol info to iterate functions in the object. diff --git a/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt b/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt index 9c06fda..e36493e 100644 --- a/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt +++ b/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt @@ -21,3 +21,4 @@ type = OptionalLibrary name = IntelJITEvents parent = ExecutionEngine +required_libraries = Core DebugInfo Support diff --git a/lib/ExecutionEngine/Interpreter/CMakeLists.txt b/lib/ExecutionEngine/Interpreter/CMakeLists.txt index 74df8f0..1aac3ac 100644 --- a/lib/ExecutionEngine/Interpreter/CMakeLists.txt +++ b/lib/ExecutionEngine/Interpreter/CMakeLists.txt @@ -13,7 +13,7 @@ add_llvm_library(LLVMInterpreter ) if( LLVM_ENABLE_FFI ) - target_link_libraries( LLVMInterpreter ${FFI_LIBRARY_PATH} ) + target_link_libraries( LLVMInterpreter ${cmake_2_8_12_PRIVATE} ${FFI_LIBRARY_PATH} ) endif() add_dependencies(LLVMInterpreter intrinsics_gen) diff --git a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp index 671bbee..b022101 100644 --- a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp +++ b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp @@ -28,6 +28,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/Mutex.h" +#include "llvm/Support/UniqueLock.h" #include <cmath> #include <csignal> #include <cstdio> @@ -51,7 +52,7 @@ static ManagedStatic<sys::Mutex> FunctionsLock; typedef GenericValue (*ExFunc)(FunctionType *, const std::vector<GenericValue> &); static ManagedStatic<std::map<const Function *, ExFunc> > ExportedFunctions; -static std::map<std::string, ExFunc> FuncNames; +static ManagedStatic<std::map<std::string, ExFunc> > FuncNames; #ifdef USE_LIBFFI typedef void (*RawFunc)(); @@ -97,9 +98,9 @@ static ExFunc lookupFunction(const Function *F) { ExtName += "_" + F->getName().str(); sys::ScopedLock Writer(*FunctionsLock); - ExFunc FnPtr = FuncNames[ExtName]; + ExFunc FnPtr = (*FuncNames)[ExtName]; if (!FnPtr) - FnPtr = FuncNames["lle_X_" + F->getName().str()]; + FnPtr = (*FuncNames)["lle_X_" + F->getName().str()]; if (!FnPtr) // Try calling a generic function... if it exists... FnPtr = (ExFunc)(intptr_t) sys::DynamicLibrary::SearchForAddressOfSymbol("lle_X_" + @@ -248,14 +249,14 @@ GenericValue Interpreter::callExternalFunction(Function *F, const std::vector<GenericValue> &ArgVals) { TheInterpreter = this; - FunctionsLock->acquire(); + unique_lock<sys::Mutex> Guard(*FunctionsLock); // Do a lookup to see if the function is in our cache... this should just be a // deferred annotation! std::map<const Function *, ExFunc>::iterator FI = ExportedFunctions->find(F); if (ExFunc Fn = (FI == ExportedFunctions->end()) ? lookupFunction(F) : FI->second) { - FunctionsLock->release(); + Guard.unlock(); return Fn(F->getFunctionType(), ArgVals); } @@ -273,7 +274,7 @@ GenericValue Interpreter::callExternalFunction(Function *F, RawFn = RF->second; } - FunctionsLock->release(); + Guard.unlock(); GenericValue Result; if (RawFn != 0 && ffiInvoke(RawFn, F, ArgVals, getDataLayout(), Result)) @@ -497,15 +498,15 @@ static GenericValue lle_X_memcpy(FunctionType *FT, void Interpreter::initializeExternalFunctions() { sys::ScopedLock Writer(*FunctionsLock); - FuncNames["lle_X_atexit"] = lle_X_atexit; - FuncNames["lle_X_exit"] = lle_X_exit; - FuncNames["lle_X_abort"] = lle_X_abort; - - FuncNames["lle_X_printf"] = lle_X_printf; - FuncNames["lle_X_sprintf"] = lle_X_sprintf; - FuncNames["lle_X_sscanf"] = lle_X_sscanf; - FuncNames["lle_X_scanf"] = lle_X_scanf; - FuncNames["lle_X_fprintf"] = lle_X_fprintf; - FuncNames["lle_X_memset"] = lle_X_memset; - FuncNames["lle_X_memcpy"] = lle_X_memcpy; + (*FuncNames)["lle_X_atexit"] = lle_X_atexit; + (*FuncNames)["lle_X_exit"] = lle_X_exit; + (*FuncNames)["lle_X_abort"] = lle_X_abort; + + (*FuncNames)["lle_X_printf"] = lle_X_printf; + (*FuncNames)["lle_X_sprintf"] = lle_X_sprintf; + (*FuncNames)["lle_X_sscanf"] = lle_X_sscanf; + (*FuncNames)["lle_X_scanf"] = lle_X_scanf; + (*FuncNames)["lle_X_fprintf"] = lle_X_fprintf; + (*FuncNames)["lle_X_memset"] = lle_X_memset; + (*FuncNames)["lle_X_memcpy"] = lle_X_memcpy; } diff --git a/lib/ExecutionEngine/Interpreter/Interpreter.cpp b/lib/ExecutionEngine/Interpreter/Interpreter.cpp index 814efcc..8562981 100644 --- a/lib/ExecutionEngine/Interpreter/Interpreter.cpp +++ b/lib/ExecutionEngine/Interpreter/Interpreter.cpp @@ -30,9 +30,10 @@ static struct RegisterInterp { extern "C" void LLVMLinkInInterpreter() { } -/// create - Create a new interpreter object. This can never fail. +/// Create a new interpreter object. /// -ExecutionEngine *Interpreter::create(Module *M, std::string* ErrStr) { +ExecutionEngine *Interpreter::create(std::unique_ptr<Module> M, + std::string *ErrStr) { // Tell this Module to materialize everything and release the GVMaterializer. if (std::error_code EC = M->materializeAllPermanently()) { if (ErrStr) @@ -41,15 +42,15 @@ ExecutionEngine *Interpreter::create(Module *M, std::string* ErrStr) { return nullptr; } - return new Interpreter(M); + return new Interpreter(std::move(M)); } //===----------------------------------------------------------------------===// // Interpreter ctor - Initialize stuff // -Interpreter::Interpreter(Module *M) - : ExecutionEngine(M), TD(M) { - +Interpreter::Interpreter(std::unique_ptr<Module> M) + : ExecutionEngine(std::move(M)), TD(Modules.back().get()) { + memset(&ExitValue.Untyped, 0, sizeof(ExitValue.Untyped)); setDataLayout(&TD); // Initialize the "backend" diff --git a/lib/ExecutionEngine/Interpreter/Interpreter.h b/lib/ExecutionEngine/Interpreter/Interpreter.h index 2145cde..2be9c59 100644 --- a/lib/ExecutionEngine/Interpreter/Interpreter.h +++ b/lib/ExecutionEngine/Interpreter/Interpreter.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLI_INTERPRETER_H -#define LLI_INTERPRETER_H +#ifndef LLVM_LIB_EXECUTIONENGINE_INTERPRETER_INTERPRETER_H +#define LLVM_LIB_EXECUTIONENGINE_INTERPRETER_INTERPRETER_H #include "llvm/ExecutionEngine/ExecutionEngine.h" #include "llvm/ExecutionEngine/GenericValue.h" @@ -37,29 +37,24 @@ typedef generic_gep_type_iterator<User::const_op_iterator> gep_type_iterator; // stack, which causes the dtor to be run, which frees all the alloca'd memory. // class AllocaHolder { - friend class AllocaHolderHandle; - std::vector<void*> Allocations; - unsigned RefCnt; + std::vector<void *> Allocations; + public: - AllocaHolder() : RefCnt(0) {} - void add(void *mem) { Allocations.push_back(mem); } - ~AllocaHolder() { - for (unsigned i = 0; i < Allocations.size(); ++i) - free(Allocations[i]); + AllocaHolder() {} + + // Make this type move-only. Define explicit move special members for MSVC. + AllocaHolder(AllocaHolder &&RHS) : Allocations(std::move(RHS.Allocations)) {} + AllocaHolder &operator=(AllocaHolder &&RHS) { + Allocations = std::move(RHS.Allocations); + return *this; } -}; -// AllocaHolderHandle gives AllocaHolder value semantics so we can stick it into -// a vector... -// -class AllocaHolderHandle { - AllocaHolder *H; -public: - AllocaHolderHandle() : H(new AllocaHolder()) { H->RefCnt++; } - AllocaHolderHandle(const AllocaHolderHandle &AH) : H(AH.H) { H->RefCnt++; } - ~AllocaHolderHandle() { if (--H->RefCnt == 0) delete H; } + ~AllocaHolder() { + for (void *Allocation : Allocations) + free(Allocation); + } - void add(void *mem) { H->add(mem); } + void add(void *Mem) { Allocations.push_back(Mem); } }; typedef std::vector<GenericValue> ValuePlaneTy; @@ -71,11 +66,29 @@ struct ExecutionContext { Function *CurFunction;// The currently executing function BasicBlock *CurBB; // The currently executing BB BasicBlock::iterator CurInst; // The next instruction to execute - std::map<Value *, GenericValue> Values; // LLVM values used in this invocation - std::vector<GenericValue> VarArgs; // Values passed through an ellipsis CallSite Caller; // Holds the call that called subframes. // NULL if main func or debugger invoked fn - AllocaHolderHandle Allocas; // Track memory allocated by alloca + std::map<Value *, GenericValue> Values; // LLVM values used in this invocation + std::vector<GenericValue> VarArgs; // Values passed through an ellipsis + AllocaHolder Allocas; // Track memory allocated by alloca + + ExecutionContext() : CurFunction(nullptr), CurBB(nullptr), CurInst(nullptr) {} + + ExecutionContext(ExecutionContext &&O) + : CurFunction(O.CurFunction), CurBB(O.CurBB), CurInst(O.CurInst), + Caller(O.Caller), Values(std::move(O.Values)), + VarArgs(std::move(O.VarArgs)), Allocas(std::move(O.Allocas)) {} + + ExecutionContext &operator=(ExecutionContext &&O) { + CurFunction = O.CurFunction; + CurBB = O.CurBB; + CurInst = O.CurInst; + Caller = O.Caller; + Values = std::move(O.Values); + VarArgs = std::move(O.VarArgs); + Allocas = std::move(O.Allocas); + return *this; + } }; // Interpreter - This class represents the entirety of the interpreter. @@ -94,7 +107,7 @@ class Interpreter : public ExecutionEngine, public InstVisitor<Interpreter> { std::vector<Function*> AtExitHandlers; public: - explicit Interpreter(Module *M); + explicit Interpreter(std::unique_ptr<Module> M); ~Interpreter(); /// runAtExitHandlers - Run any functions registered by the program's calls to @@ -105,33 +118,23 @@ public: static void Register() { InterpCtor = create; } - - /// create - Create an interpreter ExecutionEngine. This can never fail. + + /// Create an interpreter ExecutionEngine. /// - static ExecutionEngine *create(Module *M, std::string *ErrorStr = nullptr); + static ExecutionEngine *create(std::unique_ptr<Module> M, + std::string *ErrorStr = nullptr); /// run - Start execution with the specified function and arguments. /// GenericValue runFunction(Function *F, const std::vector<GenericValue> &ArgValues) override; - void *getPointerToNamedFunction(const std::string &Name, + void *getPointerToNamedFunction(StringRef Name, bool AbortOnFailure = true) override { // FIXME: not implemented. return nullptr; } - /// recompileAndRelinkFunction - For the interpreter, functions are always - /// up-to-date. - /// - void *recompileAndRelinkFunction(Function *F) override { - return getPointerToFunction(F); - } - - /// freeMachineCodeForFunction - The interpreter does not generate any code. - /// - void freeMachineCodeForFunction(Function *F) override { } - // Methods used to execute code: // Place a call on the stack void callFunction(Function *F, const std::vector<GenericValue> &ArgVals); @@ -213,7 +216,6 @@ private: // Helper functions void SwitchToNewBasicBlock(BasicBlock *Dest, ExecutionContext &SF); void *getPointerToFunction(Function *F) override { return (void*)F; } - void *getPointerToBasicBlock(BasicBlock *BB) override { return (void*)BB; } void initializeExecutionEngine() { } void initializeExternalFunctions(); diff --git a/lib/ExecutionEngine/JIT/Android.mk b/lib/ExecutionEngine/JIT/Android.mk deleted file mode 100644 index 0466ba0..0000000 --- a/lib/ExecutionEngine/JIT/Android.mk +++ /dev/null @@ -1,17 +0,0 @@ -LOCAL_PATH:= $(call my-dir) - -# For the host -# ===================================================== -include $(CLEAR_VARS) - -LOCAL_SRC_FILES := \ - JIT.cpp \ - JITEmitter.cpp \ - JITMemoryManager.cpp - -LOCAL_MODULE:= libLLVMJIT - -LOCAL_MODULE_TAGS := optional - -include $(LLVM_HOST_BUILD_MK) -include $(BUILD_HOST_STATIC_LIBRARY) diff --git a/lib/ExecutionEngine/JIT/CMakeLists.txt b/lib/ExecutionEngine/JIT/CMakeLists.txt deleted file mode 100644 index e16baed..0000000 --- a/lib/ExecutionEngine/JIT/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# TODO: Support other architectures. See Makefile. -add_definitions(-DENABLE_X86_JIT) - -add_llvm_library(LLVMJIT - JIT.cpp - JITEmitter.cpp - JITMemoryManager.cpp - ) diff --git a/lib/ExecutionEngine/JIT/JIT.cpp b/lib/ExecutionEngine/JIT/JIT.cpp deleted file mode 100644 index 83ec978..0000000 --- a/lib/ExecutionEngine/JIT/JIT.cpp +++ /dev/null @@ -1,695 +0,0 @@ -//===-- JIT.cpp - LLVM Just in Time Compiler ------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This tool implements a just-in-time compiler for LLVM, allowing direct -// execution of LLVM bitcode in an efficient manner. -// -//===----------------------------------------------------------------------===// - -#include "JIT.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/CodeGen/JITCodeEmitter.h" -#include "llvm/CodeGen/MachineCodeInfo.h" -#include "llvm/Config/config.h" -#include "llvm/ExecutionEngine/GenericValue.h" -#include "llvm/ExecutionEngine/JITEventListener.h" -#include "llvm/ExecutionEngine/JITMemoryManager.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/GlobalVariable.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Module.h" -#include "llvm/Support/Dwarf.h" -#include "llvm/Support/DynamicLibrary.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/MutexGuard.h" -#include "llvm/Target/TargetJITInfo.h" -#include "llvm/Target/TargetMachine.h" - -using namespace llvm; - -#ifdef __APPLE__ -// Apple gcc defaults to -fuse-cxa-atexit (i.e. calls __cxa_atexit instead -// of atexit). It passes the address of linker generated symbol __dso_handle -// to the function. -// This configuration change happened at version 5330. -# include <AvailabilityMacros.h> -# if defined(MAC_OS_X_VERSION_10_4) && \ - ((MAC_OS_X_VERSION_MIN_REQUIRED > MAC_OS_X_VERSION_10_4) || \ - (MAC_OS_X_VERSION_MIN_REQUIRED == MAC_OS_X_VERSION_10_4 && \ - __APPLE_CC__ >= 5330)) -# ifndef HAVE___DSO_HANDLE -# define HAVE___DSO_HANDLE 1 -# endif -# endif -#endif - -#if HAVE___DSO_HANDLE -extern void *__dso_handle __attribute__ ((__visibility__ ("hidden"))); -#endif - -namespace { - -static struct RegisterJIT { - RegisterJIT() { JIT::Register(); } -} JITRegistrator; - -} - -extern "C" void LLVMLinkInJIT() { -} - -/// createJIT - This is the factory method for creating a JIT for the current -/// machine, it does not fall back to the interpreter. This takes ownership -/// of the module. -ExecutionEngine *JIT::createJIT(Module *M, - std::string *ErrorStr, - JITMemoryManager *JMM, - bool GVsWithCode, - TargetMachine *TM) { - // Try to register the program as a source of symbols to resolve against. - // - // FIXME: Don't do this here. - sys::DynamicLibrary::LoadLibraryPermanently(nullptr, nullptr); - - // If the target supports JIT code generation, create the JIT. - if (TargetJITInfo *TJ = TM->getJITInfo()) { - return new JIT(M, *TM, *TJ, JMM, GVsWithCode); - } else { - if (ErrorStr) - *ErrorStr = "target does not support JIT code generation"; - return nullptr; - } -} - -namespace { -/// This class supports the global getPointerToNamedFunction(), which allows -/// bugpoint or gdb users to search for a function by name without any context. -class JitPool { - SmallPtrSet<JIT*, 1> JITs; // Optimize for process containing just 1 JIT. - mutable sys::Mutex Lock; -public: - void Add(JIT *jit) { - MutexGuard guard(Lock); - JITs.insert(jit); - } - void Remove(JIT *jit) { - MutexGuard guard(Lock); - JITs.erase(jit); - } - void *getPointerToNamedFunction(const char *Name) const { - MutexGuard guard(Lock); - assert(JITs.size() != 0 && "No Jit registered"); - //search function in every instance of JIT - for (SmallPtrSet<JIT*, 1>::const_iterator Jit = JITs.begin(), - end = JITs.end(); - Jit != end; ++Jit) { - if (Function *F = (*Jit)->FindFunctionNamed(Name)) - return (*Jit)->getPointerToFunction(F); - } - // The function is not available : fallback on the first created (will - // search in symbol of the current program/library) - return (*JITs.begin())->getPointerToNamedFunction(Name); - } -}; -ManagedStatic<JitPool> AllJits; -} -extern "C" { - // getPointerToNamedFunction - This function is used as a global wrapper to - // JIT::getPointerToNamedFunction for the purpose of resolving symbols when - // bugpoint is debugging the JIT. In that scenario, we are loading an .so and - // need to resolve function(s) that are being mis-codegenerated, so we need to - // resolve their addresses at runtime, and this is the way to do it. - void *getPointerToNamedFunction(const char *Name) { - return AllJits->getPointerToNamedFunction(Name); - } -} - -JIT::JIT(Module *M, TargetMachine &tm, TargetJITInfo &tji, - JITMemoryManager *jmm, bool GVsWithCode) - : ExecutionEngine(M), TM(tm), TJI(tji), - JMM(jmm ? jmm : JITMemoryManager::CreateDefaultMemManager()), - AllocateGVsWithCode(GVsWithCode), isAlreadyCodeGenerating(false) { - setDataLayout(TM.getDataLayout()); - - jitstate = new JITState(M); - - // Initialize JCE - JCE = createEmitter(*this, JMM, TM); - - // Register in global list of all JITs. - AllJits->Add(this); - - // Add target data - MutexGuard locked(lock); - FunctionPassManager &PM = jitstate->getPM(); - M->setDataLayout(TM.getDataLayout()); - PM.add(new DataLayoutPass(M)); - - // Turn the machine code intermediate representation into bytes in memory that - // may be executed. - if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) { - report_fatal_error("Target does not support machine code emission!"); - } - - // Initialize passes. - PM.doInitialization(); -} - -JIT::~JIT() { - // Cleanup. - AllJits->Remove(this); - delete jitstate; - delete JCE; - // JMM is a ownership of JCE, so we no need delete JMM here. - delete &TM; -} - -/// addModule - Add a new Module to the JIT. If we previously removed the last -/// Module, we need re-initialize jitstate with a valid Module. -void JIT::addModule(Module *M) { - MutexGuard locked(lock); - - if (Modules.empty()) { - assert(!jitstate && "jitstate should be NULL if Modules vector is empty!"); - - jitstate = new JITState(M); - - FunctionPassManager &PM = jitstate->getPM(); - M->setDataLayout(TM.getDataLayout()); - PM.add(new DataLayoutPass(M)); - - // Turn the machine code intermediate representation into bytes in memory - // that may be executed. - if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) { - report_fatal_error("Target does not support machine code emission!"); - } - - // Initialize passes. - PM.doInitialization(); - } - - ExecutionEngine::addModule(M); -} - -/// removeModule - If we are removing the last Module, invalidate the jitstate -/// since the PassManager it contains references a released Module. -bool JIT::removeModule(Module *M) { - bool result = ExecutionEngine::removeModule(M); - - MutexGuard locked(lock); - - if (jitstate && jitstate->getModule() == M) { - delete jitstate; - jitstate = nullptr; - } - - if (!jitstate && !Modules.empty()) { - jitstate = new JITState(Modules[0]); - - FunctionPassManager &PM = jitstate->getPM(); - M->setDataLayout(TM.getDataLayout()); - PM.add(new DataLayoutPass(M)); - - // Turn the machine code intermediate representation into bytes in memory - // that may be executed. - if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) { - report_fatal_error("Target does not support machine code emission!"); - } - - // Initialize passes. - PM.doInitialization(); - } - return result; -} - -/// run - Start execution with the specified function and arguments. -/// -GenericValue JIT::runFunction(Function *F, - const std::vector<GenericValue> &ArgValues) { - assert(F && "Function *F was null at entry to run()"); - - void *FPtr = getPointerToFunction(F); - assert(FPtr && "Pointer to fn's code was null after getPointerToFunction"); - FunctionType *FTy = F->getFunctionType(); - Type *RetTy = FTy->getReturnType(); - - assert((FTy->getNumParams() == ArgValues.size() || - (FTy->isVarArg() && FTy->getNumParams() <= ArgValues.size())) && - "Wrong number of arguments passed into function!"); - assert(FTy->getNumParams() == ArgValues.size() && - "This doesn't support passing arguments through varargs (yet)!"); - - // Handle some common cases first. These cases correspond to common `main' - // prototypes. - if (RetTy->isIntegerTy(32) || RetTy->isVoidTy()) { - switch (ArgValues.size()) { - case 3: - if (FTy->getParamType(0)->isIntegerTy(32) && - FTy->getParamType(1)->isPointerTy() && - FTy->getParamType(2)->isPointerTy()) { - int (*PF)(int, char **, const char **) = - (int(*)(int, char **, const char **))(intptr_t)FPtr; - - // Call the function. - GenericValue rv; - rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(), - (char **)GVTOP(ArgValues[1]), - (const char **)GVTOP(ArgValues[2]))); - return rv; - } - break; - case 2: - if (FTy->getParamType(0)->isIntegerTy(32) && - FTy->getParamType(1)->isPointerTy()) { - int (*PF)(int, char **) = (int(*)(int, char **))(intptr_t)FPtr; - - // Call the function. - GenericValue rv; - rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(), - (char **)GVTOP(ArgValues[1]))); - return rv; - } - break; - case 1: - if (FTy->getParamType(0)->isIntegerTy(32)) { - GenericValue rv; - int (*PF)(int) = (int(*)(int))(intptr_t)FPtr; - rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue())); - return rv; - } - if (FTy->getParamType(0)->isPointerTy()) { - GenericValue rv; - int (*PF)(char *) = (int(*)(char *))(intptr_t)FPtr; - rv.IntVal = APInt(32, PF((char*)GVTOP(ArgValues[0]))); - return rv; - } - break; - } - } - - // Handle cases where no arguments are passed first. - if (ArgValues.empty()) { - GenericValue rv; - switch (RetTy->getTypeID()) { - default: llvm_unreachable("Unknown return type for function call!"); - case Type::IntegerTyID: { - unsigned BitWidth = cast<IntegerType>(RetTy)->getBitWidth(); - if (BitWidth == 1) - rv.IntVal = APInt(BitWidth, ((bool(*)())(intptr_t)FPtr)()); - else if (BitWidth <= 8) - rv.IntVal = APInt(BitWidth, ((char(*)())(intptr_t)FPtr)()); - else if (BitWidth <= 16) - rv.IntVal = APInt(BitWidth, ((short(*)())(intptr_t)FPtr)()); - else if (BitWidth <= 32) - rv.IntVal = APInt(BitWidth, ((int(*)())(intptr_t)FPtr)()); - else if (BitWidth <= 64) - rv.IntVal = APInt(BitWidth, ((int64_t(*)())(intptr_t)FPtr)()); - else - llvm_unreachable("Integer types > 64 bits not supported"); - return rv; - } - case Type::VoidTyID: - rv.IntVal = APInt(32, ((int(*)())(intptr_t)FPtr)()); - return rv; - case Type::FloatTyID: - rv.FloatVal = ((float(*)())(intptr_t)FPtr)(); - return rv; - case Type::DoubleTyID: - rv.DoubleVal = ((double(*)())(intptr_t)FPtr)(); - return rv; - case Type::X86_FP80TyID: - case Type::FP128TyID: - case Type::PPC_FP128TyID: - llvm_unreachable("long double not supported yet"); - case Type::PointerTyID: - return PTOGV(((void*(*)())(intptr_t)FPtr)()); - } - } - - // Okay, this is not one of our quick and easy cases. Because we don't have a - // full FFI, we have to codegen a nullary stub function that just calls the - // function we are interested in, passing in constants for all of the - // arguments. Make this function and return. - - // First, create the function. - FunctionType *STy=FunctionType::get(RetTy, false); - Function *Stub = Function::Create(STy, Function::InternalLinkage, "", - F->getParent()); - - // Insert a basic block. - BasicBlock *StubBB = BasicBlock::Create(F->getContext(), "", Stub); - - // Convert all of the GenericValue arguments over to constants. Note that we - // currently don't support varargs. - SmallVector<Value*, 8> Args; - for (unsigned i = 0, e = ArgValues.size(); i != e; ++i) { - Constant *C = nullptr; - Type *ArgTy = FTy->getParamType(i); - const GenericValue &AV = ArgValues[i]; - switch (ArgTy->getTypeID()) { - default: llvm_unreachable("Unknown argument type for function call!"); - case Type::IntegerTyID: - C = ConstantInt::get(F->getContext(), AV.IntVal); - break; - case Type::FloatTyID: - C = ConstantFP::get(F->getContext(), APFloat(AV.FloatVal)); - break; - case Type::DoubleTyID: - C = ConstantFP::get(F->getContext(), APFloat(AV.DoubleVal)); - break; - case Type::PPC_FP128TyID: - case Type::X86_FP80TyID: - case Type::FP128TyID: - C = ConstantFP::get(F->getContext(), APFloat(ArgTy->getFltSemantics(), - AV.IntVal)); - break; - case Type::PointerTyID: - void *ArgPtr = GVTOP(AV); - if (sizeof(void*) == 4) - C = ConstantInt::get(Type::getInt32Ty(F->getContext()), - (int)(intptr_t)ArgPtr); - else - C = ConstantInt::get(Type::getInt64Ty(F->getContext()), - (intptr_t)ArgPtr); - // Cast the integer to pointer - C = ConstantExpr::getIntToPtr(C, ArgTy); - break; - } - Args.push_back(C); - } - - CallInst *TheCall = CallInst::Create(F, Args, "", StubBB); - TheCall->setCallingConv(F->getCallingConv()); - TheCall->setTailCall(); - if (!TheCall->getType()->isVoidTy()) - // Return result of the call. - ReturnInst::Create(F->getContext(), TheCall, StubBB); - else - ReturnInst::Create(F->getContext(), StubBB); // Just return void. - - // Finally, call our nullary stub function. - GenericValue Result = runFunction(Stub, std::vector<GenericValue>()); - // Erase it, since no other function can have a reference to it. - Stub->eraseFromParent(); - // And return the result. - return Result; -} - -void JIT::RegisterJITEventListener(JITEventListener *L) { - if (!L) - return; - MutexGuard locked(lock); - EventListeners.push_back(L); -} -void JIT::UnregisterJITEventListener(JITEventListener *L) { - if (!L) - return; - MutexGuard locked(lock); - std::vector<JITEventListener*>::reverse_iterator I= - std::find(EventListeners.rbegin(), EventListeners.rend(), L); - if (I != EventListeners.rend()) { - std::swap(*I, EventListeners.back()); - EventListeners.pop_back(); - } -} -void JIT::NotifyFunctionEmitted( - const Function &F, - void *Code, size_t Size, - const JITEvent_EmittedFunctionDetails &Details) { - MutexGuard locked(lock); - for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) { - EventListeners[I]->NotifyFunctionEmitted(F, Code, Size, Details); - } -} - -void JIT::NotifyFreeingMachineCode(void *OldPtr) { - MutexGuard locked(lock); - for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) { - EventListeners[I]->NotifyFreeingMachineCode(OldPtr); - } -} - -/// runJITOnFunction - Run the FunctionPassManager full of -/// just-in-time compilation passes on F, hopefully filling in -/// GlobalAddress[F] with the address of F's machine code. -/// -void JIT::runJITOnFunction(Function *F, MachineCodeInfo *MCI) { - MutexGuard locked(lock); - - class MCIListener : public JITEventListener { - MachineCodeInfo *const MCI; - public: - MCIListener(MachineCodeInfo *mci) : MCI(mci) {} - void NotifyFunctionEmitted(const Function &, void *Code, size_t Size, - const EmittedFunctionDetails &) override { - MCI->setAddress(Code); - MCI->setSize(Size); - } - }; - MCIListener MCIL(MCI); - if (MCI) - RegisterJITEventListener(&MCIL); - - runJITOnFunctionUnlocked(F); - - if (MCI) - UnregisterJITEventListener(&MCIL); -} - -void JIT::runJITOnFunctionUnlocked(Function *F) { - assert(!isAlreadyCodeGenerating && "Error: Recursive compilation detected!"); - - jitTheFunctionUnlocked(F); - - // If the function referred to another function that had not yet been - // read from bitcode, and we are jitting non-lazily, emit it now. - while (!jitstate->getPendingFunctions().empty()) { - Function *PF = jitstate->getPendingFunctions().back(); - jitstate->getPendingFunctions().pop_back(); - - assert(!PF->hasAvailableExternallyLinkage() && - "Externally-defined function should not be in pending list."); - - jitTheFunctionUnlocked(PF); - - // Now that the function has been jitted, ask the JITEmitter to rewrite - // the stub with real address of the function. - updateFunctionStubUnlocked(PF); - } -} - -void JIT::jitTheFunctionUnlocked(Function *F) { - isAlreadyCodeGenerating = true; - jitstate->getPM().run(*F); - isAlreadyCodeGenerating = false; - - // clear basic block addresses after this function is done - getBasicBlockAddressMap().clear(); -} - -/// getPointerToFunction - This method is used to get the address of the -/// specified function, compiling it if necessary. -/// -void *JIT::getPointerToFunction(Function *F) { - - if (void *Addr = getPointerToGlobalIfAvailable(F)) - return Addr; // Check if function already code gen'd - - MutexGuard locked(lock); - - // Now that this thread owns the lock, make sure we read in the function if it - // exists in this Module. - std::string ErrorMsg; - if (F->Materialize(&ErrorMsg)) { - report_fatal_error("Error reading function '" + F->getName()+ - "' from bitcode file: " + ErrorMsg); - } - - // ... and check if another thread has already code gen'd the function. - if (void *Addr = getPointerToGlobalIfAvailable(F)) - return Addr; - - if (F->isDeclaration() || F->hasAvailableExternallyLinkage()) { - bool AbortOnFailure = !F->hasExternalWeakLinkage(); - void *Addr = getPointerToNamedFunction(F->getName(), AbortOnFailure); - addGlobalMapping(F, Addr); - return Addr; - } - - runJITOnFunctionUnlocked(F); - - void *Addr = getPointerToGlobalIfAvailable(F); - assert(Addr && "Code generation didn't add function to GlobalAddress table!"); - return Addr; -} - -void JIT::addPointerToBasicBlock(const BasicBlock *BB, void *Addr) { - MutexGuard locked(lock); - - BasicBlockAddressMapTy::iterator I = - getBasicBlockAddressMap().find(BB); - if (I == getBasicBlockAddressMap().end()) { - getBasicBlockAddressMap()[BB] = Addr; - } else { - // ignore repeats: some BBs can be split into few MBBs? - } -} - -void JIT::clearPointerToBasicBlock(const BasicBlock *BB) { - MutexGuard locked(lock); - getBasicBlockAddressMap().erase(BB); -} - -void *JIT::getPointerToBasicBlock(BasicBlock *BB) { - // make sure it's function is compiled by JIT - (void)getPointerToFunction(BB->getParent()); - - // resolve basic block address - MutexGuard locked(lock); - - BasicBlockAddressMapTy::iterator I = - getBasicBlockAddressMap().find(BB); - if (I != getBasicBlockAddressMap().end()) { - return I->second; - } else { - llvm_unreachable("JIT does not have BB address for address-of-label, was" - " it eliminated by optimizer?"); - } -} - -void *JIT::getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure){ - if (!isSymbolSearchingDisabled()) { - void *ptr = JMM->getPointerToNamedFunction(Name, false); - if (ptr) - return ptr; - } - - /// If a LazyFunctionCreator is installed, use it to get/create the function. - if (LazyFunctionCreator) - if (void *RP = LazyFunctionCreator(Name)) - return RP; - - if (AbortOnFailure) { - report_fatal_error("Program used external function '"+Name+ - "' which could not be resolved!"); - } - return nullptr; -} - - -/// getOrEmitGlobalVariable - Return the address of the specified global -/// variable, possibly emitting it to memory if needed. This is used by the -/// Emitter. -void *JIT::getOrEmitGlobalVariable(const GlobalVariable *GV) { - MutexGuard locked(lock); - - void *Ptr = getPointerToGlobalIfAvailable(GV); - if (Ptr) return Ptr; - - // If the global is external, just remember the address. - if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) { -#if HAVE___DSO_HANDLE - if (GV->getName() == "__dso_handle") - return (void*)&__dso_handle; -#endif - Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(GV->getName()); - if (!Ptr) { - report_fatal_error("Could not resolve external global address: " - +GV->getName()); - } - addGlobalMapping(GV, Ptr); - } else { - // If the global hasn't been emitted to memory yet, allocate space and - // emit it into memory. - Ptr = getMemoryForGV(GV); - addGlobalMapping(GV, Ptr); - EmitGlobalVariable(GV); // Initialize the variable. - } - return Ptr; -} - -/// recompileAndRelinkFunction - This method is used to force a function -/// which has already been compiled, to be compiled again, possibly -/// after it has been modified. Then the entry to the old copy is overwritten -/// with a branch to the new copy. If there was no old copy, this acts -/// just like JIT::getPointerToFunction(). -/// -void *JIT::recompileAndRelinkFunction(Function *F) { - void *OldAddr = getPointerToGlobalIfAvailable(F); - - // If it's not already compiled there is no reason to patch it up. - if (!OldAddr) return getPointerToFunction(F); - - // Delete the old function mapping. - addGlobalMapping(F, nullptr); - - // Recodegen the function - runJITOnFunction(F); - - // Update state, forward the old function to the new function. - void *Addr = getPointerToGlobalIfAvailable(F); - assert(Addr && "Code generation didn't add function to GlobalAddress table!"); - TJI.replaceMachineCodeForFunction(OldAddr, Addr); - return Addr; -} - -/// getMemoryForGV - This method abstracts memory allocation of global -/// variable so that the JIT can allocate thread local variables depending -/// on the target. -/// -char* JIT::getMemoryForGV(const GlobalVariable* GV) { - char *Ptr; - - // GlobalVariable's which are not "constant" will cause trouble in a server - // situation. It's returned in the same block of memory as code which may - // not be writable. - if (isGVCompilationDisabled() && !GV->isConstant()) { - report_fatal_error("Compilation of non-internal GlobalValue is disabled!"); - } - - // Some applications require globals and code to live together, so they may - // be allocated into the same buffer, but in general globals are allocated - // through the memory manager which puts them near the code but not in the - // same buffer. - Type *GlobalType = GV->getType()->getElementType(); - size_t S = getDataLayout()->getTypeAllocSize(GlobalType); - size_t A = getDataLayout()->getPreferredAlignment(GV); - if (GV->isThreadLocal()) { - MutexGuard locked(lock); - Ptr = TJI.allocateThreadLocalMemory(S); - } else if (TJI.allocateSeparateGVMemory()) { - if (A <= 8) { - Ptr = (char*)malloc(S); - } else { - // Allocate S+A bytes of memory, then use an aligned pointer within that - // space. - Ptr = (char*)malloc(S+A); - unsigned MisAligned = ((intptr_t)Ptr & (A-1)); - Ptr = Ptr + (MisAligned ? (A-MisAligned) : 0); - } - } else if (AllocateGVsWithCode) { - Ptr = (char*)JCE->allocateSpace(S, A); - } else { - Ptr = (char*)JCE->allocateGlobal(S, A); - } - return Ptr; -} - -void JIT::addPendingFunction(Function *F) { - MutexGuard locked(lock); - jitstate->getPendingFunctions().push_back(F); -} - - -JITEventListener::~JITEventListener() {} diff --git a/lib/ExecutionEngine/JIT/JIT.h b/lib/ExecutionEngine/JIT/JIT.h deleted file mode 100644 index 69a7c36..0000000 --- a/lib/ExecutionEngine/JIT/JIT.h +++ /dev/null @@ -1,229 +0,0 @@ -//===-- JIT.h - Class definition for the JIT --------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines the top-level JIT data structure. -// -//===----------------------------------------------------------------------===// - -#ifndef JIT_H -#define JIT_H - -#include "llvm/ExecutionEngine/ExecutionEngine.h" -#include "llvm/IR/ValueHandle.h" -#include "llvm/PassManager.h" - -namespace llvm { - -class Function; -struct JITEvent_EmittedFunctionDetails; -class MachineCodeEmitter; -class MachineCodeInfo; -class TargetJITInfo; -class TargetMachine; - -class JITState { -private: - FunctionPassManager PM; // Passes to compile a function - Module *M; // Module used to create the PM - - /// PendingFunctions - Functions which have not been code generated yet, but - /// were called from a function being code generated. - std::vector<AssertingVH<Function> > PendingFunctions; - -public: - explicit JITState(Module *M) : PM(M), M(M) {} - - FunctionPassManager &getPM() { - return PM; - } - - Module *getModule() const { return M; } - std::vector<AssertingVH<Function> > &getPendingFunctions() { - return PendingFunctions; - } -}; - - -class JIT : public ExecutionEngine { - /// types - typedef ValueMap<const BasicBlock *, void *> - BasicBlockAddressMapTy; - /// data - TargetMachine &TM; // The current target we are compiling to - TargetJITInfo &TJI; // The JITInfo for the target we are compiling to - JITCodeEmitter *JCE; // JCE object - JITMemoryManager *JMM; - std::vector<JITEventListener*> EventListeners; - - /// AllocateGVsWithCode - Some applications require that global variables and - /// code be allocated into the same region of memory, in which case this flag - /// should be set to true. Doing so breaks freeMachineCodeForFunction. - bool AllocateGVsWithCode; - - /// True while the JIT is generating code. Used to assert against recursive - /// entry. - bool isAlreadyCodeGenerating; - - JITState *jitstate; - - /// BasicBlockAddressMap - A mapping between LLVM basic blocks and their - /// actualized version, only filled for basic blocks that have their address - /// taken. - BasicBlockAddressMapTy BasicBlockAddressMap; - - - JIT(Module *M, TargetMachine &tm, TargetJITInfo &tji, - JITMemoryManager *JMM, bool AllocateGVsWithCode); -public: - ~JIT(); - - static void Register() { - JITCtor = createJIT; - } - - /// getJITInfo - Return the target JIT information structure. - /// - TargetJITInfo &getJITInfo() const { return TJI; } - - /// create - Create an return a new JIT compiler if there is one available - /// for the current target. Otherwise, return null. - /// - static ExecutionEngine *create(Module *M, - std::string *Err, - JITMemoryManager *JMM, - CodeGenOpt::Level OptLevel = - CodeGenOpt::Default, - bool GVsWithCode = true, - Reloc::Model RM = Reloc::Default, - CodeModel::Model CMM = CodeModel::JITDefault) { - return ExecutionEngine::createJIT(M, Err, JMM, OptLevel, GVsWithCode, - RM, CMM); - } - - void addModule(Module *M) override; - - /// removeModule - Remove a Module from the list of modules. Returns true if - /// M is found. - bool removeModule(Module *M) override; - - /// runFunction - Start execution with the specified function and arguments. - /// - GenericValue runFunction(Function *F, - const std::vector<GenericValue> &ArgValues) override; - - /// getPointerToNamedFunction - This method returns the address of the - /// specified function by using the MemoryManager. As such it is only - /// useful for resolving library symbols, not code generated symbols. - /// - /// If AbortOnFailure is false and no function with the given name is - /// found, this function silently returns a null pointer. Otherwise, - /// it prints a message to stderr and aborts. - /// - void *getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure = true) override; - - // CompilationCallback - Invoked the first time that a call site is found, - // which causes lazy compilation of the target function. - // - static void CompilationCallback(); - - /// getPointerToFunction - This returns the address of the specified function, - /// compiling it if necessary. - /// - void *getPointerToFunction(Function *F) override; - - /// addPointerToBasicBlock - Adds address of the specific basic block. - void addPointerToBasicBlock(const BasicBlock *BB, void *Addr); - - /// clearPointerToBasicBlock - Removes address of specific basic block. - void clearPointerToBasicBlock(const BasicBlock *BB); - - /// getPointerToBasicBlock - This returns the address of the specified basic - /// block, assuming function is compiled. - void *getPointerToBasicBlock(BasicBlock *BB) override; - - /// getOrEmitGlobalVariable - Return the address of the specified global - /// variable, possibly emitting it to memory if needed. This is used by the - /// Emitter. - void *getOrEmitGlobalVariable(const GlobalVariable *GV) override; - - /// getPointerToFunctionOrStub - If the specified function has been - /// code-gen'd, return a pointer to the function. If not, compile it, or use - /// a stub to implement lazy compilation if available. - /// - void *getPointerToFunctionOrStub(Function *F) override; - - /// recompileAndRelinkFunction - This method is used to force a function - /// which has already been compiled, to be compiled again, possibly - /// after it has been modified. Then the entry to the old copy is overwritten - /// with a branch to the new copy. If there was no old copy, this acts - /// just like JIT::getPointerToFunction(). - /// - void *recompileAndRelinkFunction(Function *F) override; - - /// freeMachineCodeForFunction - deallocate memory used to code-generate this - /// Function. - /// - void freeMachineCodeForFunction(Function *F) override; - - /// addPendingFunction - while jitting non-lazily, a called but non-codegen'd - /// function was encountered. Add it to a pending list to be processed after - /// the current function. - /// - void addPendingFunction(Function *F); - - /// getCodeEmitter - Return the code emitter this JIT is emitting into. - /// - JITCodeEmitter *getCodeEmitter() const { return JCE; } - - static ExecutionEngine *createJIT(Module *M, - std::string *ErrorStr, - JITMemoryManager *JMM, - bool GVsWithCode, - TargetMachine *TM); - - // Run the JIT on F and return information about the generated code - void runJITOnFunction(Function *F, MachineCodeInfo *MCI = nullptr) override; - - void RegisterJITEventListener(JITEventListener *L) override; - void UnregisterJITEventListener(JITEventListener *L) override; - - TargetMachine *getTargetMachine() override { return &TM; } - - /// These functions correspond to the methods on JITEventListener. They - /// iterate over the registered listeners and call the corresponding method on - /// each. - void NotifyFunctionEmitted( - const Function &F, void *Code, size_t Size, - const JITEvent_EmittedFunctionDetails &Details); - void NotifyFreeingMachineCode(void *OldPtr); - - BasicBlockAddressMapTy & - getBasicBlockAddressMap() { - return BasicBlockAddressMap; - } - - -private: - static JITCodeEmitter *createEmitter(JIT &J, JITMemoryManager *JMM, - TargetMachine &tm); - void runJITOnFunctionUnlocked(Function *F); - void updateFunctionStubUnlocked(Function *F); - void jitTheFunctionUnlocked(Function *F); - -protected: - - /// getMemoryforGV - Allocate memory for a global variable. - char* getMemoryForGV(const GlobalVariable* GV) override; - -}; - -} // End llvm namespace - -#endif diff --git a/lib/ExecutionEngine/JIT/JITEmitter.cpp b/lib/ExecutionEngine/JIT/JITEmitter.cpp deleted file mode 100644 index 50b8c10..0000000 --- a/lib/ExecutionEngine/JIT/JITEmitter.cpp +++ /dev/null @@ -1,1256 +0,0 @@ -//===-- JITEmitter.cpp - Write machine code to executable memory ----------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines a MachineCodeEmitter object that is used by the JIT to -// write machine code to memory and remember where relocatable values are. -// -//===----------------------------------------------------------------------===// - -#include "JIT.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/CodeGen/JITCodeEmitter.h" -#include "llvm/CodeGen/MachineCodeInfo.h" -#include "llvm/CodeGen/MachineConstantPool.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineJumpTableInfo.h" -#include "llvm/CodeGen/MachineModuleInfo.h" -#include "llvm/CodeGen/MachineRelocation.h" -#include "llvm/ExecutionEngine/GenericValue.h" -#include "llvm/ExecutionEngine/JITEventListener.h" -#include "llvm/ExecutionEngine/JITMemoryManager.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/DebugInfo.h" -#include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/Operator.h" -#include "llvm/IR/ValueHandle.h" -#include "llvm/IR/ValueMap.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/Disassembler.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Memory.h" -#include "llvm/Support/MutexGuard.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetJITInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" -#include <algorithm> -#ifndef NDEBUG -#include <iomanip> -#endif -using namespace llvm; - -#define DEBUG_TYPE "jit" - -STATISTIC(NumBytes, "Number of bytes of machine code compiled"); -STATISTIC(NumRelos, "Number of relocations applied"); -STATISTIC(NumRetries, "Number of retries with more memory"); - - -// A declaration may stop being a declaration once it's fully read from bitcode. -// This function returns true if F is fully read and is still a declaration. -static bool isNonGhostDeclaration(const Function *F) { - return F->isDeclaration() && !F->isMaterializable(); -} - -//===----------------------------------------------------------------------===// -// JIT lazy compilation code. -// -namespace { - class JITEmitter; - class JITResolverState; - - template<typename ValueTy> - struct NoRAUWValueMapConfig : public ValueMapConfig<ValueTy> { - typedef JITResolverState *ExtraData; - static void onRAUW(JITResolverState *, Value *Old, Value *New) { - llvm_unreachable("The JIT doesn't know how to handle a" - " RAUW on a value it has emitted."); - } - }; - - struct CallSiteValueMapConfig : public NoRAUWValueMapConfig<Function*> { - typedef JITResolverState *ExtraData; - static void onDelete(JITResolverState *JRS, Function *F); - }; - - class JITResolverState { - public: - typedef ValueMap<Function*, void*, NoRAUWValueMapConfig<Function*> > - FunctionToLazyStubMapTy; - typedef std::map<void*, AssertingVH<Function> > CallSiteToFunctionMapTy; - typedef ValueMap<Function *, SmallPtrSet<void*, 1>, - CallSiteValueMapConfig> FunctionToCallSitesMapTy; - typedef std::map<AssertingVH<GlobalValue>, void*> GlobalToIndirectSymMapTy; - private: - /// FunctionToLazyStubMap - Keep track of the lazy stub created for a - /// particular function so that we can reuse them if necessary. - FunctionToLazyStubMapTy FunctionToLazyStubMap; - - /// CallSiteToFunctionMap - Keep track of the function that each lazy call - /// site corresponds to, and vice versa. - CallSiteToFunctionMapTy CallSiteToFunctionMap; - FunctionToCallSitesMapTy FunctionToCallSitesMap; - - /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a - /// particular GlobalVariable so that we can reuse them if necessary. - GlobalToIndirectSymMapTy GlobalToIndirectSymMap; - -#ifndef NDEBUG - /// Instance of the JIT this ResolverState serves. - JIT *TheJIT; -#endif - - public: - JITResolverState(JIT *jit) : FunctionToLazyStubMap(this), - FunctionToCallSitesMap(this) { -#ifndef NDEBUG - TheJIT = jit; -#endif - } - - FunctionToLazyStubMapTy& getFunctionToLazyStubMap() { - return FunctionToLazyStubMap; - } - - GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap() { - return GlobalToIndirectSymMap; - } - - std::pair<void *, Function *> LookupFunctionFromCallSite( - void *CallSite) const { - // The address given to us for the stub may not be exactly right, it - // might be a little bit after the stub. As such, use upper_bound to - // find it. - CallSiteToFunctionMapTy::const_iterator I = - CallSiteToFunctionMap.upper_bound(CallSite); - assert(I != CallSiteToFunctionMap.begin() && - "This is not a known call site!"); - --I; - return *I; - } - - void AddCallSite(void *CallSite, Function *F) { - bool Inserted = CallSiteToFunctionMap.insert( - std::make_pair(CallSite, F)).second; - (void)Inserted; - assert(Inserted && "Pair was already in CallSiteToFunctionMap"); - FunctionToCallSitesMap[F].insert(CallSite); - } - - void EraseAllCallSitesForPrelocked(Function *F); - - // Erases _all_ call sites regardless of their function. This is used to - // unregister the stub addresses from the StubToResolverMap in - // ~JITResolver(). - void EraseAllCallSitesPrelocked(); - }; - - /// JITResolver - Keep track of, and resolve, call sites for functions that - /// have not yet been compiled. - class JITResolver { - typedef JITResolverState::FunctionToLazyStubMapTy FunctionToLazyStubMapTy; - typedef JITResolverState::CallSiteToFunctionMapTy CallSiteToFunctionMapTy; - typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy; - - /// LazyResolverFn - The target lazy resolver function that we actually - /// rewrite instructions to use. - TargetJITInfo::LazyResolverFn LazyResolverFn; - - JITResolverState state; - - /// ExternalFnToStubMap - This is the equivalent of FunctionToLazyStubMap - /// for external functions. TODO: Of course, external functions don't need - /// a lazy stub. It's actually here to make it more likely that far calls - /// succeed, but no single stub can guarantee that. I'll remove this in a - /// subsequent checkin when I actually fix far calls. - std::map<void*, void*> ExternalFnToStubMap; - - /// revGOTMap - map addresses to indexes in the GOT - std::map<void*, unsigned> revGOTMap; - unsigned nextGOTIndex; - - JITEmitter &JE; - - /// Instance of JIT corresponding to this Resolver. - JIT *TheJIT; - - public: - explicit JITResolver(JIT &jit, JITEmitter &je) - : state(&jit), nextGOTIndex(0), JE(je), TheJIT(&jit) { - LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn); - } - - ~JITResolver(); - - /// getLazyFunctionStubIfAvailable - This returns a pointer to a function's - /// lazy-compilation stub if it has already been created. - void *getLazyFunctionStubIfAvailable(Function *F); - - /// getLazyFunctionStub - This returns a pointer to a function's - /// lazy-compilation stub, creating one on demand as needed. - void *getLazyFunctionStub(Function *F); - - /// getExternalFunctionStub - Return a stub for the function at the - /// specified address, created lazily on demand. - void *getExternalFunctionStub(void *FnAddr); - - /// getGlobalValueIndirectSym - Return an indirect symbol containing the - /// specified GV address. - void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress); - - /// getGOTIndexForAddress - Return a new or existing index in the GOT for - /// an address. This function only manages slots, it does not manage the - /// contents of the slots or the memory associated with the GOT. - unsigned getGOTIndexForAddr(void *addr); - - /// JITCompilerFn - This function is called to resolve a stub to a compiled - /// address. If the LLVM Function corresponding to the stub has not yet - /// been compiled, this function compiles it first. - static void *JITCompilerFn(void *Stub); - }; - - class StubToResolverMapTy { - /// Map a stub address to a specific instance of a JITResolver so that - /// lazily-compiled functions can find the right resolver to use. - /// - /// Guarded by Lock. - std::map<void*, JITResolver*> Map; - - /// Guards Map from concurrent accesses. - mutable sys::Mutex Lock; - - public: - /// Registers a Stub to be resolved by Resolver. - void RegisterStubResolver(void *Stub, JITResolver *Resolver) { - MutexGuard guard(Lock); - Map.insert(std::make_pair(Stub, Resolver)); - } - /// Unregisters the Stub when it's invalidated. - void UnregisterStubResolver(void *Stub) { - MutexGuard guard(Lock); - Map.erase(Stub); - } - /// Returns the JITResolver instance that owns the Stub. - JITResolver *getResolverFromStub(void *Stub) const { - MutexGuard guard(Lock); - // The address given to us for the stub may not be exactly right, it might - // be a little bit after the stub. As such, use upper_bound to find it. - // This is the same trick as in LookupFunctionFromCallSite from - // JITResolverState. - std::map<void*, JITResolver*>::const_iterator I = Map.upper_bound(Stub); - assert(I != Map.begin() && "This is not a known stub!"); - --I; - return I->second; - } - /// True if any stubs refer to the given resolver. Only used in an assert(). - /// O(N) - bool ResolverHasStubs(JITResolver* Resolver) const { - MutexGuard guard(Lock); - for (std::map<void*, JITResolver*>::const_iterator I = Map.begin(), - E = Map.end(); I != E; ++I) { - if (I->second == Resolver) - return true; - } - return false; - } - }; - /// This needs to be static so that a lazy call stub can access it with no - /// context except the address of the stub. - ManagedStatic<StubToResolverMapTy> StubToResolverMap; - - /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is - /// used to output functions to memory for execution. - class JITEmitter : public JITCodeEmitter { - JITMemoryManager *MemMgr; - - // When outputting a function stub in the context of some other function, we - // save BufferBegin/BufferEnd/CurBufferPtr here. - uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; - - // When reattempting to JIT a function after running out of space, we store - // the estimated size of the function we're trying to JIT here, so we can - // ask the memory manager for at least this much space. When we - // successfully emit the function, we reset this back to zero. - uintptr_t SizeEstimate; - - /// Relocations - These are the relocations that the function needs, as - /// emitted. - std::vector<MachineRelocation> Relocations; - - /// MBBLocations - This vector is a mapping from MBB ID's to their address. - /// It is filled in by the StartMachineBasicBlock callback and queried by - /// the getMachineBasicBlockAddress callback. - std::vector<uintptr_t> MBBLocations; - - /// ConstantPool - The constant pool for the current function. - /// - MachineConstantPool *ConstantPool; - - /// ConstantPoolBase - A pointer to the first entry in the constant pool. - /// - void *ConstantPoolBase; - - /// ConstPoolAddresses - Addresses of individual constant pool entries. - /// - SmallVector<uintptr_t, 8> ConstPoolAddresses; - - /// JumpTable - The jump tables for the current function. - /// - MachineJumpTableInfo *JumpTable; - - /// JumpTableBase - A pointer to the first entry in the jump table. - /// - void *JumpTableBase; - - /// Resolver - This contains info about the currently resolved functions. - JITResolver Resolver; - - /// LabelLocations - This vector is a mapping from Label ID's to their - /// address. - DenseMap<MCSymbol*, uintptr_t> LabelLocations; - - /// MMI - Machine module info for exception informations - MachineModuleInfo* MMI; - - // CurFn - The llvm function being emitted. Only valid during - // finishFunction(). - const Function *CurFn; - - /// Information about emitted code, which is passed to the - /// JITEventListeners. This is reset in startFunction and used in - /// finishFunction. - JITEvent_EmittedFunctionDetails EmissionDetails; - - struct EmittedCode { - void *FunctionBody; // Beginning of the function's allocation. - void *Code; // The address the function's code actually starts at. - void *ExceptionTable; - EmittedCode() : FunctionBody(nullptr), Code(nullptr), - ExceptionTable(nullptr) {} - }; - struct EmittedFunctionConfig : public ValueMapConfig<const Function*> { - typedef JITEmitter *ExtraData; - static void onDelete(JITEmitter *, const Function*); - static void onRAUW(JITEmitter *, const Function*, const Function*); - }; - ValueMap<const Function *, EmittedCode, - EmittedFunctionConfig> EmittedFunctions; - - DebugLoc PrevDL; - - /// Instance of the JIT - JIT *TheJIT; - - public: - JITEmitter(JIT &jit, JITMemoryManager *JMM, TargetMachine &TM) - : SizeEstimate(0), Resolver(jit, *this), MMI(nullptr), CurFn(nullptr), - EmittedFunctions(this), TheJIT(&jit) { - MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager(); - if (jit.getJITInfo().needsGOT()) { - MemMgr->AllocateGOT(); - DEBUG(dbgs() << "JIT is managing a GOT\n"); - } - - } - ~JITEmitter() { - delete MemMgr; - } - - JITResolver &getJITResolver() { return Resolver; } - - void startFunction(MachineFunction &F) override; - bool finishFunction(MachineFunction &F) override; - - void emitConstantPool(MachineConstantPool *MCP); - void initJumpTableInfo(MachineJumpTableInfo *MJTI); - void emitJumpTableInfo(MachineJumpTableInfo *MJTI); - - void startGVStub(const GlobalValue* GV, - unsigned StubSize, unsigned Alignment = 1); - void startGVStub(void *Buffer, unsigned StubSize); - void finishGVStub(); - void *allocIndirectGV(const GlobalValue *GV, const uint8_t *Buffer, - size_t Size, unsigned Alignment) override; - - /// allocateSpace - Reserves space in the current block if any, or - /// allocate a new one of the given size. - void *allocateSpace(uintptr_t Size, unsigned Alignment) override; - - /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace, - /// this method does not allocate memory in the current output buffer, - /// because a global may live longer than the current function. - void *allocateGlobal(uintptr_t Size, unsigned Alignment) override; - - void addRelocation(const MachineRelocation &MR) override { - Relocations.push_back(MR); - } - - void StartMachineBasicBlock(MachineBasicBlock *MBB) override { - if (MBBLocations.size() <= (unsigned)MBB->getNumber()) - MBBLocations.resize((MBB->getNumber()+1)*2); - MBBLocations[MBB->getNumber()] = getCurrentPCValue(); - if (MBB->hasAddressTaken()) - TheJIT->addPointerToBasicBlock(MBB->getBasicBlock(), - (void*)getCurrentPCValue()); - DEBUG(dbgs() << "JIT: Emitting BB" << MBB->getNumber() << " at [" - << (void*) getCurrentPCValue() << "]\n"); - } - - uintptr_t getConstantPoolEntryAddress(unsigned Entry) const override; - uintptr_t getJumpTableEntryAddress(unsigned Entry) const override; - - uintptr_t - getMachineBasicBlockAddress(MachineBasicBlock *MBB) const override { - assert(MBBLocations.size() > (unsigned)MBB->getNumber() && - MBBLocations[MBB->getNumber()] && "MBB not emitted!"); - return MBBLocations[MBB->getNumber()]; - } - - /// retryWithMoreMemory - Log a retry and deallocate all memory for the - /// given function. Increase the minimum allocation size so that we get - /// more memory next time. - void retryWithMoreMemory(MachineFunction &F); - - /// deallocateMemForFunction - Deallocate all memory for the specified - /// function body. - void deallocateMemForFunction(const Function *F); - - void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) override; - - void emitLabel(MCSymbol *Label) override { - LabelLocations[Label] = getCurrentPCValue(); - } - - DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() override { - return &LabelLocations; - } - - uintptr_t getLabelAddress(MCSymbol *Label) const override { - assert(LabelLocations.count(Label) && "Label not emitted!"); - return LabelLocations.find(Label)->second; - } - - void setModuleInfo(MachineModuleInfo* Info) override { - MMI = Info; - } - - private: - void *getPointerToGlobal(GlobalValue *GV, void *Reference, - bool MayNeedFarStub); - void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference); - }; -} - -void CallSiteValueMapConfig::onDelete(JITResolverState *JRS, Function *F) { - JRS->EraseAllCallSitesForPrelocked(F); -} - -void JITResolverState::EraseAllCallSitesForPrelocked(Function *F) { - FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F); - if (F2C == FunctionToCallSitesMap.end()) - return; - StubToResolverMapTy &S2RMap = *StubToResolverMap; - for (SmallPtrSet<void*, 1>::const_iterator I = F2C->second.begin(), - E = F2C->second.end(); I != E; ++I) { - S2RMap.UnregisterStubResolver(*I); - bool Erased = CallSiteToFunctionMap.erase(*I); - (void)Erased; - assert(Erased && "Missing call site->function mapping"); - } - FunctionToCallSitesMap.erase(F2C); -} - -void JITResolverState::EraseAllCallSitesPrelocked() { - StubToResolverMapTy &S2RMap = *StubToResolverMap; - for (CallSiteToFunctionMapTy::const_iterator - I = CallSiteToFunctionMap.begin(), - E = CallSiteToFunctionMap.end(); I != E; ++I) { - S2RMap.UnregisterStubResolver(I->first); - } - CallSiteToFunctionMap.clear(); - FunctionToCallSitesMap.clear(); -} - -JITResolver::~JITResolver() { - // No need to lock because we're in the destructor, and state isn't shared. - state.EraseAllCallSitesPrelocked(); - assert(!StubToResolverMap->ResolverHasStubs(this) && - "Resolver destroyed with stubs still alive."); -} - -/// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub -/// if it has already been created. -void *JITResolver::getLazyFunctionStubIfAvailable(Function *F) { - MutexGuard locked(TheJIT->lock); - - // If we already have a stub for this function, recycle it. - return state.getFunctionToLazyStubMap().lookup(F); -} - -/// getFunctionStub - This returns a pointer to a function stub, creating -/// one on demand as needed. -void *JITResolver::getLazyFunctionStub(Function *F) { - MutexGuard locked(TheJIT->lock); - - // If we already have a lazy stub for this function, recycle it. - void *&Stub = state.getFunctionToLazyStubMap()[F]; - if (Stub) return Stub; - - // Call the lazy resolver function if we are JIT'ing lazily. Otherwise we - // must resolve the symbol now. - void *Actual = TheJIT->isCompilingLazily() - ? (void *)(intptr_t)LazyResolverFn : (void *)nullptr; - - // If this is an external declaration, attempt to resolve the address now - // to place in the stub. - if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) { - Actual = TheJIT->getPointerToFunction(F); - - // If we resolved the symbol to a null address (eg. a weak external) - // don't emit a stub. Return a null pointer to the application. - if (!Actual) return nullptr; - } - - TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); - JE.startGVStub(F, SL.Size, SL.Alignment); - // Codegen a new stub, calling the lazy resolver or the actual address of the - // external function, if it was resolved. - Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, JE); - JE.finishGVStub(); - - if (Actual != (void*)(intptr_t)LazyResolverFn) { - // If we are getting the stub for an external function, we really want the - // address of the stub in the GlobalAddressMap for the JIT, not the address - // of the external function. - TheJIT->updateGlobalMapping(F, Stub); - } - - DEBUG(dbgs() << "JIT: Lazy stub emitted at [" << Stub << "] for function '" - << F->getName() << "'\n"); - - if (TheJIT->isCompilingLazily()) { - // Register this JITResolver as the one corresponding to this call site so - // JITCompilerFn will be able to find it. - StubToResolverMap->RegisterStubResolver(Stub, this); - - // Finally, keep track of the stub-to-Function mapping so that the - // JITCompilerFn knows which function to compile! - state.AddCallSite(Stub, F); - } else if (!Actual) { - // If we are JIT'ing non-lazily but need to call a function that does not - // exist yet, add it to the JIT's work list so that we can fill in the - // stub address later. - assert(!isNonGhostDeclaration(F) && !F->hasAvailableExternallyLinkage() && - "'Actual' should have been set above."); - TheJIT->addPendingFunction(F); - } - - return Stub; -} - -/// getGlobalValueIndirectSym - Return a lazy pointer containing the specified -/// GV address. -void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) { - MutexGuard locked(TheJIT->lock); - - // If we already have a stub for this global variable, recycle it. - void *&IndirectSym = state.getGlobalToIndirectSymMap()[GV]; - if (IndirectSym) return IndirectSym; - - // Otherwise, codegen a new indirect symbol. - IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress, - JE); - - DEBUG(dbgs() << "JIT: Indirect symbol emitted at [" << IndirectSym - << "] for GV '" << GV->getName() << "'\n"); - - return IndirectSym; -} - -/// getExternalFunctionStub - Return a stub for the function at the -/// specified address, created lazily on demand. -void *JITResolver::getExternalFunctionStub(void *FnAddr) { - // If we already have a stub for this function, recycle it. - void *&Stub = ExternalFnToStubMap[FnAddr]; - if (Stub) return Stub; - - TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); - JE.startGVStub(nullptr, SL.Size, SL.Alignment); - Stub = TheJIT->getJITInfo().emitFunctionStub(nullptr, FnAddr, JE); - JE.finishGVStub(); - - DEBUG(dbgs() << "JIT: Stub emitted at [" << Stub - << "] for external function at '" << FnAddr << "'\n"); - return Stub; -} - -unsigned JITResolver::getGOTIndexForAddr(void* addr) { - unsigned idx = revGOTMap[addr]; - if (!idx) { - idx = ++nextGOTIndex; - revGOTMap[addr] = idx; - DEBUG(dbgs() << "JIT: Adding GOT entry " << idx << " for addr [" - << addr << "]\n"); - } - return idx; -} - -/// JITCompilerFn - This function is called when a lazy compilation stub has -/// been entered. It looks up which function this stub corresponds to, compiles -/// it if necessary, then returns the resultant function pointer. -void *JITResolver::JITCompilerFn(void *Stub) { - JITResolver *JR = StubToResolverMap->getResolverFromStub(Stub); - assert(JR && "Unable to find the corresponding JITResolver to the call site"); - - Function* F = nullptr; - void* ActualPtr = nullptr; - - { - // Only lock for getting the Function. The call getPointerToFunction made - // in this function might trigger function materializing, which requires - // JIT lock to be unlocked. - MutexGuard locked(JR->TheJIT->lock); - - // The address given to us for the stub may not be exactly right, it might - // be a little bit after the stub. As such, use upper_bound to find it. - std::pair<void*, Function*> I = - JR->state.LookupFunctionFromCallSite(Stub); - F = I.second; - ActualPtr = I.first; - } - - // If we have already code generated the function, just return the address. - void *Result = JR->TheJIT->getPointerToGlobalIfAvailable(F); - - if (!Result) { - // Otherwise we don't have it, do lazy compilation now. - - // If lazy compilation is disabled, emit a useful error message and abort. - if (!JR->TheJIT->isCompilingLazily()) { - report_fatal_error("LLVM JIT requested to do lazy compilation of" - " function '" - + F->getName() + "' when lazy compiles are disabled!"); - } - - DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName() - << "' In stub ptr = " << Stub << " actual ptr = " - << ActualPtr << "\n"); - (void)ActualPtr; - - Result = JR->TheJIT->getPointerToFunction(F); - } - - // Reacquire the lock to update the GOT map. - MutexGuard locked(JR->TheJIT->lock); - - // We might like to remove the call site from the CallSiteToFunction map, but - // we can't do that! Multiple threads could be stuck, waiting to acquire the - // lock above. As soon as the 1st function finishes compiling the function, - // the next one will be released, and needs to be able to find the function it - // needs to call. - - // FIXME: We could rewrite all references to this stub if we knew them. - - // What we will do is set the compiled function address to map to the - // same GOT entry as the stub so that later clients may update the GOT - // if they see it still using the stub address. - // Note: this is done so the Resolver doesn't have to manage GOT memory - // Do this without allocating map space if the target isn't using a GOT - if(JR->revGOTMap.find(Stub) != JR->revGOTMap.end()) - JR->revGOTMap[Result] = JR->revGOTMap[Stub]; - - return Result; -} - -//===----------------------------------------------------------------------===// -// JITEmitter code. -// - -static GlobalObject *getSimpleAliasee(Constant *C) { - C = C->stripPointerCasts(); - return dyn_cast<GlobalObject>(C); -} - -void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, - bool MayNeedFarStub) { - if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) - return TheJIT->getOrEmitGlobalVariable(GV); - - if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) { - // We can only handle simple cases. - if (GlobalValue *GV = getSimpleAliasee(GA->getAliasee())) - return TheJIT->getPointerToGlobal(GV); - return nullptr; - } - - // If we have already compiled the function, return a pointer to its body. - Function *F = cast<Function>(V); - - void *FnStub = Resolver.getLazyFunctionStubIfAvailable(F); - if (FnStub) { - // Return the function stub if it's already created. We do this first so - // that we're returning the same address for the function as any previous - // call. TODO: Yes, this is wrong. The lazy stub isn't guaranteed to be - // close enough to call. - return FnStub; - } - - // If we know the target can handle arbitrary-distance calls, try to - // return a direct pointer. - if (!MayNeedFarStub) { - // If we have code, go ahead and return that. - void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); - if (ResultPtr) return ResultPtr; - - // If this is an external function pointer, we can force the JIT to - // 'compile' it, which really just adds it to the map. - if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) - return TheJIT->getPointerToFunction(F); - } - - // Otherwise, we may need a to emit a stub, and, conservatively, we always do - // so. Note that it's possible to return null from getLazyFunctionStub in the - // case of a weak extern that fails to resolve. - return Resolver.getLazyFunctionStub(F); -} - -void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) { - // Make sure GV is emitted first, and create a stub containing the fully - // resolved address. - void *GVAddress = getPointerToGlobal(V, Reference, false); - void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress); - return StubAddr; -} - -void JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) { - if (DL.isUnknown()) return; - if (!BeforePrintingInsn) return; - - const LLVMContext &Context = EmissionDetails.MF->getFunction()->getContext(); - - if (DL.getScope(Context) != nullptr && PrevDL != DL) { - JITEvent_EmittedFunctionDetails::LineStart NextLine; - NextLine.Address = getCurrentPCValue(); - NextLine.Loc = DL; - EmissionDetails.LineStarts.push_back(NextLine); - } - - PrevDL = DL; -} - -static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP, - const DataLayout *TD) { - const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); - if (Constants.empty()) return 0; - - unsigned Size = 0; - for (unsigned i = 0, e = Constants.size(); i != e; ++i) { - MachineConstantPoolEntry CPE = Constants[i]; - unsigned AlignMask = CPE.getAlignment() - 1; - Size = (Size + AlignMask) & ~AlignMask; - Type *Ty = CPE.getType(); - Size += TD->getTypeAllocSize(Ty); - } - return Size; -} - -void JITEmitter::startFunction(MachineFunction &F) { - DEBUG(dbgs() << "JIT: Starting CodeGen of Function " - << F.getName() << "\n"); - - uintptr_t ActualSize = 0; - // Set the memory writable, if it's not already - MemMgr->setMemoryWritable(); - - if (SizeEstimate > 0) { - // SizeEstimate will be non-zero on reallocation attempts. - ActualSize = SizeEstimate; - } - - BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(), - ActualSize); - BufferEnd = BufferBegin+ActualSize; - EmittedFunctions[F.getFunction()].FunctionBody = BufferBegin; - - // Ensure the constant pool/jump table info is at least 4-byte aligned. - emitAlignment(16); - - emitConstantPool(F.getConstantPool()); - if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) - initJumpTableInfo(MJTI); - - // About to start emitting the machine code for the function. - emitAlignment(std::max(F.getFunction()->getAlignment(), 8U)); - TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr); - EmittedFunctions[F.getFunction()].Code = CurBufferPtr; - - MBBLocations.clear(); - - EmissionDetails.MF = &F; - EmissionDetails.LineStarts.clear(); -} - -bool JITEmitter::finishFunction(MachineFunction &F) { - if (CurBufferPtr == BufferEnd) { - // We must call endFunctionBody before retrying, because - // deallocateMemForFunction requires it. - MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); - retryWithMoreMemory(F); - return true; - } - - if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) - emitJumpTableInfo(MJTI); - - // FnStart is the start of the text, not the start of the constant pool and - // other per-function data. - uint8_t *FnStart = - (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction()); - - // FnEnd is the end of the function's machine code. - uint8_t *FnEnd = CurBufferPtr; - - if (!Relocations.empty()) { - CurFn = F.getFunction(); - NumRelos += Relocations.size(); - - // Resolve the relocations to concrete pointers. - for (unsigned i = 0, e = Relocations.size(); i != e; ++i) { - MachineRelocation &MR = Relocations[i]; - void *ResultPtr = nullptr; - if (!MR.letTargetResolve()) { - if (MR.isExternalSymbol()) { - ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(), - false); - DEBUG(dbgs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to [" - << ResultPtr << "]\n"); - - // If the target REALLY wants a stub for this function, emit it now. - if (MR.mayNeedFarStub()) { - ResultPtr = Resolver.getExternalFunctionStub(ResultPtr); - } - } else if (MR.isGlobalValue()) { - ResultPtr = getPointerToGlobal(MR.getGlobalValue(), - BufferBegin+MR.getMachineCodeOffset(), - MR.mayNeedFarStub()); - } else if (MR.isIndirectSymbol()) { - ResultPtr = getPointerToGVIndirectSym( - MR.getGlobalValue(), BufferBegin+MR.getMachineCodeOffset()); - } else if (MR.isBasicBlock()) { - ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock()); - } else if (MR.isConstantPoolIndex()) { - ResultPtr = - (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); - } else { - assert(MR.isJumpTableIndex()); - ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex()); - } - - MR.setResultPointer(ResultPtr); - } - - // if we are managing the GOT and the relocation wants an index, - // give it one - if (MR.isGOTRelative() && MemMgr->isManagingGOT()) { - unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr); - MR.setGOTIndex(idx); - if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) { - DEBUG(dbgs() << "JIT: GOT was out of date for " << ResultPtr - << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] - << "\n"); - ((void**)MemMgr->getGOTBase())[idx] = ResultPtr; - } - } - } - - CurFn = nullptr; - TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0], - Relocations.size(), MemMgr->getGOTBase()); - } - - // Update the GOT entry for F to point to the new code. - if (MemMgr->isManagingGOT()) { - unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin); - if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) { - DEBUG(dbgs() << "JIT: GOT was out of date for " << (void*)BufferBegin - << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] - << "\n"); - ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin; - } - } - - // CurBufferPtr may have moved beyond FnEnd, due to memory allocation for - // global variables that were referenced in the relocations. - MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); - - if (CurBufferPtr == BufferEnd) { - retryWithMoreMemory(F); - return true; - } else { - // Now that we've succeeded in emitting the function, reset the - // SizeEstimate back down to zero. - SizeEstimate = 0; - } - - BufferBegin = CurBufferPtr = nullptr; - NumBytes += FnEnd-FnStart; - - // Invalidate the icache if necessary. - sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart); - - TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart, - EmissionDetails); - - // Reset the previous debug location. - PrevDL = DebugLoc(); - - DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart - << "] Function: " << F.getName() - << ": " << (FnEnd-FnStart) << " bytes of text, " - << Relocations.size() << " relocations\n"); - - Relocations.clear(); - ConstPoolAddresses.clear(); - - // Mark code region readable and executable if it's not so already. - MemMgr->setMemoryExecutable(); - - DEBUG({ - if (sys::hasDisassembler()) { - dbgs() << "JIT: Disassembled code:\n"; - dbgs() << sys::disassembleBuffer(FnStart, FnEnd-FnStart, - (uintptr_t)FnStart); - } else { - dbgs() << "JIT: Binary code:\n"; - uint8_t* q = FnStart; - for (int i = 0; q < FnEnd; q += 4, ++i) { - if (i == 4) - i = 0; - if (i == 0) - dbgs() << "JIT: " << (long)(q - FnStart) << ": "; - bool Done = false; - for (int j = 3; j >= 0; --j) { - if (q + j >= FnEnd) - Done = true; - else - dbgs() << (unsigned short)q[j]; - } - if (Done) - break; - dbgs() << ' '; - if (i == 3) - dbgs() << '\n'; - } - dbgs()<< '\n'; - } - }); - - if (MMI) - MMI->EndFunction(); - - return false; -} - -void JITEmitter::retryWithMoreMemory(MachineFunction &F) { - DEBUG(dbgs() << "JIT: Ran out of space for native code. Reattempting.\n"); - Relocations.clear(); // Clear the old relocations or we'll reapply them. - ConstPoolAddresses.clear(); - ++NumRetries; - deallocateMemForFunction(F.getFunction()); - // Try again with at least twice as much free space. - SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin)); - - for (MachineFunction::iterator MBB = F.begin(), E = F.end(); MBB != E; ++MBB){ - if (MBB->hasAddressTaken()) - TheJIT->clearPointerToBasicBlock(MBB->getBasicBlock()); - } -} - -/// deallocateMemForFunction - Deallocate all memory for the specified -/// function body. Also drop any references the function has to stubs. -/// May be called while the Function is being destroyed inside ~Value(). -void JITEmitter::deallocateMemForFunction(const Function *F) { - ValueMap<const Function *, EmittedCode, EmittedFunctionConfig>::iterator - Emitted = EmittedFunctions.find(F); - if (Emitted != EmittedFunctions.end()) { - MemMgr->deallocateFunctionBody(Emitted->second.FunctionBody); - TheJIT->NotifyFreeingMachineCode(Emitted->second.Code); - - EmittedFunctions.erase(Emitted); - } -} - - -void *JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { - if (BufferBegin) - return JITCodeEmitter::allocateSpace(Size, Alignment); - - // create a new memory block if there is no active one. - // care must be taken so that BufferBegin is invalidated when a - // block is trimmed - BufferBegin = CurBufferPtr = MemMgr->allocateSpace(Size, Alignment); - BufferEnd = BufferBegin+Size; - return CurBufferPtr; -} - -void *JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) { - // Delegate this call through the memory manager. - return MemMgr->allocateGlobal(Size, Alignment); -} - -void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { - if (TheJIT->getJITInfo().hasCustomConstantPool()) - return; - - const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); - if (Constants.empty()) return; - - unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getDataLayout()); - unsigned Align = MCP->getConstantPoolAlignment(); - ConstantPoolBase = allocateSpace(Size, Align); - ConstantPool = MCP; - - if (!ConstantPoolBase) return; // Buffer overflow. - - DEBUG(dbgs() << "JIT: Emitted constant pool at [" << ConstantPoolBase - << "] (size: " << Size << ", alignment: " << Align << ")\n"); - - // Initialize the memory for all of the constant pool entries. - unsigned Offset = 0; - for (unsigned i = 0, e = Constants.size(); i != e; ++i) { - MachineConstantPoolEntry CPE = Constants[i]; - unsigned AlignMask = CPE.getAlignment() - 1; - Offset = (Offset + AlignMask) & ~AlignMask; - - uintptr_t CAddr = (uintptr_t)ConstantPoolBase + Offset; - ConstPoolAddresses.push_back(CAddr); - if (CPE.isMachineConstantPoolEntry()) { - // FIXME: add support to lower machine constant pool values into bytes! - report_fatal_error("Initialize memory with machine specific constant pool" - "entry has not been implemented!"); - } - TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr); - DEBUG(dbgs() << "JIT: CP" << i << " at [0x"; - dbgs().write_hex(CAddr) << "]\n"); - - Type *Ty = CPE.Val.ConstVal->getType(); - Offset += TheJIT->getDataLayout()->getTypeAllocSize(Ty); - } -} - -void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) { - if (TheJIT->getJITInfo().hasCustomJumpTables()) - return; - if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline) - return; - - const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); - if (JT.empty()) return; - - unsigned NumEntries = 0; - for (unsigned i = 0, e = JT.size(); i != e; ++i) - NumEntries += JT[i].MBBs.size(); - - unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getDataLayout()); - - // Just allocate space for all the jump tables now. We will fix up the actual - // MBB entries in the tables after we emit the code for each block, since then - // we will know the final locations of the MBBs in memory. - JumpTable = MJTI; - JumpTableBase = allocateSpace(NumEntries * EntrySize, - MJTI->getEntryAlignment(*TheJIT->getDataLayout())); -} - -void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { - if (TheJIT->getJITInfo().hasCustomJumpTables()) - return; - - const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); - if (JT.empty() || !JumpTableBase) return; - - - switch (MJTI->getEntryKind()) { - case MachineJumpTableInfo::EK_Inline: - return; - case MachineJumpTableInfo::EK_BlockAddress: { - // EK_BlockAddress - Each entry is a plain address of block, e.g.: - // .word LBB123 - assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == sizeof(void*) && - "Cross JIT'ing?"); - - // For each jump table, map each target in the jump table to the address of - // an emitted MachineBasicBlock. - intptr_t *SlotPtr = (intptr_t*)JumpTableBase; - - for (unsigned i = 0, e = JT.size(); i != e; ++i) { - const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; - // Store the address of the basic block for this jump table slot in the - // memory we allocated for the jump table in 'initJumpTableInfo' - for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) - *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); - } - break; - } - - case MachineJumpTableInfo::EK_Custom32: - case MachineJumpTableInfo::EK_GPRel32BlockAddress: - case MachineJumpTableInfo::EK_LabelDifference32: { - assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == 4&&"Cross JIT'ing?"); - // For each jump table, place the offset from the beginning of the table - // to the target address. - int *SlotPtr = (int*)JumpTableBase; - - for (unsigned i = 0, e = JT.size(); i != e; ++i) { - const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; - // Store the offset of the basic block for this jump table slot in the - // memory we allocated for the jump table in 'initJumpTableInfo' - uintptr_t Base = (uintptr_t)SlotPtr; - for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) { - uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]); - /// FIXME: USe EntryKind instead of magic "getPICJumpTableEntry" hook. - *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base); - } - } - break; - } - case MachineJumpTableInfo::EK_GPRel64BlockAddress: - llvm_unreachable( - "JT Info emission not implemented for GPRel64BlockAddress yet."); - } -} - -void JITEmitter::startGVStub(const GlobalValue* GV, - unsigned StubSize, unsigned Alignment) { - SavedBufferBegin = BufferBegin; - SavedBufferEnd = BufferEnd; - SavedCurBufferPtr = CurBufferPtr; - - BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment); - BufferEnd = BufferBegin+StubSize+1; -} - -void JITEmitter::startGVStub(void *Buffer, unsigned StubSize) { - SavedBufferBegin = BufferBegin; - SavedBufferEnd = BufferEnd; - SavedCurBufferPtr = CurBufferPtr; - - BufferBegin = CurBufferPtr = (uint8_t *)Buffer; - BufferEnd = BufferBegin+StubSize+1; -} - -void JITEmitter::finishGVStub() { - assert(CurBufferPtr != BufferEnd && "Stub overflowed allocated space."); - NumBytes += getCurrentPCOffset(); - BufferBegin = SavedBufferBegin; - BufferEnd = SavedBufferEnd; - CurBufferPtr = SavedCurBufferPtr; -} - -void *JITEmitter::allocIndirectGV(const GlobalValue *GV, - const uint8_t *Buffer, size_t Size, - unsigned Alignment) { - uint8_t *IndGV = MemMgr->allocateStub(GV, Size, Alignment); - memcpy(IndGV, Buffer, Size); - return IndGV; -} - -// getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry -// in the constant pool that was last emitted with the 'emitConstantPool' -// method. -// -uintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const { - assert(ConstantNum < ConstantPool->getConstants().size() && - "Invalid ConstantPoolIndex!"); - return ConstPoolAddresses[ConstantNum]; -} - -// getJumpTableEntryAddress - Return the address of the JumpTable with index -// 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo' -// -uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const { - const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables(); - assert(Index < JT.size() && "Invalid jump table index!"); - - unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getDataLayout()); - - unsigned Offset = 0; - for (unsigned i = 0; i < Index; ++i) - Offset += JT[i].MBBs.size(); - - Offset *= EntrySize; - - return (uintptr_t)((char *)JumpTableBase + Offset); -} - -void JITEmitter::EmittedFunctionConfig::onDelete( - JITEmitter *Emitter, const Function *F) { - Emitter->deallocateMemForFunction(F); -} -void JITEmitter::EmittedFunctionConfig::onRAUW( - JITEmitter *, const Function*, const Function*) { - llvm_unreachable("The JIT doesn't know how to handle a" - " RAUW on a value it has emitted."); -} - - -//===----------------------------------------------------------------------===// -// Public interface to this file -//===----------------------------------------------------------------------===// - -JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM, - TargetMachine &tm) { - return new JITEmitter(jit, JMM, tm); -} - -// getPointerToFunctionOrStub - If the specified function has been -// code-gen'd, return a pointer to the function. If not, compile it, or use -// a stub to implement lazy compilation if available. -// -void *JIT::getPointerToFunctionOrStub(Function *F) { - // If we have already code generated the function, just return the address. - if (void *Addr = getPointerToGlobalIfAvailable(F)) - return Addr; - - // Get a stub if the target supports it. - JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter()); - return JE->getJITResolver().getLazyFunctionStub(F); -} - -void JIT::updateFunctionStubUnlocked(Function *F) { - // Get the empty stub we generated earlier. - JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter()); - void *Stub = JE->getJITResolver().getLazyFunctionStub(F); - void *Addr = getPointerToGlobalIfAvailable(F); - assert(Addr != Stub && "Function must have non-stub address to be updated."); - - // Tell the target jit info to rewrite the stub at the specified address, - // rather than creating a new one. - TargetJITInfo::StubLayout layout = getJITInfo().getStubLayout(); - JE->startGVStub(Stub, layout.Size); - getJITInfo().emitFunctionStub(F, Addr, *getCodeEmitter()); - JE->finishGVStub(); -} - -/// freeMachineCodeForFunction - release machine code memory for given Function. -/// -void JIT::freeMachineCodeForFunction(Function *F) { - // Delete translation for this from the ExecutionEngine, so it will get - // retranslated next time it is used. - updateGlobalMapping(F, nullptr); - - // Free the actual memory for the function body and related stuff. - static_cast<JITEmitter*>(JCE)->deallocateMemForFunction(F); -} diff --git a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp b/lib/ExecutionEngine/JIT/JITMemoryManager.cpp deleted file mode 100644 index 584b93f..0000000 --- a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp +++ /dev/null @@ -1,904 +0,0 @@ -//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines the DefaultJITMemoryManager class. -// -//===----------------------------------------------------------------------===// - -#include "llvm/ExecutionEngine/JITMemoryManager.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Config/config.h" -#include "llvm/IR/GlobalValue.h" -#include "llvm/Support/Allocator.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/DynamicLibrary.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/Memory.h" -#include "llvm/Support/raw_ostream.h" -#include <cassert> -#include <climits> -#include <cstring> -#include <vector> - -#if defined(__linux__) -#if defined(HAVE_SYS_STAT_H) -#include <sys/stat.h> -#endif -#include <fcntl.h> -#include <unistd.h> -#endif - -using namespace llvm; - -#define DEBUG_TYPE "jit" - -STATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT"); - -JITMemoryManager::~JITMemoryManager() {} - -//===----------------------------------------------------------------------===// -// Memory Block Implementation. -//===----------------------------------------------------------------------===// - -namespace { - /// MemoryRangeHeader - For a range of memory, this is the header that we put - /// on the block of memory. It is carefully crafted to be one word of memory. - /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader - /// which starts with this. - struct FreeRangeHeader; - struct MemoryRangeHeader { - /// ThisAllocated - This is true if this block is currently allocated. If - /// not, this can be converted to a FreeRangeHeader. - unsigned ThisAllocated : 1; - - /// PrevAllocated - Keep track of whether the block immediately before us is - /// allocated. If not, the word immediately before this header is the size - /// of the previous block. - unsigned PrevAllocated : 1; - - /// BlockSize - This is the size in bytes of this memory block, - /// including this header. - uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2); - - - /// getBlockAfter - Return the memory block immediately after this one. - /// - MemoryRangeHeader &getBlockAfter() const { - return *reinterpret_cast<MemoryRangeHeader *>( - reinterpret_cast<char*>( - const_cast<MemoryRangeHeader *>(this))+BlockSize); - } - - /// getFreeBlockBefore - If the block before this one is free, return it, - /// otherwise return null. - FreeRangeHeader *getFreeBlockBefore() const { - if (PrevAllocated) return nullptr; - intptr_t PrevSize = reinterpret_cast<intptr_t *>( - const_cast<MemoryRangeHeader *>(this))[-1]; - return reinterpret_cast<FreeRangeHeader *>( - reinterpret_cast<char*>( - const_cast<MemoryRangeHeader *>(this))-PrevSize); - } - - /// FreeBlock - Turn an allocated block into a free block, adjusting - /// bits in the object headers, and adding an end of region memory block. - FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList); - - /// TrimAllocationToSize - If this allocated block is significantly larger - /// than NewSize, split it into two pieces (where the former is NewSize - /// bytes, including the header), and add the new block to the free list. - FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList, - uint64_t NewSize); - }; - - /// FreeRangeHeader - For a memory block that isn't already allocated, this - /// keeps track of the current block and has a pointer to the next free block. - /// Free blocks are kept on a circularly linked list. - struct FreeRangeHeader : public MemoryRangeHeader { - FreeRangeHeader *Prev; - FreeRangeHeader *Next; - - /// getMinBlockSize - Get the minimum size for a memory block. Blocks - /// smaller than this size cannot be created. - static unsigned getMinBlockSize() { - return sizeof(FreeRangeHeader)+sizeof(intptr_t); - } - - /// SetEndOfBlockSizeMarker - The word at the end of every free block is - /// known to be the size of the free block. Set it for this block. - void SetEndOfBlockSizeMarker() { - void *EndOfBlock = (char*)this + BlockSize; - ((intptr_t *)EndOfBlock)[-1] = BlockSize; - } - - FreeRangeHeader *RemoveFromFreeList() { - assert(Next->Prev == this && Prev->Next == this && "Freelist broken!"); - Next->Prev = Prev; - return Prev->Next = Next; - } - - void AddToFreeList(FreeRangeHeader *FreeList) { - Next = FreeList; - Prev = FreeList->Prev; - Prev->Next = this; - Next->Prev = this; - } - - /// GrowBlock - The block after this block just got deallocated. Merge it - /// into the current block. - void GrowBlock(uintptr_t NewSize); - - /// AllocateBlock - Mark this entire block allocated, updating freelists - /// etc. This returns a pointer to the circular free-list. - FreeRangeHeader *AllocateBlock(); - }; -} - - -/// AllocateBlock - Mark this entire block allocated, updating freelists -/// etc. This returns a pointer to the circular free-list. -FreeRangeHeader *FreeRangeHeader::AllocateBlock() { - assert(!ThisAllocated && !getBlockAfter().PrevAllocated && - "Cannot allocate an allocated block!"); - // Mark this block allocated. - ThisAllocated = 1; - getBlockAfter().PrevAllocated = 1; - - // Remove it from the free list. - return RemoveFromFreeList(); -} - -/// FreeBlock - Turn an allocated block into a free block, adjusting -/// bits in the object headers, and adding an end of region memory block. -/// If possible, coalesce this block with neighboring blocks. Return the -/// FreeRangeHeader to allocate from. -FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) { - MemoryRangeHeader *FollowingBlock = &getBlockAfter(); - assert(ThisAllocated && "This block is already free!"); - assert(FollowingBlock->PrevAllocated && "Flags out of sync!"); - - FreeRangeHeader *FreeListToReturn = FreeList; - - // If the block after this one is free, merge it into this block. - if (!FollowingBlock->ThisAllocated) { - FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock; - // "FreeList" always needs to be a valid free block. If we're about to - // coalesce with it, update our notion of what the free list is. - if (&FollowingFreeBlock == FreeList) { - FreeList = FollowingFreeBlock.Next; - FreeListToReturn = nullptr; - assert(&FollowingFreeBlock != FreeList && "No tombstone block?"); - } - FollowingFreeBlock.RemoveFromFreeList(); - - // Include the following block into this one. - BlockSize += FollowingFreeBlock.BlockSize; - FollowingBlock = &FollowingFreeBlock.getBlockAfter(); - - // Tell the block after the block we are coalescing that this block is - // allocated. - FollowingBlock->PrevAllocated = 1; - } - - assert(FollowingBlock->ThisAllocated && "Missed coalescing?"); - - if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) { - PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize); - return FreeListToReturn ? FreeListToReturn : PrevFreeBlock; - } - - // Otherwise, mark this block free. - FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this; - FollowingBlock->PrevAllocated = 0; - FreeBlock.ThisAllocated = 0; - - // Link this into the linked list of free blocks. - FreeBlock.AddToFreeList(FreeList); - - // Add a marker at the end of the block, indicating the size of this free - // block. - FreeBlock.SetEndOfBlockSizeMarker(); - return FreeListToReturn ? FreeListToReturn : &FreeBlock; -} - -/// GrowBlock - The block after this block just got deallocated. Merge it -/// into the current block. -void FreeRangeHeader::GrowBlock(uintptr_t NewSize) { - assert(NewSize > BlockSize && "Not growing block?"); - BlockSize = NewSize; - SetEndOfBlockSizeMarker(); - getBlockAfter().PrevAllocated = 0; -} - -/// TrimAllocationToSize - If this allocated block is significantly larger -/// than NewSize, split it into two pieces (where the former is NewSize -/// bytes, including the header), and add the new block to the free list. -FreeRangeHeader *MemoryRangeHeader:: -TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) { - assert(ThisAllocated && getBlockAfter().PrevAllocated && - "Cannot deallocate part of an allocated block!"); - - // Don't allow blocks to be trimmed below minimum required size - NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize); - - // Round up size for alignment of header. - unsigned HeaderAlign = __alignof(FreeRangeHeader); - NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1); - - // Size is now the size of the block we will remove from the start of the - // current block. - assert(NewSize <= BlockSize && - "Allocating more space from this block than exists!"); - - // If splitting this block will cause the remainder to be too small, do not - // split the block. - if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize()) - return FreeList; - - // Otherwise, we splice the required number of bytes out of this block, form - // a new block immediately after it, then mark this block allocated. - MemoryRangeHeader &FormerNextBlock = getBlockAfter(); - - // Change the size of this block. - BlockSize = NewSize; - - // Get the new block we just sliced out and turn it into a free block. - FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter(); - NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock; - NewNextBlock.ThisAllocated = 0; - NewNextBlock.PrevAllocated = 1; - NewNextBlock.SetEndOfBlockSizeMarker(); - FormerNextBlock.PrevAllocated = 0; - NewNextBlock.AddToFreeList(FreeList); - return &NewNextBlock; -} - -//===----------------------------------------------------------------------===// -// Memory Block Implementation. -//===----------------------------------------------------------------------===// - -namespace { - - class DefaultJITMemoryManager; - - class JITAllocator { - DefaultJITMemoryManager &JMM; - public: - JITAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { } - void *Allocate(size_t Size, size_t /*Alignment*/); - void Deallocate(void *Slab, size_t Size); - }; - - /// DefaultJITMemoryManager - Manage memory for the JIT code generation. - /// This splits a large block of MAP_NORESERVE'd memory into two - /// sections, one for function stubs, one for the functions themselves. We - /// have to do this because we may need to emit a function stub while in the - /// middle of emitting a function, and we don't know how large the function we - /// are emitting is. - class DefaultJITMemoryManager : public JITMemoryManager { - public: - /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at - /// least this much unless more is requested. Currently, in 512k slabs. - static const size_t DefaultCodeSlabSize = 512 * 1024; - - /// DefaultSlabSize - Allocate globals and stubs into slabs of 64K (probably - /// 16 pages) unless we get an allocation above SizeThreshold. - static const size_t DefaultSlabSize = 64 * 1024; - - /// DefaultSizeThreshold - For any allocation larger than 16K (probably - /// 4 pages), we should allocate a separate slab to avoid wasted space at - /// the end of a normal slab. - static const size_t DefaultSizeThreshold = 16 * 1024; - - private: - // Whether to poison freed memory. - bool PoisonMemory; - - /// LastSlab - This points to the last slab allocated and is used as the - /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all - /// stubs, data, and code contiguously in memory. In general, however, this - /// is not possible because the NearBlock parameter is ignored on Windows - /// platforms and even on Unix it works on a best-effort pasis. - sys::MemoryBlock LastSlab; - - // Memory slabs allocated by the JIT. We refer to them as slabs so we don't - // confuse them with the blocks of memory described above. - std::vector<sys::MemoryBlock> CodeSlabs; - BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize, - DefaultSizeThreshold> StubAllocator; - BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize, - DefaultSizeThreshold> DataAllocator; - - // Circular list of free blocks. - FreeRangeHeader *FreeMemoryList; - - // When emitting code into a memory block, this is the block. - MemoryRangeHeader *CurBlock; - - uint8_t *GOTBase; // Target Specific reserved memory - public: - DefaultJITMemoryManager(); - ~DefaultJITMemoryManager(); - - /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the - /// last slab it allocated, so that subsequent allocations follow it. - sys::MemoryBlock allocateNewSlab(size_t size); - - /// getPointerToNamedFunction - This method returns the address of the - /// specified function by using the dlsym function call. - void *getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure = true) override; - - void AllocateGOT() override; - - // Testing methods. - bool CheckInvariants(std::string &ErrorStr) override; - size_t GetDefaultCodeSlabSize() override { return DefaultCodeSlabSize; } - size_t GetDefaultDataSlabSize() override { return DefaultSlabSize; } - size_t GetDefaultStubSlabSize() override { return DefaultSlabSize; } - unsigned GetNumCodeSlabs() override { return CodeSlabs.size(); } - unsigned GetNumDataSlabs() override { return DataAllocator.GetNumSlabs(); } - unsigned GetNumStubSlabs() override { return StubAllocator.GetNumSlabs(); } - - /// startFunctionBody - When a function starts, allocate a block of free - /// executable memory, returning a pointer to it and its actual size. - uint8_t *startFunctionBody(const Function *F, - uintptr_t &ActualSize) override { - - FreeRangeHeader* candidateBlock = FreeMemoryList; - FreeRangeHeader* head = FreeMemoryList; - FreeRangeHeader* iter = head->Next; - - uintptr_t largest = candidateBlock->BlockSize; - - // Search for the largest free block - while (iter != head) { - if (iter->BlockSize > largest) { - largest = iter->BlockSize; - candidateBlock = iter; - } - iter = iter->Next; - } - - largest = largest - sizeof(MemoryRangeHeader); - - // If this block isn't big enough for the allocation desired, allocate - // another block of memory and add it to the free list. - if (largest < ActualSize || - largest <= FreeRangeHeader::getMinBlockSize()) { - DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); - candidateBlock = allocateNewCodeSlab((size_t)ActualSize); - } - - // Select this candidate block for allocation - CurBlock = candidateBlock; - - // Allocate the entire memory block. - FreeMemoryList = candidateBlock->AllocateBlock(); - ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader); - return (uint8_t *)(CurBlock + 1); - } - - /// allocateNewCodeSlab - Helper method to allocate a new slab of code - /// memory from the OS and add it to the free list. Returns the new - /// FreeRangeHeader at the base of the slab. - FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) { - // If the user needs at least MinSize free memory, then we account for - // two MemoryRangeHeaders: the one in the user's block, and the one at the - // end of the slab. - size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader); - size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin); - sys::MemoryBlock B = allocateNewSlab(SlabSize); - CodeSlabs.push_back(B); - char *MemBase = (char*)(B.base()); - - // Put a tiny allocated block at the end of the memory chunk, so when - // FreeBlock calls getBlockAfter it doesn't fall off the end. - MemoryRangeHeader *EndBlock = - (MemoryRangeHeader*)(MemBase + B.size()) - 1; - EndBlock->ThisAllocated = 1; - EndBlock->PrevAllocated = 0; - EndBlock->BlockSize = sizeof(MemoryRangeHeader); - - // Start out with a vast new block of free memory. - FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase; - NewBlock->ThisAllocated = 0; - // Make sure getFreeBlockBefore doesn't look into unmapped memory. - NewBlock->PrevAllocated = 1; - NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock; - NewBlock->SetEndOfBlockSizeMarker(); - NewBlock->AddToFreeList(FreeMemoryList); - - assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize && - "The block was too small!"); - return NewBlock; - } - - /// endFunctionBody - The function F is now allocated, and takes the memory - /// in the range [FunctionStart,FunctionEnd). - void endFunctionBody(const Function *F, uint8_t *FunctionStart, - uint8_t *FunctionEnd) override { - assert(FunctionEnd > FunctionStart); - assert(FunctionStart == (uint8_t *)(CurBlock+1) && - "Mismatched function start/end!"); - - uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock; - - // Release the memory at the end of this block that isn't needed. - FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); - } - - /// allocateSpace - Allocate a memory block of the given size. This method - /// cannot be called between calls to startFunctionBody and endFunctionBody. - uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) override { - CurBlock = FreeMemoryList; - FreeMemoryList = FreeMemoryList->AllocateBlock(); - - uint8_t *result = (uint8_t *)(CurBlock + 1); - - if (Alignment == 0) Alignment = 1; - result = (uint8_t*)(((intptr_t)result+Alignment-1) & - ~(intptr_t)(Alignment-1)); - - uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock; - FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); - - return result; - } - - /// allocateStub - Allocate memory for a function stub. - uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize, - unsigned Alignment) override { - return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment); - } - - /// allocateGlobal - Allocate memory for a global. - uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) override { - return (uint8_t*)DataAllocator.Allocate(Size, Alignment); - } - - /// allocateCodeSection - Allocate memory for a code section. - uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, - StringRef SectionName) override { - // Grow the required block size to account for the block header - Size += sizeof(*CurBlock); - - // Alignment handling. - if (!Alignment) - Alignment = 16; - Size += Alignment - 1; - - FreeRangeHeader* candidateBlock = FreeMemoryList; - FreeRangeHeader* head = FreeMemoryList; - FreeRangeHeader* iter = head->Next; - - uintptr_t largest = candidateBlock->BlockSize; - - // Search for the largest free block. - while (iter != head) { - if (iter->BlockSize > largest) { - largest = iter->BlockSize; - candidateBlock = iter; - } - iter = iter->Next; - } - - largest = largest - sizeof(MemoryRangeHeader); - - // If this block isn't big enough for the allocation desired, allocate - // another block of memory and add it to the free list. - if (largest < Size || largest <= FreeRangeHeader::getMinBlockSize()) { - DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); - candidateBlock = allocateNewCodeSlab((size_t)Size); - } - - // Select this candidate block for allocation - CurBlock = candidateBlock; - - // Allocate the entire memory block. - FreeMemoryList = candidateBlock->AllocateBlock(); - // Release the memory at the end of this block that isn't needed. - FreeMemoryList = CurBlock->TrimAllocationToSize(FreeMemoryList, Size); - uintptr_t unalignedAddr = (uintptr_t)CurBlock + sizeof(*CurBlock); - return (uint8_t*)RoundUpToAlignment((uint64_t)unalignedAddr, Alignment); - } - - /// allocateDataSection - Allocate memory for a data section. - uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, StringRef SectionName, - bool IsReadOnly) override { - return (uint8_t*)DataAllocator.Allocate(Size, Alignment); - } - - bool finalizeMemory(std::string *ErrMsg) override { - return false; - } - - uint8_t *getGOTBase() const override { - return GOTBase; - } - - void deallocateBlock(void *Block) { - // Find the block that is allocated for this function. - MemoryRangeHeader *MemRange = static_cast<MemoryRangeHeader*>(Block) - 1; - assert(MemRange->ThisAllocated && "Block isn't allocated!"); - - // Fill the buffer with garbage! - if (PoisonMemory) { - memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange)); - } - - // Free the memory. - FreeMemoryList = MemRange->FreeBlock(FreeMemoryList); - } - - /// deallocateFunctionBody - Deallocate all memory for the specified - /// function body. - void deallocateFunctionBody(void *Body) override { - if (Body) deallocateBlock(Body); - } - - /// setMemoryWritable - When code generation is in progress, - /// the code pages may need permissions changed. - void setMemoryWritable() override { - for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) - sys::Memory::setWritable(CodeSlabs[i]); - } - /// setMemoryExecutable - When code generation is done and we're ready to - /// start execution, the code pages may need permissions changed. - void setMemoryExecutable() override { - for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) - sys::Memory::setExecutable(CodeSlabs[i]); - } - - /// setPoisonMemory - Controls whether we write garbage over freed memory. - /// - void setPoisonMemory(bool poison) override { - PoisonMemory = poison; - } - }; -} - -void *JITAllocator::Allocate(size_t Size, size_t /*Alignment*/) { - sys::MemoryBlock B = JMM.allocateNewSlab(Size); - return B.base(); -} - -void JITAllocator::Deallocate(void *Slab, size_t Size) { - sys::MemoryBlock B(Slab, Size); - sys::Memory::ReleaseRWX(B); -} - -DefaultJITMemoryManager::DefaultJITMemoryManager() - : -#ifdef NDEBUG - PoisonMemory(false), -#else - PoisonMemory(true), -#endif - LastSlab(nullptr, 0), StubAllocator(*this), DataAllocator(*this) { - - // Allocate space for code. - sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize); - CodeSlabs.push_back(MemBlock); - uint8_t *MemBase = (uint8_t*)MemBlock.base(); - - // We set up the memory chunk with 4 mem regions, like this: - // [ START - // [ Free #0 ] -> Large space to allocate functions from. - // [ Allocated #1 ] -> Tiny space to separate regions. - // [ Free #2 ] -> Tiny space so there is always at least 1 free block. - // [ Allocated #3 ] -> Tiny space to prevent looking past end of block. - // END ] - // - // The last three blocks are never deallocated or touched. - - // Add MemoryRangeHeader to the end of the memory region, indicating that - // the space after the block of memory is allocated. This is block #3. - MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1; - Mem3->ThisAllocated = 1; - Mem3->PrevAllocated = 0; - Mem3->BlockSize = sizeof(MemoryRangeHeader); - - /// Add a tiny free region so that the free list always has one entry. - FreeRangeHeader *Mem2 = - (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize()); - Mem2->ThisAllocated = 0; - Mem2->PrevAllocated = 1; - Mem2->BlockSize = FreeRangeHeader::getMinBlockSize(); - Mem2->SetEndOfBlockSizeMarker(); - Mem2->Prev = Mem2; // Mem2 *is* the free list for now. - Mem2->Next = Mem2; - - /// Add a tiny allocated region so that Mem2 is never coalesced away. - MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1; - Mem1->ThisAllocated = 1; - Mem1->PrevAllocated = 0; - Mem1->BlockSize = sizeof(MemoryRangeHeader); - - // Add a FreeRangeHeader to the start of the function body region, indicating - // that the space is free. Mark the previous block allocated so we never look - // at it. - FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase; - Mem0->ThisAllocated = 0; - Mem0->PrevAllocated = 1; - Mem0->BlockSize = (char*)Mem1-(char*)Mem0; - Mem0->SetEndOfBlockSizeMarker(); - Mem0->AddToFreeList(Mem2); - - // Start out with the freelist pointing to Mem0. - FreeMemoryList = Mem0; - - GOTBase = nullptr; -} - -void DefaultJITMemoryManager::AllocateGOT() { - assert(!GOTBase && "Cannot allocate the got multiple times"); - GOTBase = new uint8_t[sizeof(void*) * 8192]; - HasGOT = true; -} - -DefaultJITMemoryManager::~DefaultJITMemoryManager() { - for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) - sys::Memory::ReleaseRWX(CodeSlabs[i]); - - delete[] GOTBase; -} - -sys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) { - // Allocate a new block close to the last one. - std::string ErrMsg; - sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : nullptr; - sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg); - if (!B.base()) { - report_fatal_error("Allocation failed when allocating new memory in the" - " JIT\n" + Twine(ErrMsg)); - } - LastSlab = B; - ++NumSlabs; - // Initialize the slab to garbage when debugging. - if (PoisonMemory) { - memset(B.base(), 0xCD, B.size()); - } - return B; -} - -/// CheckInvariants - For testing only. Return "" if all internal invariants -/// are preserved, and a helpful error message otherwise. For free and -/// allocated blocks, make sure that adding BlockSize gives a valid block. -/// For free blocks, make sure they're in the free list and that their end of -/// block size marker is correct. This function should return an error before -/// accessing bad memory. This function is defined here instead of in -/// JITMemoryManagerTest.cpp so that we don't have to expose all of the -/// implementation details of DefaultJITMemoryManager. -bool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) { - raw_string_ostream Err(ErrorStr); - - // Construct a the set of FreeRangeHeader pointers so we can query it - // efficiently. - llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet; - FreeRangeHeader* FreeHead = FreeMemoryList; - FreeRangeHeader* FreeRange = FreeHead; - - do { - // Check that the free range pointer is in the blocks we've allocated. - bool Found = false; - for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), - E = CodeSlabs.end(); I != E && !Found; ++I) { - char *Start = (char*)I->base(); - char *End = Start + I->size(); - Found = (Start <= (char*)FreeRange && (char*)FreeRange < End); - } - if (!Found) { - Err << "Corrupt free list; points to " << FreeRange; - return false; - } - - if (FreeRange->Next->Prev != FreeRange) { - Err << "Next and Prev pointers do not match."; - return false; - } - - // Otherwise, add it to the set. - FreeHdrSet.insert(FreeRange); - FreeRange = FreeRange->Next; - } while (FreeRange != FreeHead); - - // Go over each block, and look at each MemoryRangeHeader. - for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), - E = CodeSlabs.end(); I != E; ++I) { - char *Start = (char*)I->base(); - char *End = Start + I->size(); - - // Check each memory range. - for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = nullptr; - Start <= (char*)Hdr && (char*)Hdr < End; - Hdr = &Hdr->getBlockAfter()) { - if (Hdr->ThisAllocated == 0) { - // Check that this range is in the free list. - if (!FreeHdrSet.count(Hdr)) { - Err << "Found free header at " << Hdr << " that is not in free list."; - return false; - } - - // Now make sure the size marker at the end of the block is correct. - uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1; - if (!(Start <= (char*)Marker && (char*)Marker < End)) { - Err << "Block size in header points out of current MemoryBlock."; - return false; - } - if (Hdr->BlockSize != *Marker) { - Err << "End of block size marker (" << *Marker << ") " - << "and BlockSize (" << Hdr->BlockSize << ") don't match."; - return false; - } - } - - if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) { - Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != " - << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")"; - return false; - } else if (!LastHdr && !Hdr->PrevAllocated) { - Err << "The first header should have PrevAllocated true."; - return false; - } - - // Remember the last header. - LastHdr = Hdr; - } - } - - // All invariants are preserved. - return true; -} - -//===----------------------------------------------------------------------===// -// getPointerToNamedFunction() implementation. -//===----------------------------------------------------------------------===// - -// AtExitHandlers - List of functions to call when the program exits, -// registered with the atexit() library function. -static std::vector<void (*)()> AtExitHandlers; - -/// runAtExitHandlers - Run any functions registered by the program's -/// calls to atexit(3), which we intercept and store in -/// AtExitHandlers. -/// -static void runAtExitHandlers() { - while (!AtExitHandlers.empty()) { - void (*Fn)() = AtExitHandlers.back(); - AtExitHandlers.pop_back(); - Fn(); - } -} - -//===----------------------------------------------------------------------===// -// Function stubs that are invoked instead of certain library calls -// -// Force the following functions to be linked in to anything that uses the -// JIT. This is a hack designed to work around the all-too-clever Glibc -// strategy of making these functions work differently when inlined vs. when -// not inlined, and hiding their real definitions in a separate archive file -// that the dynamic linker can't see. For more info, search for -// 'libc_nonshared.a' on Google, or read http://llvm.org/PR274. -#if defined(__linux__) && defined(__GLIBC__) -/* stat functions are redirecting to __xstat with a version number. On x86-64 - * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat' - * available as an exported symbol, so we have to add it explicitly. - */ -namespace { -class StatSymbols { -public: - StatSymbols() { - sys::DynamicLibrary::AddSymbol("stat", (void*)(intptr_t)stat); - sys::DynamicLibrary::AddSymbol("fstat", (void*)(intptr_t)fstat); - sys::DynamicLibrary::AddSymbol("lstat", (void*)(intptr_t)lstat); - sys::DynamicLibrary::AddSymbol("stat64", (void*)(intptr_t)stat64); - sys::DynamicLibrary::AddSymbol("\x1stat64", (void*)(intptr_t)stat64); - sys::DynamicLibrary::AddSymbol("\x1open64", (void*)(intptr_t)open64); - sys::DynamicLibrary::AddSymbol("\x1lseek64", (void*)(intptr_t)lseek64); - sys::DynamicLibrary::AddSymbol("fstat64", (void*)(intptr_t)fstat64); - sys::DynamicLibrary::AddSymbol("lstat64", (void*)(intptr_t)lstat64); - sys::DynamicLibrary::AddSymbol("atexit", (void*)(intptr_t)atexit); - sys::DynamicLibrary::AddSymbol("mknod", (void*)(intptr_t)mknod); - } -}; -} -static StatSymbols initStatSymbols; -#endif // __linux__ - -// jit_exit - Used to intercept the "exit" library call. -static void jit_exit(int Status) { - runAtExitHandlers(); // Run atexit handlers... - exit(Status); -} - -// jit_atexit - Used to intercept the "atexit" library call. -static int jit_atexit(void (*Fn)()) { - AtExitHandlers.push_back(Fn); // Take note of atexit handler... - return 0; // Always successful -} - -static int jit_noop() { - return 0; -} - -//===----------------------------------------------------------------------===// -// -/// getPointerToNamedFunction - This method returns the address of the specified -/// function by using the dynamic loader interface. As such it is only useful -/// for resolving library symbols, not code generated symbols. -/// -void *DefaultJITMemoryManager::getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure) { - // Check to see if this is one of the functions we want to intercept. Note, - // we cast to intptr_t here to silence a -pedantic warning that complains - // about casting a function pointer to a normal pointer. - if (Name == "exit") return (void*)(intptr_t)&jit_exit; - if (Name == "atexit") return (void*)(intptr_t)&jit_atexit; - - // We should not invoke parent's ctors/dtors from generated main()! - // On Mingw and Cygwin, the symbol __main is resolved to - // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors - // (and register wrong callee's dtors with atexit(3)). - // We expect ExecutionEngine::runStaticConstructorsDestructors() - // is called before ExecutionEngine::runFunctionAsMain() is called. - if (Name == "__main") return (void*)(intptr_t)&jit_noop; - - const char *NameStr = Name.c_str(); - // If this is an asm specifier, skip the sentinal. - if (NameStr[0] == 1) ++NameStr; - - // If it's an external function, look it up in the process image... - void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr); - if (Ptr) return Ptr; - - // If it wasn't found and if it starts with an underscore ('_') character, - // try again without the underscore. - if (NameStr[0] == '_') { - Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1); - if (Ptr) return Ptr; - } - - // Darwin/PPC adds $LDBLStub suffixes to various symbols like printf. These - // are references to hidden visibility symbols that dlsym cannot resolve. - // If we have one of these, strip off $LDBLStub and try again. -#if defined(__APPLE__) && defined(__ppc__) - if (Name.size() > 9 && Name[Name.size()-9] == '$' && - memcmp(&Name[Name.size()-8], "LDBLStub", 8) == 0) { - // First try turning $LDBLStub into $LDBL128. If that fails, strip it off. - // This mirrors logic in libSystemStubs.a. - std::string Prefix = std::string(Name.begin(), Name.end()-9); - if (void *Ptr = getPointerToNamedFunction(Prefix+"$LDBL128", false)) - return Ptr; - if (void *Ptr = getPointerToNamedFunction(Prefix, false)) - return Ptr; - } -#endif - - if (AbortOnFailure) { - report_fatal_error("Program used external function '"+Name+ - "' which could not be resolved!"); - } - return nullptr; -} - - - -JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() { - return new DefaultJITMemoryManager(); -} - -const size_t DefaultJITMemoryManager::DefaultCodeSlabSize; -const size_t DefaultJITMemoryManager::DefaultSlabSize; -const size_t DefaultJITMemoryManager::DefaultSizeThreshold; diff --git a/lib/ExecutionEngine/JIT/LLVMBuild.txt b/lib/ExecutionEngine/JIT/LLVMBuild.txt deleted file mode 100644 index dd22f1b..0000000 --- a/lib/ExecutionEngine/JIT/LLVMBuild.txt +++ /dev/null @@ -1,22 +0,0 @@ -;===- ./lib/ExecutionEngine/JIT/LLVMBuild.txt ------------------*- Conf -*--===; -; -; The LLVM Compiler Infrastructure -; -; This file is distributed under the University of Illinois Open Source -; License. See LICENSE.TXT for details. -; -;===------------------------------------------------------------------------===; -; -; This is an LLVMBuild description file for the components in this subdirectory. -; -; For more information on the LLVMBuild system, please see: -; -; http://llvm.org/docs/LLVMBuild.html -; -;===------------------------------------------------------------------------===; - -[component_0] -type = Library -name = JIT -parent = ExecutionEngine -required_libraries = CodeGen Core ExecutionEngine Support diff --git a/lib/ExecutionEngine/JIT/Makefile b/lib/ExecutionEngine/JIT/Makefile deleted file mode 100644 index aafa3d9..0000000 --- a/lib/ExecutionEngine/JIT/Makefile +++ /dev/null @@ -1,38 +0,0 @@ -##===- lib/ExecutionEngine/JIT/Makefile --------------------*- Makefile -*-===## -# -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. -# -##===----------------------------------------------------------------------===## - -LEVEL = ../../.. -LIBRARYNAME = LLVMJIT - -# Get the $(ARCH) setting -include $(LEVEL)/Makefile.config - -# Enable the X86 JIT if compiling on X86 -ifeq ($(ARCH), x86) - ENABLE_X86_JIT = 1 -endif - -# This flag can also be used on the command line to force inclusion -# of the X86 JIT on non-X86 hosts -ifdef ENABLE_X86_JIT - CPPFLAGS += -DENABLE_X86_JIT -endif - -# Enable the Sparc JIT if compiling on Sparc -ifeq ($(ARCH), Sparc) - ENABLE_SPARC_JIT = 1 -endif - -# This flag can also be used on the command line to force inclusion -# of the Sparc JIT on non-Sparc hosts -ifdef ENABLE_SPARC_JIT - CPPFLAGS += -DENABLE_SPARC_JIT -endif - -include $(LEVEL)/Makefile.common diff --git a/lib/ExecutionEngine/JITEventListener.cpp b/lib/ExecutionEngine/JITEventListener.cpp new file mode 100644 index 0000000..2a6a007 --- /dev/null +++ b/lib/ExecutionEngine/JITEventListener.cpp @@ -0,0 +1,15 @@ +//===-- JITEventListener.cpp ----------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITEventListener.h" + +using namespace llvm; + +// Out-of-line definition of the virtual destructor as this is the key function. +JITEventListener::~JITEventListener() {} diff --git a/lib/ExecutionEngine/LLVMBuild.txt b/lib/ExecutionEngine/LLVMBuild.txt index 6dc75af..ecae078 100644 --- a/lib/ExecutionEngine/LLVMBuild.txt +++ b/lib/ExecutionEngine/LLVMBuild.txt @@ -16,7 +16,7 @@ ;===------------------------------------------------------------------------===; [common] -subdirectories = Interpreter JIT MCJIT RuntimeDyld IntelJITEvents OProfileJIT +subdirectories = Interpreter MCJIT RuntimeDyld IntelJITEvents OProfileJIT [component_0] type = Library diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/lib/ExecutionEngine/MCJIT/MCJIT.cpp index e9ba96a..da5f037 100644 --- a/lib/ExecutionEngine/MCJIT/MCJIT.cpp +++ b/lib/ExecutionEngine/MCJIT/MCJIT.cpp @@ -10,7 +10,6 @@ #include "MCJIT.h" #include "llvm/ExecutionEngine/GenericValue.h" #include "llvm/ExecutionEngine/JITEventListener.h" -#include "llvm/ExecutionEngine/JITMemoryManager.h" #include "llvm/ExecutionEngine/MCJIT.h" #include "llvm/ExecutionEngine/ObjectBuffer.h" #include "llvm/ExecutionEngine/ObjectImage.h" @@ -28,6 +27,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/MutexGuard.h" #include "llvm/Target/TargetLowering.h" +#include "llvm/Target/TargetSubtargetInfo.h" using namespace llvm; @@ -42,31 +42,23 @@ static struct RegisterJIT { extern "C" void LLVMLinkInMCJIT() { } -ExecutionEngine *MCJIT::createJIT(Module *M, +ExecutionEngine *MCJIT::createJIT(std::unique_ptr<Module> M, std::string *ErrorStr, RTDyldMemoryManager *MemMgr, - bool GVsWithCode, - TargetMachine *TM) { + std::unique_ptr<TargetMachine> TM) { // Try to register the program as a source of symbols to resolve against. // // FIXME: Don't do this here. sys::DynamicLibrary::LoadLibraryPermanently(nullptr, nullptr); - return new MCJIT(M, TM, MemMgr ? MemMgr : new SectionMemoryManager(), - GVsWithCode); + return new MCJIT(std::move(M), std::move(TM), + MemMgr ? MemMgr : new SectionMemoryManager()); } -MCJIT::MCJIT(Module *m, TargetMachine *tm, RTDyldMemoryManager *MM, - bool AllocateGVsWithCode) - : ExecutionEngine(m), TM(tm), Ctx(nullptr), MemMgr(this, MM), Dyld(&MemMgr), - ObjCache(nullptr) { - - OwnedModules.addModule(m); - setDataLayout(TM->getDataLayout()); -} - -MCJIT::~MCJIT() { - MutexGuard locked(lock); +MCJIT::MCJIT(std::unique_ptr<Module> M, std::unique_ptr<TargetMachine> tm, + RTDyldMemoryManager *MM) + : ExecutionEngine(std::move(M)), TM(std::move(tm)), Ctx(nullptr), + MemMgr(this, MM), Dyld(&MemMgr), ObjCache(nullptr) { // FIXME: We are managing our modules, so we do not want the base class // ExecutionEngine to manage them as well. To avoid double destruction // of the first (and only) module added in ExecutionEngine constructor @@ -77,33 +69,28 @@ MCJIT::~MCJIT() { // If so, additional functions: addModule, removeModule, FindFunctionNamed, // runStaticConstructorsDestructors could be moved back to EE as well. // + std::unique_ptr<Module> First = std::move(Modules[0]); Modules.clear(); + + OwnedModules.addModule(std::move(First)); + setDataLayout(TM->getSubtargetImpl()->getDataLayout()); +} + +MCJIT::~MCJIT() { + MutexGuard locked(lock); + Dyld.deregisterEHFrames(); - LoadedObjectList::iterator it, end; - for (it = LoadedObjects.begin(), end = LoadedObjects.end(); it != end; ++it) { - ObjectImage *Obj = *it; - if (Obj) { + for (auto &Obj : LoadedObjects) + if (Obj) NotifyFreeingObject(*Obj); - delete Obj; - } - } - LoadedObjects.clear(); - - SmallVector<object::Archive *, 2>::iterator ArIt, ArEnd; - for (ArIt = Archives.begin(), ArEnd = Archives.end(); ArIt != ArEnd; ++ArIt) { - object::Archive *A = *ArIt; - delete A; - } Archives.clear(); - - delete TM; } -void MCJIT::addModule(Module *M) { +void MCJIT::addModule(std::unique_ptr<Module> M) { MutexGuard locked(lock); - OwnedModules.addModule(M); + OwnedModules.addModule(std::move(M)); } bool MCJIT::removeModule(Module *M) { @@ -111,29 +98,34 @@ bool MCJIT::removeModule(Module *M) { return OwnedModules.removeModule(M); } - - void MCJIT::addObjectFile(std::unique_ptr<object::ObjectFile> Obj) { - ObjectImage *LoadedObject = Dyld.loadObject(std::move(Obj)); + std::unique_ptr<ObjectImage> LoadedObject = Dyld.loadObject(std::move(Obj)); if (!LoadedObject || Dyld.hasError()) report_fatal_error(Dyld.getErrorString()); - LoadedObjects.push_back(LoadedObject); - NotifyObjectEmitted(*LoadedObject); + + LoadedObjects.push_back(std::move(LoadedObject)); } -void MCJIT::addArchive(object::Archive *A) { - Archives.push_back(A); +void MCJIT::addObjectFile(object::OwningBinary<object::ObjectFile> Obj) { + std::unique_ptr<object::ObjectFile> ObjFile; + std::unique_ptr<MemoryBuffer> MemBuf; + std::tie(ObjFile, MemBuf) = Obj.takeBinary(); + addObjectFile(std::move(ObjFile)); + Buffers.push_back(std::move(MemBuf)); } +void MCJIT::addArchive(object::OwningBinary<object::Archive> A) { + Archives.push_back(std::move(A)); +} void MCJIT::setObjectCache(ObjectCache* NewCache) { MutexGuard locked(lock); ObjCache = NewCache; } -ObjectBufferStream* MCJIT::emitObject(Module *M) { +std::unique_ptr<ObjectBufferStream> MCJIT::emitObject(Module *M) { MutexGuard locked(lock); // This must be a module which has already been added but not loaded to this @@ -142,8 +134,8 @@ ObjectBufferStream* MCJIT::emitObject(Module *M) { PassManager PM; - M->setDataLayout(TM->getDataLayout()); - PM.add(new DataLayoutPass(M)); + M->setDataLayout(TM->getSubtargetImpl()->getDataLayout()); + PM.add(new DataLayoutPass()); // The RuntimeDyld will take ownership of this shortly std::unique_ptr<ObjectBufferStream> CompiledObject(new ObjectBufferStream()); @@ -165,11 +157,11 @@ ObjectBufferStream* MCJIT::emitObject(Module *M) { if (ObjCache) { // MemoryBuffer is a thin wrapper around the actual memory, so it's OK // to create a temporary object here and delete it after the call. - std::unique_ptr<MemoryBuffer> MB(CompiledObject->getMemBuffer()); - ObjCache->notifyObjectCompiled(M, MB.get()); + MemoryBufferRef MB = CompiledObject->getMemBuffer(); + ObjCache->notifyObjectCompiled(M, MB); } - return CompiledObject.release(); + return CompiledObject; } void MCJIT::generateCodeForModule(Module *M) { @@ -187,21 +179,22 @@ void MCJIT::generateCodeForModule(Module *M) { std::unique_ptr<ObjectBuffer> ObjectToLoad; // Try to load the pre-compiled object from cache if possible if (ObjCache) { - std::unique_ptr<MemoryBuffer> PreCompiledObject(ObjCache->getObject(M)); - if (PreCompiledObject.get()) - ObjectToLoad.reset(new ObjectBuffer(PreCompiledObject.release())); + if (std::unique_ptr<MemoryBuffer> PreCompiledObject = + ObjCache->getObject(M)) + ObjectToLoad = + llvm::make_unique<ObjectBuffer>(std::move(PreCompiledObject)); } // If the cache did not contain a suitable object, compile the object if (!ObjectToLoad) { - ObjectToLoad.reset(emitObject(M)); - assert(ObjectToLoad.get() && "Compilation did not produce an object."); + ObjectToLoad = emitObject(M); + assert(ObjectToLoad && "Compilation did not produce an object."); } // Load the object into the dynamic linker. // MCJIT now owns the ObjectImage pointer (via its LoadedObjects list). - ObjectImage *LoadedObject = Dyld.loadObject(ObjectToLoad.release()); - LoadedObjects.push_back(LoadedObject); + std::unique_ptr<ObjectImage> LoadedObject = + Dyld.loadObject(std::move(ObjectToLoad)); if (!LoadedObject) report_fatal_error(Dyld.getErrorString()); @@ -210,6 +203,8 @@ void MCJIT::generateCodeForModule(Module *M) { NotifyObjectEmitted(*LoadedObject); + LoadedObjects.push_back(std::move(LoadedObject)); + OwnedModules.markModuleAsLoaded(M); } @@ -232,12 +227,14 @@ void MCJIT::finalizeLoadedModules() { void MCJIT::finalizeObject() { MutexGuard locked(lock); - for (ModulePtrSet::iterator I = OwnedModules.begin_added(), - E = OwnedModules.end_added(); - I != E; ++I) { - Module *M = *I; + // Generate code for module is going to move objects out of the 'added' list, + // so we need to copy that out before using it: + SmallVector<Module*, 16> ModsToAdd; + for (auto M : OwnedModules.added()) + ModsToAdd.push_back(M); + + for (auto M : ModsToAdd) generateCodeForModule(M); - } finalizeLoadedModules(); } @@ -255,12 +252,8 @@ void MCJIT::finalizeModule(Module *M) { finalizeLoadedModules(); } -void *MCJIT::getPointerToBasicBlock(BasicBlock *BB) { - report_fatal_error("not yet implemented"); -} - uint64_t MCJIT::getExistingSymbolAddress(const std::string &Name) { - Mangler Mang(TM->getDataLayout()); + Mangler Mang(TM->getSubtargetImpl()->getDataLayout()); SmallString<128> FullName; Mang.getNameWithPrefix(FullName, Name); return Dyld.getSymbolLoadAddress(FullName); @@ -299,9 +292,8 @@ uint64_t MCJIT::getSymbolAddress(const std::string &Name, if (Addr) return Addr; - SmallVector<object::Archive*, 2>::iterator I, E; - for (I = Archives.begin(), E = Archives.end(); I != E; ++I) { - object::Archive *A = *I; + for (object::OwningBinary<object::Archive> &OB : Archives) { + object::Archive *A = OB.getBinary(); // Look for our symbols in each Archive object::Archive::child_iterator ChildIt = A->findSym(Name); if (ChildIt != A->child_end()) { @@ -310,7 +302,7 @@ uint64_t MCJIT::getSymbolAddress(const std::string &Name, ChildIt->getAsBinary(); if (ChildBinOrErr.getError()) continue; - std::unique_ptr<object::Binary> ChildBin = std::move(ChildBinOrErr.get()); + std::unique_ptr<object::Binary> &ChildBin = ChildBinOrErr.get(); if (ChildBin->isObject()) { std::unique_ptr<object::ObjectFile> OF( static_cast<object::ObjectFile *>(ChildBin.release())); @@ -326,13 +318,19 @@ uint64_t MCJIT::getSymbolAddress(const std::string &Name, // If it hasn't already been generated, see if it's in one of our modules. Module *M = findModuleForSymbol(Name, CheckFunctionsOnly); - if (!M) - return 0; + if (M) { + generateCodeForModule(M); + + // Check the RuntimeDyld table again, it should be there now. + return getExistingSymbolAddress(Name); + } - generateCodeForModule(M); + // If a LazyFunctionCreator is installed, use it to get/create the function. + // FIXME: Should we instead have a LazySymbolCreator callback? + if (LazyFunctionCreator) + Addr = (uint64_t)LazyFunctionCreator(Name); - // Check the RuntimeDyld table again, it should be there now. - return getExistingSymbolAddress(Name); + return Addr; } uint64_t MCJIT::getGlobalValueAddress(const std::string &Name) { @@ -355,10 +353,14 @@ uint64_t MCJIT::getFunctionAddress(const std::string &Name) { void *MCJIT::getPointerToFunction(Function *F) { MutexGuard locked(lock); + Mangler Mang(TM->getSubtargetImpl()->getDataLayout()); + SmallString<128> Name; + TM->getNameWithPrefix(Name, F, Mang); + if (F->isDeclaration() || F->hasAvailableExternallyLinkage()) { bool AbortOnFailure = !F->hasExternalWeakLinkage(); - void *Addr = getPointerToNamedFunction(F->getName(), AbortOnFailure); - addGlobalMapping(F, Addr); + void *Addr = getPointerToNamedFunction(Name, AbortOnFailure); + updateGlobalMapping(F, Addr); return Addr; } @@ -368,32 +370,25 @@ void *MCJIT::getPointerToFunction(Function *F) { // Make sure the relevant module has been compiled and loaded. if (HasBeenAddedButNotLoaded) generateCodeForModule(M); - else if (!OwnedModules.hasModuleBeenLoaded(M)) + else if (!OwnedModules.hasModuleBeenLoaded(M)) { // If this function doesn't belong to one of our modules, we're done. + // FIXME: Asking for the pointer to a function that hasn't been registered, + // and isn't a declaration (which is handled above) should probably + // be an assertion. return nullptr; + } // FIXME: Should the Dyld be retaining module information? Probably not. // // This is the accessor for the target address, so make sure to check the // load address of the symbol, not the local address. - Mangler Mang(TM->getDataLayout()); - SmallString<128> Name; - TM->getNameWithPrefix(Name, F, Mang); return (void*)Dyld.getSymbolLoadAddress(Name); } -void *MCJIT::recompileAndRelinkFunction(Function *F) { - report_fatal_error("not yet implemented"); -} - -void MCJIT::freeMachineCodeForFunction(Function *F) { - report_fatal_error("not yet implemented"); -} - void MCJIT::runStaticConstructorsDestructorsInModulePtrSet( bool isDtors, ModulePtrSet::iterator I, ModulePtrSet::iterator E) { for (; I != E; ++I) { - ExecutionEngine::runStaticConstructorsDestructors(*I, isDtors); + ExecutionEngine::runStaticConstructorsDestructors(**I, isDtors); } } @@ -529,8 +524,7 @@ GenericValue MCJIT::runFunction(Function *F, llvm_unreachable("Full-featured argument passing not supported yet!"); } -void *MCJIT::getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure) { +void *MCJIT::getPointerToNamedFunction(StringRef Name, bool AbortOnFailure) { if (!isSymbolSearchingDisabled()) { void *ptr = MemMgr.getPointerToNamedFunction(Name, false); if (ptr) @@ -559,8 +553,7 @@ void MCJIT::UnregisterJITEventListener(JITEventListener *L) { if (!L) return; MutexGuard locked(lock); - SmallVector<JITEventListener*, 2>::reverse_iterator I= - std::find(EventListeners.rbegin(), EventListeners.rend(), L); + auto I = std::find(EventListeners.rbegin(), EventListeners.rend(), L); if (I != EventListeners.rend()) { std::swap(*I, EventListeners.back()); EventListeners.pop_back(); @@ -575,9 +568,8 @@ void MCJIT::NotifyObjectEmitted(const ObjectImage& Obj) { } void MCJIT::NotifyFreeingObject(const ObjectImage& Obj) { MutexGuard locked(lock); - for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) { - EventListeners[I]->NotifyFreeingObject(Obj); - } + for (JITEventListener *L : EventListeners) + L->NotifyFreeingObject(Obj); } uint64_t LinkingMemoryManager::getSymbolAddress(const std::string &Name) { @@ -588,5 +580,7 @@ uint64_t LinkingMemoryManager::getSymbolAddress(const std::string &Name) { Result = ParentEngine->getSymbolAddress(Name.substr(1), false); if (Result) return Result; + if (ParentEngine->isSymbolSearchingDisabled()) + return 0; return ClientMM->getSymbolAddress(Name); } diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.h b/lib/ExecutionEngine/MCJIT/MCJIT.h index 100e9a2..bc943b9 100644 --- a/lib/ExecutionEngine/MCJIT/MCJIT.h +++ b/lib/ExecutionEngine/MCJIT/MCJIT.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_LIB_EXECUTIONENGINE_MCJIT_H -#define LLVM_LIB_EXECUTIONENGINE_MCJIT_H +#ifndef LLVM_LIB_EXECUTIONENGINE_MCJIT_MCJIT_H +#define LLVM_LIB_EXECUTIONENGINE_MCJIT_MCJIT_H #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" @@ -101,8 +101,8 @@ private: // called. class MCJIT : public ExecutionEngine { - MCJIT(Module *M, TargetMachine *tm, RTDyldMemoryManager *MemMgr, - bool AllocateGVsWithCode); + MCJIT(std::unique_ptr<Module> M, std::unique_ptr<TargetMachine> tm, + RTDyldMemoryManager *MemMgr); typedef llvm::SmallPtrSet<Module *, 4> ModulePtrSet; @@ -118,6 +118,9 @@ class MCJIT : public ExecutionEngine { ModulePtrSet::iterator begin_added() { return AddedModules.begin(); } ModulePtrSet::iterator end_added() { return AddedModules.end(); } + iterator_range<ModulePtrSet::iterator> added() { + return iterator_range<ModulePtrSet::iterator>(begin_added(), end_added()); + } ModulePtrSet::iterator begin_loaded() { return LoadedModules.begin(); } ModulePtrSet::iterator end_loaded() { return LoadedModules.end(); } @@ -125,8 +128,8 @@ class MCJIT : public ExecutionEngine { ModulePtrSet::iterator begin_finalized() { return FinalizedModules.begin(); } ModulePtrSet::iterator end_finalized() { return FinalizedModules.end(); } - void addModule(Module *M) { - AddedModules.insert(M); + void addModule(std::unique_ptr<Module> M) { + AddedModules.insert(M.release()); } bool removeModule(Module *M) { @@ -208,18 +211,18 @@ class MCJIT : public ExecutionEngine { } }; - TargetMachine *TM; + std::unique_ptr<TargetMachine> TM; MCContext *Ctx; LinkingMemoryManager MemMgr; RuntimeDyld Dyld; - SmallVector<JITEventListener*, 2> EventListeners; + std::vector<JITEventListener*> EventListeners; OwningModuleContainer OwnedModules; - SmallVector<object::Archive*, 2> Archives; + SmallVector<object::OwningBinary<object::Archive>, 2> Archives; + SmallVector<std::unique_ptr<MemoryBuffer>, 2> Buffers; - typedef SmallVector<ObjectImage *, 2> LoadedObjectList; - LoadedObjectList LoadedObjects; + SmallVector<std::unique_ptr<ObjectImage>, 2> LoadedObjects; // An optional ObjectCache to be notified of compiled objects and used to // perform lookup of pre-compiled code to avoid re-compilation. @@ -238,9 +241,10 @@ public: /// @name ExecutionEngine interface implementation /// @{ - void addModule(Module *M) override; + void addModule(std::unique_ptr<Module> M) override; void addObjectFile(std::unique_ptr<object::ObjectFile> O) override; - void addArchive(object::Archive *O) override; + void addObjectFile(object::OwningBinary<object::ObjectFile> O) override; + void addArchive(object::OwningBinary<object::Archive> O) override; bool removeModule(Module *M) override; /// FindFunctionNamed - Search all of the active modules to find the one that @@ -276,14 +280,8 @@ public: /// \param isDtors - Run the destructors instead of constructors. void runStaticConstructorsDestructors(bool isDtors) override; - void *getPointerToBasicBlock(BasicBlock *BB) override; - void *getPointerToFunction(Function *F) override; - void *recompileAndRelinkFunction(Function *F) override; - - void freeMachineCodeForFunction(Function *F) override; - GenericValue runFunction(Function *F, const std::vector<GenericValue> &ArgValues) override; @@ -295,7 +293,7 @@ public: /// found, this function silently returns a null pointer. Otherwise, /// it prints a message to stderr and aborts. /// - void *getPointerToNamedFunction(const std::string &Name, + void *getPointerToNamedFunction(StringRef Name, bool AbortOnFailure = true) override; /// mapSectionAddress - map a section to its target address space value. @@ -315,7 +313,7 @@ public: uint64_t getGlobalValueAddress(const std::string &Name) override; uint64_t getFunctionAddress(const std::string &Name) override; - TargetMachine *getTargetMachine() override { return TM; } + TargetMachine *getTargetMachine() override { return TM.get(); } /// @} /// @name (Private) Registration Interfaces @@ -325,11 +323,10 @@ public: MCJITCtor = createJIT; } - static ExecutionEngine *createJIT(Module *M, + static ExecutionEngine *createJIT(std::unique_ptr<Module> M, std::string *ErrorStr, RTDyldMemoryManager *MemMgr, - bool GVsWithCode, - TargetMachine *TM); + std::unique_ptr<TargetMachine> TM); // @} @@ -344,7 +341,7 @@ protected: /// this function call is expected to be the contained module. The module /// is passed as a parameter here to prepare for multiple module support in /// the future. - ObjectBufferStream* emitObject(Module *M); + std::unique_ptr<ObjectBufferStream> emitObject(Module *M); void NotifyObjectEmitted(const ObjectImage& Obj); void NotifyFreeingObject(const ObjectImage& Obj); diff --git a/lib/ExecutionEngine/Makefile b/lib/ExecutionEngine/Makefile index c26e0ad..cf71432 100644 --- a/lib/ExecutionEngine/Makefile +++ b/lib/ExecutionEngine/Makefile @@ -11,7 +11,7 @@ LIBRARYNAME = LLVMExecutionEngine include $(LEVEL)/Makefile.config -PARALLEL_DIRS = Interpreter JIT MCJIT RuntimeDyld +PARALLEL_DIRS = Interpreter MCJIT RuntimeDyld ifeq ($(USE_INTEL_JITEVENTS), 1) PARALLEL_DIRS += IntelJITEvents diff --git a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp index fd37a13..5a8ccb6 100644 --- a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp +++ b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp @@ -49,12 +49,6 @@ public: ~OProfileJITEventListener(); - virtual void NotifyFunctionEmitted(const Function &F, - void *FnStart, size_t FnSize, - const JITEvent_EmittedFunctionDetails &Details); - - virtual void NotifyFreeingMachineCode(void *OldPtr); - virtual void NotifyObjectEmitted(const ObjectImage &Obj); virtual void NotifyFreeingObject(const ObjectImage &Obj); @@ -81,90 +75,6 @@ OProfileJITEventListener::~OProfileJITEventListener() { } } -static debug_line_info LineStartToOProfileFormat( - const MachineFunction &MF, FilenameCache &Filenames, - uintptr_t Address, DebugLoc Loc) { - debug_line_info Result; - Result.vma = Address; - Result.lineno = Loc.getLine(); - Result.filename = Filenames.getFilename( - Loc.getScope(MF.getFunction()->getContext())); - DEBUG(dbgs() << "Mapping " << reinterpret_cast<void*>(Result.vma) << " to " - << Result.filename << ":" << Result.lineno << "\n"); - return Result; -} - -// Adds the just-emitted function to the symbol table. -void OProfileJITEventListener::NotifyFunctionEmitted( - const Function &F, void *FnStart, size_t FnSize, - const JITEvent_EmittedFunctionDetails &Details) { - assert(F.hasName() && FnStart != 0 && "Bad symbol to add"); - if (Wrapper.op_write_native_code(F.getName().data(), - reinterpret_cast<uint64_t>(FnStart), - FnStart, FnSize) == -1) { - DEBUG(dbgs() << "Failed to tell OProfile about native function " - << F.getName() << " at [" - << FnStart << "-" << ((char*)FnStart + FnSize) << "]\n"); - return; - } - - if (!Details.LineStarts.empty()) { - // Now we convert the line number information from the address/DebugLoc - // format in Details to the address/filename/lineno format that OProfile - // expects. Note that OProfile 0.9.4 has a bug that causes it to ignore - // line numbers for addresses above 4G. - FilenameCache Filenames; - std::vector<debug_line_info> LineInfo; - LineInfo.reserve(1 + Details.LineStarts.size()); - - DebugLoc FirstLoc = Details.LineStarts[0].Loc; - assert(!FirstLoc.isUnknown() - && "LineStarts should not contain unknown DebugLocs"); - MDNode *FirstLocScope = FirstLoc.getScope(F.getContext()); - DISubprogram FunctionDI = getDISubprogram(FirstLocScope); - if (FunctionDI.Verify()) { - // If we have debug info for the function itself, use that as the line - // number of the first several instructions. Otherwise, after filling - // LineInfo, we'll adjust the address of the first line number to point at - // the start of the function. - debug_line_info line_info; - line_info.vma = reinterpret_cast<uintptr_t>(FnStart); - line_info.lineno = FunctionDI.getLineNumber(); - line_info.filename = Filenames.getFilename(FirstLocScope); - LineInfo.push_back(line_info); - } - - for (std::vector<EmittedFunctionDetails::LineStart>::const_iterator - I = Details.LineStarts.begin(), E = Details.LineStarts.end(); - I != E; ++I) { - LineInfo.push_back(LineStartToOProfileFormat( - *Details.MF, Filenames, I->Address, I->Loc)); - } - - // In case the function didn't have line info of its own, adjust the first - // line info's address to include the start of the function. - LineInfo[0].vma = reinterpret_cast<uintptr_t>(FnStart); - - if (Wrapper.op_write_debug_line_info(FnStart, LineInfo.size(), - &*LineInfo.begin()) == -1) { - DEBUG(dbgs() - << "Failed to tell OProfile about line numbers for native function " - << F.getName() << " at [" - << FnStart << "-" << ((char*)FnStart + FnSize) << "]\n"); - } - } -} - -// Removes the being-deleted function from the symbol table. -void OProfileJITEventListener::NotifyFreeingMachineCode(void *FnStart) { - assert(FnStart && "Invalid function pointer"); - if (Wrapper.op_unload_native_code(reinterpret_cast<uint64_t>(FnStart)) == -1) { - DEBUG(dbgs() - << "Failed to tell OProfile about unload of native function at " - << FnStart << "\n"); - } -} - void OProfileJITEventListener::NotifyObjectEmitted(const ObjectImage &Obj) { if (!Wrapper.isAgentAvailable()) { return; diff --git a/lib/ExecutionEngine/RTDyldMemoryManager.cpp b/lib/ExecutionEngine/RTDyldMemoryManager.cpp index 1646937..51b2d0f 100644 --- a/lib/ExecutionEngine/RTDyldMemoryManager.cpp +++ b/lib/ExecutionEngine/RTDyldMemoryManager.cpp @@ -210,7 +210,8 @@ ARM_MATH_IMPORTS(ARM_MATH_DECL) #undef ARM_MATH_DECL #endif -uint64_t RTDyldMemoryManager::getSymbolAddress(const std::string &Name) { +uint64_t +RTDyldMemoryManager::getSymbolAddressInProcess(const std::string &Name) { // This implementation assumes that the host program is the target. // Clients generating code for a remote target should implement their own // memory manager. @@ -253,19 +254,19 @@ uint64_t RTDyldMemoryManager::getSymbolAddress(const std::string &Name) { // is called before ExecutionEngine::runFunctionAsMain() is called. if (Name == "__main") return (uint64_t)&jit_noop; + // Try to demangle Name before looking it up in the process, otherwise symbol + // '_<Name>' (if present) will shadow '<Name>', and there will be no way to + // refer to the latter. + const char *NameStr = Name.c_str(); - void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr); - if (Ptr) - return (uint64_t)Ptr; - // If it wasn't found and if it starts with an underscore ('_') character, - // try again without the underscore. - if (NameStr[0] == '_') { - Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1); - if (Ptr) + if (NameStr[0] == '_') + if (void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr + 1)) return (uint64_t)Ptr; - } - return 0; + + // If we Name did not require demangling, or we failed to find the demangled + // name, try again without demangling. + return (uint64_t)sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr); } void *RTDyldMemoryManager::getPointerToNamedFunction(const std::string &Name, diff --git a/lib/ExecutionEngine/RuntimeDyld/GDBRegistrar.cpp b/lib/ExecutionEngine/RuntimeDyld/GDBRegistrar.cpp index 8546571..dfa3a20 100644 --- a/lib/ExecutionEngine/RuntimeDyld/GDBRegistrar.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/GDBRegistrar.cpp @@ -101,7 +101,7 @@ private: /// Lock used to serialize all jit registration events, since they /// modify global variables. -llvm::sys::Mutex JITDebugLock; +ManagedStatic<sys::Mutex> JITDebugLock; /// Do the registration. void NotifyDebugger(jit_code_entry* JITCodeEntry) { @@ -121,7 +121,7 @@ void NotifyDebugger(jit_code_entry* JITCodeEntry) { GDBJITRegistrar::~GDBJITRegistrar() { // Free all registered object files. - llvm::MutexGuard locked(JITDebugLock); + llvm::MutexGuard locked(*JITDebugLock); for (RegisteredObjectBufferMap::iterator I = ObjectBufferMap.begin(), E = ObjectBufferMap.end(); I != E; ++I) { // Call the private method that doesn't update the map so our iterator @@ -137,7 +137,7 @@ void GDBJITRegistrar::registerObject(const ObjectBuffer &Object) { size_t Size = Object.getBufferSize(); assert(Buffer && "Attempt to register a null object with a debugger."); - llvm::MutexGuard locked(JITDebugLock); + llvm::MutexGuard locked(*JITDebugLock); assert(ObjectBufferMap.find(Buffer) == ObjectBufferMap.end() && "Second attempt to perform debug registration."); jit_code_entry* JITCodeEntry = new jit_code_entry(); @@ -156,7 +156,7 @@ void GDBJITRegistrar::registerObject(const ObjectBuffer &Object) { bool GDBJITRegistrar::deregisterObject(const ObjectBuffer& Object) { const char *Buffer = Object.getBufferStart(); - llvm::MutexGuard locked(JITDebugLock); + llvm::MutexGuard locked(*JITDebugLock); RegisteredObjectBufferMap::iterator I = ObjectBufferMap.find(Buffer); if (I != ObjectBufferMap.end()) { diff --git a/lib/ExecutionEngine/RuntimeDyld/JITRegistrar.h b/lib/ExecutionEngine/RuntimeDyld/JITRegistrar.h index 6a514ea..636011f 100644 --- a/lib/ExecutionEngine/RuntimeDyld/JITRegistrar.h +++ b/lib/ExecutionEngine/RuntimeDyld/JITRegistrar.h @@ -7,8 +7,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_EXECUTION_ENGINE_JIT_REGISTRAR_H -#define LLVM_EXECUTION_ENGINE_JIT_REGISTRAR_H +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_JITREGISTRAR_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_JITREGISTRAR_H #include "llvm/ExecutionEngine/ObjectBuffer.h" @@ -41,4 +41,4 @@ public: } // end namespace llvm -#endif // LLVM_EXECUTION_ENGINE_JIT_REGISTRAR_H +#endif diff --git a/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h b/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h index c3a2182..9bbf6a0d 100644 --- a/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h +++ b/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_RUNTIMEDYLD_OBJECTIMAGECOMMON_H -#define LLVM_RUNTIMEDYLD_OBJECTIMAGECOMMON_H +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_OBJECTIMAGECOMMON_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_OBJECTIMAGECOMMON_H #include "llvm/ExecutionEngine/ObjectBuffer.h" #include "llvm/ExecutionEngine/ObjectImage.h" @@ -36,20 +36,17 @@ protected: // This form of the constructor allows subclasses to use // format-specific subclasses of ObjectFile directly - ObjectImageCommon(ObjectBuffer *Input, std::unique_ptr<object::ObjectFile> Obj) - : ObjectImage(Input), // saves Input as Buffer and takes ownership - ObjFile(std::move(Obj)) - { - } + ObjectImageCommon(std::unique_ptr<ObjectBuffer> Input, + std::unique_ptr<object::ObjectFile> Obj) + : ObjectImage(std::move(Input)), ObjFile(std::move(Obj)) {} public: - ObjectImageCommon(ObjectBuffer* Input) - : ObjectImage(Input) // saves Input as Buffer and takes ownership - { + ObjectImageCommon(std::unique_ptr<ObjectBuffer> Input) + : ObjectImage(std::move(Input)) { // FIXME: error checking? createObjectFile returns an ErrorOr<ObjectFile*> // and should probably be checked for failure. - std::unique_ptr<MemoryBuffer> Buf(Buffer->getMemBuffer()); - ObjFile.reset(object::ObjectFile::createObjectFile(Buf).get()); + MemoryBufferRef Buf = Buffer->getMemBuffer(); + ObjFile = std::move(object::ObjectFile::createObjectFile(Buf).get()); } ObjectImageCommon(std::unique_ptr<object::ObjectFile> Input) : ObjectImage(nullptr), ObjFile(std::move(Input)) {} @@ -86,4 +83,4 @@ public: } // end namespace llvm -#endif // LLVM_RUNTIMEDYLD_OBJECT_IMAGE_H +#endif diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index 9dfd167..c7c67f6 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -14,6 +14,7 @@ #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "JITRegistrar.h" #include "ObjectImageCommon.h" +#include "RuntimeDyldCheckerImpl.h" #include "RuntimeDyldELF.h" #include "RuntimeDyldImpl.h" #include "RuntimeDyldMachO.h" @@ -40,6 +41,44 @@ void RuntimeDyldImpl::registerEHFrames() {} void RuntimeDyldImpl::deregisterEHFrames() {} +#ifndef NDEBUG +static void dumpSectionMemory(const SectionEntry &S, StringRef State) { + dbgs() << "----- Contents of section " << S.Name << " " << State << " -----"; + + if (S.Address == nullptr) { + dbgs() << "\n <section not emitted>\n"; + return; + } + + const unsigned ColsPerRow = 16; + + uint8_t *DataAddr = S.Address; + uint64_t LoadAddr = S.LoadAddress; + + unsigned StartPadding = LoadAddr & (ColsPerRow - 1); + unsigned BytesRemaining = S.Size; + + if (StartPadding) { + dbgs() << "\n" << format("0x%016" PRIx64, LoadAddr & ~(ColsPerRow - 1)) << ":"; + while (StartPadding--) + dbgs() << " "; + } + + while (BytesRemaining > 0) { + if ((LoadAddr & (ColsPerRow - 1)) == 0) + dbgs() << "\n" << format("0x%016" PRIx64, LoadAddr) << ":"; + + dbgs() << " " << format("%02x", *DataAddr); + + ++DataAddr; + ++LoadAddr; + --BytesRemaining; + } + + dbgs() << "\n"; +} +#endif + // Resolve the relocations for all symbols we currently know about. void RuntimeDyldImpl::resolveRelocations() { MutexGuard locked(lock); @@ -55,8 +94,10 @@ void RuntimeDyldImpl::resolveRelocations() { // entry provides the section to which the relocation will be applied. uint64_t Addr = Sections[i].LoadAddress; DEBUG(dbgs() << "Resolving relocations Section #" << i << "\t" - << format("%p", (uint8_t *)Addr) << "\n"); + << format("0x%x", Addr) << "\n"); + DEBUG(dumpSectionMemory(Sections[i], "before relocations")); resolveRelocationList(Relocations[i], Addr); + DEBUG(dumpSectionMemory(Sections[i], "after relocations")); Relocations.erase(i); } } @@ -88,23 +129,20 @@ static std::error_code getOffset(const SymbolRef &Sym, uint64_t &Result) { if (std::error_code EC = Sym.getSection(SecI)) return EC; - if (SecI == Obj->section_end()) { - Result = UnknownAddressOrSize; - return object_error::success; - } - - uint64_t SectionAddress; - if (std::error_code EC = SecI->getAddress(SectionAddress)) - return EC; + if (SecI == Obj->section_end()) { + Result = UnknownAddressOrSize; + return object_error::success; + } + uint64_t SectionAddress = SecI->getAddress(); Result = Address - SectionAddress; return object_error::success; } -ObjectImage *RuntimeDyldImpl::loadObject(ObjectImage *InputObject) { +std::unique_ptr<ObjectImage> +RuntimeDyldImpl::loadObject(std::unique_ptr<ObjectImage> Obj) { MutexGuard locked(lock); - std::unique_ptr<ObjectImage> Obj(InputObject); if (!Obj) return nullptr; @@ -158,14 +196,13 @@ ObjectImage *RuntimeDyldImpl::loadObject(ObjectImage *InputObject) { SymType == object::SymbolRef::ST_Unknown) { uint64_t SectOffset; StringRef SectionData; - bool IsCode; section_iterator SI = Obj->end_sections(); Check(getOffset(*I, SectOffset)); Check(I->getSection(SI)); if (SI == Obj->end_sections()) continue; Check(SI->getContents(SectionData)); - Check(SI->isText(IsCode)); + bool IsCode = SI->isText(); unsigned SectionID = findOrEmitSection(*Obj, *SI, IsCode, LocalSections); LocalSymbols[Name.data()] = SymbolLoc(SectionID, SectOffset); @@ -195,8 +232,7 @@ ObjectImage *RuntimeDyldImpl::loadObject(ObjectImage *InputObject) { if (I == E && !ProcessAllSections) continue; - bool IsCode = false; - Check(RelocatedSection->isText(IsCode)); + bool IsCode = RelocatedSection->isText(); SectionID = findOrEmitSection(*Obj, *RelocatedSection, IsCode, LocalSections); DEBUG(dbgs() << "\tSectionID: " << SectionID << "\n"); @@ -204,12 +240,17 @@ ObjectImage *RuntimeDyldImpl::loadObject(ObjectImage *InputObject) { for (; I != E;) I = processRelocationRef(SectionID, I, *Obj, LocalSections, LocalSymbols, Stubs); + + // If there is an attached checker, notify it about the stubs for this + // section so that they can be verified. + if (Checker) + Checker->registerStubMap(Obj->getImageName(), SectionID, Stubs); } // Give the subclasses a chance to tie-up any loose ends. finalizeLoad(*Obj, LocalSections); - return Obj.release(); + return Obj; } // A helper method for computeTotalAllocSize. @@ -245,20 +286,15 @@ void RuntimeDyldImpl::computeTotalAllocSize(ObjectImage &Obj, SI != SE; ++SI) { const SectionRef &Section = *SI; - bool IsRequired; - Check(Section.isRequiredForExecution(IsRequired)); + bool IsRequired = Section.isRequiredForExecution(); // Consider only the sections that are required to be loaded for execution if (IsRequired) { - uint64_t DataSize = 0; - uint64_t Alignment64 = 0; - bool IsCode = false; - bool IsReadOnly = false; StringRef Name; - Check(Section.getSize(DataSize)); - Check(Section.getAlignment(Alignment64)); - Check(Section.isText(IsCode)); - Check(Section.isReadOnlyData(IsReadOnly)); + uint64_t DataSize = Section.getSize(); + uint64_t Alignment64 = Section.getAlignment(); + bool IsCode = Section.isText(); + bool IsReadOnly = Section.isReadOnlyData(); Check(Section.getName(Name)); unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL; @@ -340,10 +376,8 @@ unsigned RuntimeDyldImpl::computeSectionStubBufSize(ObjectImage &Obj, } // Get section data size and alignment - uint64_t Alignment64; - uint64_t DataSize; - Check(Section.getSize(DataSize)); - Check(Section.getAlignment(Alignment64)); + uint64_t DataSize = Section.getSize(); + uint64_t Alignment64 = Section.getAlignment(); // Add stubbuf size alignment unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL; @@ -354,6 +388,36 @@ unsigned RuntimeDyldImpl::computeSectionStubBufSize(ObjectImage &Obj, return StubBufSize; } +uint64_t RuntimeDyldImpl::readBytesUnaligned(uint8_t *Src, + unsigned Size) const { + uint64_t Result = 0; + if (IsTargetLittleEndian) { + Src += Size - 1; + while (Size--) + Result = (Result << 8) | *Src--; + } else + while (Size--) + Result = (Result << 8) | *Src++; + + return Result; +} + +void RuntimeDyldImpl::writeBytesUnaligned(uint64_t Value, uint8_t *Dst, + unsigned Size) const { + if (IsTargetLittleEndian) { + while (Size--) { + *Dst++ = Value & 0xFF; + Value >>= 8; + } + } else { + Dst += Size - 1; + while (Size--) { + *Dst-- = Value & 0xFF; + Value >>= 8; + } + } +} + void RuntimeDyldImpl::emitCommonSymbols(ObjectImage &Obj, const CommonSymbolMap &CommonSymbols, uint64_t TotalSize, @@ -365,7 +429,7 @@ void RuntimeDyldImpl::emitCommonSymbols(ObjectImage &Obj, if (!Addr) report_fatal_error("Unable to allocate memory for common symbols!"); uint64_t Offset = 0; - Sections.push_back(SectionEntry(StringRef(), Addr, TotalSize, 0)); + Sections.push_back(SectionEntry("<common symbols>", Addr, TotalSize, 0)); memset(Addr, 0, TotalSize); DEBUG(dbgs() << "emitCommonSection SectionID: " << SectionID << " new addr: " @@ -397,24 +461,18 @@ unsigned RuntimeDyldImpl::emitSection(ObjectImage &Obj, const SectionRef &Section, bool IsCode) { StringRef data; - uint64_t Alignment64; Check(Section.getContents(data)); - Check(Section.getAlignment(Alignment64)); + uint64_t Alignment64 = Section.getAlignment(); unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL; - bool IsRequired; - bool IsVirtual; - bool IsZeroInit; - bool IsReadOnly; - uint64_t DataSize; unsigned PaddingSize = 0; unsigned StubBufSize = 0; StringRef Name; - Check(Section.isRequiredForExecution(IsRequired)); - Check(Section.isVirtual(IsVirtual)); - Check(Section.isZeroInit(IsZeroInit)); - Check(Section.isReadOnlyData(IsReadOnly)); - Check(Section.getSize(DataSize)); + bool IsRequired = Section.isRequiredForExecution(); + bool IsVirtual = Section.isVirtual(); + bool IsZeroInit = Section.isZeroInit(); + bool IsReadOnly = Section.isReadOnlyData(); + uint64_t DataSize = Section.getSize(); Check(Section.getName(Name)); StubBufSize = computeSectionStubBufSize(Obj, Section); @@ -477,6 +535,10 @@ unsigned RuntimeDyldImpl::emitSection(ObjectImage &Obj, } Sections.push_back(SectionEntry(Name, Addr, DataSize, (uintptr_t)pData)); + + if (Checker) + Checker->registerSection(Obj.getImageName(), SectionID); + return SectionID; } @@ -517,34 +579,26 @@ void RuntimeDyldImpl::addRelocationForSymbol(const RelocationEntry &RE, } } -uint8_t *RuntimeDyldImpl::createStubFunction(uint8_t *Addr) { - if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be || - Arch == Triple::arm64 || Arch == Triple::arm64_be) { +uint8_t *RuntimeDyldImpl::createStubFunction(uint8_t *Addr, + unsigned AbiVariant) { + if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be) { // This stub has to be able to access the full address space, // since symbol lookup won't necessarily find a handy, in-range, // PLT stub for functions which could be anywhere. - uint32_t *StubAddr = (uint32_t *)Addr; - // Stub can use ip0 (== x16) to calculate address - *StubAddr = 0xd2e00010; // movz ip0, #:abs_g3:<addr> - StubAddr++; - *StubAddr = 0xf2c00010; // movk ip0, #:abs_g2_nc:<addr> - StubAddr++; - *StubAddr = 0xf2a00010; // movk ip0, #:abs_g1_nc:<addr> - StubAddr++; - *StubAddr = 0xf2800010; // movk ip0, #:abs_g0_nc:<addr> - StubAddr++; - *StubAddr = 0xd61f0200; // br ip0 + writeBytesUnaligned(0xd2e00010, Addr, 4); // movz ip0, #:abs_g3:<addr> + writeBytesUnaligned(0xf2c00010, Addr+4, 4); // movk ip0, #:abs_g2_nc:<addr> + writeBytesUnaligned(0xf2a00010, Addr+8, 4); // movk ip0, #:abs_g1_nc:<addr> + writeBytesUnaligned(0xf2800010, Addr+12, 4); // movk ip0, #:abs_g0_nc:<addr> + writeBytesUnaligned(0xd61f0200, Addr+16, 4); // br ip0 return Addr; } else if (Arch == Triple::arm || Arch == Triple::armeb) { // TODO: There is only ARM far stub now. We should add the Thumb stub, // and stubs for branches Thumb - ARM and ARM - Thumb. - uint32_t *StubAddr = (uint32_t *)Addr; - *StubAddr = 0xe51ff004; // ldr pc,<label> - return (uint8_t *)++StubAddr; + writeBytesUnaligned(0xe51ff004, Addr, 4); // ldr pc,<label> + return Addr + 4; } else if (Arch == Triple::mipsel || Arch == Triple::mips) { - uint32_t *StubAddr = (uint32_t *)Addr; // 0: 3c190000 lui t9,%hi(addr). // 4: 27390000 addiu t9,t9,%lo(addr). // 8: 03200008 jr t9. @@ -552,31 +606,37 @@ uint8_t *RuntimeDyldImpl::createStubFunction(uint8_t *Addr) { const unsigned LuiT9Instr = 0x3c190000, AdduiT9Instr = 0x27390000; const unsigned JrT9Instr = 0x03200008, NopInstr = 0x0; - *StubAddr = LuiT9Instr; - StubAddr++; - *StubAddr = AdduiT9Instr; - StubAddr++; - *StubAddr = JrT9Instr; - StubAddr++; - *StubAddr = NopInstr; + writeBytesUnaligned(LuiT9Instr, Addr, 4); + writeBytesUnaligned(AdduiT9Instr, Addr+4, 4); + writeBytesUnaligned(JrT9Instr, Addr+8, 4); + writeBytesUnaligned(NopInstr, Addr+12, 4); return Addr; } else if (Arch == Triple::ppc64 || Arch == Triple::ppc64le) { - // PowerPC64 stub: the address points to a function descriptor - // instead of the function itself. Load the function address - // on r11 and sets it to control register. Also loads the function - // TOC in r2 and environment pointer to r11. + // Depending on which version of the ELF ABI is in use, we need to + // generate one of two variants of the stub. They both start with + // the same sequence to load the target address into r12. writeInt32BE(Addr, 0x3D800000); // lis r12, highest(addr) writeInt32BE(Addr+4, 0x618C0000); // ori r12, higher(addr) writeInt32BE(Addr+8, 0x798C07C6); // sldi r12, r12, 32 writeInt32BE(Addr+12, 0x658C0000); // oris r12, r12, h(addr) writeInt32BE(Addr+16, 0x618C0000); // ori r12, r12, l(addr) - writeInt32BE(Addr+20, 0xF8410028); // std r2, 40(r1) - writeInt32BE(Addr+24, 0xE96C0000); // ld r11, 0(r12) - writeInt32BE(Addr+28, 0xE84C0008); // ld r2, 0(r12) - writeInt32BE(Addr+32, 0x7D6903A6); // mtctr r11 - writeInt32BE(Addr+36, 0xE96C0010); // ld r11, 16(r2) - writeInt32BE(Addr+40, 0x4E800420); // bctr - + if (AbiVariant == 2) { + // PowerPC64 stub ELFv2 ABI: The address points to the function itself. + // The address is already in r12 as required by the ABI. Branch to it. + writeInt32BE(Addr+20, 0xF8410018); // std r2, 24(r1) + writeInt32BE(Addr+24, 0x7D8903A6); // mtctr r12 + writeInt32BE(Addr+28, 0x4E800420); // bctr + } else { + // PowerPC64 stub ELFv1 ABI: The address points to a function descriptor. + // Load the function address on r11 and sets it to control register. Also + // loads the function TOC in r2 and environment pointer to r11. + writeInt32BE(Addr+20, 0xF8410028); // std r2, 40(r1) + writeInt32BE(Addr+24, 0xE96C0000); // ld r11, 0(r12) + writeInt32BE(Addr+28, 0xE84C0008); // ld r2, 0(r12) + writeInt32BE(Addr+32, 0x7D6903A6); // mtctr r11 + writeInt32BE(Addr+36, 0xE96C0010); // ld r11, 16(r2) + writeInt32BE(Addr+40, 0x4E800420); // bctr + } return Addr; } else if (Arch == Triple::systemz) { writeInt16BE(Addr, 0xC418); // lgrl %r1,.+8 @@ -609,6 +669,10 @@ void RuntimeDyldImpl::reassignSectionAddress(unsigned SectionID, // Addr is a uint64_t because we can't assume the pointer width // of the target is the same as that of the host. Just use a generic // "big enough" type. + DEBUG(dbgs() << "Reassigning address for section " + << SectionID << " (" << Sections[SectionID].Name << "): " + << format("0x%016" PRIx64, Sections[SectionID].LoadAddress) << " -> " + << format("0x%016" PRIx64, Addr) << "\n"); Sections[SectionID].LoadAddress = Addr; } @@ -685,25 +749,31 @@ RuntimeDyld::RuntimeDyld(RTDyldMemoryManager *mm) { Dyld = nullptr; MM = mm; ProcessAllSections = false; + Checker = nullptr; } -RuntimeDyld::~RuntimeDyld() { delete Dyld; } +RuntimeDyld::~RuntimeDyld() {} static std::unique_ptr<RuntimeDyldELF> -createRuntimeDyldELF(RTDyldMemoryManager *MM, bool ProcessAllSections) { +createRuntimeDyldELF(RTDyldMemoryManager *MM, bool ProcessAllSections, + RuntimeDyldCheckerImpl *Checker) { std::unique_ptr<RuntimeDyldELF> Dyld(new RuntimeDyldELF(MM)); Dyld->setProcessAllSections(ProcessAllSections); + Dyld->setRuntimeDyldChecker(Checker); return Dyld; } static std::unique_ptr<RuntimeDyldMachO> -createRuntimeDyldMachO(RTDyldMemoryManager *MM, bool ProcessAllSections) { - std::unique_ptr<RuntimeDyldMachO> Dyld(new RuntimeDyldMachO(MM)); +createRuntimeDyldMachO(Triple::ArchType Arch, RTDyldMemoryManager *MM, + bool ProcessAllSections, RuntimeDyldCheckerImpl *Checker) { + std::unique_ptr<RuntimeDyldMachO> Dyld(RuntimeDyldMachO::create(Arch, MM)); Dyld->setProcessAllSections(ProcessAllSections); + Dyld->setRuntimeDyldChecker(Checker); return Dyld; } -ObjectImage *RuntimeDyld::loadObject(std::unique_ptr<ObjectFile> InputObject) { +std::unique_ptr<ObjectImage> +RuntimeDyld::loadObject(std::unique_ptr<ObjectFile> InputObject) { std::unique_ptr<ObjectImage> InputImage; ObjectFile &Obj = *InputObject; @@ -711,33 +781,37 @@ ObjectImage *RuntimeDyld::loadObject(std::unique_ptr<ObjectFile> InputObject) { if (InputObject->isELF()) { InputImage.reset(RuntimeDyldELF::createObjectImageFromFile(std::move(InputObject))); if (!Dyld) - Dyld = createRuntimeDyldELF(MM, ProcessAllSections).release(); + Dyld = createRuntimeDyldELF(MM, ProcessAllSections, Checker); } else if (InputObject->isMachO()) { InputImage.reset(RuntimeDyldMachO::createObjectImageFromFile(std::move(InputObject))); if (!Dyld) - Dyld = createRuntimeDyldMachO(MM, ProcessAllSections).release(); + Dyld = createRuntimeDyldMachO( + static_cast<Triple::ArchType>(InputImage->getArch()), MM, + ProcessAllSections, Checker); } else report_fatal_error("Incompatible object format!"); if (!Dyld->isCompatibleFile(&Obj)) report_fatal_error("Incompatible object format!"); - Dyld->loadObject(InputImage.get()); - return InputImage.release(); + return Dyld->loadObject(std::move(InputImage)); } -ObjectImage *RuntimeDyld::loadObject(ObjectBuffer *InputBuffer) { +std::unique_ptr<ObjectImage> +RuntimeDyld::loadObject(std::unique_ptr<ObjectBuffer> InputBuffer) { std::unique_ptr<ObjectImage> InputImage; sys::fs::file_magic Type = sys::fs::identify_magic(InputBuffer->getBuffer()); + auto *InputBufferPtr = InputBuffer.get(); switch (Type) { + case sys::fs::file_magic::elf: case sys::fs::file_magic::elf_relocatable: case sys::fs::file_magic::elf_executable: case sys::fs::file_magic::elf_shared_object: case sys::fs::file_magic::elf_core: - InputImage.reset(RuntimeDyldELF::createObjectImage(InputBuffer)); + InputImage = RuntimeDyldELF::createObjectImage(std::move(InputBuffer)); if (!Dyld) - Dyld = createRuntimeDyldELF(MM, ProcessAllSections).release(); + Dyld = createRuntimeDyldELF(MM, ProcessAllSections, Checker); break; case sys::fs::file_magic::macho_object: case sys::fs::file_magic::macho_executable: @@ -749,9 +823,11 @@ ObjectImage *RuntimeDyld::loadObject(ObjectBuffer *InputBuffer) { case sys::fs::file_magic::macho_bundle: case sys::fs::file_magic::macho_dynamically_linked_shared_lib_stub: case sys::fs::file_magic::macho_dsym_companion: - InputImage.reset(RuntimeDyldMachO::createObjectImage(InputBuffer)); + InputImage = RuntimeDyldMachO::createObjectImage(std::move(InputBuffer)); if (!Dyld) - Dyld = createRuntimeDyldMachO(MM, ProcessAllSections).release(); + Dyld = createRuntimeDyldMachO( + static_cast<Triple::ArchType>(InputImage->getArch()), MM, + ProcessAllSections, Checker); break; case sys::fs::file_magic::unknown: case sys::fs::file_magic::bitcode: @@ -764,20 +840,19 @@ ObjectImage *RuntimeDyld::loadObject(ObjectBuffer *InputBuffer) { report_fatal_error("Incompatible object format!"); } - if (!Dyld->isCompatibleFormat(InputBuffer)) + if (!Dyld->isCompatibleFormat(InputBufferPtr)) report_fatal_error("Incompatible object format!"); - Dyld->loadObject(InputImage.get()); - return InputImage.release(); + return Dyld->loadObject(std::move(InputImage)); } -void *RuntimeDyld::getSymbolAddress(StringRef Name) { +void *RuntimeDyld::getSymbolAddress(StringRef Name) const { if (!Dyld) return nullptr; return Dyld->getSymbolAddress(Name); } -uint64_t RuntimeDyld::getSymbolLoadAddress(StringRef Name) { +uint64_t RuntimeDyld::getSymbolLoadAddress(StringRef Name) const { if (!Dyld) return 0; return Dyld->getSymbolLoadAddress(Name); diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index 190bbbf..8818349 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -7,11 +7,13 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/STLExtras.h" #include "llvm/ExecutionEngine/RuntimeDyldChecker.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDisassembler.h" #include "llvm/MC/MCInst.h" -#include "llvm/Support/StringRefMemoryObject.h" +#include "llvm/Support/Path.h" +#include "RuntimeDyldCheckerImpl.h" #include "RuntimeDyldImpl.h" #include <cctype> #include <memory> @@ -22,579 +24,696 @@ using namespace llvm; namespace llvm { - // Helper class that implements the language evaluated by RuntimeDyldChecker. - class RuntimeDyldCheckerExprEval { - public: - - RuntimeDyldCheckerExprEval(const RuntimeDyldChecker &Checker, - llvm::raw_ostream &ErrStream) - : Checker(Checker), ErrStream(ErrStream) {} - - bool evaluate(StringRef Expr) const { - // Expect equality expression of the form 'LHS = RHS'. - Expr = Expr.trim(); - size_t EQIdx = Expr.find('='); - - // Evaluate LHS. - StringRef LHSExpr = Expr.substr(0, EQIdx).rtrim(); - StringRef RemainingExpr; - EvalResult LHSResult; - std::tie(LHSResult, RemainingExpr) = - evalComplexExpr(evalSimpleExpr(LHSExpr)); - if (LHSResult.hasError()) - return handleError(Expr, LHSResult); - if (RemainingExpr != "") - return handleError(Expr, unexpectedToken(RemainingExpr, LHSExpr, "")); - - // Evaluate RHS. - StringRef RHSExpr = Expr.substr(EQIdx + 1).ltrim(); - EvalResult RHSResult; - std::tie(RHSResult, RemainingExpr) = - evalComplexExpr(evalSimpleExpr(RHSExpr)); - if (RHSResult.hasError()) - return handleError(Expr, RHSResult); - if (RemainingExpr != "") - return handleError(Expr, unexpectedToken(RemainingExpr, RHSExpr, "")); - - if (LHSResult.getValue() != RHSResult.getValue()) { - ErrStream << "Expression '" << Expr << "' is false: " - << format("0x%lx", LHSResult.getValue()) << " != " - << format("0x%lx", RHSResult.getValue()) << "\n"; - return false; - } - return true; +// Helper class that implements the language evaluated by RuntimeDyldChecker. +class RuntimeDyldCheckerExprEval { +public: + RuntimeDyldCheckerExprEval(const RuntimeDyldCheckerImpl &Checker, + raw_ostream &ErrStream) + : Checker(Checker) {} + + bool evaluate(StringRef Expr) const { + // Expect equality expression of the form 'LHS = RHS'. + Expr = Expr.trim(); + size_t EQIdx = Expr.find('='); + + ParseContext OutsideLoad(false); + + // Evaluate LHS. + StringRef LHSExpr = Expr.substr(0, EQIdx).rtrim(); + StringRef RemainingExpr; + EvalResult LHSResult; + std::tie(LHSResult, RemainingExpr) = + evalComplexExpr(evalSimpleExpr(LHSExpr, OutsideLoad), OutsideLoad); + if (LHSResult.hasError()) + return handleError(Expr, LHSResult); + if (RemainingExpr != "") + return handleError(Expr, unexpectedToken(RemainingExpr, LHSExpr, "")); + + // Evaluate RHS. + StringRef RHSExpr = Expr.substr(EQIdx + 1).ltrim(); + EvalResult RHSResult; + std::tie(RHSResult, RemainingExpr) = + evalComplexExpr(evalSimpleExpr(RHSExpr, OutsideLoad), OutsideLoad); + if (RHSResult.hasError()) + return handleError(Expr, RHSResult); + if (RemainingExpr != "") + return handleError(Expr, unexpectedToken(RemainingExpr, RHSExpr, "")); + + if (LHSResult.getValue() != RHSResult.getValue()) { + Checker.ErrStream << "Expression '" << Expr << "' is false: " + << format("0x%" PRIx64, LHSResult.getValue()) + << " != " << format("0x%" PRIx64, RHSResult.getValue()) + << "\n"; + return false; } + return true; + } + +private: + // RuntimeDyldCheckerExprEval requires some context when parsing exprs. In + // particular, it needs to know whether a symbol is being evaluated in the + // context of a load, in which case we want the linker's local address for + // the symbol, or outside of a load, in which case we want the symbol's + // address in the remote target. + + struct ParseContext { + bool IsInsideLoad; + ParseContext(bool IsInsideLoad) : IsInsideLoad(IsInsideLoad) {} + }; + + const RuntimeDyldCheckerImpl &Checker; + + enum class BinOpToken : unsigned { + Invalid, + Add, + Sub, + BitwiseAnd, + BitwiseOr, + ShiftLeft, + ShiftRight + }; + + class EvalResult { + public: + EvalResult() : Value(0), ErrorMsg("") {} + EvalResult(uint64_t Value) : Value(Value), ErrorMsg("") {} + EvalResult(std::string ErrorMsg) : Value(0), ErrorMsg(ErrorMsg) {} + uint64_t getValue() const { return Value; } + bool hasError() const { return ErrorMsg != ""; } + const std::string &getErrorMsg() const { return ErrorMsg; } private: - const RuntimeDyldChecker &Checker; - llvm::raw_ostream &ErrStream; - - enum class BinOpToken : unsigned { Invalid, Add, Sub, BitwiseAnd, - BitwiseOr, ShiftLeft, ShiftRight }; - - class EvalResult { - public: - EvalResult() - : Value(0), ErrorMsg("") {} - EvalResult(uint64_t Value) - : Value(Value), ErrorMsg("") {} - EvalResult(std::string ErrorMsg) - : Value(0), ErrorMsg(ErrorMsg) {} - uint64_t getValue() const { return Value; } - bool hasError() const { return ErrorMsg != ""; } - const std::string& getErrorMsg() const { return ErrorMsg; } - private: - uint64_t Value; - std::string ErrorMsg; - }; - - StringRef getTokenForError(StringRef Expr) const { - if (Expr.empty()) - return ""; - - StringRef Token, Remaining; - if (isalpha(Expr[0])) - std::tie(Token, Remaining) = parseSymbol(Expr); - else if (isdigit(Expr[0])) - std::tie(Token, Remaining) = parseNumberString(Expr); - else { - unsigned TokLen = 1; - if (Expr.startswith("<<") || Expr.startswith(">>")) - TokLen = 2; - Token = Expr.substr(0, TokLen); - } - return Token; - } + uint64_t Value; + std::string ErrorMsg; + }; - EvalResult unexpectedToken(StringRef TokenStart, - StringRef SubExpr, - StringRef ErrText) const { - std::string ErrorMsg("Encountered unexpected token '"); - ErrorMsg += getTokenForError(TokenStart); - if (SubExpr != "") { - ErrorMsg += "' while parsing subexpression '"; - ErrorMsg += SubExpr; - } - ErrorMsg += "'"; - if (ErrText != "") { - ErrorMsg += " "; - ErrorMsg += ErrText; - } - return EvalResult(std::move(ErrorMsg)); + StringRef getTokenForError(StringRef Expr) const { + if (Expr.empty()) + return ""; + + StringRef Token, Remaining; + if (isalpha(Expr[0])) + std::tie(Token, Remaining) = parseSymbol(Expr); + else if (isdigit(Expr[0])) + std::tie(Token, Remaining) = parseNumberString(Expr); + else { + unsigned TokLen = 1; + if (Expr.startswith("<<") || Expr.startswith(">>")) + TokLen = 2; + Token = Expr.substr(0, TokLen); } + return Token; + } - bool handleError(StringRef Expr, const EvalResult &R) const { - assert(R.hasError() && "Not an error result."); - ErrStream << "Error evaluating expression '" << Expr << "': " - << R.getErrorMsg() << "\n"; - return false; + EvalResult unexpectedToken(StringRef TokenStart, StringRef SubExpr, + StringRef ErrText) const { + std::string ErrorMsg("Encountered unexpected token '"); + ErrorMsg += getTokenForError(TokenStart); + if (SubExpr != "") { + ErrorMsg += "' while parsing subexpression '"; + ErrorMsg += SubExpr; } + ErrorMsg += "'"; + if (ErrText != "") { + ErrorMsg += " "; + ErrorMsg += ErrText; + } + return EvalResult(std::move(ErrorMsg)); + } - std::pair<BinOpToken, StringRef> parseBinOpToken(StringRef Expr) const { - if (Expr.empty()) - return std::make_pair(BinOpToken::Invalid, ""); - - // Handle the two 2-character tokens. - if (Expr.startswith("<<")) - return std::make_pair(BinOpToken::ShiftLeft, - Expr.substr(2).ltrim()); - if (Expr.startswith(">>")) - return std::make_pair(BinOpToken::ShiftRight, - Expr.substr(2).ltrim()); - - // Handle one-character tokens. - BinOpToken Op; - switch (Expr[0]) { - default: return std::make_pair(BinOpToken::Invalid, Expr); - case '+': Op = BinOpToken::Add; break; - case '-': Op = BinOpToken::Sub; break; - case '&': Op = BinOpToken::BitwiseAnd; break; - case '|': Op = BinOpToken::BitwiseOr; break; - } + bool handleError(StringRef Expr, const EvalResult &R) const { + assert(R.hasError() && "Not an error result."); + Checker.ErrStream << "Error evaluating expression '" << Expr + << "': " << R.getErrorMsg() << "\n"; + return false; + } - return std::make_pair(Op, Expr.substr(1).ltrim()); + std::pair<BinOpToken, StringRef> parseBinOpToken(StringRef Expr) const { + if (Expr.empty()) + return std::make_pair(BinOpToken::Invalid, ""); + + // Handle the two 2-character tokens. + if (Expr.startswith("<<")) + return std::make_pair(BinOpToken::ShiftLeft, Expr.substr(2).ltrim()); + if (Expr.startswith(">>")) + return std::make_pair(BinOpToken::ShiftRight, Expr.substr(2).ltrim()); + + // Handle one-character tokens. + BinOpToken Op; + switch (Expr[0]) { + default: + return std::make_pair(BinOpToken::Invalid, Expr); + case '+': + Op = BinOpToken::Add; + break; + case '-': + Op = BinOpToken::Sub; + break; + case '&': + Op = BinOpToken::BitwiseAnd; + break; + case '|': + Op = BinOpToken::BitwiseOr; + break; } - EvalResult computeBinOpResult(BinOpToken Op, const EvalResult &LHSResult, - const EvalResult &RHSResult) const { - switch (Op) { - default: llvm_unreachable("Tried to evaluate unrecognized operation."); - case BinOpToken::Add: - return EvalResult(LHSResult.getValue() + RHSResult.getValue()); - case BinOpToken::Sub: - return EvalResult(LHSResult.getValue() - RHSResult.getValue()); - case BinOpToken::BitwiseAnd: - return EvalResult(LHSResult.getValue() & RHSResult.getValue()); - case BinOpToken::BitwiseOr: - return EvalResult(LHSResult.getValue() | RHSResult.getValue()); - case BinOpToken::ShiftLeft: - return EvalResult(LHSResult.getValue() << RHSResult.getValue()); - case BinOpToken::ShiftRight: - return EvalResult(LHSResult.getValue() >> RHSResult.getValue()); - } - } + return std::make_pair(Op, Expr.substr(1).ltrim()); + } - // Parse a symbol and return a (string, string) pair representing the symbol - // name and expression remaining to be parsed. - std::pair<StringRef, StringRef> parseSymbol(StringRef Expr) const { - size_t FirstNonSymbol = - Expr.find_first_not_of("0123456789" - "abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - ":_"); - return std::make_pair(Expr.substr(0, FirstNonSymbol), - Expr.substr(FirstNonSymbol).ltrim()); + EvalResult computeBinOpResult(BinOpToken Op, const EvalResult &LHSResult, + const EvalResult &RHSResult) const { + switch (Op) { + default: + llvm_unreachable("Tried to evaluate unrecognized operation."); + case BinOpToken::Add: + return EvalResult(LHSResult.getValue() + RHSResult.getValue()); + case BinOpToken::Sub: + return EvalResult(LHSResult.getValue() - RHSResult.getValue()); + case BinOpToken::BitwiseAnd: + return EvalResult(LHSResult.getValue() & RHSResult.getValue()); + case BinOpToken::BitwiseOr: + return EvalResult(LHSResult.getValue() | RHSResult.getValue()); + case BinOpToken::ShiftLeft: + return EvalResult(LHSResult.getValue() << RHSResult.getValue()); + case BinOpToken::ShiftRight: + return EvalResult(LHSResult.getValue() >> RHSResult.getValue()); } + } - // Evaluate a call to decode_operand. Decode the instruction operand at the - // given symbol and get the value of the requested operand. - // Returns an error if the instruction cannot be decoded, or the requested - // operand is not an immediate. - // On success, retuns a pair containing the value of the operand, plus - // the expression remaining to be evaluated. - std::pair<EvalResult, StringRef> evalDecodeOperand(StringRef Expr) const { - if (!Expr.startswith("(")) - return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); - StringRef RemainingExpr = Expr.substr(1).ltrim(); - StringRef Symbol; - std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); - - if (!Checker.checkSymbolIsValidForLoad(Symbol)) - return std::make_pair(EvalResult(("Cannot decode unknown symbol '" + - Symbol + "'").str()), - ""); - - if (!RemainingExpr.startswith(",")) - return std::make_pair(unexpectedToken(RemainingExpr, RemainingExpr, - "expected ','"), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - EvalResult OpIdxExpr; - std::tie(OpIdxExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); - if (OpIdxExpr.hasError()) - return std::make_pair(OpIdxExpr, ""); - - if (!RemainingExpr.startswith(")")) - return std::make_pair(unexpectedToken(RemainingExpr, RemainingExpr, - "expected ')'"), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - MCInst Inst; - uint64_t Size; - if (!decodeInst(Symbol, Inst, Size)) - return std::make_pair(EvalResult(("Couldn't decode instruction at '" + - Symbol + "'").str()), - ""); - - unsigned OpIdx = OpIdxExpr.getValue(); - if (OpIdx >= Inst.getNumOperands()) { - std::string ErrMsg; - raw_string_ostream ErrMsgStream(ErrMsg); - ErrMsgStream << "Invalid operand index '" << format("%i", OpIdx) - << " for instruction '" << Symbol - << ". Instruction has only " - << format("%i", Inst.getNumOperands()) << " operands."; - return std::make_pair(EvalResult(ErrMsgStream.str()), ""); - } + // Parse a symbol and return a (string, string) pair representing the symbol + // name and expression remaining to be parsed. + std::pair<StringRef, StringRef> parseSymbol(StringRef Expr) const { + size_t FirstNonSymbol = Expr.find_first_not_of("0123456789" + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + ":_.$"); + return std::make_pair(Expr.substr(0, FirstNonSymbol), + Expr.substr(FirstNonSymbol).ltrim()); + } - const MCOperand &Op = Inst.getOperand(OpIdx); - if (!Op.isImm()) { - std::string ErrMsg; - raw_string_ostream ErrMsgStream(ErrMsg); - ErrMsgStream << "Operand '" << format("%i", OpIdx) - << "' of instruction '" << Symbol - << "' is not an immediate.\nInstruction is:\n "; - Inst.dump_pretty(ErrMsgStream, - Checker.Disassembler->getContext().getAsmInfo(), - Checker.InstPrinter); - - return std::make_pair(EvalResult(ErrMsgStream.str()), ""); - } + // Evaluate a call to decode_operand. Decode the instruction operand at the + // given symbol and get the value of the requested operand. + // Returns an error if the instruction cannot be decoded, or the requested + // operand is not an immediate. + // On success, retuns a pair containing the value of the operand, plus + // the expression remaining to be evaluated. + std::pair<EvalResult, StringRef> evalDecodeOperand(StringRef Expr) const { + if (!Expr.startswith("(")) + return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); + StringRef RemainingExpr = Expr.substr(1).ltrim(); + StringRef Symbol; + std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); + + if (!Checker.isSymbolValid(Symbol)) + return std::make_pair( + EvalResult(("Cannot decode unknown symbol '" + Symbol + "'").str()), + ""); - return std::make_pair(EvalResult(Op.getImm()), RemainingExpr); - } + if (!RemainingExpr.startswith(",")) + return std::make_pair( + unexpectedToken(RemainingExpr, RemainingExpr, "expected ','"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); - // Evaluate a call to next_pc. Decode the instruction at the given - // symbol and return the following program counter.. - // Returns an error if the instruction cannot be decoded. - // On success, returns a pair containing the next PC, plus the length of the - // expression remaining to be evaluated. - std::pair<EvalResult, StringRef> evalNextPC(StringRef Expr) const { - if (!Expr.startswith("(")) - return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); - StringRef RemainingExpr = Expr.substr(1).ltrim(); - StringRef Symbol; - std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); - - if (!Checker.checkSymbolIsValidForLoad(Symbol)) - return std::make_pair(EvalResult(("Cannot decode unknown symbol '" - + Symbol + "'").str()), - ""); - - if (!RemainingExpr.startswith(")")) - return std::make_pair(unexpectedToken(RemainingExpr, RemainingExpr, - "expected ')'"), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - MCInst Inst; - uint64_t Size; - if (!decodeInst(Symbol, Inst, Size)) - return std::make_pair(EvalResult(("Couldn't decode instruction at '" + - Symbol + "'").str()), - ""); - uint64_t NextPC = Checker.getSymbolAddress(Symbol) + Size; - - return std::make_pair(EvalResult(NextPC), RemainingExpr); - } + EvalResult OpIdxExpr; + std::tie(OpIdxExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); + if (OpIdxExpr.hasError()) + return std::make_pair(OpIdxExpr, ""); - // Evaluate an identiefer expr, which may be a symbol, or a call to - // one of the builtin functions: get_insn_opcode or get_insn_length. - // Return the result, plus the expression remaining to be parsed. - std::pair<EvalResult, StringRef> evalIdentifierExpr(StringRef Expr) const { - StringRef Symbol; - StringRef RemainingExpr; - std::tie(Symbol, RemainingExpr) = parseSymbol(Expr); - - // Check for builtin function calls. - if (Symbol == "decode_operand") - return evalDecodeOperand(RemainingExpr); - else if (Symbol == "next_pc") - return evalNextPC(RemainingExpr); - - // Looks like a plain symbol reference. - return std::make_pair(EvalResult(Checker.getSymbolAddress(Symbol)), - RemainingExpr); - } + if (!RemainingExpr.startswith(")")) + return std::make_pair( + unexpectedToken(RemainingExpr, RemainingExpr, "expected ')'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); - // Parse a number (hexadecimal or decimal) and return a (string, string) - // pair representing the number and the expression remaining to be parsed. - std::pair<StringRef, StringRef> parseNumberString(StringRef Expr) const { - size_t FirstNonDigit = StringRef::npos; - if (Expr.startswith("0x")) { - FirstNonDigit = Expr.find_first_not_of("0123456789abcdefABCDEF", 2); - if (FirstNonDigit == StringRef::npos) - FirstNonDigit = Expr.size(); - } else { - FirstNonDigit = Expr.find_first_not_of("0123456789"); - if (FirstNonDigit == StringRef::npos) - FirstNonDigit = Expr.size(); - } - return std::make_pair(Expr.substr(0, FirstNonDigit), - Expr.substr(FirstNonDigit)); + MCInst Inst; + uint64_t Size; + if (!decodeInst(Symbol, Inst, Size)) + return std::make_pair( + EvalResult(("Couldn't decode instruction at '" + Symbol + "'").str()), + ""); + + unsigned OpIdx = OpIdxExpr.getValue(); + if (OpIdx >= Inst.getNumOperands()) { + std::string ErrMsg; + raw_string_ostream ErrMsgStream(ErrMsg); + ErrMsgStream << "Invalid operand index '" << format("%i", OpIdx) + << "' for instruction '" << Symbol + << "'. Instruction has only " + << format("%i", Inst.getNumOperands()) + << " operands.\nInstruction is:\n "; + Inst.dump_pretty(ErrMsgStream, + Checker.Disassembler->getContext().getAsmInfo(), + Checker.InstPrinter); + return std::make_pair(EvalResult(ErrMsgStream.str()), ""); } - // Evaluate a constant numeric expression (hexidecimal or decimal) and - // return a pair containing the result, and the expression remaining to be - // evaluated. - std::pair<EvalResult, StringRef> evalNumberExpr(StringRef Expr) const { - StringRef ValueStr; - StringRef RemainingExpr; - std::tie(ValueStr, RemainingExpr) = parseNumberString(Expr); - - if (ValueStr.empty() || !isdigit(ValueStr[0])) - return std::make_pair(unexpectedToken(RemainingExpr, RemainingExpr, - "expected number"), - ""); - uint64_t Value; - ValueStr.getAsInteger(0, Value); - return std::make_pair(EvalResult(Value), RemainingExpr); + const MCOperand &Op = Inst.getOperand(OpIdx); + if (!Op.isImm()) { + std::string ErrMsg; + raw_string_ostream ErrMsgStream(ErrMsg); + ErrMsgStream << "Operand '" << format("%i", OpIdx) << "' of instruction '" + << Symbol << "' is not an immediate.\nInstruction is:\n "; + Inst.dump_pretty(ErrMsgStream, + Checker.Disassembler->getContext().getAsmInfo(), + Checker.InstPrinter); + + return std::make_pair(EvalResult(ErrMsgStream.str()), ""); } - // Evaluate an expression of the form "(<expr>)" and return a pair - // containing the result of evaluating <expr>, plus the expression - // remaining to be parsed. - std::pair<EvalResult, StringRef> evalParensExpr(StringRef Expr) const { - assert(Expr.startswith("(") && "Not a parenthesized expression"); - EvalResult SubExprResult; - StringRef RemainingExpr; - std::tie(SubExprResult, RemainingExpr) = - evalComplexExpr(evalSimpleExpr(Expr.substr(1).ltrim())); - if (SubExprResult.hasError()) - return std::make_pair(SubExprResult, ""); - if (!RemainingExpr.startswith(")")) - return std::make_pair(unexpectedToken(RemainingExpr, Expr, - "expected ')'"), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - return std::make_pair(SubExprResult, RemainingExpr); - } + return std::make_pair(EvalResult(Op.getImm()), RemainingExpr); + } - // Evaluate an expression in one of the following forms: - // *{<number>}<symbol> - // *{<number>}(<symbol> + <number>) - // *{<number>}(<symbol> - <number>) - // Return a pair containing the result, plus the expression remaining to be - // parsed. - std::pair<EvalResult, StringRef> evalLoadExpr(StringRef Expr) const { - assert(Expr.startswith("*") && "Not a load expression"); - StringRef RemainingExpr = Expr.substr(1).ltrim(); - // Parse read size. - if (!RemainingExpr.startswith("{")) - return std::make_pair(EvalResult("Expected '{' following '*'."), ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - EvalResult ReadSizeExpr; - std::tie(ReadSizeExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); - if (ReadSizeExpr.hasError()) - return std::make_pair(ReadSizeExpr, RemainingExpr); - uint64_t ReadSize = ReadSizeExpr.getValue(); - if (ReadSize < 1 || ReadSize > 8) - return std::make_pair(EvalResult("Invalid size for dereference."), ""); - if (!RemainingExpr.startswith("}")) - return std::make_pair(EvalResult("Missing '}' for dereference."), ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - // Check for '(symbol +/- constant)' form. - bool SymbolPlusConstant = false; - if (RemainingExpr.startswith("(")) { - SymbolPlusConstant = true; - RemainingExpr = RemainingExpr.substr(1).ltrim(); - } + // Evaluate a call to next_pc. + // Decode the instruction at the given symbol and return the following program + // counter. + // Returns an error if the instruction cannot be decoded. + // On success, returns a pair containing the next PC, plus of the + // expression remaining to be evaluated. + std::pair<EvalResult, StringRef> evalNextPC(StringRef Expr, + ParseContext PCtx) const { + if (!Expr.startswith("(")) + return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); + StringRef RemainingExpr = Expr.substr(1).ltrim(); + StringRef Symbol; + std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); + + if (!Checker.isSymbolValid(Symbol)) + return std::make_pair( + EvalResult(("Cannot decode unknown symbol '" + Symbol + "'").str()), + ""); + + if (!RemainingExpr.startswith(")")) + return std::make_pair( + unexpectedToken(RemainingExpr, RemainingExpr, "expected ')'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + MCInst Inst; + uint64_t InstSize; + if (!decodeInst(Symbol, Inst, InstSize)) + return std::make_pair( + EvalResult(("Couldn't decode instruction at '" + Symbol + "'").str()), + ""); + + uint64_t SymbolAddr = PCtx.IsInsideLoad + ? Checker.getSymbolLinkerAddr(Symbol) + : Checker.getSymbolRemoteAddr(Symbol); + uint64_t NextPC = SymbolAddr + InstSize; - // Read symbol. - StringRef Symbol; - std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); + return std::make_pair(EvalResult(NextPC), RemainingExpr); + } - if (!Checker.checkSymbolIsValidForLoad(Symbol)) - return std::make_pair(EvalResult(("Cannot dereference unknown symbol '" - + Symbol + "'").str()), - ""); + // Evaluate a call to stub_addr. + // Look up and return the address of the stub for the given + // (<file name>, <section name>, <symbol name>) tuple. + // On success, returns a pair containing the stub address, plus the expression + // remaining to be evaluated. + std::pair<EvalResult, StringRef> evalStubAddr(StringRef Expr, + ParseContext PCtx) const { + if (!Expr.startswith("(")) + return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); + StringRef RemainingExpr = Expr.substr(1).ltrim(); + + // Handle file-name specially, as it may contain characters that aren't + // legal for symbols. + StringRef FileName; + size_t ComaIdx = RemainingExpr.find(','); + FileName = RemainingExpr.substr(0, ComaIdx).rtrim(); + RemainingExpr = RemainingExpr.substr(ComaIdx).ltrim(); + + if (!RemainingExpr.startswith(",")) + return std::make_pair( + unexpectedToken(RemainingExpr, Expr, "expected ','"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); - // Set up defaut offset. - int64_t Offset = 0; + StringRef SectionName; + std::tie(SectionName, RemainingExpr) = parseSymbol(RemainingExpr); - // Handle "+/- constant)" portion if necessary. - if (SymbolPlusConstant) { - char OpChar = RemainingExpr[0]; - if (OpChar != '+' && OpChar != '-') - return std::make_pair(EvalResult("Invalid operator in load address."), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); + if (!RemainingExpr.startswith(",")) + return std::make_pair( + unexpectedToken(RemainingExpr, Expr, "expected ','"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); - EvalResult OffsetExpr; - std::tie(OffsetExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); + StringRef Symbol; + std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); - Offset = (OpChar == '+') ? - OffsetExpr.getValue() : -1 * OffsetExpr.getValue(); + if (!RemainingExpr.startswith(")")) + return std::make_pair( + unexpectedToken(RemainingExpr, Expr, "expected ')'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); - if (!RemainingExpr.startswith(")")) - return std::make_pair(EvalResult("Missing ')' in load address."), - ""); + uint64_t StubAddr; + std::string ErrorMsg = ""; + std::tie(StubAddr, ErrorMsg) = Checker.getStubAddrFor( + FileName, SectionName, Symbol, PCtx.IsInsideLoad); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - } + if (ErrorMsg != "") + return std::make_pair(EvalResult(ErrorMsg), ""); + return std::make_pair(EvalResult(StubAddr), RemainingExpr); + } + + std::pair<EvalResult, StringRef> evalSectionAddr(StringRef Expr, + ParseContext PCtx) const { + if (!Expr.startswith("(")) + return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); + StringRef RemainingExpr = Expr.substr(1).ltrim(); + + // Handle file-name specially, as it may contain characters that aren't + // legal for symbols. + StringRef FileName; + size_t ComaIdx = RemainingExpr.find(','); + FileName = RemainingExpr.substr(0, ComaIdx).rtrim(); + RemainingExpr = RemainingExpr.substr(ComaIdx).ltrim(); + + if (!RemainingExpr.startswith(",")) return std::make_pair( - EvalResult(Checker.readMemoryAtSymbol(Symbol, Offset, ReadSize)), - RemainingExpr); + unexpectedToken(RemainingExpr, Expr, "expected ','"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + StringRef SectionName; + std::tie(SectionName, RemainingExpr) = parseSymbol(RemainingExpr); + + if (!RemainingExpr.startswith(")")) + return std::make_pair( + unexpectedToken(RemainingExpr, Expr, "expected ')'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + uint64_t StubAddr; + std::string ErrorMsg = ""; + std::tie(StubAddr, ErrorMsg) = Checker.getSectionAddr( + FileName, SectionName, PCtx.IsInsideLoad); + + if (ErrorMsg != "") + return std::make_pair(EvalResult(ErrorMsg), ""); + + return std::make_pair(EvalResult(StubAddr), RemainingExpr); + } + + // Evaluate an identiefer expr, which may be a symbol, or a call to + // one of the builtin functions: get_insn_opcode or get_insn_length. + // Return the result, plus the expression remaining to be parsed. + std::pair<EvalResult, StringRef> evalIdentifierExpr(StringRef Expr, + ParseContext PCtx) const { + StringRef Symbol; + StringRef RemainingExpr; + std::tie(Symbol, RemainingExpr) = parseSymbol(Expr); + + // Check for builtin function calls. + if (Symbol == "decode_operand") + return evalDecodeOperand(RemainingExpr); + else if (Symbol == "next_pc") + return evalNextPC(RemainingExpr, PCtx); + else if (Symbol == "stub_addr") + return evalStubAddr(RemainingExpr, PCtx); + else if (Symbol == "section_addr") + return evalSectionAddr(RemainingExpr, PCtx); + + if (!Checker.isSymbolValid(Symbol)) { + std::string ErrMsg("No known address for symbol '"); + ErrMsg += Symbol; + ErrMsg += "'"; + if (Symbol.startswith("L")) + ErrMsg += " (this appears to be an assembler local label - " + " perhaps drop the 'L'?)"; + + return std::make_pair(EvalResult(ErrMsg), ""); } - // Evaluate a "simple" expression. This is any expression that _isn't_ an - // un-parenthesized binary expression. - // - // "Simple" expressions can be optionally bit-sliced. See evalSlicedExpr. - // - // Returns a pair containing the result of the evaluation, plus the - // expression remaining to be parsed. - std::pair<EvalResult, StringRef> evalSimpleExpr(StringRef Expr) const { - EvalResult SubExprResult; - StringRef RemainingExpr; - - if (Expr.empty()) - return std::make_pair(EvalResult("Unexpected end of expression"), ""); - - if (Expr[0] == '(') - std::tie(SubExprResult, RemainingExpr) = evalParensExpr(Expr); - else if (Expr[0] == '*') - std::tie(SubExprResult, RemainingExpr) = evalLoadExpr(Expr); - else if (isalpha(Expr[0])) - std::tie(SubExprResult, RemainingExpr) = evalIdentifierExpr(Expr); - else if (isdigit(Expr[0])) - std::tie(SubExprResult, RemainingExpr) = evalNumberExpr(Expr); - - if (SubExprResult.hasError()) - return std::make_pair(SubExprResult, RemainingExpr); - - // Evaluate bit-slice if present. - if (RemainingExpr.startswith("[")) - std::tie(SubExprResult, RemainingExpr) = - evalSliceExpr(std::make_pair(SubExprResult, RemainingExpr)); + // The value for the symbol depends on the context we're evaluating in: + // Inside a load this is the address in the linker's memory, outside a + // load it's the address in the target processes memory. + uint64_t Value = PCtx.IsInsideLoad ? Checker.getSymbolLinkerAddr(Symbol) + : Checker.getSymbolRemoteAddr(Symbol); - return std::make_pair(SubExprResult, RemainingExpr); + // Looks like a plain symbol reference. + return std::make_pair(EvalResult(Value), RemainingExpr); + } + + // Parse a number (hexadecimal or decimal) and return a (string, string) + // pair representing the number and the expression remaining to be parsed. + std::pair<StringRef, StringRef> parseNumberString(StringRef Expr) const { + size_t FirstNonDigit = StringRef::npos; + if (Expr.startswith("0x")) { + FirstNonDigit = Expr.find_first_not_of("0123456789abcdefABCDEF", 2); + if (FirstNonDigit == StringRef::npos) + FirstNonDigit = Expr.size(); + } else { + FirstNonDigit = Expr.find_first_not_of("0123456789"); + if (FirstNonDigit == StringRef::npos) + FirstNonDigit = Expr.size(); } + return std::make_pair(Expr.substr(0, FirstNonDigit), + Expr.substr(FirstNonDigit)); + } + + // Evaluate a constant numeric expression (hexidecimal or decimal) and + // return a pair containing the result, and the expression remaining to be + // evaluated. + std::pair<EvalResult, StringRef> evalNumberExpr(StringRef Expr) const { + StringRef ValueStr; + StringRef RemainingExpr; + std::tie(ValueStr, RemainingExpr) = parseNumberString(Expr); + + if (ValueStr.empty() || !isdigit(ValueStr[0])) + return std::make_pair( + unexpectedToken(RemainingExpr, RemainingExpr, "expected number"), ""); + uint64_t Value; + ValueStr.getAsInteger(0, Value); + return std::make_pair(EvalResult(Value), RemainingExpr); + } + + // Evaluate an expression of the form "(<expr>)" and return a pair + // containing the result of evaluating <expr>, plus the expression + // remaining to be parsed. + std::pair<EvalResult, StringRef> evalParensExpr(StringRef Expr, + ParseContext PCtx) const { + assert(Expr.startswith("(") && "Not a parenthesized expression"); + EvalResult SubExprResult; + StringRef RemainingExpr; + std::tie(SubExprResult, RemainingExpr) = + evalComplexExpr(evalSimpleExpr(Expr.substr(1).ltrim(), PCtx), PCtx); + if (SubExprResult.hasError()) + return std::make_pair(SubExprResult, ""); + if (!RemainingExpr.startswith(")")) + return std::make_pair( + unexpectedToken(RemainingExpr, Expr, "expected ')'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + return std::make_pair(SubExprResult, RemainingExpr); + } - // Evaluate a bit-slice of an expression. - // A bit-slice has the form "<expr>[high:low]". The result of evaluating a - // slice is the bits between high and low (inclusive) in the original - // expression, right shifted so that the "low" bit is in position 0 in the + // Evaluate an expression in one of the following forms: + // *{<number>}<expr> + // Return a pair containing the result, plus the expression remaining to be + // parsed. + std::pair<EvalResult, StringRef> evalLoadExpr(StringRef Expr) const { + assert(Expr.startswith("*") && "Not a load expression"); + StringRef RemainingExpr = Expr.substr(1).ltrim(); + + // Parse read size. + if (!RemainingExpr.startswith("{")) + return std::make_pair(EvalResult("Expected '{' following '*'."), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + EvalResult ReadSizeExpr; + std::tie(ReadSizeExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); + if (ReadSizeExpr.hasError()) + return std::make_pair(ReadSizeExpr, RemainingExpr); + uint64_t ReadSize = ReadSizeExpr.getValue(); + if (ReadSize < 1 || ReadSize > 8) + return std::make_pair(EvalResult("Invalid size for dereference."), ""); + if (!RemainingExpr.startswith("}")) + return std::make_pair(EvalResult("Missing '}' for dereference."), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + // Evaluate the expression representing the load address. + ParseContext LoadCtx(true); + EvalResult LoadAddrExprResult; + std::tie(LoadAddrExprResult, RemainingExpr) = + evalComplexExpr(evalSimpleExpr(RemainingExpr, LoadCtx), LoadCtx); + + if (LoadAddrExprResult.hasError()) + return std::make_pair(LoadAddrExprResult, ""); + + uint64_t LoadAddr = LoadAddrExprResult.getValue(); + + return std::make_pair( + EvalResult(Checker.readMemoryAtAddr(LoadAddr, ReadSize)), + RemainingExpr); + } + + // Evaluate a "simple" expression. This is any expression that _isn't_ an + // un-parenthesized binary expression. + // + // "Simple" expressions can be optionally bit-sliced. See evalSlicedExpr. + // + // Returns a pair containing the result of the evaluation, plus the + // expression remaining to be parsed. + std::pair<EvalResult, StringRef> evalSimpleExpr(StringRef Expr, + ParseContext PCtx) const { + EvalResult SubExprResult; + StringRef RemainingExpr; + + if (Expr.empty()) + return std::make_pair(EvalResult("Unexpected end of expression"), ""); + + if (Expr[0] == '(') + std::tie(SubExprResult, RemainingExpr) = evalParensExpr(Expr, PCtx); + else if (Expr[0] == '*') + std::tie(SubExprResult, RemainingExpr) = evalLoadExpr(Expr); + else if (isalpha(Expr[0]) || Expr[0] == '_') + std::tie(SubExprResult, RemainingExpr) = evalIdentifierExpr(Expr, PCtx); + else if (isdigit(Expr[0])) + std::tie(SubExprResult, RemainingExpr) = evalNumberExpr(Expr); + else + return std::make_pair( + unexpectedToken(Expr, Expr, + "expected '(', '*', identifier, or number"), ""); + + if (SubExprResult.hasError()) + return std::make_pair(SubExprResult, RemainingExpr); + + // Evaluate bit-slice if present. + if (RemainingExpr.startswith("[")) + std::tie(SubExprResult, RemainingExpr) = + evalSliceExpr(std::make_pair(SubExprResult, RemainingExpr)); + + return std::make_pair(SubExprResult, RemainingExpr); + } + + // Evaluate a bit-slice of an expression. + // A bit-slice has the form "<expr>[high:low]". The result of evaluating a + // slice is the bits between high and low (inclusive) in the original + // expression, right shifted so that the "low" bit is in position 0 in the + // result. + // Returns a pair containing the result of the slice operation, plus the + // expression remaining to be parsed. + std::pair<EvalResult, StringRef> + evalSliceExpr(std::pair<EvalResult, StringRef> Ctx) const { + EvalResult SubExprResult; + StringRef RemainingExpr; + std::tie(SubExprResult, RemainingExpr) = Ctx; + + assert(RemainingExpr.startswith("[") && "Not a slice expr."); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + EvalResult HighBitExpr; + std::tie(HighBitExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); + + if (HighBitExpr.hasError()) + return std::make_pair(HighBitExpr, RemainingExpr); + + if (!RemainingExpr.startswith(":")) + return std::make_pair( + unexpectedToken(RemainingExpr, RemainingExpr, "expected ':'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + EvalResult LowBitExpr; + std::tie(LowBitExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); + + if (LowBitExpr.hasError()) + return std::make_pair(LowBitExpr, RemainingExpr); + + if (!RemainingExpr.startswith("]")) + return std::make_pair( + unexpectedToken(RemainingExpr, RemainingExpr, "expected ']'"), ""); + RemainingExpr = RemainingExpr.substr(1).ltrim(); + + unsigned HighBit = HighBitExpr.getValue(); + unsigned LowBit = LowBitExpr.getValue(); + uint64_t Mask = ((uint64_t)1 << (HighBit - LowBit + 1)) - 1; + uint64_t SlicedValue = (SubExprResult.getValue() >> LowBit) & Mask; + return std::make_pair(EvalResult(SlicedValue), RemainingExpr); + } + + // Evaluate a "complex" expression. + // Takes an already evaluated subexpression and checks for the presence of a + // binary operator, computing the result of the binary operation if one is + // found. Used to make arithmetic expressions left-associative. + // Returns a pair containing the ultimate result of evaluating the + // expression, plus the expression remaining to be evaluated. + std::pair<EvalResult, StringRef> + evalComplexExpr(std::pair<EvalResult, StringRef> LHSAndRemaining, + ParseContext PCtx) const { + EvalResult LHSResult; + StringRef RemainingExpr; + std::tie(LHSResult, RemainingExpr) = LHSAndRemaining; + + // If there was an error, or there's nothing left to evaluate, return the // result. - // Returns a pair containing the result of the slice operation, plus the - // expression remaining to be parsed. - std::pair<EvalResult, StringRef> evalSliceExpr( - std::pair<EvalResult, StringRef> Ctx) const{ - EvalResult SubExprResult; - StringRef RemainingExpr; - std::tie(SubExprResult, RemainingExpr) = Ctx; - - assert(RemainingExpr.startswith("[") && "Not a slice expr."); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - EvalResult HighBitExpr; - std::tie(HighBitExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); - - if (HighBitExpr.hasError()) - return std::make_pair(HighBitExpr, RemainingExpr); - - if (!RemainingExpr.startswith(":")) - return std::make_pair(unexpectedToken(RemainingExpr, RemainingExpr, - "expected ':'"), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - EvalResult LowBitExpr; - std::tie(LowBitExpr, RemainingExpr) = evalNumberExpr(RemainingExpr); - - if (LowBitExpr.hasError()) - return std::make_pair(LowBitExpr, RemainingExpr); - - if (!RemainingExpr.startswith("]")) - return std::make_pair(unexpectedToken(RemainingExpr, RemainingExpr, - "expected ']'"), - ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - - unsigned HighBit = HighBitExpr.getValue(); - unsigned LowBit = LowBitExpr.getValue(); - uint64_t Mask = ((uint64_t)1 << (HighBit - LowBit + 1)) - 1; - uint64_t SlicedValue = (SubExprResult.getValue() >> LowBit) & Mask; - return std::make_pair(EvalResult(SlicedValue), RemainingExpr); - } + if (LHSResult.hasError() || RemainingExpr == "") + return std::make_pair(LHSResult, RemainingExpr); - // Evaluate a "complex" expression. - // Takes an already evaluated subexpression and checks for the presence of a - // binary operator, computing the result of the binary operation if one is - // found. Used to make arithmetic expressions left-associative. - // Returns a pair containing the ultimate result of evaluating the - // expression, plus the expression remaining to be evaluated. - std::pair<EvalResult, StringRef> evalComplexExpr( - std::pair<EvalResult, StringRef> Ctx) const { - EvalResult LHSResult; - StringRef RemainingExpr; - std::tie(LHSResult, RemainingExpr) = Ctx; - - // If there was an error, or there's nothing left to evaluate, return the - // result. - if (LHSResult.hasError() || RemainingExpr == "") - return std::make_pair(LHSResult, RemainingExpr); - - // Otherwise check if this is a binary expressioan. - BinOpToken BinOp; - std::tie(BinOp, RemainingExpr) = parseBinOpToken(RemainingExpr); - - // If this isn't a recognized expression just return. - if (BinOp == BinOpToken::Invalid) - return std::make_pair(LHSResult, RemainingExpr); - - // This is a recognized bin-op. Evaluate the RHS, then evaluate the binop. - EvalResult RHSResult; - std::tie(RHSResult, RemainingExpr) = evalSimpleExpr(RemainingExpr); - - // If there was an error evaluating the RHS, return it. - if (RHSResult.hasError()) - return std::make_pair(RHSResult, RemainingExpr); - - // This is a binary expression - evaluate and try to continue as a - // complex expr. - EvalResult ThisResult(computeBinOpResult(BinOp, LHSResult, RHSResult)); - - return evalComplexExpr(std::make_pair(ThisResult, RemainingExpr)); - } + // Otherwise check if this is a binary expressioan. + BinOpToken BinOp; + std::tie(BinOp, RemainingExpr) = parseBinOpToken(RemainingExpr); - bool decodeInst(StringRef Symbol, MCInst &Inst, uint64_t &Size) const { - MCDisassembler *Dis = Checker.Disassembler; - StringRef SectionMem = Checker.getSubsectionStartingAt(Symbol); - StringRefMemoryObject SectionBytes(SectionMem, 0); + // If this isn't a recognized expression just return. + if (BinOp == BinOpToken::Invalid) + return std::make_pair(LHSResult, RemainingExpr); - MCDisassembler::DecodeStatus S = - Dis->getInstruction(Inst, Size, SectionBytes, 0, nulls(), nulls()); + // This is a recognized bin-op. Evaluate the RHS, then evaluate the binop. + EvalResult RHSResult; + std::tie(RHSResult, RemainingExpr) = evalSimpleExpr(RemainingExpr, PCtx); - return (S == MCDisassembler::Success); - } + // If there was an error evaluating the RHS, return it. + if (RHSResult.hasError()) + return std::make_pair(RHSResult, RemainingExpr); - }; + // This is a binary expression - evaluate and try to continue as a + // complex expr. + EvalResult ThisResult(computeBinOpResult(BinOp, LHSResult, RHSResult)); + + return evalComplexExpr(std::make_pair(ThisResult, RemainingExpr), PCtx); + } + bool decodeInst(StringRef Symbol, MCInst &Inst, uint64_t &Size) const { + MCDisassembler *Dis = Checker.Disassembler; + StringRef SectionMem = Checker.getSubsectionStartingAt(Symbol); + ArrayRef<uint8_t> SectionBytes( + reinterpret_cast<const uint8_t *>(SectionMem.data()), + SectionMem.size()); + + MCDisassembler::DecodeStatus S = + Dis->getInstruction(Inst, Size, SectionBytes, 0, nulls(), nulls()); + + return (S == MCDisassembler::Success); + } +}; } -bool RuntimeDyldChecker::check(StringRef CheckExpr) const { +RuntimeDyldCheckerImpl::RuntimeDyldCheckerImpl(RuntimeDyld &RTDyld, + MCDisassembler *Disassembler, + MCInstPrinter *InstPrinter, + raw_ostream &ErrStream) + : RTDyld(RTDyld), Disassembler(Disassembler), InstPrinter(InstPrinter), + ErrStream(ErrStream) { + RTDyld.Checker = this; +} + +bool RuntimeDyldCheckerImpl::check(StringRef CheckExpr) const { CheckExpr = CheckExpr.trim(); - DEBUG(llvm::dbgs() << "RuntimeDyldChecker: Checking '" << CheckExpr - << "'...\n"); + DEBUG(dbgs() << "RuntimeDyldChecker: Checking '" << CheckExpr << "'...\n"); RuntimeDyldCheckerExprEval P(*this, ErrStream); bool Result = P.evaluate(CheckExpr); (void)Result; - DEBUG(llvm::dbgs() << "RuntimeDyldChecker: '" << CheckExpr << "' " - << (Result ? "passed" : "FAILED") << ".\n"); + DEBUG(dbgs() << "RuntimeDyldChecker: '" << CheckExpr << "' " + << (Result ? "passed" : "FAILED") << ".\n"); return Result; } -bool RuntimeDyldChecker::checkAllRulesInBuffer(StringRef RulePrefix, - MemoryBuffer* MemBuf) const { +bool RuntimeDyldCheckerImpl::checkAllRulesInBuffer(StringRef RulePrefix, + MemoryBuffer *MemBuf) const { bool DidAllTestsPass = true; unsigned NumRules = 0; const char *LineStart = MemBuf->getBufferStart(); // Eat whitespace. - while (LineStart != MemBuf->getBufferEnd() && - std::isspace(*LineStart)) + while (LineStart != MemBuf->getBufferEnd() && std::isspace(*LineStart)) ++LineStart; while (LineStart != MemBuf->getBufferEnd() && *LineStart != '\0') { const char *LineEnd = LineStart; - while (LineEnd != MemBuf->getBufferEnd() && - *LineEnd != '\r' && *LineEnd != '\n') + while (LineEnd != MemBuf->getBufferEnd() && *LineEnd != '\r' && + *LineEnd != '\n') ++LineEnd; StringRef Line(LineStart, LineEnd - LineStart); @@ -605,37 +724,210 @@ bool RuntimeDyldChecker::checkAllRulesInBuffer(StringRef RulePrefix, // Eat whitespace. LineStart = LineEnd; - while (LineStart != MemBuf->getBufferEnd() && - std::isspace(*LineStart)) + while (LineStart != MemBuf->getBufferEnd() && std::isspace(*LineStart)) ++LineStart; } return DidAllTestsPass && (NumRules != 0); } -bool RuntimeDyldChecker::checkSymbolIsValidForLoad(StringRef Symbol) const { - return RTDyld.getSymbolAddress(Symbol) != nullptr; +bool RuntimeDyldCheckerImpl::isSymbolValid(StringRef Symbol) const { + return getRTDyld().getSymbolAddress(Symbol) != nullptr; } -uint64_t RuntimeDyldChecker::getSymbolAddress(StringRef Symbol) const { - return RTDyld.getAnySymbolRemoteAddress(Symbol); +uint64_t RuntimeDyldCheckerImpl::getSymbolLinkerAddr(StringRef Symbol) const { + return static_cast<uint64_t>( + reinterpret_cast<uintptr_t>(getRTDyld().getSymbolAddress(Symbol))); } -uint64_t RuntimeDyldChecker::readMemoryAtSymbol(StringRef Symbol, - int64_t Offset, - unsigned Size) const { - uint8_t *Src = RTDyld.getSymbolAddress(Symbol); - uint64_t Result = 0; - memcpy(&Result, Src + Offset, Size); - return Result; +uint64_t RuntimeDyldCheckerImpl::getSymbolRemoteAddr(StringRef Symbol) const { + return getRTDyld().getAnySymbolRemoteAddress(Symbol); +} + +uint64_t RuntimeDyldCheckerImpl::readMemoryAtAddr(uint64_t SrcAddr, + unsigned Size) const { + uintptr_t PtrSizedAddr = static_cast<uintptr_t>(SrcAddr); + assert(PtrSizedAddr == SrcAddr && "Linker memory pointer out-of-range."); + uint8_t *Src = reinterpret_cast<uint8_t*>(PtrSizedAddr); + return getRTDyld().readBytesUnaligned(Src, Size); } -StringRef RuntimeDyldChecker::getSubsectionStartingAt(StringRef Name) const { + +std::pair<const RuntimeDyldCheckerImpl::SectionAddressInfo*, std::string> +RuntimeDyldCheckerImpl::findSectionAddrInfo(StringRef FileName, + StringRef SectionName) const { + + auto SectionMapItr = Stubs.find(FileName); + if (SectionMapItr == Stubs.end()) { + std::string ErrorMsg = "File '"; + ErrorMsg += FileName; + ErrorMsg += "' not found. "; + if (Stubs.empty()) + ErrorMsg += "No stubs registered."; + else { + ErrorMsg += "Available files are:"; + for (const auto& StubEntry : Stubs) { + ErrorMsg += " '"; + ErrorMsg += StubEntry.first; + ErrorMsg += "'"; + } + } + ErrorMsg += "\n"; + return std::make_pair(nullptr, ErrorMsg); + } + + auto SectionInfoItr = SectionMapItr->second.find(SectionName); + if (SectionInfoItr == SectionMapItr->second.end()) + return std::make_pair(nullptr, + ("Section '" + SectionName + "' not found in file '" + + FileName + "'\n").str()); + + return std::make_pair(&SectionInfoItr->second, std::string("")); +} + +std::pair<uint64_t, std::string> RuntimeDyldCheckerImpl::getSectionAddr( + StringRef FileName, StringRef SectionName, bool IsInsideLoad) const { + + const SectionAddressInfo *SectionInfo = nullptr; + { + std::string ErrorMsg; + std::tie(SectionInfo, ErrorMsg) = + findSectionAddrInfo(FileName, SectionName); + if (ErrorMsg != "") + return std::make_pair(0, ErrorMsg); + } + + unsigned SectionID = SectionInfo->SectionID; + uint64_t Addr; + if (IsInsideLoad) + Addr = + static_cast<uint64_t>( + reinterpret_cast<uintptr_t>(getRTDyld().Sections[SectionID].Address)); + else + Addr = getRTDyld().Sections[SectionID].LoadAddress; + + return std::make_pair(Addr, std::string("")); +} + +std::pair<uint64_t, std::string> RuntimeDyldCheckerImpl::getStubAddrFor( + StringRef FileName, StringRef SectionName, StringRef SymbolName, + bool IsInsideLoad) const { + + const SectionAddressInfo *SectionInfo = nullptr; + { + std::string ErrorMsg; + std::tie(SectionInfo, ErrorMsg) = + findSectionAddrInfo(FileName, SectionName); + if (ErrorMsg != "") + return std::make_pair(0, ErrorMsg); + } + + unsigned SectionID = SectionInfo->SectionID; + const StubOffsetsMap &SymbolStubs = SectionInfo->StubOffsets; + auto StubOffsetItr = SymbolStubs.find(SymbolName); + if (StubOffsetItr == SymbolStubs.end()) + return std::make_pair(0, + ("Stub for symbol '" + SymbolName + "' not found. " + "If '" + SymbolName + "' is an internal symbol this " + "may indicate that the stub target offset is being " + "computed incorrectly.\n").str()); + + uint64_t StubOffset = StubOffsetItr->second; + + uint64_t Addr; + if (IsInsideLoad) { + uintptr_t SectionBase = + reinterpret_cast<uintptr_t>(getRTDyld().Sections[SectionID].Address); + Addr = static_cast<uint64_t>(SectionBase) + StubOffset; + } else { + uint64_t SectionBase = getRTDyld().Sections[SectionID].LoadAddress; + Addr = SectionBase + StubOffset; + } + + return std::make_pair(Addr, std::string("")); +} + +StringRef +RuntimeDyldCheckerImpl::getSubsectionStartingAt(StringRef Name) const { RuntimeDyldImpl::SymbolTableMap::const_iterator pos = - RTDyld.GlobalSymbolTable.find(Name); - if (pos == RTDyld.GlobalSymbolTable.end()) + getRTDyld().GlobalSymbolTable.find(Name); + if (pos == getRTDyld().GlobalSymbolTable.end()) return StringRef(); RuntimeDyldImpl::SymbolLoc Loc = pos->second; - uint8_t *SectionAddr = RTDyld.getSectionAddress(Loc.first); - return StringRef(reinterpret_cast<const char*>(SectionAddr) + Loc.second, - RTDyld.Sections[Loc.first].Size - Loc.second); + uint8_t *SectionAddr = getRTDyld().getSectionAddress(Loc.first); + return StringRef(reinterpret_cast<const char *>(SectionAddr) + Loc.second, + getRTDyld().Sections[Loc.first].Size - Loc.second); +} + +void RuntimeDyldCheckerImpl::registerSection( + StringRef FilePath, unsigned SectionID) { + StringRef FileName = sys::path::filename(FilePath); + const SectionEntry &Section = getRTDyld().Sections[SectionID]; + StringRef SectionName = Section.Name; + + Stubs[FileName][SectionName].SectionID = SectionID; +} + +void RuntimeDyldCheckerImpl::registerStubMap( + StringRef FilePath, unsigned SectionID, + const RuntimeDyldImpl::StubMap &RTDyldStubs) { + StringRef FileName = sys::path::filename(FilePath); + const SectionEntry &Section = getRTDyld().Sections[SectionID]; + StringRef SectionName = Section.Name; + + Stubs[FileName][SectionName].SectionID = SectionID; + + for (auto &StubMapEntry : RTDyldStubs) { + std::string SymbolName = ""; + + if (StubMapEntry.first.SymbolName) + SymbolName = StubMapEntry.first.SymbolName; + else { + // If this is a (Section, Offset) pair, do a reverse lookup in the + // global symbol table to find the name. + for (auto &GSTEntry : getRTDyld().GlobalSymbolTable) { + if (GSTEntry.second.first == StubMapEntry.first.SectionID && + GSTEntry.second.second == + static_cast<uint64_t>(StubMapEntry.first.Offset)) { + SymbolName = GSTEntry.first(); + break; + } + } + } + + if (SymbolName != "") + Stubs[FileName][SectionName].StubOffsets[SymbolName] = + StubMapEntry.second; + } +} + +RuntimeDyldChecker::RuntimeDyldChecker(RuntimeDyld &RTDyld, + MCDisassembler *Disassembler, + MCInstPrinter *InstPrinter, + raw_ostream &ErrStream) + : Impl(make_unique<RuntimeDyldCheckerImpl>(RTDyld, Disassembler, + InstPrinter, ErrStream)) {} + +RuntimeDyldChecker::~RuntimeDyldChecker() {} + +RuntimeDyld& RuntimeDyldChecker::getRTDyld() { + return Impl->RTDyld; +} + +const RuntimeDyld& RuntimeDyldChecker::getRTDyld() const { + return Impl->RTDyld; +} + +bool RuntimeDyldChecker::check(StringRef CheckExpr) const { + return Impl->check(CheckExpr); +} + +bool RuntimeDyldChecker::checkAllRulesInBuffer(StringRef RulePrefix, + MemoryBuffer *MemBuf) const { + return Impl->checkAllRulesInBuffer(RulePrefix, MemBuf); +} + +std::pair<uint64_t, std::string> +RuntimeDyldChecker::getSectionAddr(StringRef FileName, StringRef SectionName, + bool LinkerAddress) { + return Impl->getSectionAddr(FileName, SectionName, LinkerAddress); } diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h new file mode 100644 index 0000000..de20c1e --- /dev/null +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h @@ -0,0 +1,76 @@ +//===-- RuntimeDyldCheckerImpl.h -- RuntimeDyld test framework --*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDCHECKERIMPL_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDCHECKERIMPL_H + +#include "RuntimeDyldImpl.h" +#include <set> + +namespace llvm { + +class RuntimeDyldCheckerImpl { + friend class RuntimeDyldChecker; + friend class RuntimeDyldImpl; + friend class RuntimeDyldCheckerExprEval; + +public: + RuntimeDyldCheckerImpl(RuntimeDyld &RTDyld, MCDisassembler *Disassembler, + MCInstPrinter *InstPrinter, + llvm::raw_ostream &ErrStream); + + bool check(StringRef CheckExpr) const; + bool checkAllRulesInBuffer(StringRef RulePrefix, MemoryBuffer *MemBuf) const; + +private: + + // StubMap typedefs. + typedef std::map<std::string, uint64_t> StubOffsetsMap; + struct SectionAddressInfo { + uint64_t SectionID; + StubOffsetsMap StubOffsets; + }; + typedef std::map<std::string, SectionAddressInfo> SectionMap; + typedef std::map<std::string, SectionMap> StubMap; + + RuntimeDyldImpl &getRTDyld() const { return *RTDyld.Dyld; } + + bool isSymbolValid(StringRef Symbol) const; + uint64_t getSymbolLinkerAddr(StringRef Symbol) const; + uint64_t getSymbolRemoteAddr(StringRef Symbol) const; + uint64_t readMemoryAtAddr(uint64_t Addr, unsigned Size) const; + + std::pair<const SectionAddressInfo*, std::string> findSectionAddrInfo( + StringRef FileName, + StringRef SectionName) const; + + std::pair<uint64_t, std::string> getSectionAddr(StringRef FileName, + StringRef SectionName, + bool IsInsideLoad) const; + + std::pair<uint64_t, std::string> getStubAddrFor(StringRef FileName, + StringRef SectionName, + StringRef Symbol, + bool IsInsideLoad) const; + StringRef getSubsectionStartingAt(StringRef Name) const; + + void registerSection(StringRef FilePath, unsigned SectionID); + void registerStubMap(StringRef FilePath, unsigned SectionID, + const RuntimeDyldImpl::StubMap &RTDyldStubs); + + RuntimeDyld &RTDyld; + MCDisassembler *Disassembler; + MCInstPrinter *InstPrinter; + llvm::raw_ostream &ErrStream; + + StubMap Stubs; +}; +} + +#endif diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index 80e489c..d95cffe 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -23,6 +23,7 @@ #include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/ELF.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/MemoryBuffer.h" using namespace llvm; @@ -55,9 +56,9 @@ template <class ELFT> class DyldELFObject : public ELFObjectFile<ELFT> { public: DyldELFObject(std::unique_ptr<ObjectFile> UnderlyingFile, - std::unique_ptr<MemoryBuffer> Wrapper, std::error_code &ec); + MemoryBufferRef Wrapper, std::error_code &ec); - DyldELFObject(std::unique_ptr<MemoryBuffer> Wrapper, std::error_code &ec); + DyldELFObject(MemoryBufferRef Wrapper, std::error_code &ec); void updateSectionAddress(const SectionRef &Sec, uint64_t Addr); void updateSymbolAddress(const SymbolRef &Sym, uint64_t Addr); @@ -76,8 +77,10 @@ template <class ELFT> class ELFObjectImage : public ObjectImageCommon { bool Registered; public: - ELFObjectImage(ObjectBuffer *Input, std::unique_ptr<DyldELFObject<ELFT>> Obj) - : ObjectImageCommon(Input, std::move(Obj)), Registered(false) {} + ELFObjectImage(std::unique_ptr<ObjectBuffer> Input, + std::unique_ptr<DyldELFObject<ELFT>> Obj) + : ObjectImageCommon(std::move(Input), std::move(Obj)), Registered(false) { + } virtual ~ELFObjectImage() { if (Registered) @@ -109,17 +112,15 @@ public: // actual memory. Ultimately, the Binary parent class will take ownership of // this MemoryBuffer object but not the underlying memory. template <class ELFT> -DyldELFObject<ELFT>::DyldELFObject(std::unique_ptr<MemoryBuffer> Wrapper, - std::error_code &EC) - : ELFObjectFile<ELFT>(std::move(Wrapper), EC) { +DyldELFObject<ELFT>::DyldELFObject(MemoryBufferRef Wrapper, std::error_code &EC) + : ELFObjectFile<ELFT>(Wrapper, EC) { this->isDyldELFObject = true; } template <class ELFT> DyldELFObject<ELFT>::DyldELFObject(std::unique_ptr<ObjectFile> UnderlyingFile, - std::unique_ptr<MemoryBuffer> Wrapper, - std::error_code &EC) - : ELFObjectFile<ELFT>(std::move(Wrapper), EC), + MemoryBufferRef Wrapper, std::error_code &EC) + : ELFObjectFile<ELFT>(Wrapper, EC), UnderlyingFile(std::move(UnderlyingFile)) { this->isDyldELFObject = true; } @@ -185,36 +186,36 @@ RuntimeDyldELF::createObjectImageFromFile(std::unique_ptr<object::ObjectFile> Ob return nullptr; std::error_code ec; - std::unique_ptr<MemoryBuffer> Buffer( - MemoryBuffer::getMemBuffer(ObjFile->getData(), "", false)); + MemoryBufferRef Buffer = ObjFile->getMemoryBufferRef(); if (ObjFile->getBytesInAddress() == 4 && ObjFile->isLittleEndian()) { auto Obj = llvm::make_unique<DyldELFObject<ELFType<support::little, 2, false>>>( - std::move(ObjFile), std::move(Buffer), ec); + std::move(ObjFile), Buffer, ec); return new ELFObjectImage<ELFType<support::little, 2, false>>( nullptr, std::move(Obj)); } else if (ObjFile->getBytesInAddress() == 4 && !ObjFile->isLittleEndian()) { auto Obj = llvm::make_unique<DyldELFObject<ELFType<support::big, 2, false>>>( - std::move(ObjFile), std::move(Buffer), ec); + std::move(ObjFile), Buffer, ec); return new ELFObjectImage<ELFType<support::big, 2, false>>(nullptr, std::move(Obj)); } else if (ObjFile->getBytesInAddress() == 8 && !ObjFile->isLittleEndian()) { auto Obj = llvm::make_unique<DyldELFObject<ELFType<support::big, 2, true>>>( - std::move(ObjFile), std::move(Buffer), ec); + std::move(ObjFile), Buffer, ec); return new ELFObjectImage<ELFType<support::big, 2, true>>(nullptr, std::move(Obj)); } else if (ObjFile->getBytesInAddress() == 8 && ObjFile->isLittleEndian()) { auto Obj = llvm::make_unique<DyldELFObject<ELFType<support::little, 2, true>>>( - std::move(ObjFile), std::move(Buffer), ec); + std::move(ObjFile), Buffer, ec); return new ELFObjectImage<ELFType<support::little, 2, true>>( nullptr, std::move(Obj)); } else llvm_unreachable("Unexpected ELF format"); } -ObjectImage *RuntimeDyldELF::createObjectImage(ObjectBuffer *Buffer) { +std::unique_ptr<ObjectImage> +RuntimeDyldELF::createObjectImage(std::unique_ptr<ObjectBuffer> Buffer) { if (Buffer->getBufferSize() < ELF::EI_NIDENT) llvm_unreachable("Unexpected ELF object size"); std::pair<unsigned char, unsigned char> Ident = @@ -222,34 +223,36 @@ ObjectImage *RuntimeDyldELF::createObjectImage(ObjectBuffer *Buffer) { (uint8_t)Buffer->getBufferStart()[ELF::EI_DATA]); std::error_code ec; - std::unique_ptr<MemoryBuffer> Buf(Buffer->getMemBuffer()); + MemoryBufferRef Buf = Buffer->getMemBuffer(); if (Ident.first == ELF::ELFCLASS32 && Ident.second == ELF::ELFDATA2LSB) { auto Obj = llvm::make_unique<DyldELFObject<ELFType<support::little, 4, false>>>( - std::move(Buf), ec); - return new ELFObjectImage<ELFType<support::little, 4, false>>( - Buffer, std::move(Obj)); - } else if (Ident.first == ELF::ELFCLASS32 && - Ident.second == ELF::ELFDATA2MSB) { + Buf, ec); + return llvm::make_unique< + ELFObjectImage<ELFType<support::little, 4, false>>>(std::move(Buffer), + std::move(Obj)); + } + if (Ident.first == ELF::ELFCLASS32 && Ident.second == ELF::ELFDATA2MSB) { auto Obj = - llvm::make_unique<DyldELFObject<ELFType<support::big, 4, false>>>( - std::move(Buf), ec); - return new ELFObjectImage<ELFType<support::big, 4, false>>(Buffer, - std::move(Obj)); - } else if (Ident.first == ELF::ELFCLASS64 && - Ident.second == ELF::ELFDATA2MSB) { + llvm::make_unique<DyldELFObject<ELFType<support::big, 4, false>>>(Buf, + ec); + return llvm::make_unique<ELFObjectImage<ELFType<support::big, 4, false>>>( + std::move(Buffer), std::move(Obj)); + } + if (Ident.first == ELF::ELFCLASS64 && Ident.second == ELF::ELFDATA2MSB) { auto Obj = llvm::make_unique<DyldELFObject<ELFType<support::big, 8, true>>>( - std::move(Buf), ec); - return new ELFObjectImage<ELFType<support::big, 8, true>>(Buffer, std::move(Obj)); - } else if (Ident.first == ELF::ELFCLASS64 && - Ident.second == ELF::ELFDATA2LSB) { - auto Obj = - llvm::make_unique<DyldELFObject<ELFType<support::little, 8, true>>>( - std::move(Buf), ec); - return new ELFObjectImage<ELFType<support::little, 8, true>>(Buffer, std::move(Obj)); - } else - llvm_unreachable("Unexpected ELF format"); + Buf, ec); + return llvm::make_unique<ELFObjectImage<ELFType<support::big, 8, true>>>( + std::move(Buffer), std::move(Obj)); + } + assert(Ident.first == ELF::ELFCLASS64 && Ident.second == ELF::ELFDATA2LSB && + "Unexpected ELF format"); + auto Obj = + llvm::make_unique<DyldELFObject<ELFType<support::little, 8, true>>>(Buf, + ec); + return llvm::make_unique<ELFObjectImage<ELFType<support::little, 8, true>>>( + std::move(Buffer), std::move(Obj)); } RuntimeDyldELF::~RuntimeDyldELF() {} @@ -263,10 +266,9 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, llvm_unreachable("Relocation type not implemented yet!"); break; case ELF::R_X86_64_64: { - uint64_t *Target = reinterpret_cast<uint64_t *>(Section.Address + Offset); - *Target = Value + Addend; + support::ulittle64_t::ref(Section.Address + Offset) = Value + Addend; DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at " - << format("%p\n", Target)); + << format("%p\n", Section.Address + Offset)); break; } case ELF::R_X86_64_32: @@ -276,17 +278,15 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, (Type == ELF::R_X86_64_32S && ((int64_t)Value <= INT32_MAX && (int64_t)Value >= INT32_MIN))); uint32_t TruncatedAddr = (Value & 0xFFFFFFFF); - uint32_t *Target = reinterpret_cast<uint32_t *>(Section.Address + Offset); - *Target = TruncatedAddr; + support::ulittle32_t::ref(Section.Address + Offset) = TruncatedAddr; DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at " - << format("%p\n", Target)); + << format("%p\n", Section.Address + Offset)); break; } case ELF::R_X86_64_GOTPCREL: { // findGOTEntry returns the 'G + GOT' part of the relocation calculation // based on the load/target address of the GOT (not the current/local addr). uint64_t GOTAddr = findGOTEntry(Value, SymOffset); - uint32_t *Target = reinterpret_cast<uint32_t *>(Section.Address + Offset); uint64_t FinalAddress = Section.LoadAddress + Offset; // The processRelocationRef method combines the symbol offset and the addend // and in most cases that's what we want. For this relocation type, we need @@ -294,30 +294,29 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, int64_t RealOffset = GOTAddr + Addend - SymOffset - FinalAddress; assert(RealOffset <= INT32_MAX && RealOffset >= INT32_MIN); int32_t TruncOffset = (RealOffset & 0xFFFFFFFF); - *Target = TruncOffset; + support::ulittle32_t::ref(Section.Address + Offset) = TruncOffset; break; } case ELF::R_X86_64_PC32: { // Get the placeholder value from the generated object since // a previous relocation attempt may have overwritten the loaded version - uint32_t *Placeholder = - reinterpret_cast<uint32_t *>(Section.ObjAddress + Offset); - uint32_t *Target = reinterpret_cast<uint32_t *>(Section.Address + Offset); + support::ulittle32_t::ref Placeholder( + (void *)(Section.ObjAddress + Offset)); uint64_t FinalAddress = Section.LoadAddress + Offset; - int64_t RealOffset = *Placeholder + Value + Addend - FinalAddress; + int64_t RealOffset = Placeholder + Value + Addend - FinalAddress; assert(RealOffset <= INT32_MAX && RealOffset >= INT32_MIN); int32_t TruncOffset = (RealOffset & 0xFFFFFFFF); - *Target = TruncOffset; + support::ulittle32_t::ref(Section.Address + Offset) = TruncOffset; break; } case ELF::R_X86_64_PC64: { // Get the placeholder value from the generated object since // a previous relocation attempt may have overwritten the loaded version - uint64_t *Placeholder = - reinterpret_cast<uint64_t *>(Section.ObjAddress + Offset); - uint64_t *Target = reinterpret_cast<uint64_t *>(Section.Address + Offset); + support::ulittle64_t::ref Placeholder( + (void *)(Section.ObjAddress + Offset)); uint64_t FinalAddress = Section.LoadAddress + Offset; - *Target = *Placeholder + Value + Addend - FinalAddress; + support::ulittle64_t::ref(Section.Address + Offset) = + Placeholder + Value + Addend - FinalAddress; break; } } @@ -330,21 +329,20 @@ void RuntimeDyldELF::resolveX86Relocation(const SectionEntry &Section, case ELF::R_386_32: { // Get the placeholder value from the generated object since // a previous relocation attempt may have overwritten the loaded version - uint32_t *Placeholder = - reinterpret_cast<uint32_t *>(Section.ObjAddress + Offset); - uint32_t *Target = reinterpret_cast<uint32_t *>(Section.Address + Offset); - *Target = *Placeholder + Value + Addend; + support::ulittle32_t::ref Placeholder( + (void *)(Section.ObjAddress + Offset)); + support::ulittle32_t::ref(Section.Address + Offset) = + Placeholder + Value + Addend; break; } case ELF::R_386_PC32: { // Get the placeholder value from the generated object since // a previous relocation attempt may have overwritten the loaded version - uint32_t *Placeholder = - reinterpret_cast<uint32_t *>(Section.ObjAddress + Offset); - uint32_t *Target = reinterpret_cast<uint32_t *>(Section.Address + Offset); + support::ulittle32_t::ref Placeholder( + (void *)(Section.ObjAddress + Offset)); uint32_t FinalAddress = ((Section.LoadAddress + Offset) & 0xFFFFFFFF); - uint32_t RealOffset = *Placeholder + Value + Addend - FinalAddress; - *Target = RealOffset; + uint32_t RealOffset = Placeholder + Value + Addend - FinalAddress; + support::ulittle32_t::ref(Section.Address + Offset) = RealOffset; break; } default: @@ -704,8 +702,7 @@ void RuntimeDyldELF::findOPDEntrySection(ObjectImage &Obj, section_iterator tsi(Obj.end_sections()); check(TargetSymbol->getSection(tsi)); - bool IsCode = false; - tsi->isText(IsCode); + bool IsCode = tsi->isText(); Rel.SectionID = findOrEmitSection(Obj, (*tsi), IsCode, LocalSections); Rel.Addend = (intptr_t)Addend; return; @@ -911,8 +908,6 @@ void RuntimeDyldELF::resolveRelocation(const SectionEntry &Section, break; case Triple::aarch64: case Triple::aarch64_be: - case Triple::arm64: - case Triple::arm64_be: resolveAArch64Relocation(Section, Offset, Value, Type, Addend); break; case Triple::arm: // Fall through. @@ -987,9 +982,7 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( if (si == Obj.end_sections()) llvm_unreachable("Symbol section not found, bad object file format!"); DEBUG(dbgs() << "\t\tThis is section symbol\n"); - // Default to 'true' in case isText fails (though it never does). - bool isCode = true; - si->isText(isCode); + bool isCode = si->isText(); Value.SectionID = findOrEmitSection(Obj, (*si), isCode, ObjSectionToID); Value.Addend = Addend; break; @@ -1018,8 +1011,7 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( DEBUG(dbgs() << "\t\tSectionID: " << SectionID << " Offset: " << Offset << "\n"); - if ((Arch == Triple::aarch64 || Arch == Triple::aarch64_be || - Arch == Triple::arm64 || Arch == Triple::arm64_be) && + if ((Arch == Triple::aarch64 || Arch == Triple::aarch64_be) && (RelType == ELF::R_AARCH64_CALL26 || RelType == ELF::R_AARCH64_JUMP26)) { // This is an AArch64 branch relocation, need to use a stub function. DEBUG(dbgs() << "\t\tThis is an AArch64 branch relocation."); @@ -1141,6 +1133,10 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( } } else if (Arch == Triple::ppc64 || Arch == Triple::ppc64le) { if (RelType == ELF::R_PPC64_REL24) { + // Determine ABI variant in use for this object. + unsigned AbiVariant; + Obj.getObjectFile()->getPlatformFlags(AbiVariant); + AbiVariant &= ELF::EF_PPC64_ABI; // A PPC branch relocation will need a stub function if the target is // an external symbol (Symbol::ST_Unknown) or if the target address // is not within the signed 24-bits branch address. @@ -1148,10 +1144,18 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( uint8_t *Target = Section.Address + Offset; bool RangeOverflow = false; if (SymType != SymbolRef::ST_Unknown) { - // A function call may points to the .opd entry, so the final symbol - // value - // in calculated based in the relocation values in .opd section. - findOPDEntrySection(Obj, ObjSectionToID, Value); + if (AbiVariant != 2) { + // In the ELFv1 ABI, a function call may point to the .opd entry, + // so the final symbol value is calculated based on the relocation + // values in the .opd section. + findOPDEntrySection(Obj, ObjSectionToID, Value); + } else { + // In the ELFv2 ABI, a function symbol may provide a local entry + // point, which must be used for direct calls. + uint8_t SymOther; + Symbol->getOther(SymOther); + Value.Addend += ELF::decodePPC64LocalEntryOffset(SymOther); + } uint8_t *RelocTarget = Sections[Value.SectionID].Address + Value.Addend; int32_t delta = static_cast<int32_t>(Target - RelocTarget); // If it is within 24-bits branch range, just set the branch target @@ -1179,7 +1183,8 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( DEBUG(dbgs() << " Create a new stub function\n"); Stubs[Value] = Section.StubOffset; uint8_t *StubTargetAddr = - createStubFunction(Section.Address + Section.StubOffset); + createStubFunction(Section.Address + Section.StubOffset, + AbiVariant); RelocationEntry RE(SectionID, StubTargetAddr - Section.Address, ELF::R_PPC64_ADDR64, Value.Addend); @@ -1217,9 +1222,13 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( RelType, 0); Section.StubOffset += getMaxStubSize(); } - if (SymType == SymbolRef::ST_Unknown) + if (SymType == SymbolRef::ST_Unknown) { // Restore the TOC for external calls - writeInt32BE(Target + 4, 0xE8410028); // ld r2,40(r1) + if (AbiVariant == 2) + writeInt32BE(Target + 4, 0xE8410018); // ld r2,28(r1) + else + writeInt32BE(Target + 4, 0xE8410028); // ld r2,40(r1) + } } } else if (RelType == ELF::R_PPC64_TOC16 || RelType == ELF::R_PPC64_TOC16_DS || @@ -1306,7 +1315,7 @@ relocation_iterator RuntimeDyldELF::processRelocationRef( Stubs[Value] = StubOffset; createStubFunction((uint8_t *)StubAddress); RelocationEntry RE(SectionID, StubOffset + 8, ELF::R_390_64, - Value.Addend - Addend); + Value.Offset); if (Value.SymbolName) addRelocationForSymbol(RE, Value.SymbolName); else @@ -1414,8 +1423,6 @@ size_t RuntimeDyldELF::getGOTEntrySize() { case Triple::x86_64: case Triple::aarch64: case Triple::aarch64_be: - case Triple::arm64: - case Triple::arm64_be: case Triple::ppc64: case Triple::ppc64le: case Triple::systemz: diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h index 59fdfbe..4aeab81 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_RUNTIME_DYLD_ELF_H -#define LLVM_RUNTIME_DYLD_ELF_H +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDELF_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDELF_H #include "RuntimeDyldImpl.h" #include "llvm/ADT/DenseMap.h" @@ -58,8 +58,7 @@ class RuntimeDyldELF : public RuntimeDyldImpl { uint64_t Value, uint32_t Type, int64_t Addend); unsigned getMaxStubSize() override { - if (Arch == Triple::aarch64 || Arch == Triple::arm64 || - Arch == Triple::aarch64_be || Arch == Triple::arm64_be) + if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be) return 20; // movz; movk; movk; movk; br if (Arch == Triple::arm || Arch == Triple::thumb) return 8; // 32-bit instruction and 32-bit address @@ -120,7 +119,8 @@ public: ObjSectionToIDMap &SectionMap) override; virtual ~RuntimeDyldELF(); - static ObjectImage *createObjectImage(ObjectBuffer *InputBuffer); + static std::unique_ptr<ObjectImage> + createObjectImage(std::unique_ptr<ObjectBuffer> InputBuffer); static ObjectImage *createObjectImageFromFile(std::unique_ptr<object::ObjectFile> Obj); }; diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index 0336cba..69ea3b4 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -11,8 +11,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_RUNTIME_DYLD_IMPL_H -#define LLVM_RUNTIME_DYLD_IMPL_H +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDIMPL_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDIMPL_H #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" @@ -70,7 +70,7 @@ public: SectionEntry(StringRef name, uint8_t *address, size_t size, uintptr_t objAddress) : Name(name), Address(address), Size(size), - LoadAddress((uintptr_t)address), StubOffset(size), + LoadAddress(reinterpret_cast<uintptr_t>(address)), StubOffset(size), ObjAddress(objAddress) {} }; @@ -159,7 +159,7 @@ public: }; class RuntimeDyldImpl { - friend class RuntimeDyldChecker; + friend class RuntimeDyldCheckerImpl; private: uint64_t getAnySymbolRemoteAddress(StringRef Symbol) { @@ -172,6 +172,9 @@ protected: // The MemoryManager to load objects into. RTDyldMemoryManager *MemMgr; + // Attached RuntimeDyldChecker instance. Null if no instance attached. + RuntimeDyldCheckerImpl *Checker; + // A list of all sections emitted by the dynamic linker. These sections are // referenced in the code by means of their index in this list - SectionID. typedef SmallVector<SectionEntry, 64> SectionList; @@ -211,6 +214,7 @@ protected: // modules. This map is indexed by symbol name. StringMap<RelocationList> ExternalSymbolRelocations; + typedef std::map<RelocationValueRef, uintptr_t> StubMap; Triple::ArchType Arch; @@ -245,11 +249,11 @@ protected: return true; } - uint64_t getSectionLoadAddress(unsigned SectionID) { + uint64_t getSectionLoadAddress(unsigned SectionID) const { return Sections[SectionID].LoadAddress; } - uint8_t *getSectionAddress(unsigned SectionID) { + uint8_t *getSectionAddress(unsigned SectionID) const { return (uint8_t *)Sections[SectionID].Address; } @@ -282,6 +286,13 @@ protected: *(Addr + 7) = Value & 0xFF; } + /// Endian-aware read Read the least significant Size bytes from Src. + uint64_t readBytesUnaligned(uint8_t *Src, unsigned Size) const; + + /// Endian-aware write. Write the least significant Size bytes from Value to + /// Dst. + void writeBytesUnaligned(uint64_t Value, uint8_t *Dst, unsigned Size) const; + /// \brief Given the common symbols discovered in the object file, emit a /// new section for them and update the symbol mappings in the object and /// symbol table. @@ -312,7 +323,7 @@ protected: /// \brief Emits long jump instruction to Addr. /// \return Pointer to the memory area for emitting target address. - uint8_t *createStubFunction(uint8_t *Addr); + uint8_t *createStubFunction(uint8_t *Addr, unsigned AbiVariant = 0); /// \brief Resolves relocations from Relocs list with address from Value. void resolveRelocationList(const RelocationList &Relocs, uint64_t Value); @@ -349,7 +360,7 @@ protected: public: RuntimeDyldImpl(RTDyldMemoryManager *mm) - : MemMgr(mm), ProcessAllSections(false), HasError(false) { + : MemMgr(mm), Checker(nullptr), ProcessAllSections(false), HasError(false) { } virtual ~RuntimeDyldImpl(); @@ -358,9 +369,14 @@ public: this->ProcessAllSections = ProcessAllSections; } - ObjectImage *loadObject(ObjectImage *InputObject); + void setRuntimeDyldChecker(RuntimeDyldCheckerImpl *Checker) { + this->Checker = Checker; + } + + std::unique_ptr<ObjectImage> + loadObject(std::unique_ptr<ObjectImage> InputObject); - uint8_t* getSymbolAddress(StringRef Name) { + uint8_t* getSymbolAddress(StringRef Name) const { // FIXME: Just look up as a function for now. Overly simple of course. // Work in progress. SymbolTableMap::const_iterator pos = GlobalSymbolTable.find(Name); @@ -370,7 +386,7 @@ public: return getSectionAddress(Loc.first) + Loc.second; } - uint64_t getSymbolLoadAddress(StringRef Name) { + uint64_t getSymbolLoadAddress(StringRef Name) const { // FIXME: Just look up as a function for now. Overly simple of course. // Work in progress. SymbolTableMap::const_iterator pos = GlobalSymbolTable.find(Name); diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp index 4eb516c..d3d6f5d 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp @@ -14,8 +14,12 @@ #include "RuntimeDyldMachO.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" -#include "ObjectImageCommon.h" -#include "JITRegistrar.h" + +#include "Targets/RuntimeDyldMachOARM.h" +#include "Targets/RuntimeDyldMachOAArch64.h" +#include "Targets/RuntimeDyldMachOI386.h" +#include "Targets/RuntimeDyldMachOX86_64.h" + using namespace llvm; using namespace llvm::object; @@ -23,164 +27,231 @@ using namespace llvm::object; namespace llvm { -class MachOObjectImage : public ObjectImageCommon { -private: - typedef SmallVector<uint64_t, 1> SectionAddrList; - SectionAddrList OldSectionAddrList; - -protected: - bool is64; - bool Registered; - -private: - void initOldAddress() { - MachOObjectFile *objf = static_cast<MachOObjectFile *>(ObjFile.get()); - // Unfortunately we need to do this, since there's information encoded - // in the original addr of the section that we could not otherwise - // recover. The reason for this is that symbols do not actually store - // their file offset, but only their vmaddr. This means that in order - // to locate the symbol correctly in the object file, we need to know - // where the original start of the section was (including any padding, - // etc). - for (section_iterator i = objf->section_begin(), e = objf->section_end(); - i != e; ++i) { - uint64_t Addr; - i->getAddress(Addr); - OldSectionAddrList[i->getRawDataRefImpl().d.a] = Addr; +int64_t RuntimeDyldMachO::memcpyAddend(const RelocationEntry &RE) const { + unsigned NumBytes = 1 << RE.Size; + uint8_t *Src = Sections[RE.SectionID].Address + RE.Offset; + + return static_cast<int64_t>(readBytesUnaligned(Src, NumBytes)); +} + +RelocationValueRef RuntimeDyldMachO::getRelocationValueRef( + ObjectImage &ObjImg, const relocation_iterator &RI, + const RelocationEntry &RE, ObjSectionToIDMap &ObjSectionToID, + const SymbolTableMap &Symbols) { + + const MachOObjectFile &Obj = + static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); + MachO::any_relocation_info RelInfo = + Obj.getRelocation(RI->getRawDataRefImpl()); + RelocationValueRef Value; + + bool IsExternal = Obj.getPlainRelocationExternal(RelInfo); + if (IsExternal) { + symbol_iterator Symbol = RI->getSymbol(); + StringRef TargetName; + Symbol->getName(TargetName); + SymbolTableMap::const_iterator SI = Symbols.find(TargetName.data()); + if (SI != Symbols.end()) { + Value.SectionID = SI->second.first; + Value.Offset = SI->second.second + RE.Addend; + } else { + SI = GlobalSymbolTable.find(TargetName.data()); + if (SI != GlobalSymbolTable.end()) { + Value.SectionID = SI->second.first; + Value.Offset = SI->second.second + RE.Addend; + } else { + Value.SymbolName = TargetName.data(); + Value.Offset = RE.Addend; + } } + } else { + SectionRef Sec = Obj.getRelocationSection(RelInfo); + bool IsCode = Sec.isText(); + Value.SectionID = findOrEmitSection(ObjImg, Sec, IsCode, ObjSectionToID); + uint64_t Addr = Sec.getAddress(); + Value.Offset = RE.Addend - Addr; } -public: - MachOObjectImage(ObjectBuffer *Input, bool is64) - : ObjectImageCommon(Input), - OldSectionAddrList(ObjFile->section_end()->getRawDataRefImpl().d.a, 0), - is64(is64), Registered(false) { - initOldAddress(); - } + return Value; +} - MachOObjectImage(std::unique_ptr<object::ObjectFile> Input, bool is64) - : ObjectImageCommon(std::move(Input)), - OldSectionAddrList(ObjFile->section_end()->getRawDataRefImpl().d.a, 0), - is64(is64), Registered(false) { - initOldAddress(); +void RuntimeDyldMachO::makeValueAddendPCRel(RelocationValueRef &Value, + ObjectImage &ObjImg, + const relocation_iterator &RI, + unsigned OffsetToNextPC) { + const MachOObjectFile &Obj = + static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); + MachO::any_relocation_info RelInfo = + Obj.getRelocation(RI->getRawDataRefImpl()); + + bool IsPCRel = Obj.getAnyRelocationPCRel(RelInfo); + if (IsPCRel) { + uint64_t RelocAddr = 0; + RI->getAddress(RelocAddr); + Value.Offset += RelocAddr + OffsetToNextPC; } +} + +void RuntimeDyldMachO::dumpRelocationToResolve(const RelocationEntry &RE, + uint64_t Value) const { + const SectionEntry &Section = Sections[RE.SectionID]; + uint8_t *LocalAddress = Section.Address + RE.Offset; + uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + + dbgs() << "resolveRelocation Section: " << RE.SectionID + << " LocalAddress: " << format("%p", LocalAddress) + << " FinalAddress: " << format("0x%016" PRIx64, FinalAddress) + << " Value: " << format("0x%016" PRIx64, Value) << " Addend: " << RE.Addend + << " isPCRel: " << RE.IsPCRel << " MachoType: " << RE.RelType + << " Size: " << (1 << RE.Size) << "\n"; +} - virtual ~MachOObjectImage() { - if (Registered) - deregisterWithDebugger(); +section_iterator +RuntimeDyldMachO::getSectionByAddress(const MachOObjectFile &Obj, + uint64_t Addr) { + section_iterator SI = Obj.section_begin(); + section_iterator SE = Obj.section_end(); + + for (; SI != SE; ++SI) { + uint64_t SAddr = SI->getAddress(); + uint64_t SSize = SI->getSize(); + if ((Addr >= SAddr) && (Addr < SAddr + SSize)) + return SI; } - // Subclasses can override these methods to update the image with loaded - // addresses for sections and common symbols - virtual void updateSectionAddress(const SectionRef &Sec, uint64_t Addr) { - MachOObjectFile *objf = static_cast<MachOObjectFile *>(ObjFile.get()); - char *data = - const_cast<char *>(objf->getSectionPointer(Sec.getRawDataRefImpl())); + return SE; +} - uint64_t oldAddr = OldSectionAddrList[Sec.getRawDataRefImpl().d.a]; - if (is64) { - ((MachO::section_64 *)data)->addr = Addr; - } else { - ((MachO::section *)data)->addr = Addr; - } +// Populate __pointers section. +void RuntimeDyldMachO::populateIndirectSymbolPointersSection( + MachOObjectFile &Obj, + const SectionRef &PTSection, + unsigned PTSectionID) { + assert(!Obj.is64Bit() && + "Pointer table section not supported in 64-bit MachO."); - for (symbol_iterator i = objf->symbol_begin(), e = objf->symbol_end(); - i != e; ++i) { - section_iterator symSec(objf->section_end()); - (*i).getSection(symSec); - if (*symSec == Sec) { - uint64_t symAddr; - (*i).getAddress(symAddr); - updateSymbolAddress(*i, symAddr + Addr - oldAddr); - } - } - } + MachO::dysymtab_command DySymTabCmd = Obj.getDysymtabLoadCommand(); + MachO::section Sec32 = Obj.getSection(PTSection.getRawDataRefImpl()); + uint32_t PTSectionSize = Sec32.size; + unsigned FirstIndirectSymbol = Sec32.reserved1; + const unsigned PTEntrySize = 4; + unsigned NumPTEntries = PTSectionSize / PTEntrySize; + unsigned PTEntryOffset = 0; - uint64_t getOldSectionAddr(const SectionRef &Sec) const { - return OldSectionAddrList[Sec.getRawDataRefImpl().d.a]; - } + assert((PTSectionSize % PTEntrySize) == 0 && + "Pointers section does not contain a whole number of stubs?"); - virtual void updateSymbolAddress(const SymbolRef &Sym, uint64_t Addr) { - char *data = const_cast<char *>( - reinterpret_cast<const char *>(Sym.getRawDataRefImpl().p)); - if (is64) - ((MachO::nlist_64 *)data)->n_value = Addr; - else - ((MachO::nlist *)data)->n_value = Addr; - } + DEBUG(dbgs() << "Populating pointer table section " + << Sections[PTSectionID].Name + << ", Section ID " << PTSectionID << ", " + << NumPTEntries << " entries, " << PTEntrySize + << " bytes each:\n"); - virtual void registerWithDebugger() { - JITRegistrar::getGDBRegistrar().registerObject(*Buffer); - Registered = true; + for (unsigned i = 0; i < NumPTEntries; ++i) { + unsigned SymbolIndex = + Obj.getIndirectSymbolTableEntry(DySymTabCmd, FirstIndirectSymbol + i); + symbol_iterator SI = Obj.getSymbolByIndex(SymbolIndex); + StringRef IndirectSymbolName; + SI->getName(IndirectSymbolName); + DEBUG(dbgs() << " " << IndirectSymbolName << ": index " << SymbolIndex + << ", PT offset: " << PTEntryOffset << "\n"); + RelocationEntry RE(PTSectionID, PTEntryOffset, + MachO::GENERIC_RELOC_VANILLA, 0, false, 2); + addRelocationForSymbol(RE, IndirectSymbolName); + PTEntryOffset += PTEntrySize; } +} - virtual void deregisterWithDebugger() { - JITRegistrar::getGDBRegistrar().deregisterObject(*Buffer); - } -}; - -ObjectImage *RuntimeDyldMachO::createObjectImage(ObjectBuffer *Buffer) { - uint32_t magic = *((const uint32_t *)Buffer->getBufferStart()); - bool is64 = (magic == MachO::MH_MAGIC_64); - assert((magic == MachO::MH_MAGIC_64 || magic == MachO::MH_MAGIC) && - "Unrecognized Macho Magic"); - return new MachOObjectImage(Buffer, is64); +bool +RuntimeDyldMachO::isCompatibleFormat(const ObjectBuffer *InputBuffer) const { + if (InputBuffer->getBufferSize() < 4) + return false; + StringRef Magic(InputBuffer->getBufferStart(), 4); + if (Magic == "\xFE\xED\xFA\xCE") + return true; + if (Magic == "\xCE\xFA\xED\xFE") + return true; + if (Magic == "\xFE\xED\xFA\xCF") + return true; + if (Magic == "\xCF\xFA\xED\xFE") + return true; + return false; } -ObjectImage *RuntimeDyldMachO::createObjectImageFromFile( - std::unique_ptr<object::ObjectFile> ObjFile) { - if (!ObjFile) - return nullptr; +bool RuntimeDyldMachO::isCompatibleFile(const object::ObjectFile *Obj) const { + return Obj->isMachO(); +} - MemoryBuffer *Buffer = - MemoryBuffer::getMemBuffer(ObjFile->getData(), "", false); +template <typename Impl> +void RuntimeDyldMachOCRTPBase<Impl>::finalizeLoad(ObjectImage &ObjImg, + ObjSectionToIDMap &SectionMap) { + unsigned EHFrameSID = RTDYLD_INVALID_SECTION_ID; + unsigned TextSID = RTDYLD_INVALID_SECTION_ID; + unsigned ExceptTabSID = RTDYLD_INVALID_SECTION_ID; + ObjSectionToIDMap::iterator i, e; - uint32_t magic = *((const uint32_t *)Buffer->getBufferStart()); - bool is64 = (magic == MachO::MH_MAGIC_64); - assert((magic == MachO::MH_MAGIC_64 || magic == MachO::MH_MAGIC) && - "Unrecognized Macho Magic"); - return new MachOObjectImage(std::move(ObjFile), is64); + for (i = SectionMap.begin(), e = SectionMap.end(); i != e; ++i) { + const SectionRef &Section = i->first; + StringRef Name; + Section.getName(Name); + if (Name == "__eh_frame") + EHFrameSID = i->second; + else if (Name == "__text") + TextSID = i->second; + else if (Name == "__gcc_except_tab") + ExceptTabSID = i->second; + else + impl().finalizeSection(ObjImg, i->second, Section); + } + UnregisteredEHFrameSections.push_back( + EHFrameRelatedSections(EHFrameSID, TextSID, ExceptTabSID)); } -static unsigned char *processFDE(unsigned char *P, intptr_t DeltaForText, - intptr_t DeltaForEH) { +template <typename Impl> +unsigned char *RuntimeDyldMachOCRTPBase<Impl>::processFDE(unsigned char *P, + int64_t DeltaForText, + int64_t DeltaForEH) { + typedef typename Impl::TargetPtrT TargetPtrT; + DEBUG(dbgs() << "Processing FDE: Delta for text: " << DeltaForText << ", Delta for EH: " << DeltaForEH << "\n"); - uint32_t Length = *((uint32_t *)P); + uint32_t Length = readBytesUnaligned(P, 4); P += 4; unsigned char *Ret = P + Length; - uint32_t Offset = *((uint32_t *)P); + uint32_t Offset = readBytesUnaligned(P, 4); if (Offset == 0) // is a CIE return Ret; P += 4; - intptr_t FDELocation = *((intptr_t *)P); - intptr_t NewLocation = FDELocation - DeltaForText; - *((intptr_t *)P) = NewLocation; - P += sizeof(intptr_t); + TargetPtrT FDELocation = readBytesUnaligned(P, sizeof(TargetPtrT)); + TargetPtrT NewLocation = FDELocation - DeltaForText; + writeBytesUnaligned(NewLocation, P, sizeof(TargetPtrT)); + + P += sizeof(TargetPtrT); // Skip the FDE address range - P += sizeof(intptr_t); + P += sizeof(TargetPtrT); uint8_t Augmentationsize = *P; P += 1; if (Augmentationsize != 0) { - intptr_t LSDA = *((intptr_t *)P); - intptr_t NewLSDA = LSDA - DeltaForEH; - *((intptr_t *)P) = NewLSDA; + TargetPtrT LSDA = readBytesUnaligned(P, sizeof(TargetPtrT)); + TargetPtrT NewLSDA = LSDA - DeltaForEH; + writeBytesUnaligned(NewLSDA, P, sizeof(TargetPtrT)); } return Ret; } -static intptr_t computeDelta(SectionEntry *A, SectionEntry *B) { - intptr_t ObjDistance = A->ObjAddress - B->ObjAddress; - intptr_t MemDistance = A->LoadAddress - B->LoadAddress; +static int64_t computeDelta(SectionEntry *A, SectionEntry *B) { + int64_t ObjDistance = A->ObjAddress - B->ObjAddress; + int64_t MemDistance = A->LoadAddress - B->LoadAddress; return ObjDistance - MemDistance; } -void RuntimeDyldMachO::registerEHFrames() { +template <typename Impl> +void RuntimeDyldMachOCRTPBase<Impl>::registerEHFrames() { if (!MemMgr) return; @@ -195,8 +266,8 @@ void RuntimeDyldMachO::registerEHFrames() { if (SectionInfo.ExceptTabSID != RTDYLD_INVALID_SECTION_ID) ExceptTab = &Sections[SectionInfo.ExceptTabSID]; - intptr_t DeltaForText = computeDelta(Text, EHFrame); - intptr_t DeltaForEH = 0; + int64_t DeltaForText = computeDelta(Text, EHFrame); + int64_t DeltaForEH = 0; if (ExceptTab) DeltaForEH = computeDelta(ExceptTab, EHFrame); @@ -212,615 +283,17 @@ void RuntimeDyldMachO::registerEHFrames() { UnregisteredEHFrameSections.clear(); } -void RuntimeDyldMachO::finalizeLoad(ObjectImage &ObjImg, - ObjSectionToIDMap &SectionMap) { - unsigned EHFrameSID = RTDYLD_INVALID_SECTION_ID; - unsigned TextSID = RTDYLD_INVALID_SECTION_ID; - unsigned ExceptTabSID = RTDYLD_INVALID_SECTION_ID; - ObjSectionToIDMap::iterator i, e; - for (i = SectionMap.begin(), e = SectionMap.end(); i != e; ++i) { - const SectionRef &Section = i->first; - StringRef Name; - Section.getName(Name); - if (Name == "__eh_frame") - EHFrameSID = i->second; - else if (Name == "__text") - TextSID = i->second; - else if (Name == "__gcc_except_tab") - ExceptTabSID = i->second; - else if (Name == "__jump_table") - populateJumpTable(cast<MachOObjectFile>(*ObjImg.getObjectFile()), - Section, i->second); - else if (Name == "__pointers") - populatePointersSection(cast<MachOObjectFile>(*ObjImg.getObjectFile()), - Section, i->second); - } - UnregisteredEHFrameSections.push_back( - EHFrameRelatedSections(EHFrameSID, TextSID, ExceptTabSID)); -} - -// The target location for the relocation is described by RE.SectionID and -// RE.Offset. RE.SectionID can be used to find the SectionEntry. Each -// SectionEntry has three members describing its location. -// SectionEntry::Address is the address at which the section has been loaded -// into memory in the current (host) process. SectionEntry::LoadAddress is the -// address that the section will have in the target process. -// SectionEntry::ObjAddress is the address of the bits for this section in the -// original emitted object image (also in the current address space). -// -// Relocations will be applied as if the section were loaded at -// SectionEntry::LoadAddress, but they will be applied at an address based -// on SectionEntry::Address. SectionEntry::ObjAddress will be used to refer to -// Target memory contents if they are required for value calculations. -// -// The Value parameter here is the load address of the symbol for the -// relocation to be applied. For relocations which refer to symbols in the -// current object Value will be the LoadAddress of the section in which -// the symbol resides (RE.Addend provides additional information about the -// symbol location). For external symbols, Value will be the address of the -// symbol in the target address space. -void RuntimeDyldMachO::resolveRelocation(const RelocationEntry &RE, - uint64_t Value) { - DEBUG ( - const SectionEntry &Section = Sections[RE.SectionID]; - uint8_t* LocalAddress = Section.Address + RE.Offset; - uint64_t FinalAddress = Section.LoadAddress + RE.Offset; - - dbgs() << "resolveRelocation Section: " << RE.SectionID - << " LocalAddress: " << format("%p", LocalAddress) - << " FinalAddress: " << format("%p", FinalAddress) - << " Value: " << format("%p", Value) - << " Addend: " << RE.Addend - << " isPCRel: " << RE.IsPCRel - << " MachoType: " << RE.RelType - << " Size: " << (1 << RE.Size) << "\n"; - ); - - // This just dispatches to the proper target specific routine. +std::unique_ptr<RuntimeDyldMachO> +llvm::RuntimeDyldMachO::create(Triple::ArchType Arch, RTDyldMemoryManager *MM) { switch (Arch) { default: - llvm_unreachable("Unsupported CPU type!"); - case Triple::x86_64: - resolveX86_64Relocation(RE, Value); - break; - case Triple::x86: - resolveI386Relocation(RE, Value); - break; - case Triple::arm: // Fall through. - case Triple::thumb: - resolveARMRelocation(RE, Value); - break; - case Triple::aarch64: - case Triple::arm64: - resolveAArch64Relocation(RE, Value); - break; - } -} - -bool RuntimeDyldMachO::resolveI386Relocation(const RelocationEntry &RE, - uint64_t Value) { - const SectionEntry &Section = Sections[RE.SectionID]; - uint8_t* LocalAddress = Section.Address + RE.Offset; - - if (RE.IsPCRel) { - uint64_t FinalAddress = Section.LoadAddress + RE.Offset; - Value -= FinalAddress + 4; // see MachOX86_64::resolveRelocation. - } - - switch (RE.RelType) { - default: - llvm_unreachable("Invalid relocation type!"); - case MachO::GENERIC_RELOC_VANILLA: - return applyRelocationValue(LocalAddress, Value + RE.Addend, - 1 << RE.Size); - case MachO::GENERIC_RELOC_SECTDIFF: - case MachO::GENERIC_RELOC_LOCAL_SECTDIFF: { - uint64_t SectionABase = Sections[RE.Sections.SectionA].LoadAddress; - uint64_t SectionBBase = Sections[RE.Sections.SectionB].LoadAddress; - assert((Value == SectionABase || Value == SectionBBase) && - "Unexpected SECTDIFF relocation value."); - Value = SectionABase - SectionBBase + RE.Addend; - return applyRelocationValue(LocalAddress, Value, 1 << RE.Size); - } - case MachO::GENERIC_RELOC_PB_LA_PTR: - return Error("Relocation type not implemented yet!"); - } -} - -bool RuntimeDyldMachO::resolveX86_64Relocation(const RelocationEntry &RE, - uint64_t Value) { - const SectionEntry &Section = Sections[RE.SectionID]; - uint8_t* LocalAddress = Section.Address + RE.Offset; - - // If the relocation is PC-relative, the value to be encoded is the - // pointer difference. - if (RE.IsPCRel) { - // FIXME: It seems this value needs to be adjusted by 4 for an effective PC - // address. Is that expected? Only for branches, perhaps? - uint64_t FinalAddress = Section.LoadAddress + RE.Offset; - Value -= FinalAddress + 4; // see MachOX86_64::resolveRelocation. - } - - switch (RE.RelType) { - default: - llvm_unreachable("Invalid relocation type!"); - case MachO::X86_64_RELOC_SIGNED_1: - case MachO::X86_64_RELOC_SIGNED_2: - case MachO::X86_64_RELOC_SIGNED_4: - case MachO::X86_64_RELOC_SIGNED: - case MachO::X86_64_RELOC_UNSIGNED: - case MachO::X86_64_RELOC_BRANCH: - return applyRelocationValue(LocalAddress, Value + RE.Addend, 1 << RE.Size); - case MachO::X86_64_RELOC_GOT_LOAD: - case MachO::X86_64_RELOC_GOT: - case MachO::X86_64_RELOC_SUBTRACTOR: - case MachO::X86_64_RELOC_TLV: - return Error("Relocation type not implemented yet!"); - } -} - -bool RuntimeDyldMachO::resolveARMRelocation(const RelocationEntry &RE, - uint64_t Value) { - const SectionEntry &Section = Sections[RE.SectionID]; - uint8_t* LocalAddress = Section.Address + RE.Offset; - - // If the relocation is PC-relative, the value to be encoded is the - // pointer difference. - if (RE.IsPCRel) { - uint64_t FinalAddress = Section.LoadAddress + RE.Offset; - Value -= FinalAddress; - // ARM PCRel relocations have an effective-PC offset of two instructions - // (four bytes in Thumb mode, 8 bytes in ARM mode). - // FIXME: For now, assume ARM mode. - Value -= 8; - } - - switch (RE.RelType) { - default: - llvm_unreachable("Invalid relocation type!"); - case MachO::ARM_RELOC_VANILLA: - return applyRelocationValue(LocalAddress, Value, 1 << RE.Size); - case MachO::ARM_RELOC_BR24: { - // Mask the value into the target address. We know instructions are - // 32-bit aligned, so we can do it all at once. - uint32_t *p = (uint32_t *)LocalAddress; - // The low two bits of the value are not encoded. - Value >>= 2; - // Mask the value to 24 bits. - uint64_t FinalValue = Value & 0xffffff; - // Check for overflow. - if (Value != FinalValue) - return Error("ARM BR24 relocation out of range."); - // FIXME: If the destination is a Thumb function (and the instruction - // is a non-predicated BL instruction), we need to change it to a BLX - // instruction instead. - - // Insert the value into the instruction. - *p = (*p & ~0xffffff) | FinalValue; + llvm_unreachable("Unsupported target for RuntimeDyldMachO."); break; + case Triple::arm: return make_unique<RuntimeDyldMachOARM>(MM); + case Triple::aarch64: return make_unique<RuntimeDyldMachOAArch64>(MM); + case Triple::x86: return make_unique<RuntimeDyldMachOI386>(MM); + case Triple::x86_64: return make_unique<RuntimeDyldMachOX86_64>(MM); } - case MachO::ARM_THUMB_RELOC_BR22: - case MachO::ARM_THUMB_32BIT_BRANCH: - case MachO::ARM_RELOC_HALF: - case MachO::ARM_RELOC_HALF_SECTDIFF: - case MachO::ARM_RELOC_PAIR: - case MachO::ARM_RELOC_SECTDIFF: - case MachO::ARM_RELOC_LOCAL_SECTDIFF: - case MachO::ARM_RELOC_PB_LA_PTR: - return Error("Relocation type not implemented yet!"); - } - return false; -} - -bool RuntimeDyldMachO::resolveAArch64Relocation(const RelocationEntry &RE, - uint64_t Value) { - const SectionEntry &Section = Sections[RE.SectionID]; - uint8_t* LocalAddress = Section.Address + RE.Offset; - - // If the relocation is PC-relative, the value to be encoded is the - // pointer difference. - if (RE.IsPCRel) { - uint64_t FinalAddress = Section.LoadAddress + RE.Offset; - Value -= FinalAddress; - } - - switch (RE.RelType) { - default: - llvm_unreachable("Invalid relocation type!"); - case MachO::ARM64_RELOC_UNSIGNED: - return applyRelocationValue(LocalAddress, Value, 1 << RE.Size); - case MachO::ARM64_RELOC_BRANCH26: { - // Mask the value into the target address. We know instructions are - // 32-bit aligned, so we can do it all at once. - uint32_t *p = (uint32_t *)LocalAddress; - // The low two bits of the value are not encoded. - Value >>= 2; - // Mask the value to 26 bits. - uint64_t FinalValue = Value & 0x3ffffff; - // Check for overflow. - if (FinalValue != Value) - return Error("ARM64 BRANCH26 relocation out of range."); - // Insert the value into the instruction. - *p = (*p & ~0x3ffffff) | FinalValue; - break; - } - case MachO::ARM64_RELOC_SUBTRACTOR: - case MachO::ARM64_RELOC_PAGE21: - case MachO::ARM64_RELOC_PAGEOFF12: - case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: - case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: - case MachO::ARM64_RELOC_POINTER_TO_GOT: - case MachO::ARM64_RELOC_TLVP_LOAD_PAGE21: - case MachO::ARM64_RELOC_TLVP_LOAD_PAGEOFF12: - case MachO::ARM64_RELOC_ADDEND: - return Error("Relocation type not implemented yet!"); - } - return false; -} - -void RuntimeDyldMachO::populateJumpTable(MachOObjectFile &Obj, - const SectionRef &JTSection, - unsigned JTSectionID) { - assert(!Obj.is64Bit() && - "__jump_table section not supported in 64-bit MachO."); - - MachO::dysymtab_command DySymTabCmd = Obj.getDysymtabLoadCommand(); - MachO::section Sec32 = Obj.getSection(JTSection.getRawDataRefImpl()); - uint32_t JTSectionSize = Sec32.size; - unsigned FirstIndirectSymbol = Sec32.reserved1; - unsigned JTEntrySize = Sec32.reserved2; - unsigned NumJTEntries = JTSectionSize / JTEntrySize; - uint8_t* JTSectionAddr = getSectionAddress(JTSectionID); - unsigned JTEntryOffset = 0; - - assert((JTSectionSize % JTEntrySize) == 0 && - "Jump-table section does not contain a whole number of stubs?"); - - for (unsigned i = 0; i < NumJTEntries; ++i) { - unsigned SymbolIndex = - Obj.getIndirectSymbolTableEntry(DySymTabCmd, FirstIndirectSymbol + i); - symbol_iterator SI = Obj.getSymbolByIndex(SymbolIndex); - StringRef IndirectSymbolName; - SI->getName(IndirectSymbolName); - uint8_t* JTEntryAddr = JTSectionAddr + JTEntryOffset; - createStubFunction(JTEntryAddr); - RelocationEntry RE(JTSectionID, JTEntryOffset + 1, - MachO::GENERIC_RELOC_VANILLA, 0, true, 2); - addRelocationForSymbol(RE, IndirectSymbolName); - JTEntryOffset += JTEntrySize; - } -} - -void RuntimeDyldMachO::populatePointersSection(MachOObjectFile &Obj, - const SectionRef &PTSection, - unsigned PTSectionID) { - assert(!Obj.is64Bit() && - "__pointers section not supported in 64-bit MachO."); - - MachO::dysymtab_command DySymTabCmd = Obj.getDysymtabLoadCommand(); - MachO::section Sec32 = Obj.getSection(PTSection.getRawDataRefImpl()); - uint32_t PTSectionSize = Sec32.size; - unsigned FirstIndirectSymbol = Sec32.reserved1; - const unsigned PTEntrySize = 4; - unsigned NumPTEntries = PTSectionSize / PTEntrySize; - unsigned PTEntryOffset = 0; - - assert((PTSectionSize % PTEntrySize) == 0 && - "Pointers section does not contain a whole number of stubs?"); - - DEBUG(dbgs() << "Populating __pointers, Section ID " << PTSectionID - << ", " << NumPTEntries << " entries, " - << PTEntrySize << " bytes each:\n"); - - for (unsigned i = 0; i < NumPTEntries; ++i) { - unsigned SymbolIndex = - Obj.getIndirectSymbolTableEntry(DySymTabCmd, FirstIndirectSymbol + i); - symbol_iterator SI = Obj.getSymbolByIndex(SymbolIndex); - StringRef IndirectSymbolName; - SI->getName(IndirectSymbolName); - DEBUG(dbgs() << " " << IndirectSymbolName << ": index " << SymbolIndex - << ", PT offset: " << PTEntryOffset << "\n"); - RelocationEntry RE(PTSectionID, PTEntryOffset, - MachO::GENERIC_RELOC_VANILLA, 0, false, 2); - addRelocationForSymbol(RE, IndirectSymbolName); - PTEntryOffset += PTEntrySize; - } -} - - -section_iterator getSectionByAddress(const MachOObjectFile &Obj, - uint64_t Addr) { - section_iterator SI = Obj.section_begin(); - section_iterator SE = Obj.section_end(); - - for (; SI != SE; ++SI) { - uint64_t SAddr, SSize; - SI->getAddress(SAddr); - SI->getSize(SSize); - if ((Addr >= SAddr) && (Addr < SAddr + SSize)) - return SI; - } - - return SE; -} - -relocation_iterator RuntimeDyldMachO::processSECTDIFFRelocation( - unsigned SectionID, - relocation_iterator RelI, - ObjectImage &Obj, - ObjSectionToIDMap &ObjSectionToID) { - const MachOObjectFile *MachO = - static_cast<const MachOObjectFile*>(Obj.getObjectFile()); - MachO::any_relocation_info RE = - MachO->getRelocation(RelI->getRawDataRefImpl()); - - SectionEntry &Section = Sections[SectionID]; - uint32_t RelocType = MachO->getAnyRelocationType(RE); - bool IsPCRel = MachO->getAnyRelocationPCRel(RE); - unsigned Size = MachO->getAnyRelocationLength(RE); - uint64_t Offset; - RelI->getOffset(Offset); - uint8_t *LocalAddress = Section.Address + Offset; - unsigned NumBytes = 1 << Size; - int64_t Addend = 0; - memcpy(&Addend, LocalAddress, NumBytes); - - ++RelI; - MachO::any_relocation_info RE2 = - MachO->getRelocation(RelI->getRawDataRefImpl()); - - uint32_t AddrA = MachO->getScatteredRelocationValue(RE); - section_iterator SAI = getSectionByAddress(*MachO, AddrA); - assert(SAI != MachO->section_end() && "Can't find section for address A"); - uint64_t SectionABase; - SAI->getAddress(SectionABase); - uint64_t SectionAOffset = AddrA - SectionABase; - SectionRef SectionA = *SAI; - bool IsCode; - SectionA.isText(IsCode); - uint32_t SectionAID = findOrEmitSection(Obj, SectionA, IsCode, - ObjSectionToID); - - uint32_t AddrB = MachO->getScatteredRelocationValue(RE2); - section_iterator SBI = getSectionByAddress(*MachO, AddrB); - assert(SBI != MachO->section_end() && "Can't find section for address B"); - uint64_t SectionBBase; - SBI->getAddress(SectionBBase); - uint64_t SectionBOffset = AddrB - SectionBBase; - SectionRef SectionB = *SBI; - uint32_t SectionBID = findOrEmitSection(Obj, SectionB, IsCode, - ObjSectionToID); - - if (Addend != AddrA - AddrB) - Error("Unexpected SECTDIFF relocation addend."); - - DEBUG(dbgs() << "Found SECTDIFF: AddrA: " << AddrA << ", AddrB: " << AddrB - << ", Addend: " << Addend << ", SectionA ID: " - << SectionAID << ", SectionAOffset: " << SectionAOffset - << ", SectionB ID: " << SectionBID << ", SectionBOffset: " - << SectionBOffset << "\n"); - RelocationEntry R(SectionID, Offset, RelocType, 0, - SectionAID, SectionAOffset, SectionBID, SectionBOffset, - IsPCRel, Size); - - addRelocationForSection(R, SectionAID); - addRelocationForSection(R, SectionBID); - - return ++RelI; -} - -relocation_iterator RuntimeDyldMachO::processI386ScatteredVANILLA( - unsigned SectionID, - relocation_iterator RelI, - ObjectImage &Obj, - ObjSectionToIDMap &ObjSectionToID) { - const MachOObjectFile *MachO = - static_cast<const MachOObjectFile*>(Obj.getObjectFile()); - MachO::any_relocation_info RE = - MachO->getRelocation(RelI->getRawDataRefImpl()); - - SectionEntry &Section = Sections[SectionID]; - uint32_t RelocType = MachO->getAnyRelocationType(RE); - bool IsPCRel = MachO->getAnyRelocationPCRel(RE); - unsigned Size = MachO->getAnyRelocationLength(RE); - uint64_t Offset; - RelI->getOffset(Offset); - uint8_t *LocalAddress = Section.Address + Offset; - unsigned NumBytes = 1 << Size; - int64_t Addend = 0; - memcpy(&Addend, LocalAddress, NumBytes); - - unsigned SymbolBaseAddr = MachO->getScatteredRelocationValue(RE); - section_iterator TargetSI = getSectionByAddress(*MachO, SymbolBaseAddr); - assert(TargetSI != MachO->section_end() && "Can't find section for symbol"); - uint64_t SectionBaseAddr; - TargetSI->getAddress(SectionBaseAddr); - SectionRef TargetSection = *TargetSI; - bool IsCode; - TargetSection.isText(IsCode); - uint32_t TargetSectionID = findOrEmitSection(Obj, TargetSection, IsCode, - ObjSectionToID); - - Addend -= SectionBaseAddr; - RelocationEntry R(SectionID, Offset, RelocType, Addend, - IsPCRel, Size); - - addRelocationForSection(R, TargetSectionID); - - return ++RelI; -} - -relocation_iterator RuntimeDyldMachO::processRelocationRef( - unsigned SectionID, relocation_iterator RelI, ObjectImage &Obj, - ObjSectionToIDMap &ObjSectionToID, const SymbolTableMap &Symbols, - StubMap &Stubs) { - const ObjectFile *OF = Obj.getObjectFile(); - const MachOObjectImage &MachOObj = *static_cast<MachOObjectImage *>(&Obj); - const MachOObjectFile *MachO = static_cast<const MachOObjectFile *>(OF); - MachO::any_relocation_info RE = - MachO->getRelocation(RelI->getRawDataRefImpl()); - - uint32_t RelType = MachO->getAnyRelocationType(RE); - - // FIXME: Properly handle scattered relocations. - // Special case the couple of scattered relocations that we know how - // to handle: SECTDIFF relocations, and scattered VANILLA relocations - // on I386. - // For all other scattered relocations, just bail out and hope for the - // best, since the offsets computed by scattered relocations have often - // been optimisticaly filled in by the compiler. This will fail - // horribly where the relocations *do* need to be applied, but that was - // already the case. - if (MachO->isRelocationScattered(RE)) { - if (RelType == MachO::GENERIC_RELOC_SECTDIFF || - RelType == MachO::GENERIC_RELOC_LOCAL_SECTDIFF) - return processSECTDIFFRelocation(SectionID, RelI, Obj, ObjSectionToID); - else if (Arch == Triple::x86 && RelType == MachO::GENERIC_RELOC_VANILLA) - return processI386ScatteredVANILLA(SectionID, RelI, Obj, ObjSectionToID); - else - return ++RelI; - } - - RelocationValueRef Value; - SectionEntry &Section = Sections[SectionID]; - - bool IsExtern = MachO->getPlainRelocationExternal(RE); - bool IsPCRel = MachO->getAnyRelocationPCRel(RE); - unsigned Size = MachO->getAnyRelocationLength(RE); - uint64_t Offset; - RelI->getOffset(Offset); - uint8_t *LocalAddress = Section.Address + Offset; - unsigned NumBytes = 1 << Size; - uint64_t Addend = 0; - memcpy(&Addend, LocalAddress, NumBytes); - - if (IsExtern) { - // Obtain the symbol name which is referenced in the relocation - symbol_iterator Symbol = RelI->getSymbol(); - StringRef TargetName; - Symbol->getName(TargetName); - // First search for the symbol in the local symbol table - SymbolTableMap::const_iterator lsi = Symbols.find(TargetName.data()); - if (lsi != Symbols.end()) { - Value.SectionID = lsi->second.first; - Value.Addend = lsi->second.second + Addend; - } else { - // Search for the symbol in the global symbol table - SymbolTableMap::const_iterator gsi = - GlobalSymbolTable.find(TargetName.data()); - if (gsi != GlobalSymbolTable.end()) { - Value.SectionID = gsi->second.first; - Value.Addend = gsi->second.second + Addend; - } else { - Value.SymbolName = TargetName.data(); - Value.Addend = Addend; - } - } - - // Addends for external, PC-rel relocations on i386 point back to the zero - // offset. Calculate the final offset from the relocation target instead. - // This allows us to use the same logic for both external and internal - // relocations in resolveI386RelocationRef. - if (Arch == Triple::x86 && IsPCRel) { - uint64_t RelocAddr = 0; - RelI->getAddress(RelocAddr); - Value.Addend += RelocAddr + 4; - } - - } else { - SectionRef Sec = MachO->getRelocationSection(RE); - bool IsCode = false; - Sec.isText(IsCode); - Value.SectionID = findOrEmitSection(Obj, Sec, IsCode, ObjSectionToID); - uint64_t Addr = MachOObj.getOldSectionAddr(Sec); - DEBUG(dbgs() << "\nAddr: " << Addr << "\nAddend: " << Addend); - Value.Addend = Addend - Addr; - if (IsPCRel) - Value.Addend += Offset + NumBytes; - } - - if (Arch == Triple::x86_64 && (RelType == MachO::X86_64_RELOC_GOT || - RelType == MachO::X86_64_RELOC_GOT_LOAD)) { - assert(IsPCRel); - assert(Size == 2); - - // FIXME: Teach the generic code above not to prematurely conflate - // relocation addends and symbol offsets. - Value.Addend -= Addend; - StubMap::const_iterator i = Stubs.find(Value); - uint8_t *Addr; - if (i != Stubs.end()) { - Addr = Section.Address + i->second; - } else { - Stubs[Value] = Section.StubOffset; - uint8_t *GOTEntry = Section.Address + Section.StubOffset; - RelocationEntry GOTRE(SectionID, Section.StubOffset, - MachO::X86_64_RELOC_UNSIGNED, Value.Addend, false, - 3); - if (Value.SymbolName) - addRelocationForSymbol(GOTRE, Value.SymbolName); - else - addRelocationForSection(GOTRE, Value.SectionID); - Section.StubOffset += 8; - Addr = GOTEntry; - } - RelocationEntry TargetRE(SectionID, Offset, - MachO::X86_64_RELOC_UNSIGNED, Addend, true, - 2); - resolveRelocation(TargetRE, (uint64_t)Addr); - } else if (Arch == Triple::arm && (RelType & 0xf) == MachO::ARM_RELOC_BR24) { - // This is an ARM branch relocation, need to use a stub function. - - // Look up for existing stub. - StubMap::const_iterator i = Stubs.find(Value); - uint8_t *Addr; - if (i != Stubs.end()) { - Addr = Section.Address + i->second; - } else { - // Create a new stub function. - Stubs[Value] = Section.StubOffset; - uint8_t *StubTargetAddr = - createStubFunction(Section.Address + Section.StubOffset); - RelocationEntry StubRE(SectionID, StubTargetAddr - Section.Address, - MachO::GENERIC_RELOC_VANILLA, Value.Addend); - if (Value.SymbolName) - addRelocationForSymbol(StubRE, Value.SymbolName); - else - addRelocationForSection(StubRE, Value.SectionID); - Addr = Section.Address + Section.StubOffset; - Section.StubOffset += getMaxStubSize(); - } - RelocationEntry TargetRE(Value.SectionID, Offset, RelType, 0, IsPCRel, - Size); - resolveRelocation(TargetRE, (uint64_t)Addr); - } else { - RelocationEntry RE(SectionID, Offset, RelType, Value.Addend, IsPCRel, Size); - if (Value.SymbolName) - addRelocationForSymbol(RE, Value.SymbolName); - else - addRelocationForSection(RE, Value.SectionID); - } - return ++RelI; -} - -bool -RuntimeDyldMachO::isCompatibleFormat(const ObjectBuffer *InputBuffer) const { - if (InputBuffer->getBufferSize() < 4) - return false; - StringRef Magic(InputBuffer->getBufferStart(), 4); - if (Magic == "\xFE\xED\xFA\xCE") - return true; - if (Magic == "\xCE\xFA\xED\xFE") - return true; - if (Magic == "\xFE\xED\xFA\xCF") - return true; - if (Magic == "\xCF\xFA\xED\xFE") - return true; - return false; -} - -bool RuntimeDyldMachO::isCompatibleFile(const object::ObjectFile *Obj) const { - return Obj->isMachO(); } } // end namespace llvm diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h index 35f0720..7583474 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h @@ -11,74 +11,33 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_RUNTIME_DYLD_MACHO_H -#define LLVM_RUNTIME_DYLD_MACHO_H +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDMACHO_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_RUNTIMEDYLDMACHO_H #include "ObjectImageCommon.h" #include "RuntimeDyldImpl.h" -#include "llvm/ADT/IndexedMap.h" #include "llvm/Object/MachO.h" #include "llvm/Support/Format.h" +#define DEBUG_TYPE "dyld" + using namespace llvm; using namespace llvm::object; namespace llvm { class RuntimeDyldMachO : public RuntimeDyldImpl { -private: - - /// Write the least significant 'Size' bytes in 'Value' out at the address - /// pointed to by Addr. - bool applyRelocationValue(uint8_t *Addr, uint64_t Value, unsigned Size) { - for (unsigned i = 0; i < Size; ++i) { - *Addr++ = (uint8_t)Value; - Value >>= 8; - } - - return false; - } - - bool resolveI386Relocation(const RelocationEntry &RE, uint64_t Value); - bool resolveX86_64Relocation(const RelocationEntry &RE, uint64_t Value); - bool resolveARMRelocation(const RelocationEntry &RE, uint64_t Value); - bool resolveAArch64Relocation(const RelocationEntry &RE, uint64_t Value); - - // Populate stubs in __jump_table section. - void populateJumpTable(MachOObjectFile &Obj, const SectionRef &JTSection, - unsigned JTSectionID); - - // Populate __pointers section. - void populatePointersSection(MachOObjectFile &Obj, const SectionRef &PTSection, - unsigned PTSectionID); - - unsigned getMaxStubSize() override { - if (Arch == Triple::arm || Arch == Triple::thumb) - return 8; // 32-bit instruction and 32-bit address - else if (Arch == Triple::x86_64) - return 8; // GOT entry - else - return 0; - } - - unsigned getStubAlignment() override { return 1; } - - relocation_iterator processSECTDIFFRelocation( - unsigned SectionID, - relocation_iterator RelI, - ObjectImage &ObjImg, - ObjSectionToIDMap &ObjSectionToID); - - relocation_iterator processI386ScatteredVANILLA( - unsigned SectionID, - relocation_iterator RelI, - ObjectImage &ObjImg, - ObjSectionToIDMap &ObjSectionToID); +protected: + struct SectionOffsetPair { + unsigned SectionID; + uint64_t Offset; + }; struct EHFrameRelatedSections { EHFrameRelatedSections() : EHFrameSID(RTDYLD_INVALID_SECTION_ID), TextSID(RTDYLD_INVALID_SECTION_ID), ExceptTabSID(RTDYLD_INVALID_SECTION_ID) {} + EHFrameRelatedSections(SID EH, SID T, SID Ex) : EHFrameSID(EH), TextSID(T), ExceptTabSID(Ex) {} SID EHFrameSID; @@ -91,25 +50,116 @@ private: // EH frame sections with the memory manager. SmallVector<EHFrameRelatedSections, 2> UnregisteredEHFrameSections; -public: RuntimeDyldMachO(RTDyldMemoryManager *mm) : RuntimeDyldImpl(mm) {} - void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override; - relocation_iterator - processRelocationRef(unsigned SectionID, relocation_iterator RelI, - ObjectImage &Obj, ObjSectionToIDMap &ObjSectionToID, - const SymbolTableMap &Symbols, StubMap &Stubs) override; + /// This convenience method uses memcpy to extract a contiguous addend (the + /// addend size and offset are taken from the corresponding fields of the RE). + int64_t memcpyAddend(const RelocationEntry &RE) const; + + /// Given a relocation_iterator for a non-scattered relocation, construct a + /// RelocationEntry and fill in the common fields. The 'Addend' field is *not* + /// filled in, since immediate encodings are highly target/opcode specific. + /// For targets/opcodes with simple, contiguous immediates (e.g. X86) the + /// memcpyAddend method can be used to read the immediate. + RelocationEntry getRelocationEntry(unsigned SectionID, ObjectImage &ObjImg, + const relocation_iterator &RI) const { + const MachOObjectFile &Obj = + static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); + MachO::any_relocation_info RelInfo = + Obj.getRelocation(RI->getRawDataRefImpl()); + + bool IsPCRel = Obj.getAnyRelocationPCRel(RelInfo); + unsigned Size = Obj.getAnyRelocationLength(RelInfo); + uint64_t Offset; + RI->getOffset(Offset); + MachO::RelocationInfoType RelType = + static_cast<MachO::RelocationInfoType>(Obj.getAnyRelocationType(RelInfo)); + + return RelocationEntry(SectionID, Offset, RelType, 0, IsPCRel, Size); + } + + /// Construct a RelocationValueRef representing the relocation target. + /// For Symbols in known sections, this will return a RelocationValueRef + /// representing a (SectionID, Offset) pair. + /// For Symbols whose section is not known, this will return a + /// (SymbolName, Offset) pair, where the Offset is taken from the instruction + /// immediate (held in RE.Addend). + /// In both cases the Addend field is *NOT* fixed up to be PC-relative. That + /// should be done by the caller where appropriate by calling makePCRel on + /// the RelocationValueRef. + RelocationValueRef getRelocationValueRef(ObjectImage &ObjImg, + const relocation_iterator &RI, + const RelocationEntry &RE, + ObjSectionToIDMap &ObjSectionToID, + const SymbolTableMap &Symbols); + + /// Make the RelocationValueRef addend PC-relative. + void makeValueAddendPCRel(RelocationValueRef &Value, ObjectImage &ObjImg, + const relocation_iterator &RI, + unsigned OffsetToNextPC); + + /// Dump information about the relocation entry (RE) and resolved value. + void dumpRelocationToResolve(const RelocationEntry &RE, uint64_t Value) const; + + // Return a section iterator for the section containing the given address. + static section_iterator getSectionByAddress(const MachOObjectFile &Obj, + uint64_t Addr); + + + // Populate __pointers section. + void populateIndirectSymbolPointersSection(MachOObjectFile &Obj, + const SectionRef &PTSection, + unsigned PTSectionID); + +public: + /// Create an ObjectImage from the given ObjectBuffer. + static std::unique_ptr<ObjectImage> + createObjectImage(std::unique_ptr<ObjectBuffer> InputBuffer) { + return llvm::make_unique<ObjectImageCommon>(std::move(InputBuffer)); + } + + /// Create an ObjectImage from the given ObjectFile. + static ObjectImage * + createObjectImageFromFile(std::unique_ptr<object::ObjectFile> InputObject) { + return new ObjectImageCommon(std::move(InputObject)); + } + + /// Create a RuntimeDyldMachO instance for the given target architecture. + static std::unique_ptr<RuntimeDyldMachO> create(Triple::ArchType Arch, + RTDyldMemoryManager *mm); + + SectionEntry &getSection(unsigned SectionID) { return Sections[SectionID]; } + bool isCompatibleFormat(const ObjectBuffer *Buffer) const override; bool isCompatibleFile(const object::ObjectFile *Obj) const override; - void registerEHFrames() override; +}; + +/// RuntimeDyldMachOTarget - Templated base class for generic MachO linker +/// algorithms and data structures. +/// +/// Concrete, target specific sub-classes can be accessed via the impl() +/// methods. (i.e. the RuntimeDyldMachO hierarchy uses the Curiously +/// Recurring Template Idiom). Concrete subclasses for each target +/// can be found in ./Targets. +template <typename Impl> +class RuntimeDyldMachOCRTPBase : public RuntimeDyldMachO { +private: + Impl &impl() { return static_cast<Impl &>(*this); } + const Impl &impl() const { return static_cast<const Impl &>(*this); } + + unsigned char *processFDE(unsigned char *P, int64_t DeltaForText, + int64_t DeltaForEH); + +public: + RuntimeDyldMachOCRTPBase(RTDyldMemoryManager *mm) : RuntimeDyldMachO(mm) {} + void finalizeLoad(ObjectImage &ObjImg, ObjSectionToIDMap &SectionMap) override; - - static ObjectImage *createObjectImage(ObjectBuffer *Buffer); - static ObjectImage * - createObjectImageFromFile(std::unique_ptr<object::ObjectFile> InputObject); + void registerEHFrames() override; }; } // end namespace llvm +#undef DEBUG_TYPE + #endif diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h new file mode 100644 index 0000000..f5cf9ac --- /dev/null +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h @@ -0,0 +1,405 @@ +//===-- RuntimeDyldMachOAArch64.h -- MachO/AArch64 specific code. -*- C++ -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOAARCH64_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOAARCH64_H + +#include "../RuntimeDyldMachO.h" +#include "llvm/Support/Endian.h" + +#define DEBUG_TYPE "dyld" + +namespace llvm { + +class RuntimeDyldMachOAArch64 + : public RuntimeDyldMachOCRTPBase<RuntimeDyldMachOAArch64> { +public: + + typedef uint64_t TargetPtrT; + + RuntimeDyldMachOAArch64(RTDyldMemoryManager *MM) + : RuntimeDyldMachOCRTPBase(MM) {} + + unsigned getMaxStubSize() override { return 8; } + + unsigned getStubAlignment() override { return 8; } + + /// Extract the addend encoded in the instruction / memory location. + int64_t decodeAddend(const RelocationEntry &RE) const { + const SectionEntry &Section = Sections[RE.SectionID]; + uint8_t *LocalAddress = Section.Address + RE.Offset; + unsigned NumBytes = 1 << RE.Size; + int64_t Addend = 0; + // Verify that the relocation has the correct size and alignment. + switch (RE.RelType) { + default: + llvm_unreachable("Unsupported relocation type!"); + case MachO::ARM64_RELOC_UNSIGNED: + assert((NumBytes == 4 || NumBytes == 8) && "Invalid relocation size."); + break; + case MachO::ARM64_RELOC_BRANCH26: + case MachO::ARM64_RELOC_PAGE21: + case MachO::ARM64_RELOC_PAGEOFF12: + case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: + case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: + assert(NumBytes == 4 && "Invalid relocation size."); + assert((((uintptr_t)LocalAddress & 0x3) == 0) && + "Instruction address is not aligned to 4 bytes."); + break; + } + + switch (RE.RelType) { + default: + llvm_unreachable("Unsupported relocation type!"); + case MachO::ARM64_RELOC_UNSIGNED: + // This could be an unaligned memory location. + if (NumBytes == 4) + Addend = *reinterpret_cast<support::ulittle32_t *>(LocalAddress); + else + Addend = *reinterpret_cast<support::ulittle64_t *>(LocalAddress); + break; + case MachO::ARM64_RELOC_BRANCH26: { + // Verify that the relocation points to the expected branch instruction. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + assert((*p & 0xFC000000) == 0x14000000 && "Expected branch instruction."); + + // Get the 26 bit addend encoded in the branch instruction and sign-extend + // to 64 bit. The lower 2 bits are always zeros and are therefore implicit + // (<< 2). + Addend = (*p & 0x03FFFFFF) << 2; + Addend = SignExtend64(Addend, 28); + break; + } + case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: + case MachO::ARM64_RELOC_PAGE21: { + // Verify that the relocation points to the expected adrp instruction. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + assert((*p & 0x9F000000) == 0x90000000 && "Expected adrp instruction."); + + // Get the 21 bit addend encoded in the adrp instruction and sign-extend + // to 64 bit. The lower 12 bits (4096 byte page) are always zeros and are + // therefore implicit (<< 12). + Addend = ((*p & 0x60000000) >> 29) | ((*p & 0x01FFFFE0) >> 3) << 12; + Addend = SignExtend64(Addend, 33); + break; + } + case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: { + // Verify that the relocation points to one of the expected load / store + // instructions. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + (void)p; + assert((*p & 0x3B000000) == 0x39000000 && + "Only expected load / store instructions."); + } // fall-through + case MachO::ARM64_RELOC_PAGEOFF12: { + // Verify that the relocation points to one of the expected load / store + // or add / sub instructions. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + assert((((*p & 0x3B000000) == 0x39000000) || + ((*p & 0x11C00000) == 0x11000000) ) && + "Expected load / store or add/sub instruction."); + + // Get the 12 bit addend encoded in the instruction. + Addend = (*p & 0x003FFC00) >> 10; + + // Check which instruction we are decoding to obtain the implicit shift + // factor of the instruction. + int ImplicitShift = 0; + if ((*p & 0x3B000000) == 0x39000000) { // << load / store + // For load / store instructions the size is encoded in bits 31:30. + ImplicitShift = ((*p >> 30) & 0x3); + if (ImplicitShift == 0) { + // Check if this a vector op to get the correct shift value. + if ((*p & 0x04800000) == 0x04800000) + ImplicitShift = 4; + } + } + // Compensate for implicit shift. + Addend <<= ImplicitShift; + break; + } + } + return Addend; + } + + /// Extract the addend encoded in the instruction. + void encodeAddend(uint8_t *LocalAddress, unsigned NumBytes, + MachO::RelocationInfoType RelType, int64_t Addend) const { + // Verify that the relocation has the correct alignment. + switch (RelType) { + default: + llvm_unreachable("Unsupported relocation type!"); + case MachO::ARM64_RELOC_UNSIGNED: + assert((NumBytes == 4 || NumBytes == 8) && "Invalid relocation size."); + break; + case MachO::ARM64_RELOC_BRANCH26: + case MachO::ARM64_RELOC_PAGE21: + case MachO::ARM64_RELOC_PAGEOFF12: + case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: + case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: + assert(NumBytes == 4 && "Invalid relocation size."); + assert((((uintptr_t)LocalAddress & 0x3) == 0) && + "Instruction address is not aligned to 4 bytes."); + break; + } + + switch (RelType) { + default: + llvm_unreachable("Unsupported relocation type!"); + case MachO::ARM64_RELOC_UNSIGNED: + // This could be an unaligned memory location. + if (NumBytes == 4) + *reinterpret_cast<support::ulittle32_t *>(LocalAddress) = Addend; + else + *reinterpret_cast<support::ulittle64_t *>(LocalAddress) = Addend; + break; + case MachO::ARM64_RELOC_BRANCH26: { + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + // Verify that the relocation points to the expected branch instruction. + assert((*p & 0xFC000000) == 0x14000000 && "Expected branch instruction."); + + // Verify addend value. + assert((Addend & 0x3) == 0 && "Branch target is not aligned"); + assert(isInt<28>(Addend) && "Branch target is out of range."); + + // Encode the addend as 26 bit immediate in the branch instruction. + *p = (*p & 0xFC000000) | ((uint32_t)(Addend >> 2) & 0x03FFFFFF); + break; + } + case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: + case MachO::ARM64_RELOC_PAGE21: { + // Verify that the relocation points to the expected adrp instruction. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + assert((*p & 0x9F000000) == 0x90000000 && "Expected adrp instruction."); + + // Check that the addend fits into 21 bits (+ 12 lower bits). + assert((Addend & 0xFFF) == 0 && "ADRP target is not page aligned."); + assert(isInt<33>(Addend) && "Invalid page reloc value."); + + // Encode the addend into the instruction. + uint32_t ImmLoValue = (uint32_t)(Addend << 17) & 0x60000000; + uint32_t ImmHiValue = (uint32_t)(Addend >> 9) & 0x00FFFFE0; + *p = (*p & 0x9F00001F) | ImmHiValue | ImmLoValue; + break; + } + case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: { + // Verify that the relocation points to one of the expected load / store + // instructions. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + assert((*p & 0x3B000000) == 0x39000000 && + "Only expected load / store instructions."); + (void)p; + } // fall-through + case MachO::ARM64_RELOC_PAGEOFF12: { + // Verify that the relocation points to one of the expected load / store + // or add / sub instructions. + auto *p = reinterpret_cast<support::aligned_ulittle32_t *>(LocalAddress); + assert((((*p & 0x3B000000) == 0x39000000) || + ((*p & 0x11C00000) == 0x11000000) ) && + "Expected load / store or add/sub instruction."); + + // Check which instruction we are decoding to obtain the implicit shift + // factor of the instruction and verify alignment. + int ImplicitShift = 0; + if ((*p & 0x3B000000) == 0x39000000) { // << load / store + // For load / store instructions the size is encoded in bits 31:30. + ImplicitShift = ((*p >> 30) & 0x3); + switch (ImplicitShift) { + case 0: + // Check if this a vector op to get the correct shift value. + if ((*p & 0x04800000) == 0x04800000) { + ImplicitShift = 4; + assert(((Addend & 0xF) == 0) && + "128-bit LDR/STR not 16-byte aligned."); + } + break; + case 1: + assert(((Addend & 0x1) == 0) && "16-bit LDR/STR not 2-byte aligned."); + break; + case 2: + assert(((Addend & 0x3) == 0) && "32-bit LDR/STR not 4-byte aligned."); + break; + case 3: + assert(((Addend & 0x7) == 0) && "64-bit LDR/STR not 8-byte aligned."); + break; + } + } + // Compensate for implicit shift. + Addend >>= ImplicitShift; + assert(isUInt<12>(Addend) && "Addend cannot be encoded."); + + // Encode the addend into the instruction. + *p = (*p & 0xFFC003FF) | ((uint32_t)(Addend << 10) & 0x003FFC00); + break; + } + } + } + + relocation_iterator + processRelocationRef(unsigned SectionID, relocation_iterator RelI, + ObjectImage &ObjImg, ObjSectionToIDMap &ObjSectionToID, + const SymbolTableMap &Symbols, StubMap &Stubs) override { + const MachOObjectFile &Obj = + static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); + MachO::any_relocation_info RelInfo = + Obj.getRelocation(RelI->getRawDataRefImpl()); + + assert(!Obj.isRelocationScattered(RelInfo) && ""); + + // ARM64 has an ARM64_RELOC_ADDEND relocation type that carries an explicit + // addend for the following relocation. If found: (1) store the associated + // addend, (2) consume the next relocation, and (3) use the stored addend to + // override the addend. + int64_t ExplicitAddend = 0; + if (Obj.getAnyRelocationType(RelInfo) == MachO::ARM64_RELOC_ADDEND) { + assert(!Obj.getPlainRelocationExternal(RelInfo)); + assert(!Obj.getAnyRelocationPCRel(RelInfo)); + assert(Obj.getAnyRelocationLength(RelInfo) == 2); + int64_t RawAddend = Obj.getPlainRelocationSymbolNum(RelInfo); + // Sign-extend the 24-bit to 64-bit. + ExplicitAddend = SignExtend64(RawAddend, 24); + ++RelI; + RelInfo = Obj.getRelocation(RelI->getRawDataRefImpl()); + } + + RelocationEntry RE(getRelocationEntry(SectionID, ObjImg, RelI)); + RE.Addend = decodeAddend(RE); + RelocationValueRef Value( + getRelocationValueRef(ObjImg, RelI, RE, ObjSectionToID, Symbols)); + + assert((ExplicitAddend == 0 || RE.Addend == 0) && "Relocation has "\ + "ARM64_RELOC_ADDEND and embedded addend in the instruction."); + if (ExplicitAddend) { + RE.Addend = ExplicitAddend; + Value.Offset = ExplicitAddend; + } + + bool IsExtern = Obj.getPlainRelocationExternal(RelInfo); + if (!IsExtern && RE.IsPCRel) + makeValueAddendPCRel(Value, ObjImg, RelI, 1 << RE.Size); + + RE.Addend = Value.Offset; + + if (RE.RelType == MachO::ARM64_RELOC_GOT_LOAD_PAGE21 || + RE.RelType == MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12) + processGOTRelocation(RE, Value, Stubs); + else { + if (Value.SymbolName) + addRelocationForSymbol(RE, Value.SymbolName); + else + addRelocationForSection(RE, Value.SectionID); + } + + return ++RelI; + } + + void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { + DEBUG(dumpRelocationToResolve(RE, Value)); + + const SectionEntry &Section = Sections[RE.SectionID]; + uint8_t *LocalAddress = Section.Address + RE.Offset; + MachO::RelocationInfoType RelType = + static_cast<MachO::RelocationInfoType>(RE.RelType); + + switch (RelType) { + default: + llvm_unreachable("Invalid relocation type!"); + case MachO::ARM64_RELOC_UNSIGNED: { + assert(!RE.IsPCRel && "PCRel and ARM64_RELOC_UNSIGNED not supported"); + // Mask in the target value a byte at a time (we don't have an alignment + // guarantee for the target address, so this is safest). + if (RE.Size < 2) + llvm_unreachable("Invalid size for ARM64_RELOC_UNSIGNED"); + + encodeAddend(LocalAddress, 1 << RE.Size, RelType, Value + RE.Addend); + break; + } + case MachO::ARM64_RELOC_BRANCH26: { + assert(RE.IsPCRel && "not PCRel and ARM64_RELOC_BRANCH26 not supported"); + // Check if branch is in range. + uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + int64_t PCRelVal = Value - FinalAddress + RE.Addend; + encodeAddend(LocalAddress, /*Size=*/4, RelType, PCRelVal); + break; + } + case MachO::ARM64_RELOC_GOT_LOAD_PAGE21: + case MachO::ARM64_RELOC_PAGE21: { + assert(RE.IsPCRel && "not PCRel and ARM64_RELOC_PAGE21 not supported"); + // Adjust for PC-relative relocation and offset. + uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + int64_t PCRelVal = + ((Value + RE.Addend) & (-4096)) - (FinalAddress & (-4096)); + encodeAddend(LocalAddress, /*Size=*/4, RelType, PCRelVal); + break; + } + case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12: + case MachO::ARM64_RELOC_PAGEOFF12: { + assert(!RE.IsPCRel && "PCRel and ARM64_RELOC_PAGEOFF21 not supported"); + // Add the offset from the symbol. + Value += RE.Addend; + // Mask out the page address and only use the lower 12 bits. + Value &= 0xFFF; + encodeAddend(LocalAddress, /*Size=*/4, RelType, Value); + break; + } + case MachO::ARM64_RELOC_SUBTRACTOR: + case MachO::ARM64_RELOC_POINTER_TO_GOT: + case MachO::ARM64_RELOC_TLVP_LOAD_PAGE21: + case MachO::ARM64_RELOC_TLVP_LOAD_PAGEOFF12: + llvm_unreachable("Relocation type not yet implemented!"); + case MachO::ARM64_RELOC_ADDEND: + llvm_unreachable("ARM64_RELOC_ADDEND should have been handeled by " + "processRelocationRef!"); + } + } + + void finalizeSection(ObjectImage &ObjImg, unsigned SectionID, + const SectionRef &Section) {} + +private: + void processGOTRelocation(const RelocationEntry &RE, + RelocationValueRef &Value, StubMap &Stubs) { + assert(RE.Size == 2); + SectionEntry &Section = Sections[RE.SectionID]; + StubMap::const_iterator i = Stubs.find(Value); + int64_t Offset; + if (i != Stubs.end()) + Offset = static_cast<int64_t>(i->second); + else { + // FIXME: There must be a better way to do this then to check and fix the + // alignment every time!!! + uintptr_t BaseAddress = uintptr_t(Section.Address); + uintptr_t StubAlignment = getStubAlignment(); + uintptr_t StubAddress = + (BaseAddress + Section.StubOffset + StubAlignment - 1) & + -StubAlignment; + unsigned StubOffset = StubAddress - BaseAddress; + Stubs[Value] = StubOffset; + assert(((StubAddress % getStubAlignment()) == 0) && + "GOT entry not aligned"); + RelocationEntry GOTRE(RE.SectionID, StubOffset, + MachO::ARM64_RELOC_UNSIGNED, Value.Offset, + /*IsPCRel=*/false, /*Size=*/3); + if (Value.SymbolName) + addRelocationForSymbol(GOTRE, Value.SymbolName); + else + addRelocationForSection(GOTRE, Value.SectionID); + Section.StubOffset = StubOffset + getMaxStubSize(); + Offset = static_cast<int64_t>(StubOffset); + } + RelocationEntry TargetRE(RE.SectionID, RE.Offset, RE.RelType, Offset, + RE.IsPCRel, RE.Size); + addRelocationForSection(TargetRE, RE.SectionID); + } +}; +} + +#undef DEBUG_TYPE + +#endif diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h new file mode 100644 index 0000000..9766751 --- /dev/null +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h @@ -0,0 +1,277 @@ +//===----- RuntimeDyldMachOARM.h ---- MachO/ARM specific code. ----*- C++ -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOARM_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOARM_H + +#include "../RuntimeDyldMachO.h" + +#define DEBUG_TYPE "dyld" + +namespace llvm { + +class RuntimeDyldMachOARM + : public RuntimeDyldMachOCRTPBase<RuntimeDyldMachOARM> { +private: + typedef RuntimeDyldMachOCRTPBase<RuntimeDyldMachOARM> ParentT; + +public: + + typedef uint32_t TargetPtrT; + + RuntimeDyldMachOARM(RTDyldMemoryManager *MM) : RuntimeDyldMachOCRTPBase(MM) {} + + unsigned getMaxStubSize() override { return 8; } + + unsigned getStubAlignment() override { return 4; } + + int64_t decodeAddend(const RelocationEntry &RE) const { + const SectionEntry &Section = Sections[RE.SectionID]; + uint8_t *LocalAddress = Section.Address + RE.Offset; + + switch (RE.RelType) { + default: + return memcpyAddend(RE); + case MachO::ARM_RELOC_BR24: { + uint32_t Temp = readBytesUnaligned(LocalAddress, 4); + Temp &= 0x00ffffff; // Mask out the opcode. + // Now we've got the shifted immediate, shift by 2, sign extend and ret. + return SignExtend32<26>(Temp << 2); + } + } + } + + relocation_iterator + processRelocationRef(unsigned SectionID, relocation_iterator RelI, + ObjectImage &ObjImg, ObjSectionToIDMap &ObjSectionToID, + const SymbolTableMap &Symbols, StubMap &Stubs) override { + const MachOObjectFile &Obj = + static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); + MachO::any_relocation_info RelInfo = + Obj.getRelocation(RelI->getRawDataRefImpl()); + uint32_t RelType = Obj.getAnyRelocationType(RelInfo); + + if (Obj.isRelocationScattered(RelInfo)) { + if (RelType == MachO::ARM_RELOC_HALF_SECTDIFF) + return processHALFSECTDIFFRelocation(SectionID, RelI, ObjImg, + ObjSectionToID); + else + return ++++RelI; + } + + RelocationEntry RE(getRelocationEntry(SectionID, ObjImg, RelI)); + RE.Addend = decodeAddend(RE); + RelocationValueRef Value( + getRelocationValueRef(ObjImg, RelI, RE, ObjSectionToID, Symbols)); + + if (RE.IsPCRel) + makeValueAddendPCRel(Value, ObjImg, RelI, 8); + + if ((RE.RelType & 0xf) == MachO::ARM_RELOC_BR24) + processBranchRelocation(RE, Value, Stubs); + else { + RE.Addend = Value.Offset; + if (Value.SymbolName) + addRelocationForSymbol(RE, Value.SymbolName); + else + addRelocationForSection(RE, Value.SectionID); + } + + return ++RelI; + } + + void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { + DEBUG(dumpRelocationToResolve(RE, Value)); + const SectionEntry &Section = Sections[RE.SectionID]; + uint8_t *LocalAddress = Section.Address + RE.Offset; + + // If the relocation is PC-relative, the value to be encoded is the + // pointer difference. + if (RE.IsPCRel) { + uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + Value -= FinalAddress; + // ARM PCRel relocations have an effective-PC offset of two instructions + // (four bytes in Thumb mode, 8 bytes in ARM mode). + // FIXME: For now, assume ARM mode. + Value -= 8; + } + + switch (RE.RelType) { + default: + llvm_unreachable("Invalid relocation type!"); + case MachO::ARM_RELOC_VANILLA: + writeBytesUnaligned(Value + RE.Addend, LocalAddress, 1 << RE.Size); + break; + case MachO::ARM_RELOC_BR24: { + // Mask the value into the target address. We know instructions are + // 32-bit aligned, so we can do it all at once. + Value += RE.Addend; + // The low two bits of the value are not encoded. + Value >>= 2; + // Mask the value to 24 bits. + uint64_t FinalValue = Value & 0xffffff; + // FIXME: If the destination is a Thumb function (and the instruction + // is a non-predicated BL instruction), we need to change it to a BLX + // instruction instead. + + // Insert the value into the instruction. + uint32_t Temp = readBytesUnaligned(LocalAddress, 4); + writeBytesUnaligned((Temp & ~0xffffff) | FinalValue, LocalAddress, 4); + + break; + } + case MachO::ARM_RELOC_HALF_SECTDIFF: { + uint64_t SectionABase = Sections[RE.Sections.SectionA].LoadAddress; + uint64_t SectionBBase = Sections[RE.Sections.SectionB].LoadAddress; + assert((Value == SectionABase || Value == SectionBBase) && + "Unexpected HALFSECTDIFF relocation value."); + Value = SectionABase - SectionBBase + RE.Addend; + if (RE.Size & 0x1) // :upper16: + Value = (Value >> 16); + Value &= 0xffff; + + uint32_t Insn = readBytesUnaligned(LocalAddress, 4); + Insn = (Insn & 0xfff0f000) | ((Value & 0xf000) << 4) | (Value & 0x0fff); + writeBytesUnaligned(Insn, LocalAddress, 4); + break; + } + + case MachO::ARM_THUMB_RELOC_BR22: + case MachO::ARM_THUMB_32BIT_BRANCH: + case MachO::ARM_RELOC_HALF: + case MachO::ARM_RELOC_PAIR: + case MachO::ARM_RELOC_SECTDIFF: + case MachO::ARM_RELOC_LOCAL_SECTDIFF: + case MachO::ARM_RELOC_PB_LA_PTR: + Error("Relocation type not implemented yet!"); + return; + } + } + + void finalizeSection(ObjectImage &ObjImg, unsigned SectionID, + const SectionRef &Section) { + StringRef Name; + Section.getName(Name); + + if (Name == "__nl_symbol_ptr") + populateIndirectSymbolPointersSection( + cast<MachOObjectFile>(*ObjImg.getObjectFile()), + Section, SectionID); + } + +private: + + void processBranchRelocation(const RelocationEntry &RE, + const RelocationValueRef &Value, + StubMap &Stubs) { + // This is an ARM branch relocation, need to use a stub function. + // Look up for existing stub. + SectionEntry &Section = Sections[RE.SectionID]; + RuntimeDyldMachO::StubMap::const_iterator i = Stubs.find(Value); + uint8_t *Addr; + if (i != Stubs.end()) { + Addr = Section.Address + i->second; + } else { + // Create a new stub function. + Stubs[Value] = Section.StubOffset; + uint8_t *StubTargetAddr = + createStubFunction(Section.Address + Section.StubOffset); + RelocationEntry StubRE(RE.SectionID, StubTargetAddr - Section.Address, + MachO::GENERIC_RELOC_VANILLA, Value.Offset, false, + 2); + if (Value.SymbolName) + addRelocationForSymbol(StubRE, Value.SymbolName); + else + addRelocationForSection(StubRE, Value.SectionID); + Addr = Section.Address + Section.StubOffset; + Section.StubOffset += getMaxStubSize(); + } + RelocationEntry TargetRE(RE.SectionID, RE.Offset, RE.RelType, 0, + RE.IsPCRel, RE.Size); + resolveRelocation(TargetRE, (uint64_t)Addr); + } + + relocation_iterator + processHALFSECTDIFFRelocation(unsigned SectionID, relocation_iterator RelI, + ObjectImage &Obj, + ObjSectionToIDMap &ObjSectionToID) { + const MachOObjectFile *MachO = + static_cast<const MachOObjectFile *>(Obj.getObjectFile()); + MachO::any_relocation_info RE = + MachO->getRelocation(RelI->getRawDataRefImpl()); + + + // For a half-diff relocation the length bits actually record whether this + // is a movw/movt, and whether this is arm or thumb. + // Bit 0 indicates movw (b0 == 0) or movt (b0 == 1). + // Bit 1 indicates arm (b1 == 0) or thumb (b1 == 1). + unsigned HalfDiffKindBits = MachO->getAnyRelocationLength(RE); + if (HalfDiffKindBits & 0x2) + llvm_unreachable("Thumb not yet supported."); + + SectionEntry &Section = Sections[SectionID]; + uint32_t RelocType = MachO->getAnyRelocationType(RE); + bool IsPCRel = MachO->getAnyRelocationPCRel(RE); + uint64_t Offset; + RelI->getOffset(Offset); + uint8_t *LocalAddress = Section.Address + Offset; + int64_t Immediate = readBytesUnaligned(LocalAddress, 4); // Copy the whole instruction out. + Immediate = ((Immediate >> 4) & 0xf000) | (Immediate & 0xfff); + + ++RelI; + MachO::any_relocation_info RE2 = + MachO->getRelocation(RelI->getRawDataRefImpl()); + uint32_t AddrA = MachO->getScatteredRelocationValue(RE); + section_iterator SAI = getSectionByAddress(*MachO, AddrA); + assert(SAI != MachO->section_end() && "Can't find section for address A"); + uint64_t SectionABase = SAI->getAddress(); + uint64_t SectionAOffset = AddrA - SectionABase; + SectionRef SectionA = *SAI; + bool IsCode = SectionA.isText(); + uint32_t SectionAID = + findOrEmitSection(Obj, SectionA, IsCode, ObjSectionToID); + + uint32_t AddrB = MachO->getScatteredRelocationValue(RE2); + section_iterator SBI = getSectionByAddress(*MachO, AddrB); + assert(SBI != MachO->section_end() && "Can't find section for address B"); + uint64_t SectionBBase = SBI->getAddress(); + uint64_t SectionBOffset = AddrB - SectionBBase; + SectionRef SectionB = *SBI; + uint32_t SectionBID = + findOrEmitSection(Obj, SectionB, IsCode, ObjSectionToID); + + uint32_t OtherHalf = MachO->getAnyRelocationAddress(RE2) & 0xffff; + unsigned Shift = (HalfDiffKindBits & 0x1) ? 16 : 0; + uint32_t FullImmVal = (Immediate << Shift) | (OtherHalf << (16 - Shift)); + int64_t Addend = FullImmVal - (AddrA - AddrB); + + // addend = Encoded - Expected + // = Encoded - (AddrA - AddrB) + + DEBUG(dbgs() << "Found SECTDIFF: AddrA: " << AddrA << ", AddrB: " << AddrB + << ", Addend: " << Addend << ", SectionA ID: " << SectionAID + << ", SectionAOffset: " << SectionAOffset + << ", SectionB ID: " << SectionBID + << ", SectionBOffset: " << SectionBOffset << "\n"); + RelocationEntry R(SectionID, Offset, RelocType, Addend, SectionAID, + SectionAOffset, SectionBID, SectionBOffset, IsPCRel, + HalfDiffKindBits); + + addRelocationForSection(R, SectionAID); + addRelocationForSection(R, SectionBID); + + return ++RelI; + } + +}; +} + +#undef DEBUG_TYPE + +#endif diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h new file mode 100644 index 0000000..258b847 --- /dev/null +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h @@ -0,0 +1,261 @@ +//===---- RuntimeDyldMachOI386.h ---- MachO/I386 specific code. ---*- C++ -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOI386_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOI386_H + +#include "../RuntimeDyldMachO.h" + +#define DEBUG_TYPE "dyld" + +namespace llvm { + +class RuntimeDyldMachOI386 + : public RuntimeDyldMachOCRTPBase<RuntimeDyldMachOI386> { +public: + + typedef uint32_t TargetPtrT; + + RuntimeDyldMachOI386(RTDyldMemoryManager *MM) + : RuntimeDyldMachOCRTPBase(MM) {} + + unsigned getMaxStubSize() override { return 0; } + + unsigned getStubAlignment() override { return 1; } + + relocation_iterator + processRelocationRef(unsigned SectionID, relocation_iterator RelI, + ObjectImage &ObjImg, ObjSectionToIDMap &ObjSectionToID, + const SymbolTableMap &Symbols, StubMap &Stubs) override { + const MachOObjectFile &Obj = + static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); + MachO::any_relocation_info RelInfo = + Obj.getRelocation(RelI->getRawDataRefImpl()); + uint32_t RelType = Obj.getAnyRelocationType(RelInfo); + + if (Obj.isRelocationScattered(RelInfo)) { + if (RelType == MachO::GENERIC_RELOC_SECTDIFF || + RelType == MachO::GENERIC_RELOC_LOCAL_SECTDIFF) + return processSECTDIFFRelocation(SectionID, RelI, ObjImg, + ObjSectionToID); + else if (RelType == MachO::GENERIC_RELOC_VANILLA) + return processI386ScatteredVANILLA(SectionID, RelI, ObjImg, + ObjSectionToID); + llvm_unreachable("Unhandled scattered relocation."); + } + + RelocationEntry RE(getRelocationEntry(SectionID, ObjImg, RelI)); + RE.Addend = memcpyAddend(RE); + RelocationValueRef Value( + getRelocationValueRef(ObjImg, RelI, RE, ObjSectionToID, Symbols)); + + // Addends for external, PC-rel relocations on i386 point back to the zero + // offset. Calculate the final offset from the relocation target instead. + // This allows us to use the same logic for both external and internal + // relocations in resolveI386RelocationRef. + // bool IsExtern = Obj.getPlainRelocationExternal(RelInfo); + // if (IsExtern && RE.IsPCRel) { + // uint64_t RelocAddr = 0; + // RelI->getAddress(RelocAddr); + // Value.Addend += RelocAddr + 4; + // } + if (RE.IsPCRel) + makeValueAddendPCRel(Value, ObjImg, RelI, 1 << RE.Size); + + RE.Addend = Value.Offset; + + if (Value.SymbolName) + addRelocationForSymbol(RE, Value.SymbolName); + else + addRelocationForSection(RE, Value.SectionID); + + return ++RelI; + } + + void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { + DEBUG(dumpRelocationToResolve(RE, Value)); + + const SectionEntry &Section = Sections[RE.SectionID]; + uint8_t *LocalAddress = Section.Address + RE.Offset; + + if (RE.IsPCRel) { + uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + Value -= FinalAddress + 4; // see MachOX86_64::resolveRelocation. + } + + switch (RE.RelType) { + default: + llvm_unreachable("Invalid relocation type!"); + case MachO::GENERIC_RELOC_VANILLA: + writeBytesUnaligned(Value + RE.Addend, LocalAddress, 1 << RE.Size); + break; + case MachO::GENERIC_RELOC_SECTDIFF: + case MachO::GENERIC_RELOC_LOCAL_SECTDIFF: { + uint64_t SectionABase = Sections[RE.Sections.SectionA].LoadAddress; + uint64_t SectionBBase = Sections[RE.Sections.SectionB].LoadAddress; + assert((Value == SectionABase || Value == SectionBBase) && + "Unexpected SECTDIFF relocation value."); + Value = SectionABase - SectionBBase + RE.Addend; + writeBytesUnaligned(Value, LocalAddress, 1 << RE.Size); + break; + } + case MachO::GENERIC_RELOC_PB_LA_PTR: + Error("Relocation type not implemented yet!"); + } + } + + void finalizeSection(ObjectImage &ObjImg, unsigned SectionID, + const SectionRef &Section) { + StringRef Name; + Section.getName(Name); + + if (Name == "__jump_table") + populateJumpTable(cast<MachOObjectFile>(*ObjImg.getObjectFile()), Section, + SectionID); + else if (Name == "__pointers") + populateIndirectSymbolPointersSection( + cast<MachOObjectFile>(*ObjImg.getObjectFile()), + Section, SectionID); + } + +private: + relocation_iterator + processSECTDIFFRelocation(unsigned SectionID, relocation_iterator RelI, + ObjectImage &Obj, + ObjSectionToIDMap &ObjSectionToID) { + const MachOObjectFile *MachO = + static_cast<const MachOObjectFile *>(Obj.getObjectFile()); + MachO::any_relocation_info RE = + MachO->getRelocation(RelI->getRawDataRefImpl()); + + SectionEntry &Section = Sections[SectionID]; + uint32_t RelocType = MachO->getAnyRelocationType(RE); + bool IsPCRel = MachO->getAnyRelocationPCRel(RE); + unsigned Size = MachO->getAnyRelocationLength(RE); + uint64_t Offset; + RelI->getOffset(Offset); + uint8_t *LocalAddress = Section.Address + Offset; + unsigned NumBytes = 1 << Size; + uint64_t Addend = readBytesUnaligned(LocalAddress, NumBytes); + + ++RelI; + MachO::any_relocation_info RE2 = + MachO->getRelocation(RelI->getRawDataRefImpl()); + + uint32_t AddrA = MachO->getScatteredRelocationValue(RE); + section_iterator SAI = getSectionByAddress(*MachO, AddrA); + assert(SAI != MachO->section_end() && "Can't find section for address A"); + uint64_t SectionABase = SAI->getAddress(); + uint64_t SectionAOffset = AddrA - SectionABase; + SectionRef SectionA = *SAI; + bool IsCode = SectionA.isText(); + uint32_t SectionAID = + findOrEmitSection(Obj, SectionA, IsCode, ObjSectionToID); + + uint32_t AddrB = MachO->getScatteredRelocationValue(RE2); + section_iterator SBI = getSectionByAddress(*MachO, AddrB); + assert(SBI != MachO->section_end() && "Can't find section for address B"); + uint64_t SectionBBase = SBI->getAddress(); + uint64_t SectionBOffset = AddrB - SectionBBase; + SectionRef SectionB = *SBI; + uint32_t SectionBID = + findOrEmitSection(Obj, SectionB, IsCode, ObjSectionToID); + + if (Addend != AddrA - AddrB) + Error("Unexpected SECTDIFF relocation addend."); + + DEBUG(dbgs() << "Found SECTDIFF: AddrA: " << AddrA << ", AddrB: " << AddrB + << ", Addend: " << Addend << ", SectionA ID: " << SectionAID + << ", SectionAOffset: " << SectionAOffset + << ", SectionB ID: " << SectionBID + << ", SectionBOffset: " << SectionBOffset << "\n"); + RelocationEntry R(SectionID, Offset, RelocType, 0, SectionAID, + SectionAOffset, SectionBID, SectionBOffset, IsPCRel, + Size); + + addRelocationForSection(R, SectionAID); + addRelocationForSection(R, SectionBID); + + return ++RelI; + } + + relocation_iterator processI386ScatteredVANILLA( + unsigned SectionID, relocation_iterator RelI, ObjectImage &Obj, + RuntimeDyldMachO::ObjSectionToIDMap &ObjSectionToID) { + const MachOObjectFile *MachO = + static_cast<const MachOObjectFile *>(Obj.getObjectFile()); + MachO::any_relocation_info RE = + MachO->getRelocation(RelI->getRawDataRefImpl()); + + SectionEntry &Section = Sections[SectionID]; + uint32_t RelocType = MachO->getAnyRelocationType(RE); + bool IsPCRel = MachO->getAnyRelocationPCRel(RE); + unsigned Size = MachO->getAnyRelocationLength(RE); + uint64_t Offset; + RelI->getOffset(Offset); + uint8_t *LocalAddress = Section.Address + Offset; + unsigned NumBytes = 1 << Size; + int64_t Addend = readBytesUnaligned(LocalAddress, NumBytes); + + unsigned SymbolBaseAddr = MachO->getScatteredRelocationValue(RE); + section_iterator TargetSI = getSectionByAddress(*MachO, SymbolBaseAddr); + assert(TargetSI != MachO->section_end() && "Can't find section for symbol"); + uint64_t SectionBaseAddr = TargetSI->getAddress(); + SectionRef TargetSection = *TargetSI; + bool IsCode = TargetSection.isText(); + uint32_t TargetSectionID = + findOrEmitSection(Obj, TargetSection, IsCode, ObjSectionToID); + + Addend -= SectionBaseAddr; + RelocationEntry R(SectionID, Offset, RelocType, Addend, IsPCRel, Size); + + addRelocationForSection(R, TargetSectionID); + + return ++RelI; + } + + // Populate stubs in __jump_table section. + void populateJumpTable(MachOObjectFile &Obj, const SectionRef &JTSection, + unsigned JTSectionID) { + assert(!Obj.is64Bit() && + "__jump_table section not supported in 64-bit MachO."); + + MachO::dysymtab_command DySymTabCmd = Obj.getDysymtabLoadCommand(); + MachO::section Sec32 = Obj.getSection(JTSection.getRawDataRefImpl()); + uint32_t JTSectionSize = Sec32.size; + unsigned FirstIndirectSymbol = Sec32.reserved1; + unsigned JTEntrySize = Sec32.reserved2; + unsigned NumJTEntries = JTSectionSize / JTEntrySize; + uint8_t *JTSectionAddr = getSectionAddress(JTSectionID); + unsigned JTEntryOffset = 0; + + assert((JTSectionSize % JTEntrySize) == 0 && + "Jump-table section does not contain a whole number of stubs?"); + + for (unsigned i = 0; i < NumJTEntries; ++i) { + unsigned SymbolIndex = + Obj.getIndirectSymbolTableEntry(DySymTabCmd, FirstIndirectSymbol + i); + symbol_iterator SI = Obj.getSymbolByIndex(SymbolIndex); + StringRef IndirectSymbolName; + SI->getName(IndirectSymbolName); + uint8_t *JTEntryAddr = JTSectionAddr + JTEntryOffset; + createStubFunction(JTEntryAddr); + RelocationEntry RE(JTSectionID, JTEntryOffset + 1, + MachO::GENERIC_RELOC_VANILLA, 0, true, 2); + addRelocationForSymbol(RE, IndirectSymbolName); + JTEntryOffset += JTEntrySize; + } + } + +}; +} + +#undef DEBUG_TYPE + +#endif diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h new file mode 100644 index 0000000..84d9e80 --- /dev/null +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h @@ -0,0 +1,136 @@ +//===-- RuntimeDyldMachOX86_64.h ---- MachO/X86_64 specific code. -*- C++ -*-=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOX86_64_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDMACHOX86_64_H + +#include "../RuntimeDyldMachO.h" + +#define DEBUG_TYPE "dyld" + +namespace llvm { + +class RuntimeDyldMachOX86_64 + : public RuntimeDyldMachOCRTPBase<RuntimeDyldMachOX86_64> { +public: + + typedef uint64_t TargetPtrT; + + RuntimeDyldMachOX86_64(RTDyldMemoryManager *MM) + : RuntimeDyldMachOCRTPBase(MM) {} + + unsigned getMaxStubSize() override { return 8; } + + unsigned getStubAlignment() override { return 1; } + + relocation_iterator + processRelocationRef(unsigned SectionID, relocation_iterator RelI, + ObjectImage &ObjImg, ObjSectionToIDMap &ObjSectionToID, + const SymbolTableMap &Symbols, StubMap &Stubs) override { + const MachOObjectFile &Obj = + static_cast<const MachOObjectFile &>(*ObjImg.getObjectFile()); + MachO::any_relocation_info RelInfo = + Obj.getRelocation(RelI->getRawDataRefImpl()); + + assert(!Obj.isRelocationScattered(RelInfo) && + "Scattered relocations not supported on X86_64"); + + RelocationEntry RE(getRelocationEntry(SectionID, ObjImg, RelI)); + RE.Addend = memcpyAddend(RE); + RelocationValueRef Value( + getRelocationValueRef(ObjImg, RelI, RE, ObjSectionToID, Symbols)); + + bool IsExtern = Obj.getPlainRelocationExternal(RelInfo); + if (!IsExtern && RE.IsPCRel) + makeValueAddendPCRel(Value, ObjImg, RelI, 1 << RE.Size); + + if (RE.RelType == MachO::X86_64_RELOC_GOT || + RE.RelType == MachO::X86_64_RELOC_GOT_LOAD) + processGOTRelocation(RE, Value, Stubs); + else { + RE.Addend = Value.Offset; + if (Value.SymbolName) + addRelocationForSymbol(RE, Value.SymbolName); + else + addRelocationForSection(RE, Value.SectionID); + } + + return ++RelI; + } + + void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override { + DEBUG(dumpRelocationToResolve(RE, Value)); + const SectionEntry &Section = Sections[RE.SectionID]; + uint8_t *LocalAddress = Section.Address + RE.Offset; + + // If the relocation is PC-relative, the value to be encoded is the + // pointer difference. + if (RE.IsPCRel) { + // FIXME: It seems this value needs to be adjusted by 4 for an effective + // PC address. Is that expected? Only for branches, perhaps? + uint64_t FinalAddress = Section.LoadAddress + RE.Offset; + Value -= FinalAddress + 4; + } + + switch (RE.RelType) { + default: + llvm_unreachable("Invalid relocation type!"); + case MachO::X86_64_RELOC_SIGNED_1: + case MachO::X86_64_RELOC_SIGNED_2: + case MachO::X86_64_RELOC_SIGNED_4: + case MachO::X86_64_RELOC_SIGNED: + case MachO::X86_64_RELOC_UNSIGNED: + case MachO::X86_64_RELOC_BRANCH: + writeBytesUnaligned(Value + RE.Addend, LocalAddress, 1 << RE.Size); + break; + case MachO::X86_64_RELOC_GOT_LOAD: + case MachO::X86_64_RELOC_GOT: + case MachO::X86_64_RELOC_SUBTRACTOR: + case MachO::X86_64_RELOC_TLV: + Error("Relocation type not implemented yet!"); + } + } + + void finalizeSection(ObjectImage &ObjImg, unsigned SectionID, + const SectionRef &Section) {} + +private: + void processGOTRelocation(const RelocationEntry &RE, + RelocationValueRef &Value, StubMap &Stubs) { + SectionEntry &Section = Sections[RE.SectionID]; + assert(RE.IsPCRel); + assert(RE.Size == 2); + Value.Offset -= RE.Addend; + RuntimeDyldMachO::StubMap::const_iterator i = Stubs.find(Value); + uint8_t *Addr; + if (i != Stubs.end()) { + Addr = Section.Address + i->second; + } else { + Stubs[Value] = Section.StubOffset; + uint8_t *GOTEntry = Section.Address + Section.StubOffset; + RelocationEntry GOTRE(RE.SectionID, Section.StubOffset, + MachO::X86_64_RELOC_UNSIGNED, Value.Offset, false, + 3); + if (Value.SymbolName) + addRelocationForSymbol(GOTRE, Value.SymbolName); + else + addRelocationForSection(GOTRE, Value.SectionID); + Section.StubOffset += 8; + Addr = GOTEntry; + } + RelocationEntry TargetRE(RE.SectionID, RE.Offset, + MachO::X86_64_RELOC_UNSIGNED, RE.Addend, true, 2); + resolveRelocation(TargetRE, (uint64_t)Addr); + } +}; +} + +#undef DEBUG_TYPE + +#endif diff --git a/lib/ExecutionEngine/TargetSelect.cpp b/lib/ExecutionEngine/TargetSelect.cpp index b10d51f..e6679cf 100644 --- a/lib/ExecutionEngine/TargetSelect.cpp +++ b/lib/ExecutionEngine/TargetSelect.cpp @@ -30,7 +30,7 @@ TargetMachine *EngineBuilder::selectTarget() { // MCJIT can generate code for remote targets, but the old JIT and Interpreter // must use the host architecture. - if (UseMCJIT && WhichEngine != EngineKind::Interpreter && M) + if (WhichEngine != EngineKind::Interpreter && M) TT.setTriple(M->getTargetTriple()); return selectTarget(TT, MArch, MCPU, MAttrs); @@ -89,8 +89,7 @@ TargetMachine *EngineBuilder::selectTarget(const Triple &TargetTriple, } // FIXME: non-iOS ARM FastISel is broken with MCJIT. - if (UseMCJIT && - TheTriple.getArch() == Triple::arm && + if (TheTriple.getArch() == Triple::arm && !TheTriple.isiOS() && OptLevel == CodeGenOpt::None) { OptLevel = CodeGenOpt::Less; |