diff options
Diffstat (limited to 'lib/ExecutionEngine/JIT')
| -rw-r--r-- | lib/ExecutionEngine/JIT/Android.mk | 17 | ||||
| -rw-r--r-- | lib/ExecutionEngine/JIT/CMakeLists.txt | 8 | ||||
| -rw-r--r-- | lib/ExecutionEngine/JIT/JIT.cpp | 695 | ||||
| -rw-r--r-- | lib/ExecutionEngine/JIT/JIT.h | 229 | ||||
| -rw-r--r-- | lib/ExecutionEngine/JIT/JITEmitter.cpp | 1256 | ||||
| -rw-r--r-- | lib/ExecutionEngine/JIT/JITMemoryManager.cpp | 904 | ||||
| -rw-r--r-- | lib/ExecutionEngine/JIT/LLVMBuild.txt | 22 | ||||
| -rw-r--r-- | lib/ExecutionEngine/JIT/Makefile | 38 |
8 files changed, 0 insertions, 3169 deletions
diff --git a/lib/ExecutionEngine/JIT/Android.mk b/lib/ExecutionEngine/JIT/Android.mk deleted file mode 100644 index 0466ba0..0000000 --- a/lib/ExecutionEngine/JIT/Android.mk +++ /dev/null @@ -1,17 +0,0 @@ -LOCAL_PATH:= $(call my-dir) - -# For the host -# ===================================================== -include $(CLEAR_VARS) - -LOCAL_SRC_FILES := \ - JIT.cpp \ - JITEmitter.cpp \ - JITMemoryManager.cpp - -LOCAL_MODULE:= libLLVMJIT - -LOCAL_MODULE_TAGS := optional - -include $(LLVM_HOST_BUILD_MK) -include $(BUILD_HOST_STATIC_LIBRARY) diff --git a/lib/ExecutionEngine/JIT/CMakeLists.txt b/lib/ExecutionEngine/JIT/CMakeLists.txt deleted file mode 100644 index e16baed..0000000 --- a/lib/ExecutionEngine/JIT/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# TODO: Support other architectures. See Makefile. -add_definitions(-DENABLE_X86_JIT) - -add_llvm_library(LLVMJIT - JIT.cpp - JITEmitter.cpp - JITMemoryManager.cpp - ) diff --git a/lib/ExecutionEngine/JIT/JIT.cpp b/lib/ExecutionEngine/JIT/JIT.cpp deleted file mode 100644 index 83ec978..0000000 --- a/lib/ExecutionEngine/JIT/JIT.cpp +++ /dev/null @@ -1,695 +0,0 @@ -//===-- JIT.cpp - LLVM Just in Time Compiler ------------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This tool implements a just-in-time compiler for LLVM, allowing direct -// execution of LLVM bitcode in an efficient manner. -// -//===----------------------------------------------------------------------===// - -#include "JIT.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/CodeGen/JITCodeEmitter.h" -#include "llvm/CodeGen/MachineCodeInfo.h" -#include "llvm/Config/config.h" -#include "llvm/ExecutionEngine/GenericValue.h" -#include "llvm/ExecutionEngine/JITEventListener.h" -#include "llvm/ExecutionEngine/JITMemoryManager.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/GlobalVariable.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Module.h" -#include "llvm/Support/Dwarf.h" -#include "llvm/Support/DynamicLibrary.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/MutexGuard.h" -#include "llvm/Target/TargetJITInfo.h" -#include "llvm/Target/TargetMachine.h" - -using namespace llvm; - -#ifdef __APPLE__ -// Apple gcc defaults to -fuse-cxa-atexit (i.e. calls __cxa_atexit instead -// of atexit). It passes the address of linker generated symbol __dso_handle -// to the function. -// This configuration change happened at version 5330. -# include <AvailabilityMacros.h> -# if defined(MAC_OS_X_VERSION_10_4) && \ - ((MAC_OS_X_VERSION_MIN_REQUIRED > MAC_OS_X_VERSION_10_4) || \ - (MAC_OS_X_VERSION_MIN_REQUIRED == MAC_OS_X_VERSION_10_4 && \ - __APPLE_CC__ >= 5330)) -# ifndef HAVE___DSO_HANDLE -# define HAVE___DSO_HANDLE 1 -# endif -# endif -#endif - -#if HAVE___DSO_HANDLE -extern void *__dso_handle __attribute__ ((__visibility__ ("hidden"))); -#endif - -namespace { - -static struct RegisterJIT { - RegisterJIT() { JIT::Register(); } -} JITRegistrator; - -} - -extern "C" void LLVMLinkInJIT() { -} - -/// createJIT - This is the factory method for creating a JIT for the current -/// machine, it does not fall back to the interpreter. This takes ownership -/// of the module. -ExecutionEngine *JIT::createJIT(Module *M, - std::string *ErrorStr, - JITMemoryManager *JMM, - bool GVsWithCode, - TargetMachine *TM) { - // Try to register the program as a source of symbols to resolve against. - // - // FIXME: Don't do this here. - sys::DynamicLibrary::LoadLibraryPermanently(nullptr, nullptr); - - // If the target supports JIT code generation, create the JIT. - if (TargetJITInfo *TJ = TM->getJITInfo()) { - return new JIT(M, *TM, *TJ, JMM, GVsWithCode); - } else { - if (ErrorStr) - *ErrorStr = "target does not support JIT code generation"; - return nullptr; - } -} - -namespace { -/// This class supports the global getPointerToNamedFunction(), which allows -/// bugpoint or gdb users to search for a function by name without any context. -class JitPool { - SmallPtrSet<JIT*, 1> JITs; // Optimize for process containing just 1 JIT. - mutable sys::Mutex Lock; -public: - void Add(JIT *jit) { - MutexGuard guard(Lock); - JITs.insert(jit); - } - void Remove(JIT *jit) { - MutexGuard guard(Lock); - JITs.erase(jit); - } - void *getPointerToNamedFunction(const char *Name) const { - MutexGuard guard(Lock); - assert(JITs.size() != 0 && "No Jit registered"); - //search function in every instance of JIT - for (SmallPtrSet<JIT*, 1>::const_iterator Jit = JITs.begin(), - end = JITs.end(); - Jit != end; ++Jit) { - if (Function *F = (*Jit)->FindFunctionNamed(Name)) - return (*Jit)->getPointerToFunction(F); - } - // The function is not available : fallback on the first created (will - // search in symbol of the current program/library) - return (*JITs.begin())->getPointerToNamedFunction(Name); - } -}; -ManagedStatic<JitPool> AllJits; -} -extern "C" { - // getPointerToNamedFunction - This function is used as a global wrapper to - // JIT::getPointerToNamedFunction for the purpose of resolving symbols when - // bugpoint is debugging the JIT. In that scenario, we are loading an .so and - // need to resolve function(s) that are being mis-codegenerated, so we need to - // resolve their addresses at runtime, and this is the way to do it. - void *getPointerToNamedFunction(const char *Name) { - return AllJits->getPointerToNamedFunction(Name); - } -} - -JIT::JIT(Module *M, TargetMachine &tm, TargetJITInfo &tji, - JITMemoryManager *jmm, bool GVsWithCode) - : ExecutionEngine(M), TM(tm), TJI(tji), - JMM(jmm ? jmm : JITMemoryManager::CreateDefaultMemManager()), - AllocateGVsWithCode(GVsWithCode), isAlreadyCodeGenerating(false) { - setDataLayout(TM.getDataLayout()); - - jitstate = new JITState(M); - - // Initialize JCE - JCE = createEmitter(*this, JMM, TM); - - // Register in global list of all JITs. - AllJits->Add(this); - - // Add target data - MutexGuard locked(lock); - FunctionPassManager &PM = jitstate->getPM(); - M->setDataLayout(TM.getDataLayout()); - PM.add(new DataLayoutPass(M)); - - // Turn the machine code intermediate representation into bytes in memory that - // may be executed. - if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) { - report_fatal_error("Target does not support machine code emission!"); - } - - // Initialize passes. - PM.doInitialization(); -} - -JIT::~JIT() { - // Cleanup. - AllJits->Remove(this); - delete jitstate; - delete JCE; - // JMM is a ownership of JCE, so we no need delete JMM here. - delete &TM; -} - -/// addModule - Add a new Module to the JIT. If we previously removed the last -/// Module, we need re-initialize jitstate with a valid Module. -void JIT::addModule(Module *M) { - MutexGuard locked(lock); - - if (Modules.empty()) { - assert(!jitstate && "jitstate should be NULL if Modules vector is empty!"); - - jitstate = new JITState(M); - - FunctionPassManager &PM = jitstate->getPM(); - M->setDataLayout(TM.getDataLayout()); - PM.add(new DataLayoutPass(M)); - - // Turn the machine code intermediate representation into bytes in memory - // that may be executed. - if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) { - report_fatal_error("Target does not support machine code emission!"); - } - - // Initialize passes. - PM.doInitialization(); - } - - ExecutionEngine::addModule(M); -} - -/// removeModule - If we are removing the last Module, invalidate the jitstate -/// since the PassManager it contains references a released Module. -bool JIT::removeModule(Module *M) { - bool result = ExecutionEngine::removeModule(M); - - MutexGuard locked(lock); - - if (jitstate && jitstate->getModule() == M) { - delete jitstate; - jitstate = nullptr; - } - - if (!jitstate && !Modules.empty()) { - jitstate = new JITState(Modules[0]); - - FunctionPassManager &PM = jitstate->getPM(); - M->setDataLayout(TM.getDataLayout()); - PM.add(new DataLayoutPass(M)); - - // Turn the machine code intermediate representation into bytes in memory - // that may be executed. - if (TM.addPassesToEmitMachineCode(PM, *JCE, !getVerifyModules())) { - report_fatal_error("Target does not support machine code emission!"); - } - - // Initialize passes. - PM.doInitialization(); - } - return result; -} - -/// run - Start execution with the specified function and arguments. -/// -GenericValue JIT::runFunction(Function *F, - const std::vector<GenericValue> &ArgValues) { - assert(F && "Function *F was null at entry to run()"); - - void *FPtr = getPointerToFunction(F); - assert(FPtr && "Pointer to fn's code was null after getPointerToFunction"); - FunctionType *FTy = F->getFunctionType(); - Type *RetTy = FTy->getReturnType(); - - assert((FTy->getNumParams() == ArgValues.size() || - (FTy->isVarArg() && FTy->getNumParams() <= ArgValues.size())) && - "Wrong number of arguments passed into function!"); - assert(FTy->getNumParams() == ArgValues.size() && - "This doesn't support passing arguments through varargs (yet)!"); - - // Handle some common cases first. These cases correspond to common `main' - // prototypes. - if (RetTy->isIntegerTy(32) || RetTy->isVoidTy()) { - switch (ArgValues.size()) { - case 3: - if (FTy->getParamType(0)->isIntegerTy(32) && - FTy->getParamType(1)->isPointerTy() && - FTy->getParamType(2)->isPointerTy()) { - int (*PF)(int, char **, const char **) = - (int(*)(int, char **, const char **))(intptr_t)FPtr; - - // Call the function. - GenericValue rv; - rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(), - (char **)GVTOP(ArgValues[1]), - (const char **)GVTOP(ArgValues[2]))); - return rv; - } - break; - case 2: - if (FTy->getParamType(0)->isIntegerTy(32) && - FTy->getParamType(1)->isPointerTy()) { - int (*PF)(int, char **) = (int(*)(int, char **))(intptr_t)FPtr; - - // Call the function. - GenericValue rv; - rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(), - (char **)GVTOP(ArgValues[1]))); - return rv; - } - break; - case 1: - if (FTy->getParamType(0)->isIntegerTy(32)) { - GenericValue rv; - int (*PF)(int) = (int(*)(int))(intptr_t)FPtr; - rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue())); - return rv; - } - if (FTy->getParamType(0)->isPointerTy()) { - GenericValue rv; - int (*PF)(char *) = (int(*)(char *))(intptr_t)FPtr; - rv.IntVal = APInt(32, PF((char*)GVTOP(ArgValues[0]))); - return rv; - } - break; - } - } - - // Handle cases where no arguments are passed first. - if (ArgValues.empty()) { - GenericValue rv; - switch (RetTy->getTypeID()) { - default: llvm_unreachable("Unknown return type for function call!"); - case Type::IntegerTyID: { - unsigned BitWidth = cast<IntegerType>(RetTy)->getBitWidth(); - if (BitWidth == 1) - rv.IntVal = APInt(BitWidth, ((bool(*)())(intptr_t)FPtr)()); - else if (BitWidth <= 8) - rv.IntVal = APInt(BitWidth, ((char(*)())(intptr_t)FPtr)()); - else if (BitWidth <= 16) - rv.IntVal = APInt(BitWidth, ((short(*)())(intptr_t)FPtr)()); - else if (BitWidth <= 32) - rv.IntVal = APInt(BitWidth, ((int(*)())(intptr_t)FPtr)()); - else if (BitWidth <= 64) - rv.IntVal = APInt(BitWidth, ((int64_t(*)())(intptr_t)FPtr)()); - else - llvm_unreachable("Integer types > 64 bits not supported"); - return rv; - } - case Type::VoidTyID: - rv.IntVal = APInt(32, ((int(*)())(intptr_t)FPtr)()); - return rv; - case Type::FloatTyID: - rv.FloatVal = ((float(*)())(intptr_t)FPtr)(); - return rv; - case Type::DoubleTyID: - rv.DoubleVal = ((double(*)())(intptr_t)FPtr)(); - return rv; - case Type::X86_FP80TyID: - case Type::FP128TyID: - case Type::PPC_FP128TyID: - llvm_unreachable("long double not supported yet"); - case Type::PointerTyID: - return PTOGV(((void*(*)())(intptr_t)FPtr)()); - } - } - - // Okay, this is not one of our quick and easy cases. Because we don't have a - // full FFI, we have to codegen a nullary stub function that just calls the - // function we are interested in, passing in constants for all of the - // arguments. Make this function and return. - - // First, create the function. - FunctionType *STy=FunctionType::get(RetTy, false); - Function *Stub = Function::Create(STy, Function::InternalLinkage, "", - F->getParent()); - - // Insert a basic block. - BasicBlock *StubBB = BasicBlock::Create(F->getContext(), "", Stub); - - // Convert all of the GenericValue arguments over to constants. Note that we - // currently don't support varargs. - SmallVector<Value*, 8> Args; - for (unsigned i = 0, e = ArgValues.size(); i != e; ++i) { - Constant *C = nullptr; - Type *ArgTy = FTy->getParamType(i); - const GenericValue &AV = ArgValues[i]; - switch (ArgTy->getTypeID()) { - default: llvm_unreachable("Unknown argument type for function call!"); - case Type::IntegerTyID: - C = ConstantInt::get(F->getContext(), AV.IntVal); - break; - case Type::FloatTyID: - C = ConstantFP::get(F->getContext(), APFloat(AV.FloatVal)); - break; - case Type::DoubleTyID: - C = ConstantFP::get(F->getContext(), APFloat(AV.DoubleVal)); - break; - case Type::PPC_FP128TyID: - case Type::X86_FP80TyID: - case Type::FP128TyID: - C = ConstantFP::get(F->getContext(), APFloat(ArgTy->getFltSemantics(), - AV.IntVal)); - break; - case Type::PointerTyID: - void *ArgPtr = GVTOP(AV); - if (sizeof(void*) == 4) - C = ConstantInt::get(Type::getInt32Ty(F->getContext()), - (int)(intptr_t)ArgPtr); - else - C = ConstantInt::get(Type::getInt64Ty(F->getContext()), - (intptr_t)ArgPtr); - // Cast the integer to pointer - C = ConstantExpr::getIntToPtr(C, ArgTy); - break; - } - Args.push_back(C); - } - - CallInst *TheCall = CallInst::Create(F, Args, "", StubBB); - TheCall->setCallingConv(F->getCallingConv()); - TheCall->setTailCall(); - if (!TheCall->getType()->isVoidTy()) - // Return result of the call. - ReturnInst::Create(F->getContext(), TheCall, StubBB); - else - ReturnInst::Create(F->getContext(), StubBB); // Just return void. - - // Finally, call our nullary stub function. - GenericValue Result = runFunction(Stub, std::vector<GenericValue>()); - // Erase it, since no other function can have a reference to it. - Stub->eraseFromParent(); - // And return the result. - return Result; -} - -void JIT::RegisterJITEventListener(JITEventListener *L) { - if (!L) - return; - MutexGuard locked(lock); - EventListeners.push_back(L); -} -void JIT::UnregisterJITEventListener(JITEventListener *L) { - if (!L) - return; - MutexGuard locked(lock); - std::vector<JITEventListener*>::reverse_iterator I= - std::find(EventListeners.rbegin(), EventListeners.rend(), L); - if (I != EventListeners.rend()) { - std::swap(*I, EventListeners.back()); - EventListeners.pop_back(); - } -} -void JIT::NotifyFunctionEmitted( - const Function &F, - void *Code, size_t Size, - const JITEvent_EmittedFunctionDetails &Details) { - MutexGuard locked(lock); - for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) { - EventListeners[I]->NotifyFunctionEmitted(F, Code, Size, Details); - } -} - -void JIT::NotifyFreeingMachineCode(void *OldPtr) { - MutexGuard locked(lock); - for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) { - EventListeners[I]->NotifyFreeingMachineCode(OldPtr); - } -} - -/// runJITOnFunction - Run the FunctionPassManager full of -/// just-in-time compilation passes on F, hopefully filling in -/// GlobalAddress[F] with the address of F's machine code. -/// -void JIT::runJITOnFunction(Function *F, MachineCodeInfo *MCI) { - MutexGuard locked(lock); - - class MCIListener : public JITEventListener { - MachineCodeInfo *const MCI; - public: - MCIListener(MachineCodeInfo *mci) : MCI(mci) {} - void NotifyFunctionEmitted(const Function &, void *Code, size_t Size, - const EmittedFunctionDetails &) override { - MCI->setAddress(Code); - MCI->setSize(Size); - } - }; - MCIListener MCIL(MCI); - if (MCI) - RegisterJITEventListener(&MCIL); - - runJITOnFunctionUnlocked(F); - - if (MCI) - UnregisterJITEventListener(&MCIL); -} - -void JIT::runJITOnFunctionUnlocked(Function *F) { - assert(!isAlreadyCodeGenerating && "Error: Recursive compilation detected!"); - - jitTheFunctionUnlocked(F); - - // If the function referred to another function that had not yet been - // read from bitcode, and we are jitting non-lazily, emit it now. - while (!jitstate->getPendingFunctions().empty()) { - Function *PF = jitstate->getPendingFunctions().back(); - jitstate->getPendingFunctions().pop_back(); - - assert(!PF->hasAvailableExternallyLinkage() && - "Externally-defined function should not be in pending list."); - - jitTheFunctionUnlocked(PF); - - // Now that the function has been jitted, ask the JITEmitter to rewrite - // the stub with real address of the function. - updateFunctionStubUnlocked(PF); - } -} - -void JIT::jitTheFunctionUnlocked(Function *F) { - isAlreadyCodeGenerating = true; - jitstate->getPM().run(*F); - isAlreadyCodeGenerating = false; - - // clear basic block addresses after this function is done - getBasicBlockAddressMap().clear(); -} - -/// getPointerToFunction - This method is used to get the address of the -/// specified function, compiling it if necessary. -/// -void *JIT::getPointerToFunction(Function *F) { - - if (void *Addr = getPointerToGlobalIfAvailable(F)) - return Addr; // Check if function already code gen'd - - MutexGuard locked(lock); - - // Now that this thread owns the lock, make sure we read in the function if it - // exists in this Module. - std::string ErrorMsg; - if (F->Materialize(&ErrorMsg)) { - report_fatal_error("Error reading function '" + F->getName()+ - "' from bitcode file: " + ErrorMsg); - } - - // ... and check if another thread has already code gen'd the function. - if (void *Addr = getPointerToGlobalIfAvailable(F)) - return Addr; - - if (F->isDeclaration() || F->hasAvailableExternallyLinkage()) { - bool AbortOnFailure = !F->hasExternalWeakLinkage(); - void *Addr = getPointerToNamedFunction(F->getName(), AbortOnFailure); - addGlobalMapping(F, Addr); - return Addr; - } - - runJITOnFunctionUnlocked(F); - - void *Addr = getPointerToGlobalIfAvailable(F); - assert(Addr && "Code generation didn't add function to GlobalAddress table!"); - return Addr; -} - -void JIT::addPointerToBasicBlock(const BasicBlock *BB, void *Addr) { - MutexGuard locked(lock); - - BasicBlockAddressMapTy::iterator I = - getBasicBlockAddressMap().find(BB); - if (I == getBasicBlockAddressMap().end()) { - getBasicBlockAddressMap()[BB] = Addr; - } else { - // ignore repeats: some BBs can be split into few MBBs? - } -} - -void JIT::clearPointerToBasicBlock(const BasicBlock *BB) { - MutexGuard locked(lock); - getBasicBlockAddressMap().erase(BB); -} - -void *JIT::getPointerToBasicBlock(BasicBlock *BB) { - // make sure it's function is compiled by JIT - (void)getPointerToFunction(BB->getParent()); - - // resolve basic block address - MutexGuard locked(lock); - - BasicBlockAddressMapTy::iterator I = - getBasicBlockAddressMap().find(BB); - if (I != getBasicBlockAddressMap().end()) { - return I->second; - } else { - llvm_unreachable("JIT does not have BB address for address-of-label, was" - " it eliminated by optimizer?"); - } -} - -void *JIT::getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure){ - if (!isSymbolSearchingDisabled()) { - void *ptr = JMM->getPointerToNamedFunction(Name, false); - if (ptr) - return ptr; - } - - /// If a LazyFunctionCreator is installed, use it to get/create the function. - if (LazyFunctionCreator) - if (void *RP = LazyFunctionCreator(Name)) - return RP; - - if (AbortOnFailure) { - report_fatal_error("Program used external function '"+Name+ - "' which could not be resolved!"); - } - return nullptr; -} - - -/// getOrEmitGlobalVariable - Return the address of the specified global -/// variable, possibly emitting it to memory if needed. This is used by the -/// Emitter. -void *JIT::getOrEmitGlobalVariable(const GlobalVariable *GV) { - MutexGuard locked(lock); - - void *Ptr = getPointerToGlobalIfAvailable(GV); - if (Ptr) return Ptr; - - // If the global is external, just remember the address. - if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) { -#if HAVE___DSO_HANDLE - if (GV->getName() == "__dso_handle") - return (void*)&__dso_handle; -#endif - Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(GV->getName()); - if (!Ptr) { - report_fatal_error("Could not resolve external global address: " - +GV->getName()); - } - addGlobalMapping(GV, Ptr); - } else { - // If the global hasn't been emitted to memory yet, allocate space and - // emit it into memory. - Ptr = getMemoryForGV(GV); - addGlobalMapping(GV, Ptr); - EmitGlobalVariable(GV); // Initialize the variable. - } - return Ptr; -} - -/// recompileAndRelinkFunction - This method is used to force a function -/// which has already been compiled, to be compiled again, possibly -/// after it has been modified. Then the entry to the old copy is overwritten -/// with a branch to the new copy. If there was no old copy, this acts -/// just like JIT::getPointerToFunction(). -/// -void *JIT::recompileAndRelinkFunction(Function *F) { - void *OldAddr = getPointerToGlobalIfAvailable(F); - - // If it's not already compiled there is no reason to patch it up. - if (!OldAddr) return getPointerToFunction(F); - - // Delete the old function mapping. - addGlobalMapping(F, nullptr); - - // Recodegen the function - runJITOnFunction(F); - - // Update state, forward the old function to the new function. - void *Addr = getPointerToGlobalIfAvailable(F); - assert(Addr && "Code generation didn't add function to GlobalAddress table!"); - TJI.replaceMachineCodeForFunction(OldAddr, Addr); - return Addr; -} - -/// getMemoryForGV - This method abstracts memory allocation of global -/// variable so that the JIT can allocate thread local variables depending -/// on the target. -/// -char* JIT::getMemoryForGV(const GlobalVariable* GV) { - char *Ptr; - - // GlobalVariable's which are not "constant" will cause trouble in a server - // situation. It's returned in the same block of memory as code which may - // not be writable. - if (isGVCompilationDisabled() && !GV->isConstant()) { - report_fatal_error("Compilation of non-internal GlobalValue is disabled!"); - } - - // Some applications require globals and code to live together, so they may - // be allocated into the same buffer, but in general globals are allocated - // through the memory manager which puts them near the code but not in the - // same buffer. - Type *GlobalType = GV->getType()->getElementType(); - size_t S = getDataLayout()->getTypeAllocSize(GlobalType); - size_t A = getDataLayout()->getPreferredAlignment(GV); - if (GV->isThreadLocal()) { - MutexGuard locked(lock); - Ptr = TJI.allocateThreadLocalMemory(S); - } else if (TJI.allocateSeparateGVMemory()) { - if (A <= 8) { - Ptr = (char*)malloc(S); - } else { - // Allocate S+A bytes of memory, then use an aligned pointer within that - // space. - Ptr = (char*)malloc(S+A); - unsigned MisAligned = ((intptr_t)Ptr & (A-1)); - Ptr = Ptr + (MisAligned ? (A-MisAligned) : 0); - } - } else if (AllocateGVsWithCode) { - Ptr = (char*)JCE->allocateSpace(S, A); - } else { - Ptr = (char*)JCE->allocateGlobal(S, A); - } - return Ptr; -} - -void JIT::addPendingFunction(Function *F) { - MutexGuard locked(lock); - jitstate->getPendingFunctions().push_back(F); -} - - -JITEventListener::~JITEventListener() {} diff --git a/lib/ExecutionEngine/JIT/JIT.h b/lib/ExecutionEngine/JIT/JIT.h deleted file mode 100644 index 69a7c36..0000000 --- a/lib/ExecutionEngine/JIT/JIT.h +++ /dev/null @@ -1,229 +0,0 @@ -//===-- JIT.h - Class definition for the JIT --------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines the top-level JIT data structure. -// -//===----------------------------------------------------------------------===// - -#ifndef JIT_H -#define JIT_H - -#include "llvm/ExecutionEngine/ExecutionEngine.h" -#include "llvm/IR/ValueHandle.h" -#include "llvm/PassManager.h" - -namespace llvm { - -class Function; -struct JITEvent_EmittedFunctionDetails; -class MachineCodeEmitter; -class MachineCodeInfo; -class TargetJITInfo; -class TargetMachine; - -class JITState { -private: - FunctionPassManager PM; // Passes to compile a function - Module *M; // Module used to create the PM - - /// PendingFunctions - Functions which have not been code generated yet, but - /// were called from a function being code generated. - std::vector<AssertingVH<Function> > PendingFunctions; - -public: - explicit JITState(Module *M) : PM(M), M(M) {} - - FunctionPassManager &getPM() { - return PM; - } - - Module *getModule() const { return M; } - std::vector<AssertingVH<Function> > &getPendingFunctions() { - return PendingFunctions; - } -}; - - -class JIT : public ExecutionEngine { - /// types - typedef ValueMap<const BasicBlock *, void *> - BasicBlockAddressMapTy; - /// data - TargetMachine &TM; // The current target we are compiling to - TargetJITInfo &TJI; // The JITInfo for the target we are compiling to - JITCodeEmitter *JCE; // JCE object - JITMemoryManager *JMM; - std::vector<JITEventListener*> EventListeners; - - /// AllocateGVsWithCode - Some applications require that global variables and - /// code be allocated into the same region of memory, in which case this flag - /// should be set to true. Doing so breaks freeMachineCodeForFunction. - bool AllocateGVsWithCode; - - /// True while the JIT is generating code. Used to assert against recursive - /// entry. - bool isAlreadyCodeGenerating; - - JITState *jitstate; - - /// BasicBlockAddressMap - A mapping between LLVM basic blocks and their - /// actualized version, only filled for basic blocks that have their address - /// taken. - BasicBlockAddressMapTy BasicBlockAddressMap; - - - JIT(Module *M, TargetMachine &tm, TargetJITInfo &tji, - JITMemoryManager *JMM, bool AllocateGVsWithCode); -public: - ~JIT(); - - static void Register() { - JITCtor = createJIT; - } - - /// getJITInfo - Return the target JIT information structure. - /// - TargetJITInfo &getJITInfo() const { return TJI; } - - /// create - Create an return a new JIT compiler if there is one available - /// for the current target. Otherwise, return null. - /// - static ExecutionEngine *create(Module *M, - std::string *Err, - JITMemoryManager *JMM, - CodeGenOpt::Level OptLevel = - CodeGenOpt::Default, - bool GVsWithCode = true, - Reloc::Model RM = Reloc::Default, - CodeModel::Model CMM = CodeModel::JITDefault) { - return ExecutionEngine::createJIT(M, Err, JMM, OptLevel, GVsWithCode, - RM, CMM); - } - - void addModule(Module *M) override; - - /// removeModule - Remove a Module from the list of modules. Returns true if - /// M is found. - bool removeModule(Module *M) override; - - /// runFunction - Start execution with the specified function and arguments. - /// - GenericValue runFunction(Function *F, - const std::vector<GenericValue> &ArgValues) override; - - /// getPointerToNamedFunction - This method returns the address of the - /// specified function by using the MemoryManager. As such it is only - /// useful for resolving library symbols, not code generated symbols. - /// - /// If AbortOnFailure is false and no function with the given name is - /// found, this function silently returns a null pointer. Otherwise, - /// it prints a message to stderr and aborts. - /// - void *getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure = true) override; - - // CompilationCallback - Invoked the first time that a call site is found, - // which causes lazy compilation of the target function. - // - static void CompilationCallback(); - - /// getPointerToFunction - This returns the address of the specified function, - /// compiling it if necessary. - /// - void *getPointerToFunction(Function *F) override; - - /// addPointerToBasicBlock - Adds address of the specific basic block. - void addPointerToBasicBlock(const BasicBlock *BB, void *Addr); - - /// clearPointerToBasicBlock - Removes address of specific basic block. - void clearPointerToBasicBlock(const BasicBlock *BB); - - /// getPointerToBasicBlock - This returns the address of the specified basic - /// block, assuming function is compiled. - void *getPointerToBasicBlock(BasicBlock *BB) override; - - /// getOrEmitGlobalVariable - Return the address of the specified global - /// variable, possibly emitting it to memory if needed. This is used by the - /// Emitter. - void *getOrEmitGlobalVariable(const GlobalVariable *GV) override; - - /// getPointerToFunctionOrStub - If the specified function has been - /// code-gen'd, return a pointer to the function. If not, compile it, or use - /// a stub to implement lazy compilation if available. - /// - void *getPointerToFunctionOrStub(Function *F) override; - - /// recompileAndRelinkFunction - This method is used to force a function - /// which has already been compiled, to be compiled again, possibly - /// after it has been modified. Then the entry to the old copy is overwritten - /// with a branch to the new copy. If there was no old copy, this acts - /// just like JIT::getPointerToFunction(). - /// - void *recompileAndRelinkFunction(Function *F) override; - - /// freeMachineCodeForFunction - deallocate memory used to code-generate this - /// Function. - /// - void freeMachineCodeForFunction(Function *F) override; - - /// addPendingFunction - while jitting non-lazily, a called but non-codegen'd - /// function was encountered. Add it to a pending list to be processed after - /// the current function. - /// - void addPendingFunction(Function *F); - - /// getCodeEmitter - Return the code emitter this JIT is emitting into. - /// - JITCodeEmitter *getCodeEmitter() const { return JCE; } - - static ExecutionEngine *createJIT(Module *M, - std::string *ErrorStr, - JITMemoryManager *JMM, - bool GVsWithCode, - TargetMachine *TM); - - // Run the JIT on F and return information about the generated code - void runJITOnFunction(Function *F, MachineCodeInfo *MCI = nullptr) override; - - void RegisterJITEventListener(JITEventListener *L) override; - void UnregisterJITEventListener(JITEventListener *L) override; - - TargetMachine *getTargetMachine() override { return &TM; } - - /// These functions correspond to the methods on JITEventListener. They - /// iterate over the registered listeners and call the corresponding method on - /// each. - void NotifyFunctionEmitted( - const Function &F, void *Code, size_t Size, - const JITEvent_EmittedFunctionDetails &Details); - void NotifyFreeingMachineCode(void *OldPtr); - - BasicBlockAddressMapTy & - getBasicBlockAddressMap() { - return BasicBlockAddressMap; - } - - -private: - static JITCodeEmitter *createEmitter(JIT &J, JITMemoryManager *JMM, - TargetMachine &tm); - void runJITOnFunctionUnlocked(Function *F); - void updateFunctionStubUnlocked(Function *F); - void jitTheFunctionUnlocked(Function *F); - -protected: - - /// getMemoryforGV - Allocate memory for a global variable. - char* getMemoryForGV(const GlobalVariable* GV) override; - -}; - -} // End llvm namespace - -#endif diff --git a/lib/ExecutionEngine/JIT/JITEmitter.cpp b/lib/ExecutionEngine/JIT/JITEmitter.cpp deleted file mode 100644 index 50b8c10..0000000 --- a/lib/ExecutionEngine/JIT/JITEmitter.cpp +++ /dev/null @@ -1,1256 +0,0 @@ -//===-- JITEmitter.cpp - Write machine code to executable memory ----------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines a MachineCodeEmitter object that is used by the JIT to -// write machine code to memory and remember where relocatable values are. -// -//===----------------------------------------------------------------------===// - -#include "JIT.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/CodeGen/JITCodeEmitter.h" -#include "llvm/CodeGen/MachineCodeInfo.h" -#include "llvm/CodeGen/MachineConstantPool.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineJumpTableInfo.h" -#include "llvm/CodeGen/MachineModuleInfo.h" -#include "llvm/CodeGen/MachineRelocation.h" -#include "llvm/ExecutionEngine/GenericValue.h" -#include "llvm/ExecutionEngine/JITEventListener.h" -#include "llvm/ExecutionEngine/JITMemoryManager.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/DebugInfo.h" -#include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/Operator.h" -#include "llvm/IR/ValueHandle.h" -#include "llvm/IR/ValueMap.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/Disassembler.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/Memory.h" -#include "llvm/Support/MutexGuard.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetJITInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" -#include <algorithm> -#ifndef NDEBUG -#include <iomanip> -#endif -using namespace llvm; - -#define DEBUG_TYPE "jit" - -STATISTIC(NumBytes, "Number of bytes of machine code compiled"); -STATISTIC(NumRelos, "Number of relocations applied"); -STATISTIC(NumRetries, "Number of retries with more memory"); - - -// A declaration may stop being a declaration once it's fully read from bitcode. -// This function returns true if F is fully read and is still a declaration. -static bool isNonGhostDeclaration(const Function *F) { - return F->isDeclaration() && !F->isMaterializable(); -} - -//===----------------------------------------------------------------------===// -// JIT lazy compilation code. -// -namespace { - class JITEmitter; - class JITResolverState; - - template<typename ValueTy> - struct NoRAUWValueMapConfig : public ValueMapConfig<ValueTy> { - typedef JITResolverState *ExtraData; - static void onRAUW(JITResolverState *, Value *Old, Value *New) { - llvm_unreachable("The JIT doesn't know how to handle a" - " RAUW on a value it has emitted."); - } - }; - - struct CallSiteValueMapConfig : public NoRAUWValueMapConfig<Function*> { - typedef JITResolverState *ExtraData; - static void onDelete(JITResolverState *JRS, Function *F); - }; - - class JITResolverState { - public: - typedef ValueMap<Function*, void*, NoRAUWValueMapConfig<Function*> > - FunctionToLazyStubMapTy; - typedef std::map<void*, AssertingVH<Function> > CallSiteToFunctionMapTy; - typedef ValueMap<Function *, SmallPtrSet<void*, 1>, - CallSiteValueMapConfig> FunctionToCallSitesMapTy; - typedef std::map<AssertingVH<GlobalValue>, void*> GlobalToIndirectSymMapTy; - private: - /// FunctionToLazyStubMap - Keep track of the lazy stub created for a - /// particular function so that we can reuse them if necessary. - FunctionToLazyStubMapTy FunctionToLazyStubMap; - - /// CallSiteToFunctionMap - Keep track of the function that each lazy call - /// site corresponds to, and vice versa. - CallSiteToFunctionMapTy CallSiteToFunctionMap; - FunctionToCallSitesMapTy FunctionToCallSitesMap; - - /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a - /// particular GlobalVariable so that we can reuse them if necessary. - GlobalToIndirectSymMapTy GlobalToIndirectSymMap; - -#ifndef NDEBUG - /// Instance of the JIT this ResolverState serves. - JIT *TheJIT; -#endif - - public: - JITResolverState(JIT *jit) : FunctionToLazyStubMap(this), - FunctionToCallSitesMap(this) { -#ifndef NDEBUG - TheJIT = jit; -#endif - } - - FunctionToLazyStubMapTy& getFunctionToLazyStubMap() { - return FunctionToLazyStubMap; - } - - GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap() { - return GlobalToIndirectSymMap; - } - - std::pair<void *, Function *> LookupFunctionFromCallSite( - void *CallSite) const { - // The address given to us for the stub may not be exactly right, it - // might be a little bit after the stub. As such, use upper_bound to - // find it. - CallSiteToFunctionMapTy::const_iterator I = - CallSiteToFunctionMap.upper_bound(CallSite); - assert(I != CallSiteToFunctionMap.begin() && - "This is not a known call site!"); - --I; - return *I; - } - - void AddCallSite(void *CallSite, Function *F) { - bool Inserted = CallSiteToFunctionMap.insert( - std::make_pair(CallSite, F)).second; - (void)Inserted; - assert(Inserted && "Pair was already in CallSiteToFunctionMap"); - FunctionToCallSitesMap[F].insert(CallSite); - } - - void EraseAllCallSitesForPrelocked(Function *F); - - // Erases _all_ call sites regardless of their function. This is used to - // unregister the stub addresses from the StubToResolverMap in - // ~JITResolver(). - void EraseAllCallSitesPrelocked(); - }; - - /// JITResolver - Keep track of, and resolve, call sites for functions that - /// have not yet been compiled. - class JITResolver { - typedef JITResolverState::FunctionToLazyStubMapTy FunctionToLazyStubMapTy; - typedef JITResolverState::CallSiteToFunctionMapTy CallSiteToFunctionMapTy; - typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy; - - /// LazyResolverFn - The target lazy resolver function that we actually - /// rewrite instructions to use. - TargetJITInfo::LazyResolverFn LazyResolverFn; - - JITResolverState state; - - /// ExternalFnToStubMap - This is the equivalent of FunctionToLazyStubMap - /// for external functions. TODO: Of course, external functions don't need - /// a lazy stub. It's actually here to make it more likely that far calls - /// succeed, but no single stub can guarantee that. I'll remove this in a - /// subsequent checkin when I actually fix far calls. - std::map<void*, void*> ExternalFnToStubMap; - - /// revGOTMap - map addresses to indexes in the GOT - std::map<void*, unsigned> revGOTMap; - unsigned nextGOTIndex; - - JITEmitter &JE; - - /// Instance of JIT corresponding to this Resolver. - JIT *TheJIT; - - public: - explicit JITResolver(JIT &jit, JITEmitter &je) - : state(&jit), nextGOTIndex(0), JE(je), TheJIT(&jit) { - LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn); - } - - ~JITResolver(); - - /// getLazyFunctionStubIfAvailable - This returns a pointer to a function's - /// lazy-compilation stub if it has already been created. - void *getLazyFunctionStubIfAvailable(Function *F); - - /// getLazyFunctionStub - This returns a pointer to a function's - /// lazy-compilation stub, creating one on demand as needed. - void *getLazyFunctionStub(Function *F); - - /// getExternalFunctionStub - Return a stub for the function at the - /// specified address, created lazily on demand. - void *getExternalFunctionStub(void *FnAddr); - - /// getGlobalValueIndirectSym - Return an indirect symbol containing the - /// specified GV address. - void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress); - - /// getGOTIndexForAddress - Return a new or existing index in the GOT for - /// an address. This function only manages slots, it does not manage the - /// contents of the slots or the memory associated with the GOT. - unsigned getGOTIndexForAddr(void *addr); - - /// JITCompilerFn - This function is called to resolve a stub to a compiled - /// address. If the LLVM Function corresponding to the stub has not yet - /// been compiled, this function compiles it first. - static void *JITCompilerFn(void *Stub); - }; - - class StubToResolverMapTy { - /// Map a stub address to a specific instance of a JITResolver so that - /// lazily-compiled functions can find the right resolver to use. - /// - /// Guarded by Lock. - std::map<void*, JITResolver*> Map; - - /// Guards Map from concurrent accesses. - mutable sys::Mutex Lock; - - public: - /// Registers a Stub to be resolved by Resolver. - void RegisterStubResolver(void *Stub, JITResolver *Resolver) { - MutexGuard guard(Lock); - Map.insert(std::make_pair(Stub, Resolver)); - } - /// Unregisters the Stub when it's invalidated. - void UnregisterStubResolver(void *Stub) { - MutexGuard guard(Lock); - Map.erase(Stub); - } - /// Returns the JITResolver instance that owns the Stub. - JITResolver *getResolverFromStub(void *Stub) const { - MutexGuard guard(Lock); - // The address given to us for the stub may not be exactly right, it might - // be a little bit after the stub. As such, use upper_bound to find it. - // This is the same trick as in LookupFunctionFromCallSite from - // JITResolverState. - std::map<void*, JITResolver*>::const_iterator I = Map.upper_bound(Stub); - assert(I != Map.begin() && "This is not a known stub!"); - --I; - return I->second; - } - /// True if any stubs refer to the given resolver. Only used in an assert(). - /// O(N) - bool ResolverHasStubs(JITResolver* Resolver) const { - MutexGuard guard(Lock); - for (std::map<void*, JITResolver*>::const_iterator I = Map.begin(), - E = Map.end(); I != E; ++I) { - if (I->second == Resolver) - return true; - } - return false; - } - }; - /// This needs to be static so that a lazy call stub can access it with no - /// context except the address of the stub. - ManagedStatic<StubToResolverMapTy> StubToResolverMap; - - /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is - /// used to output functions to memory for execution. - class JITEmitter : public JITCodeEmitter { - JITMemoryManager *MemMgr; - - // When outputting a function stub in the context of some other function, we - // save BufferBegin/BufferEnd/CurBufferPtr here. - uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; - - // When reattempting to JIT a function after running out of space, we store - // the estimated size of the function we're trying to JIT here, so we can - // ask the memory manager for at least this much space. When we - // successfully emit the function, we reset this back to zero. - uintptr_t SizeEstimate; - - /// Relocations - These are the relocations that the function needs, as - /// emitted. - std::vector<MachineRelocation> Relocations; - - /// MBBLocations - This vector is a mapping from MBB ID's to their address. - /// It is filled in by the StartMachineBasicBlock callback and queried by - /// the getMachineBasicBlockAddress callback. - std::vector<uintptr_t> MBBLocations; - - /// ConstantPool - The constant pool for the current function. - /// - MachineConstantPool *ConstantPool; - - /// ConstantPoolBase - A pointer to the first entry in the constant pool. - /// - void *ConstantPoolBase; - - /// ConstPoolAddresses - Addresses of individual constant pool entries. - /// - SmallVector<uintptr_t, 8> ConstPoolAddresses; - - /// JumpTable - The jump tables for the current function. - /// - MachineJumpTableInfo *JumpTable; - - /// JumpTableBase - A pointer to the first entry in the jump table. - /// - void *JumpTableBase; - - /// Resolver - This contains info about the currently resolved functions. - JITResolver Resolver; - - /// LabelLocations - This vector is a mapping from Label ID's to their - /// address. - DenseMap<MCSymbol*, uintptr_t> LabelLocations; - - /// MMI - Machine module info for exception informations - MachineModuleInfo* MMI; - - // CurFn - The llvm function being emitted. Only valid during - // finishFunction(). - const Function *CurFn; - - /// Information about emitted code, which is passed to the - /// JITEventListeners. This is reset in startFunction and used in - /// finishFunction. - JITEvent_EmittedFunctionDetails EmissionDetails; - - struct EmittedCode { - void *FunctionBody; // Beginning of the function's allocation. - void *Code; // The address the function's code actually starts at. - void *ExceptionTable; - EmittedCode() : FunctionBody(nullptr), Code(nullptr), - ExceptionTable(nullptr) {} - }; - struct EmittedFunctionConfig : public ValueMapConfig<const Function*> { - typedef JITEmitter *ExtraData; - static void onDelete(JITEmitter *, const Function*); - static void onRAUW(JITEmitter *, const Function*, const Function*); - }; - ValueMap<const Function *, EmittedCode, - EmittedFunctionConfig> EmittedFunctions; - - DebugLoc PrevDL; - - /// Instance of the JIT - JIT *TheJIT; - - public: - JITEmitter(JIT &jit, JITMemoryManager *JMM, TargetMachine &TM) - : SizeEstimate(0), Resolver(jit, *this), MMI(nullptr), CurFn(nullptr), - EmittedFunctions(this), TheJIT(&jit) { - MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager(); - if (jit.getJITInfo().needsGOT()) { - MemMgr->AllocateGOT(); - DEBUG(dbgs() << "JIT is managing a GOT\n"); - } - - } - ~JITEmitter() { - delete MemMgr; - } - - JITResolver &getJITResolver() { return Resolver; } - - void startFunction(MachineFunction &F) override; - bool finishFunction(MachineFunction &F) override; - - void emitConstantPool(MachineConstantPool *MCP); - void initJumpTableInfo(MachineJumpTableInfo *MJTI); - void emitJumpTableInfo(MachineJumpTableInfo *MJTI); - - void startGVStub(const GlobalValue* GV, - unsigned StubSize, unsigned Alignment = 1); - void startGVStub(void *Buffer, unsigned StubSize); - void finishGVStub(); - void *allocIndirectGV(const GlobalValue *GV, const uint8_t *Buffer, - size_t Size, unsigned Alignment) override; - - /// allocateSpace - Reserves space in the current block if any, or - /// allocate a new one of the given size. - void *allocateSpace(uintptr_t Size, unsigned Alignment) override; - - /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace, - /// this method does not allocate memory in the current output buffer, - /// because a global may live longer than the current function. - void *allocateGlobal(uintptr_t Size, unsigned Alignment) override; - - void addRelocation(const MachineRelocation &MR) override { - Relocations.push_back(MR); - } - - void StartMachineBasicBlock(MachineBasicBlock *MBB) override { - if (MBBLocations.size() <= (unsigned)MBB->getNumber()) - MBBLocations.resize((MBB->getNumber()+1)*2); - MBBLocations[MBB->getNumber()] = getCurrentPCValue(); - if (MBB->hasAddressTaken()) - TheJIT->addPointerToBasicBlock(MBB->getBasicBlock(), - (void*)getCurrentPCValue()); - DEBUG(dbgs() << "JIT: Emitting BB" << MBB->getNumber() << " at [" - << (void*) getCurrentPCValue() << "]\n"); - } - - uintptr_t getConstantPoolEntryAddress(unsigned Entry) const override; - uintptr_t getJumpTableEntryAddress(unsigned Entry) const override; - - uintptr_t - getMachineBasicBlockAddress(MachineBasicBlock *MBB) const override { - assert(MBBLocations.size() > (unsigned)MBB->getNumber() && - MBBLocations[MBB->getNumber()] && "MBB not emitted!"); - return MBBLocations[MBB->getNumber()]; - } - - /// retryWithMoreMemory - Log a retry and deallocate all memory for the - /// given function. Increase the minimum allocation size so that we get - /// more memory next time. - void retryWithMoreMemory(MachineFunction &F); - - /// deallocateMemForFunction - Deallocate all memory for the specified - /// function body. - void deallocateMemForFunction(const Function *F); - - void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) override; - - void emitLabel(MCSymbol *Label) override { - LabelLocations[Label] = getCurrentPCValue(); - } - - DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() override { - return &LabelLocations; - } - - uintptr_t getLabelAddress(MCSymbol *Label) const override { - assert(LabelLocations.count(Label) && "Label not emitted!"); - return LabelLocations.find(Label)->second; - } - - void setModuleInfo(MachineModuleInfo* Info) override { - MMI = Info; - } - - private: - void *getPointerToGlobal(GlobalValue *GV, void *Reference, - bool MayNeedFarStub); - void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference); - }; -} - -void CallSiteValueMapConfig::onDelete(JITResolverState *JRS, Function *F) { - JRS->EraseAllCallSitesForPrelocked(F); -} - -void JITResolverState::EraseAllCallSitesForPrelocked(Function *F) { - FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F); - if (F2C == FunctionToCallSitesMap.end()) - return; - StubToResolverMapTy &S2RMap = *StubToResolverMap; - for (SmallPtrSet<void*, 1>::const_iterator I = F2C->second.begin(), - E = F2C->second.end(); I != E; ++I) { - S2RMap.UnregisterStubResolver(*I); - bool Erased = CallSiteToFunctionMap.erase(*I); - (void)Erased; - assert(Erased && "Missing call site->function mapping"); - } - FunctionToCallSitesMap.erase(F2C); -} - -void JITResolverState::EraseAllCallSitesPrelocked() { - StubToResolverMapTy &S2RMap = *StubToResolverMap; - for (CallSiteToFunctionMapTy::const_iterator - I = CallSiteToFunctionMap.begin(), - E = CallSiteToFunctionMap.end(); I != E; ++I) { - S2RMap.UnregisterStubResolver(I->first); - } - CallSiteToFunctionMap.clear(); - FunctionToCallSitesMap.clear(); -} - -JITResolver::~JITResolver() { - // No need to lock because we're in the destructor, and state isn't shared. - state.EraseAllCallSitesPrelocked(); - assert(!StubToResolverMap->ResolverHasStubs(this) && - "Resolver destroyed with stubs still alive."); -} - -/// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub -/// if it has already been created. -void *JITResolver::getLazyFunctionStubIfAvailable(Function *F) { - MutexGuard locked(TheJIT->lock); - - // If we already have a stub for this function, recycle it. - return state.getFunctionToLazyStubMap().lookup(F); -} - -/// getFunctionStub - This returns a pointer to a function stub, creating -/// one on demand as needed. -void *JITResolver::getLazyFunctionStub(Function *F) { - MutexGuard locked(TheJIT->lock); - - // If we already have a lazy stub for this function, recycle it. - void *&Stub = state.getFunctionToLazyStubMap()[F]; - if (Stub) return Stub; - - // Call the lazy resolver function if we are JIT'ing lazily. Otherwise we - // must resolve the symbol now. - void *Actual = TheJIT->isCompilingLazily() - ? (void *)(intptr_t)LazyResolverFn : (void *)nullptr; - - // If this is an external declaration, attempt to resolve the address now - // to place in the stub. - if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) { - Actual = TheJIT->getPointerToFunction(F); - - // If we resolved the symbol to a null address (eg. a weak external) - // don't emit a stub. Return a null pointer to the application. - if (!Actual) return nullptr; - } - - TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); - JE.startGVStub(F, SL.Size, SL.Alignment); - // Codegen a new stub, calling the lazy resolver or the actual address of the - // external function, if it was resolved. - Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, JE); - JE.finishGVStub(); - - if (Actual != (void*)(intptr_t)LazyResolverFn) { - // If we are getting the stub for an external function, we really want the - // address of the stub in the GlobalAddressMap for the JIT, not the address - // of the external function. - TheJIT->updateGlobalMapping(F, Stub); - } - - DEBUG(dbgs() << "JIT: Lazy stub emitted at [" << Stub << "] for function '" - << F->getName() << "'\n"); - - if (TheJIT->isCompilingLazily()) { - // Register this JITResolver as the one corresponding to this call site so - // JITCompilerFn will be able to find it. - StubToResolverMap->RegisterStubResolver(Stub, this); - - // Finally, keep track of the stub-to-Function mapping so that the - // JITCompilerFn knows which function to compile! - state.AddCallSite(Stub, F); - } else if (!Actual) { - // If we are JIT'ing non-lazily but need to call a function that does not - // exist yet, add it to the JIT's work list so that we can fill in the - // stub address later. - assert(!isNonGhostDeclaration(F) && !F->hasAvailableExternallyLinkage() && - "'Actual' should have been set above."); - TheJIT->addPendingFunction(F); - } - - return Stub; -} - -/// getGlobalValueIndirectSym - Return a lazy pointer containing the specified -/// GV address. -void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) { - MutexGuard locked(TheJIT->lock); - - // If we already have a stub for this global variable, recycle it. - void *&IndirectSym = state.getGlobalToIndirectSymMap()[GV]; - if (IndirectSym) return IndirectSym; - - // Otherwise, codegen a new indirect symbol. - IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress, - JE); - - DEBUG(dbgs() << "JIT: Indirect symbol emitted at [" << IndirectSym - << "] for GV '" << GV->getName() << "'\n"); - - return IndirectSym; -} - -/// getExternalFunctionStub - Return a stub for the function at the -/// specified address, created lazily on demand. -void *JITResolver::getExternalFunctionStub(void *FnAddr) { - // If we already have a stub for this function, recycle it. - void *&Stub = ExternalFnToStubMap[FnAddr]; - if (Stub) return Stub; - - TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); - JE.startGVStub(nullptr, SL.Size, SL.Alignment); - Stub = TheJIT->getJITInfo().emitFunctionStub(nullptr, FnAddr, JE); - JE.finishGVStub(); - - DEBUG(dbgs() << "JIT: Stub emitted at [" << Stub - << "] for external function at '" << FnAddr << "'\n"); - return Stub; -} - -unsigned JITResolver::getGOTIndexForAddr(void* addr) { - unsigned idx = revGOTMap[addr]; - if (!idx) { - idx = ++nextGOTIndex; - revGOTMap[addr] = idx; - DEBUG(dbgs() << "JIT: Adding GOT entry " << idx << " for addr [" - << addr << "]\n"); - } - return idx; -} - -/// JITCompilerFn - This function is called when a lazy compilation stub has -/// been entered. It looks up which function this stub corresponds to, compiles -/// it if necessary, then returns the resultant function pointer. -void *JITResolver::JITCompilerFn(void *Stub) { - JITResolver *JR = StubToResolverMap->getResolverFromStub(Stub); - assert(JR && "Unable to find the corresponding JITResolver to the call site"); - - Function* F = nullptr; - void* ActualPtr = nullptr; - - { - // Only lock for getting the Function. The call getPointerToFunction made - // in this function might trigger function materializing, which requires - // JIT lock to be unlocked. - MutexGuard locked(JR->TheJIT->lock); - - // The address given to us for the stub may not be exactly right, it might - // be a little bit after the stub. As such, use upper_bound to find it. - std::pair<void*, Function*> I = - JR->state.LookupFunctionFromCallSite(Stub); - F = I.second; - ActualPtr = I.first; - } - - // If we have already code generated the function, just return the address. - void *Result = JR->TheJIT->getPointerToGlobalIfAvailable(F); - - if (!Result) { - // Otherwise we don't have it, do lazy compilation now. - - // If lazy compilation is disabled, emit a useful error message and abort. - if (!JR->TheJIT->isCompilingLazily()) { - report_fatal_error("LLVM JIT requested to do lazy compilation of" - " function '" - + F->getName() + "' when lazy compiles are disabled!"); - } - - DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName() - << "' In stub ptr = " << Stub << " actual ptr = " - << ActualPtr << "\n"); - (void)ActualPtr; - - Result = JR->TheJIT->getPointerToFunction(F); - } - - // Reacquire the lock to update the GOT map. - MutexGuard locked(JR->TheJIT->lock); - - // We might like to remove the call site from the CallSiteToFunction map, but - // we can't do that! Multiple threads could be stuck, waiting to acquire the - // lock above. As soon as the 1st function finishes compiling the function, - // the next one will be released, and needs to be able to find the function it - // needs to call. - - // FIXME: We could rewrite all references to this stub if we knew them. - - // What we will do is set the compiled function address to map to the - // same GOT entry as the stub so that later clients may update the GOT - // if they see it still using the stub address. - // Note: this is done so the Resolver doesn't have to manage GOT memory - // Do this without allocating map space if the target isn't using a GOT - if(JR->revGOTMap.find(Stub) != JR->revGOTMap.end()) - JR->revGOTMap[Result] = JR->revGOTMap[Stub]; - - return Result; -} - -//===----------------------------------------------------------------------===// -// JITEmitter code. -// - -static GlobalObject *getSimpleAliasee(Constant *C) { - C = C->stripPointerCasts(); - return dyn_cast<GlobalObject>(C); -} - -void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, - bool MayNeedFarStub) { - if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) - return TheJIT->getOrEmitGlobalVariable(GV); - - if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) { - // We can only handle simple cases. - if (GlobalValue *GV = getSimpleAliasee(GA->getAliasee())) - return TheJIT->getPointerToGlobal(GV); - return nullptr; - } - - // If we have already compiled the function, return a pointer to its body. - Function *F = cast<Function>(V); - - void *FnStub = Resolver.getLazyFunctionStubIfAvailable(F); - if (FnStub) { - // Return the function stub if it's already created. We do this first so - // that we're returning the same address for the function as any previous - // call. TODO: Yes, this is wrong. The lazy stub isn't guaranteed to be - // close enough to call. - return FnStub; - } - - // If we know the target can handle arbitrary-distance calls, try to - // return a direct pointer. - if (!MayNeedFarStub) { - // If we have code, go ahead and return that. - void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); - if (ResultPtr) return ResultPtr; - - // If this is an external function pointer, we can force the JIT to - // 'compile' it, which really just adds it to the map. - if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) - return TheJIT->getPointerToFunction(F); - } - - // Otherwise, we may need a to emit a stub, and, conservatively, we always do - // so. Note that it's possible to return null from getLazyFunctionStub in the - // case of a weak extern that fails to resolve. - return Resolver.getLazyFunctionStub(F); -} - -void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) { - // Make sure GV is emitted first, and create a stub containing the fully - // resolved address. - void *GVAddress = getPointerToGlobal(V, Reference, false); - void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress); - return StubAddr; -} - -void JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) { - if (DL.isUnknown()) return; - if (!BeforePrintingInsn) return; - - const LLVMContext &Context = EmissionDetails.MF->getFunction()->getContext(); - - if (DL.getScope(Context) != nullptr && PrevDL != DL) { - JITEvent_EmittedFunctionDetails::LineStart NextLine; - NextLine.Address = getCurrentPCValue(); - NextLine.Loc = DL; - EmissionDetails.LineStarts.push_back(NextLine); - } - - PrevDL = DL; -} - -static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP, - const DataLayout *TD) { - const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); - if (Constants.empty()) return 0; - - unsigned Size = 0; - for (unsigned i = 0, e = Constants.size(); i != e; ++i) { - MachineConstantPoolEntry CPE = Constants[i]; - unsigned AlignMask = CPE.getAlignment() - 1; - Size = (Size + AlignMask) & ~AlignMask; - Type *Ty = CPE.getType(); - Size += TD->getTypeAllocSize(Ty); - } - return Size; -} - -void JITEmitter::startFunction(MachineFunction &F) { - DEBUG(dbgs() << "JIT: Starting CodeGen of Function " - << F.getName() << "\n"); - - uintptr_t ActualSize = 0; - // Set the memory writable, if it's not already - MemMgr->setMemoryWritable(); - - if (SizeEstimate > 0) { - // SizeEstimate will be non-zero on reallocation attempts. - ActualSize = SizeEstimate; - } - - BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(), - ActualSize); - BufferEnd = BufferBegin+ActualSize; - EmittedFunctions[F.getFunction()].FunctionBody = BufferBegin; - - // Ensure the constant pool/jump table info is at least 4-byte aligned. - emitAlignment(16); - - emitConstantPool(F.getConstantPool()); - if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) - initJumpTableInfo(MJTI); - - // About to start emitting the machine code for the function. - emitAlignment(std::max(F.getFunction()->getAlignment(), 8U)); - TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr); - EmittedFunctions[F.getFunction()].Code = CurBufferPtr; - - MBBLocations.clear(); - - EmissionDetails.MF = &F; - EmissionDetails.LineStarts.clear(); -} - -bool JITEmitter::finishFunction(MachineFunction &F) { - if (CurBufferPtr == BufferEnd) { - // We must call endFunctionBody before retrying, because - // deallocateMemForFunction requires it. - MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); - retryWithMoreMemory(F); - return true; - } - - if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) - emitJumpTableInfo(MJTI); - - // FnStart is the start of the text, not the start of the constant pool and - // other per-function data. - uint8_t *FnStart = - (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction()); - - // FnEnd is the end of the function's machine code. - uint8_t *FnEnd = CurBufferPtr; - - if (!Relocations.empty()) { - CurFn = F.getFunction(); - NumRelos += Relocations.size(); - - // Resolve the relocations to concrete pointers. - for (unsigned i = 0, e = Relocations.size(); i != e; ++i) { - MachineRelocation &MR = Relocations[i]; - void *ResultPtr = nullptr; - if (!MR.letTargetResolve()) { - if (MR.isExternalSymbol()) { - ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(), - false); - DEBUG(dbgs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to [" - << ResultPtr << "]\n"); - - // If the target REALLY wants a stub for this function, emit it now. - if (MR.mayNeedFarStub()) { - ResultPtr = Resolver.getExternalFunctionStub(ResultPtr); - } - } else if (MR.isGlobalValue()) { - ResultPtr = getPointerToGlobal(MR.getGlobalValue(), - BufferBegin+MR.getMachineCodeOffset(), - MR.mayNeedFarStub()); - } else if (MR.isIndirectSymbol()) { - ResultPtr = getPointerToGVIndirectSym( - MR.getGlobalValue(), BufferBegin+MR.getMachineCodeOffset()); - } else if (MR.isBasicBlock()) { - ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock()); - } else if (MR.isConstantPoolIndex()) { - ResultPtr = - (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); - } else { - assert(MR.isJumpTableIndex()); - ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex()); - } - - MR.setResultPointer(ResultPtr); - } - - // if we are managing the GOT and the relocation wants an index, - // give it one - if (MR.isGOTRelative() && MemMgr->isManagingGOT()) { - unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr); - MR.setGOTIndex(idx); - if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) { - DEBUG(dbgs() << "JIT: GOT was out of date for " << ResultPtr - << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] - << "\n"); - ((void**)MemMgr->getGOTBase())[idx] = ResultPtr; - } - } - } - - CurFn = nullptr; - TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0], - Relocations.size(), MemMgr->getGOTBase()); - } - - // Update the GOT entry for F to point to the new code. - if (MemMgr->isManagingGOT()) { - unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin); - if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) { - DEBUG(dbgs() << "JIT: GOT was out of date for " << (void*)BufferBegin - << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] - << "\n"); - ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin; - } - } - - // CurBufferPtr may have moved beyond FnEnd, due to memory allocation for - // global variables that were referenced in the relocations. - MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); - - if (CurBufferPtr == BufferEnd) { - retryWithMoreMemory(F); - return true; - } else { - // Now that we've succeeded in emitting the function, reset the - // SizeEstimate back down to zero. - SizeEstimate = 0; - } - - BufferBegin = CurBufferPtr = nullptr; - NumBytes += FnEnd-FnStart; - - // Invalidate the icache if necessary. - sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart); - - TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart, - EmissionDetails); - - // Reset the previous debug location. - PrevDL = DebugLoc(); - - DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart - << "] Function: " << F.getName() - << ": " << (FnEnd-FnStart) << " bytes of text, " - << Relocations.size() << " relocations\n"); - - Relocations.clear(); - ConstPoolAddresses.clear(); - - // Mark code region readable and executable if it's not so already. - MemMgr->setMemoryExecutable(); - - DEBUG({ - if (sys::hasDisassembler()) { - dbgs() << "JIT: Disassembled code:\n"; - dbgs() << sys::disassembleBuffer(FnStart, FnEnd-FnStart, - (uintptr_t)FnStart); - } else { - dbgs() << "JIT: Binary code:\n"; - uint8_t* q = FnStart; - for (int i = 0; q < FnEnd; q += 4, ++i) { - if (i == 4) - i = 0; - if (i == 0) - dbgs() << "JIT: " << (long)(q - FnStart) << ": "; - bool Done = false; - for (int j = 3; j >= 0; --j) { - if (q + j >= FnEnd) - Done = true; - else - dbgs() << (unsigned short)q[j]; - } - if (Done) - break; - dbgs() << ' '; - if (i == 3) - dbgs() << '\n'; - } - dbgs()<< '\n'; - } - }); - - if (MMI) - MMI->EndFunction(); - - return false; -} - -void JITEmitter::retryWithMoreMemory(MachineFunction &F) { - DEBUG(dbgs() << "JIT: Ran out of space for native code. Reattempting.\n"); - Relocations.clear(); // Clear the old relocations or we'll reapply them. - ConstPoolAddresses.clear(); - ++NumRetries; - deallocateMemForFunction(F.getFunction()); - // Try again with at least twice as much free space. - SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin)); - - for (MachineFunction::iterator MBB = F.begin(), E = F.end(); MBB != E; ++MBB){ - if (MBB->hasAddressTaken()) - TheJIT->clearPointerToBasicBlock(MBB->getBasicBlock()); - } -} - -/// deallocateMemForFunction - Deallocate all memory for the specified -/// function body. Also drop any references the function has to stubs. -/// May be called while the Function is being destroyed inside ~Value(). -void JITEmitter::deallocateMemForFunction(const Function *F) { - ValueMap<const Function *, EmittedCode, EmittedFunctionConfig>::iterator - Emitted = EmittedFunctions.find(F); - if (Emitted != EmittedFunctions.end()) { - MemMgr->deallocateFunctionBody(Emitted->second.FunctionBody); - TheJIT->NotifyFreeingMachineCode(Emitted->second.Code); - - EmittedFunctions.erase(Emitted); - } -} - - -void *JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { - if (BufferBegin) - return JITCodeEmitter::allocateSpace(Size, Alignment); - - // create a new memory block if there is no active one. - // care must be taken so that BufferBegin is invalidated when a - // block is trimmed - BufferBegin = CurBufferPtr = MemMgr->allocateSpace(Size, Alignment); - BufferEnd = BufferBegin+Size; - return CurBufferPtr; -} - -void *JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) { - // Delegate this call through the memory manager. - return MemMgr->allocateGlobal(Size, Alignment); -} - -void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { - if (TheJIT->getJITInfo().hasCustomConstantPool()) - return; - - const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); - if (Constants.empty()) return; - - unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getDataLayout()); - unsigned Align = MCP->getConstantPoolAlignment(); - ConstantPoolBase = allocateSpace(Size, Align); - ConstantPool = MCP; - - if (!ConstantPoolBase) return; // Buffer overflow. - - DEBUG(dbgs() << "JIT: Emitted constant pool at [" << ConstantPoolBase - << "] (size: " << Size << ", alignment: " << Align << ")\n"); - - // Initialize the memory for all of the constant pool entries. - unsigned Offset = 0; - for (unsigned i = 0, e = Constants.size(); i != e; ++i) { - MachineConstantPoolEntry CPE = Constants[i]; - unsigned AlignMask = CPE.getAlignment() - 1; - Offset = (Offset + AlignMask) & ~AlignMask; - - uintptr_t CAddr = (uintptr_t)ConstantPoolBase + Offset; - ConstPoolAddresses.push_back(CAddr); - if (CPE.isMachineConstantPoolEntry()) { - // FIXME: add support to lower machine constant pool values into bytes! - report_fatal_error("Initialize memory with machine specific constant pool" - "entry has not been implemented!"); - } - TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr); - DEBUG(dbgs() << "JIT: CP" << i << " at [0x"; - dbgs().write_hex(CAddr) << "]\n"); - - Type *Ty = CPE.Val.ConstVal->getType(); - Offset += TheJIT->getDataLayout()->getTypeAllocSize(Ty); - } -} - -void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) { - if (TheJIT->getJITInfo().hasCustomJumpTables()) - return; - if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline) - return; - - const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); - if (JT.empty()) return; - - unsigned NumEntries = 0; - for (unsigned i = 0, e = JT.size(); i != e; ++i) - NumEntries += JT[i].MBBs.size(); - - unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getDataLayout()); - - // Just allocate space for all the jump tables now. We will fix up the actual - // MBB entries in the tables after we emit the code for each block, since then - // we will know the final locations of the MBBs in memory. - JumpTable = MJTI; - JumpTableBase = allocateSpace(NumEntries * EntrySize, - MJTI->getEntryAlignment(*TheJIT->getDataLayout())); -} - -void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { - if (TheJIT->getJITInfo().hasCustomJumpTables()) - return; - - const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); - if (JT.empty() || !JumpTableBase) return; - - - switch (MJTI->getEntryKind()) { - case MachineJumpTableInfo::EK_Inline: - return; - case MachineJumpTableInfo::EK_BlockAddress: { - // EK_BlockAddress - Each entry is a plain address of block, e.g.: - // .word LBB123 - assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == sizeof(void*) && - "Cross JIT'ing?"); - - // For each jump table, map each target in the jump table to the address of - // an emitted MachineBasicBlock. - intptr_t *SlotPtr = (intptr_t*)JumpTableBase; - - for (unsigned i = 0, e = JT.size(); i != e; ++i) { - const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; - // Store the address of the basic block for this jump table slot in the - // memory we allocated for the jump table in 'initJumpTableInfo' - for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) - *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); - } - break; - } - - case MachineJumpTableInfo::EK_Custom32: - case MachineJumpTableInfo::EK_GPRel32BlockAddress: - case MachineJumpTableInfo::EK_LabelDifference32: { - assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == 4&&"Cross JIT'ing?"); - // For each jump table, place the offset from the beginning of the table - // to the target address. - int *SlotPtr = (int*)JumpTableBase; - - for (unsigned i = 0, e = JT.size(); i != e; ++i) { - const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; - // Store the offset of the basic block for this jump table slot in the - // memory we allocated for the jump table in 'initJumpTableInfo' - uintptr_t Base = (uintptr_t)SlotPtr; - for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) { - uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]); - /// FIXME: USe EntryKind instead of magic "getPICJumpTableEntry" hook. - *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base); - } - } - break; - } - case MachineJumpTableInfo::EK_GPRel64BlockAddress: - llvm_unreachable( - "JT Info emission not implemented for GPRel64BlockAddress yet."); - } -} - -void JITEmitter::startGVStub(const GlobalValue* GV, - unsigned StubSize, unsigned Alignment) { - SavedBufferBegin = BufferBegin; - SavedBufferEnd = BufferEnd; - SavedCurBufferPtr = CurBufferPtr; - - BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment); - BufferEnd = BufferBegin+StubSize+1; -} - -void JITEmitter::startGVStub(void *Buffer, unsigned StubSize) { - SavedBufferBegin = BufferBegin; - SavedBufferEnd = BufferEnd; - SavedCurBufferPtr = CurBufferPtr; - - BufferBegin = CurBufferPtr = (uint8_t *)Buffer; - BufferEnd = BufferBegin+StubSize+1; -} - -void JITEmitter::finishGVStub() { - assert(CurBufferPtr != BufferEnd && "Stub overflowed allocated space."); - NumBytes += getCurrentPCOffset(); - BufferBegin = SavedBufferBegin; - BufferEnd = SavedBufferEnd; - CurBufferPtr = SavedCurBufferPtr; -} - -void *JITEmitter::allocIndirectGV(const GlobalValue *GV, - const uint8_t *Buffer, size_t Size, - unsigned Alignment) { - uint8_t *IndGV = MemMgr->allocateStub(GV, Size, Alignment); - memcpy(IndGV, Buffer, Size); - return IndGV; -} - -// getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry -// in the constant pool that was last emitted with the 'emitConstantPool' -// method. -// -uintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const { - assert(ConstantNum < ConstantPool->getConstants().size() && - "Invalid ConstantPoolIndex!"); - return ConstPoolAddresses[ConstantNum]; -} - -// getJumpTableEntryAddress - Return the address of the JumpTable with index -// 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo' -// -uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const { - const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables(); - assert(Index < JT.size() && "Invalid jump table index!"); - - unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getDataLayout()); - - unsigned Offset = 0; - for (unsigned i = 0; i < Index; ++i) - Offset += JT[i].MBBs.size(); - - Offset *= EntrySize; - - return (uintptr_t)((char *)JumpTableBase + Offset); -} - -void JITEmitter::EmittedFunctionConfig::onDelete( - JITEmitter *Emitter, const Function *F) { - Emitter->deallocateMemForFunction(F); -} -void JITEmitter::EmittedFunctionConfig::onRAUW( - JITEmitter *, const Function*, const Function*) { - llvm_unreachable("The JIT doesn't know how to handle a" - " RAUW on a value it has emitted."); -} - - -//===----------------------------------------------------------------------===// -// Public interface to this file -//===----------------------------------------------------------------------===// - -JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM, - TargetMachine &tm) { - return new JITEmitter(jit, JMM, tm); -} - -// getPointerToFunctionOrStub - If the specified function has been -// code-gen'd, return a pointer to the function. If not, compile it, or use -// a stub to implement lazy compilation if available. -// -void *JIT::getPointerToFunctionOrStub(Function *F) { - // If we have already code generated the function, just return the address. - if (void *Addr = getPointerToGlobalIfAvailable(F)) - return Addr; - - // Get a stub if the target supports it. - JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter()); - return JE->getJITResolver().getLazyFunctionStub(F); -} - -void JIT::updateFunctionStubUnlocked(Function *F) { - // Get the empty stub we generated earlier. - JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter()); - void *Stub = JE->getJITResolver().getLazyFunctionStub(F); - void *Addr = getPointerToGlobalIfAvailable(F); - assert(Addr != Stub && "Function must have non-stub address to be updated."); - - // Tell the target jit info to rewrite the stub at the specified address, - // rather than creating a new one. - TargetJITInfo::StubLayout layout = getJITInfo().getStubLayout(); - JE->startGVStub(Stub, layout.Size); - getJITInfo().emitFunctionStub(F, Addr, *getCodeEmitter()); - JE->finishGVStub(); -} - -/// freeMachineCodeForFunction - release machine code memory for given Function. -/// -void JIT::freeMachineCodeForFunction(Function *F) { - // Delete translation for this from the ExecutionEngine, so it will get - // retranslated next time it is used. - updateGlobalMapping(F, nullptr); - - // Free the actual memory for the function body and related stuff. - static_cast<JITEmitter*>(JCE)->deallocateMemForFunction(F); -} diff --git a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp b/lib/ExecutionEngine/JIT/JITMemoryManager.cpp deleted file mode 100644 index 584b93f..0000000 --- a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp +++ /dev/null @@ -1,904 +0,0 @@ -//===-- JITMemoryManager.cpp - Memory Allocator for JIT'd code ------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines the DefaultJITMemoryManager class. -// -//===----------------------------------------------------------------------===// - -#include "llvm/ExecutionEngine/JITMemoryManager.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/Twine.h" -#include "llvm/Config/config.h" -#include "llvm/IR/GlobalValue.h" -#include "llvm/Support/Allocator.h" -#include "llvm/Support/Compiler.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/DynamicLibrary.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/Memory.h" -#include "llvm/Support/raw_ostream.h" -#include <cassert> -#include <climits> -#include <cstring> -#include <vector> - -#if defined(__linux__) -#if defined(HAVE_SYS_STAT_H) -#include <sys/stat.h> -#endif -#include <fcntl.h> -#include <unistd.h> -#endif - -using namespace llvm; - -#define DEBUG_TYPE "jit" - -STATISTIC(NumSlabs, "Number of slabs of memory allocated by the JIT"); - -JITMemoryManager::~JITMemoryManager() {} - -//===----------------------------------------------------------------------===// -// Memory Block Implementation. -//===----------------------------------------------------------------------===// - -namespace { - /// MemoryRangeHeader - For a range of memory, this is the header that we put - /// on the block of memory. It is carefully crafted to be one word of memory. - /// Allocated blocks have just this header, free'd blocks have FreeRangeHeader - /// which starts with this. - struct FreeRangeHeader; - struct MemoryRangeHeader { - /// ThisAllocated - This is true if this block is currently allocated. If - /// not, this can be converted to a FreeRangeHeader. - unsigned ThisAllocated : 1; - - /// PrevAllocated - Keep track of whether the block immediately before us is - /// allocated. If not, the word immediately before this header is the size - /// of the previous block. - unsigned PrevAllocated : 1; - - /// BlockSize - This is the size in bytes of this memory block, - /// including this header. - uintptr_t BlockSize : (sizeof(intptr_t)*CHAR_BIT - 2); - - - /// getBlockAfter - Return the memory block immediately after this one. - /// - MemoryRangeHeader &getBlockAfter() const { - return *reinterpret_cast<MemoryRangeHeader *>( - reinterpret_cast<char*>( - const_cast<MemoryRangeHeader *>(this))+BlockSize); - } - - /// getFreeBlockBefore - If the block before this one is free, return it, - /// otherwise return null. - FreeRangeHeader *getFreeBlockBefore() const { - if (PrevAllocated) return nullptr; - intptr_t PrevSize = reinterpret_cast<intptr_t *>( - const_cast<MemoryRangeHeader *>(this))[-1]; - return reinterpret_cast<FreeRangeHeader *>( - reinterpret_cast<char*>( - const_cast<MemoryRangeHeader *>(this))-PrevSize); - } - - /// FreeBlock - Turn an allocated block into a free block, adjusting - /// bits in the object headers, and adding an end of region memory block. - FreeRangeHeader *FreeBlock(FreeRangeHeader *FreeList); - - /// TrimAllocationToSize - If this allocated block is significantly larger - /// than NewSize, split it into two pieces (where the former is NewSize - /// bytes, including the header), and add the new block to the free list. - FreeRangeHeader *TrimAllocationToSize(FreeRangeHeader *FreeList, - uint64_t NewSize); - }; - - /// FreeRangeHeader - For a memory block that isn't already allocated, this - /// keeps track of the current block and has a pointer to the next free block. - /// Free blocks are kept on a circularly linked list. - struct FreeRangeHeader : public MemoryRangeHeader { - FreeRangeHeader *Prev; - FreeRangeHeader *Next; - - /// getMinBlockSize - Get the minimum size for a memory block. Blocks - /// smaller than this size cannot be created. - static unsigned getMinBlockSize() { - return sizeof(FreeRangeHeader)+sizeof(intptr_t); - } - - /// SetEndOfBlockSizeMarker - The word at the end of every free block is - /// known to be the size of the free block. Set it for this block. - void SetEndOfBlockSizeMarker() { - void *EndOfBlock = (char*)this + BlockSize; - ((intptr_t *)EndOfBlock)[-1] = BlockSize; - } - - FreeRangeHeader *RemoveFromFreeList() { - assert(Next->Prev == this && Prev->Next == this && "Freelist broken!"); - Next->Prev = Prev; - return Prev->Next = Next; - } - - void AddToFreeList(FreeRangeHeader *FreeList) { - Next = FreeList; - Prev = FreeList->Prev; - Prev->Next = this; - Next->Prev = this; - } - - /// GrowBlock - The block after this block just got deallocated. Merge it - /// into the current block. - void GrowBlock(uintptr_t NewSize); - - /// AllocateBlock - Mark this entire block allocated, updating freelists - /// etc. This returns a pointer to the circular free-list. - FreeRangeHeader *AllocateBlock(); - }; -} - - -/// AllocateBlock - Mark this entire block allocated, updating freelists -/// etc. This returns a pointer to the circular free-list. -FreeRangeHeader *FreeRangeHeader::AllocateBlock() { - assert(!ThisAllocated && !getBlockAfter().PrevAllocated && - "Cannot allocate an allocated block!"); - // Mark this block allocated. - ThisAllocated = 1; - getBlockAfter().PrevAllocated = 1; - - // Remove it from the free list. - return RemoveFromFreeList(); -} - -/// FreeBlock - Turn an allocated block into a free block, adjusting -/// bits in the object headers, and adding an end of region memory block. -/// If possible, coalesce this block with neighboring blocks. Return the -/// FreeRangeHeader to allocate from. -FreeRangeHeader *MemoryRangeHeader::FreeBlock(FreeRangeHeader *FreeList) { - MemoryRangeHeader *FollowingBlock = &getBlockAfter(); - assert(ThisAllocated && "This block is already free!"); - assert(FollowingBlock->PrevAllocated && "Flags out of sync!"); - - FreeRangeHeader *FreeListToReturn = FreeList; - - // If the block after this one is free, merge it into this block. - if (!FollowingBlock->ThisAllocated) { - FreeRangeHeader &FollowingFreeBlock = *(FreeRangeHeader *)FollowingBlock; - // "FreeList" always needs to be a valid free block. If we're about to - // coalesce with it, update our notion of what the free list is. - if (&FollowingFreeBlock == FreeList) { - FreeList = FollowingFreeBlock.Next; - FreeListToReturn = nullptr; - assert(&FollowingFreeBlock != FreeList && "No tombstone block?"); - } - FollowingFreeBlock.RemoveFromFreeList(); - - // Include the following block into this one. - BlockSize += FollowingFreeBlock.BlockSize; - FollowingBlock = &FollowingFreeBlock.getBlockAfter(); - - // Tell the block after the block we are coalescing that this block is - // allocated. - FollowingBlock->PrevAllocated = 1; - } - - assert(FollowingBlock->ThisAllocated && "Missed coalescing?"); - - if (FreeRangeHeader *PrevFreeBlock = getFreeBlockBefore()) { - PrevFreeBlock->GrowBlock(PrevFreeBlock->BlockSize + BlockSize); - return FreeListToReturn ? FreeListToReturn : PrevFreeBlock; - } - - // Otherwise, mark this block free. - FreeRangeHeader &FreeBlock = *(FreeRangeHeader*)this; - FollowingBlock->PrevAllocated = 0; - FreeBlock.ThisAllocated = 0; - - // Link this into the linked list of free blocks. - FreeBlock.AddToFreeList(FreeList); - - // Add a marker at the end of the block, indicating the size of this free - // block. - FreeBlock.SetEndOfBlockSizeMarker(); - return FreeListToReturn ? FreeListToReturn : &FreeBlock; -} - -/// GrowBlock - The block after this block just got deallocated. Merge it -/// into the current block. -void FreeRangeHeader::GrowBlock(uintptr_t NewSize) { - assert(NewSize > BlockSize && "Not growing block?"); - BlockSize = NewSize; - SetEndOfBlockSizeMarker(); - getBlockAfter().PrevAllocated = 0; -} - -/// TrimAllocationToSize - If this allocated block is significantly larger -/// than NewSize, split it into two pieces (where the former is NewSize -/// bytes, including the header), and add the new block to the free list. -FreeRangeHeader *MemoryRangeHeader:: -TrimAllocationToSize(FreeRangeHeader *FreeList, uint64_t NewSize) { - assert(ThisAllocated && getBlockAfter().PrevAllocated && - "Cannot deallocate part of an allocated block!"); - - // Don't allow blocks to be trimmed below minimum required size - NewSize = std::max<uint64_t>(FreeRangeHeader::getMinBlockSize(), NewSize); - - // Round up size for alignment of header. - unsigned HeaderAlign = __alignof(FreeRangeHeader); - NewSize = (NewSize+ (HeaderAlign-1)) & ~(HeaderAlign-1); - - // Size is now the size of the block we will remove from the start of the - // current block. - assert(NewSize <= BlockSize && - "Allocating more space from this block than exists!"); - - // If splitting this block will cause the remainder to be too small, do not - // split the block. - if (BlockSize <= NewSize+FreeRangeHeader::getMinBlockSize()) - return FreeList; - - // Otherwise, we splice the required number of bytes out of this block, form - // a new block immediately after it, then mark this block allocated. - MemoryRangeHeader &FormerNextBlock = getBlockAfter(); - - // Change the size of this block. - BlockSize = NewSize; - - // Get the new block we just sliced out and turn it into a free block. - FreeRangeHeader &NewNextBlock = (FreeRangeHeader &)getBlockAfter(); - NewNextBlock.BlockSize = (char*)&FormerNextBlock - (char*)&NewNextBlock; - NewNextBlock.ThisAllocated = 0; - NewNextBlock.PrevAllocated = 1; - NewNextBlock.SetEndOfBlockSizeMarker(); - FormerNextBlock.PrevAllocated = 0; - NewNextBlock.AddToFreeList(FreeList); - return &NewNextBlock; -} - -//===----------------------------------------------------------------------===// -// Memory Block Implementation. -//===----------------------------------------------------------------------===// - -namespace { - - class DefaultJITMemoryManager; - - class JITAllocator { - DefaultJITMemoryManager &JMM; - public: - JITAllocator(DefaultJITMemoryManager &jmm) : JMM(jmm) { } - void *Allocate(size_t Size, size_t /*Alignment*/); - void Deallocate(void *Slab, size_t Size); - }; - - /// DefaultJITMemoryManager - Manage memory for the JIT code generation. - /// This splits a large block of MAP_NORESERVE'd memory into two - /// sections, one for function stubs, one for the functions themselves. We - /// have to do this because we may need to emit a function stub while in the - /// middle of emitting a function, and we don't know how large the function we - /// are emitting is. - class DefaultJITMemoryManager : public JITMemoryManager { - public: - /// DefaultCodeSlabSize - When we have to go map more memory, we allocate at - /// least this much unless more is requested. Currently, in 512k slabs. - static const size_t DefaultCodeSlabSize = 512 * 1024; - - /// DefaultSlabSize - Allocate globals and stubs into slabs of 64K (probably - /// 16 pages) unless we get an allocation above SizeThreshold. - static const size_t DefaultSlabSize = 64 * 1024; - - /// DefaultSizeThreshold - For any allocation larger than 16K (probably - /// 4 pages), we should allocate a separate slab to avoid wasted space at - /// the end of a normal slab. - static const size_t DefaultSizeThreshold = 16 * 1024; - - private: - // Whether to poison freed memory. - bool PoisonMemory; - - /// LastSlab - This points to the last slab allocated and is used as the - /// NearBlock parameter to AllocateRWX so that we can attempt to lay out all - /// stubs, data, and code contiguously in memory. In general, however, this - /// is not possible because the NearBlock parameter is ignored on Windows - /// platforms and even on Unix it works on a best-effort pasis. - sys::MemoryBlock LastSlab; - - // Memory slabs allocated by the JIT. We refer to them as slabs so we don't - // confuse them with the blocks of memory described above. - std::vector<sys::MemoryBlock> CodeSlabs; - BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize, - DefaultSizeThreshold> StubAllocator; - BumpPtrAllocatorImpl<JITAllocator, DefaultSlabSize, - DefaultSizeThreshold> DataAllocator; - - // Circular list of free blocks. - FreeRangeHeader *FreeMemoryList; - - // When emitting code into a memory block, this is the block. - MemoryRangeHeader *CurBlock; - - uint8_t *GOTBase; // Target Specific reserved memory - public: - DefaultJITMemoryManager(); - ~DefaultJITMemoryManager(); - - /// allocateNewSlab - Allocates a new MemoryBlock and remembers it as the - /// last slab it allocated, so that subsequent allocations follow it. - sys::MemoryBlock allocateNewSlab(size_t size); - - /// getPointerToNamedFunction - This method returns the address of the - /// specified function by using the dlsym function call. - void *getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure = true) override; - - void AllocateGOT() override; - - // Testing methods. - bool CheckInvariants(std::string &ErrorStr) override; - size_t GetDefaultCodeSlabSize() override { return DefaultCodeSlabSize; } - size_t GetDefaultDataSlabSize() override { return DefaultSlabSize; } - size_t GetDefaultStubSlabSize() override { return DefaultSlabSize; } - unsigned GetNumCodeSlabs() override { return CodeSlabs.size(); } - unsigned GetNumDataSlabs() override { return DataAllocator.GetNumSlabs(); } - unsigned GetNumStubSlabs() override { return StubAllocator.GetNumSlabs(); } - - /// startFunctionBody - When a function starts, allocate a block of free - /// executable memory, returning a pointer to it and its actual size. - uint8_t *startFunctionBody(const Function *F, - uintptr_t &ActualSize) override { - - FreeRangeHeader* candidateBlock = FreeMemoryList; - FreeRangeHeader* head = FreeMemoryList; - FreeRangeHeader* iter = head->Next; - - uintptr_t largest = candidateBlock->BlockSize; - - // Search for the largest free block - while (iter != head) { - if (iter->BlockSize > largest) { - largest = iter->BlockSize; - candidateBlock = iter; - } - iter = iter->Next; - } - - largest = largest - sizeof(MemoryRangeHeader); - - // If this block isn't big enough for the allocation desired, allocate - // another block of memory and add it to the free list. - if (largest < ActualSize || - largest <= FreeRangeHeader::getMinBlockSize()) { - DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); - candidateBlock = allocateNewCodeSlab((size_t)ActualSize); - } - - // Select this candidate block for allocation - CurBlock = candidateBlock; - - // Allocate the entire memory block. - FreeMemoryList = candidateBlock->AllocateBlock(); - ActualSize = CurBlock->BlockSize - sizeof(MemoryRangeHeader); - return (uint8_t *)(CurBlock + 1); - } - - /// allocateNewCodeSlab - Helper method to allocate a new slab of code - /// memory from the OS and add it to the free list. Returns the new - /// FreeRangeHeader at the base of the slab. - FreeRangeHeader *allocateNewCodeSlab(size_t MinSize) { - // If the user needs at least MinSize free memory, then we account for - // two MemoryRangeHeaders: the one in the user's block, and the one at the - // end of the slab. - size_t PaddedMin = MinSize + 2 * sizeof(MemoryRangeHeader); - size_t SlabSize = std::max(DefaultCodeSlabSize, PaddedMin); - sys::MemoryBlock B = allocateNewSlab(SlabSize); - CodeSlabs.push_back(B); - char *MemBase = (char*)(B.base()); - - // Put a tiny allocated block at the end of the memory chunk, so when - // FreeBlock calls getBlockAfter it doesn't fall off the end. - MemoryRangeHeader *EndBlock = - (MemoryRangeHeader*)(MemBase + B.size()) - 1; - EndBlock->ThisAllocated = 1; - EndBlock->PrevAllocated = 0; - EndBlock->BlockSize = sizeof(MemoryRangeHeader); - - // Start out with a vast new block of free memory. - FreeRangeHeader *NewBlock = (FreeRangeHeader*)MemBase; - NewBlock->ThisAllocated = 0; - // Make sure getFreeBlockBefore doesn't look into unmapped memory. - NewBlock->PrevAllocated = 1; - NewBlock->BlockSize = (uintptr_t)EndBlock - (uintptr_t)NewBlock; - NewBlock->SetEndOfBlockSizeMarker(); - NewBlock->AddToFreeList(FreeMemoryList); - - assert(NewBlock->BlockSize - sizeof(MemoryRangeHeader) >= MinSize && - "The block was too small!"); - return NewBlock; - } - - /// endFunctionBody - The function F is now allocated, and takes the memory - /// in the range [FunctionStart,FunctionEnd). - void endFunctionBody(const Function *F, uint8_t *FunctionStart, - uint8_t *FunctionEnd) override { - assert(FunctionEnd > FunctionStart); - assert(FunctionStart == (uint8_t *)(CurBlock+1) && - "Mismatched function start/end!"); - - uintptr_t BlockSize = FunctionEnd - (uint8_t *)CurBlock; - - // Release the memory at the end of this block that isn't needed. - FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); - } - - /// allocateSpace - Allocate a memory block of the given size. This method - /// cannot be called between calls to startFunctionBody and endFunctionBody. - uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) override { - CurBlock = FreeMemoryList; - FreeMemoryList = FreeMemoryList->AllocateBlock(); - - uint8_t *result = (uint8_t *)(CurBlock + 1); - - if (Alignment == 0) Alignment = 1; - result = (uint8_t*)(((intptr_t)result+Alignment-1) & - ~(intptr_t)(Alignment-1)); - - uintptr_t BlockSize = result + Size - (uint8_t *)CurBlock; - FreeMemoryList =CurBlock->TrimAllocationToSize(FreeMemoryList, BlockSize); - - return result; - } - - /// allocateStub - Allocate memory for a function stub. - uint8_t *allocateStub(const GlobalValue* F, unsigned StubSize, - unsigned Alignment) override { - return (uint8_t*)StubAllocator.Allocate(StubSize, Alignment); - } - - /// allocateGlobal - Allocate memory for a global. - uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) override { - return (uint8_t*)DataAllocator.Allocate(Size, Alignment); - } - - /// allocateCodeSection - Allocate memory for a code section. - uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, - StringRef SectionName) override { - // Grow the required block size to account for the block header - Size += sizeof(*CurBlock); - - // Alignment handling. - if (!Alignment) - Alignment = 16; - Size += Alignment - 1; - - FreeRangeHeader* candidateBlock = FreeMemoryList; - FreeRangeHeader* head = FreeMemoryList; - FreeRangeHeader* iter = head->Next; - - uintptr_t largest = candidateBlock->BlockSize; - - // Search for the largest free block. - while (iter != head) { - if (iter->BlockSize > largest) { - largest = iter->BlockSize; - candidateBlock = iter; - } - iter = iter->Next; - } - - largest = largest - sizeof(MemoryRangeHeader); - - // If this block isn't big enough for the allocation desired, allocate - // another block of memory and add it to the free list. - if (largest < Size || largest <= FreeRangeHeader::getMinBlockSize()) { - DEBUG(dbgs() << "JIT: Allocating another slab of memory for function."); - candidateBlock = allocateNewCodeSlab((size_t)Size); - } - - // Select this candidate block for allocation - CurBlock = candidateBlock; - - // Allocate the entire memory block. - FreeMemoryList = candidateBlock->AllocateBlock(); - // Release the memory at the end of this block that isn't needed. - FreeMemoryList = CurBlock->TrimAllocationToSize(FreeMemoryList, Size); - uintptr_t unalignedAddr = (uintptr_t)CurBlock + sizeof(*CurBlock); - return (uint8_t*)RoundUpToAlignment((uint64_t)unalignedAddr, Alignment); - } - - /// allocateDataSection - Allocate memory for a data section. - uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, StringRef SectionName, - bool IsReadOnly) override { - return (uint8_t*)DataAllocator.Allocate(Size, Alignment); - } - - bool finalizeMemory(std::string *ErrMsg) override { - return false; - } - - uint8_t *getGOTBase() const override { - return GOTBase; - } - - void deallocateBlock(void *Block) { - // Find the block that is allocated for this function. - MemoryRangeHeader *MemRange = static_cast<MemoryRangeHeader*>(Block) - 1; - assert(MemRange->ThisAllocated && "Block isn't allocated!"); - - // Fill the buffer with garbage! - if (PoisonMemory) { - memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange)); - } - - // Free the memory. - FreeMemoryList = MemRange->FreeBlock(FreeMemoryList); - } - - /// deallocateFunctionBody - Deallocate all memory for the specified - /// function body. - void deallocateFunctionBody(void *Body) override { - if (Body) deallocateBlock(Body); - } - - /// setMemoryWritable - When code generation is in progress, - /// the code pages may need permissions changed. - void setMemoryWritable() override { - for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) - sys::Memory::setWritable(CodeSlabs[i]); - } - /// setMemoryExecutable - When code generation is done and we're ready to - /// start execution, the code pages may need permissions changed. - void setMemoryExecutable() override { - for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) - sys::Memory::setExecutable(CodeSlabs[i]); - } - - /// setPoisonMemory - Controls whether we write garbage over freed memory. - /// - void setPoisonMemory(bool poison) override { - PoisonMemory = poison; - } - }; -} - -void *JITAllocator::Allocate(size_t Size, size_t /*Alignment*/) { - sys::MemoryBlock B = JMM.allocateNewSlab(Size); - return B.base(); -} - -void JITAllocator::Deallocate(void *Slab, size_t Size) { - sys::MemoryBlock B(Slab, Size); - sys::Memory::ReleaseRWX(B); -} - -DefaultJITMemoryManager::DefaultJITMemoryManager() - : -#ifdef NDEBUG - PoisonMemory(false), -#else - PoisonMemory(true), -#endif - LastSlab(nullptr, 0), StubAllocator(*this), DataAllocator(*this) { - - // Allocate space for code. - sys::MemoryBlock MemBlock = allocateNewSlab(DefaultCodeSlabSize); - CodeSlabs.push_back(MemBlock); - uint8_t *MemBase = (uint8_t*)MemBlock.base(); - - // We set up the memory chunk with 4 mem regions, like this: - // [ START - // [ Free #0 ] -> Large space to allocate functions from. - // [ Allocated #1 ] -> Tiny space to separate regions. - // [ Free #2 ] -> Tiny space so there is always at least 1 free block. - // [ Allocated #3 ] -> Tiny space to prevent looking past end of block. - // END ] - // - // The last three blocks are never deallocated or touched. - - // Add MemoryRangeHeader to the end of the memory region, indicating that - // the space after the block of memory is allocated. This is block #3. - MemoryRangeHeader *Mem3 = (MemoryRangeHeader*)(MemBase+MemBlock.size())-1; - Mem3->ThisAllocated = 1; - Mem3->PrevAllocated = 0; - Mem3->BlockSize = sizeof(MemoryRangeHeader); - - /// Add a tiny free region so that the free list always has one entry. - FreeRangeHeader *Mem2 = - (FreeRangeHeader *)(((char*)Mem3)-FreeRangeHeader::getMinBlockSize()); - Mem2->ThisAllocated = 0; - Mem2->PrevAllocated = 1; - Mem2->BlockSize = FreeRangeHeader::getMinBlockSize(); - Mem2->SetEndOfBlockSizeMarker(); - Mem2->Prev = Mem2; // Mem2 *is* the free list for now. - Mem2->Next = Mem2; - - /// Add a tiny allocated region so that Mem2 is never coalesced away. - MemoryRangeHeader *Mem1 = (MemoryRangeHeader*)Mem2-1; - Mem1->ThisAllocated = 1; - Mem1->PrevAllocated = 0; - Mem1->BlockSize = sizeof(MemoryRangeHeader); - - // Add a FreeRangeHeader to the start of the function body region, indicating - // that the space is free. Mark the previous block allocated so we never look - // at it. - FreeRangeHeader *Mem0 = (FreeRangeHeader*)MemBase; - Mem0->ThisAllocated = 0; - Mem0->PrevAllocated = 1; - Mem0->BlockSize = (char*)Mem1-(char*)Mem0; - Mem0->SetEndOfBlockSizeMarker(); - Mem0->AddToFreeList(Mem2); - - // Start out with the freelist pointing to Mem0. - FreeMemoryList = Mem0; - - GOTBase = nullptr; -} - -void DefaultJITMemoryManager::AllocateGOT() { - assert(!GOTBase && "Cannot allocate the got multiple times"); - GOTBase = new uint8_t[sizeof(void*) * 8192]; - HasGOT = true; -} - -DefaultJITMemoryManager::~DefaultJITMemoryManager() { - for (unsigned i = 0, e = CodeSlabs.size(); i != e; ++i) - sys::Memory::ReleaseRWX(CodeSlabs[i]); - - delete[] GOTBase; -} - -sys::MemoryBlock DefaultJITMemoryManager::allocateNewSlab(size_t size) { - // Allocate a new block close to the last one. - std::string ErrMsg; - sys::MemoryBlock *LastSlabPtr = LastSlab.base() ? &LastSlab : nullptr; - sys::MemoryBlock B = sys::Memory::AllocateRWX(size, LastSlabPtr, &ErrMsg); - if (!B.base()) { - report_fatal_error("Allocation failed when allocating new memory in the" - " JIT\n" + Twine(ErrMsg)); - } - LastSlab = B; - ++NumSlabs; - // Initialize the slab to garbage when debugging. - if (PoisonMemory) { - memset(B.base(), 0xCD, B.size()); - } - return B; -} - -/// CheckInvariants - For testing only. Return "" if all internal invariants -/// are preserved, and a helpful error message otherwise. For free and -/// allocated blocks, make sure that adding BlockSize gives a valid block. -/// For free blocks, make sure they're in the free list and that their end of -/// block size marker is correct. This function should return an error before -/// accessing bad memory. This function is defined here instead of in -/// JITMemoryManagerTest.cpp so that we don't have to expose all of the -/// implementation details of DefaultJITMemoryManager. -bool DefaultJITMemoryManager::CheckInvariants(std::string &ErrorStr) { - raw_string_ostream Err(ErrorStr); - - // Construct a the set of FreeRangeHeader pointers so we can query it - // efficiently. - llvm::SmallPtrSet<MemoryRangeHeader*, 16> FreeHdrSet; - FreeRangeHeader* FreeHead = FreeMemoryList; - FreeRangeHeader* FreeRange = FreeHead; - - do { - // Check that the free range pointer is in the blocks we've allocated. - bool Found = false; - for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), - E = CodeSlabs.end(); I != E && !Found; ++I) { - char *Start = (char*)I->base(); - char *End = Start + I->size(); - Found = (Start <= (char*)FreeRange && (char*)FreeRange < End); - } - if (!Found) { - Err << "Corrupt free list; points to " << FreeRange; - return false; - } - - if (FreeRange->Next->Prev != FreeRange) { - Err << "Next and Prev pointers do not match."; - return false; - } - - // Otherwise, add it to the set. - FreeHdrSet.insert(FreeRange); - FreeRange = FreeRange->Next; - } while (FreeRange != FreeHead); - - // Go over each block, and look at each MemoryRangeHeader. - for (std::vector<sys::MemoryBlock>::iterator I = CodeSlabs.begin(), - E = CodeSlabs.end(); I != E; ++I) { - char *Start = (char*)I->base(); - char *End = Start + I->size(); - - // Check each memory range. - for (MemoryRangeHeader *Hdr = (MemoryRangeHeader*)Start, *LastHdr = nullptr; - Start <= (char*)Hdr && (char*)Hdr < End; - Hdr = &Hdr->getBlockAfter()) { - if (Hdr->ThisAllocated == 0) { - // Check that this range is in the free list. - if (!FreeHdrSet.count(Hdr)) { - Err << "Found free header at " << Hdr << " that is not in free list."; - return false; - } - - // Now make sure the size marker at the end of the block is correct. - uintptr_t *Marker = ((uintptr_t*)&Hdr->getBlockAfter()) - 1; - if (!(Start <= (char*)Marker && (char*)Marker < End)) { - Err << "Block size in header points out of current MemoryBlock."; - return false; - } - if (Hdr->BlockSize != *Marker) { - Err << "End of block size marker (" << *Marker << ") " - << "and BlockSize (" << Hdr->BlockSize << ") don't match."; - return false; - } - } - - if (LastHdr && LastHdr->ThisAllocated != Hdr->PrevAllocated) { - Err << "Hdr->PrevAllocated (" << Hdr->PrevAllocated << ") != " - << "LastHdr->ThisAllocated (" << LastHdr->ThisAllocated << ")"; - return false; - } else if (!LastHdr && !Hdr->PrevAllocated) { - Err << "The first header should have PrevAllocated true."; - return false; - } - - // Remember the last header. - LastHdr = Hdr; - } - } - - // All invariants are preserved. - return true; -} - -//===----------------------------------------------------------------------===// -// getPointerToNamedFunction() implementation. -//===----------------------------------------------------------------------===// - -// AtExitHandlers - List of functions to call when the program exits, -// registered with the atexit() library function. -static std::vector<void (*)()> AtExitHandlers; - -/// runAtExitHandlers - Run any functions registered by the program's -/// calls to atexit(3), which we intercept and store in -/// AtExitHandlers. -/// -static void runAtExitHandlers() { - while (!AtExitHandlers.empty()) { - void (*Fn)() = AtExitHandlers.back(); - AtExitHandlers.pop_back(); - Fn(); - } -} - -//===----------------------------------------------------------------------===// -// Function stubs that are invoked instead of certain library calls -// -// Force the following functions to be linked in to anything that uses the -// JIT. This is a hack designed to work around the all-too-clever Glibc -// strategy of making these functions work differently when inlined vs. when -// not inlined, and hiding their real definitions in a separate archive file -// that the dynamic linker can't see. For more info, search for -// 'libc_nonshared.a' on Google, or read http://llvm.org/PR274. -#if defined(__linux__) && defined(__GLIBC__) -/* stat functions are redirecting to __xstat with a version number. On x86-64 - * linking with libc_nonshared.a and -Wl,--export-dynamic doesn't make 'stat' - * available as an exported symbol, so we have to add it explicitly. - */ -namespace { -class StatSymbols { -public: - StatSymbols() { - sys::DynamicLibrary::AddSymbol("stat", (void*)(intptr_t)stat); - sys::DynamicLibrary::AddSymbol("fstat", (void*)(intptr_t)fstat); - sys::DynamicLibrary::AddSymbol("lstat", (void*)(intptr_t)lstat); - sys::DynamicLibrary::AddSymbol("stat64", (void*)(intptr_t)stat64); - sys::DynamicLibrary::AddSymbol("\x1stat64", (void*)(intptr_t)stat64); - sys::DynamicLibrary::AddSymbol("\x1open64", (void*)(intptr_t)open64); - sys::DynamicLibrary::AddSymbol("\x1lseek64", (void*)(intptr_t)lseek64); - sys::DynamicLibrary::AddSymbol("fstat64", (void*)(intptr_t)fstat64); - sys::DynamicLibrary::AddSymbol("lstat64", (void*)(intptr_t)lstat64); - sys::DynamicLibrary::AddSymbol("atexit", (void*)(intptr_t)atexit); - sys::DynamicLibrary::AddSymbol("mknod", (void*)(intptr_t)mknod); - } -}; -} -static StatSymbols initStatSymbols; -#endif // __linux__ - -// jit_exit - Used to intercept the "exit" library call. -static void jit_exit(int Status) { - runAtExitHandlers(); // Run atexit handlers... - exit(Status); -} - -// jit_atexit - Used to intercept the "atexit" library call. -static int jit_atexit(void (*Fn)()) { - AtExitHandlers.push_back(Fn); // Take note of atexit handler... - return 0; // Always successful -} - -static int jit_noop() { - return 0; -} - -//===----------------------------------------------------------------------===// -// -/// getPointerToNamedFunction - This method returns the address of the specified -/// function by using the dynamic loader interface. As such it is only useful -/// for resolving library symbols, not code generated symbols. -/// -void *DefaultJITMemoryManager::getPointerToNamedFunction(const std::string &Name, - bool AbortOnFailure) { - // Check to see if this is one of the functions we want to intercept. Note, - // we cast to intptr_t here to silence a -pedantic warning that complains - // about casting a function pointer to a normal pointer. - if (Name == "exit") return (void*)(intptr_t)&jit_exit; - if (Name == "atexit") return (void*)(intptr_t)&jit_atexit; - - // We should not invoke parent's ctors/dtors from generated main()! - // On Mingw and Cygwin, the symbol __main is resolved to - // callee's(eg. tools/lli) one, to invoke wrong duplicated ctors - // (and register wrong callee's dtors with atexit(3)). - // We expect ExecutionEngine::runStaticConstructorsDestructors() - // is called before ExecutionEngine::runFunctionAsMain() is called. - if (Name == "__main") return (void*)(intptr_t)&jit_noop; - - const char *NameStr = Name.c_str(); - // If this is an asm specifier, skip the sentinal. - if (NameStr[0] == 1) ++NameStr; - - // If it's an external function, look it up in the process image... - void *Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr); - if (Ptr) return Ptr; - - // If it wasn't found and if it starts with an underscore ('_') character, - // try again without the underscore. - if (NameStr[0] == '_') { - Ptr = sys::DynamicLibrary::SearchForAddressOfSymbol(NameStr+1); - if (Ptr) return Ptr; - } - - // Darwin/PPC adds $LDBLStub suffixes to various symbols like printf. These - // are references to hidden visibility symbols that dlsym cannot resolve. - // If we have one of these, strip off $LDBLStub and try again. -#if defined(__APPLE__) && defined(__ppc__) - if (Name.size() > 9 && Name[Name.size()-9] == '$' && - memcmp(&Name[Name.size()-8], "LDBLStub", 8) == 0) { - // First try turning $LDBLStub into $LDBL128. If that fails, strip it off. - // This mirrors logic in libSystemStubs.a. - std::string Prefix = std::string(Name.begin(), Name.end()-9); - if (void *Ptr = getPointerToNamedFunction(Prefix+"$LDBL128", false)) - return Ptr; - if (void *Ptr = getPointerToNamedFunction(Prefix, false)) - return Ptr; - } -#endif - - if (AbortOnFailure) { - report_fatal_error("Program used external function '"+Name+ - "' which could not be resolved!"); - } - return nullptr; -} - - - -JITMemoryManager *JITMemoryManager::CreateDefaultMemManager() { - return new DefaultJITMemoryManager(); -} - -const size_t DefaultJITMemoryManager::DefaultCodeSlabSize; -const size_t DefaultJITMemoryManager::DefaultSlabSize; -const size_t DefaultJITMemoryManager::DefaultSizeThreshold; diff --git a/lib/ExecutionEngine/JIT/LLVMBuild.txt b/lib/ExecutionEngine/JIT/LLVMBuild.txt deleted file mode 100644 index dd22f1b..0000000 --- a/lib/ExecutionEngine/JIT/LLVMBuild.txt +++ /dev/null @@ -1,22 +0,0 @@ -;===- ./lib/ExecutionEngine/JIT/LLVMBuild.txt ------------------*- Conf -*--===; -; -; The LLVM Compiler Infrastructure -; -; This file is distributed under the University of Illinois Open Source -; License. See LICENSE.TXT for details. -; -;===------------------------------------------------------------------------===; -; -; This is an LLVMBuild description file for the components in this subdirectory. -; -; For more information on the LLVMBuild system, please see: -; -; http://llvm.org/docs/LLVMBuild.html -; -;===------------------------------------------------------------------------===; - -[component_0] -type = Library -name = JIT -parent = ExecutionEngine -required_libraries = CodeGen Core ExecutionEngine Support diff --git a/lib/ExecutionEngine/JIT/Makefile b/lib/ExecutionEngine/JIT/Makefile deleted file mode 100644 index aafa3d9..0000000 --- a/lib/ExecutionEngine/JIT/Makefile +++ /dev/null @@ -1,38 +0,0 @@ -##===- lib/ExecutionEngine/JIT/Makefile --------------------*- Makefile -*-===## -# -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. -# -##===----------------------------------------------------------------------===## - -LEVEL = ../../.. -LIBRARYNAME = LLVMJIT - -# Get the $(ARCH) setting -include $(LEVEL)/Makefile.config - -# Enable the X86 JIT if compiling on X86 -ifeq ($(ARCH), x86) - ENABLE_X86_JIT = 1 -endif - -# This flag can also be used on the command line to force inclusion -# of the X86 JIT on non-X86 hosts -ifdef ENABLE_X86_JIT - CPPFLAGS += -DENABLE_X86_JIT -endif - -# Enable the Sparc JIT if compiling on Sparc -ifeq ($(ARCH), Sparc) - ENABLE_SPARC_JIT = 1 -endif - -# This flag can also be used on the command line to force inclusion -# of the Sparc JIT on non-Sparc hosts -ifdef ENABLE_SPARC_JIT - CPPFLAGS += -DENABLE_SPARC_JIT -endif - -include $(LEVEL)/Makefile.common |