/**************************************************************************
 *
 * Copyright 2010 VMware, Inc.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 **************************************************************************/


/**
 * The purpose of this module is to expose LLVM functionality not available
 * through the C++ bindings.
 */


#ifndef __STDC_LIMIT_MACROS
#define __STDC_LIMIT_MACROS
#endif

#ifndef __STDC_CONSTANT_MACROS
#define __STDC_CONSTANT_MACROS
#endif

// Undef these vars just to silence warnings
#undef PACKAGE_BUGREPORT
#undef PACKAGE_NAME
#undef PACKAGE_STRING
#undef PACKAGE_TARNAME
#undef PACKAGE_VERSION


#include <stddef.h>

// Workaround http://llvm.org/PR23628
#if HAVE_LLVM >= 0x0307
#  pragma push_macro("DEBUG")
#  undef DEBUG
#endif

#include <llvm-c/Core.h>
#include <llvm-c/ExecutionEngine.h>
#include <llvm/Target/TargetOptions.h>
#include <llvm/ExecutionEngine/ExecutionEngine.h>
#include <llvm/ADT/Triple.h>
#if HAVE_LLVM >= 0x0307
#include <llvm/Analysis/TargetLibraryInfo.h>
#else
#include <llvm/Target/TargetLibraryInfo.h>
#endif
#if HAVE_LLVM < 0x0306
#include <llvm/ExecutionEngine/JITMemoryManager.h>
#else
#include <llvm/ExecutionEngine/SectionMemoryManager.h>
#endif
#include <llvm/Support/CommandLine.h>
#include <llvm/Support/Host.h>
#include <llvm/Support/PrettyStackTrace.h>

#include <llvm/Support/TargetSelect.h>

#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/Module.h>
#include <llvm/Support/CBindingWrapping.h>

#include <llvm/Config/llvm-config.h>
#if LLVM_USE_INTEL_JITEVENTS
#include <llvm/ExecutionEngine/JITEventListener.h>
#endif

// Workaround http://llvm.org/PR23628
#if HAVE_LLVM >= 0x0307
#  pragma pop_macro("DEBUG")
#endif

#include "c11/threads.h"
#include "os/os_thread.h"
#include "pipe/p_config.h"
#include "util/u_debug.h"
#include "util/u_cpu_detect.h"

#include "lp_bld_misc.h"

namespace {

class LLVMEnsureMultithreaded {
public:
   LLVMEnsureMultithreaded()
   {
      if (!LLVMIsMultithreaded()) {
         LLVMStartMultithreaded();
      }
   }
};

static LLVMEnsureMultithreaded lLVMEnsureMultithreaded;

}

static once_flag init_native_targets_once_flag = ONCE_FLAG_INIT;

static void init_native_targets()
{
   // If we have a native target, initialize it to ensure it is linked in and
   // usable by the JIT.
   llvm::InitializeNativeTarget();

   llvm::InitializeNativeTargetAsmPrinter();

   llvm::InitializeNativeTargetDisassembler();
}

/**
 * The llvm target registry is not thread-safe, so drivers and state-trackers
 * that want to initialize targets should use the gallivm_init_llvm_targets()
 * function to safely initialize targets.
 *
 * LLVM targets should be initialized before the driver or state-tracker tries
 * to access the registry.
 */
extern "C" void
gallivm_init_llvm_targets(void)
{
   call_once(&init_native_targets_once_flag, init_native_targets);
}

extern "C" void
lp_set_target_options(void)
{
#if HAVE_LLVM < 0x0304
   /*
    * By default LLVM adds a signal handler to output a pretty stack trace.
    * This signal handler is never removed, causing problems when unloading the
    * shared object where the gallium driver resides.
    */
   llvm::DisablePrettyStackTrace = true;
#endif

   gallivm_init_llvm_targets();
}

extern "C"
LLVMTargetLibraryInfoRef
gallivm_create_target_library_info(const char *triple)
{
   return reinterpret_cast<LLVMTargetLibraryInfoRef>(
#if HAVE_LLVM < 0x0307
   new llvm::TargetLibraryInfo(
#else
   new llvm::TargetLibraryInfoImpl(
#endif
   llvm::Triple(triple)));
}

extern "C"
void
gallivm_dispose_target_library_info(LLVMTargetLibraryInfoRef library_info)
{
   delete reinterpret_cast<
#if HAVE_LLVM < 0x0307
   llvm::TargetLibraryInfo
#else
   llvm::TargetLibraryInfoImpl
#endif
   *>(library_info);
}


#if HAVE_LLVM < 0x0304

extern "C"
void
LLVMSetAlignmentBackport(LLVMValueRef V,
                         unsigned Bytes)
{
   switch (LLVMGetInstructionOpcode(V)) {
   case LLVMLoad:
      llvm::unwrap<llvm::LoadInst>(V)->setAlignment(Bytes);
      break;
   case LLVMStore:
      llvm::unwrap<llvm::StoreInst>(V)->setAlignment(Bytes);
      break;
   default:
      assert(0);
      break;
   }
}

#endif


#if HAVE_LLVM < 0x0306
typedef llvm::JITMemoryManager BaseMemoryManager;
#else
typedef llvm::RTDyldMemoryManager BaseMemoryManager;
#endif


/*
 * Delegating is tedious but the default manager class is hidden in an
 * anonymous namespace in LLVM, so we cannot just derive from it to change
 * its behavior.
 */
class DelegatingJITMemoryManager : public BaseMemoryManager {

   protected:
      virtual BaseMemoryManager *mgr() const = 0;

   public:
#if HAVE_LLVM < 0x0306
      /*
       * From JITMemoryManager
       */
      virtual void setMemoryWritable() {
         mgr()->setMemoryWritable();
      }
      virtual void setMemoryExecutable() {
         mgr()->setMemoryExecutable();
      }
      virtual void setPoisonMemory(bool poison) {
         mgr()->setPoisonMemory(poison);
      }
      virtual void AllocateGOT() {
         mgr()->AllocateGOT();
         /*
          * isManagingGOT() is not virtual in base class so we can't delegate.
          * Instead we mirror the value of HasGOT in our instance.
          */
         HasGOT = mgr()->isManagingGOT();
      }
      virtual uint8_t *getGOTBase() const {
         return mgr()->getGOTBase();
      }
      virtual uint8_t *startFunctionBody(const llvm::Function *F,
                                         uintptr_t &ActualSize) {
         return mgr()->startFunctionBody(F, ActualSize);
      }
      virtual uint8_t *allocateStub(const llvm::GlobalValue *F,
                                    unsigned StubSize,
                                    unsigned Alignment) {
         return mgr()->allocateStub(F, StubSize, Alignment);
      }
      virtual void endFunctionBody(const llvm::Function *F,
                                   uint8_t *FunctionStart,
                                   uint8_t *FunctionEnd) {
         mgr()->endFunctionBody(F, FunctionStart, FunctionEnd);
      }
      virtual uint8_t *allocateSpace(intptr_t Size, unsigned Alignment) {
         return mgr()->allocateSpace(Size, Alignment);
      }
      virtual uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) {
         return mgr()->allocateGlobal(Size, Alignment);
      }
      virtual void deallocateFunctionBody(void *Body) {
         mgr()->deallocateFunctionBody(Body);
      }
#if HAVE_LLVM < 0x0304
      virtual uint8_t *startExceptionTable(const llvm::Function *F,
                                           uintptr_t &ActualSize) {
         return mgr()->startExceptionTable(F, ActualSize);
      }
      virtual void endExceptionTable(const llvm::Function *F,
                                     uint8_t *TableStart,
                                     uint8_t *TableEnd,
                                     uint8_t *FrameRegister) {
         mgr()->endExceptionTable(F, TableStart, TableEnd,
                                  FrameRegister);
      }
      virtual void deallocateExceptionTable(void *ET) {
         mgr()->deallocateExceptionTable(ET);
      }
#endif
      virtual bool CheckInvariants(std::string &s) {
         return mgr()->CheckInvariants(s);
      }
      virtual size_t GetDefaultCodeSlabSize() {
         return mgr()->GetDefaultCodeSlabSize();
      }
      virtual size_t GetDefaultDataSlabSize() {
         return mgr()->GetDefaultDataSlabSize();
      }
      virtual size_t GetDefaultStubSlabSize() {
         return mgr()->GetDefaultStubSlabSize();
      }
      virtual unsigned GetNumCodeSlabs() {
         return mgr()->GetNumCodeSlabs();
      }
      virtual unsigned GetNumDataSlabs() {
         return mgr()->GetNumDataSlabs();
      }
      virtual unsigned GetNumStubSlabs() {
         return mgr()->GetNumStubSlabs();
      }
#endif

      /*
       * From RTDyldMemoryManager
       */
#if HAVE_LLVM >= 0x0304
      virtual uint8_t *allocateCodeSection(uintptr_t Size,
                                           unsigned Alignment,
                                           unsigned SectionID,
                                           llvm::StringRef SectionName) {
         return mgr()->allocateCodeSection(Size, Alignment, SectionID,
                                           SectionName);
      }
#else
      virtual uint8_t *allocateCodeSection(uintptr_t Size,
                                           unsigned Alignment,
                                           unsigned SectionID) {
         return mgr()->allocateCodeSection(Size, Alignment, SectionID);
      }
#endif
      virtual uint8_t *allocateDataSection(uintptr_t Size,
                                           unsigned Alignment,
                                           unsigned SectionID,
#if HAVE_LLVM >= 0x0304
                                           llvm::StringRef SectionName,
#endif
                                           bool IsReadOnly) {
         return mgr()->allocateDataSection(Size, Alignment, SectionID,
#if HAVE_LLVM >= 0x0304
                                           SectionName,
#endif
                                           IsReadOnly);
      }
#if HAVE_LLVM >= 0x0304
      virtual void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) {
         mgr()->registerEHFrames(Addr, LoadAddr, Size);
      }
      virtual void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) {
         mgr()->deregisterEHFrames(Addr, LoadAddr, Size);
      }
#else
      virtual void registerEHFrames(llvm::StringRef SectionData) {
         mgr()->registerEHFrames(SectionData);
      }
#endif
      virtual void *getPointerToNamedFunction(const std::string &Name,
                                              bool AbortOnFailure=true) {
         return mgr()->getPointerToNamedFunction(Name, AbortOnFailure);
      }
#if HAVE_LLVM <= 0x0303
      virtual bool applyPermissions(std::string *ErrMsg = 0) {
         return mgr()->applyPermissions(ErrMsg);
      }
#else
      virtual bool finalizeMemory(std::string *ErrMsg = 0) {
         return mgr()->finalizeMemory(ErrMsg);
      }
#endif
};


/*
 * Delegate memory management to one shared manager for more efficient use
 * of memory than creating a separate pool for each LLVM engine.
 * Keep generated code until freeGeneratedCode() is called, instead of when
 * memory manager is destroyed, which happens during engine destruction.
 * This allows additional memory savings as we don't have to keep the engine
 * around in order to use the code.
 * All methods are delegated to the shared manager except destruction and
 * deallocating code.  For the latter we just remember what needs to be
 * deallocated later.  The shared manager is deleted once it is empty.
 */
class ShaderMemoryManager : public DelegatingJITMemoryManager {

   BaseMemoryManager *TheMM;

   struct GeneratedCode {
      typedef std::vector<void *> Vec;
      Vec FunctionBody, ExceptionTable;
      BaseMemoryManager *TheMM;

      GeneratedCode(BaseMemoryManager *MM) {
         TheMM = MM;
      }

      ~GeneratedCode() {
         /*
          * Deallocate things as previously requested and
          * free shared manager when no longer used.
          */
#if HAVE_LLVM < 0x0306
         Vec::iterator i;

         assert(TheMM);
         for ( i = FunctionBody.begin(); i != FunctionBody.end(); ++i )
            TheMM->deallocateFunctionBody(*i);
#if HAVE_LLVM < 0x0304
         for ( i = ExceptionTable.begin(); i != ExceptionTable.end(); ++i )
            TheMM->deallocateExceptionTable(*i);
#endif /* HAVE_LLVM < 0x0304 */
#endif /* HAVE_LLVM < 0x0306 */
      }
   };

   GeneratedCode *code;

   BaseMemoryManager *mgr() const {
      return TheMM;
   }

   public:

      ShaderMemoryManager(BaseMemoryManager* MM) {
         TheMM = MM;
         code = new GeneratedCode(MM);
      }

      virtual ~ShaderMemoryManager() {
         /*
          * 'code' is purposely not deleted.  It is the user's responsibility
          * to call getGeneratedCode() and freeGeneratedCode().
          */
      }

      struct lp_generated_code *getGeneratedCode() {
         return (struct lp_generated_code *) code;
      }

      static void freeGeneratedCode(struct lp_generated_code *code) {
         delete (GeneratedCode *) code;
      }

#if HAVE_LLVM < 0x0304
      virtual void deallocateExceptionTable(void *ET) {
         // remember for later deallocation
         code->ExceptionTable.push_back(ET);
      }
#endif

      virtual void deallocateFunctionBody(void *Body) {
         // remember for later deallocation
         code->FunctionBody.push_back(Body);
      }
};


/**
 * Same as LLVMCreateJITCompilerForModule, but:
 * - allows using MCJIT and enabling AVX feature where available.
 * - set target options
 *
 * See also:
 * - llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp
 * - llvm/tools/lli/lli.cpp
 * - http://markmail.org/message/ttkuhvgj4cxxy2on#query:+page:1+mid:aju2dggerju3ivd3+state:results
 */
extern "C"
LLVMBool
lp_build_create_jit_compiler_for_module(LLVMExecutionEngineRef *OutJIT,
                                        lp_generated_code **OutCode,
                                        LLVMModuleRef M,
                                        LLVMMCJITMemoryManagerRef CMM,
                                        unsigned OptLevel,
                                        int useMCJIT,
                                        char **OutError)
{
   using namespace llvm;

   std::string Error;
#if HAVE_LLVM >= 0x0306
   EngineBuilder builder(std::unique_ptr<Module>(unwrap(M)));
#else
   EngineBuilder builder(unwrap(M));
#endif

   /**
    * LLVM 3.1+ haven't more "extern unsigned llvm::StackAlignmentOverride" and
    * friends for configuring code generation options, like stack alignment.
    */
   TargetOptions options;
#if defined(PIPE_ARCH_X86)
   options.StackAlignmentOverride = 4;
#if HAVE_LLVM < 0x0304
   options.RealignStack = true;
#endif
#endif

#if defined(DEBUG) && HAVE_LLVM < 0x0307
   options.JITEmitDebugInfo = true;
#endif

   /* XXX: Workaround http://llvm.org/PR21435 */
#if defined(DEBUG) || defined(PROFILE) || \
    (HAVE_LLVM >= 0x0303 && (defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)))
#if HAVE_LLVM < 0x0304
   options.NoFramePointerElimNonLeaf = true;
#endif
#if HAVE_LLVM < 0x0307
   options.NoFramePointerElim = true;
#endif
#endif

   builder.setEngineKind(EngineKind::JIT)
          .setErrorStr(&Error)
          .setTargetOptions(options)
          .setOptLevel((CodeGenOpt::Level)OptLevel);

   if (useMCJIT) {
#if HAVE_LLVM < 0x0306
       builder.setUseMCJIT(true);
#endif
#ifdef _WIN32
       /*
        * MCJIT works on Windows, but currently only through ELF object format.
        *
        * XXX: We could use `LLVM_HOST_TRIPLE "-elf"` but LLVM_HOST_TRIPLE has
        * different strings for MinGW/MSVC, so better play it safe and be
        * explicit.
        */
#  ifdef _WIN64
       LLVMSetTarget(M, "x86_64-pc-win32-elf");
#  else
       LLVMSetTarget(M, "i686-pc-win32-elf");
#  endif
#endif
   }

   llvm::SmallVector<std::string, 16> MAttrs;

#if defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
   /*
    * We need to unset attributes because sometimes LLVM mistakenly assumes
    * certain features are present given the processor name.
    *
    * https://bugs.freedesktop.org/show_bug.cgi?id=92214
    * http://llvm.org/PR25021
    * http://llvm.org/PR19429
    * http://llvm.org/PR16721
    */
   MAttrs.push_back(util_cpu_caps.has_sse    ? "+sse"    : "-sse"   );
   MAttrs.push_back(util_cpu_caps.has_sse2   ? "+sse2"   : "-sse2"  );
   MAttrs.push_back(util_cpu_caps.has_sse3   ? "+sse3"   : "-sse3"  );
   MAttrs.push_back(util_cpu_caps.has_ssse3  ? "+ssse3"  : "-ssse3" );
#if HAVE_LLVM >= 0x0304
   MAttrs.push_back(util_cpu_caps.has_sse4_1 ? "+sse4.1" : "-sse4.1");
#else
   MAttrs.push_back(util_cpu_caps.has_sse4_1 ? "+sse41"  : "-sse41" );
#endif
#if HAVE_LLVM >= 0x0304
   MAttrs.push_back(util_cpu_caps.has_sse4_2 ? "+sse4.2" : "-sse4.2");
#else
   MAttrs.push_back(util_cpu_caps.has_sse4_2 ? "+sse42"  : "-sse42" );
#endif
   /*
    * AVX feature is not automatically detected from CPUID by the X86 target
    * yet, because the old (yet default) JIT engine is not capable of
    * emitting the opcodes. On newer llvm versions it is and at least some
    * versions (tested with 3.3) will emit avx opcodes without this anyway.
    */
   MAttrs.push_back(util_cpu_caps.has_avx  ? "+avx"  : "-avx");
   MAttrs.push_back(util_cpu_caps.has_f16c ? "+f16c" : "-f16c");
   if (HAVE_LLVM >= 0x0304) {
      MAttrs.push_back(util_cpu_caps.has_fma  ? "+fma"  : "-fma");
   } else {
      /*
       * The old JIT in LLVM 3.3 has a bug encoding llvm.fmuladd.f32 and
       * llvm.fmuladd.v2f32 intrinsics when FMA is available.
       */
      MAttrs.push_back("-fma");
   }
   MAttrs.push_back(util_cpu_caps.has_avx2 ? "+avx2" : "-avx2");
   /* disable avx512 and all subvariants */
#if HAVE_LLVM >= 0x0304
   MAttrs.push_back("-avx512cd");
   MAttrs.push_back("-avx512er");
   MAttrs.push_back("-avx512f");
   MAttrs.push_back("-avx512pf");
#endif
#if HAVE_LLVM >= 0x0305
   MAttrs.push_back("-avx512bw");
   MAttrs.push_back("-avx512dq");
   MAttrs.push_back("-avx512vl");
#endif
#endif

#if defined(PIPE_ARCH_PPC)
   MAttrs.push_back(util_cpu_caps.has_altivec ? "+altivec" : "-altivec");
#if HAVE_LLVM >= 0x0304
   /*
    * Make sure VSX instructions are disabled
    * See LLVM bug https://llvm.org/bugs/show_bug.cgi?id=25503#c7
    */
   if (util_cpu_caps.has_altivec) {
      MAttrs.push_back("-vsx");
   }
#endif
#endif

   builder.setMAttrs(MAttrs);

#if HAVE_LLVM >= 0x0305
   StringRef MCPU = llvm::sys::getHostCPUName();
   /*
    * The cpu bits are no longer set automatically, so need to set mcpu manually.
    * Note that the MAttrs set above will be sort of ignored (since we should
    * not set any which would not be set by specifying the cpu anyway).
    * It ought to be safe though since getHostCPUName() should include bits
    * not only from the cpu but environment as well (for instance if it's safe
    * to use avx instructions which need OS support). According to
    * http://llvm.org/bugs/show_bug.cgi?id=19429 however if I understand this
    * right it may be necessary to specify older cpu (or disable mattrs) though
    * when not using MCJIT so no instructions are generated which the old JIT
    * can't handle. Not entirely sure if we really need to do anything yet.
    */
   builder.setMCPU(MCPU);
#endif

   ShaderMemoryManager *MM = NULL;
   if (useMCJIT) {
       BaseMemoryManager* JMM = reinterpret_cast<BaseMemoryManager*>(CMM);
       MM = new ShaderMemoryManager(JMM);
       *OutCode = MM->getGeneratedCode();

#if HAVE_LLVM >= 0x0306
       builder.setMCJITMemoryManager(std::unique_ptr<RTDyldMemoryManager>(MM));
       MM = NULL; // ownership taken by std::unique_ptr
#elif HAVE_LLVM > 0x0303
       builder.setMCJITMemoryManager(MM);
#else
       builder.setJITMemoryManager(MM);
#endif
   } else {
#if HAVE_LLVM < 0x0306
       BaseMemoryManager* JMM = reinterpret_cast<BaseMemoryManager*>(CMM);
       MM = new ShaderMemoryManager(JMM);
       *OutCode = MM->getGeneratedCode();

       builder.setJITMemoryManager(MM);
#else
       assert(0);
#endif
   }

   ExecutionEngine *JIT;

   JIT = builder.create();
#if LLVM_USE_INTEL_JITEVENTS
   JITEventListener *JEL = JITEventListener::createIntelJITEventListener();
   JIT->RegisterJITEventListener(JEL);
#endif
   if (JIT) {
      *OutJIT = wrap(JIT);
      return 0;
   }
   lp_free_generated_code(*OutCode);
   *OutCode = 0;
   delete MM;
   *OutError = strdup(Error.c_str());
   return 1;
}


extern "C"
void
lp_free_generated_code(struct lp_generated_code *code)
{
   ShaderMemoryManager::freeGeneratedCode(code);
}

extern "C"
LLVMMCJITMemoryManagerRef
lp_get_default_memory_manager()
{
   BaseMemoryManager *mm;
#if HAVE_LLVM < 0x0306
   mm = llvm::JITMemoryManager::CreateDefaultMemManager();
#else
   mm = new llvm::SectionMemoryManager();
#endif
   return reinterpret_cast<LLVMMCJITMemoryManagerRef>(mm);
}

extern "C"
void
lp_free_memory_manager(LLVMMCJITMemoryManagerRef memorymgr)
{
   delete reinterpret_cast<BaseMemoryManager*>(memorymgr);
}

extern "C" void
lp_add_attr_dereferenceable(LLVMValueRef val, uint64_t bytes)
{
#if HAVE_LLVM >= 0x0306
   llvm::Argument *A = llvm::unwrap<llvm::Argument>(val);
   llvm::AttrBuilder B;
   B.addDereferenceableAttr(bytes);
   A->addAttr(llvm::AttributeSet::get(A->getContext(), A->getArgNo() + 1,  B));
#endif
}