diff options
Diffstat (limited to 'src/gallium/auxiliary/gallivm/lp_bld_init.c')
| -rw-r--r-- | src/gallium/auxiliary/gallivm/lp_bld_init.c | 488 |
1 files changed, 334 insertions, 154 deletions
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_init.c b/src/gallium/auxiliary/gallivm/lp_bld_init.c index 768d935..5bf4bcf 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_init.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_init.c @@ -26,15 +26,44 @@ **************************************************************************/ +#include "pipe/p_config.h" #include "pipe/p_compiler.h" #include "util/u_cpu_detect.h" #include "util/u_debug.h" #include "util/u_memory.h" #include "util/u_simple_list.h" +#include "lp_bld.h" #include "lp_bld_debug.h" +#include "lp_bld_misc.h" #include "lp_bld_init.h" +#include <llvm-c/Analysis.h> #include <llvm-c/Transforms/Scalar.h> +#include <llvm-c/BitWriter.h> + + +/** + * AVX is supported in: + * - standard JIT from LLVM 3.2 onwards + * - MC-JIT from LLVM 3.1 + * - MC-JIT supports limited OSes (MacOSX and Linux) + * - standard JIT in LLVM 3.1, with backports + */ +#if HAVE_LLVM >= 0x0301 && (defined(PIPE_OS_LINUX) || defined(PIPE_OS_APPLE)) +# define USE_MCJIT 1 +# define HAVE_AVX 1 +#elif HAVE_LLVM >= 0x0302 || (HAVE_LLVM == 0x0301 && defined(HAVE_JIT_AVX_SUPPORT)) +# define USE_MCJIT 0 +# define HAVE_AVX 1 +#else +# define USE_MCJIT 0 +# define HAVE_AVX 0 +#endif + + +#if USE_MCJIT +void LLVMLinkInMCJIT(); +#endif #ifdef DEBUG @@ -57,6 +86,8 @@ DEBUG_GET_ONCE_FLAGS_OPTION(gallivm_debug, "GALLIVM_DEBUG", lp_bld_debug_flags, static boolean gallivm_initialized = FALSE; +unsigned lp_native_vector_width; + /* * Optimization values are: @@ -81,25 +112,13 @@ enum LLVM_CodeGenOpt_Level { }; +#if HAVE_LLVM <= 0x0206 /** - * LLVM 2.6 permits only one ExecutionEngine to be created. This is it. - */ -static LLVMExecutionEngineRef GlobalEngine = NULL; - -/** - * Same gallivm state shared by all contexts. + * LLVM 2.6 permits only one ExecutionEngine to be created. So use the + * same gallivm state everywhere. */ static struct gallivm_state *GlobalGallivm = NULL; - - - - -extern void -lp_register_oprofile_jit_event_listener(LLVMExecutionEngineRef EE); - -extern void -lp_set_target_options(void); - +#endif /** @@ -111,6 +130,7 @@ static boolean create_pass_manager(struct gallivm_state *gallivm) { assert(!gallivm->passmgr); + assert(gallivm->target); gallivm->passmgr = LLVMCreateFunctionPassManager(gallivm->provider); if (!gallivm->passmgr) @@ -174,33 +194,37 @@ free_gallivm_state(struct gallivm_state *gallivm) &mod, &error); #endif + if (gallivm->passmgr) { + LLVMDisposePassManager(gallivm->passmgr); + } + #if 0 /* XXX this seems to crash with all versions of LLVM */ if (gallivm->provider) LLVMDisposeModuleProvider(gallivm->provider); #endif - if (gallivm->passmgr) - LLVMDisposePassManager(gallivm->passmgr); - -#if HAVE_LLVM >= 0x207 - if (gallivm->module) - LLVMDisposeModule(gallivm->module); -#endif - -#if 0 - /* Don't free the exec engine, it's a global/singleton */ - if (gallivm->engine) + if (HAVE_LLVM >= 0x207 && gallivm->engine) { + /* This will already destroy any associated module */ LLVMDisposeExecutionEngine(gallivm->engine); -#endif + } else { + LLVMDisposeModule(gallivm->module); + } -#if 0 +#if !USE_MCJIT /* Don't free the TargetData, it's owned by the exec engine */ - LLVMDisposeTargetData(gallivm->target); +#else + if (gallivm->target) { + LLVMDisposeTargetData(gallivm->target); + } #endif + /* Never free the LLVM context. + */ +#if 0 if (gallivm->context) LLVMContextDispose(gallivm->context); +#endif if (gallivm->builder) LLVMDisposeBuilder(gallivm->builder); @@ -215,37 +239,14 @@ free_gallivm_state(struct gallivm_state *gallivm) } -/** - * Allocate gallivm LLVM objects. - * \return TRUE for success, FALSE for failure - */ static boolean -init_gallivm_state(struct gallivm_state *gallivm) +init_gallivm_engine(struct gallivm_state *gallivm) { - assert(!gallivm->context); - assert(!gallivm->module); - assert(!gallivm->provider); - - lp_build_init(); - - gallivm->context = LLVMContextCreate(); - if (!gallivm->context) - goto fail; - - gallivm->module = LLVMModuleCreateWithNameInContext("gallivm", - gallivm->context); - if (!gallivm->module) - goto fail; - - gallivm->provider = - LLVMCreateModuleProviderForExistingModule(gallivm->module); - if (!gallivm->provider) - goto fail; - - if (!GlobalEngine) { + if (1) { /* We can only create one LLVMExecutionEngine (w/ LLVM 2.6 anyway) */ enum LLVM_CodeGenOpt_Level optlevel; char *error = NULL; + int ret; if (gallivm_debug & GALLIVM_DEBUG_NO_OPT) { optlevel = None; @@ -254,135 +255,162 @@ init_gallivm_state(struct gallivm_state *gallivm) optlevel = Default; } - if (LLVMCreateJITCompiler(&GlobalEngine, gallivm->provider, - (unsigned) optlevel, &error)) { +#if USE_MCJIT + ret = lp_build_create_mcjit_compiler_for_module(&gallivm->engine, + gallivm->module, + (unsigned) optlevel, + &error); +#else + ret = LLVMCreateJITCompiler(&gallivm->engine, gallivm->provider, + (unsigned) optlevel, &error); +#endif + if (ret) { _debug_printf("%s\n", error); LLVMDisposeMessage(error); goto fail; } #if defined(DEBUG) || defined(PROFILE) - lp_register_oprofile_jit_event_listener(GlobalEngine); + lp_register_oprofile_jit_event_listener(gallivm->engine); #endif } - gallivm->engine = GlobalEngine; - LLVMAddModuleProvider(gallivm->engine, gallivm->provider);//new +#if !USE_MCJIT gallivm->target = LLVMGetExecutionEngineTargetData(gallivm->engine); if (!gallivm->target) goto fail; +#else + if (0) { + /* + * Dump the data layout strings. + */ - if (!create_pass_manager(gallivm)) - goto fail; + LLVMTargetDataRef target = LLVMGetExecutionEngineTargetData(gallivm->engine); + char *data_layout; + char *engine_data_layout; - gallivm->builder = LLVMCreateBuilderInContext(gallivm->context); - if (!gallivm->builder) - goto fail; + data_layout = LLVMCopyStringRepOfTargetData(gallivm->target); + engine_data_layout = LLVMCopyStringRepOfTargetData(target); + + if (1) { + debug_printf("module target data = %s\n", data_layout); + debug_printf("engine target data = %s\n", engine_data_layout); + } + + free(data_layout); + free(engine_data_layout); + } +#endif return TRUE; fail: - free_gallivm_state(gallivm); return FALSE; } -struct callback -{ - garbage_collect_callback_func func; - void *cb_data; - struct callback *prev, *next; -}; - - -/** list of all garbage collector callbacks */ -static struct callback callback_list = {NULL, NULL, NULL, NULL}; +/** + * Singleton + * + * We must never free LLVM contexts, because LLVM has several global caches + * which pointing/derived from objects owned by the context, causing false + * memory leaks and false cache hits when these objects are destroyed. + * + * TODO: For thread safety on multi-threaded OpenGL we should use one LLVM + * context per thread, and put them in a pool when threads are destroyed. + */ +static LLVMContextRef gallivm_context = NULL; /** - * Register a function with gallivm which will be called when we - * do garbage collection. + * Allocate gallivm LLVM objects. + * \return TRUE for success, FALSE for failure */ -void -gallivm_register_garbage_collector_callback(garbage_collect_callback_func func, - void *cb_data) +static boolean +init_gallivm_state(struct gallivm_state *gallivm) { - struct callback *cb; - - if (!callback_list.prev) { - make_empty_list(&callback_list); - } + assert(!gallivm->context); + assert(!gallivm->module); + assert(!gallivm->provider); - /* see if already in list */ - foreach(cb, &callback_list) { - if (cb->func == func && cb->cb_data == cb_data) - return; - } + lp_build_init(); - /* add to list */ - cb = CALLOC_STRUCT(callback); - if (cb) { - cb->func = func; - cb->cb_data = cb_data; - insert_at_head(&callback_list, cb); + if (!gallivm_context) { + gallivm_context = LLVMContextCreate(); } -} + gallivm->context = gallivm_context; + if (!gallivm->context) + goto fail; + gallivm->module = LLVMModuleCreateWithNameInContext("gallivm", + gallivm->context); + if (!gallivm->module) + goto fail; -/** - * Remove a callback. - */ -void -gallivm_remove_garbage_collector_callback(garbage_collect_callback_func func, - void *cb_data) -{ - struct callback *cb; - - /* search list */ - foreach(cb, &callback_list) { - if (cb->func == func && cb->cb_data == cb_data) { - /* found, remove it */ - remove_from_list(cb); - FREE(cb); - return; - } - } -} + gallivm->provider = + LLVMCreateModuleProviderForExistingModule(gallivm->module); + if (!gallivm->provider) + goto fail; + gallivm->builder = LLVMCreateBuilderInContext(gallivm->context); + if (!gallivm->builder) + goto fail; -/** - * Call the callback functions (which are typically in the - * draw module and llvmpipe driver. - */ -static void -call_garbage_collector_callbacks(void) -{ - struct callback *cb; - foreach(cb, &callback_list) { - cb->func(cb->cb_data); + /* FIXME: MC-JIT only allows compiling one module at a time, and it must be + * complete when MC-JIT is created. So defer the MC-JIT engine creation for + * now. + */ +#if !USE_MCJIT + if (!init_gallivm_engine(gallivm)) { + goto fail; } -} +#else + /* + * MC-JIT engine compiles the module immediately on creation, so we can't + * obtain the target data from it. Instead we create a target data layout + * from a string. + * + * The produced layout strings are not precisely the same, but should make + * no difference for the kind of optimization passes we run. + * + * For reference this is the layout string on x64: + * + * e-p:64:64:64-S128-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f16:16:16-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64 + * + * See also: + * - http://llvm.org/docs/LangRef.html#datalayout + */ + + { + const unsigned pointer_size = 8 * sizeof(void *); + char layout[512]; + util_snprintf(layout, sizeof layout, "%c-p:%u:%u:%u-i64:64:64-a0:0:%u-s0:%u:%u", +#ifdef PIPE_ARCH_LITTLE_ENDIAN + 'e', // little endian +#else + 'E', // big endian +#endif + pointer_size, pointer_size, pointer_size, // pointer size, abi alignment, preferred alignment + pointer_size, // aggregate preferred alignment + pointer_size, pointer_size); // stack objects abi alignment, preferred alignment + gallivm->target = LLVMCreateTargetData(layout); + if (!gallivm->target) { + return FALSE; + } + } +#endif + if (!create_pass_manager(gallivm)) + goto fail; -/** - * Other gallium components using gallivm should call this periodically - * to let us do garbage collection (or at least try to free memory - * accumulated by the LLVM libraries). - */ -void -gallivm_garbage_collect(struct gallivm_state *gallivm) -{ - if (gallivm->context) { - if (gallivm_debug & GALLIVM_DEBUG_GC) - debug_printf("***** Doing LLVM garbage collection\n"); + return TRUE; - call_garbage_collector_callbacks(); - free_gallivm_state(gallivm); - init_gallivm_state(gallivm); - } +fail: + free_gallivm_state(gallivm); + return FALSE; } @@ -398,12 +426,27 @@ lp_build_init(void) lp_set_target_options(); - LLVMInitializeNativeTarget(); - +#if USE_MCJIT + LLVMLinkInMCJIT(); +#else LLVMLinkInJIT(); +#endif util_cpu_detect(); + + if (HAVE_AVX && + util_cpu_caps.has_avx) { + lp_native_vector_width = 256; + } else { + /* Leave it at 128, even when no SIMD extensions are available. + * Really needs to be a multiple of 128 so can fit 4 floats. + */ + lp_native_vector_width = 128; + } + lp_native_vector_width = debug_get_num_option("LP_NATIVE_VECTOR_WIDTH", + lp_native_vector_width); + gallivm_initialized = TRUE; #if 0 @@ -423,16 +466,27 @@ lp_build_init(void) struct gallivm_state * gallivm_create(void) { - if (!GlobalGallivm) { - GlobalGallivm = CALLOC_STRUCT(gallivm_state); - if (GlobalGallivm) { - if (!init_gallivm_state(GlobalGallivm)) { - FREE(GlobalGallivm); - GlobalGallivm = NULL; - } + struct gallivm_state *gallivm; + +#if HAVE_LLVM <= 0x206 + if (GlobalGallivm) { + return GlobalGallivm; + } +#endif + + gallivm = CALLOC_STRUCT(gallivm_state); + if (gallivm) { + if (!init_gallivm_state(gallivm)) { + FREE(gallivm); + gallivm = NULL; } } - return GlobalGallivm; + +#if HAVE_LLVM <= 0x206 + GlobalGallivm = gallivm; +#endif + + return gallivm; } @@ -442,6 +496,132 @@ gallivm_create(void) void gallivm_destroy(struct gallivm_state *gallivm) { +#if HAVE_LLVM <= 0x0206 /* No-op: don't destroy the singleton */ (void) gallivm; +#else + free_gallivm_state(gallivm); + FREE(gallivm); +#endif +} + + +/** + * Validate and optimze a function. + */ +static void +gallivm_optimize_function(struct gallivm_state *gallivm, + LLVMValueRef func) +{ + if (0) { + debug_printf("optimizing %s...\n", LLVMGetValueName(func)); + } + + assert(gallivm->passmgr); + + /* Apply optimizations to LLVM IR */ + LLVMRunFunctionPassManager(gallivm->passmgr, func); + + if (0) { + if (gallivm_debug & GALLIVM_DEBUG_IR) { + /* Print the LLVM IR to stderr */ + lp_debug_dump_value(func); + debug_printf("\n"); + } + } +} + + +/** + * Validate a function. + */ +void +gallivm_verify_function(struct gallivm_state *gallivm, + LLVMValueRef func) +{ + /* Verify the LLVM IR. If invalid, dump and abort */ +#ifdef DEBUG + if (LLVMVerifyFunction(func, LLVMPrintMessageAction)) { + lp_debug_dump_value(func); + assert(0); + return; + } +#endif + + gallivm_optimize_function(gallivm, func); + + if (gallivm_debug & GALLIVM_DEBUG_IR) { + /* Print the LLVM IR to stderr */ + lp_debug_dump_value(func); + debug_printf("\n"); + } +} + + +void +gallivm_compile_module(struct gallivm_state *gallivm) +{ +#if HAVE_LLVM > 0x206 + assert(!gallivm->compiled); +#endif + + /* Dump byte code to a file */ + if (0) { + LLVMWriteBitcodeToFile(gallivm->module, "llvmpipe.bc"); + debug_printf("llvmpipe.bc written\n"); + debug_printf("Invoke as \"llc -o - llvmpipe.bc\"\n"); + } + +#if USE_MCJIT + assert(!gallivm->engine); + if (!init_gallivm_engine(gallivm)) { + assert(0); + } +#endif + assert(gallivm->engine); + + ++gallivm->compiled; +} + + +func_pointer +gallivm_jit_function(struct gallivm_state *gallivm, + LLVMValueRef func) +{ + void *code; + func_pointer jit_func; + + assert(gallivm->compiled); + assert(gallivm->engine); + + code = LLVMGetPointerToGlobal(gallivm->engine, func); + assert(code); + jit_func = pointer_to_func(code); + + if (gallivm_debug & GALLIVM_DEBUG_ASM) { + lp_disassemble(code); + } + + /* Free the function body to save memory */ + lp_func_delete_body(func); + + return jit_func; +} + + +/** + * Free the function (and its machine code). + */ +void +gallivm_free_function(struct gallivm_state *gallivm, + LLVMValueRef func, + const void *code) +{ +#if !USE_MCJIT + if (code) { + LLVMFreeMachineCodeForFunction(gallivm->engine, func); + } + + LLVMDeleteFunction(func); +#endif } |