diff options
Diffstat (limited to 'src/gallium/drivers/r600/r600_llvm.c')
| -rw-r--r-- | src/gallium/drivers/r600/r600_llvm.c | 943 |
1 files changed, 0 insertions, 943 deletions
diff --git a/src/gallium/drivers/r600/r600_llvm.c b/src/gallium/drivers/r600/r600_llvm.c deleted file mode 100644 index 7eab29c..0000000 --- a/src/gallium/drivers/r600/r600_llvm.c +++ /dev/null @@ -1,943 +0,0 @@ -#include "r600_llvm.h" - -#include "gallivm/lp_bld_const.h" -#include "gallivm/lp_bld_intr.h" -#include "gallivm/lp_bld_gather.h" -#include "tgsi/tgsi_parse.h" -#include "util/list.h" -#include "util/u_memory.h" - -#include "evergreend.h" -#include "r600_asm.h" -#include "r600_sq.h" -#include "r600_opcodes.h" -#include "r600_shader.h" -#include "r600_pipe.h" -#include "radeon_llvm.h" -#include "radeon_llvm_emit.h" -#include "radeon_elf_util.h" - -#include <stdio.h> - -#if defined R600_USE_LLVM || defined HAVE_OPENCL - -#define CONSTANT_BUFFER_0_ADDR_SPACE 8 -#define CONSTANT_BUFFER_1_ADDR_SPACE (CONSTANT_BUFFER_0_ADDR_SPACE + R600_BUFFER_INFO_CONST_BUFFER) -#define LLVM_R600_BUFFER_INFO_CONST_BUFFER \ - (CONSTANT_BUFFER_0_ADDR_SPACE + R600_BUFFER_INFO_CONST_BUFFER) - -static LLVMValueRef llvm_load_const_buffer( - struct lp_build_tgsi_context * bld_base, - LLVMValueRef OffsetValue, - unsigned ConstantAddressSpace) -{ - LLVMValueRef offset[2] = { - LLVMConstInt(LLVMInt64TypeInContext(bld_base->base.gallivm->context), 0, false), - OffsetValue - }; - - LLVMTypeRef const_ptr_type = LLVMPointerType(LLVMArrayType(LLVMVectorType(bld_base->base.elem_type, 4), 1024), - ConstantAddressSpace); - LLVMValueRef const_ptr = LLVMBuildIntToPtr(bld_base->base.gallivm->builder, lp_build_const_int32(bld_base->base.gallivm, 0), const_ptr_type, ""); - LLVMValueRef ptr = LLVMBuildGEP(bld_base->base.gallivm->builder, const_ptr, offset, 2, ""); - return LLVMBuildLoad(bld_base->base.gallivm->builder, ptr, ""); -} - -static LLVMValueRef llvm_fetch_const( - struct lp_build_tgsi_context * bld_base, - const struct tgsi_full_src_register *reg, - enum tgsi_opcode_type type, - unsigned swizzle) -{ - LLVMValueRef offset = lp_build_const_int32(bld_base->base.gallivm, reg->Register.Index); - if (reg->Register.Indirect) { - struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base); - LLVMValueRef index = LLVMBuildLoad(bld_base->base.gallivm->builder, bld->addr[reg->Indirect.Index][reg->Indirect.Swizzle], ""); - offset = LLVMBuildAdd(bld_base->base.gallivm->builder, offset, index, ""); - } - unsigned ConstantAddressSpace = CONSTANT_BUFFER_0_ADDR_SPACE ; - if (reg->Register.Dimension) { - ConstantAddressSpace += reg->Dimension.Index; - } - LLVMValueRef cvecval = llvm_load_const_buffer(bld_base, offset, ConstantAddressSpace); - LLVMValueRef cval = LLVMBuildExtractElement(bld_base->base.gallivm->builder, cvecval, lp_build_const_int32(bld_base->base.gallivm, swizzle), ""); - return bitcast(bld_base, type, cval); -} - -static void llvm_load_system_value( - struct radeon_llvm_context * ctx, - unsigned index, - const struct tgsi_full_declaration *decl) -{ - unsigned chan; - - switch (decl->Semantic.Name) { - case TGSI_SEMANTIC_INSTANCEID: chan = 3; break; - case TGSI_SEMANTIC_VERTEXID: chan = 0; break; - default: assert(!"unknown system value"); - } - - ctx->system_values[index] = LLVMBuildExtractElement(ctx->gallivm.builder, - LLVMGetParam(ctx->main_fn, 0), lp_build_const_int32(&(ctx->gallivm), chan), - ""); -} - -static LLVMValueRef -llvm_load_input_vector( - struct radeon_llvm_context * ctx, unsigned location, unsigned ijregs, - boolean interp) -{ - LLVMTypeRef VecType; - LLVMValueRef Args[3] = { - lp_build_const_int32(&(ctx->gallivm), location) - }; - unsigned ArgCount = 1; - if (interp) { - VecType = LLVMVectorType(ctx->soa.bld_base.base.elem_type, 2); - LLVMValueRef IJIndex = LLVMGetParam(ctx->main_fn, ijregs / 2); - Args[ArgCount++] = LLVMBuildExtractElement(ctx->gallivm.builder, IJIndex, - lp_build_const_int32(&(ctx->gallivm), 2 * (ijregs % 2)), ""); - Args[ArgCount++] = LLVMBuildExtractElement(ctx->gallivm.builder, IJIndex, - lp_build_const_int32(&(ctx->gallivm), 2 * (ijregs % 2) + 1), ""); - LLVMValueRef HalfVec[2] = { - lp_build_intrinsic(ctx->gallivm.builder, "llvm.R600.interp.xy", - VecType, Args, ArgCount, LLVMReadNoneAttribute), - lp_build_intrinsic(ctx->gallivm.builder, "llvm.R600.interp.zw", - VecType, Args, ArgCount, LLVMReadNoneAttribute) - }; - LLVMValueRef MaskInputs[4] = { - lp_build_const_int32(&(ctx->gallivm), 0), - lp_build_const_int32(&(ctx->gallivm), 1), - lp_build_const_int32(&(ctx->gallivm), 2), - lp_build_const_int32(&(ctx->gallivm), 3) - }; - LLVMValueRef Mask = LLVMConstVector(MaskInputs, 4); - return LLVMBuildShuffleVector(ctx->gallivm.builder, HalfVec[0], HalfVec[1], - Mask, ""); - } else { - VecType = LLVMVectorType(ctx->soa.bld_base.base.elem_type, 4); - return lp_build_intrinsic(ctx->gallivm.builder, "llvm.R600.interp.const", - VecType, Args, ArgCount, LLVMReadNoneAttribute); - } -} - -static LLVMValueRef -llvm_face_select_helper( - struct radeon_llvm_context * ctx, - LLVMValueRef face, LLVMValueRef front_color, LLVMValueRef back_color) -{ - const struct lp_build_context * bb = &ctx->soa.bld_base.base; - LLVMValueRef is_front = LLVMBuildFCmp( - bb->gallivm->builder, LLVMRealUGT, face, - lp_build_const_float(bb->gallivm, 0.0f), ""); - return LLVMBuildSelect(bb->gallivm->builder, is_front, - front_color, back_color, ""); -} - -static void llvm_load_input( - struct radeon_llvm_context * ctx, - unsigned input_index, - const struct tgsi_full_declaration *decl) -{ - const struct r600_shader_io * input = &ctx->r600_inputs[input_index]; - unsigned chan; - int two_side = (ctx->two_side && input->name == TGSI_SEMANTIC_COLOR); - LLVMValueRef v; - boolean require_interp_intrinsic = ctx->chip_class >= EVERGREEN && - ctx->type == TGSI_PROCESSOR_FRAGMENT; - - if (require_interp_intrinsic && input->spi_sid) { - v = llvm_load_input_vector(ctx, input->lds_pos, input->ij_index, - (input->interpolate > 0)); - } else - v = LLVMGetParam(ctx->main_fn, input->gpr); - - if (two_side) { - struct r600_shader_io * back_input = - &ctx->r600_inputs[input->back_color_input]; - LLVMValueRef v2; - LLVMValueRef face = LLVMGetParam(ctx->main_fn, ctx->face_gpr); - face = LLVMBuildExtractElement(ctx->gallivm.builder, face, - lp_build_const_int32(&(ctx->gallivm), 0), ""); - - if (require_interp_intrinsic && back_input->spi_sid) - v2 = llvm_load_input_vector(ctx, back_input->lds_pos, - back_input->ij_index, (back_input->interpolate > 0)); - else - v2 = LLVMGetParam(ctx->main_fn, back_input->gpr); - v = llvm_face_select_helper(ctx, face, v, v2); - } - - for (chan = 0; chan < 4; chan++) { - unsigned soa_index = radeon_llvm_reg_index_soa(input_index, chan); - - ctx->inputs[soa_index] = LLVMBuildExtractElement(ctx->gallivm.builder, v, - lp_build_const_int32(&(ctx->gallivm), chan), ""); - - if (input->name == TGSI_SEMANTIC_POSITION && - ctx->type == TGSI_PROCESSOR_FRAGMENT && chan == 3) { - /* RCP for fragcoord.w */ - ctx->inputs[soa_index] = LLVMBuildFDiv(ctx->gallivm.builder, - lp_build_const_float(&(ctx->gallivm), 1.0f), - ctx->inputs[soa_index], ""); - } - } -} - -static void llvm_emit_prologue(struct lp_build_tgsi_context * bld_base) -{ - struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base); - radeon_llvm_shader_type(ctx->main_fn, ctx->type); - -} - -static void llvm_emit_epilogue(struct lp_build_tgsi_context * bld_base) -{ - struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base); - struct lp_build_context * base = &bld_base->base; - struct pipe_stream_output_info * so = ctx->stream_outputs; - unsigned i; - unsigned next_pos = 60; - unsigned next_param = 0; - - unsigned color_count = 0; - boolean has_color = false; - - if (ctx->type == TGSI_PROCESSOR_VERTEX && so->num_outputs) { - for (i = 0; i < so->num_outputs; i++) { - unsigned register_index = so->output[i].register_index; - unsigned start_component = so->output[i].start_component; - unsigned num_components = so->output[i].num_components; - unsigned dst_offset = so->output[i].dst_offset; - unsigned chan; - LLVMValueRef elements[4]; - if (dst_offset < start_component) { - for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { - elements[chan] = LLVMBuildLoad(base->gallivm->builder, - ctx->soa.outputs[register_index][(chan + start_component) % TGSI_NUM_CHANNELS], ""); - } - start_component = 0; - } else { - for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { - elements[chan] = LLVMBuildLoad(base->gallivm->builder, - ctx->soa.outputs[register_index][chan], ""); - } - } - LLVMValueRef output = lp_build_gather_values(base->gallivm, elements, 4); - LLVMValueRef args[4]; - args[0] = output; - args[1] = lp_build_const_int32(base->gallivm, dst_offset - start_component); - args[2] = lp_build_const_int32(base->gallivm, so->output[i].output_buffer); - args[3] = lp_build_const_int32(base->gallivm, ((1 << num_components) - 1) << start_component); - lp_build_intrinsic(base->gallivm->builder, "llvm.R600.store.stream.output", - LLVMVoidTypeInContext(base->gallivm->context), args, 4, 0); - } - } - - /* Add the necessary export instructions */ - for (i = 0; i < ctx->output_reg_count; i++) { - unsigned chan; - LLVMValueRef elements[4]; - for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { - elements[chan] = LLVMBuildLoad(base->gallivm->builder, - ctx->soa.outputs[i][chan], ""); - } - if (ctx->alpha_to_one && ctx->type == TGSI_PROCESSOR_FRAGMENT && ctx->r600_outputs[i].name == TGSI_SEMANTIC_COLOR) - elements[3] = lp_build_const_float(base->gallivm, 1.0f); - LLVMValueRef output = lp_build_gather_values(base->gallivm, elements, 4); - - if (ctx->type == TGSI_PROCESSOR_VERTEX) { - switch (ctx->r600_outputs[i].name) { - case TGSI_SEMANTIC_POSITION: - case TGSI_SEMANTIC_PSIZE: { - LLVMValueRef args[3]; - args[0] = output; - args[1] = lp_build_const_int32(base->gallivm, next_pos++); - args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_POS); - lp_build_intrinsic( - base->gallivm->builder, - "llvm.R600.store.swizzle", - LLVMVoidTypeInContext(base->gallivm->context), - args, 3, 0); - break; - } - case TGSI_SEMANTIC_CLIPVERTEX: { - LLVMValueRef args[3]; - unsigned reg_index; - LLVMValueRef adjusted_elements[4]; - for (reg_index = 0; reg_index < 2; reg_index ++) { - for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { - LLVMValueRef offset = lp_build_const_int32(bld_base->base.gallivm, reg_index * 4 + chan); - LLVMValueRef base_vector = llvm_load_const_buffer(bld_base, offset, CONSTANT_BUFFER_1_ADDR_SPACE); - args[0] = output; - args[1] = base_vector; - adjusted_elements[chan] = lp_build_intrinsic(base->gallivm->builder, - "llvm.AMDGPU.dp4", bld_base->base.elem_type, - args, 2, LLVMReadNoneAttribute); - } - args[0] = lp_build_gather_values(base->gallivm, - adjusted_elements, 4); - args[1] = lp_build_const_int32(base->gallivm, next_pos++); - args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_POS); - lp_build_intrinsic( - base->gallivm->builder, - "llvm.R600.store.swizzle", - LLVMVoidTypeInContext(base->gallivm->context), - args, 3, 0); - } - break; - } - case TGSI_SEMANTIC_CLIPDIST : { - LLVMValueRef args[3]; - args[0] = output; - args[1] = lp_build_const_int32(base->gallivm, next_pos++); - args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_POS); - lp_build_intrinsic( - base->gallivm->builder, - "llvm.R600.store.swizzle", - LLVMVoidTypeInContext(base->gallivm->context), - args, 3, 0); - args[1] = lp_build_const_int32(base->gallivm, next_param++); - args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PARAM); - lp_build_intrinsic( - base->gallivm->builder, - "llvm.R600.store.swizzle", - LLVMVoidTypeInContext(base->gallivm->context), - args, 3, 0); - break; - } - case TGSI_SEMANTIC_FOG: { - elements[0] = LLVMBuildLoad(base->gallivm->builder, - ctx->soa.outputs[i][0], ""); - elements[1] = elements[2] = lp_build_const_float(base->gallivm, 0.0f); - elements[3] = lp_build_const_float(base->gallivm, 1.0f); - - LLVMValueRef args[3]; - args[0] = lp_build_gather_values(base->gallivm, elements, 4); - args[1] = lp_build_const_int32(base->gallivm, next_param++); - args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PARAM); - lp_build_intrinsic( - base->gallivm->builder, - "llvm.R600.store.swizzle", - LLVMVoidTypeInContext(base->gallivm->context), - args, 3, 0); - break; - } - default: { - LLVMValueRef args[3]; - args[0] = output; - args[1] = lp_build_const_int32(base->gallivm, next_param++); - args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PARAM); - lp_build_intrinsic( - base->gallivm->builder, - "llvm.R600.store.swizzle", - LLVMVoidTypeInContext(base->gallivm->context), - args, 3, 0); - break; - } - } - } else if (ctx->type == TGSI_PROCESSOR_FRAGMENT) { - switch (ctx->r600_outputs[i].name) { - case TGSI_SEMANTIC_COLOR: - has_color = true; - if ( color_count < ctx->color_buffer_count) { - LLVMValueRef args[3]; - args[0] = output; - if (ctx->fs_color_all) { - for (unsigned j = 0; j < ctx->color_buffer_count; j++) { - args[1] = lp_build_const_int32(base->gallivm, j); - args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PIXEL); - lp_build_intrinsic( - base->gallivm->builder, - "llvm.R600.store.swizzle", - LLVMVoidTypeInContext(base->gallivm->context), - args, 3, 0); - } - } else { - args[1] = lp_build_const_int32(base->gallivm, color_count++); - args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PIXEL); - lp_build_intrinsic( - base->gallivm->builder, - "llvm.R600.store.swizzle", - LLVMVoidTypeInContext(base->gallivm->context), - args, 3, 0); - } - } - break; - case TGSI_SEMANTIC_POSITION: - lp_build_intrinsic_unary( - base->gallivm->builder, - "llvm.R600.store.pixel.depth", - LLVMVoidTypeInContext(base->gallivm->context), - LLVMBuildLoad(base->gallivm->builder, ctx->soa.outputs[i][2], "")); - break; - case TGSI_SEMANTIC_STENCIL: - lp_build_intrinsic_unary( - base->gallivm->builder, - "llvm.R600.store.pixel.stencil", - LLVMVoidTypeInContext(base->gallivm->context), - LLVMBuildLoad(base->gallivm->builder, ctx->soa.outputs[i][1], "")); - break; - } - } - } - // Add dummy exports - if (ctx->type == TGSI_PROCESSOR_VERTEX) { - if (!next_param) { - lp_build_intrinsic_unary(base->gallivm->builder, "llvm.R600.store.dummy", - LLVMVoidTypeInContext(base->gallivm->context), - lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PARAM)); - } - if (!(next_pos-60)) { - lp_build_intrinsic_unary(base->gallivm->builder, "llvm.R600.store.dummy", - LLVMVoidTypeInContext(base->gallivm->context), - lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_POS)); - } - } - if (ctx->type == TGSI_PROCESSOR_FRAGMENT) { - if (!has_color) { - lp_build_intrinsic_unary(base->gallivm->builder, "llvm.R600.store.dummy", - LLVMVoidTypeInContext(base->gallivm->context), - lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PIXEL)); - } - } - -} - -static void llvm_emit_tex( - const struct lp_build_tgsi_action * action, - struct lp_build_tgsi_context * bld_base, - struct lp_build_emit_data * emit_data) -{ - struct gallivm_state * gallivm = bld_base->base.gallivm; - LLVMValueRef args[7]; - unsigned c, sampler_src; - struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base); - - if (emit_data->inst->Texture.Texture == TGSI_TEXTURE_BUFFER) { - switch (emit_data->inst->Instruction.Opcode) { - case TGSI_OPCODE_TXQ: { - struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base); - ctx->uses_tex_buffers = true; - bool isEgPlus = (ctx->chip_class >= EVERGREEN); - LLVMValueRef offset = lp_build_const_int32(bld_base->base.gallivm, - isEgPlus ? 0 : 1); - LLVMValueRef cvecval = llvm_load_const_buffer(bld_base, offset, - LLVM_R600_BUFFER_INFO_CONST_BUFFER); - if (!isEgPlus) { - LLVMValueRef maskval[4] = { - lp_build_const_int32(gallivm, 1), - lp_build_const_int32(gallivm, 2), - lp_build_const_int32(gallivm, 3), - lp_build_const_int32(gallivm, 0), - }; - LLVMValueRef mask = LLVMConstVector(maskval, 4); - cvecval = LLVMBuildShuffleVector(gallivm->builder, cvecval, cvecval, - mask, ""); - } - emit_data->output[0] = cvecval; - return; - } - case TGSI_OPCODE_TXF: { - args[0] = LLVMBuildExtractElement(gallivm->builder, emit_data->args[0], lp_build_const_int32(gallivm, 0), ""); - args[1] = lp_build_const_int32(gallivm, R600_MAX_CONST_BUFFERS); - emit_data->output[0] = lp_build_intrinsic(gallivm->builder, - "llvm.R600.load.texbuf", - emit_data->dst_type, args, 2, LLVMReadNoneAttribute); - if (ctx->chip_class >= EVERGREEN) - return; - ctx->uses_tex_buffers = true; - LLVMDumpValue(emit_data->output[0]); - emit_data->output[0] = LLVMBuildBitCast(gallivm->builder, - emit_data->output[0], LLVMVectorType(bld_base->base.int_elem_type, 4), - ""); - LLVMValueRef Mask = llvm_load_const_buffer(bld_base, - lp_build_const_int32(gallivm, 0), - LLVM_R600_BUFFER_INFO_CONST_BUFFER); - Mask = LLVMBuildBitCast(gallivm->builder, Mask, - LLVMVectorType(bld_base->base.int_elem_type, 4), ""); - emit_data->output[0] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_AND, - emit_data->output[0], - Mask); - LLVMValueRef WComponent = LLVMBuildExtractElement(gallivm->builder, - emit_data->output[0], lp_build_const_int32(gallivm, 3), ""); - Mask = llvm_load_const_buffer(bld_base, lp_build_const_int32(gallivm, 1), - LLVM_R600_BUFFER_INFO_CONST_BUFFER); - Mask = LLVMBuildExtractElement(gallivm->builder, Mask, - lp_build_const_int32(gallivm, 0), ""); - Mask = LLVMBuildBitCast(gallivm->builder, Mask, - bld_base->base.int_elem_type, ""); - WComponent = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_OR, - WComponent, Mask); - emit_data->output[0] = LLVMBuildInsertElement(gallivm->builder, - emit_data->output[0], WComponent, lp_build_const_int32(gallivm, 3), ""); - emit_data->output[0] = LLVMBuildBitCast(gallivm->builder, - emit_data->output[0], LLVMVectorType(bld_base->base.elem_type, 4), ""); - } - return; - default: - break; - } - } - - if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_TEX || - emit_data->inst->Instruction.Opcode == TGSI_OPCODE_TXP) { - LLVMValueRef Vector[4] = { - LLVMBuildExtractElement(gallivm->builder, emit_data->args[0], - lp_build_const_int32(gallivm, 0), ""), - LLVMBuildExtractElement(gallivm->builder, emit_data->args[0], - lp_build_const_int32(gallivm, 1), ""), - LLVMBuildExtractElement(gallivm->builder, emit_data->args[0], - lp_build_const_int32(gallivm, 2), ""), - LLVMBuildExtractElement(gallivm->builder, emit_data->args[0], - lp_build_const_int32(gallivm, 3), ""), - }; - switch (emit_data->inst->Texture.Texture) { - case TGSI_TEXTURE_2D: - case TGSI_TEXTURE_RECT: - Vector[2] = Vector[3] = LLVMGetUndef(bld_base->base.elem_type); - break; - case TGSI_TEXTURE_1D: - Vector[1] = Vector[2] = Vector[3] = LLVMGetUndef(bld_base->base.elem_type); - break; - default: - break; - } - args[0] = lp_build_gather_values(gallivm, Vector, 4); - } else { - args[0] = emit_data->args[0]; - } - - assert(emit_data->arg_count + 2 <= Elements(args)); - - for (c = 1; c < emit_data->arg_count; ++c) - args[c] = emit_data->args[c]; - - if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_TXF) { - args[1] = LLVMBuildShl(gallivm->builder, args[1], lp_build_const_int32(gallivm, 1), ""); - args[2] = LLVMBuildShl(gallivm->builder, args[2], lp_build_const_int32(gallivm, 1), ""); - args[3] = LLVMBuildShl(gallivm->builder, args[3], lp_build_const_int32(gallivm, 1), ""); - } - - sampler_src = emit_data->inst->Instruction.NumSrcRegs-1; - - args[c++] = lp_build_const_int32(gallivm, - emit_data->inst->Src[sampler_src].Register.Index + R600_MAX_CONST_BUFFERS); - args[c++] = lp_build_const_int32(gallivm, - emit_data->inst->Src[sampler_src].Register.Index); - args[c++] = lp_build_const_int32(gallivm, - emit_data->inst->Texture.Texture); - - if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_TXF && - (emit_data->inst->Texture.Texture == TGSI_TEXTURE_2D_MSAA || - emit_data->inst->Texture.Texture == TGSI_TEXTURE_2D_ARRAY_MSAA)) { - - switch (emit_data->inst->Texture.Texture) { - case TGSI_TEXTURE_2D_MSAA: - args[6] = lp_build_const_int32(gallivm, TGSI_TEXTURE_2D); - break; - case TGSI_TEXTURE_2D_ARRAY_MSAA: - args[6] = lp_build_const_int32(gallivm, TGSI_TEXTURE_2D_ARRAY); - break; - default: - break; - } - - if (ctx->has_compressed_msaa_texturing) { - LLVMValueRef ldptr_args[10] = { - args[0], // Coord - args[1], // Offset X - args[2], // Offset Y - args[3], // Offset Z - args[4], - args[5], - lp_build_const_int32(gallivm, 1), - lp_build_const_int32(gallivm, 1), - lp_build_const_int32(gallivm, 1), - lp_build_const_int32(gallivm, 1) - }; - LLVMValueRef ptr = lp_build_intrinsic(gallivm->builder, - "llvm.R600.ldptr", - emit_data->dst_type, ldptr_args, 10, LLVMReadNoneAttribute); - LLVMValueRef Tmp = LLVMBuildExtractElement(gallivm->builder, args[0], - lp_build_const_int32(gallivm, 3), ""); - Tmp = LLVMBuildMul(gallivm->builder, Tmp, - lp_build_const_int32(gallivm, 4), ""); - LLVMValueRef ResX = LLVMBuildExtractElement(gallivm->builder, ptr, - lp_build_const_int32(gallivm, 0), ""); - ResX = LLVMBuildBitCast(gallivm->builder, ResX, - bld_base->base.int_elem_type, ""); - Tmp = LLVMBuildLShr(gallivm->builder, ResX, Tmp, ""); - Tmp = LLVMBuildAnd(gallivm->builder, Tmp, - lp_build_const_int32(gallivm, 0xF), ""); - args[0] = LLVMBuildInsertElement(gallivm->builder, args[0], Tmp, - lp_build_const_int32(gallivm, 3), ""); - args[c++] = lp_build_const_int32(gallivm, - emit_data->inst->Texture.Texture); - } - } - - emit_data->output[0] = lp_build_intrinsic(gallivm->builder, - action->intr_name, - emit_data->dst_type, args, c, LLVMReadNoneAttribute); - - if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_TXQ && - ((emit_data->inst->Texture.Texture == TGSI_TEXTURE_CUBE_ARRAY || - emit_data->inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE_ARRAY))) - if (emit_data->inst->Dst[0].Register.WriteMask & 4) { - LLVMValueRef offset = lp_build_const_int32(bld_base->base.gallivm, 0); - LLVMValueRef ZLayer = LLVMBuildExtractElement(gallivm->builder, - llvm_load_const_buffer(bld_base, offset, LLVM_R600_BUFFER_INFO_CONST_BUFFER), - lp_build_const_int32(gallivm, 0), ""); - - emit_data->output[0] = LLVMBuildInsertElement(gallivm->builder, emit_data->output[0], ZLayer, lp_build_const_int32(gallivm, 2), ""); - struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base); - ctx->has_txq_cube_array_z_comp = true; - } -} - -static void emit_cndlt( - const struct lp_build_tgsi_action * action, - struct lp_build_tgsi_context * bld_base, - struct lp_build_emit_data * emit_data) -{ - LLVMBuilderRef builder = bld_base->base.gallivm->builder; - LLVMValueRef float_zero = lp_build_const_float( - bld_base->base.gallivm, 0.0f); - LLVMValueRef cmp = LLVMBuildFCmp( - builder, LLVMRealULT, emit_data->args[0], float_zero, ""); - emit_data->output[emit_data->chan] = LLVMBuildSelect(builder, - cmp, emit_data->args[1], emit_data->args[2], ""); -} - -static void dp_fetch_args( - struct lp_build_tgsi_context * bld_base, - struct lp_build_emit_data * emit_data) -{ - struct lp_build_context * base = &bld_base->base; - unsigned chan; - LLVMValueRef elements[2][4]; - unsigned opcode = emit_data->inst->Instruction.Opcode; - unsigned dp_components = (opcode == TGSI_OPCODE_DP2 ? 2 : - (opcode == TGSI_OPCODE_DP3 ? 3 : 4)); - for (chan = 0 ; chan < dp_components; chan++) { - elements[0][chan] = lp_build_emit_fetch(bld_base, - emit_data->inst, 0, chan); - elements[1][chan] = lp_build_emit_fetch(bld_base, - emit_data->inst, 1, chan); - } - - for ( ; chan < 4; chan++) { - elements[0][chan] = base->zero; - elements[1][chan] = base->zero; - } - - /* Fix up for DPH */ - if (opcode == TGSI_OPCODE_DPH) { - elements[0][TGSI_CHAN_W] = base->one; - } - - emit_data->args[0] = lp_build_gather_values(bld_base->base.gallivm, - elements[0], 4); - emit_data->args[1] = lp_build_gather_values(bld_base->base.gallivm, - elements[1], 4); - emit_data->arg_count = 2; - - emit_data->dst_type = base->elem_type; -} - -static struct lp_build_tgsi_action dot_action = { - .fetch_args = dp_fetch_args, - .emit = build_tgsi_intrinsic_nomem, - .intr_name = "llvm.AMDGPU.dp4" -}; - -static void txd_fetch_args( - struct lp_build_tgsi_context * bld_base, - struct lp_build_emit_data * emit_data) -{ - const struct tgsi_full_instruction * inst = emit_data->inst; - - LLVMValueRef coords[4]; - unsigned chan, src; - for (src = 0; src < 3; src++) { - for (chan = 0; chan < 4; chan++) - coords[chan] = lp_build_emit_fetch(bld_base, inst, src, chan); - - emit_data->args[src] = lp_build_gather_values(bld_base->base.gallivm, - coords, 4); - } - emit_data->arg_count = 3; - emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4); -} - - -static void txp_fetch_args( - struct lp_build_tgsi_context * bld_base, - struct lp_build_emit_data * emit_data) -{ - const struct tgsi_full_instruction * inst = emit_data->inst; - LLVMValueRef src_w; - unsigned chan; - LLVMValueRef coords[5]; - - emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4); - src_w = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_W); - - for (chan = 0; chan < 3; chan++ ) { - LLVMValueRef arg = lp_build_emit_fetch(bld_base, - emit_data->inst, 0, chan); - coords[chan] = lp_build_emit_llvm_binary(bld_base, - TGSI_OPCODE_DIV, arg, src_w); - } - coords[3] = bld_base->base.one; - - if ((inst->Texture.Texture == TGSI_TEXTURE_CUBE || - inst->Texture.Texture == TGSI_TEXTURE_CUBE_ARRAY || - inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE || - inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE_ARRAY) && - inst->Instruction.Opcode != TGSI_OPCODE_TXQ && - inst->Instruction.Opcode != TGSI_OPCODE_TXQ_LZ) { - radeon_llvm_emit_prepare_cube_coords(bld_base, emit_data, coords, NULL); - } - - emit_data->args[0] = lp_build_gather_values(bld_base->base.gallivm, - coords, 4); - emit_data->arg_count = 1; -} - -static void tex_fetch_args( - struct lp_build_tgsi_context * bld_base, - struct lp_build_emit_data * emit_data) -{ - const struct tgsi_full_instruction * inst = emit_data->inst; - - LLVMValueRef coords[5]; - unsigned chan; - for (chan = 0; chan < 4; chan++) { - coords[chan] = lp_build_emit_fetch(bld_base, inst, 0, chan); - } - - if (inst->Instruction.Opcode == TGSI_OPCODE_TEX2 || - inst->Instruction.Opcode == TGSI_OPCODE_TXB2 || - inst->Instruction.Opcode == TGSI_OPCODE_TXL2) { - /* These instructions have additional operand that should be packed - * into the cube coord vector by radeon_llvm_emit_prepare_cube_coords. - * That operand should be passed as a float value in the args array - * right after the coord vector. After packing it's not used anymore, - * that's why arg_count is not increased */ - coords[4] = lp_build_emit_fetch(bld_base, inst, 1, TGSI_CHAN_X); - } - - if ((inst->Texture.Texture == TGSI_TEXTURE_CUBE || - inst->Texture.Texture == TGSI_TEXTURE_CUBE_ARRAY || - inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE || - inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE_ARRAY) && - inst->Instruction.Opcode != TGSI_OPCODE_TXQ && - inst->Instruction.Opcode != TGSI_OPCODE_TXQ_LZ) { - radeon_llvm_emit_prepare_cube_coords(bld_base, emit_data, coords, NULL); - } - - emit_data->arg_count = 1; - emit_data->args[0] = lp_build_gather_values(bld_base->base.gallivm, - coords, 4); - emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4); -} - -static void txf_fetch_args( - struct lp_build_tgsi_context * bld_base, - struct lp_build_emit_data * emit_data) -{ - const struct tgsi_full_instruction * inst = emit_data->inst; - struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base); - const struct tgsi_texture_offset * off = inst->TexOffsets; - LLVMTypeRef offset_type = bld_base->int_bld.elem_type; - - /* fetch tex coords */ - tex_fetch_args(bld_base, emit_data); - - /* fetch tex offsets */ - if (inst->Texture.NumOffsets) { - assert(inst->Texture.NumOffsets == 1); - - emit_data->args[1] = LLVMConstBitCast( - bld->immediates[off->Index][off->SwizzleX], - offset_type); - emit_data->args[2] = LLVMConstBitCast( - bld->immediates[off->Index][off->SwizzleY], - offset_type); - emit_data->args[3] = LLVMConstBitCast( - bld->immediates[off->Index][off->SwizzleZ], - offset_type); - } else { - emit_data->args[1] = bld_base->int_bld.zero; - emit_data->args[2] = bld_base->int_bld.zero; - emit_data->args[3] = bld_base->int_bld.zero; - } - - emit_data->arg_count = 4; -} - -LLVMModuleRef r600_tgsi_llvm( - struct radeon_llvm_context * ctx, - const struct tgsi_token * tokens) -{ - struct tgsi_shader_info shader_info; - struct lp_build_tgsi_context * bld_base = &ctx->soa.bld_base; - radeon_llvm_context_init(ctx, "r600--"); - LLVMTypeRef Arguments[32]; - unsigned ArgumentsCount = 0; - for (unsigned i = 0; i < ctx->inputs_count; i++) - Arguments[ArgumentsCount++] = LLVMVectorType(bld_base->base.elem_type, 4); - radeon_llvm_create_func(ctx, NULL, 0, Arguments, ArgumentsCount); - for (unsigned i = 0; i < ctx->inputs_count; i++) { - LLVMValueRef P = LLVMGetParam(ctx->main_fn, i); - LLVMAddAttribute(P, LLVMInRegAttribute); - } - tgsi_scan_shader(tokens, &shader_info); - - bld_base->info = &shader_info; - bld_base->userdata = ctx; - bld_base->emit_fetch_funcs[TGSI_FILE_CONSTANT] = llvm_fetch_const; - bld_base->emit_prologue = llvm_emit_prologue; - bld_base->emit_epilogue = llvm_emit_epilogue; - ctx->load_input = llvm_load_input; - ctx->load_system_value = llvm_load_system_value; - - bld_base->op_actions[TGSI_OPCODE_DP2] = dot_action; - bld_base->op_actions[TGSI_OPCODE_DP3] = dot_action; - bld_base->op_actions[TGSI_OPCODE_DP4] = dot_action; - bld_base->op_actions[TGSI_OPCODE_DPH] = dot_action; - bld_base->op_actions[TGSI_OPCODE_DDX].intr_name = "llvm.AMDGPU.ddx"; - bld_base->op_actions[TGSI_OPCODE_DDX].fetch_args = tex_fetch_args; - bld_base->op_actions[TGSI_OPCODE_DDX].emit = llvm_emit_tex; - bld_base->op_actions[TGSI_OPCODE_DDY].intr_name = "llvm.AMDGPU.ddy"; - bld_base->op_actions[TGSI_OPCODE_DDY].fetch_args = tex_fetch_args; - bld_base->op_actions[TGSI_OPCODE_DDY].emit = llvm_emit_tex; - bld_base->op_actions[TGSI_OPCODE_TEX].fetch_args = tex_fetch_args; - bld_base->op_actions[TGSI_OPCODE_TEX].intr_name = "llvm.AMDGPU.tex"; - bld_base->op_actions[TGSI_OPCODE_TEX].emit = llvm_emit_tex; - bld_base->op_actions[TGSI_OPCODE_TEX2].fetch_args = tex_fetch_args; - bld_base->op_actions[TGSI_OPCODE_TEX2].intr_name = "llvm.AMDGPU.tex"; - bld_base->op_actions[TGSI_OPCODE_TEX2].emit = llvm_emit_tex; - bld_base->op_actions[TGSI_OPCODE_TXB].fetch_args = tex_fetch_args; - bld_base->op_actions[TGSI_OPCODE_TXB].intr_name = "llvm.AMDGPU.txb"; - bld_base->op_actions[TGSI_OPCODE_TXB].emit = llvm_emit_tex; - bld_base->op_actions[TGSI_OPCODE_TXB2].fetch_args = tex_fetch_args; - bld_base->op_actions[TGSI_OPCODE_TXB2].intr_name = "llvm.AMDGPU.txb"; - bld_base->op_actions[TGSI_OPCODE_TXB2].emit = llvm_emit_tex; - bld_base->op_actions[TGSI_OPCODE_TXD].fetch_args = txd_fetch_args; - bld_base->op_actions[TGSI_OPCODE_TXD].intr_name = "llvm.AMDGPU.txd"; - bld_base->op_actions[TGSI_OPCODE_TXD].emit = llvm_emit_tex; - bld_base->op_actions[TGSI_OPCODE_TXF].fetch_args = txf_fetch_args; - bld_base->op_actions[TGSI_OPCODE_TXF].intr_name = "llvm.AMDGPU.txf"; - bld_base->op_actions[TGSI_OPCODE_TXF].emit = llvm_emit_tex; - bld_base->op_actions[TGSI_OPCODE_TXL].fetch_args = tex_fetch_args; - bld_base->op_actions[TGSI_OPCODE_TXL].intr_name = "llvm.AMDGPU.txl"; - bld_base->op_actions[TGSI_OPCODE_TXL].emit = llvm_emit_tex; - bld_base->op_actions[TGSI_OPCODE_TXL2].fetch_args = tex_fetch_args; - bld_base->op_actions[TGSI_OPCODE_TXL2].intr_name = "llvm.AMDGPU.txl"; - bld_base->op_actions[TGSI_OPCODE_TXL2].emit = llvm_emit_tex; - bld_base->op_actions[TGSI_OPCODE_TXP].fetch_args = txp_fetch_args; - bld_base->op_actions[TGSI_OPCODE_TXP].intr_name = "llvm.AMDGPU.tex"; - bld_base->op_actions[TGSI_OPCODE_TXP].emit = llvm_emit_tex; - bld_base->op_actions[TGSI_OPCODE_TXQ].fetch_args = tex_fetch_args; - bld_base->op_actions[TGSI_OPCODE_TXQ].intr_name = "llvm.AMDGPU.txq"; - bld_base->op_actions[TGSI_OPCODE_TXQ].emit = llvm_emit_tex; - bld_base->op_actions[TGSI_OPCODE_CMP].emit = emit_cndlt; - - lp_build_tgsi_llvm(bld_base, tokens); - - LLVMBuildRetVoid(bld_base->base.gallivm->builder); - radeon_llvm_finalize_module(ctx); - - return ctx->gallivm.module; -} - -/* We need to define these R600 registers here, because we can't include - * evergreend.h and r600d.h. - */ -#define R_028868_SQ_PGM_RESOURCES_VS 0x028868 -#define R_028850_SQ_PGM_RESOURCES_PS 0x028850 - -void r600_shader_binary_read_config(const struct radeon_shader_binary *binary, - struct r600_bytecode *bc, - uint64_t symbol_offset, - boolean *use_kill) -{ - unsigned i; - const unsigned char *config = - radeon_shader_binary_config_start(binary, symbol_offset); - - for (i = 0; i < binary->config_size_per_symbol; i+= 8) { - unsigned reg = - util_le32_to_cpu(*(uint32_t*)(config + i)); - unsigned value = - util_le32_to_cpu(*(uint32_t*)(config + i + 4)); - switch (reg) { - /* R600 / R700 */ - case R_028850_SQ_PGM_RESOURCES_PS: - case R_028868_SQ_PGM_RESOURCES_VS: - /* Evergreen / Northern Islands */ - case R_028844_SQ_PGM_RESOURCES_PS: - case R_028860_SQ_PGM_RESOURCES_VS: - case R_0288D4_SQ_PGM_RESOURCES_LS: - bc->ngpr = MAX2(bc->ngpr, G_028844_NUM_GPRS(value)); - bc->nstack = MAX2(bc->nstack, G_028844_STACK_SIZE(value)); - break; - case R_02880C_DB_SHADER_CONTROL: - *use_kill = G_02880C_KILL_ENABLE(value); - break; - case R_0288E8_SQ_LDS_ALLOC: - bc->nlds_dw = value; - break; - } - } - -} - -unsigned r600_create_shader(struct r600_bytecode *bc, - const struct radeon_shader_binary *binary, - boolean *use_kill) - -{ - assert(binary->code_size % 4 == 0); - bc->bytecode = CALLOC(1, binary->code_size); - memcpy(bc->bytecode, binary->code, binary->code_size); - bc->ndw = binary->code_size / 4; - - r600_shader_binary_read_config(binary, bc, 0, use_kill); - - return 0; -} - -void r600_destroy_shader(struct r600_bytecode *bc) -{ - FREE(bc->bytecode); -} - -unsigned r600_llvm_compile( - LLVMModuleRef mod, - enum radeon_family family, - struct r600_bytecode *bc, - boolean *use_kill, - unsigned dump, - struct pipe_debug_callback *debug) -{ - unsigned r; - struct radeon_shader_binary binary; - const char * gpu_family = r600_get_llvm_processor_name(family); - - radeon_shader_binary_init(&binary); - if (dump) - LLVMDumpModule(mod); - r = radeon_llvm_compile(mod, &binary, gpu_family, NULL, debug); - - r = r600_create_shader(bc, &binary, use_kill); - - radeon_shader_binary_clean(&binary); - - return r; -} - -#endif |