+++ /dev/null
-#include "r600_llvm.h"
-
-#include "gallivm/lp_bld_const.h"
-#include "gallivm/lp_bld_intr.h"
-#include "gallivm/lp_bld_gather.h"
-#include "tgsi/tgsi_parse.h"
-#include "util/list.h"
-#include "util/u_memory.h"
-
-#include "evergreend.h"
-#include "r600_asm.h"
-#include "r600_sq.h"
-#include "r600_opcodes.h"
-#include "r600_shader.h"
-#include "r600_pipe.h"
-#include "radeon_llvm.h"
-#include "radeon_llvm_emit.h"
-#include "radeon_elf_util.h"
-
-#include <stdio.h>
-
-#if defined R600_USE_LLVM || defined HAVE_OPENCL
-
-#define CONSTANT_BUFFER_0_ADDR_SPACE 8
-#define CONSTANT_BUFFER_1_ADDR_SPACE (CONSTANT_BUFFER_0_ADDR_SPACE + R600_BUFFER_INFO_CONST_BUFFER)
-#define LLVM_R600_BUFFER_INFO_CONST_BUFFER \
- (CONSTANT_BUFFER_0_ADDR_SPACE + R600_BUFFER_INFO_CONST_BUFFER)
-
-static LLVMValueRef llvm_load_const_buffer(
- struct lp_build_tgsi_context * bld_base,
- LLVMValueRef OffsetValue,
- unsigned ConstantAddressSpace)
-{
- LLVMValueRef offset[2] = {
- LLVMConstInt(LLVMInt64TypeInContext(bld_base->base.gallivm->context), 0, false),
- OffsetValue
- };
-
- LLVMTypeRef const_ptr_type = LLVMPointerType(LLVMArrayType(LLVMVectorType(bld_base->base.elem_type, 4), 1024),
- ConstantAddressSpace);
- LLVMValueRef const_ptr = LLVMBuildIntToPtr(bld_base->base.gallivm->builder, lp_build_const_int32(bld_base->base.gallivm, 0), const_ptr_type, "");
- LLVMValueRef ptr = LLVMBuildGEP(bld_base->base.gallivm->builder, const_ptr, offset, 2, "");
- return LLVMBuildLoad(bld_base->base.gallivm->builder, ptr, "");
-}
-
-static LLVMValueRef llvm_fetch_const(
- struct lp_build_tgsi_context * bld_base,
- const struct tgsi_full_src_register *reg,
- enum tgsi_opcode_type type,
- unsigned swizzle)
-{
- LLVMValueRef offset = lp_build_const_int32(bld_base->base.gallivm, reg->Register.Index);
- if (reg->Register.Indirect) {
- struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
- LLVMValueRef index = LLVMBuildLoad(bld_base->base.gallivm->builder, bld->addr[reg->Indirect.Index][reg->Indirect.Swizzle], "");
- offset = LLVMBuildAdd(bld_base->base.gallivm->builder, offset, index, "");
- }
- unsigned ConstantAddressSpace = CONSTANT_BUFFER_0_ADDR_SPACE ;
- if (reg->Register.Dimension) {
- ConstantAddressSpace += reg->Dimension.Index;
- }
- LLVMValueRef cvecval = llvm_load_const_buffer(bld_base, offset, ConstantAddressSpace);
- LLVMValueRef cval = LLVMBuildExtractElement(bld_base->base.gallivm->builder, cvecval, lp_build_const_int32(bld_base->base.gallivm, swizzle), "");
- return bitcast(bld_base, type, cval);
-}
-
-static void llvm_load_system_value(
- struct radeon_llvm_context * ctx,
- unsigned index,
- const struct tgsi_full_declaration *decl)
-{
- unsigned chan;
-
- switch (decl->Semantic.Name) {
- case TGSI_SEMANTIC_INSTANCEID: chan = 3; break;
- case TGSI_SEMANTIC_VERTEXID: chan = 0; break;
- default: assert(!"unknown system value");
- }
-
- ctx->system_values[index] = LLVMBuildExtractElement(ctx->gallivm.builder,
- LLVMGetParam(ctx->main_fn, 0), lp_build_const_int32(&(ctx->gallivm), chan),
- "");
-}
-
-static LLVMValueRef
-llvm_load_input_vector(
- struct radeon_llvm_context * ctx, unsigned location, unsigned ijregs,
- boolean interp)
-{
- LLVMTypeRef VecType;
- LLVMValueRef Args[3] = {
- lp_build_const_int32(&(ctx->gallivm), location)
- };
- unsigned ArgCount = 1;
- if (interp) {
- VecType = LLVMVectorType(ctx->soa.bld_base.base.elem_type, 2);
- LLVMValueRef IJIndex = LLVMGetParam(ctx->main_fn, ijregs / 2);
- Args[ArgCount++] = LLVMBuildExtractElement(ctx->gallivm.builder, IJIndex,
- lp_build_const_int32(&(ctx->gallivm), 2 * (ijregs % 2)), "");
- Args[ArgCount++] = LLVMBuildExtractElement(ctx->gallivm.builder, IJIndex,
- lp_build_const_int32(&(ctx->gallivm), 2 * (ijregs % 2) + 1), "");
- LLVMValueRef HalfVec[2] = {
- lp_build_intrinsic(ctx->gallivm.builder, "llvm.R600.interp.xy",
- VecType, Args, ArgCount, LLVMReadNoneAttribute),
- lp_build_intrinsic(ctx->gallivm.builder, "llvm.R600.interp.zw",
- VecType, Args, ArgCount, LLVMReadNoneAttribute)
- };
- LLVMValueRef MaskInputs[4] = {
- lp_build_const_int32(&(ctx->gallivm), 0),
- lp_build_const_int32(&(ctx->gallivm), 1),
- lp_build_const_int32(&(ctx->gallivm), 2),
- lp_build_const_int32(&(ctx->gallivm), 3)
- };
- LLVMValueRef Mask = LLVMConstVector(MaskInputs, 4);
- return LLVMBuildShuffleVector(ctx->gallivm.builder, HalfVec[0], HalfVec[1],
- Mask, "");
- } else {
- VecType = LLVMVectorType(ctx->soa.bld_base.base.elem_type, 4);
- return lp_build_intrinsic(ctx->gallivm.builder, "llvm.R600.interp.const",
- VecType, Args, ArgCount, LLVMReadNoneAttribute);
- }
-}
-
-static LLVMValueRef
-llvm_face_select_helper(
- struct radeon_llvm_context * ctx,
- LLVMValueRef face, LLVMValueRef front_color, LLVMValueRef back_color)
-{
- const struct lp_build_context * bb = &ctx->soa.bld_base.base;
- LLVMValueRef is_front = LLVMBuildFCmp(
- bb->gallivm->builder, LLVMRealUGT, face,
- lp_build_const_float(bb->gallivm, 0.0f), "");
- return LLVMBuildSelect(bb->gallivm->builder, is_front,
- front_color, back_color, "");
-}
-
-static void llvm_load_input(
- struct radeon_llvm_context * ctx,
- unsigned input_index,
- const struct tgsi_full_declaration *decl)
-{
- const struct r600_shader_io * input = &ctx->r600_inputs[input_index];
- unsigned chan;
- int two_side = (ctx->two_side && input->name == TGSI_SEMANTIC_COLOR);
- LLVMValueRef v;
- boolean require_interp_intrinsic = ctx->chip_class >= EVERGREEN &&
- ctx->type == TGSI_PROCESSOR_FRAGMENT;
-
- if (require_interp_intrinsic && input->spi_sid) {
- v = llvm_load_input_vector(ctx, input->lds_pos, input->ij_index,
- (input->interpolate > 0));
- } else
- v = LLVMGetParam(ctx->main_fn, input->gpr);
-
- if (two_side) {
- struct r600_shader_io * back_input =
- &ctx->r600_inputs[input->back_color_input];
- LLVMValueRef v2;
- LLVMValueRef face = LLVMGetParam(ctx->main_fn, ctx->face_gpr);
- face = LLVMBuildExtractElement(ctx->gallivm.builder, face,
- lp_build_const_int32(&(ctx->gallivm), 0), "");
-
- if (require_interp_intrinsic && back_input->spi_sid)
- v2 = llvm_load_input_vector(ctx, back_input->lds_pos,
- back_input->ij_index, (back_input->interpolate > 0));
- else
- v2 = LLVMGetParam(ctx->main_fn, back_input->gpr);
- v = llvm_face_select_helper(ctx, face, v, v2);
- }
-
- for (chan = 0; chan < 4; chan++) {
- unsigned soa_index = radeon_llvm_reg_index_soa(input_index, chan);
-
- ctx->inputs[soa_index] = LLVMBuildExtractElement(ctx->gallivm.builder, v,
- lp_build_const_int32(&(ctx->gallivm), chan), "");
-
- if (input->name == TGSI_SEMANTIC_POSITION &&
- ctx->type == TGSI_PROCESSOR_FRAGMENT && chan == 3) {
- /* RCP for fragcoord.w */
- ctx->inputs[soa_index] = LLVMBuildFDiv(ctx->gallivm.builder,
- lp_build_const_float(&(ctx->gallivm), 1.0f),
- ctx->inputs[soa_index], "");
- }
- }
-}
-
-static void llvm_emit_prologue(struct lp_build_tgsi_context * bld_base)
-{
- struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
- radeon_llvm_shader_type(ctx->main_fn, ctx->type);
-
-}
-
-static void llvm_emit_epilogue(struct lp_build_tgsi_context * bld_base)
-{
- struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
- struct lp_build_context * base = &bld_base->base;
- struct pipe_stream_output_info * so = ctx->stream_outputs;
- unsigned i;
- unsigned next_pos = 60;
- unsigned next_param = 0;
-
- unsigned color_count = 0;
- boolean has_color = false;
-
- if (ctx->type == TGSI_PROCESSOR_VERTEX && so->num_outputs) {
- for (i = 0; i < so->num_outputs; i++) {
- unsigned register_index = so->output[i].register_index;
- unsigned start_component = so->output[i].start_component;
- unsigned num_components = so->output[i].num_components;
- unsigned dst_offset = so->output[i].dst_offset;
- unsigned chan;
- LLVMValueRef elements[4];
- if (dst_offset < start_component) {
- for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
- elements[chan] = LLVMBuildLoad(base->gallivm->builder,
- ctx->soa.outputs[register_index][(chan + start_component) % TGSI_NUM_CHANNELS], "");
- }
- start_component = 0;
- } else {
- for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
- elements[chan] = LLVMBuildLoad(base->gallivm->builder,
- ctx->soa.outputs[register_index][chan], "");
- }
- }
- LLVMValueRef output = lp_build_gather_values(base->gallivm, elements, 4);
- LLVMValueRef args[4];
- args[0] = output;
- args[1] = lp_build_const_int32(base->gallivm, dst_offset - start_component);
- args[2] = lp_build_const_int32(base->gallivm, so->output[i].output_buffer);
- args[3] = lp_build_const_int32(base->gallivm, ((1 << num_components) - 1) << start_component);
- lp_build_intrinsic(base->gallivm->builder, "llvm.R600.store.stream.output",
- LLVMVoidTypeInContext(base->gallivm->context), args, 4, 0);
- }
- }
-
- /* Add the necessary export instructions */
- for (i = 0; i < ctx->output_reg_count; i++) {
- unsigned chan;
- LLVMValueRef elements[4];
- for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
- elements[chan] = LLVMBuildLoad(base->gallivm->builder,
- ctx->soa.outputs[i][chan], "");
- }
- if (ctx->alpha_to_one && ctx->type == TGSI_PROCESSOR_FRAGMENT && ctx->r600_outputs[i].name == TGSI_SEMANTIC_COLOR)
- elements[3] = lp_build_const_float(base->gallivm, 1.0f);
- LLVMValueRef output = lp_build_gather_values(base->gallivm, elements, 4);
-
- if (ctx->type == TGSI_PROCESSOR_VERTEX) {
- switch (ctx->r600_outputs[i].name) {
- case TGSI_SEMANTIC_POSITION:
- case TGSI_SEMANTIC_PSIZE: {
- LLVMValueRef args[3];
- args[0] = output;
- args[1] = lp_build_const_int32(base->gallivm, next_pos++);
- args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_POS);
- lp_build_intrinsic(
- base->gallivm->builder,
- "llvm.R600.store.swizzle",
- LLVMVoidTypeInContext(base->gallivm->context),
- args, 3, 0);
- break;
- }
- case TGSI_SEMANTIC_CLIPVERTEX: {
- LLVMValueRef args[3];
- unsigned reg_index;
- LLVMValueRef adjusted_elements[4];
- for (reg_index = 0; reg_index < 2; reg_index ++) {
- for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
- LLVMValueRef offset = lp_build_const_int32(bld_base->base.gallivm, reg_index * 4 + chan);
- LLVMValueRef base_vector = llvm_load_const_buffer(bld_base, offset, CONSTANT_BUFFER_1_ADDR_SPACE);
- args[0] = output;
- args[1] = base_vector;
- adjusted_elements[chan] = lp_build_intrinsic(base->gallivm->builder,
- "llvm.AMDGPU.dp4", bld_base->base.elem_type,
- args, 2, LLVMReadNoneAttribute);
- }
- args[0] = lp_build_gather_values(base->gallivm,
- adjusted_elements, 4);
- args[1] = lp_build_const_int32(base->gallivm, next_pos++);
- args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_POS);
- lp_build_intrinsic(
- base->gallivm->builder,
- "llvm.R600.store.swizzle",
- LLVMVoidTypeInContext(base->gallivm->context),
- args, 3, 0);
- }
- break;
- }
- case TGSI_SEMANTIC_CLIPDIST : {
- LLVMValueRef args[3];
- args[0] = output;
- args[1] = lp_build_const_int32(base->gallivm, next_pos++);
- args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_POS);
- lp_build_intrinsic(
- base->gallivm->builder,
- "llvm.R600.store.swizzle",
- LLVMVoidTypeInContext(base->gallivm->context),
- args, 3, 0);
- args[1] = lp_build_const_int32(base->gallivm, next_param++);
- args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PARAM);
- lp_build_intrinsic(
- base->gallivm->builder,
- "llvm.R600.store.swizzle",
- LLVMVoidTypeInContext(base->gallivm->context),
- args, 3, 0);
- break;
- }
- case TGSI_SEMANTIC_FOG: {
- elements[0] = LLVMBuildLoad(base->gallivm->builder,
- ctx->soa.outputs[i][0], "");
- elements[1] = elements[2] = lp_build_const_float(base->gallivm, 0.0f);
- elements[3] = lp_build_const_float(base->gallivm, 1.0f);
-
- LLVMValueRef args[3];
- args[0] = lp_build_gather_values(base->gallivm, elements, 4);
- args[1] = lp_build_const_int32(base->gallivm, next_param++);
- args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PARAM);
- lp_build_intrinsic(
- base->gallivm->builder,
- "llvm.R600.store.swizzle",
- LLVMVoidTypeInContext(base->gallivm->context),
- args, 3, 0);
- break;
- }
- default: {
- LLVMValueRef args[3];
- args[0] = output;
- args[1] = lp_build_const_int32(base->gallivm, next_param++);
- args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PARAM);
- lp_build_intrinsic(
- base->gallivm->builder,
- "llvm.R600.store.swizzle",
- LLVMVoidTypeInContext(base->gallivm->context),
- args, 3, 0);
- break;
- }
- }
- } else if (ctx->type == TGSI_PROCESSOR_FRAGMENT) {
- switch (ctx->r600_outputs[i].name) {
- case TGSI_SEMANTIC_COLOR:
- has_color = true;
- if ( color_count < ctx->color_buffer_count) {
- LLVMValueRef args[3];
- args[0] = output;
- if (ctx->fs_color_all) {
- for (unsigned j = 0; j < ctx->color_buffer_count; j++) {
- args[1] = lp_build_const_int32(base->gallivm, j);
- args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PIXEL);
- lp_build_intrinsic(
- base->gallivm->builder,
- "llvm.R600.store.swizzle",
- LLVMVoidTypeInContext(base->gallivm->context),
- args, 3, 0);
- }
- } else {
- args[1] = lp_build_const_int32(base->gallivm, color_count++);
- args[2] = lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PIXEL);
- lp_build_intrinsic(
- base->gallivm->builder,
- "llvm.R600.store.swizzle",
- LLVMVoidTypeInContext(base->gallivm->context),
- args, 3, 0);
- }
- }
- break;
- case TGSI_SEMANTIC_POSITION:
- lp_build_intrinsic_unary(
- base->gallivm->builder,
- "llvm.R600.store.pixel.depth",
- LLVMVoidTypeInContext(base->gallivm->context),
- LLVMBuildLoad(base->gallivm->builder, ctx->soa.outputs[i][2], ""));
- break;
- case TGSI_SEMANTIC_STENCIL:
- lp_build_intrinsic_unary(
- base->gallivm->builder,
- "llvm.R600.store.pixel.stencil",
- LLVMVoidTypeInContext(base->gallivm->context),
- LLVMBuildLoad(base->gallivm->builder, ctx->soa.outputs[i][1], ""));
- break;
- }
- }
- }
- // Add dummy exports
- if (ctx->type == TGSI_PROCESSOR_VERTEX) {
- if (!next_param) {
- lp_build_intrinsic_unary(base->gallivm->builder, "llvm.R600.store.dummy",
- LLVMVoidTypeInContext(base->gallivm->context),
- lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PARAM));
- }
- if (!(next_pos-60)) {
- lp_build_intrinsic_unary(base->gallivm->builder, "llvm.R600.store.dummy",
- LLVMVoidTypeInContext(base->gallivm->context),
- lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_POS));
- }
- }
- if (ctx->type == TGSI_PROCESSOR_FRAGMENT) {
- if (!has_color) {
- lp_build_intrinsic_unary(base->gallivm->builder, "llvm.R600.store.dummy",
- LLVMVoidTypeInContext(base->gallivm->context),
- lp_build_const_int32(base->gallivm, V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_PIXEL));
- }
- }
-
-}
-
-static void llvm_emit_tex(
- const struct lp_build_tgsi_action * action,
- struct lp_build_tgsi_context * bld_base,
- struct lp_build_emit_data * emit_data)
-{
- struct gallivm_state * gallivm = bld_base->base.gallivm;
- LLVMValueRef args[7];
- unsigned c, sampler_src;
- struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
-
- if (emit_data->inst->Texture.Texture == TGSI_TEXTURE_BUFFER) {
- switch (emit_data->inst->Instruction.Opcode) {
- case TGSI_OPCODE_TXQ: {
- struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
- ctx->uses_tex_buffers = true;
- bool isEgPlus = (ctx->chip_class >= EVERGREEN);
- LLVMValueRef offset = lp_build_const_int32(bld_base->base.gallivm,
- isEgPlus ? 0 : 1);
- LLVMValueRef cvecval = llvm_load_const_buffer(bld_base, offset,
- LLVM_R600_BUFFER_INFO_CONST_BUFFER);
- if (!isEgPlus) {
- LLVMValueRef maskval[4] = {
- lp_build_const_int32(gallivm, 1),
- lp_build_const_int32(gallivm, 2),
- lp_build_const_int32(gallivm, 3),
- lp_build_const_int32(gallivm, 0),
- };
- LLVMValueRef mask = LLVMConstVector(maskval, 4);
- cvecval = LLVMBuildShuffleVector(gallivm->builder, cvecval, cvecval,
- mask, "");
- }
- emit_data->output[0] = cvecval;
- return;
- }
- case TGSI_OPCODE_TXF: {
- args[0] = LLVMBuildExtractElement(gallivm->builder, emit_data->args[0], lp_build_const_int32(gallivm, 0), "");
- args[1] = lp_build_const_int32(gallivm, R600_MAX_CONST_BUFFERS);
- emit_data->output[0] = lp_build_intrinsic(gallivm->builder,
- "llvm.R600.load.texbuf",
- emit_data->dst_type, args, 2, LLVMReadNoneAttribute);
- if (ctx->chip_class >= EVERGREEN)
- return;
- ctx->uses_tex_buffers = true;
- LLVMDumpValue(emit_data->output[0]);
- emit_data->output[0] = LLVMBuildBitCast(gallivm->builder,
- emit_data->output[0], LLVMVectorType(bld_base->base.int_elem_type, 4),
- "");
- LLVMValueRef Mask = llvm_load_const_buffer(bld_base,
- lp_build_const_int32(gallivm, 0),
- LLVM_R600_BUFFER_INFO_CONST_BUFFER);
- Mask = LLVMBuildBitCast(gallivm->builder, Mask,
- LLVMVectorType(bld_base->base.int_elem_type, 4), "");
- emit_data->output[0] = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_AND,
- emit_data->output[0],
- Mask);
- LLVMValueRef WComponent = LLVMBuildExtractElement(gallivm->builder,
- emit_data->output[0], lp_build_const_int32(gallivm, 3), "");
- Mask = llvm_load_const_buffer(bld_base, lp_build_const_int32(gallivm, 1),
- LLVM_R600_BUFFER_INFO_CONST_BUFFER);
- Mask = LLVMBuildExtractElement(gallivm->builder, Mask,
- lp_build_const_int32(gallivm, 0), "");
- Mask = LLVMBuildBitCast(gallivm->builder, Mask,
- bld_base->base.int_elem_type, "");
- WComponent = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_OR,
- WComponent, Mask);
- emit_data->output[0] = LLVMBuildInsertElement(gallivm->builder,
- emit_data->output[0], WComponent, lp_build_const_int32(gallivm, 3), "");
- emit_data->output[0] = LLVMBuildBitCast(gallivm->builder,
- emit_data->output[0], LLVMVectorType(bld_base->base.elem_type, 4), "");
- }
- return;
- default:
- break;
- }
- }
-
- if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_TEX ||
- emit_data->inst->Instruction.Opcode == TGSI_OPCODE_TXP) {
- LLVMValueRef Vector[4] = {
- LLVMBuildExtractElement(gallivm->builder, emit_data->args[0],
- lp_build_const_int32(gallivm, 0), ""),
- LLVMBuildExtractElement(gallivm->builder, emit_data->args[0],
- lp_build_const_int32(gallivm, 1), ""),
- LLVMBuildExtractElement(gallivm->builder, emit_data->args[0],
- lp_build_const_int32(gallivm, 2), ""),
- LLVMBuildExtractElement(gallivm->builder, emit_data->args[0],
- lp_build_const_int32(gallivm, 3), ""),
- };
- switch (emit_data->inst->Texture.Texture) {
- case TGSI_TEXTURE_2D:
- case TGSI_TEXTURE_RECT:
- Vector[2] = Vector[3] = LLVMGetUndef(bld_base->base.elem_type);
- break;
- case TGSI_TEXTURE_1D:
- Vector[1] = Vector[2] = Vector[3] = LLVMGetUndef(bld_base->base.elem_type);
- break;
- default:
- break;
- }
- args[0] = lp_build_gather_values(gallivm, Vector, 4);
- } else {
- args[0] = emit_data->args[0];
- }
-
- assert(emit_data->arg_count + 2 <= Elements(args));
-
- for (c = 1; c < emit_data->arg_count; ++c)
- args[c] = emit_data->args[c];
-
- if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_TXF) {
- args[1] = LLVMBuildShl(gallivm->builder, args[1], lp_build_const_int32(gallivm, 1), "");
- args[2] = LLVMBuildShl(gallivm->builder, args[2], lp_build_const_int32(gallivm, 1), "");
- args[3] = LLVMBuildShl(gallivm->builder, args[3], lp_build_const_int32(gallivm, 1), "");
- }
-
- sampler_src = emit_data->inst->Instruction.NumSrcRegs-1;
-
- args[c++] = lp_build_const_int32(gallivm,
- emit_data->inst->Src[sampler_src].Register.Index + R600_MAX_CONST_BUFFERS);
- args[c++] = lp_build_const_int32(gallivm,
- emit_data->inst->Src[sampler_src].Register.Index);
- args[c++] = lp_build_const_int32(gallivm,
- emit_data->inst->Texture.Texture);
-
- if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_TXF &&
- (emit_data->inst->Texture.Texture == TGSI_TEXTURE_2D_MSAA ||
- emit_data->inst->Texture.Texture == TGSI_TEXTURE_2D_ARRAY_MSAA)) {
-
- switch (emit_data->inst->Texture.Texture) {
- case TGSI_TEXTURE_2D_MSAA:
- args[6] = lp_build_const_int32(gallivm, TGSI_TEXTURE_2D);
- break;
- case TGSI_TEXTURE_2D_ARRAY_MSAA:
- args[6] = lp_build_const_int32(gallivm, TGSI_TEXTURE_2D_ARRAY);
- break;
- default:
- break;
- }
-
- if (ctx->has_compressed_msaa_texturing) {
- LLVMValueRef ldptr_args[10] = {
- args[0], // Coord
- args[1], // Offset X
- args[2], // Offset Y
- args[3], // Offset Z
- args[4],
- args[5],
- lp_build_const_int32(gallivm, 1),
- lp_build_const_int32(gallivm, 1),
- lp_build_const_int32(gallivm, 1),
- lp_build_const_int32(gallivm, 1)
- };
- LLVMValueRef ptr = lp_build_intrinsic(gallivm->builder,
- "llvm.R600.ldptr",
- emit_data->dst_type, ldptr_args, 10, LLVMReadNoneAttribute);
- LLVMValueRef Tmp = LLVMBuildExtractElement(gallivm->builder, args[0],
- lp_build_const_int32(gallivm, 3), "");
- Tmp = LLVMBuildMul(gallivm->builder, Tmp,
- lp_build_const_int32(gallivm, 4), "");
- LLVMValueRef ResX = LLVMBuildExtractElement(gallivm->builder, ptr,
- lp_build_const_int32(gallivm, 0), "");
- ResX = LLVMBuildBitCast(gallivm->builder, ResX,
- bld_base->base.int_elem_type, "");
- Tmp = LLVMBuildLShr(gallivm->builder, ResX, Tmp, "");
- Tmp = LLVMBuildAnd(gallivm->builder, Tmp,
- lp_build_const_int32(gallivm, 0xF), "");
- args[0] = LLVMBuildInsertElement(gallivm->builder, args[0], Tmp,
- lp_build_const_int32(gallivm, 3), "");
- args[c++] = lp_build_const_int32(gallivm,
- emit_data->inst->Texture.Texture);
- }
- }
-
- emit_data->output[0] = lp_build_intrinsic(gallivm->builder,
- action->intr_name,
- emit_data->dst_type, args, c, LLVMReadNoneAttribute);
-
- if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_TXQ &&
- ((emit_data->inst->Texture.Texture == TGSI_TEXTURE_CUBE_ARRAY ||
- emit_data->inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE_ARRAY)))
- if (emit_data->inst->Dst[0].Register.WriteMask & 4) {
- LLVMValueRef offset = lp_build_const_int32(bld_base->base.gallivm, 0);
- LLVMValueRef ZLayer = LLVMBuildExtractElement(gallivm->builder,
- llvm_load_const_buffer(bld_base, offset, LLVM_R600_BUFFER_INFO_CONST_BUFFER),
- lp_build_const_int32(gallivm, 0), "");
-
- emit_data->output[0] = LLVMBuildInsertElement(gallivm->builder, emit_data->output[0], ZLayer, lp_build_const_int32(gallivm, 2), "");
- struct radeon_llvm_context * ctx = radeon_llvm_context(bld_base);
- ctx->has_txq_cube_array_z_comp = true;
- }
-}
-
-static void emit_cndlt(
- const struct lp_build_tgsi_action * action,
- struct lp_build_tgsi_context * bld_base,
- struct lp_build_emit_data * emit_data)
-{
- LLVMBuilderRef builder = bld_base->base.gallivm->builder;
- LLVMValueRef float_zero = lp_build_const_float(
- bld_base->base.gallivm, 0.0f);
- LLVMValueRef cmp = LLVMBuildFCmp(
- builder, LLVMRealULT, emit_data->args[0], float_zero, "");
- emit_data->output[emit_data->chan] = LLVMBuildSelect(builder,
- cmp, emit_data->args[1], emit_data->args[2], "");
-}
-
-static void dp_fetch_args(
- struct lp_build_tgsi_context * bld_base,
- struct lp_build_emit_data * emit_data)
-{
- struct lp_build_context * base = &bld_base->base;
- unsigned chan;
- LLVMValueRef elements[2][4];
- unsigned opcode = emit_data->inst->Instruction.Opcode;
- unsigned dp_components = (opcode == TGSI_OPCODE_DP2 ? 2 :
- (opcode == TGSI_OPCODE_DP3 ? 3 : 4));
- for (chan = 0 ; chan < dp_components; chan++) {
- elements[0][chan] = lp_build_emit_fetch(bld_base,
- emit_data->inst, 0, chan);
- elements[1][chan] = lp_build_emit_fetch(bld_base,
- emit_data->inst, 1, chan);
- }
-
- for ( ; chan < 4; chan++) {
- elements[0][chan] = base->zero;
- elements[1][chan] = base->zero;
- }
-
- /* Fix up for DPH */
- if (opcode == TGSI_OPCODE_DPH) {
- elements[0][TGSI_CHAN_W] = base->one;
- }
-
- emit_data->args[0] = lp_build_gather_values(bld_base->base.gallivm,
- elements[0], 4);
- emit_data->args[1] = lp_build_gather_values(bld_base->base.gallivm,
- elements[1], 4);
- emit_data->arg_count = 2;
-
- emit_data->dst_type = base->elem_type;
-}
-
-static struct lp_build_tgsi_action dot_action = {
- .fetch_args = dp_fetch_args,
- .emit = build_tgsi_intrinsic_nomem,
- .intr_name = "llvm.AMDGPU.dp4"
-};
-
-static void txd_fetch_args(
- struct lp_build_tgsi_context * bld_base,
- struct lp_build_emit_data * emit_data)
-{
- const struct tgsi_full_instruction * inst = emit_data->inst;
-
- LLVMValueRef coords[4];
- unsigned chan, src;
- for (src = 0; src < 3; src++) {
- for (chan = 0; chan < 4; chan++)
- coords[chan] = lp_build_emit_fetch(bld_base, inst, src, chan);
-
- emit_data->args[src] = lp_build_gather_values(bld_base->base.gallivm,
- coords, 4);
- }
- emit_data->arg_count = 3;
- emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4);
-}
-
-
-static void txp_fetch_args(
- struct lp_build_tgsi_context * bld_base,
- struct lp_build_emit_data * emit_data)
-{
- const struct tgsi_full_instruction * inst = emit_data->inst;
- LLVMValueRef src_w;
- unsigned chan;
- LLVMValueRef coords[5];
-
- emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4);
- src_w = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_W);
-
- for (chan = 0; chan < 3; chan++ ) {
- LLVMValueRef arg = lp_build_emit_fetch(bld_base,
- emit_data->inst, 0, chan);
- coords[chan] = lp_build_emit_llvm_binary(bld_base,
- TGSI_OPCODE_DIV, arg, src_w);
- }
- coords[3] = bld_base->base.one;
-
- if ((inst->Texture.Texture == TGSI_TEXTURE_CUBE ||
- inst->Texture.Texture == TGSI_TEXTURE_CUBE_ARRAY ||
- inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE ||
- inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE_ARRAY) &&
- inst->Instruction.Opcode != TGSI_OPCODE_TXQ &&
- inst->Instruction.Opcode != TGSI_OPCODE_TXQ_LZ) {
- radeon_llvm_emit_prepare_cube_coords(bld_base, emit_data, coords, NULL);
- }
-
- emit_data->args[0] = lp_build_gather_values(bld_base->base.gallivm,
- coords, 4);
- emit_data->arg_count = 1;
-}
-
-static void tex_fetch_args(
- struct lp_build_tgsi_context * bld_base,
- struct lp_build_emit_data * emit_data)
-{
- const struct tgsi_full_instruction * inst = emit_data->inst;
-
- LLVMValueRef coords[5];
- unsigned chan;
- for (chan = 0; chan < 4; chan++) {
- coords[chan] = lp_build_emit_fetch(bld_base, inst, 0, chan);
- }
-
- if (inst->Instruction.Opcode == TGSI_OPCODE_TEX2 ||
- inst->Instruction.Opcode == TGSI_OPCODE_TXB2 ||
- inst->Instruction.Opcode == TGSI_OPCODE_TXL2) {
- /* These instructions have additional operand that should be packed
- * into the cube coord vector by radeon_llvm_emit_prepare_cube_coords.
- * That operand should be passed as a float value in the args array
- * right after the coord vector. After packing it's not used anymore,
- * that's why arg_count is not increased */
- coords[4] = lp_build_emit_fetch(bld_base, inst, 1, TGSI_CHAN_X);
- }
-
- if ((inst->Texture.Texture == TGSI_TEXTURE_CUBE ||
- inst->Texture.Texture == TGSI_TEXTURE_CUBE_ARRAY ||
- inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE ||
- inst->Texture.Texture == TGSI_TEXTURE_SHADOWCUBE_ARRAY) &&
- inst->Instruction.Opcode != TGSI_OPCODE_TXQ &&
- inst->Instruction.Opcode != TGSI_OPCODE_TXQ_LZ) {
- radeon_llvm_emit_prepare_cube_coords(bld_base, emit_data, coords, NULL);
- }
-
- emit_data->arg_count = 1;
- emit_data->args[0] = lp_build_gather_values(bld_base->base.gallivm,
- coords, 4);
- emit_data->dst_type = LLVMVectorType(bld_base->base.elem_type, 4);
-}
-
-static void txf_fetch_args(
- struct lp_build_tgsi_context * bld_base,
- struct lp_build_emit_data * emit_data)
-{
- const struct tgsi_full_instruction * inst = emit_data->inst;
- struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
- const struct tgsi_texture_offset * off = inst->TexOffsets;
- LLVMTypeRef offset_type = bld_base->int_bld.elem_type;
-
- /* fetch tex coords */
- tex_fetch_args(bld_base, emit_data);
-
- /* fetch tex offsets */
- if (inst->Texture.NumOffsets) {
- assert(inst->Texture.NumOffsets == 1);
-
- emit_data->args[1] = LLVMConstBitCast(
- bld->immediates[off->Index][off->SwizzleX],
- offset_type);
- emit_data->args[2] = LLVMConstBitCast(
- bld->immediates[off->Index][off->SwizzleY],
- offset_type);
- emit_data->args[3] = LLVMConstBitCast(
- bld->immediates[off->Index][off->SwizzleZ],
- offset_type);
- } else {
- emit_data->args[1] = bld_base->int_bld.zero;
- emit_data->args[2] = bld_base->int_bld.zero;
- emit_data->args[3] = bld_base->int_bld.zero;
- }
-
- emit_data->arg_count = 4;
-}
-
-LLVMModuleRef r600_tgsi_llvm(
- struct radeon_llvm_context * ctx,
- const struct tgsi_token * tokens)
-{
- struct tgsi_shader_info shader_info;
- struct lp_build_tgsi_context * bld_base = &ctx->soa.bld_base;
- radeon_llvm_context_init(ctx, "r600--");
- LLVMTypeRef Arguments[32];
- unsigned ArgumentsCount = 0;
- for (unsigned i = 0; i < ctx->inputs_count; i++)
- Arguments[ArgumentsCount++] = LLVMVectorType(bld_base->base.elem_type, 4);
- radeon_llvm_create_func(ctx, NULL, 0, Arguments, ArgumentsCount);
- for (unsigned i = 0; i < ctx->inputs_count; i++) {
- LLVMValueRef P = LLVMGetParam(ctx->main_fn, i);
- LLVMAddAttribute(P, LLVMInRegAttribute);
- }
- tgsi_scan_shader(tokens, &shader_info);
-
- bld_base->info = &shader_info;
- bld_base->userdata = ctx;
- bld_base->emit_fetch_funcs[TGSI_FILE_CONSTANT] = llvm_fetch_const;
- bld_base->emit_prologue = llvm_emit_prologue;
- bld_base->emit_epilogue = llvm_emit_epilogue;
- ctx->load_input = llvm_load_input;
- ctx->load_system_value = llvm_load_system_value;
-
- bld_base->op_actions[TGSI_OPCODE_DP2] = dot_action;
- bld_base->op_actions[TGSI_OPCODE_DP3] = dot_action;
- bld_base->op_actions[TGSI_OPCODE_DP4] = dot_action;
- bld_base->op_actions[TGSI_OPCODE_DPH] = dot_action;
- bld_base->op_actions[TGSI_OPCODE_DDX].intr_name = "llvm.AMDGPU.ddx";
- bld_base->op_actions[TGSI_OPCODE_DDX].fetch_args = tex_fetch_args;
- bld_base->op_actions[TGSI_OPCODE_DDX].emit = llvm_emit_tex;
- bld_base->op_actions[TGSI_OPCODE_DDY].intr_name = "llvm.AMDGPU.ddy";
- bld_base->op_actions[TGSI_OPCODE_DDY].fetch_args = tex_fetch_args;
- bld_base->op_actions[TGSI_OPCODE_DDY].emit = llvm_emit_tex;
- bld_base->op_actions[TGSI_OPCODE_TEX].fetch_args = tex_fetch_args;
- bld_base->op_actions[TGSI_OPCODE_TEX].intr_name = "llvm.AMDGPU.tex";
- bld_base->op_actions[TGSI_OPCODE_TEX].emit = llvm_emit_tex;
- bld_base->op_actions[TGSI_OPCODE_TEX2].fetch_args = tex_fetch_args;
- bld_base->op_actions[TGSI_OPCODE_TEX2].intr_name = "llvm.AMDGPU.tex";
- bld_base->op_actions[TGSI_OPCODE_TEX2].emit = llvm_emit_tex;
- bld_base->op_actions[TGSI_OPCODE_TXB].fetch_args = tex_fetch_args;
- bld_base->op_actions[TGSI_OPCODE_TXB].intr_name = "llvm.AMDGPU.txb";
- bld_base->op_actions[TGSI_OPCODE_TXB].emit = llvm_emit_tex;
- bld_base->op_actions[TGSI_OPCODE_TXB2].fetch_args = tex_fetch_args;
- bld_base->op_actions[TGSI_OPCODE_TXB2].intr_name = "llvm.AMDGPU.txb";
- bld_base->op_actions[TGSI_OPCODE_TXB2].emit = llvm_emit_tex;
- bld_base->op_actions[TGSI_OPCODE_TXD].fetch_args = txd_fetch_args;
- bld_base->op_actions[TGSI_OPCODE_TXD].intr_name = "llvm.AMDGPU.txd";
- bld_base->op_actions[TGSI_OPCODE_TXD].emit = llvm_emit_tex;
- bld_base->op_actions[TGSI_OPCODE_TXF].fetch_args = txf_fetch_args;
- bld_base->op_actions[TGSI_OPCODE_TXF].intr_name = "llvm.AMDGPU.txf";
- bld_base->op_actions[TGSI_OPCODE_TXF].emit = llvm_emit_tex;
- bld_base->op_actions[TGSI_OPCODE_TXL].fetch_args = tex_fetch_args;
- bld_base->op_actions[TGSI_OPCODE_TXL].intr_name = "llvm.AMDGPU.txl";
- bld_base->op_actions[TGSI_OPCODE_TXL].emit = llvm_emit_tex;
- bld_base->op_actions[TGSI_OPCODE_TXL2].fetch_args = tex_fetch_args;
- bld_base->op_actions[TGSI_OPCODE_TXL2].intr_name = "llvm.AMDGPU.txl";
- bld_base->op_actions[TGSI_OPCODE_TXL2].emit = llvm_emit_tex;
- bld_base->op_actions[TGSI_OPCODE_TXP].fetch_args = txp_fetch_args;
- bld_base->op_actions[TGSI_OPCODE_TXP].intr_name = "llvm.AMDGPU.tex";
- bld_base->op_actions[TGSI_OPCODE_TXP].emit = llvm_emit_tex;
- bld_base->op_actions[TGSI_OPCODE_TXQ].fetch_args = tex_fetch_args;
- bld_base->op_actions[TGSI_OPCODE_TXQ].intr_name = "llvm.AMDGPU.txq";
- bld_base->op_actions[TGSI_OPCODE_TXQ].emit = llvm_emit_tex;
- bld_base->op_actions[TGSI_OPCODE_CMP].emit = emit_cndlt;
-
- lp_build_tgsi_llvm(bld_base, tokens);
-
- LLVMBuildRetVoid(bld_base->base.gallivm->builder);
- radeon_llvm_finalize_module(ctx);
-
- return ctx->gallivm.module;
-}
-
-/* We need to define these R600 registers here, because we can't include
- * evergreend.h and r600d.h.
- */
-#define R_028868_SQ_PGM_RESOURCES_VS 0x028868
-#define R_028850_SQ_PGM_RESOURCES_PS 0x028850
-
-void r600_shader_binary_read_config(const struct radeon_shader_binary *binary,
- struct r600_bytecode *bc,
- uint64_t symbol_offset,
- boolean *use_kill)
-{
- unsigned i;
- const unsigned char *config =
- radeon_shader_binary_config_start(binary, symbol_offset);
-
- for (i = 0; i < binary->config_size_per_symbol; i+= 8) {
- unsigned reg =
- util_le32_to_cpu(*(uint32_t*)(config + i));
- unsigned value =
- util_le32_to_cpu(*(uint32_t*)(config + i + 4));
- switch (reg) {
- /* R600 / R700 */
- case R_028850_SQ_PGM_RESOURCES_PS:
- case R_028868_SQ_PGM_RESOURCES_VS:
- /* Evergreen / Northern Islands */
- case R_028844_SQ_PGM_RESOURCES_PS:
- case R_028860_SQ_PGM_RESOURCES_VS:
- case R_0288D4_SQ_PGM_RESOURCES_LS:
- bc->ngpr = MAX2(bc->ngpr, G_028844_NUM_GPRS(value));
- bc->nstack = MAX2(bc->nstack, G_028844_STACK_SIZE(value));
- break;
- case R_02880C_DB_SHADER_CONTROL:
- *use_kill = G_02880C_KILL_ENABLE(value);
- break;
- case R_0288E8_SQ_LDS_ALLOC:
- bc->nlds_dw = value;
- break;
- }
- }
-
-}
-
-unsigned r600_create_shader(struct r600_bytecode *bc,
- const struct radeon_shader_binary *binary,
- boolean *use_kill)
-
-{
- assert(binary->code_size % 4 == 0);
- bc->bytecode = CALLOC(1, binary->code_size);
- memcpy(bc->bytecode, binary->code, binary->code_size);
- bc->ndw = binary->code_size / 4;
-
- r600_shader_binary_read_config(binary, bc, 0, use_kill);
-
- return 0;
-}
-
-void r600_destroy_shader(struct r600_bytecode *bc)
-{
- FREE(bc->bytecode);
-}
-
-unsigned r600_llvm_compile(
- LLVMModuleRef mod,
- enum radeon_family family,
- struct r600_bytecode *bc,
- boolean *use_kill,
- unsigned dump,
- struct pipe_debug_callback *debug)
-{
- unsigned r;
- struct radeon_shader_binary binary;
- const char * gpu_family = r600_get_llvm_processor_name(family);
-
- radeon_shader_binary_init(&binary);
- if (dump)
- LLVMDumpModule(mod);
- r = radeon_llvm_compile(mod, &binary, gpu_family, NULL, debug);
-
- r = r600_create_shader(bc, &binary, use_kill);
-
- radeon_shader_binary_clean(&binary);
-
- return r;
-}
-
-#endif
projects / mesa.git / blobdiff