-static void membar_emit(
- const struct lp_build_tgsi_action *action,
- struct lp_build_tgsi_context *bld_base,
- struct lp_build_emit_data *emit_data)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- LLVMValueRef src0 = lp_build_emit_fetch(bld_base, emit_data->inst, 0, 0);
- unsigned flags = LLVMConstIntGetZExtValue(src0);
- unsigned wait_flags = 0;
-
- if (flags & TGSI_MEMBAR_THREAD_GROUP)
- wait_flags |= AC_WAIT_LGKM | AC_WAIT_VLOAD | AC_WAIT_VSTORE;
-
- if (flags & (TGSI_MEMBAR_ATOMIC_BUFFER |
- TGSI_MEMBAR_SHADER_BUFFER |
- TGSI_MEMBAR_SHADER_IMAGE))
- wait_flags |= AC_WAIT_VLOAD | AC_WAIT_VSTORE;
-
- if (flags & TGSI_MEMBAR_SHARED)
- wait_flags |= AC_WAIT_LGKM;
-
- ac_build_waitcnt(&ctx->ac, wait_flags);
-}
-
-static void clock_emit(
- const struct lp_build_tgsi_action *action,
- struct lp_build_tgsi_context *bld_base,
- struct lp_build_emit_data *emit_data)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- LLVMValueRef tmp = ac_build_shader_clock(&ctx->ac);
-
- emit_data->output[0] =
- LLVMBuildExtractElement(ctx->ac.builder, tmp, ctx->i32_0, "");
- emit_data->output[1] =
- LLVMBuildExtractElement(ctx->ac.builder, tmp, ctx->i32_1, "");
-}
-
-static void si_llvm_emit_ddxy(
- const struct lp_build_tgsi_action *action,
- struct lp_build_tgsi_context *bld_base,
- struct lp_build_emit_data *emit_data)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- unsigned opcode = emit_data->info->opcode;
- LLVMValueRef val;
- int idx;
- unsigned mask;
-
- if (opcode == TGSI_OPCODE_DDX_FINE)
- mask = AC_TID_MASK_LEFT;
- else if (opcode == TGSI_OPCODE_DDY_FINE)
- mask = AC_TID_MASK_TOP;
- else
- mask = AC_TID_MASK_TOP_LEFT;
-
- /* for DDX we want to next X pixel, DDY next Y pixel. */
- idx = (opcode == TGSI_OPCODE_DDX || opcode == TGSI_OPCODE_DDX_FINE) ? 1 : 2;
-
- val = ac_to_integer(&ctx->ac, emit_data->args[0]);
- val = ac_build_ddxy(&ctx->ac, mask, idx, val);
- emit_data->output[emit_data->chan] = val;
-}
-
-static void build_interp_intrinsic(const struct lp_build_tgsi_action *action,
- struct lp_build_tgsi_context *bld_base,
- struct lp_build_emit_data *emit_data)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- struct si_shader *shader = ctx->shader;
- const struct tgsi_shader_info *info = &shader->selector->info;
- LLVMValueRef interp_param;
- const struct tgsi_full_instruction *inst = emit_data->inst;
- const struct tgsi_full_src_register *input = &inst->Src[0];
- int input_base, input_array_size;
- int chan;
- int i;
- LLVMValueRef prim_mask = ctx->abi.prim_mask;
- LLVMValueRef array_idx, offset_x = NULL, offset_y = NULL;
- int interp_param_idx;
- unsigned interp;
- unsigned location;
-
- if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET) {
- /* offset is in second src, first two channels */
- offset_x = lp_build_emit_fetch(bld_base, emit_data->inst, 1,
- TGSI_CHAN_X);
- offset_y = lp_build_emit_fetch(bld_base, emit_data->inst, 1,
- TGSI_CHAN_Y);
- } else if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
- LLVMValueRef sample_position;
- LLVMValueRef sample_id;
- LLVMValueRef halfval = LLVMConstReal(ctx->f32, 0.5f);
-
- /* fetch sample ID, then fetch its sample position,
- * and place into first two channels.
- */
- sample_id = lp_build_emit_fetch(bld_base,
- emit_data->inst, 1, TGSI_CHAN_X);
- sample_id = ac_to_integer(&ctx->ac, sample_id);
-
- /* Section 8.13.2 (Interpolation Functions) of the OpenGL Shading
- * Language 4.50 spec says about interpolateAtSample:
- *
- * "Returns the value of the input interpolant variable at
- * the location of sample number sample. If multisample
- * buffers are not available, the input variable will be
- * evaluated at the center of the pixel. If sample sample
- * does not exist, the position used to interpolate the
- * input variable is undefined."
- *
- * This means that sample_id values outside of the valid are
- * in fact valid input, and the usual mechanism for loading the
- * sample position doesn't work.
- */
- if (ctx->shader->key.mono.u.ps.interpolate_at_sample_force_center) {
- LLVMValueRef center[4] = {
- LLVMConstReal(ctx->f32, 0.5),
- LLVMConstReal(ctx->f32, 0.5),
- ctx->ac.f32_0,
- ctx->ac.f32_0,
- };
-
- sample_position = ac_build_gather_values(&ctx->ac, center, 4);
- } else {
- sample_position = load_sample_position(&ctx->abi, sample_id);
- }
-
- offset_x = LLVMBuildExtractElement(ctx->ac.builder, sample_position,
- ctx->i32_0, "");
-
- offset_x = LLVMBuildFSub(ctx->ac.builder, offset_x, halfval, "");
- offset_y = LLVMBuildExtractElement(ctx->ac.builder, sample_position,
- ctx->i32_1, "");
- offset_y = LLVMBuildFSub(ctx->ac.builder, offset_y, halfval, "");
- }
-
- assert(input->Register.File == TGSI_FILE_INPUT);
-
- if (input->Register.Indirect) {
- unsigned array_id = input->Indirect.ArrayID;
-
- if (array_id) {
- input_base = info->input_array_first[array_id];
- input_array_size = info->input_array_last[array_id] - input_base + 1;
- } else {
- input_base = inst->Src[0].Register.Index;
- input_array_size = info->num_inputs - input_base;
- }
-
- array_idx = si_get_indirect_index(ctx, &input->Indirect,
- 1, input->Register.Index - input_base);
- } else {
- input_base = inst->Src[0].Register.Index;
- input_array_size = 1;
- array_idx = ctx->i32_0;
- }
-
- interp = shader->selector->info.input_interpolate[input_base];
-
- if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
- inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE)
- location = TGSI_INTERPOLATE_LOC_CENTER;
- else
- location = TGSI_INTERPOLATE_LOC_CENTROID;
-
- interp_param_idx = lookup_interp_param_index(interp, location);
- if (interp_param_idx == -1)
- return;
- else if (interp_param_idx)
- interp_param = LLVMGetParam(ctx->main_fn, interp_param_idx);
- else
- interp_param = NULL;
-
- if (inst->Instruction.Opcode == TGSI_OPCODE_INTERP_OFFSET ||
- inst->Instruction.Opcode == TGSI_OPCODE_INTERP_SAMPLE) {
- LLVMValueRef ij_out[2];
- LLVMValueRef ddxy_out = ac_build_ddxy_interp(&ctx->ac, interp_param);
-
- /*
- * take the I then J parameters, and the DDX/Y for it, and
- * calculate the IJ inputs for the interpolator.
- * temp1 = ddx * offset/sample.x + I;
- * interp_param.I = ddy * offset/sample.y + temp1;
- * temp1 = ddx * offset/sample.x + J;
- * interp_param.J = ddy * offset/sample.y + temp1;
- */
- for (i = 0; i < 2; i++) {
- LLVMValueRef ix_ll = LLVMConstInt(ctx->i32, i, 0);
- LLVMValueRef iy_ll = LLVMConstInt(ctx->i32, i + 2, 0);
- LLVMValueRef ddx_el = LLVMBuildExtractElement(ctx->ac.builder,
- ddxy_out, ix_ll, "");
- LLVMValueRef ddy_el = LLVMBuildExtractElement(ctx->ac.builder,
- ddxy_out, iy_ll, "");
- LLVMValueRef interp_el = LLVMBuildExtractElement(ctx->ac.builder,
- interp_param, ix_ll, "");
- LLVMValueRef temp;
-
- interp_el = ac_to_float(&ctx->ac, interp_el);
-
- temp = ac_build_fmad(&ctx->ac, ddx_el, offset_x, interp_el);
- ij_out[i] = ac_build_fmad(&ctx->ac, ddy_el, offset_y, temp);
- }
- interp_param = ac_build_gather_values(&ctx->ac, ij_out, 2);
- }
-
- if (interp_param)
- interp_param = ac_to_float(&ctx->ac, interp_param);
-
- for (chan = 0; chan < 4; chan++) {
- LLVMValueRef gather = LLVMGetUndef(LLVMVectorType(ctx->f32, input_array_size));
- unsigned schan = tgsi_util_get_full_src_register_swizzle(&inst->Src[0], chan);
-
- for (unsigned idx = 0; idx < input_array_size; ++idx) {
- LLVMValueRef v, i = NULL, j = NULL;
-
- if (interp_param) {
- i = LLVMBuildExtractElement(
- ctx->ac.builder, interp_param, ctx->i32_0, "");
- j = LLVMBuildExtractElement(
- ctx->ac.builder, interp_param, ctx->i32_1, "");
- }
- v = si_build_fs_interp(ctx, input_base + idx, schan,
- prim_mask, i, j);
-
- gather = LLVMBuildInsertElement(ctx->ac.builder,
- gather, v, LLVMConstInt(ctx->i32, idx, false), "");
- }
-
- emit_data->output[chan] = LLVMBuildExtractElement(
- ctx->ac.builder, gather, array_idx, "");
- }
-}
-
-static void vote_all_emit(
- const struct lp_build_tgsi_action *action,
- struct lp_build_tgsi_context *bld_base,
- struct lp_build_emit_data *emit_data)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
-
- LLVMValueRef tmp = ac_build_vote_all(&ctx->ac, emit_data->args[0]);
- emit_data->output[emit_data->chan] =
- LLVMBuildSExt(ctx->ac.builder, tmp, ctx->i32, "");
-}
-
-static void vote_any_emit(
- const struct lp_build_tgsi_action *action,
- struct lp_build_tgsi_context *bld_base,
- struct lp_build_emit_data *emit_data)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
-
- LLVMValueRef tmp = ac_build_vote_any(&ctx->ac, emit_data->args[0]);
- emit_data->output[emit_data->chan] =
- LLVMBuildSExt(ctx->ac.builder, tmp, ctx->i32, "");
-}
-
-static void vote_eq_emit(
- const struct lp_build_tgsi_action *action,
- struct lp_build_tgsi_context *bld_base,
- struct lp_build_emit_data *emit_data)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
-
- LLVMValueRef tmp = ac_build_vote_eq(&ctx->ac, emit_data->args[0]);
- emit_data->output[emit_data->chan] =
- LLVMBuildSExt(ctx->ac.builder, tmp, ctx->i32, "");
-}
-
-static void ballot_emit(
- const struct lp_build_tgsi_action *action,
- struct lp_build_tgsi_context *bld_base,
- struct lp_build_emit_data *emit_data)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- LLVMBuilderRef builder = ctx->ac.builder;
- LLVMValueRef tmp;
-
- tmp = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_X);
- tmp = ac_build_ballot(&ctx->ac, tmp);
-
- emit_data->output[0] = LLVMBuildTrunc(builder, tmp, ctx->i32, "");
-
- if (ctx->ac.wave_size == 32) {
- emit_data->output[1] = ctx->i32_0;
- } else {
- tmp = LLVMBuildLShr(builder, tmp, LLVMConstInt(ctx->i64, 32, 0), "");
- emit_data->output[1] = LLVMBuildTrunc(builder, tmp, ctx->i32, "");
- }
-}
-
-static void read_lane_emit(
- const struct lp_build_tgsi_action *action,
- struct lp_build_tgsi_context *bld_base,
- struct lp_build_emit_data *emit_data)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
-
- if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_READ_INVOC) {
- emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst,
- 0, emit_data->src_chan);
-
- /* Always read the source invocation (= lane) from the X channel. */
- emit_data->args[1] = lp_build_emit_fetch(bld_base, emit_data->inst,
- 1, TGSI_CHAN_X);
- emit_data->arg_count = 2;
- }
-
- /* We currently have no other way to prevent LLVM from lifting the icmp
- * calls to a dominating basic block.
- */
- ac_build_optimization_barrier(&ctx->ac, &emit_data->args[0]);
-
- for (unsigned i = 0; i < emit_data->arg_count; ++i)
- emit_data->args[i] = ac_to_integer(&ctx->ac, emit_data->args[i]);
-
- emit_data->output[emit_data->chan] =
- ac_build_intrinsic(&ctx->ac, action->intr_name,
- ctx->i32, emit_data->args, emit_data->arg_count,
- AC_FUNC_ATTR_READNONE |
- AC_FUNC_ATTR_CONVERGENT);
-}
-
-static unsigned si_llvm_get_stream(struct lp_build_tgsi_context *bld_base,
- struct lp_build_emit_data *emit_data)
-{
- struct si_shader_context *ctx = si_shader_context(bld_base);
- struct tgsi_src_register src0 = emit_data->inst->Src[0].Register;
- LLVMValueRef imm;
- unsigned stream;
-
- assert(src0.File == TGSI_FILE_IMMEDIATE);
-
- imm = ctx->imms[src0.Index * TGSI_NUM_CHANNELS + src0.SwizzleX];
- stream = LLVMConstIntGetZExtValue(imm) & 0x3;
- return stream;
-}
-