#include "anv_private.h"
#include "vk_format_info.h"
#include "vk_util.h"
+#include "util/fast_idiv_by_const.h"
#include "common/gen_l3_config.h"
#include "genxml/gen_macros.h"
#include "genxml/genX_pack.h"
static void
-emit_lrm(struct anv_batch *batch,
- uint32_t reg, struct anv_bo *bo, uint32_t offset)
+emit_lrm(struct anv_batch *batch, uint32_t reg, struct anv_address addr)
{
anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
lrm.RegisterAddress = reg;
- lrm.MemoryAddress = (struct anv_address) { bo, offset };
+ lrm.MemoryAddress = addr;
}
}
{
struct anv_device *device = cmd_buffer->device;
+ /* If we are emitting a new state base address we probably need to re-emit
+ * binding tables.
+ */
+ cmd_buffer->state.descriptors_dirty |= ~0;
+
/* Emit a render target cache flush.
*
* This isn't documented anywhere in the PRM. However, it seems to be
anv_batch_emit(&cmd_buffer->batch, GENX(STATE_BASE_ADDRESS), sba) {
sba.GeneralStateBaseAddress = (struct anv_address) { NULL, 0 };
- sba.GeneralStateMemoryObjectControlState = GENX(MOCS);
+ sba.GeneralStateMOCS = GENX(MOCS);
sba.GeneralStateBaseAddressModifyEnable = true;
sba.SurfaceStateBaseAddress =
anv_cmd_buffer_surface_base_address(cmd_buffer);
- sba.SurfaceStateMemoryObjectControlState = GENX(MOCS);
+ sba.SurfaceStateMOCS = GENX(MOCS);
sba.SurfaceStateBaseAddressModifyEnable = true;
sba.DynamicStateBaseAddress =
- (struct anv_address) { &device->dynamic_state_pool.block_pool.bo, 0 };
- sba.DynamicStateMemoryObjectControlState = GENX(MOCS);
+ (struct anv_address) { device->dynamic_state_pool.block_pool.bo, 0 };
+ sba.DynamicStateMOCS = GENX(MOCS);
sba.DynamicStateBaseAddressModifyEnable = true;
sba.IndirectObjectBaseAddress = (struct anv_address) { NULL, 0 };
- sba.IndirectObjectMemoryObjectControlState = GENX(MOCS);
+ sba.IndirectObjectMOCS = GENX(MOCS);
sba.IndirectObjectBaseAddressModifyEnable = true;
sba.InstructionBaseAddress =
- (struct anv_address) { &device->instruction_state_pool.block_pool.bo, 0 };
- sba.InstructionMemoryObjectControlState = GENX(MOCS);
+ (struct anv_address) { device->instruction_state_pool.block_pool.bo, 0 };
+ sba.InstructionMOCS = GENX(MOCS);
sba.InstructionBaseAddressModifyEnable = true;
# if (GEN_GEN >= 8)
sba.IndirectObjectBufferSizeModifyEnable = true;
sba.InstructionBufferSize = 0xfffff;
sba.InstructionBuffersizeModifyEnable = true;
+# endif
+# if (GEN_GEN >= 9)
+ sba.BindlessSurfaceStateBaseAddress = (struct anv_address) { NULL, 0 };
+ sba.BindlessSurfaceStateMOCS = GENX(MOCS);
+ sba.BindlessSurfaceStateBaseAddressModifyEnable = true;
+ sba.BindlessSurfaceStateSize = 0;
+# endif
+# if (GEN_GEN >= 10)
+ sba.BindlessSamplerStateBaseAddress = (struct anv_address) { NULL, 0 };
+ sba.BindlessSamplerStateMOCS = GENX(MOCS);
+ sba.BindlessSamplerStateBaseAddressModifyEnable = true;
+ sba.BindlessSamplerStateBufferSize = 0;
# endif
}
}
static void
-add_surface_state_reloc(struct anv_cmd_buffer *cmd_buffer,
- struct anv_state state,
- struct anv_bo *bo, uint32_t offset)
+add_surface_reloc(struct anv_cmd_buffer *cmd_buffer,
+ struct anv_state state, struct anv_address addr)
{
const struct isl_device *isl_dev = &cmd_buffer->device->isl_dev;
VkResult result =
anv_reloc_list_add(&cmd_buffer->surface_relocs, &cmd_buffer->pool->alloc,
- state.offset + isl_dev->ss.addr_offset, bo, offset);
+ state.offset + isl_dev->ss.addr_offset,
+ addr.bo, addr.offset);
if (result != VK_SUCCESS)
anv_batch_set_error(&cmd_buffer->batch, result);
}
static void
-add_image_view_relocs(struct anv_cmd_buffer *cmd_buffer,
- struct anv_surface_state state)
+add_surface_state_relocs(struct anv_cmd_buffer *cmd_buffer,
+ struct anv_surface_state state)
{
const struct isl_device *isl_dev = &cmd_buffer->device->isl_dev;
assert(!anv_address_is_null(state.address));
- add_surface_state_reloc(cmd_buffer, state.state,
- state.address.bo, state.address.offset);
+ add_surface_reloc(cmd_buffer, state.state, state.address);
if (!anv_address_is_null(state.aux_address)) {
VkResult result =
0, 0, 1, hiz_op);
}
-#define MI_PREDICATE_SRC0 0x2400
-#define MI_PREDICATE_SRC1 0x2408
+#define MI_PREDICATE_SRC0 0x2400
+#define MI_PREDICATE_SRC1 0x2408
+#define MI_PREDICATE_RESULT 0x2418
static void
set_image_compressed_bit(struct anv_cmd_buffer *cmd_buffer,
static void
anv_cmd_predicated_ccs_resolve(struct anv_cmd_buffer *cmd_buffer,
const struct anv_image *image,
+ enum isl_format format,
VkImageAspectFlagBits aspect,
uint32_t level, uint32_t array_layer,
enum isl_aux_op resolve_op,
image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
resolve_op = ISL_AUX_OP_FULL_RESOLVE;
- anv_image_ccs_op(cmd_buffer, image, aspect, level,
+ anv_image_ccs_op(cmd_buffer, image, format, aspect, level,
array_layer, 1, resolve_op, NULL, true);
}
static void
anv_cmd_predicated_mcs_resolve(struct anv_cmd_buffer *cmd_buffer,
const struct anv_image *image,
+ enum isl_format format,
VkImageAspectFlagBits aspect,
uint32_t array_layer,
enum isl_aux_op resolve_op,
aspect, 0, array_layer,
resolve_op, fast_clear_supported);
- anv_image_mcs_op(cmd_buffer, image, aspect,
+ anv_image_mcs_op(cmd_buffer, image, format, aspect,
array_layer, 1, resolve_op, NULL, true);
#else
unreachable("MCS resolves are unsupported on Ivybridge and Bay Trail");
uint32_t layer_count)
{
/* The aspect must be exactly one of the image aspects. */
- assert(_mesa_bitcount(aspect) == 1 && (aspect & image->aspects));
+ assert(util_bitcount(aspect) == 1 && (aspect & image->aspects));
/* The only compression types with more than just fast-clears are MCS,
* CCS_E, and HiZ. With HiZ we just trust the layout and don't actually
assert(cmd_buffer && image);
assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
- struct anv_bo *ss_bo =
- &cmd_buffer->device->surface_state_pool.block_pool.bo;
- uint32_t ss_clear_offset = surface_state.offset +
- cmd_buffer->device->isl_dev.ss.clear_value_offset;
+ struct anv_address ss_clear_addr = {
+ .bo = cmd_buffer->device->surface_state_pool.block_pool.bo,
+ .offset = surface_state.offset +
+ cmd_buffer->device->isl_dev.ss.clear_value_offset,
+ };
const struct anv_address entry_addr =
anv_image_get_clear_color_addr(cmd_buffer->device, image, aspect);
unsigned copy_size = cmd_buffer->device->isl_dev.ss.clear_value_size;
if (copy_from_surface_state) {
- genX(cmd_buffer_mi_memcpy)(cmd_buffer, entry_addr.bo, entry_addr.offset,
- ss_bo, ss_clear_offset, copy_size);
+ genX(cmd_buffer_mi_memcpy)(cmd_buffer, entry_addr,
+ ss_clear_addr, copy_size);
} else {
- genX(cmd_buffer_mi_memcpy)(cmd_buffer, ss_bo, ss_clear_offset,
- entry_addr.bo, entry_addr.offset, copy_size);
+ genX(cmd_buffer_mi_memcpy)(cmd_buffer, ss_clear_addr,
+ entry_addr, copy_size);
/* Updating a surface state object may require that the state cache be
* invalidated. From the SKL PRM, Shared Functions -> State -> State
assert(level_count != VK_REMAINING_MIP_LEVELS &&
layer_count != VK_REMAINING_ARRAY_LAYERS);
/* Ensure the subresource range is valid. */
- uint64_t last_level_num = base_level + level_count;
+ UNUSED uint64_t last_level_num = base_level + level_count;
const uint32_t max_depth = anv_minify(image->extent.depth, base_level);
UNUSED const uint32_t image_layers = MAX2(image->array_size, max_depth);
assert((uint64_t)base_layer + layer_count <= image_layers);
uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
- if (image->planes[plane].shadow_surface.isl.size > 0 &&
+ if (image->planes[plane].shadow_surface.isl.size_B > 0 &&
final_layout == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL) {
/* This surface is a linear compressed image with a tiled shadow surface
* for texturing. The client is about to use it in READ_ONLY_OPTIMAL so
uint32_t level_layer_count =
MIN2(layer_count, aux_layers - base_layer);
- anv_image_ccs_op(cmd_buffer, image, aspect, level,
- base_layer, level_layer_count,
+ anv_image_ccs_op(cmd_buffer, image,
+ image->planes[plane].surface.isl.format,
+ aspect, level, base_layer, level_layer_count,
ISL_AUX_OP_AMBIGUATE, NULL, false);
if (image->planes[plane].aux_usage == ISL_AUX_USAGE_CCS_E) {
}
assert(base_level == 0 && level_count == 1);
- anv_image_mcs_op(cmd_buffer, image, aspect,
- base_layer, layer_count,
+ anv_image_mcs_op(cmd_buffer, image,
+ image->planes[plane].surface.isl.format,
+ aspect, base_layer, layer_count,
ISL_AUX_OP_FAST_CLEAR, NULL, false);
}
return;
for (uint32_t a = 0; a < level_layer_count; a++) {
uint32_t array_layer = base_layer + a;
if (image->samples == 1) {
- anv_cmd_predicated_ccs_resolve(cmd_buffer, image, aspect,
- level, array_layer, resolve_op,
+ anv_cmd_predicated_ccs_resolve(cmd_buffer, image,
+ image->planes[plane].surface.isl.format,
+ aspect, level, array_layer, resolve_op,
final_fast_clear);
} else {
- anv_cmd_predicated_mcs_resolve(cmd_buffer, image, aspect,
- array_layer, resolve_op,
+ /* We only support fast-clear on the first layer so partial
+ * resolves should not be used on other layers as they will use
+ * the clear color stored in memory that is only valid for layer0.
+ */
+ if (resolve_op == ISL_AUX_OP_PARTIAL_RESOLVE &&
+ array_layer != 0)
+ continue;
+
+ anv_cmd_predicated_mcs_resolve(cmd_buffer, image,
+ image->planes[plane].surface.isl.format,
+ aspect, array_layer, resolve_op,
final_fast_clear);
}
}
&state->attachments[i].color,
NULL);
- add_image_view_relocs(cmd_buffer, state->attachments[i].color);
+ add_surface_state_relocs(cmd_buffer, state->attachments[i].color);
} else {
depth_stencil_attachment_compute_aux_usage(cmd_buffer->device,
state, i,
&state->attachments[i].input,
NULL);
- add_image_view_relocs(cmd_buffer, state->attachments[i].input);
+ add_surface_state_relocs(cmd_buffer, state->attachments[i].input);
}
}
}
if (iview) {
VkImageLayout layout =
- cmd_buffer->state.subpass->depth_stencil_attachment.layout;
+ cmd_buffer->state.subpass->depth_stencil_attachment->layout;
enum isl_aux_usage aux_usage =
anv_layout_to_aux_usage(&cmd_buffer->device->info, iview->image,
cmd_buffer->state.gfx.dirty |= ANV_CMD_DIRTY_RENDER_TARGETS;
}
+#if GEN_GEN >= 8 || GEN_IS_HASWELL
+ if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY) {
+ const VkCommandBufferInheritanceConditionalRenderingInfoEXT *conditional_rendering_info =
+ vk_find_struct_const(pBeginInfo->pInheritanceInfo->pNext, COMMAND_BUFFER_INHERITANCE_CONDITIONAL_RENDERING_INFO_EXT);
+
+ /* If secondary buffer supports conditional rendering
+ * we should emit commands as if conditional rendering is enabled.
+ */
+ cmd_buffer->state.conditional_render_enabled =
+ conditional_rendering_info && conditional_rendering_info->conditionalRenderingEnable;
+ }
+#endif
+
return result;
}
assert(secondary->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY);
assert(!anv_batch_has_error(&secondary->batch));
+#if GEN_GEN >= 8 || GEN_IS_HASWELL
+ if (secondary->state.conditional_render_enabled) {
+ if (!primary->state.conditional_render_enabled) {
+ /* Secondary buffer is constructed as if it will be executed
+ * with conditional rendering, we should satisfy this dependency
+ * regardless of conditional rendering being enabled in primary.
+ */
+ emit_lri(&primary->batch, CS_GPR(ANV_PREDICATE_RESULT_REG), UINT32_MAX);
+ emit_lri(&primary->batch, CS_GPR(ANV_PREDICATE_RESULT_REG) + 4, UINT32_MAX);
+ }
+ }
+#endif
+
if (secondary->usage_flags &
VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT) {
/* If we're continuing a render pass from the primary, we need to
* we allocated for them in BeginCommandBuffer.
*/
struct anv_bo *ss_bo =
- &primary->device->surface_state_pool.block_pool.bo;
+ primary->device->surface_state_pool.block_pool.bo;
struct anv_state src_state = primary->state.render_pass_states;
struct anv_state dst_state = secondary->state.render_pass_states;
assert(src_state.alloc_size == dst_state.alloc_size);
- genX(cmd_buffer_so_memcpy)(primary, ss_bo, dst_state.offset,
- ss_bo, src_state.offset,
+ genX(cmd_buffer_so_memcpy)(primary,
+ (struct anv_address) {
+ .bo = ss_bo,
+ .offset = dst_state.offset,
+ },
+ (struct anv_address) {
+ .bo = ss_bo,
+ .offset = src_state.offset,
+ },
src_state.alloc_size);
}
uint32_t l3cr;
anv_pack_struct(&l3cr, GENX(L3CNTLREG),
.SLMEnable = has_slm,
+#if GEN_GEN == 11
+ /* WA_1406697149: Bit 9 "Error Detection Behavior Control" must be set
+ * in L3CNTLREG register. The default setting of the bit is not the
+ * desirable behavior.
+ */
+ .ErrorDetectionBehaviorControl = true,
+ .UseFullWays = true,
+#endif
.URBAllocation = cfg->n[GEN_L3P_URB],
.ROAllocation = cfg->n[GEN_L3P_RO],
.DCAllocation = cfg->n[GEN_L3P_DC],
pipe.StallAtPixelScoreboard = true;
}
+ /* If a render target flush was emitted, then we can toggle off the bit
+ * saying that render target writes are ongoing.
+ */
+ if (bits & ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT)
+ bits &= ~(ANV_PIPE_RENDER_TARGET_BUFFER_WRITES);
+
bits &= ~(ANV_PIPE_FLUSH_BITS | ANV_PIPE_CS_STALL_BIT);
}
if (bits & ANV_PIPE_INVALIDATE_BITS) {
+ /* From the SKL PRM, Vol. 2a, "PIPE_CONTROL",
+ *
+ * "If the VF Cache Invalidation Enable is set to a 1 in a
+ * PIPE_CONTROL, a separate Null PIPE_CONTROL, all bitfields sets to
+ * 0, with the VF Cache Invalidation Enable set to 0 needs to be sent
+ * prior to the PIPE_CONTROL with VF Cache Invalidation Enable set to
+ * a 1."
+ *
+ * This appears to hang Broadwell, so we restrict it to just gen9.
+ */
+ if (GEN_GEN == 9 && (bits & ANV_PIPE_VF_CACHE_INVALIDATE_BIT))
+ anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pipe);
+
anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pipe) {
pipe.StateCacheInvalidationEnable =
bits & ANV_PIPE_STATE_CACHE_INVALIDATE_BIT;
bits & ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
pipe.InstructionCacheInvalidateEnable =
bits & ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT;
+
+ /* From the SKL PRM, Vol. 2a, "PIPE_CONTROL",
+ *
+ * "When VF Cache Invalidate is set “Post Sync Operation” must be
+ * enabled to “Write Immediate Data” or “Write PS Depth Count” or
+ * “Write Timestamp”.
+ */
+ if (GEN_GEN == 9 && pipe.VFCacheInvalidationEnable) {
+ pipe.PostSyncOperation = WriteImmediateData;
+ pipe.Address =
+ (struct anv_address) { &cmd_buffer->device->workaround_bo, 0 };
+ }
}
bits &= ~ANV_PIPE_INVALIDATE_BITS;
#endif
const unsigned num_stages =
- _mesa_bitcount(stages & VK_SHADER_STAGE_ALL_GRAPHICS);
+ util_bitcount(stages & VK_SHADER_STAGE_ALL_GRAPHICS);
unsigned size_per_stage = push_constant_kb / num_stages;
/* Broadwell+ and Haswell gt3 require that the push constant sizes be in
return pipe_state->dynamic_offsets[dynamic_offset_idx];
}
+static struct anv_address
+anv_descriptor_set_address(struct anv_cmd_buffer *cmd_buffer,
+ struct anv_descriptor_set *set)
+{
+ if (set->pool) {
+ /* This is a normal descriptor set */
+ return (struct anv_address) {
+ .bo = &set->pool->bo,
+ .offset = set->desc_mem.offset,
+ };
+ } else {
+ /* This is a push descriptor set. We have to flag it as used on the GPU
+ * so that the next time we push descriptors, we grab a new memory.
+ */
+ struct anv_push_descriptor_set *push_set =
+ (struct anv_push_descriptor_set *)set;
+ push_set->set_used_on_gpu = true;
+
+ return (struct anv_address) {
+ .bo = cmd_buffer->dynamic_state_stream.state_pool->block_pool.bo,
+ .offset = set->desc_mem.offset,
+ };
+ }
+}
+
static VkResult
emit_binding_table(struct anv_cmd_buffer *cmd_buffer,
gl_shader_stage stage,
struct anv_state *bt_state)
{
+ const struct gen_device_info *devinfo = &cmd_buffer->device->info;
struct anv_subpass *subpass = cmd_buffer->state.subpass;
struct anv_cmd_pipeline_state *pipe_state;
struct anv_pipeline *pipeline;
- uint32_t bias, state_offset;
+ uint32_t state_offset;
switch (stage) {
case MESA_SHADER_COMPUTE:
pipe_state = &cmd_buffer->state.compute.base;
- bias = 1;
break;
default:
pipe_state = &cmd_buffer->state.gfx.base;
- bias = 0;
break;
}
pipeline = pipe_state->pipeline;
}
struct anv_pipeline_bind_map *map = &pipeline->shaders[stage]->bind_map;
- if (bias + map->surface_count == 0) {
+ if (map->surface_count == 0) {
*bt_state = (struct anv_state) { 0, };
return VK_SUCCESS;
}
*bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer,
- bias + map->surface_count,
+ map->surface_count,
&state_offset);
uint32_t *bt_map = bt_state->map;
if (bt_state->map == NULL)
return VK_ERROR_OUT_OF_DEVICE_MEMORY;
- if (stage == MESA_SHADER_COMPUTE &&
- get_cs_prog_data(pipeline)->uses_num_work_groups) {
- struct anv_bo *bo = cmd_buffer->state.compute.num_workgroups.bo;
- uint32_t bo_offset = cmd_buffer->state.compute.num_workgroups.offset;
-
- struct anv_state surface_state;
- surface_state =
- anv_cmd_buffer_alloc_surface_state(cmd_buffer);
-
- const enum isl_format format =
- anv_isl_format_for_descriptor_type(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER);
- anv_fill_buffer_surface_state(cmd_buffer->device, surface_state,
- format,
- cmd_buffer->state.compute.num_workgroups,
- 12, 1);
-
- bt_map[0] = surface_state.offset + state_offset;
- add_surface_state_reloc(cmd_buffer, surface_state, bo, bo_offset);
- }
-
- if (map->surface_count == 0)
- goto out;
-
- if (map->image_count > 0) {
+ /* We only use push constant space for images before gen9 */
+ if (map->image_param_count > 0) {
VkResult result =
anv_cmd_buffer_ensure_push_constant_field(cmd_buffer, stage, images);
if (result != VK_SUCCESS)
surface_state = cmd_buffer->state.null_surface_state;
}
- bt_map[bias + s] = surface_state.offset + state_offset;
+ bt_map[s] = surface_state.offset + state_offset;
+ continue;
+ } else if (binding->set == ANV_DESCRIPTOR_SET_SHADER_CONSTANTS) {
+ struct anv_state surface_state =
+ anv_cmd_buffer_alloc_surface_state(cmd_buffer);
+
+ struct anv_address constant_data = {
+ .bo = pipeline->device->dynamic_state_pool.block_pool.bo,
+ .offset = pipeline->shaders[stage]->constant_data.offset,
+ };
+ unsigned constant_data_size =
+ pipeline->shaders[stage]->constant_data_size;
+
+ const enum isl_format format =
+ anv_isl_format_for_descriptor_type(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER);
+ anv_fill_buffer_surface_state(cmd_buffer->device,
+ surface_state, format,
+ constant_data, constant_data_size, 1);
+
+ bt_map[s] = surface_state.offset + state_offset;
+ add_surface_reloc(cmd_buffer, surface_state, constant_data);
+ continue;
+ } else if (binding->set == ANV_DESCRIPTOR_SET_NUM_WORK_GROUPS) {
+ /* This is always the first binding for compute shaders */
+ assert(stage == MESA_SHADER_COMPUTE && s == 0);
+ if (!get_cs_prog_data(pipeline)->uses_num_work_groups)
+ continue;
+
+ struct anv_state surface_state =
+ anv_cmd_buffer_alloc_surface_state(cmd_buffer);
+
+ const enum isl_format format =
+ anv_isl_format_for_descriptor_type(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER);
+ anv_fill_buffer_surface_state(cmd_buffer->device, surface_state,
+ format,
+ cmd_buffer->state.compute.num_workgroups,
+ 12, 1);
+ bt_map[s] = surface_state.offset + state_offset;
+ add_surface_reloc(cmd_buffer, surface_state,
+ cmd_buffer->state.compute.num_workgroups);
+ continue;
+ } else if (binding->set == ANV_DESCRIPTOR_SET_DESCRIPTORS) {
+ /* This is a descriptor set buffer so the set index is actually
+ * given by binding->binding. (Yes, that's confusing.)
+ */
+ struct anv_descriptor_set *set =
+ pipe_state->descriptors[binding->binding];
+ assert(set->desc_mem.alloc_size);
+ assert(set->desc_surface_state.alloc_size);
+ bt_map[s] = set->desc_surface_state.offset + state_offset;
+ add_surface_reloc(cmd_buffer, set->desc_surface_state,
+ anv_descriptor_set_address(cmd_buffer, set));
continue;
}
desc->image_view->planes[binding->plane].optimal_sampler_surface_state;
surface_state = sstate.state;
assert(surface_state.alloc_size);
- add_image_view_relocs(cmd_buffer, sstate);
+ add_surface_state_relocs(cmd_buffer, sstate);
break;
}
case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
desc->image_view->planes[binding->plane].optimal_sampler_surface_state;
surface_state = sstate.state;
assert(surface_state.alloc_size);
- add_image_view_relocs(cmd_buffer, sstate);
+ add_surface_state_relocs(cmd_buffer, sstate);
} else {
/* For color input attachments, we create the surface state at
* vkBeginRenderPass time so that we can include aux and clear
: desc->image_view->planes[binding->plane].storage_surface_state;
surface_state = sstate.state;
assert(surface_state.alloc_size);
- add_image_view_relocs(cmd_buffer, sstate);
-
- struct brw_image_param *image_param =
- &cmd_buffer->state.push_constants[stage]->images[image++];
+ add_surface_state_relocs(cmd_buffer, sstate);
+ if (devinfo->gen < 9) {
+ /* We only need the image params on gen8 and earlier. No image
+ * workarounds that require tiling information are required on
+ * SKL and above.
+ */
+ assert(image < MAX_GEN8_IMAGES);
+ struct brw_image_param *image_param =
+ &cmd_buffer->state.push_constants[stage]->images[image++];
- *image_param = desc->image_view->planes[binding->plane].storage_image_param;
- image_param->surface_idx = bias + s;
+ *image_param =
+ desc->image_view->planes[binding->plane].storage_image_param;
+ }
break;
}
case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
surface_state = desc->buffer_view->surface_state;
assert(surface_state.alloc_size);
- add_surface_state_reloc(cmd_buffer, surface_state,
- desc->buffer_view->address.bo,
- desc->buffer_view->address.offset);
+ add_surface_reloc(cmd_buffer, surface_state,
+ desc->buffer_view->address);
break;
case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
/* Clamp the range to the buffer size */
uint32_t range = MIN2(desc->range, desc->buffer->size - offset);
- struct anv_address address = {
- .bo = desc->buffer->bo,
- .offset = desc->buffer->offset + offset,
- };
+ struct anv_address address =
+ anv_address_add(desc->buffer->address, offset);
surface_state =
anv_state_stream_alloc(&cmd_buffer->surface_state_stream, 64, 64);
anv_fill_buffer_surface_state(cmd_buffer->device, surface_state,
format, address, range, 1);
- add_surface_state_reloc(cmd_buffer, surface_state,
- address.bo, address.offset);
+ add_surface_reloc(cmd_buffer, surface_state, address);
break;
}
? desc->buffer_view->writeonly_storage_surface_state
: desc->buffer_view->storage_surface_state;
assert(surface_state.alloc_size);
- add_surface_state_reloc(cmd_buffer, surface_state,
- desc->buffer_view->address.bo,
- desc->buffer_view->address.offset);
-
- struct brw_image_param *image_param =
- &cmd_buffer->state.push_constants[stage]->images[image++];
-
- *image_param = desc->buffer_view->storage_image_param;
- image_param->surface_idx = bias + s;
+ add_surface_reloc(cmd_buffer, surface_state,
+ desc->buffer_view->address);
+ if (devinfo->gen < 9) {
+ assert(image < MAX_GEN8_IMAGES);
+ struct brw_image_param *image_param =
+ &cmd_buffer->state.push_constants[stage]->images[image++];
+
+ *image_param = desc->buffer_view->storage_image_param;
+ }
break;
default:
continue;
}
- bt_map[bias + s] = surface_state.offset + state_offset;
+ bt_map[s] = surface_state.offset + state_offset;
}
- assert(image == map->image_count);
-
- out:
- anv_state_flush(cmd_buffer->device, *bt_state);
+ assert(image == map->image_param_count);
#if GEN_GEN >= 11
/* The PIPE_CONTROL command description says:
sampler->state[binding->plane], sizeof(sampler->state[0]));
}
- anv_state_flush(cmd_buffer->device, *state);
-
return VK_SUCCESS;
}
const struct anv_pipeline_binding *binding =
&bind_map->surface_to_descriptor[surface];
- const struct anv_descriptor *desc =
- anv_descriptor_for_binding(&gfx_state->base, binding);
-
struct anv_address read_addr;
uint32_t read_len;
- if (desc->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER) {
+ if (binding->set == ANV_DESCRIPTOR_SET_SHADER_CONSTANTS) {
+ struct anv_address constant_data = {
+ .bo = pipeline->device->dynamic_state_pool.block_pool.bo,
+ .offset = pipeline->shaders[stage]->constant_data.offset,
+ };
+ unsigned constant_data_size =
+ pipeline->shaders[stage]->constant_data_size;
+
read_len = MIN2(range->length,
- DIV_ROUND_UP(desc->buffer_view->range, 32) - range->start);
- read_addr = anv_address_add(desc->buffer_view->address,
+ DIV_ROUND_UP(constant_data_size, 32) - range->start);
+ read_addr = anv_address_add(constant_data,
range->start * 32);
- } else {
- assert(desc->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC);
-
- uint32_t dynamic_offset =
- dynamic_offset_for_binding(&gfx_state->base, binding);
- uint32_t buf_offset =
- MIN2(desc->offset + dynamic_offset, desc->buffer->size);
- uint32_t buf_range =
- MIN2(desc->range, desc->buffer->size - buf_offset);
+ } else if (binding->set == ANV_DESCRIPTOR_SET_DESCRIPTORS) {
+ /* This is a descriptor set buffer so the set index is
+ * actually given by binding->binding. (Yes, that's
+ * confusing.)
+ */
+ struct anv_descriptor_set *set =
+ gfx_state->base.descriptors[binding->binding];
+ struct anv_address desc_buffer_addr =
+ anv_descriptor_set_address(cmd_buffer, set);
+ const unsigned desc_buffer_size = set->desc_mem.alloc_size;
read_len = MIN2(range->length,
- DIV_ROUND_UP(buf_range, 32) - range->start);
- read_addr = (struct anv_address) {
- .bo = desc->buffer->bo,
- .offset = desc->buffer->offset + buf_offset +
- range->start * 32,
- };
+ DIV_ROUND_UP(desc_buffer_size, 32) - range->start);
+ read_addr = anv_address_add(desc_buffer_addr,
+ range->start * 32);
+ } else {
+ const struct anv_descriptor *desc =
+ anv_descriptor_for_binding(&gfx_state->base, binding);
+
+ if (desc->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER) {
+ read_len = MIN2(range->length,
+ DIV_ROUND_UP(desc->buffer_view->range, 32) - range->start);
+ read_addr = anv_address_add(desc->buffer_view->address,
+ range->start * 32);
+ } else {
+ assert(desc->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC);
+
+ uint32_t dynamic_offset =
+ dynamic_offset_for_binding(&gfx_state->base, binding);
+ uint32_t buf_offset =
+ MIN2(desc->offset + dynamic_offset, desc->buffer->size);
+ uint32_t buf_range =
+ MIN2(desc->range, desc->buffer->size - buf_offset);
+
+ read_len = MIN2(range->length,
+ DIV_ROUND_UP(buf_range, 32) - range->start);
+ read_addr = anv_address_add(desc->buffer->address,
+ buf_offset + range->start * 32);
+ }
}
if (read_len > 0) {
if (state.alloc_size > 0) {
c.ConstantBody.Buffer[n] = (struct anv_address) {
- .bo = &cmd_buffer->device->dynamic_state_pool.block_pool.bo,
+ .bo = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
.offset = state.offset,
};
c.ConstantBody.ReadLength[n] =
uint32_t *p;
uint32_t vb_emit = cmd_buffer->state.gfx.vb_dirty & pipeline->vb_used;
+ if (cmd_buffer->state.gfx.dirty & ANV_CMD_DIRTY_PIPELINE)
+ vb_emit |= pipeline->vb_used;
assert((pipeline->active_stages & VK_SHADER_STAGE_COMPUTE_BIT) == 0);
struct GENX(VERTEX_BUFFER_STATE) state = {
.VertexBufferIndex = vb,
-#if GEN_GEN >= 8
- .MemoryObjectControlState = GENX(MOCS),
-#else
- .BufferAccessType = pipeline->instancing_enable[vb] ? INSTANCEDATA : VERTEXDATA,
- /* Our implementation of VK_KHR_multiview uses instancing to draw
- * the different views. If the client asks for instancing, we
- * need to use the Instance Data Step Rate to ensure that we
- * repeat the client's per-instance data once for each view.
- */
- .InstanceDataStepRate = anv_subpass_view_count(pipeline->subpass),
- .VertexBufferMemoryObjectControlState = GENX(MOCS),
+ .MOCS = anv_mocs_for_bo(cmd_buffer->device, buffer->address.bo),
+#if GEN_GEN <= 7
+ .BufferAccessType = pipeline->vb[vb].instanced ? INSTANCEDATA : VERTEXDATA,
+ .InstanceDataStepRate = pipeline->vb[vb].instance_divisor,
#endif
.AddressModifyEnable = true,
- .BufferPitch = pipeline->binding_stride[vb],
- .BufferStartingAddress = { buffer->bo, buffer->offset + offset },
+ .BufferPitch = pipeline->vb[vb].stride,
+ .BufferStartingAddress = anv_address_add(buffer->address, offset),
#if GEN_GEN >= 8
.BufferSize = buffer->size - offset
#else
- .EndAddress = { buffer->bo, buffer->offset + buffer->size - 1},
+ .EndAddress = anv_address_add(buffer->address, buffer->size - 1),
#endif
};
cmd_buffer->state.gfx.vb_dirty &= ~vb_emit;
+#if GEN_GEN >= 8
+ if (cmd_buffer->state.gfx.dirty & ANV_CMD_DIRTY_XFB_ENABLE) {
+ /* We don't need any per-buffer dirty tracking because you're not
+ * allowed to bind different XFB buffers while XFB is enabled.
+ */
+ for (unsigned idx = 0; idx < MAX_XFB_BUFFERS; idx++) {
+ struct anv_xfb_binding *xfb = &cmd_buffer->state.xfb_bindings[idx];
+ anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_SO_BUFFER), sob) {
+ sob.SOBufferIndex = idx;
+
+ if (cmd_buffer->state.xfb_enabled && xfb->buffer) {
+ sob.SOBufferEnable = true;
+ sob.MOCS = cmd_buffer->device->default_mocs,
+ sob.StreamOffsetWriteEnable = false;
+ sob.SurfaceBaseAddress = anv_address_add(xfb->buffer->address,
+ xfb->offset);
+ /* Size is in DWords - 1 */
+ sob.SurfaceSize = xfb->size / 4 - 1;
+ }
+ }
+ }
+
+ /* CNL and later require a CS stall after 3DSTATE_SO_BUFFER */
+ if (GEN_GEN >= 10)
+ cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_CS_STALL_BIT;
+ }
+#endif
+
if (cmd_buffer->state.gfx.dirty & ANV_CMD_DIRTY_PIPELINE) {
anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
pipeline->depth_clamp_enable);
}
- if (cmd_buffer->state.gfx.dirty & ANV_CMD_DIRTY_DYNAMIC_SCISSOR)
+ if (cmd_buffer->state.gfx.dirty & (ANV_CMD_DIRTY_DYNAMIC_SCISSOR |
+ ANV_CMD_DIRTY_RENDER_TARGETS))
gen7_cmd_buffer_emit_scissor(cmd_buffer);
genX(cmd_buffer_flush_dynamic_state)(cmd_buffer);
.VertexBufferIndex = index,
.AddressModifyEnable = true,
.BufferPitch = 0,
+ .MOCS = anv_mocs_for_bo(cmd_buffer->device, addr.bo),
#if (GEN_GEN >= 8)
- .MemoryObjectControlState = GENX(MOCS),
.BufferStartingAddress = addr,
.BufferSize = size
#else
- .VertexBufferMemoryObjectControlState = GENX(MOCS),
.BufferStartingAddress = addr,
.EndAddress = anv_address_add(addr, size),
#endif
((uint32_t *)id_state.map)[0] = base_vertex;
((uint32_t *)id_state.map)[1] = base_instance;
- anv_state_flush(cmd_buffer->device, id_state);
-
struct anv_address addr = {
- .bo = &cmd_buffer->device->dynamic_state_pool.block_pool.bo,
+ .bo = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
.offset = id_state.offset,
};
((uint32_t *)state.map)[0] = draw_index;
- anv_state_flush(cmd_buffer->device, state);
-
struct anv_address addr = {
- .bo = &cmd_buffer->device->dynamic_state_pool.block_pool.bo,
+ .bo = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
.offset = state.offset,
};
genX(cmd_buffer_flush_state)(cmd_buffer);
+ if (cmd_buffer->state.conditional_render_enabled)
+ genX(cmd_emit_conditional_render_predicate)(cmd_buffer);
+
if (vs_prog_data->uses_firstvertex ||
vs_prog_data->uses_baseinstance)
emit_base_vertex_instance(cmd_buffer, firstVertex, firstInstance);
instanceCount *= anv_subpass_view_count(cmd_buffer->state.subpass);
anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
+ prim.PredicateEnable = cmd_buffer->state.conditional_render_enabled;
prim.VertexAccessType = SEQUENTIAL;
prim.PrimitiveTopologyType = pipeline->topology;
prim.VertexCountPerInstance = vertexCount;
genX(cmd_buffer_flush_state)(cmd_buffer);
+ if (cmd_buffer->state.conditional_render_enabled)
+ genX(cmd_emit_conditional_render_predicate)(cmd_buffer);
+
if (vs_prog_data->uses_firstvertex ||
vs_prog_data->uses_baseinstance)
emit_base_vertex_instance(cmd_buffer, vertexOffset, firstInstance);
instanceCount *= anv_subpass_view_count(cmd_buffer->state.subpass);
anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
+ prim.PredicateEnable = cmd_buffer->state.conditional_render_enabled;
prim.VertexAccessType = RANDOM;
prim.PrimitiveTopologyType = pipeline->topology;
prim.VertexCountPerInstance = indexCount;
build_alu_multiply_gpr0(dw + 1, &num_dwords, N);
}
+static void
+emit_alu_add(struct anv_batch *batch, unsigned dst_reg,
+ unsigned reg_a, unsigned reg_b)
+{
+ uint32_t *dw = anv_batch_emitn(batch, 1 + 4, GENX(MI_MATH));
+ dw[1] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCA, reg_a);
+ dw[2] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCB, reg_b);
+ dw[3] = mi_alu(MI_ALU_ADD, 0, 0);
+ dw[4] = mi_alu(MI_ALU_STORE, dst_reg, MI_ALU_ACCU);
+}
+
+static void
+emit_add32_gpr0(struct anv_batch *batch, uint32_t N)
+{
+ emit_lri(batch, CS_GPR(1), N);
+ emit_alu_add(batch, MI_ALU_REG0, MI_ALU_REG0, MI_ALU_REG1);
+}
+
+static void
+emit_alu_shl(struct anv_batch *batch, unsigned dst_reg,
+ unsigned src_reg, unsigned shift)
+{
+ assert(shift > 0);
+
+ uint32_t *dw = anv_batch_emitn(batch, 1 + 4 * shift, GENX(MI_MATH));
+ for (unsigned i = 0; i < shift; i++) {
+ unsigned add_src = (i == 0) ? src_reg : dst_reg;
+ dw[1 + (i * 4) + 0] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCA, add_src);
+ dw[1 + (i * 4) + 1] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCB, add_src);
+ dw[1 + (i * 4) + 2] = mi_alu(MI_ALU_ADD, 0, 0);
+ dw[1 + (i * 4) + 3] = mi_alu(MI_ALU_STORE, dst_reg, MI_ALU_ACCU);
+ }
+}
+
+static void
+emit_div32_gpr0(struct anv_batch *batch, uint32_t D)
+{
+ /* Zero out the top of GPR0 */
+ emit_lri(batch, CS_GPR(0) + 4, 0);
+
+ if (D == 0) {
+ /* This invalid, but we should do something so we set GPR0 to 0. */
+ emit_lri(batch, CS_GPR(0), 0);
+ } else if (util_is_power_of_two_or_zero(D)) {
+ unsigned log2_D = util_logbase2(D);
+ assert(log2_D < 32);
+ /* We right-shift by log2(D) by left-shifting by 32 - log2(D) and taking
+ * the top 32 bits of the result.
+ */
+ emit_alu_shl(batch, MI_ALU_REG0, MI_ALU_REG0, 32 - log2_D);
+ emit_lrr(batch, CS_GPR(0) + 0, CS_GPR(0) + 4);
+ emit_lri(batch, CS_GPR(0) + 4, 0);
+ } else {
+ struct util_fast_udiv_info m = util_compute_fast_udiv_info(D, 32, 32);
+ assert(m.multiplier <= UINT32_MAX);
+
+ if (m.pre_shift) {
+ /* We right-shift by L by left-shifting by 32 - l and taking the top
+ * 32 bits of the result.
+ */
+ if (m.pre_shift < 32)
+ emit_alu_shl(batch, MI_ALU_REG0, MI_ALU_REG0, 32 - m.pre_shift);
+ emit_lrr(batch, CS_GPR(0) + 0, CS_GPR(0) + 4);
+ emit_lri(batch, CS_GPR(0) + 4, 0);
+ }
+
+ /* Do the 32x32 multiply into gpr0 */
+ emit_mul_gpr0(batch, m.multiplier);
+
+ if (m.increment) {
+ /* If we need to increment, save off a copy of GPR0 */
+ emit_lri(batch, CS_GPR(1) + 0, m.multiplier);
+ emit_lri(batch, CS_GPR(1) + 4, 0);
+ emit_alu_add(batch, MI_ALU_REG0, MI_ALU_REG0, MI_ALU_REG1);
+ }
+
+ /* Shift by 32 */
+ emit_lrr(batch, CS_GPR(0) + 0, CS_GPR(0) + 4);
+ emit_lri(batch, CS_GPR(0) + 4, 0);
+
+ if (m.post_shift) {
+ /* We right-shift by L by left-shifting by 32 - l and taking the top
+ * 32 bits of the result.
+ */
+ if (m.post_shift < 32)
+ emit_alu_shl(batch, MI_ALU_REG0, MI_ALU_REG0, 32 - m.post_shift);
+ emit_lrr(batch, CS_GPR(0) + 0, CS_GPR(0) + 4);
+ emit_lri(batch, CS_GPR(0) + 4, 0);
+ }
+ }
+}
+
#endif /* GEN_IS_HASWELL || GEN_GEN >= 8 */
+void genX(CmdDrawIndirectByteCountEXT)(
+ VkCommandBuffer commandBuffer,
+ uint32_t instanceCount,
+ uint32_t firstInstance,
+ VkBuffer counterBuffer,
+ VkDeviceSize counterBufferOffset,
+ uint32_t counterOffset,
+ uint32_t vertexStride)
+{
+#if GEN_IS_HASWELL || GEN_GEN >= 8
+ ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
+ ANV_FROM_HANDLE(anv_buffer, counter_buffer, counterBuffer);
+ struct anv_pipeline *pipeline = cmd_buffer->state.gfx.base.pipeline;
+ const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
+
+ /* firstVertex is always zero for this draw function */
+ const uint32_t firstVertex = 0;
+
+ if (anv_batch_has_error(&cmd_buffer->batch))
+ return;
+
+ genX(cmd_buffer_flush_state)(cmd_buffer);
+
+ if (vs_prog_data->uses_firstvertex ||
+ vs_prog_data->uses_baseinstance)
+ emit_base_vertex_instance(cmd_buffer, firstVertex, firstInstance);
+ if (vs_prog_data->uses_drawid)
+ emit_draw_index(cmd_buffer, 0);
+
+ /* Our implementation of VK_KHR_multiview uses instancing to draw the
+ * different views. We need to multiply instanceCount by the view count.
+ */
+ instanceCount *= anv_subpass_view_count(cmd_buffer->state.subpass);
+
+ emit_lrm(&cmd_buffer->batch, CS_GPR(0),
+ anv_address_add(counter_buffer->address, counterBufferOffset));
+ if (counterOffset)
+ emit_add32_gpr0(&cmd_buffer->batch, -counterOffset);
+ emit_div32_gpr0(&cmd_buffer->batch, vertexStride);
+ emit_lrr(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, CS_GPR(0));
+
+ emit_lri(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, firstVertex);
+ emit_lri(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, instanceCount);
+ emit_lri(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, firstInstance);
+ emit_lri(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, 0);
+
+ anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
+ prim.IndirectParameterEnable = true;
+ prim.VertexAccessType = SEQUENTIAL;
+ prim.PrimitiveTopologyType = pipeline->topology;
+ }
+#endif /* GEN_IS_HASWELL || GEN_GEN >= 8 */
+}
+
static void
load_indirect_parameters(struct anv_cmd_buffer *cmd_buffer,
struct anv_address addr,
{
struct anv_batch *batch = &cmd_buffer->batch;
- emit_lrm(batch, GEN7_3DPRIM_VERTEX_COUNT, addr.bo, addr.offset);
+ emit_lrm(batch, GEN7_3DPRIM_VERTEX_COUNT, anv_address_add(addr, 0));
unsigned view_count = anv_subpass_view_count(cmd_buffer->state.subpass);
if (view_count > 1) {
#if GEN_IS_HASWELL || GEN_GEN >= 8
- emit_lrm(batch, CS_GPR(0), addr.bo, addr.offset + 4);
+ emit_lrm(batch, CS_GPR(0), anv_address_add(addr, 4));
emit_mul_gpr0(batch, view_count);
emit_lrr(batch, GEN7_3DPRIM_INSTANCE_COUNT, CS_GPR(0));
#else
anv_finishme("Multiview + indirect draw requires MI_MATH; "
"MI_MATH is not supported on Ivy Bridge");
- emit_lrm(batch, GEN7_3DPRIM_INSTANCE_COUNT, addr.bo, addr.offset + 4);
+ emit_lrm(batch, GEN7_3DPRIM_INSTANCE_COUNT, anv_address_add(addr, 4));
#endif
} else {
- emit_lrm(batch, GEN7_3DPRIM_INSTANCE_COUNT, addr.bo, addr.offset + 4);
+ emit_lrm(batch, GEN7_3DPRIM_INSTANCE_COUNT, anv_address_add(addr, 4));
}
- emit_lrm(batch, GEN7_3DPRIM_START_VERTEX, addr.bo, addr.offset + 8);
+ emit_lrm(batch, GEN7_3DPRIM_START_VERTEX, anv_address_add(addr, 8));
if (indexed) {
- emit_lrm(batch, GEN7_3DPRIM_BASE_VERTEX, addr.bo, addr.offset + 12);
- emit_lrm(batch, GEN7_3DPRIM_START_INSTANCE, addr.bo, addr.offset + 16);
+ emit_lrm(batch, GEN7_3DPRIM_BASE_VERTEX, anv_address_add(addr, 12));
+ emit_lrm(batch, GEN7_3DPRIM_START_INSTANCE, anv_address_add(addr, 16));
} else {
- emit_lrm(batch, GEN7_3DPRIM_START_INSTANCE, addr.bo, addr.offset + 12);
+ emit_lrm(batch, GEN7_3DPRIM_START_INSTANCE, anv_address_add(addr, 12));
emit_lri(batch, GEN7_3DPRIM_BASE_VERTEX, 0);
}
}
genX(cmd_buffer_flush_state)(cmd_buffer);
+ if (cmd_buffer->state.conditional_render_enabled)
+ genX(cmd_emit_conditional_render_predicate)(cmd_buffer);
+
for (uint32_t i = 0; i < drawCount; i++) {
- struct anv_address draw = {
- .bo = buffer->bo,
- .offset = buffer->offset + offset,
- };
+ struct anv_address draw = anv_address_add(buffer->address, offset);
if (vs_prog_data->uses_firstvertex ||
vs_prog_data->uses_baseinstance)
anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
prim.IndirectParameterEnable = true;
+ prim.PredicateEnable = cmd_buffer->state.conditional_render_enabled;
prim.VertexAccessType = SEQUENTIAL;
prim.PrimitiveTopologyType = pipeline->topology;
}
genX(cmd_buffer_flush_state)(cmd_buffer);
+ if (cmd_buffer->state.conditional_render_enabled)
+ genX(cmd_emit_conditional_render_predicate)(cmd_buffer);
+
for (uint32_t i = 0; i < drawCount; i++) {
- struct anv_address draw = {
- .bo = buffer->bo,
- .offset = buffer->offset + offset,
- };
+ struct anv_address draw = anv_address_add(buffer->address, offset);
+
+ /* TODO: We need to stomp base vertex to 0 somehow */
+ if (vs_prog_data->uses_firstvertex ||
+ vs_prog_data->uses_baseinstance)
+ emit_base_vertex_instance_bo(cmd_buffer, anv_address_add(draw, 12));
+ if (vs_prog_data->uses_drawid)
+ emit_draw_index(cmd_buffer, i);
+
+ load_indirect_parameters(cmd_buffer, draw, true);
+
+ anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
+ prim.IndirectParameterEnable = true;
+ prim.PredicateEnable = cmd_buffer->state.conditional_render_enabled;
+ prim.VertexAccessType = RANDOM;
+ prim.PrimitiveTopologyType = pipeline->topology;
+ }
+
+ offset += stride;
+ }
+}
+
+#define TMP_DRAW_COUNT_REG MI_ALU_REG14
+
+static void
+prepare_for_draw_count_predicate(struct anv_cmd_buffer *cmd_buffer,
+ struct anv_address count_address,
+ const bool conditional_render_enabled)
+{
+ if (conditional_render_enabled) {
+#if GEN_GEN >= 8 || GEN_IS_HASWELL
+ emit_lrm(&cmd_buffer->batch, CS_GPR(TMP_DRAW_COUNT_REG), count_address);
+ emit_lri(&cmd_buffer->batch, CS_GPR(TMP_DRAW_COUNT_REG) + 4, 0);
+#endif
+ } else {
+ /* Upload the current draw count from the draw parameters buffer to
+ * MI_PREDICATE_SRC0.
+ */
+ emit_lrm(&cmd_buffer->batch, MI_PREDICATE_SRC0, count_address);
+ emit_lri(&cmd_buffer->batch, MI_PREDICATE_SRC0 + 4, 0);
+
+ emit_lri(&cmd_buffer->batch, MI_PREDICATE_SRC1 + 4, 0);
+ }
+}
+
+static void
+emit_draw_count_predicate(struct anv_cmd_buffer *cmd_buffer,
+ uint32_t draw_index)
+{
+ /* Upload the index of the current primitive to MI_PREDICATE_SRC1. */
+ emit_lri(&cmd_buffer->batch, MI_PREDICATE_SRC1, draw_index);
+
+ if (draw_index == 0) {
+ anv_batch_emit(&cmd_buffer->batch, GENX(MI_PREDICATE), mip) {
+ mip.LoadOperation = LOAD_LOADINV;
+ mip.CombineOperation = COMBINE_SET;
+ mip.CompareOperation = COMPARE_SRCS_EQUAL;
+ }
+ } else {
+ /* While draw_index < draw_count the predicate's result will be
+ * (draw_index == draw_count) ^ TRUE = TRUE
+ * When draw_index == draw_count the result is
+ * (TRUE) ^ TRUE = FALSE
+ * After this all results will be:
+ * (FALSE) ^ FALSE = FALSE
+ */
+ anv_batch_emit(&cmd_buffer->batch, GENX(MI_PREDICATE), mip) {
+ mip.LoadOperation = LOAD_LOAD;
+ mip.CombineOperation = COMBINE_XOR;
+ mip.CompareOperation = COMPARE_SRCS_EQUAL;
+ }
+ }
+}
+
+#if GEN_GEN >= 8 || GEN_IS_HASWELL
+static void
+emit_draw_count_predicate_with_conditional_render(
+ struct anv_cmd_buffer *cmd_buffer,
+ uint32_t draw_index)
+{
+ const int draw_index_reg = MI_ALU_REG0;
+ const int tmp_result_reg = MI_ALU_REG1;
+
+ emit_lri(&cmd_buffer->batch, CS_GPR(draw_index_reg), draw_index);
+ emit_lri(&cmd_buffer->batch, CS_GPR(draw_index_reg) + 4, 0);
+
+ uint32_t *dw;
+ /* Compute (draw_index < draw_count).
+ * We do this by subtracting and storing the carry bit.
+ */
+ dw = anv_batch_emitn(&cmd_buffer->batch, 9, GENX(MI_MATH));
+ dw[1] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCA, draw_index_reg);
+ dw[2] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCB, TMP_DRAW_COUNT_REG);
+ dw[3] = mi_alu(MI_ALU_SUB, 0, 0);
+ dw[4] = mi_alu(MI_ALU_STORE, tmp_result_reg, MI_ALU_CF);
+ /* & condition */
+ dw[5] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCA, tmp_result_reg);
+ dw[6] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCB, ANV_PREDICATE_RESULT_REG);
+ dw[7] = mi_alu(MI_ALU_AND, 0, 0);
+ dw[8] = mi_alu(MI_ALU_STORE, tmp_result_reg, MI_ALU_ACCU);
+
+#if GEN_GEN >= 8
+ emit_lrr(&cmd_buffer->batch, MI_PREDICATE_RESULT, CS_GPR(tmp_result_reg));
+#else
+ /* MI_PREDICATE_RESULT is not whitelisted in i915 command parser
+ * so we emit MI_PREDICATE to set it.
+ */
+
+ emit_lrr(&cmd_buffer->batch, MI_PREDICATE_SRC0, CS_GPR(tmp_result_reg));
+ emit_lri(&cmd_buffer->batch, MI_PREDICATE_SRC0 + 4, 0);
+ emit_lri(&cmd_buffer->batch, MI_PREDICATE_SRC1, 0);
+ emit_lri(&cmd_buffer->batch, MI_PREDICATE_SRC1 + 4, 0);
+
+ anv_batch_emit(&cmd_buffer->batch, GENX(MI_PREDICATE), mip) {
+ mip.LoadOperation = LOAD_LOADINV;
+ mip.CombineOperation = COMBINE_SET;
+ mip.CompareOperation = COMPARE_SRCS_EQUAL;
+ }
+#endif
+}
+#endif
+
+void genX(CmdDrawIndirectCountKHR)(
+ VkCommandBuffer commandBuffer,
+ VkBuffer _buffer,
+ VkDeviceSize offset,
+ VkBuffer _countBuffer,
+ VkDeviceSize countBufferOffset,
+ uint32_t maxDrawCount,
+ uint32_t stride)
+{
+ ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
+ ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
+ ANV_FROM_HANDLE(anv_buffer, count_buffer, _countBuffer);
+ struct anv_cmd_state *cmd_state = &cmd_buffer->state;
+ struct anv_pipeline *pipeline = cmd_state->gfx.base.pipeline;
+ const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
+
+ if (anv_batch_has_error(&cmd_buffer->batch))
+ return;
+
+ genX(cmd_buffer_flush_state)(cmd_buffer);
+
+ struct anv_address count_address =
+ anv_address_add(count_buffer->address, countBufferOffset);
+
+ prepare_for_draw_count_predicate(cmd_buffer, count_address,
+ cmd_state->conditional_render_enabled);
+
+ for (uint32_t i = 0; i < maxDrawCount; i++) {
+ struct anv_address draw = anv_address_add(buffer->address, offset);
+
+#if GEN_GEN >= 8 || GEN_IS_HASWELL
+ if (cmd_state->conditional_render_enabled) {
+ emit_draw_count_predicate_with_conditional_render(cmd_buffer, i);
+ } else {
+ emit_draw_count_predicate(cmd_buffer, i);
+ }
+#else
+ emit_draw_count_predicate(cmd_buffer, i);
+#endif
+
+ if (vs_prog_data->uses_firstvertex ||
+ vs_prog_data->uses_baseinstance)
+ emit_base_vertex_instance_bo(cmd_buffer, anv_address_add(draw, 8));
+ if (vs_prog_data->uses_drawid)
+ emit_draw_index(cmd_buffer, i);
+
+ load_indirect_parameters(cmd_buffer, draw, false);
+
+ anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
+ prim.IndirectParameterEnable = true;
+ prim.PredicateEnable = true;
+ prim.VertexAccessType = SEQUENTIAL;
+ prim.PrimitiveTopologyType = pipeline->topology;
+ }
+
+ offset += stride;
+ }
+}
+
+void genX(CmdDrawIndexedIndirectCountKHR)(
+ VkCommandBuffer commandBuffer,
+ VkBuffer _buffer,
+ VkDeviceSize offset,
+ VkBuffer _countBuffer,
+ VkDeviceSize countBufferOffset,
+ uint32_t maxDrawCount,
+ uint32_t stride)
+{
+ ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
+ ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
+ ANV_FROM_HANDLE(anv_buffer, count_buffer, _countBuffer);
+ struct anv_cmd_state *cmd_state = &cmd_buffer->state;
+ struct anv_pipeline *pipeline = cmd_state->gfx.base.pipeline;
+ const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
+
+ if (anv_batch_has_error(&cmd_buffer->batch))
+ return;
+
+ genX(cmd_buffer_flush_state)(cmd_buffer);
+
+ struct anv_address count_address =
+ anv_address_add(count_buffer->address, countBufferOffset);
+
+ prepare_for_draw_count_predicate(cmd_buffer, count_address,
+ cmd_state->conditional_render_enabled);
+
+ for (uint32_t i = 0; i < maxDrawCount; i++) {
+ struct anv_address draw = anv_address_add(buffer->address, offset);
+
+#if GEN_GEN >= 8 || GEN_IS_HASWELL
+ if (cmd_state->conditional_render_enabled) {
+ emit_draw_count_predicate_with_conditional_render(cmd_buffer, i);
+ } else {
+ emit_draw_count_predicate(cmd_buffer, i);
+ }
+#else
+ emit_draw_count_predicate(cmd_buffer, i);
+#endif
/* TODO: We need to stomp base vertex to 0 somehow */
if (vs_prog_data->uses_firstvertex ||
anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
prim.IndirectParameterEnable = true;
+ prim.PredicateEnable = true;
prim.VertexAccessType = RANDOM;
prim.PrimitiveTopologyType = pipeline->topology;
}
}
}
+void genX(CmdBeginTransformFeedbackEXT)(
+ VkCommandBuffer commandBuffer,
+ uint32_t firstCounterBuffer,
+ uint32_t counterBufferCount,
+ const VkBuffer* pCounterBuffers,
+ const VkDeviceSize* pCounterBufferOffsets)
+{
+ ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
+
+ assert(firstCounterBuffer < MAX_XFB_BUFFERS);
+ assert(counterBufferCount <= MAX_XFB_BUFFERS);
+ assert(firstCounterBuffer + counterBufferCount <= MAX_XFB_BUFFERS);
+
+ /* From the SKL PRM Vol. 2c, SO_WRITE_OFFSET:
+ *
+ * "Ssoftware must ensure that no HW stream output operations can be in
+ * process or otherwise pending at the point that the MI_LOAD/STORE
+ * commands are processed. This will likely require a pipeline flush."
+ */
+ cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_CS_STALL_BIT;
+ genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
+
+ for (uint32_t idx = 0; idx < MAX_XFB_BUFFERS; idx++) {
+ /* If we have a counter buffer, this is a resume so we need to load the
+ * value into the streamout offset register. Otherwise, this is a begin
+ * and we need to reset it to zero.
+ */
+ if (pCounterBuffers &&
+ idx >= firstCounterBuffer &&
+ idx - firstCounterBuffer < counterBufferCount &&
+ pCounterBuffers[idx - firstCounterBuffer] != VK_NULL_HANDLE) {
+ uint32_t cb_idx = idx - firstCounterBuffer;
+ ANV_FROM_HANDLE(anv_buffer, counter_buffer, pCounterBuffers[cb_idx]);
+ uint64_t offset = pCounterBufferOffsets ?
+ pCounterBufferOffsets[cb_idx] : 0;
+
+ anv_batch_emit(&cmd_buffer->batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
+ lrm.RegisterAddress = GENX(SO_WRITE_OFFSET0_num) + idx * 4;
+ lrm.MemoryAddress = anv_address_add(counter_buffer->address,
+ offset);
+ }
+ } else {
+ anv_batch_emit(&cmd_buffer->batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
+ lri.RegisterOffset = GENX(SO_WRITE_OFFSET0_num) + idx * 4;
+ lri.DataDWord = 0;
+ }
+ }
+ }
+
+ cmd_buffer->state.xfb_enabled = true;
+ cmd_buffer->state.gfx.dirty |= ANV_CMD_DIRTY_XFB_ENABLE;
+}
+
+void genX(CmdEndTransformFeedbackEXT)(
+ VkCommandBuffer commandBuffer,
+ uint32_t firstCounterBuffer,
+ uint32_t counterBufferCount,
+ const VkBuffer* pCounterBuffers,
+ const VkDeviceSize* pCounterBufferOffsets)
+{
+ ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
+
+ assert(firstCounterBuffer < MAX_XFB_BUFFERS);
+ assert(counterBufferCount <= MAX_XFB_BUFFERS);
+ assert(firstCounterBuffer + counterBufferCount <= MAX_XFB_BUFFERS);
+
+ /* From the SKL PRM Vol. 2c, SO_WRITE_OFFSET:
+ *
+ * "Ssoftware must ensure that no HW stream output operations can be in
+ * process or otherwise pending at the point that the MI_LOAD/STORE
+ * commands are processed. This will likely require a pipeline flush."
+ */
+ cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_CS_STALL_BIT;
+ genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
+
+ for (uint32_t cb_idx = 0; cb_idx < counterBufferCount; cb_idx++) {
+ unsigned idx = firstCounterBuffer + cb_idx;
+
+ /* If we have a counter buffer, this is a resume so we need to load the
+ * value into the streamout offset register. Otherwise, this is a begin
+ * and we need to reset it to zero.
+ */
+ if (pCounterBuffers &&
+ cb_idx < counterBufferCount &&
+ pCounterBuffers[cb_idx] != VK_NULL_HANDLE) {
+ ANV_FROM_HANDLE(anv_buffer, counter_buffer, pCounterBuffers[cb_idx]);
+ uint64_t offset = pCounterBufferOffsets ?
+ pCounterBufferOffsets[cb_idx] : 0;
+
+ anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), srm) {
+ srm.MemoryAddress = anv_address_add(counter_buffer->address,
+ offset);
+ srm.RegisterAddress = GENX(SO_WRITE_OFFSET0_num) + idx * 4;
+ }
+ }
+ }
+
+ cmd_buffer->state.xfb_enabled = false;
+ cmd_buffer->state.gfx.dirty |= ANV_CMD_DIRTY_XFB_ENABLE;
+}
+
static VkResult
flush_compute_descriptor_set(struct anv_cmd_buffer *cmd_buffer)
{
curbe.CURBEDataStartAddress = push_state.offset;
}
}
+
+ cmd_buffer->state.push_constants_dirty &= ~VK_SHADER_STAGE_COMPUTE_BIT;
}
cmd_buffer->state.compute.pipeline_dirty = false;
sizes[0] = groupCountX;
sizes[1] = groupCountY;
sizes[2] = groupCountZ;
- anv_state_flush(cmd_buffer->device, state);
cmd_buffer->state.compute.num_workgroups = (struct anv_address) {
- .bo = &cmd_buffer->device->dynamic_state_pool.block_pool.bo,
+ .bo = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
.offset = state.offset,
};
}
genX(cmd_buffer_flush_compute_state)(cmd_buffer);
+ if (cmd_buffer->state.conditional_render_enabled)
+ genX(cmd_emit_conditional_render_predicate)(cmd_buffer);
+
anv_batch_emit(&cmd_buffer->batch, GENX(GPGPU_WALKER), ggw) {
+ ggw.PredicateEnable = cmd_buffer->state.conditional_render_enabled;
ggw.SIMDSize = prog_data->simd_size / 16;
ggw.ThreadDepthCounterMaximum = 0;
ggw.ThreadHeightCounterMaximum = 0;
ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
struct anv_pipeline *pipeline = cmd_buffer->state.compute.base.pipeline;
const struct brw_cs_prog_data *prog_data = get_cs_prog_data(pipeline);
- struct anv_address addr = {
- .bo = buffer->bo,
- .offset = buffer->offset + offset,
- };
+ struct anv_address addr = anv_address_add(buffer->address, offset);
struct anv_batch *batch = &cmd_buffer->batch;
anv_cmd_buffer_push_base_group_id(cmd_buffer, 0, 0, 0);
genX(cmd_buffer_flush_compute_state)(cmd_buffer);
- emit_lrm(batch, GPGPU_DISPATCHDIMX, addr.bo, addr.offset);
- emit_lrm(batch, GPGPU_DISPATCHDIMY, addr.bo, addr.offset + 4);
- emit_lrm(batch, GPGPU_DISPATCHDIMZ, addr.bo, addr.offset + 8);
+ emit_lrm(batch, GPGPU_DISPATCHDIMX, anv_address_add(addr, 0));
+ emit_lrm(batch, GPGPU_DISPATCHDIMY, anv_address_add(addr, 4));
+ emit_lrm(batch, GPGPU_DISPATCHDIMZ, anv_address_add(addr, 8));
#if GEN_GEN <= 7
/* Clear upper 32-bits of SRC0 and all 64-bits of SRC1 */
emit_lri(batch, MI_PREDICATE_SRC1 + 4, 0);
/* Load compute_dispatch_indirect_x_size into SRC0 */
- emit_lrm(batch, MI_PREDICATE_SRC0, addr.bo, addr.offset + 0);
+ emit_lrm(batch, MI_PREDICATE_SRC0, anv_address_add(addr, 0));
/* predicate = (compute_dispatch_indirect_x_size == 0); */
anv_batch_emit(batch, GENX(MI_PREDICATE), mip) {
}
/* Load compute_dispatch_indirect_y_size into SRC0 */
- emit_lrm(batch, MI_PREDICATE_SRC0, addr.bo, addr.offset + 4);
+ emit_lrm(batch, MI_PREDICATE_SRC0, anv_address_add(addr, 4));
/* predicate |= (compute_dispatch_indirect_y_size == 0); */
anv_batch_emit(batch, GENX(MI_PREDICATE), mip) {
}
/* Load compute_dispatch_indirect_z_size into SRC0 */
- emit_lrm(batch, MI_PREDICATE_SRC0, addr.bo, addr.offset + 8);
+ emit_lrm(batch, MI_PREDICATE_SRC0, anv_address_add(addr, 8));
/* predicate |= (compute_dispatch_indirect_z_size == 0); */
anv_batch_emit(batch, GENX(MI_PREDICATE), mip) {
}
/* predicate = !predicate; */
-#define COMPARE_FALSE 1
anv_batch_emit(batch, GENX(MI_PREDICATE), mip) {
mip.LoadOperation = LOAD_LOADINV;
mip.CombineOperation = COMBINE_OR;
mip.CompareOperation = COMPARE_FALSE;
}
+
+#if GEN_IS_HASWELL
+ if (cmd_buffer->state.conditional_render_enabled) {
+ emit_lrr(batch, MI_PREDICATE_SRC0, CS_GPR(ANV_PREDICATE_RESULT_REG));
+ /* predicate &= !(conditional_rendering_predicate == 0); */
+ anv_batch_emit(batch, GENX(MI_PREDICATE), mip) {
+ mip.LoadOperation = LOAD_LOADINV;
+ mip.CombineOperation = COMBINE_AND;
+ mip.CompareOperation = COMPARE_SRCS_EQUAL;
+ }
+ }
+#endif
+
+#else /* GEN_GEN > 7 */
+ if (cmd_buffer->state.conditional_render_enabled)
+ genX(cmd_emit_conditional_render_predicate)(cmd_buffer);
#endif
anv_batch_emit(batch, GENX(GPGPU_WALKER), ggw) {
ggw.IndirectParameterEnable = true;
- ggw.PredicateEnable = GEN_GEN <= 7;
+ ggw.PredicateEnable = GEN_GEN <= 7 ||
+ cmd_buffer->state.conditional_render_enabled;
ggw.SIMDSize = prog_data->simd_size / 16;
ggw.ThreadDepthCounterMaximum = 0;
ggw.ThreadHeightCounterMaximum = 0;
if (dw == NULL)
return;
- struct isl_depth_stencil_hiz_emit_info info = {
- .mocs = device->default_mocs,
- };
+ struct isl_depth_stencil_hiz_emit_info info = { };
if (iview)
info.view = &iview->planes[0].isl;
info.depth_address =
anv_batch_emit_reloc(&cmd_buffer->batch,
dw + device->isl_dev.ds.depth_offset / 4,
- image->planes[depth_plane].bo,
- image->planes[depth_plane].bo_offset +
+ image->planes[depth_plane].address.bo,
+ image->planes[depth_plane].address.offset +
surface->offset);
+ info.mocs =
+ anv_mocs_for_bo(device, image->planes[depth_plane].address.bo);
const uint32_t ds =
- cmd_buffer->state.subpass->depth_stencil_attachment.attachment;
+ cmd_buffer->state.subpass->depth_stencil_attachment->attachment;
info.hiz_usage = cmd_buffer->state.attachments[ds].aux_usage;
if (info.hiz_usage == ISL_AUX_USAGE_HIZ) {
info.hiz_surf = &image->planes[depth_plane].aux_surface.isl;
info.hiz_address =
anv_batch_emit_reloc(&cmd_buffer->batch,
dw + device->isl_dev.ds.hiz_offset / 4,
- image->planes[depth_plane].bo,
- image->planes[depth_plane].bo_offset +
+ image->planes[depth_plane].address.bo,
+ image->planes[depth_plane].address.offset +
image->planes[depth_plane].aux_surface.offset);
info.depth_clear_value = ANV_HZ_FC_VAL;
info.stencil_address =
anv_batch_emit_reloc(&cmd_buffer->batch,
dw + device->isl_dev.ds.stencil_offset / 4,
- image->planes[stencil_plane].bo,
- image->planes[stencil_plane].bo_offset + surface->offset);
+ image->planes[stencil_plane].address.bo,
+ image->planes[stencil_plane].address.offset +
+ surface->offset);
+ info.mocs =
+ anv_mocs_for_bo(device, image->planes[stencil_plane].address.bo);
}
isl_emit_depth_stencil_hiz_s(&device->isl_dev, dw, &info);
union isl_color_value clear_color = {};
anv_clear_color_from_att_state(&clear_color, att_state, iview);
if (iview->image->samples == 1) {
- anv_image_ccs_op(cmd_buffer, image, VK_IMAGE_ASPECT_COLOR_BIT,
+ anv_image_ccs_op(cmd_buffer, image,
+ iview->planes[0].isl.format,
+ VK_IMAGE_ASPECT_COLOR_BIT,
0, 0, 1, ISL_AUX_OP_FAST_CLEAR,
&clear_color,
false);
} else {
- anv_image_mcs_op(cmd_buffer, image, VK_IMAGE_ASPECT_COLOR_BIT,
+ anv_image_mcs_op(cmd_buffer, image,
+ iview->planes[0].isl.format,
+ VK_IMAGE_ASPECT_COLOR_BIT,
0, 1, ISL_AUX_OP_FAST_CLEAR,
&clear_color,
false);
if (GEN_GEN < 10 &&
(att_state->pending_load_aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) &&
- image->planes[0].aux_surface.isl.size > 0 &&
+ image->planes[0].aux_surface.isl.size_B > 0 &&
iview->planes[0].isl.base_level == 0 &&
iview->planes[0].isl.base_array_layer == 0) {
if (att_state->aux_usage != ISL_AUX_USAGE_NONE) {
cmd_buffer_emit_depth_stencil(cmd_buffer);
}
+static enum blorp_filter
+vk_to_blorp_resolve_mode(VkResolveModeFlagBitsKHR vk_mode)
+{
+ switch (vk_mode) {
+ case VK_RESOLVE_MODE_SAMPLE_ZERO_BIT_KHR:
+ return BLORP_FILTER_SAMPLE_0;
+ case VK_RESOLVE_MODE_AVERAGE_BIT_KHR:
+ return BLORP_FILTER_AVERAGE;
+ case VK_RESOLVE_MODE_MIN_BIT_KHR:
+ return BLORP_FILTER_MIN_SAMPLE;
+ case VK_RESOLVE_MODE_MAX_BIT_KHR:
+ return BLORP_FILTER_MAX_SAMPLE;
+ default:
+ return BLORP_FILTER_NONE;
+ }
+}
+
static void
cmd_buffer_end_subpass(struct anv_cmd_buffer *cmd_buffer)
{
struct anv_cmd_state *cmd_state = &cmd_buffer->state;
struct anv_subpass *subpass = cmd_state->subpass;
uint32_t subpass_id = anv_get_subpass_id(&cmd_buffer->state);
+ struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
- anv_cmd_buffer_resolve_subpass(cmd_buffer);
+ if (subpass->has_color_resolve) {
+ /* We are about to do some MSAA resolves. We need to flush so that the
+ * result of writes to the MSAA color attachments show up in the sampler
+ * when we blit to the single-sampled resolve target.
+ */
+ cmd_buffer->state.pending_pipe_bits |=
+ ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT |
+ ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
+
+ for (uint32_t i = 0; i < subpass->color_count; ++i) {
+ uint32_t src_att = subpass->color_attachments[i].attachment;
+ uint32_t dst_att = subpass->resolve_attachments[i].attachment;
+
+ if (dst_att == VK_ATTACHMENT_UNUSED)
+ continue;
+
+ assert(src_att < cmd_buffer->state.pass->attachment_count);
+ assert(dst_att < cmd_buffer->state.pass->attachment_count);
+
+ if (cmd_buffer->state.attachments[dst_att].pending_clear_aspects) {
+ /* From the Vulkan 1.0 spec:
+ *
+ * If the first use of an attachment in a render pass is as a
+ * resolve attachment, then the loadOp is effectively ignored
+ * as the resolve is guaranteed to overwrite all pixels in the
+ * render area.
+ */
+ cmd_buffer->state.attachments[dst_att].pending_clear_aspects = 0;
+ }
+
+ struct anv_image_view *src_iview = fb->attachments[src_att];
+ struct anv_image_view *dst_iview = fb->attachments[dst_att];
+
+ const VkRect2D render_area = cmd_buffer->state.render_area;
+
+ enum isl_aux_usage src_aux_usage =
+ cmd_buffer->state.attachments[src_att].aux_usage;
+ enum isl_aux_usage dst_aux_usage =
+ cmd_buffer->state.attachments[dst_att].aux_usage;
+
+ assert(src_iview->aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT &&
+ dst_iview->aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT);
+
+ anv_image_msaa_resolve(cmd_buffer,
+ src_iview->image, src_aux_usage,
+ src_iview->planes[0].isl.base_level,
+ src_iview->planes[0].isl.base_array_layer,
+ dst_iview->image, dst_aux_usage,
+ dst_iview->planes[0].isl.base_level,
+ dst_iview->planes[0].isl.base_array_layer,
+ VK_IMAGE_ASPECT_COLOR_BIT,
+ render_area.offset.x, render_area.offset.y,
+ render_area.offset.x, render_area.offset.y,
+ render_area.extent.width,
+ render_area.extent.height,
+ fb->layers, BLORP_FILTER_NONE);
+ }
+ }
+
+ if (subpass->ds_resolve_attachment) {
+ /* We are about to do some MSAA resolves. We need to flush so that the
+ * result of writes to the MSAA depth attachments show up in the sampler
+ * when we blit to the single-sampled resolve target.
+ */
+ cmd_buffer->state.pending_pipe_bits |=
+ ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT |
+ ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
+
+ uint32_t src_att = subpass->depth_stencil_attachment->attachment;
+ uint32_t dst_att = subpass->ds_resolve_attachment->attachment;
+
+ assert(src_att < cmd_buffer->state.pass->attachment_count);
+ assert(dst_att < cmd_buffer->state.pass->attachment_count);
+
+ if (cmd_buffer->state.attachments[dst_att].pending_clear_aspects) {
+ /* From the Vulkan 1.0 spec:
+ *
+ * If the first use of an attachment in a render pass is as a
+ * resolve attachment, then the loadOp is effectively ignored
+ * as the resolve is guaranteed to overwrite all pixels in the
+ * render area.
+ */
+ cmd_buffer->state.attachments[dst_att].pending_clear_aspects = 0;
+ }
+
+ struct anv_image_view *src_iview = fb->attachments[src_att];
+ struct anv_image_view *dst_iview = fb->attachments[dst_att];
+
+ const VkRect2D render_area = cmd_buffer->state.render_area;
+
+ if ((src_iview->image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) &&
+ subpass->depth_resolve_mode != VK_RESOLVE_MODE_NONE_KHR) {
+
+ struct anv_attachment_state *src_state =
+ &cmd_state->attachments[src_att];
+ struct anv_attachment_state *dst_state =
+ &cmd_state->attachments[dst_att];
+
+ /* MSAA resolves sample from the source attachment. Transition the
+ * depth attachment first to get rid of any HiZ that we may not be
+ * able to handle.
+ */
+ transition_depth_buffer(cmd_buffer, src_iview->image,
+ src_state->current_layout,
+ VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+ src_state->aux_usage =
+ anv_layout_to_aux_usage(&cmd_buffer->device->info, src_iview->image,
+ VK_IMAGE_ASPECT_DEPTH_BIT,
+ VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+ src_state->current_layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+
+ /* MSAA resolves write to the resolve attachment as if it were any
+ * other transfer op. Transition the resolve attachment accordingly.
+ */
+ VkImageLayout dst_initial_layout = dst_state->current_layout;
+
+ /* If our render area is the entire size of the image, we're going to
+ * blow it all away so we can claim the initial layout is UNDEFINED
+ * and we'll get a HiZ ambiguate instead of a resolve.
+ */
+ if (dst_iview->image->type != VK_IMAGE_TYPE_3D &&
+ render_area.offset.x == 0 && render_area.offset.y == 0 &&
+ render_area.extent.width == dst_iview->extent.width &&
+ render_area.extent.height == dst_iview->extent.height)
+ dst_initial_layout = VK_IMAGE_LAYOUT_UNDEFINED;
+
+ transition_depth_buffer(cmd_buffer, dst_iview->image,
+ dst_initial_layout,
+ VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
+ dst_state->aux_usage =
+ anv_layout_to_aux_usage(&cmd_buffer->device->info, dst_iview->image,
+ VK_IMAGE_ASPECT_DEPTH_BIT,
+ VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL);
+ dst_state->current_layout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
+
+ enum blorp_filter filter =
+ vk_to_blorp_resolve_mode(subpass->depth_resolve_mode);
+
+ anv_image_msaa_resolve(cmd_buffer,
+ src_iview->image, src_state->aux_usage,
+ src_iview->planes[0].isl.base_level,
+ src_iview->planes[0].isl.base_array_layer,
+ dst_iview->image, dst_state->aux_usage,
+ dst_iview->planes[0].isl.base_level,
+ dst_iview->planes[0].isl.base_array_layer,
+ VK_IMAGE_ASPECT_DEPTH_BIT,
+ render_area.offset.x, render_area.offset.y,
+ render_area.offset.x, render_area.offset.y,
+ render_area.extent.width,
+ render_area.extent.height,
+ fb->layers, filter);
+ }
+
+ if ((src_iview->image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
+ subpass->stencil_resolve_mode != VK_RESOLVE_MODE_NONE_KHR) {
+
+ enum isl_aux_usage src_aux_usage = ISL_AUX_USAGE_NONE;
+ enum isl_aux_usage dst_aux_usage = ISL_AUX_USAGE_NONE;
+
+ enum blorp_filter filter =
+ vk_to_blorp_resolve_mode(subpass->stencil_resolve_mode);
+
+ anv_image_msaa_resolve(cmd_buffer,
+ src_iview->image, src_aux_usage,
+ src_iview->planes[0].isl.base_level,
+ src_iview->planes[0].isl.base_array_layer,
+ dst_iview->image, dst_aux_usage,
+ dst_iview->planes[0].isl.base_level,
+ dst_iview->planes[0].isl.base_array_layer,
+ VK_IMAGE_ASPECT_STENCIL_BIT,
+ render_area.offset.x, render_area.offset.y,
+ render_area.offset.x, render_area.offset.y,
+ render_area.extent.width,
+ render_area.extent.height,
+ fb->layers, filter);
+ }
+ }
- struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
for (uint32_t i = 0; i < subpass->attachment_count; ++i) {
const uint32_t a = subpass->attachments[i].attachment;
if (a == VK_ATTACHMENT_UNUSED)
struct anv_image_view *iview = fb->attachments[a];
const struct anv_image *image = iview->image;
+ if ((image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) &&
+ image->vk_format != iview->vk_format) {
+ enum anv_fast_clear_type fast_clear_type =
+ anv_layout_to_fast_clear_type(&cmd_buffer->device->info,
+ image, VK_IMAGE_ASPECT_COLOR_BIT,
+ att_state->current_layout);
+
+ /* If any clear color was used, flush it down the aux surfaces. If we
+ * don't do it now using the view's format we might use the clear
+ * color incorrectly in the following resolves (for example with an
+ * SRGB view & a UNORM image).
+ */
+ if (fast_clear_type != ANV_FAST_CLEAR_NONE) {
+ anv_perf_warn(cmd_buffer->device->instance, fb,
+ "Doing a partial resolve to get rid of clear color at the "
+ "end of a renderpass due to an image/view format mismatch");
+
+ uint32_t base_layer, layer_count;
+ if (image->type == VK_IMAGE_TYPE_3D) {
+ base_layer = 0;
+ layer_count = anv_minify(iview->image->extent.depth,
+ iview->planes[0].isl.base_level);
+ } else {
+ base_layer = iview->planes[0].isl.base_array_layer;
+ layer_count = fb->layers;
+ }
+
+ for (uint32_t a = 0; a < layer_count; a++) {
+ uint32_t array_layer = base_layer + a;
+ if (image->samples == 1) {
+ anv_cmd_predicated_ccs_resolve(cmd_buffer, image,
+ iview->planes[0].isl.format,
+ VK_IMAGE_ASPECT_COLOR_BIT,
+ iview->planes[0].isl.base_level,
+ array_layer,
+ ISL_AUX_OP_PARTIAL_RESOLVE,
+ ANV_FAST_CLEAR_NONE);
+ } else {
+ anv_cmd_predicated_mcs_resolve(cmd_buffer, image,
+ iview->planes[0].isl.format,
+ VK_IMAGE_ASPECT_COLOR_BIT,
+ base_layer,
+ ISL_AUX_OP_PARTIAL_RESOLVE,
+ ANV_FAST_CLEAR_NONE);
+ }
+ }
+ }
+ }
+
/* Transition the image into the final layout for this render pass */
VkImageLayout target_layout =
cmd_state->pass->attachments[a].final_layout;
cmd_buffer_begin_subpass(cmd_buffer, 0);
}
+void genX(CmdBeginRenderPass2KHR)(
+ VkCommandBuffer commandBuffer,
+ const VkRenderPassBeginInfo* pRenderPassBeginInfo,
+ const VkSubpassBeginInfoKHR* pSubpassBeginInfo)
+{
+ genX(CmdBeginRenderPass)(commandBuffer, pRenderPassBeginInfo,
+ pSubpassBeginInfo->contents);
+}
+
void genX(CmdNextSubpass)(
VkCommandBuffer commandBuffer,
VkSubpassContents contents)
cmd_buffer_begin_subpass(cmd_buffer, prev_subpass + 1);
}
+void genX(CmdNextSubpass2KHR)(
+ VkCommandBuffer commandBuffer,
+ const VkSubpassBeginInfoKHR* pSubpassBeginInfo,
+ const VkSubpassEndInfoKHR* pSubpassEndInfo)
+{
+ genX(CmdNextSubpass)(commandBuffer, pSubpassBeginInfo->contents);
+}
+
void genX(CmdEndRenderPass)(
VkCommandBuffer commandBuffer)
{
cmd_buffer->state.pass = NULL;
cmd_buffer->state.subpass = NULL;
}
+
+void genX(CmdEndRenderPass2KHR)(
+ VkCommandBuffer commandBuffer,
+ const VkSubpassEndInfoKHR* pSubpassEndInfo)
+{
+ genX(CmdEndRenderPass)(commandBuffer);
+}
+
+void
+genX(cmd_emit_conditional_render_predicate)(struct anv_cmd_buffer *cmd_buffer)
+{
+#if GEN_GEN >= 8 || GEN_IS_HASWELL
+ emit_lrr(&cmd_buffer->batch, MI_PREDICATE_SRC0, CS_GPR(ANV_PREDICATE_RESULT_REG));
+ emit_lri(&cmd_buffer->batch, MI_PREDICATE_SRC0 + 4, 0);
+ emit_lri(&cmd_buffer->batch, MI_PREDICATE_SRC1, 0);
+ emit_lri(&cmd_buffer->batch, MI_PREDICATE_SRC1 + 4, 0);
+
+ anv_batch_emit(&cmd_buffer->batch, GENX(MI_PREDICATE), mip) {
+ mip.LoadOperation = LOAD_LOADINV;
+ mip.CombineOperation = COMBINE_SET;
+ mip.CompareOperation = COMPARE_SRCS_EQUAL;
+ }
+#endif
+}
+
+#if GEN_GEN >= 8 || GEN_IS_HASWELL
+void genX(CmdBeginConditionalRenderingEXT)(
+ VkCommandBuffer commandBuffer,
+ const VkConditionalRenderingBeginInfoEXT* pConditionalRenderingBegin)
+{
+ ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
+ ANV_FROM_HANDLE(anv_buffer, buffer, pConditionalRenderingBegin->buffer);
+ struct anv_cmd_state *cmd_state = &cmd_buffer->state;
+ struct anv_address value_address =
+ anv_address_add(buffer->address, pConditionalRenderingBegin->offset);
+
+ const bool isInverted = pConditionalRenderingBegin->flags &
+ VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT;
+
+ cmd_state->conditional_render_enabled = true;
+
+ genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
+
+ /* Section 19.4 of the Vulkan 1.1.85 spec says:
+ *
+ * If the value of the predicate in buffer memory changes
+ * while conditional rendering is active, the rendering commands
+ * may be discarded in an implementation-dependent way.
+ * Some implementations may latch the value of the predicate
+ * upon beginning conditional rendering while others
+ * may read it before every rendering command.
+ *
+ * So it's perfectly fine to read a value from the buffer once.
+ */
+ emit_lrm(&cmd_buffer->batch, CS_GPR(MI_ALU_REG0), value_address);
+ /* Zero the top 32-bits of MI_PREDICATE_SRC0 */
+ emit_lri(&cmd_buffer->batch, CS_GPR(MI_ALU_REG0) + 4, 0);
+
+ /* Precompute predicate result, it is necessary to support secondary
+ * command buffers since it is unknown if conditional rendering is
+ * inverted when populating them.
+ */
+ uint32_t *dw = anv_batch_emitn(&cmd_buffer->batch, 5, GENX(MI_MATH));
+ dw[1] = mi_alu(MI_ALU_LOAD0, MI_ALU_SRCA, 0);
+ dw[2] = mi_alu(MI_ALU_LOAD, MI_ALU_SRCB, MI_ALU_REG0);
+ dw[3] = mi_alu(MI_ALU_SUB, 0, 0);
+ dw[4] = mi_alu(isInverted ? MI_ALU_STOREINV : MI_ALU_STORE,
+ ANV_PREDICATE_RESULT_REG, MI_ALU_CF);
+}
+
+void genX(CmdEndConditionalRenderingEXT)(
+ VkCommandBuffer commandBuffer)
+{
+ ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
+ struct anv_cmd_state *cmd_state = &cmd_buffer->state;
+
+ cmd_state->conditional_render_enabled = false;
+}
+#endif
projects / mesa.git / blobdiff