- struct si_resource *buffer = si_resource(buf);
- unsigned i, shader;
- unsigned num_elems = sctx->vertex_elements ?
- sctx->vertex_elements->count : 0;
-
- /* We changed the buffer, now we need to bind it where the old one
- * was bound. This consists of 2 things:
- * 1) Updating the resource descriptor and dirtying it.
- * 2) Adding a relocation to the CS, so that it's usable.
- */
-
- /* Vertex buffers. */
- if (!buffer) {
- if (num_elems)
- sctx->vertex_buffers_dirty = true;
- } else if (buffer->bind_history & PIPE_BIND_VERTEX_BUFFER) {
- for (i = 0; i < num_elems; i++) {
- int vb = sctx->vertex_elements->vertex_buffer_index[i];
-
- if (vb >= ARRAY_SIZE(sctx->vertex_buffer))
- continue;
- if (!sctx->vertex_buffer[vb].buffer.resource)
- continue;
-
- if (sctx->vertex_buffer[vb].buffer.resource == buf) {
- sctx->vertex_buffers_dirty = true;
- break;
- }
- }
- }
-
- /* Streamout buffers. (other internal buffers can't be invalidated) */
- if (!buffer || buffer->bind_history & PIPE_BIND_STREAM_OUTPUT) {
- for (i = SI_VS_STREAMOUT_BUF0; i <= SI_VS_STREAMOUT_BUF3; i++) {
- struct si_buffer_resources *buffers = &sctx->rw_buffers;
- struct si_descriptors *descs =
- &sctx->descriptors[SI_DESCS_RW_BUFFERS];
- struct pipe_resource *buffer = buffers->buffers[i];
-
- if (!buffer || (buf && buffer != buf))
- continue;
-
- si_set_buf_desc_address(si_resource(buffer), buffers->offsets[i],
- descs->list + i*4);
- sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
-
- radeon_add_to_gfx_buffer_list_check_mem(sctx,
- si_resource(buffer),
- RADEON_USAGE_WRITE,
- RADEON_PRIO_SHADER_RW_BUFFER,
- true);
-
- /* Update the streamout state. */
- if (sctx->streamout.begin_emitted)
- si_emit_streamout_end(sctx);
- sctx->streamout.append_bitmask =
- sctx->streamout.enabled_mask;
- si_streamout_buffers_dirty(sctx);
- }
- }
-
- /* Constant and shader buffers. */
- if (!buffer || buffer->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
- for (shader = 0; shader < SI_NUM_SHADERS; shader++)
- si_reset_buffer_resources(sctx, &sctx->const_and_shader_buffers[shader],
- si_const_and_shader_buffer_descriptors_idx(shader),
- u_bit_consecutive(SI_NUM_SHADER_BUFFERS, SI_NUM_CONST_BUFFERS),
- buf,
- sctx->const_and_shader_buffers[shader].priority_constbuf);
- }
-
- if (!buffer || buffer->bind_history & PIPE_BIND_SHADER_BUFFER) {
- for (shader = 0; shader < SI_NUM_SHADERS; shader++)
- si_reset_buffer_resources(sctx, &sctx->const_and_shader_buffers[shader],
- si_const_and_shader_buffer_descriptors_idx(shader),
- u_bit_consecutive(0, SI_NUM_SHADER_BUFFERS),
- buf,
- sctx->const_and_shader_buffers[shader].priority);
- }
-
- if (!buffer || buffer->bind_history & PIPE_BIND_SAMPLER_VIEW) {
- /* Texture buffers - update bindings. */
- for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
- struct si_samplers *samplers = &sctx->samplers[shader];
- struct si_descriptors *descs =
- si_sampler_and_image_descriptors(sctx, shader);
- unsigned mask = samplers->enabled_mask;
-
- while (mask) {
- unsigned i = u_bit_scan(&mask);
- struct pipe_resource *buffer = samplers->views[i]->texture;
-
- if (buffer && buffer->target == PIPE_BUFFER &&
- (!buf || buffer == buf)) {
- unsigned desc_slot = si_get_sampler_slot(i);
-
- si_set_buf_desc_address(si_resource(buffer),
- samplers->views[i]->u.buf.offset,
- descs->list + desc_slot * 16 + 4);
- sctx->descriptors_dirty |=
- 1u << si_sampler_and_image_descriptors_idx(shader);
-
- radeon_add_to_gfx_buffer_list_check_mem(
- sctx, si_resource(buffer),
- RADEON_USAGE_READ,
- RADEON_PRIO_SAMPLER_BUFFER, true);
- }
- }
- }
- }
-
- /* Shader images */
- if (!buffer || buffer->bind_history & PIPE_BIND_SHADER_IMAGE) {
- for (shader = 0; shader < SI_NUM_SHADERS; ++shader) {
- struct si_images *images = &sctx->images[shader];
- struct si_descriptors *descs =
- si_sampler_and_image_descriptors(sctx, shader);
- unsigned mask = images->enabled_mask;
-
- while (mask) {
- unsigned i = u_bit_scan(&mask);
- struct pipe_resource *buffer = images->views[i].resource;
-
- if (buffer && buffer->target == PIPE_BUFFER &&
- (!buf || buffer == buf)) {
- unsigned desc_slot = si_get_image_slot(i);
-
- if (images->views[i].access & PIPE_IMAGE_ACCESS_WRITE)
- si_mark_image_range_valid(&images->views[i]);
-
- si_set_buf_desc_address(si_resource(buffer),
- images->views[i].u.buf.offset,
- descs->list + desc_slot * 8 + 4);
- sctx->descriptors_dirty |=
- 1u << si_sampler_and_image_descriptors_idx(shader);
-
- radeon_add_to_gfx_buffer_list_check_mem(
- sctx, si_resource(buffer),
- RADEON_USAGE_READWRITE,
- RADEON_PRIO_SAMPLER_BUFFER, true);
- }
- }
- }
- }
-
- /* Bindless texture handles */
- if (!buffer || buffer->texture_handle_allocated) {
- struct si_descriptors *descs = &sctx->bindless_descriptors;
-
- util_dynarray_foreach(&sctx->resident_tex_handles,
- struct si_texture_handle *, tex_handle) {
- struct pipe_sampler_view *view = (*tex_handle)->view;
- unsigned desc_slot = (*tex_handle)->desc_slot;
- struct pipe_resource *buffer = view->texture;
-
- if (buffer && buffer->target == PIPE_BUFFER &&
- (!buf || buffer == buf)) {
- si_set_buf_desc_address(si_resource(buffer),
- view->u.buf.offset,
- descs->list +
- desc_slot * 16 + 4);
-
- (*tex_handle)->desc_dirty = true;
- sctx->bindless_descriptors_dirty = true;
-
- radeon_add_to_gfx_buffer_list_check_mem(
- sctx, si_resource(buffer),
- RADEON_USAGE_READ,
- RADEON_PRIO_SAMPLER_BUFFER, true);
- }
- }
- }
-
- /* Bindless image handles */
- if (!buffer || buffer->image_handle_allocated) {
- struct si_descriptors *descs = &sctx->bindless_descriptors;
-
- util_dynarray_foreach(&sctx->resident_img_handles,
- struct si_image_handle *, img_handle) {
- struct pipe_image_view *view = &(*img_handle)->view;
- unsigned desc_slot = (*img_handle)->desc_slot;
- struct pipe_resource *buffer = view->resource;
-
- if (buffer && buffer->target == PIPE_BUFFER &&
- (!buf || buffer == buf)) {
- if (view->access & PIPE_IMAGE_ACCESS_WRITE)
- si_mark_image_range_valid(view);
-
- si_set_buf_desc_address(si_resource(buffer),
- view->u.buf.offset,
- descs->list +
- desc_slot * 16 + 4);
-
- (*img_handle)->desc_dirty = true;
- sctx->bindless_descriptors_dirty = true;
-
- radeon_add_to_gfx_buffer_list_check_mem(
- sctx, si_resource(buffer),
- RADEON_USAGE_READWRITE,
- RADEON_PRIO_SAMPLER_BUFFER, true);
- }
- }
- }
-
- if (buffer) {
- /* Do the same for other contexts. They will invoke this function
- * with buffer == NULL.
- */
- unsigned new_counter = p_atomic_inc_return(&sctx->screen->dirty_buf_counter);
-
- /* Skip the update for the current context, because we have already updated
- * the buffer bindings.
- */
- if (new_counter == sctx->last_dirty_buf_counter + 1)
- sctx->last_dirty_buf_counter = new_counter;
- }
-}
-
-static void si_upload_bindless_descriptor(struct si_context *sctx,
- unsigned desc_slot,
- unsigned num_dwords)
-{
- struct si_descriptors *desc = &sctx->bindless_descriptors;
- unsigned desc_slot_offset = desc_slot * 16;
- uint32_t *data;
- uint64_t va;
-
- data = desc->list + desc_slot_offset;
- va = desc->gpu_address + desc_slot_offset * 4;
-
- si_cp_write_data(sctx, desc->buffer, va - desc->buffer->gpu_address,
- num_dwords * 4, V_370_TC_L2, V_370_ME, data);
+ struct si_resource *buffer = si_resource(buf);
+ unsigned i, shader;
+ unsigned num_elems = sctx->num_vertex_elements;
+
+ /* We changed the buffer, now we need to bind it where the old one
+ * was bound. This consists of 2 things:
+ * 1) Updating the resource descriptor and dirtying it.
+ * 2) Adding a relocation to the CS, so that it's usable.
+ */
+
+ /* Vertex buffers. */
+ if (!buffer) {
+ if (num_elems)
+ sctx->vertex_buffers_dirty = true;
+ } else if (buffer->bind_history & PIPE_BIND_VERTEX_BUFFER) {
+ for (i = 0; i < num_elems; i++) {
+ int vb = sctx->vertex_elements->vertex_buffer_index[i];
+
+ if (vb >= ARRAY_SIZE(sctx->vertex_buffer))
+ continue;
+ if (!sctx->vertex_buffer[vb].buffer.resource)
+ continue;
+
+ if (sctx->vertex_buffer[vb].buffer.resource == buf) {
+ sctx->vertex_buffers_dirty = true;
+ break;
+ }
+ }
+ }
+
+ /* Streamout buffers. (other internal buffers can't be invalidated) */
+ if (!buffer || buffer->bind_history & PIPE_BIND_STREAM_OUTPUT) {
+ for (i = SI_VS_STREAMOUT_BUF0; i <= SI_VS_STREAMOUT_BUF3; i++) {
+ struct si_buffer_resources *buffers = &sctx->rw_buffers;
+ struct si_descriptors *descs = &sctx->descriptors[SI_DESCS_RW_BUFFERS];
+ struct pipe_resource *buffer = buffers->buffers[i];
+
+ if (!buffer || (buf && buffer != buf))
+ continue;
+
+ si_set_buf_desc_address(si_resource(buffer), buffers->offsets[i], descs->list + i * 4);
+ sctx->descriptors_dirty |= 1u << SI_DESCS_RW_BUFFERS;
+
+ radeon_add_to_gfx_buffer_list_check_mem(sctx, si_resource(buffer), RADEON_USAGE_WRITE,
+ RADEON_PRIO_SHADER_RW_BUFFER, true);
+
+ /* Update the streamout state. */
+ if (sctx->streamout.begin_emitted)
+ si_emit_streamout_end(sctx);
+ sctx->streamout.append_bitmask = sctx->streamout.enabled_mask;
+ si_streamout_buffers_dirty(sctx);
+ }
+ }
+
+ /* Constant and shader buffers. */
+ if (!buffer || buffer->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
+ for (shader = 0; shader < SI_NUM_SHADERS; shader++)
+ si_reset_buffer_resources(sctx, &sctx->const_and_shader_buffers[shader],
+ si_const_and_shader_buffer_descriptors_idx(shader),
+ u_bit_consecutive64(SI_NUM_SHADER_BUFFERS, SI_NUM_CONST_BUFFERS),
+ buf, sctx->const_and_shader_buffers[shader].priority_constbuf);
+ }
+
+ if (!buffer || buffer->bind_history & PIPE_BIND_SHADER_BUFFER) {
+ for (shader = 0; shader < SI_NUM_SHADERS; shader++)
+ si_reset_buffer_resources(sctx, &sctx->const_and_shader_buffers[shader],
+ si_const_and_shader_buffer_descriptors_idx(shader),
+ u_bit_consecutive64(0, SI_NUM_SHADER_BUFFERS), buf,
+ sctx->const_and_shader_buffers[shader].priority);
+ }
+
+ if (!buffer || buffer->bind_history & PIPE_BIND_SAMPLER_VIEW) {
+ /* Texture buffers - update bindings. */
+ for (shader = 0; shader < SI_NUM_SHADERS; shader++) {
+ struct si_samplers *samplers = &sctx->samplers[shader];
+ struct si_descriptors *descs = si_sampler_and_image_descriptors(sctx, shader);
+ unsigned mask = samplers->enabled_mask;
+
+ while (mask) {
+ unsigned i = u_bit_scan(&mask);
+ struct pipe_resource *buffer = samplers->views[i]->texture;
+
+ if (buffer && buffer->target == PIPE_BUFFER && (!buf || buffer == buf)) {
+ unsigned desc_slot = si_get_sampler_slot(i);
+
+ si_set_buf_desc_address(si_resource(buffer), samplers->views[i]->u.buf.offset,
+ descs->list + desc_slot * 16 + 4);
+ sctx->descriptors_dirty |= 1u << si_sampler_and_image_descriptors_idx(shader);
+
+ radeon_add_to_gfx_buffer_list_check_mem(sctx, si_resource(buffer), RADEON_USAGE_READ,
+ RADEON_PRIO_SAMPLER_BUFFER, true);
+ }
+ }
+ }
+ }
+
+ /* Shader images */
+ if (!buffer || buffer->bind_history & PIPE_BIND_SHADER_IMAGE) {
+ for (shader = 0; shader < SI_NUM_SHADERS; ++shader) {
+ struct si_images *images = &sctx->images[shader];
+ struct si_descriptors *descs = si_sampler_and_image_descriptors(sctx, shader);
+ unsigned mask = images->enabled_mask;
+
+ while (mask) {
+ unsigned i = u_bit_scan(&mask);
+ struct pipe_resource *buffer = images->views[i].resource;
+
+ if (buffer && buffer->target == PIPE_BUFFER && (!buf || buffer == buf)) {
+ unsigned desc_slot = si_get_image_slot(i);
+
+ if (images->views[i].access & PIPE_IMAGE_ACCESS_WRITE)
+ si_mark_image_range_valid(&images->views[i]);
+
+ si_set_buf_desc_address(si_resource(buffer), images->views[i].u.buf.offset,
+ descs->list + desc_slot * 8 + 4);
+ sctx->descriptors_dirty |= 1u << si_sampler_and_image_descriptors_idx(shader);
+
+ radeon_add_to_gfx_buffer_list_check_mem(sctx, si_resource(buffer),
+ RADEON_USAGE_READWRITE,
+ RADEON_PRIO_SAMPLER_BUFFER, true);
+ }
+ }
+ }
+ }
+
+ /* Bindless texture handles */
+ if (!buffer || buffer->texture_handle_allocated) {
+ struct si_descriptors *descs = &sctx->bindless_descriptors;
+
+ util_dynarray_foreach (&sctx->resident_tex_handles, struct si_texture_handle *, tex_handle) {
+ struct pipe_sampler_view *view = (*tex_handle)->view;
+ unsigned desc_slot = (*tex_handle)->desc_slot;
+ struct pipe_resource *buffer = view->texture;
+
+ if (buffer && buffer->target == PIPE_BUFFER && (!buf || buffer == buf)) {
+ si_set_buf_desc_address(si_resource(buffer), view->u.buf.offset,
+ descs->list + desc_slot * 16 + 4);
+
+ (*tex_handle)->desc_dirty = true;
+ sctx->bindless_descriptors_dirty = true;
+
+ radeon_add_to_gfx_buffer_list_check_mem(sctx, si_resource(buffer), RADEON_USAGE_READ,
+ RADEON_PRIO_SAMPLER_BUFFER, true);
+ }
+ }
+ }
+
+ /* Bindless image handles */
+ if (!buffer || buffer->image_handle_allocated) {
+ struct si_descriptors *descs = &sctx->bindless_descriptors;
+
+ util_dynarray_foreach (&sctx->resident_img_handles, struct si_image_handle *, img_handle) {
+ struct pipe_image_view *view = &(*img_handle)->view;
+ unsigned desc_slot = (*img_handle)->desc_slot;
+ struct pipe_resource *buffer = view->resource;
+
+ if (buffer && buffer->target == PIPE_BUFFER && (!buf || buffer == buf)) {
+ if (view->access & PIPE_IMAGE_ACCESS_WRITE)
+ si_mark_image_range_valid(view);
+
+ si_set_buf_desc_address(si_resource(buffer), view->u.buf.offset,
+ descs->list + desc_slot * 16 + 4);
+
+ (*img_handle)->desc_dirty = true;
+ sctx->bindless_descriptors_dirty = true;
+
+ radeon_add_to_gfx_buffer_list_check_mem(
+ sctx, si_resource(buffer), RADEON_USAGE_READWRITE, RADEON_PRIO_SAMPLER_BUFFER, true);
+ }
+ }
+ }
+
+ if (buffer) {
+ /* Do the same for other contexts. They will invoke this function
+ * with buffer == NULL.
+ */
+ unsigned new_counter = p_atomic_inc_return(&sctx->screen->dirty_buf_counter);
+
+ /* Skip the update for the current context, because we have already updated
+ * the buffer bindings.
+ */
+ if (new_counter == sctx->last_dirty_buf_counter + 1)
+ sctx->last_dirty_buf_counter = new_counter;
+ }
+}
+
+static void si_upload_bindless_descriptor(struct si_context *sctx, unsigned desc_slot,
+ unsigned num_dwords)
+{
+ struct si_descriptors *desc = &sctx->bindless_descriptors;
+ unsigned desc_slot_offset = desc_slot * 16;
+ uint32_t *data;
+ uint64_t va;
+
+ data = desc->list + desc_slot_offset;
+ va = desc->gpu_address + desc_slot_offset * 4;
+
+ si_cp_write_data(sctx, desc->buffer, va - desc->buffer->gpu_address, num_dwords * 4, V_370_TC_L2,
+ V_370_ME, data);