+static void si_init_bindless_descriptors(struct si_context *sctx,
+ struct si_descriptors *desc,
+ short shader_userdata_rel_index,
+ unsigned num_elements)
+{
+ MAYBE_UNUSED unsigned desc_slot;
+
+ si_init_descriptors(desc, shader_userdata_rel_index, 16, num_elements);
+ sctx->bindless_descriptors.num_active_slots = num_elements;
+
+ /* The first bindless descriptor is stored at slot 1, because 0 is not
+ * considered to be a valid handle.
+ */
+ sctx->num_bindless_descriptors = 1;
+
+ /* Track which bindless slots are used (or not). */
+ util_idalloc_init(&sctx->bindless_used_slots);
+ util_idalloc_resize(&sctx->bindless_used_slots, num_elements);
+
+ /* Reserve slot 0 because it's an invalid handle for bindless. */
+ desc_slot = util_idalloc_alloc(&sctx->bindless_used_slots);
+ assert(desc_slot == 0);
+}
+
+static void si_release_bindless_descriptors(struct si_context *sctx)
+{
+ si_release_descriptors(&sctx->bindless_descriptors);
+ util_idalloc_fini(&sctx->bindless_used_slots);
+}
+
+static unsigned si_get_first_free_bindless_slot(struct si_context *sctx)
+{
+ struct si_descriptors *desc = &sctx->bindless_descriptors;
+ unsigned desc_slot;
+
+ desc_slot = util_idalloc_alloc(&sctx->bindless_used_slots);
+ if (desc_slot >= desc->num_elements) {
+ /* The array of bindless descriptors is full, resize it. */
+ unsigned slot_size = desc->element_dw_size * 4;
+ unsigned new_num_elements = desc->num_elements * 2;
+
+ desc->list = REALLOC(desc->list, desc->num_elements * slot_size,
+ new_num_elements * slot_size);
+ desc->num_elements = new_num_elements;
+ desc->num_active_slots = new_num_elements;
+ }
+
+ assert(desc_slot);
+ return desc_slot;
+}
+
+static unsigned
+si_create_bindless_descriptor(struct si_context *sctx, uint32_t *desc_list,
+ unsigned size)
+{
+ struct si_descriptors *desc = &sctx->bindless_descriptors;
+ unsigned desc_slot, desc_slot_offset;
+
+ /* Find a free slot. */
+ desc_slot = si_get_first_free_bindless_slot(sctx);
+
+ /* For simplicity, sampler and image bindless descriptors use fixed
+ * 16-dword slots for now. Image descriptors only need 8-dword but this
+ * doesn't really matter because no real apps use image handles.
+ */
+ desc_slot_offset = desc_slot * 16;
+
+ /* Copy the descriptor into the array. */
+ memcpy(desc->list + desc_slot_offset, desc_list, size);
+
+ /* Re-upload the whole array of bindless descriptors into a new buffer.
+ */
+ if (!si_upload_descriptors(sctx, desc))
+ return 0;
+
+ /* Make sure to re-emit the shader pointers for all stages. */
+ sctx->graphics_bindless_pointer_dirty = true;
+ sctx->compute_bindless_pointer_dirty = true;
+
+ return desc_slot;
+}
+
+static void si_update_bindless_buffer_descriptor(struct si_context *sctx,
+ unsigned desc_slot,
+ struct pipe_resource *resource,
+ uint64_t offset,
+ bool *desc_dirty)
+{
+ struct si_descriptors *desc = &sctx->bindless_descriptors;
+ struct r600_resource *buf = r600_resource(resource);
+ unsigned desc_slot_offset = desc_slot * 16;
+ uint32_t *desc_list = desc->list + desc_slot_offset + 4;
+ uint64_t old_desc_va;
+
+ assert(resource->target == PIPE_BUFFER);
+
+ /* Retrieve the old buffer addr from the descriptor. */
+ old_desc_va = si_desc_extract_buffer_address(desc_list);
+
+ if (old_desc_va != buf->gpu_address + offset) {
+ /* The buffer has been invalidated when the handle wasn't
+ * resident, update the descriptor and the dirty flag.
+ */
+ si_set_buf_desc_address(buf, offset, &desc_list[0]);
+
+ *desc_dirty = true;
+ }
+}
+
+static uint64_t si_create_texture_handle(struct pipe_context *ctx,
+ struct pipe_sampler_view *view,
+ const struct pipe_sampler_state *state)
+{
+ struct si_sampler_view *sview = (struct si_sampler_view *)view;
+ struct si_context *sctx = (struct si_context *)ctx;
+ struct si_texture_handle *tex_handle;
+ struct si_sampler_state *sstate;
+ uint32_t desc_list[16];
+ uint64_t handle;
+
+ tex_handle = CALLOC_STRUCT(si_texture_handle);
+ if (!tex_handle)
+ return 0;
+
+ memset(desc_list, 0, sizeof(desc_list));
+ si_init_descriptor_list(&desc_list[0], 16, 1, null_texture_descriptor);
+
+ sstate = ctx->create_sampler_state(ctx, state);
+ if (!sstate) {
+ FREE(tex_handle);
+ return 0;
+ }
+
+ si_set_sampler_view_desc(sctx, sview, sstate, &desc_list[0]);
+ memcpy(&tex_handle->sstate, sstate, sizeof(*sstate));
+ ctx->delete_sampler_state(ctx, sstate);
+
+ tex_handle->desc_slot = si_create_bindless_descriptor(sctx, desc_list,
+ sizeof(desc_list));
+ if (!tex_handle->desc_slot) {
+ FREE(tex_handle);
+ return 0;
+ }
+
+ handle = tex_handle->desc_slot;
+
+ if (!_mesa_hash_table_insert(sctx->tex_handles,
+ (void *)(uintptr_t)handle,
+ tex_handle)) {
+ FREE(tex_handle);
+ return 0;
+ }
+
+ pipe_sampler_view_reference(&tex_handle->view, view);
+
+ r600_resource(sview->base.texture)->texture_handle_allocated = true;
+
+ return handle;
+}
+
+static void si_delete_texture_handle(struct pipe_context *ctx, uint64_t handle)
+{
+ struct si_context *sctx = (struct si_context *)ctx;
+ struct si_texture_handle *tex_handle;
+ struct hash_entry *entry;
+
+ entry = _mesa_hash_table_search(sctx->tex_handles,
+ (void *)(uintptr_t)handle);
+ if (!entry)
+ return;
+
+ tex_handle = (struct si_texture_handle *)entry->data;
+
+ /* Allow this descriptor slot to be re-used. */
+ util_idalloc_free(&sctx->bindless_used_slots, tex_handle->desc_slot);
+
+ pipe_sampler_view_reference(&tex_handle->view, NULL);
+ _mesa_hash_table_remove(sctx->tex_handles, entry);
+ FREE(tex_handle);
+}
+
+static void si_make_texture_handle_resident(struct pipe_context *ctx,
+ uint64_t handle, bool resident)
+{
+ struct si_context *sctx = (struct si_context *)ctx;
+ struct si_texture_handle *tex_handle;
+ struct si_sampler_view *sview;
+ struct hash_entry *entry;
+
+ entry = _mesa_hash_table_search(sctx->tex_handles,
+ (void *)(uintptr_t)handle);
+ if (!entry)
+ return;
+
+ tex_handle = (struct si_texture_handle *)entry->data;
+ sview = (struct si_sampler_view *)tex_handle->view;
+
+ if (resident) {
+ if (sview->base.texture->target != PIPE_BUFFER) {
+ struct r600_texture *rtex =
+ (struct r600_texture *)sview->base.texture;
+
+ if (depth_needs_decompression(rtex)) {
+ util_dynarray_append(
+ &sctx->resident_tex_needs_depth_decompress,
+ struct si_texture_handle *,
+ tex_handle);
+ }
+
+ if (color_needs_decompression(rtex)) {
+ util_dynarray_append(
+ &sctx->resident_tex_needs_color_decompress,
+ struct si_texture_handle *,
+ tex_handle);
+ }
+
+ if (rtex->dcc_offset &&
+ p_atomic_read(&rtex->framebuffers_bound))
+ sctx->need_check_render_feedback = true;
+
+ si_update_bindless_texture_descriptor(sctx, tex_handle);
+ } else {
+ si_update_bindless_buffer_descriptor(sctx,
+ tex_handle->desc_slot,
+ sview->base.texture,
+ sview->base.u.buf.offset,
+ &tex_handle->desc_dirty);
+ }
+
+ /* Re-upload the descriptor if it has been updated while it
+ * wasn't resident.
+ */
+ if (tex_handle->desc_dirty)
+ sctx->bindless_descriptors_dirty = true;
+
+ /* Add the texture handle to the per-context list. */
+ util_dynarray_append(&sctx->resident_tex_handles,
+ struct si_texture_handle *, tex_handle);
+
+ /* Add the buffers to the current CS in case si_begin_new_cs()
+ * is not going to be called.
+ */
+ si_sampler_view_add_buffer(sctx, sview->base.texture,
+ RADEON_USAGE_READ,
+ sview->is_stencil_sampler, false);
+ } else {
+ /* Remove the texture handle from the per-context list. */
+ util_dynarray_delete_unordered(&sctx->resident_tex_handles,
+ struct si_texture_handle *,
+ tex_handle);
+
+ if (sview->base.texture->target != PIPE_BUFFER) {
+ util_dynarray_delete_unordered(
+ &sctx->resident_tex_needs_depth_decompress,
+ struct si_texture_handle *, tex_handle);
+
+ util_dynarray_delete_unordered(
+ &sctx->resident_tex_needs_color_decompress,
+ struct si_texture_handle *, tex_handle);
+ }
+ }
+}
+
+static uint64_t si_create_image_handle(struct pipe_context *ctx,
+ const struct pipe_image_view *view)
+{
+ struct si_context *sctx = (struct si_context *)ctx;
+ struct si_image_handle *img_handle;
+ uint32_t desc_list[8];
+ uint64_t handle;
+
+ if (!view || !view->resource)
+ return 0;
+
+ img_handle = CALLOC_STRUCT(si_image_handle);
+ if (!img_handle)
+ return 0;
+
+ memset(desc_list, 0, sizeof(desc_list));
+ si_init_descriptor_list(&desc_list[0], 8, 1, null_image_descriptor);
+
+ si_set_shader_image_desc(sctx, view, false, &desc_list[0], NULL);
+
+ img_handle->desc_slot = si_create_bindless_descriptor(sctx, desc_list,
+ sizeof(desc_list));
+ if (!img_handle->desc_slot) {
+ FREE(img_handle);
+ return 0;
+ }
+
+ handle = img_handle->desc_slot;
+
+ if (!_mesa_hash_table_insert(sctx->img_handles,
+ (void *)(uintptr_t)handle,
+ img_handle)) {
+ FREE(img_handle);
+ return 0;
+ }
+
+ util_copy_image_view(&img_handle->view, view);
+
+ r600_resource(view->resource)->image_handle_allocated = true;
+
+ return handle;
+}
+
+static void si_delete_image_handle(struct pipe_context *ctx, uint64_t handle)
+{
+ struct si_context *sctx = (struct si_context *)ctx;
+ struct si_image_handle *img_handle;
+ struct hash_entry *entry;
+
+ entry = _mesa_hash_table_search(sctx->img_handles,
+ (void *)(uintptr_t)handle);
+ if (!entry)
+ return;
+
+ img_handle = (struct si_image_handle *)entry->data;
+
+ util_copy_image_view(&img_handle->view, NULL);
+ _mesa_hash_table_remove(sctx->img_handles, entry);
+ FREE(img_handle);
+}
+
+static void si_make_image_handle_resident(struct pipe_context *ctx,
+ uint64_t handle, unsigned access,
+ bool resident)
+{
+ struct si_context *sctx = (struct si_context *)ctx;
+ struct si_image_handle *img_handle;
+ struct pipe_image_view *view;
+ struct r600_resource *res;
+ struct hash_entry *entry;
+
+ entry = _mesa_hash_table_search(sctx->img_handles,
+ (void *)(uintptr_t)handle);
+ if (!entry)
+ return;
+
+ img_handle = (struct si_image_handle *)entry->data;
+ view = &img_handle->view;
+ res = (struct r600_resource *)view->resource;
+
+ if (resident) {
+ if (res->b.b.target != PIPE_BUFFER) {
+ struct r600_texture *rtex = (struct r600_texture *)res;
+ unsigned level = view->u.tex.level;
+
+ if (color_needs_decompression(rtex)) {
+ util_dynarray_append(
+ &sctx->resident_img_needs_color_decompress,
+ struct si_image_handle *,
+ img_handle);
+ }
+
+ if (vi_dcc_enabled(rtex, level) &&
+ p_atomic_read(&rtex->framebuffers_bound))
+ sctx->need_check_render_feedback = true;
+
+ si_update_bindless_image_descriptor(sctx, img_handle);
+ } else {
+ si_update_bindless_buffer_descriptor(sctx,
+ img_handle->desc_slot,
+ view->resource,
+ view->u.buf.offset,
+ &img_handle->desc_dirty);
+ }
+
+ /* Re-upload the descriptor if it has been updated while it
+ * wasn't resident.
+ */
+ if (img_handle->desc_dirty)
+ sctx->bindless_descriptors_dirty = true;
+
+ /* Add the image handle to the per-context list. */
+ util_dynarray_append(&sctx->resident_img_handles,
+ struct si_image_handle *, img_handle);
+
+ /* Add the buffers to the current CS in case si_begin_new_cs()
+ * is not going to be called.
+ */
+ si_sampler_view_add_buffer(sctx, view->resource,
+ (access & PIPE_IMAGE_ACCESS_WRITE) ?
+ RADEON_USAGE_READWRITE :
+ RADEON_USAGE_READ, false, false);
+ } else {
+ /* Remove the image handle from the per-context list. */
+ util_dynarray_delete_unordered(&sctx->resident_img_handles,
+ struct si_image_handle *,
+ img_handle);
+
+ if (res->b.b.target != PIPE_BUFFER) {
+ util_dynarray_delete_unordered(
+ &sctx->resident_img_needs_color_decompress,
+ struct si_image_handle *,
+ img_handle);
+ }
+ }
+}
+
+
+void si_all_resident_buffers_begin_new_cs(struct si_context *sctx)
+{
+ unsigned num_resident_tex_handles, num_resident_img_handles;
+
+ num_resident_tex_handles = sctx->resident_tex_handles.size /
+ sizeof(struct si_texture_handle *);
+ num_resident_img_handles = sctx->resident_img_handles.size /
+ sizeof(struct si_image_handle *);
+
+ /* Add all resident texture handles. */
+ util_dynarray_foreach(&sctx->resident_tex_handles,
+ struct si_texture_handle *, tex_handle) {
+ struct si_sampler_view *sview =
+ (struct si_sampler_view *)(*tex_handle)->view;
+
+ si_sampler_view_add_buffer(sctx, sview->base.texture,
+ RADEON_USAGE_READ,
+ sview->is_stencil_sampler, false);
+ }
+
+ /* Add all resident image handles. */
+ util_dynarray_foreach(&sctx->resident_img_handles,
+ struct si_image_handle *, img_handle) {
+ struct pipe_image_view *view = &(*img_handle)->view;
+
+ si_sampler_view_add_buffer(sctx, view->resource,
+ RADEON_USAGE_READWRITE,
+ false, false);
+ }
+
+ sctx->b.num_resident_handles += num_resident_tex_handles +
+ num_resident_img_handles;
+}
+
+/* INIT/DEINIT/UPLOAD */