+/* BINDLESS */
+
+struct si_bindless_descriptor_slab
+{
+ struct pb_slab base;
+ struct r600_resource *buffer;
+ struct si_bindless_descriptor *entries;
+};
+
+bool si_bindless_descriptor_can_reclaim_slab(void *priv,
+ struct pb_slab_entry *entry)
+{
+ /* Do not allow to reclaim any bindless descriptors for now because the
+ * GPU might be using them. This should be improved later on.
+ */
+ return false;
+}
+
+struct pb_slab *si_bindless_descriptor_slab_alloc(void *priv, unsigned heap,
+ unsigned entry_size,
+ unsigned group_index)
+{
+ struct si_context *sctx = priv;
+ struct si_screen *sscreen = sctx->screen;
+ struct si_bindless_descriptor_slab *slab;
+
+ slab = CALLOC_STRUCT(si_bindless_descriptor_slab);
+ if (!slab)
+ return NULL;
+
+ /* Create a buffer in VRAM for 1024 bindless descriptors. */
+ slab->buffer = (struct r600_resource *)
+ pipe_buffer_create(&sscreen->b.b, 0,
+ PIPE_USAGE_DEFAULT, 64 * 1024);
+ if (!slab->buffer)
+ goto fail;
+
+ slab->base.num_entries = slab->buffer->bo_size / entry_size;
+ slab->base.num_free = slab->base.num_entries;
+ slab->entries = CALLOC(slab->base.num_entries, sizeof(*slab->entries));
+ if (!slab->entries)
+ goto fail_buffer;
+
+ LIST_INITHEAD(&slab->base.free);
+
+ for (unsigned i = 0; i < slab->base.num_entries; ++i) {
+ struct si_bindless_descriptor *desc = &slab->entries[i];
+
+ desc->entry.slab = &slab->base;
+ desc->entry.group_index = group_index;
+ desc->buffer = slab->buffer;
+ desc->offset = i * entry_size;
+
+ LIST_ADDTAIL(&desc->entry.head, &slab->base.free);
+ }
+
+ /* Add the descriptor to the per-context list. */
+ util_dynarray_append(&sctx->bindless_descriptors,
+ struct r600_resource *, slab->buffer);
+
+ return &slab->base;
+
+fail_buffer:
+ r600_resource_reference(&slab->buffer, NULL);
+fail:
+ FREE(slab);
+ return NULL;
+}
+
+void si_bindless_descriptor_slab_free(void *priv, struct pb_slab *pslab)
+{
+ struct si_context *sctx = priv;
+ struct si_bindless_descriptor_slab *slab =
+ (struct si_bindless_descriptor_slab *)pslab;
+
+ /* Remove the descriptor from the per-context list. */
+ util_dynarray_delete_unordered(&sctx->bindless_descriptors,
+ struct r600_resource *, slab->buffer);
+
+ r600_resource_reference(&slab->buffer, NULL);
+ FREE(slab->entries);
+ FREE(slab);
+}
+
+static struct si_bindless_descriptor *
+si_create_bindless_descriptor(struct si_context *sctx, uint32_t *desc_list,
+ unsigned size)
+{
+ struct si_screen *sscreen = sctx->screen;
+ struct si_bindless_descriptor *desc;
+ struct pb_slab_entry *entry;
+ void *ptr;
+
+ /* Sub-allocate the bindless descriptor from a slab to avoid dealing
+ * with a ton of buffers and for reducing the winsys overhead.
+ */
+ entry = pb_slab_alloc(&sctx->bindless_descriptor_slabs, 64, 0);
+ if (!entry)
+ return NULL;
+
+ desc = NULL;
+ desc = container_of(entry, desc, entry);
+
+ /* Upload the descriptor directly in VRAM. Because the slabs are
+ * currently never reclaimed, we don't need to synchronize the
+ * operation.
+ */
+ ptr = sscreen->b.ws->buffer_map(desc->buffer->buf, NULL,
+ PIPE_TRANSFER_WRITE |
+ PIPE_TRANSFER_UNSYNCHRONIZED);
+ util_memcpy_cpu_to_le32(ptr + desc->offset, desc_list, size);
+
+ /* Keep track of the initial descriptor especially for buffers
+ * invalidation because we might need to know the previous address.
+ */
+ memcpy(desc->desc_list, desc_list, sizeof(desc->desc_list));
+
+ return desc;
+}
+
+static void si_invalidate_bindless_buf_desc(struct si_context *sctx,
+ struct si_bindless_descriptor *desc,
+ struct pipe_resource *resource,
+ uint64_t offset)
+{
+ struct r600_resource *buf = r600_resource(resource);
+ uint32_t *desc_list = desc->desc_list;
+ uint64_t old_desc_va;
+
+ assert(resource->target == PIPE_BUFFER);
+
+ /* Retrieve the old buffer addr from the descriptor. */
+ old_desc_va = desc_list[0];
+ old_desc_va |= ((uint64_t)G_008F04_BASE_ADDRESS_HI(desc_list[1]) << 32);
+
+ if (old_desc_va != buf->gpu_address + offset) {
+ /* The buffer has been invalidated when the handle wasn't
+ * resident, update the descriptor and the dirty flag.
+ */
+ si_set_buf_desc_address(buf, offset, &desc_list[4]);
+
+ desc->dirty = true;
+ sctx->bindless_descriptors_dirty = true;
+ }
+}
+
+static uint64_t si_create_texture_handle(struct pipe_context *ctx,
+ struct pipe_sampler_view *view,
+ const struct pipe_sampler_state *state)
+{
+ struct si_sampler_view *sview = (struct si_sampler_view *)view;
+ struct si_context *sctx = (struct si_context *)ctx;
+ struct si_texture_handle *tex_handle;
+ struct si_sampler_state *sstate;
+ uint32_t desc_list[16];
+ uint64_t handle;
+
+ tex_handle = CALLOC_STRUCT(si_texture_handle);
+ if (!tex_handle)
+ return 0;
+
+ memset(desc_list, 0, sizeof(desc_list));
+ si_init_descriptor_list(&desc_list[0], 16, 1, null_texture_descriptor);
+
+ sstate = ctx->create_sampler_state(ctx, state);
+ if (!sstate) {
+ FREE(tex_handle);
+ return 0;
+ }
+
+ si_set_sampler_view_desc(sctx, sview, sstate, &desc_list[0]);
+ ctx->delete_sampler_state(ctx, sstate);
+
+ tex_handle->desc = si_create_bindless_descriptor(sctx, desc_list,
+ sizeof(desc_list));
+ if (!tex_handle->desc) {
+ FREE(tex_handle);
+ return 0;
+ }
+
+ handle = tex_handle->desc->buffer->gpu_address +
+ tex_handle->desc->offset;
+
+ if (!_mesa_hash_table_insert(sctx->tex_handles, (void *)handle,
+ tex_handle)) {
+ pb_slab_free(&sctx->bindless_descriptor_slabs,
+ &tex_handle->desc->entry);
+ FREE(tex_handle);
+ return 0;
+ }
+
+ pipe_sampler_view_reference(&tex_handle->view, view);
+
+ r600_resource(sview->base.texture)->texture_handle_allocated = true;
+
+ return handle;
+}
+
+static void si_delete_texture_handle(struct pipe_context *ctx, uint64_t handle)
+{
+ struct si_context *sctx = (struct si_context *)ctx;
+ struct si_texture_handle *tex_handle;
+ struct hash_entry *entry;
+
+ entry = _mesa_hash_table_search(sctx->tex_handles, (void *)handle);
+ if (!entry)
+ return;
+
+ tex_handle = (struct si_texture_handle *)entry->data;
+
+ pipe_sampler_view_reference(&tex_handle->view, NULL);
+ _mesa_hash_table_remove(sctx->tex_handles, entry);
+ pb_slab_free(&sctx->bindless_descriptor_slabs,
+ &tex_handle->desc->entry);
+ FREE(tex_handle);
+}
+
+static void si_make_texture_handle_resident(struct pipe_context *ctx,
+ uint64_t handle, bool resident)
+{
+ struct si_context *sctx = (struct si_context *)ctx;
+ struct si_texture_handle *tex_handle;
+ struct si_sampler_view *sview;
+ struct hash_entry *entry;
+
+ entry = _mesa_hash_table_search(sctx->tex_handles, (void *)handle);
+ if (!entry)
+ return;
+
+ tex_handle = (struct si_texture_handle *)entry->data;
+ sview = (struct si_sampler_view *)tex_handle->view;
+
+ if (resident) {
+ if (sview->base.texture->target != PIPE_BUFFER) {
+ struct r600_texture *rtex =
+ (struct r600_texture *)sview->base.texture;
+
+ if (depth_needs_decompression(rtex, sview)) {
+ util_dynarray_append(
+ &sctx->resident_tex_needs_depth_decompress,
+ struct si_texture_handle *,
+ tex_handle);
+ }
+
+ tex_handle->needs_color_decompress =
+ color_needs_decompression(rtex);
+
+ if (rtex->dcc_offset &&
+ p_atomic_read(&rtex->framebuffers_bound))
+ sctx->need_check_render_feedback = true;
+ } else {
+ si_invalidate_bindless_buf_desc(sctx, tex_handle->desc,
+ sview->base.texture,
+ sview->base.u.buf.offset);
+ }
+
+ /* Add the texture handle to the per-context list. */
+ util_dynarray_append(&sctx->resident_tex_handles,
+ struct si_texture_handle *, tex_handle);
+
+ /* Add the buffers to the current CS in case si_begin_new_cs()
+ * is not going to be called.
+ */
+ radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
+ tex_handle->desc->buffer,
+ RADEON_USAGE_READWRITE,
+ RADEON_PRIO_DESCRIPTORS);
+
+ si_sampler_view_add_buffer(sctx, sview->base.texture,
+ RADEON_USAGE_READ,
+ sview->is_stencil_sampler, false);
+ } else {
+ /* Remove the texture handle from the per-context list. */
+ util_dynarray_delete_unordered(&sctx->resident_tex_handles,
+ struct si_texture_handle *,
+ tex_handle);
+
+ if (sview->base.texture->target != PIPE_BUFFER) {
+ util_dynarray_delete_unordered(
+ &sctx->resident_tex_needs_depth_decompress,
+ struct si_texture_handle *, tex_handle);
+ }
+ }
+}
+
+static uint64_t si_create_image_handle(struct pipe_context *ctx,
+ const struct pipe_image_view *view)
+{
+ struct si_context *sctx = (struct si_context *)ctx;
+ struct si_image_handle *img_handle;
+ uint32_t desc_list[16];
+ uint64_t handle;
+
+ if (!view || !view->resource)
+ return 0;
+
+ img_handle = CALLOC_STRUCT(si_image_handle);
+ if (!img_handle)
+ return 0;
+
+ memset(desc_list, 0, sizeof(desc_list));
+ si_init_descriptor_list(&desc_list[0], 8, 1, null_image_descriptor);
+
+ si_set_shader_image_desc(sctx, view, false, &desc_list[0]);
+
+ img_handle->desc = si_create_bindless_descriptor(sctx, desc_list,
+ sizeof(desc_list));
+ if (!img_handle->desc) {
+ FREE(img_handle);
+ return 0;
+ }
+
+ handle = img_handle->desc->buffer->gpu_address +
+ img_handle->desc->offset;
+
+ if (!_mesa_hash_table_insert(sctx->img_handles, (void *)handle,
+ img_handle)) {
+ pb_slab_free(&sctx->bindless_descriptor_slabs,
+ &img_handle->desc->entry);
+ FREE(img_handle);
+ return 0;
+ }
+
+ util_copy_image_view(&img_handle->view, view);
+
+ r600_resource(view->resource)->image_handle_allocated = true;
+
+ return handle;
+}
+
+static void si_delete_image_handle(struct pipe_context *ctx, uint64_t handle)
+{
+ struct si_context *sctx = (struct si_context *)ctx;
+ struct si_image_handle *img_handle;
+ struct hash_entry *entry;
+
+ entry = _mesa_hash_table_search(sctx->img_handles, (void *)handle);
+ if (!entry)
+ return;
+
+ img_handle = (struct si_image_handle *)entry->data;
+
+ util_copy_image_view(&img_handle->view, NULL);
+ _mesa_hash_table_remove(sctx->img_handles, entry);
+ pb_slab_free(&sctx->bindless_descriptor_slabs,
+ &img_handle->desc->entry);
+ FREE(img_handle);
+}
+
+static void si_make_image_handle_resident(struct pipe_context *ctx,
+ uint64_t handle, unsigned access,
+ bool resident)
+{
+ struct si_context *sctx = (struct si_context *)ctx;
+ struct si_image_handle *img_handle;
+ struct pipe_image_view *view;
+ struct hash_entry *entry;
+
+ entry = _mesa_hash_table_search(sctx->img_handles, (void *)handle);
+ if (!entry)
+ return;
+
+ img_handle = (struct si_image_handle *)entry->data;
+ view = &img_handle->view;
+
+ if (resident) {
+ struct r600_resource *res =
+ (struct r600_resource *)view->resource;
+
+ if (res->b.b.target != PIPE_BUFFER) {
+ struct r600_texture *rtex = (struct r600_texture *)res;
+ unsigned level = view->u.tex.level;
+
+ img_handle->needs_color_decompress =
+ color_needs_decompression(rtex);
+
+ if (vi_dcc_enabled(rtex, level) &&
+ p_atomic_read(&rtex->framebuffers_bound))
+ sctx->need_check_render_feedback = true;
+ } else {
+ si_invalidate_bindless_buf_desc(sctx, img_handle->desc,
+ view->resource,
+ view->u.buf.offset);
+ }
+
+ /* Add the image handle to the per-context list. */
+ util_dynarray_append(&sctx->resident_img_handles,
+ struct si_image_handle *, img_handle);
+
+ /* Add the buffers to the current CS in case si_begin_new_cs()
+ * is not going to be called.
+ */
+ radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx,
+ img_handle->desc->buffer,
+ RADEON_USAGE_READWRITE,
+ RADEON_PRIO_DESCRIPTORS);
+
+ si_sampler_view_add_buffer(sctx, view->resource,
+ (access & PIPE_IMAGE_ACCESS_WRITE) ?
+ RADEON_USAGE_READWRITE :
+ RADEON_USAGE_READ, false, false);
+ } else {
+ /* Remove the image handle from the per-context list. */
+ util_dynarray_delete_unordered(&sctx->resident_img_handles,
+ struct si_image_handle *,
+ img_handle);
+ }
+}
+
+
+void si_all_resident_buffers_begin_new_cs(struct si_context *sctx)
+{
+ unsigned num_resident_tex_handles, num_resident_img_handles;
+
+ num_resident_tex_handles = sctx->resident_tex_handles.size /
+ sizeof(struct si_texture_handle *);
+ num_resident_img_handles = sctx->resident_img_handles.size /
+ sizeof(struct si_image_handle *);
+
+ /* Skip adding the bindless descriptors when no handles are resident.
+ */
+ if (!num_resident_tex_handles && !num_resident_img_handles)
+ return;
+
+ /* Add all bindless descriptors. */
+ util_dynarray_foreach(&sctx->bindless_descriptors,
+ struct r600_resource *, desc) {
+
+ radeon_add_to_buffer_list(&sctx->b, &sctx->b.gfx, *desc,
+ RADEON_USAGE_READWRITE,
+ RADEON_PRIO_DESCRIPTORS);
+ }
+
+ /* Add all resident texture handles. */
+ util_dynarray_foreach(&sctx->resident_tex_handles,
+ struct si_texture_handle *, tex_handle) {
+ struct si_sampler_view *sview =
+ (struct si_sampler_view *)(*tex_handle)->view;
+
+ si_sampler_view_add_buffer(sctx, sview->base.texture,
+ RADEON_USAGE_READ,
+ sview->is_stencil_sampler, false);
+ }
+
+ /* Add all resident image handles. */
+ util_dynarray_foreach(&sctx->resident_img_handles,
+ struct si_image_handle *, img_handle) {
+ struct pipe_image_view *view = &(*img_handle)->view;
+
+ si_sampler_view_add_buffer(sctx, view->resource,
+ RADEON_USAGE_READWRITE,
+ false, false);
+ }
+
+ sctx->b.num_resident_handles += num_resident_tex_handles +
+ num_resident_img_handles;
+}
+