From 8eb1214755366fc34ed15a7e3dec48d4f0d65f10 Mon Sep 17 00:00:00 2001 From: Ilia Mirkin Date: Wed, 21 Jun 2017 23:13:20 -0400 Subject: [PATCH] nvc0: add support for bindless textures on kepler+ This keeps a list of resident textures (per context), and dumps that list into the active buffer list when submitting. We also treat bindless texture fetches slightly differently, wrt the meaning of indirect, and not requiring the SAMPLER file to be used. Signed-off-by: Ilia Mirkin --- src/gallium/drivers/nouveau/codegen/nv50_ir.h | 1 + .../nouveau/codegen/nv50_ir_from_tgsi.cpp | 10 ++ .../nouveau/codegen/nv50_ir_lowering_nvc0.cpp | 10 +- .../drivers/nouveau/nv50/nv50_stateobj_tex.h | 1 + .../drivers/nouveau/nvc0/nvc0_context.c | 15 +++ .../drivers/nouveau/nvc0/nvc0_context.h | 17 ++- .../drivers/nouveau/nvc0/nvc0_screen.h | 2 + src/gallium/drivers/nouveau/nvc0/nvc0_tex.c | 120 ++++++++++++++++++ src/gallium/drivers/nouveau/nvc0/nvc0_vbo.c | 6 + .../drivers/nouveau/nvc0/nve4_compute.c | 6 + 10 files changed, 183 insertions(+), 5 deletions(-) diff --git a/src/gallium/drivers/nouveau/codegen/nv50_ir.h b/src/gallium/drivers/nouveau/codegen/nv50_ir.h index bc15992df0e..f4f3c708886 100644 --- a/src/gallium/drivers/nouveau/codegen/nv50_ir.h +++ b/src/gallium/drivers/nouveau/codegen/nv50_ir.h @@ -1024,6 +1024,7 @@ public: bool liveOnly; // only execute on live pixels of a quad (optimization) bool levelZero; bool derivAll; + bool bindless; int8_t useOffsets; // 0, 1, or 4 for textureGatherOffsets int8_t offset[3]; // only used on nv50 diff --git a/src/gallium/drivers/nouveau/codegen/nv50_ir_from_tgsi.cpp b/src/gallium/drivers/nouveau/codegen/nv50_ir_from_tgsi.cpp index 469fcff41d4..6c615806b83 100644 --- a/src/gallium/drivers/nouveau/codegen/nv50_ir_from_tgsi.cpp +++ b/src/gallium/drivers/nouveau/codegen/nv50_ir_from_tgsi.cpp @@ -2206,6 +2206,16 @@ Converter::setTexRS(TexInstruction *tex, unsigned int& s, int R, int S) { unsigned rIdx = 0, sIdx = 0; + if (R >= 0 && tgsi.getSrc(R).getFile() != TGSI_FILE_SAMPLER) { + // This is the bindless case. We have to get the actual value and pass + // it in. This will be the complete handle. + tex->tex.rIndirectSrc = s; + tex->setSrc(s++, fetchSrc(R, 0)); + tex->setTexture(tgsi.getTexture(code, R), 0xff, 0x1f); + tex->tex.bindless = true; + return; + } + if (R >= 0) rIdx = tgsi.getSrc(R).getIndex(0); if (S >= 0) diff --git a/src/gallium/drivers/nouveau/codegen/nv50_ir_lowering_nvc0.cpp b/src/gallium/drivers/nouveau/codegen/nv50_ir_lowering_nvc0.cpp index 8a864079ce9..4e65d449ebf 100644 --- a/src/gallium/drivers/nouveau/codegen/nv50_ir_lowering_nvc0.cpp +++ b/src/gallium/drivers/nouveau/codegen/nv50_ir_lowering_nvc0.cpp @@ -882,10 +882,12 @@ NVC0LoweringPass::handleTEX(TexInstruction *i) if (i->tex.rIndirectSrc >= 0 || i->tex.sIndirectSrc >= 0) { // XXX this ignores tsc, and assumes a 1:1 mapping assert(i->tex.rIndirectSrc >= 0); - Value *hnd = loadTexHandle(i->getIndirectR(), i->tex.r); - i->tex.r = 0xff; - i->tex.s = 0x1f; - i->setIndirectR(hnd); + if (!i->tex.bindless) { + Value *hnd = loadTexHandle(i->getIndirectR(), i->tex.r); + i->tex.r = 0xff; + i->tex.s = 0x1f; + i->setIndirectR(hnd); + } i->setIndirectS(NULL); } else if (i->tex.r == i->tex.s || i->op == OP_TXF) { if (i->tex.r == 0xffff) diff --git a/src/gallium/drivers/nouveau/nv50/nv50_stateobj_tex.h b/src/gallium/drivers/nouveau/nv50/nv50_stateobj_tex.h index 6bc451450b1..eea2686f551 100644 --- a/src/gallium/drivers/nouveau/nv50/nv50_stateobj_tex.h +++ b/src/gallium/drivers/nouveau/nv50/nv50_stateobj_tex.h @@ -20,6 +20,7 @@ struct nv50_tic_entry { struct pipe_sampler_view pipe; int id; uint32_t tic[8]; + uint32_t bindless; }; static inline struct nv50_tic_entry * diff --git a/src/gallium/drivers/nouveau/nvc0/nvc0_context.c b/src/gallium/drivers/nouveau/nvc0/nvc0_context.c index d5ef5851daa..2e4490b8d97 100644 --- a/src/gallium/drivers/nouveau/nvc0/nvc0_context.c +++ b/src/gallium/drivers/nouveau/nvc0/nvc0_context.c @@ -206,6 +206,16 @@ nvc0_destroy(struct pipe_context *pipe) nvc0_context_unreference_resources(nvc0); nvc0_blitctx_destroy(nvc0); + list_for_each_entry_safe(struct nvc0_resident, pos, &nvc0->tex_head, list) { + list_del(&pos->list); + free(pos); + } + + list_for_each_entry_safe(struct nvc0_resident, pos, &nvc0->img_head, list) { + list_del(&pos->list); + free(pos); + } + nouveau_context_destroy(&nvc0->base); } @@ -401,6 +411,11 @@ nvc0_create(struct pipe_screen *pscreen, void *priv, unsigned ctxflags) nvc0_init_state_functions(nvc0); nvc0_init_transfer_functions(nvc0); nvc0_init_resource_functions(pipe); + if (nvc0->screen->base.class_3d >= NVE4_3D_CLASS) + nvc0_init_bindless_functions(pipe); + + list_inithead(&nvc0->tex_head); + list_inithead(&nvc0->img_head); nvc0->base.invalidate_resource_storage = nvc0_invalidate_resource_storage; diff --git a/src/gallium/drivers/nouveau/nvc0/nvc0_context.h b/src/gallium/drivers/nouveau/nvc0/nvc0_context.h index f0eabb02fe2..c5b625ecb45 100644 --- a/src/gallium/drivers/nouveau/nvc0/nvc0_context.h +++ b/src/gallium/drivers/nouveau/nvc0/nvc0_context.h @@ -5,6 +5,7 @@ #include "pipe/p_defines.h" #include "pipe/p_state.h" +#include "util/list.h" #include "util/u_memory.h" #include "util/u_math.h" #include "util/u_inlines.h" @@ -81,6 +82,7 @@ #define NVC0_BIND_3D_SUF 245 #define NVC0_BIND_3D_BUF 246 #define NVC0_BIND_3D_SCREEN 247 +#define NVC0_BIND_3D_BINDLESS 248 #define NVC0_BIND_3D_TLS 249 #define NVC0_BIND_3D_TEXT 250 #define NVC0_BIND_3D_COUNT 251 @@ -95,7 +97,8 @@ #define NVC0_BIND_CP_QUERY 52 #define NVC0_BIND_CP_BUF 53 #define NVC0_BIND_CP_TEXT 54 -#define NVC0_BIND_CP_COUNT 55 +#define NVC0_BIND_CP_BINDLESS 55 +#define NVC0_BIND_CP_COUNT 56 /* bufctx for other operations */ #define NVC0_BIND_2D 0 @@ -151,6 +154,13 @@ struct nvc0_blitctx; bool nvc0_blitctx_create(struct nvc0_context *); void nvc0_blitctx_destroy(struct nvc0_context *); +struct nvc0_resident { + struct list_head list; + uint64_t handle; + struct nv04_resource *buf; + uint32_t flags; +}; + struct nvc0_context { struct nouveau_context base; @@ -212,6 +222,9 @@ struct nvc0_context { uint32_t tex_handles[6][PIPE_MAX_SAMPLERS]; /* for nve4 */ + struct list_head tex_head; + struct list_head img_head; + struct pipe_framebuffer_state framebuffer; struct pipe_blend_color blend_colour; struct pipe_stencil_ref stencil_ref; @@ -362,6 +375,8 @@ struct pipe_sampler_view * gm107_create_texture_view_from_image(struct pipe_context *, const struct pipe_image_view *); +void nvc0_init_bindless_functions(struct pipe_context *); + /* nvc0_transfer.c */ void nvc0_init_transfer_functions(struct nvc0_context *); diff --git a/src/gallium/drivers/nouveau/nvc0/nvc0_screen.h b/src/gallium/drivers/nouveau/nvc0/nvc0_screen.h index de0a02d9cb8..30635c757f1 100644 --- a/src/gallium/drivers/nouveau/nvc0/nvc0_screen.h +++ b/src/gallium/drivers/nouveau/nvc0/nvc0_screen.h @@ -193,6 +193,8 @@ extern const struct nvc0_vertex_format nvc0_vertex_format[]; static inline void nvc0_screen_tic_unlock(struct nvc0_screen *screen, struct nv50_tic_entry *tic) { + if (tic->bindless) + return; if (tic->id >= 0) screen->tic.lock[tic->id / 32] &= ~(1 << (tic->id % 32)); } diff --git a/src/gallium/drivers/nouveau/nvc0/nvc0_tex.c b/src/gallium/drivers/nouveau/nvc0/nvc0_tex.c index 5ed1d06ce2a..88cec7fcef9 100644 --- a/src/gallium/drivers/nouveau/nvc0/nvc0_tex.c +++ b/src/gallium/drivers/nouveau/nvc0/nvc0_tex.c @@ -90,6 +90,7 @@ gm107_create_texture_view(struct pipe_context *pipe, view->pipe.context = pipe; view->id = -1; + view->bindless = 0; pipe_resource_reference(&view->pipe.texture, texture); @@ -302,6 +303,7 @@ gf100_create_texture_view(struct pipe_context *pipe, view->pipe.context = pipe; view->id = -1; + view->bindless = 0; pipe_resource_reference(&view->pipe.texture, texture); @@ -762,6 +764,124 @@ nve4_set_tex_handles(struct nvc0_context *nvc0) } } +static uint64_t +nve4_create_texture_handle(struct pipe_context *pipe, + struct pipe_sampler_view *view, + const struct pipe_sampler_state *sampler) +{ + /* We have to create persistent handles that won't change for these objects + * That means that we have to upload them into place and lock them so that + * they can't be kicked out later. + */ + struct nvc0_context *nvc0 = nvc0_context(pipe); + struct nouveau_pushbuf *push = nvc0->base.pushbuf; + struct nv50_tic_entry *tic = nv50_tic_entry(view); + struct nv50_tsc_entry *tsc = pipe->create_sampler_state(pipe, sampler); + struct pipe_sampler_view *v = NULL; + + tsc->id = nvc0_screen_tsc_alloc(nvc0->screen, tsc); + if (tsc->id < 0) + goto fail; + + if (tic->id < 0) { + tic->id = nvc0_screen_tic_alloc(nvc0->screen, tic); + if (tic->id < 0) + goto fail; + + nve4_p2mf_push_linear(&nvc0->base, nvc0->screen->txc, tic->id * 32, + NV_VRAM_DOMAIN(&nvc0->screen->base), 32, + tic->tic); + + IMMED_NVC0(push, NVC0_3D(TIC_FLUSH), 0); + } + + nve4_p2mf_push_linear(&nvc0->base, nvc0->screen->txc, + 65536 + tsc->id * 32, + NV_VRAM_DOMAIN(&nvc0->screen->base), + 32, tsc->tsc); + + IMMED_NVC0(push, NVC0_3D(TSC_FLUSH), 0); + + // Add an extra reference to this sampler view effectively held by this + // texture handle. This is to deal with the sampler view being dereferenced + // before the handle is. However we need the view to still be live until the + // handle to it is deleted. + pipe_sampler_view_reference(&v, view); + p_atomic_inc(&tic->bindless); + + nvc0->screen->tic.lock[tic->id / 32] |= 1 << (tic->id % 32); + nvc0->screen->tsc.lock[tsc->id / 32] |= 1 << (tsc->id % 32); + + return 0x100000000ULL | (tsc->id << 20) | tic->id; + +fail: + pipe->delete_sampler_state(pipe, tsc); + return 0; +} + +static bool +view_bound(struct nvc0_context *nvc0, struct pipe_sampler_view *view) { + for (int s = 0; s < 6; s++) { + for (int i = 0; i < nvc0->num_textures[s]; i++) + if (nvc0->textures[s][i] == view) + return true; + } + return false; +} + +static void +nve4_delete_texture_handle(struct pipe_context *pipe, uint64_t handle) +{ + struct nvc0_context *nvc0 = nvc0_context(pipe); + uint32_t tic = handle & NVE4_TIC_ENTRY_INVALID; + uint32_t tsc = (handle & NVE4_TSC_ENTRY_INVALID) >> 20; + struct nv50_tic_entry *entry = nvc0->screen->tic.entries[tic]; + + if (entry) { + struct pipe_sampler_view *view = &entry->pipe; + assert(entry->bindless); + p_atomic_dec(&entry->bindless); + if (!view_bound(nvc0, view)) + nvc0_screen_tic_unlock(nvc0->screen, entry); + pipe_sampler_view_reference(&view, NULL); + } + + pipe->delete_sampler_state(pipe, nvc0->screen->tsc.entries[tsc]); +} + +static void +nve4_make_texture_handle_resident(struct pipe_context *pipe, + uint64_t handle, bool resident) +{ + struct nvc0_context *nvc0 = nvc0_context(pipe); + if (resident) { + struct nvc0_resident *res = calloc(1, sizeof(struct nvc0_resident)); + struct nv50_tic_entry *tic = + nvc0->screen->tic.entries[handle & NVE4_TIC_ENTRY_INVALID]; + assert(tic); + assert(tic->bindless); + + res->handle = handle; + res->buf = nv04_resource(tic->pipe.texture); + res->flags = NOUVEAU_BO_RD; + list_add(&res->list, &nvc0->tex_head); + } else { + list_for_each_entry_safe(struct nvc0_resident, pos, &nvc0->tex_head, list) { + if (pos->handle == handle) { + list_del(&pos->list); + free(pos); + break; + } + } + } +} + +void +nvc0_init_bindless_functions(struct pipe_context *pipe) { + pipe->create_texture_handle = nve4_create_texture_handle; + pipe->delete_texture_handle = nve4_delete_texture_handle; + pipe->make_texture_handle_resident = nve4_make_texture_handle_resident; +} static const uint8_t nve4_su_format_map[PIPE_FORMAT_COUNT]; static const uint16_t nve4_su_format_aux_map[PIPE_FORMAT_COUNT]; diff --git a/src/gallium/drivers/nouveau/nvc0/nvc0_vbo.c b/src/gallium/drivers/nouveau/nvc0/nvc0_vbo.c index 63dccedd3e3..4cd3712203b 100644 --- a/src/gallium/drivers/nouveau/nvc0/nvc0_vbo.c +++ b/src/gallium/drivers/nouveau/nvc0/nvc0_vbo.c @@ -972,6 +972,11 @@ nvc0_draw_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info) BCTX_REFN(nvc0->bufctx_3d, 3D_IDX, buf, RD); } + list_for_each_entry(struct nvc0_resident, resident, &nvc0->tex_head, list) { + nvc0_add_resident(nvc0->bufctx_3d, NVC0_BIND_3D_BINDLESS, resident->buf, + resident->flags); + } + nvc0_state_validate_3d(nvc0, ~0); if (nvc0->vertprog->vp.need_draw_parameters && !info->indirect) { @@ -1083,4 +1088,5 @@ cleanup: nouveau_pushbuf_bufctx(push, NULL); nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_IDX); + nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_BINDLESS); } diff --git a/src/gallium/drivers/nouveau/nvc0/nve4_compute.c b/src/gallium/drivers/nouveau/nvc0/nve4_compute.c index bc5d9e04401..db501afb9d1 100644 --- a/src/gallium/drivers/nouveau/nvc0/nve4_compute.c +++ b/src/gallium/drivers/nouveau/nvc0/nve4_compute.c @@ -681,6 +681,11 @@ nve4_launch_grid(struct pipe_context *pipe, const struct pipe_grid_info *info) BCTX_REFN_bo(nvc0->bufctx_cp, CP_DESC, NOUVEAU_BO_GART | NOUVEAU_BO_RD, desc_bo); + list_for_each_entry(struct nvc0_resident, resident, &nvc0->tex_head, list) { + nvc0_add_resident(nvc0->bufctx_cp, NVC0_BIND_CP_BINDLESS, resident->buf, + resident->flags); + } + ret = !nve4_state_validate_cp(nvc0, ~0); if (ret) goto out; @@ -742,6 +747,7 @@ out: NOUVEAU_ERR("Failed to launch grid !\n"); nouveau_scratch_done(&nvc0->base); nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_DESC); + nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_BINDLESS); } -- 2.30.2