freedreno: Make the slice pitch be bytes, not pixels.
[mesa.git] / src / gallium / drivers / freedreno / ir3 / ir3_gallium.c
index 726bd14ac6d83888c560574b298a8e375131edbc..04474097f552fc6079bb05279003cb63bd424625 100644 (file)
@@ -29,7 +29,7 @@
 #include "util/u_string.h"
 #include "util/u_memory.h"
 #include "util/u_inlines.h"
-#include "util/u_format.h"
+#include "util/format/u_format.h"
 #include "tgsi/tgsi_dump.h"
 #include "tgsi/tgsi_parse.h"
 
 #include "ir3/ir3_nir.h"
 
 static void
-dump_shader_info(struct ir3_shader_variant *v, struct pipe_debug_callback *debug)
+dump_shader_info(struct ir3_shader_variant *v, bool binning_pass,
+               struct pipe_debug_callback *debug)
 {
        if (!unlikely(fd_mesa_debug & FD_DBG_SHADERDB))
                return;
 
-       pipe_debug_message(debug, SHADER_INFO, "\n"
-                       "SHADER-DB: %s prog %d/%d: %u instructions, %u dwords\n"
-                       "SHADER-DB: %s prog %d/%d: %u half, %u full\n"
-                       "SHADER-DB: %s prog %d/%d: %u const, %u constlen\n"
-                       "SHADER-DB: %s prog %d/%d: %u (ss), %u (sy)\n"
-                       "SHADER-DB: %s prog %d/%d: max_sun=%u\n",
-                       ir3_shader_stage(v->shader),
-                       v->shader->id, v->id,
+       pipe_debug_message(debug, SHADER_INFO,
+                       "%s shader: %u inst, %u nops, %u non-nops, %u mov, %u cov, "
+                       "%u dwords, %u last-baryf, %u half, %u full, %u constlen, "
+                       "%u sstall, %u (ss), %u (sy), %d max_sun, %d loops\n",
+                       ir3_shader_stage(v),
                        v->info.instrs_count,
+                       v->info.nops_count,
+                       v->info.instrs_count - v->info.nops_count,
+                       v->info.mov_count,
+                       v->info.cov_count,
                        v->info.sizedwords,
-                       ir3_shader_stage(v->shader),
-                       v->shader->id, v->id,
+                       v->info.last_baryf,
                        v->info.max_half_reg + 1,
                        v->info.max_reg + 1,
-                       ir3_shader_stage(v->shader),
-                       v->shader->id, v->id,
-                       v->info.max_const + 1,
                        v->constlen,
-                       ir3_shader_stage(v->shader),
-                       v->shader->id, v->id,
+                       v->info.sstall,
                        v->info.ss, v->info.sy,
-                       ir3_shader_stage(v->shader),
-                       v->shader->id, v->id,
-                       v->max_sun);
+                       v->max_sun, v->loops);
 }
 
 struct ir3_shader_variant *
@@ -91,7 +86,7 @@ ir3_shader_variant(struct ir3_shader *shader, struct ir3_shader_key key,
        v = ir3_shader_get_variant(shader, &key, binning_pass, &created);
 
        if (created) {
-               dump_shader_info(v, debug);
+               dump_shader_info(v, binning_pass, debug);
        }
 
        return v;
@@ -120,7 +115,7 @@ copy_stream_out(struct ir3_stream_output_info *i,
 
 struct ir3_shader *
 ir3_shader_create(struct ir3_compiler *compiler,
-               const struct pipe_shader_state *cso, gl_shader_stage type,
+               const struct pipe_shader_state *cso,
                struct pipe_debug_callback *debug,
                struct pipe_screen *screen)
 {
@@ -133,7 +128,7 @@ ir3_shader_create(struct ir3_compiler *compiler,
                if (ir3_shader_debug & IR3_DBG_DISASM) {
                        tgsi_dump(cso->tokens, 0);
                }
-               nir = ir3_tgsi_to_nir(compiler, cso->tokens, screen);
+               nir = tgsi_to_nir(cso->tokens, screen);
        }
 
        struct ir3_shader *shader = ir3_shader_from_nir(compiler, nir);
@@ -145,10 +140,13 @@ ir3_shader_create(struct ir3_compiler *compiler,
                 * (as otherwise nothing will trigger the shader to be
                 * actually compiled)
                 */
-               static struct ir3_shader_key key;
-               memset(&key, 0, sizeof(key));
+               static struct ir3_shader_key key; /* static is implicitly zeroed */
                ir3_shader_variant(shader, key, false, debug);
+
+               if (nir->info.stage != MESA_SHADER_FRAGMENT)
+                       ir3_shader_variant(shader, key, true, debug);
        }
+
        return shader;
 }
 
@@ -170,22 +168,21 @@ ir3_shader_create_compute(struct ir3_compiler *compiler,
                if (ir3_shader_debug & IR3_DBG_DISASM) {
                        tgsi_dump(cso->prog, 0);
                }
-               nir = ir3_tgsi_to_nir(compiler, cso->prog, screen);
+               nir = tgsi_to_nir(cso->prog, screen);
        }
 
        struct ir3_shader *shader = ir3_shader_from_nir(compiler, nir);
 
-       return shader;
-}
+       if (fd_mesa_debug & FD_DBG_SHADERDB) {
+               /* if shader-db run, create a standard variant immediately
+                * (as otherwise nothing will trigger the shader to be
+                * actually compiled)
+                */
+               static struct ir3_shader_key key; /* static is implicitly zeroed */
+               ir3_shader_variant(shader, key, false, debug);
+       }
 
-struct nir_shader *
-ir3_tgsi_to_nir(struct ir3_compiler *compiler,
-               const struct tgsi_token *tokens,
-               struct pipe_screen *screen)
-{
-       /* TODO: pass screen to tgsi_to_nir when it needs that. */
-       (void) screen;
-       return tgsi_to_nir(tokens, ir3_get_compiler_options(compiler));
+       return shader;
 }
 
 /* This has to reach into the fd_context a bit more than the rest of
@@ -218,47 +215,88 @@ ring_wfi(struct fd_batch *batch, struct fd_ringbuffer *ring)
 }
 
 static void
-emit_user_consts(struct fd_context *ctx, const struct ir3_shader_variant *v,
-               struct fd_ringbuffer *ring, struct fd_constbuf_stateobj *constbuf)
+emit_const(struct fd_screen *screen, struct fd_ringbuffer *ring,
+               const struct ir3_shader_variant *v, uint32_t dst_offset,
+               uint32_t offset, uint32_t size,
+               const void *user_buffer, struct pipe_resource *buffer)
 {
-       const unsigned index = 0;     /* user consts are index 0 */
+       assert(dst_offset + size <= v->constlen * 4);
 
-       if (constbuf->enabled_mask & (1 << index)) {
-               struct pipe_constant_buffer *cb = &constbuf->cb[index];
-               unsigned size = align(cb->buffer_size, 4) / 4; /* size in dwords */
+       screen->emit_const(ring, v->type, dst_offset,
+                       offset, size, user_buffer, buffer);
+}
 
-               /* in particular, with binning shader we may end up with
-                * unused consts, ie. we could end up w/ constlen that is
-                * smaller than first_driver_param.  In that case truncate
-                * the user consts early to avoid HLSQ lockup caused by
-                * writing too many consts
-                */
-               uint32_t max_const = MIN2(v->num_uniforms, v->constlen);
+/**
+ * Indirectly calculates size of cmdstream needed for ir3_emit_user_consts().
+ * Returns number of packets, and total size of all the payload.
+ *
+ * The value can be a worst-case, ie. some shader variants may not read all
+ * consts, etc.
+ *
+ * Returns size in dwords.
+ */
+void
+ir3_user_consts_size(struct ir3_ubo_analysis_state *state,
+               unsigned *packets, unsigned *size)
+{
+       *packets = *size = 0;
+
+       for (uint32_t i = 0; i < ARRAY_SIZE(state->range); i++) {
+               if (state->range[i].start < state->range[i].end) {
+                       *size += state->range[i].end - state->range[i].start;
+                       (*packets)++;
+               }
+       }
+}
+
+/**
+ * Uploads sub-ranges of UBOs to the hardware's constant buffer (UBO access
+ * outside of these ranges will be done using full UBO accesses in the
+ * shader).
+ */
+void
+ir3_emit_user_consts(struct fd_screen *screen, const struct ir3_shader_variant *v,
+               struct fd_ringbuffer *ring, struct fd_constbuf_stateobj *constbuf)
+{
+       struct ir3_ubo_analysis_state *state;
+       state = &v->shader->ubo_state;
 
-               // I expect that size should be a multiple of vec4's:
-               assert(size == align(size, 4));
+       for (unsigned i = 0; i < state->num_enabled; i++) {
+               assert(!state->range[i].bindless);
+               unsigned ubo = state->range[i].block;
+               if (!(constbuf->enabled_mask & (1 << ubo)))
+                       continue;
+               struct pipe_constant_buffer *cb = &constbuf->cb[ubo];
+
+               uint32_t size = state->range[i].end - state->range[i].start;
+               uint32_t offset = cb->buffer_offset + state->range[i].start;
 
                /* and even if the start of the const buffer is before
                 * first_immediate, the end may not be:
                 */
-               size = MIN2(size, 4 * max_const);
+               size = MIN2(size, (16 * v->constlen) - state->range[i].offset);
 
-               if (size > 0) {
-                       ring_wfi(ctx->batch, ring);
-                       ctx->emit_const(ring, v->type, 0,
-                                       cb->buffer_offset, size,
-                                       cb->user_buffer, cb->buffer);
-               }
+               if (size == 0)
+                       continue;
+
+               /* things should be aligned to vec4: */
+               debug_assert((state->range[i].offset % 16) == 0);
+               debug_assert((size % 16) == 0);
+               debug_assert((offset % 16) == 0);
+
+               emit_const(screen, ring, v, state->range[i].offset / 4,
+                               offset, size / 4, cb->user_buffer, cb->buffer);
        }
 }
 
-static void
-emit_ubos(struct fd_context *ctx, const struct ir3_shader_variant *v,
+void
+ir3_emit_ubos(struct fd_screen *screen, const struct ir3_shader_variant *v,
                struct fd_ringbuffer *ring, struct fd_constbuf_stateobj *constbuf)
 {
-       uint32_t offset = v->constbase.ubo;
+       const struct ir3_const_state *const_state = &v->shader->const_state;
+       uint32_t offset = const_state->offsets.ubo;
        if (v->constlen > offset) {
-               uint32_t params = v->num_ubos;
+               uint32_t params = const_state->num_ubos;
                uint32_t offsets[params];
                struct pipe_resource *prscs[params];
 
@@ -276,64 +314,67 @@ emit_ubos(struct fd_context *ctx, const struct ir3_shader_variant *v,
                        }
                }
 
-               ring_wfi(ctx->batch, ring);
-               ctx->emit_const_bo(ring, v->type, false, offset * 4, params, prscs, offsets);
+               assert(offset * 4 + params < v->constlen * 4);
+
+               screen->emit_const_bo(ring, v->type, false, offset * 4, params, prscs, offsets);
        }
 }
 
-static void
-emit_ssbo_sizes(struct fd_context *ctx, const struct ir3_shader_variant *v,
+void
+ir3_emit_ssbo_sizes(struct fd_screen *screen, const struct ir3_shader_variant *v,
                struct fd_ringbuffer *ring, struct fd_shaderbuf_stateobj *sb)
 {
-       uint32_t offset = v->constbase.ssbo_sizes;
+       const struct ir3_const_state *const_state = &v->shader->const_state;
+       uint32_t offset = const_state->offsets.ssbo_sizes;
        if (v->constlen > offset) {
-               uint32_t sizes[align(v->const_layout.ssbo_size.count, 4)];
-               unsigned mask = v->const_layout.ssbo_size.mask;
+               uint32_t sizes[align(const_state->ssbo_size.count, 4)];
+               unsigned mask = const_state->ssbo_size.mask;
 
                while (mask) {
                        unsigned index = u_bit_scan(&mask);
-                       unsigned off = v->const_layout.ssbo_size.off[index];
+                       unsigned off = const_state->ssbo_size.off[index];
                        sizes[off] = sb->sb[index].buffer_size;
                }
 
-               ring_wfi(ctx->batch, ring);
-               ctx->emit_const(ring, v->type, offset * 4,
+               emit_const(screen, ring, v, offset * 4,
                        0, ARRAY_SIZE(sizes), sizes, NULL);
        }
 }
 
-static void
-emit_image_dims(struct fd_context *ctx, const struct ir3_shader_variant *v,
+void
+ir3_emit_image_dims(struct fd_screen *screen, const struct ir3_shader_variant *v,
                struct fd_ringbuffer *ring, struct fd_shaderimg_stateobj *si)
 {
-       uint32_t offset = v->constbase.image_dims;
+       const struct ir3_const_state *const_state = &v->shader->const_state;
+       uint32_t offset = const_state->offsets.image_dims;
        if (v->constlen > offset) {
-               uint32_t dims[align(v->const_layout.image_dims.count, 4)];
-               unsigned mask = v->const_layout.image_dims.mask;
+               uint32_t dims[align(const_state->image_dims.count, 4)];
+               unsigned mask = const_state->image_dims.mask;
 
                while (mask) {
                        struct pipe_image_view *img;
                        struct fd_resource *rsc;
                        unsigned index = u_bit_scan(&mask);
-                       unsigned off = v->const_layout.image_dims.off[index];
+                       unsigned off = const_state->image_dims.off[index];
 
                        img = &si->si[index];
                        rsc = fd_resource(img->resource);
 
                        dims[off + 0] = util_format_get_blocksize(img->format);
                        if (img->resource->target != PIPE_BUFFER) {
-                               unsigned lvl = img->u.tex.level;
+                               struct fdl_slice *slice =
+                                       fd_resource_slice(rsc, img->u.tex.level);
                                /* note for 2d/cube/etc images, even if re-interpreted
                                 * as a different color format, the pixel size should
                                 * be the same, so use original dimensions for y and z
                                 * stride:
                                 */
-                               dims[off + 1] = rsc->slices[lvl].pitch * rsc->cpp;
+                               dims[off + 1] = slice->pitch;
                                /* see corresponding logic in fd_resource_offset(): */
-                               if (rsc->layer_first) {
-                                       dims[off + 2] = rsc->layer_size;
+                               if (rsc->layout.layer_first) {
+                                       dims[off + 2] = rsc->layout.layer_size;
                                } else {
-                                       dims[off + 2] = rsc->slices[lvl].size0;
+                                       dims[off + 2] = slice->size0;
                                }
                        } else {
                                /* For buffer-backed images, the log2 of the format's
@@ -347,19 +388,19 @@ emit_image_dims(struct fd_context *ctx, const struct ir3_shader_variant *v,
                                dims[off + 1] = ffs(dims[off + 0]) - 1;
                        }
                }
+               uint32_t size = MIN2(ARRAY_SIZE(dims), v->constlen * 4 - offset * 4);
 
-               ring_wfi(ctx->batch, ring);
-               ctx->emit_const(ring, v->type, offset * 4,
-                       0, ARRAY_SIZE(dims), dims, NULL);
+               emit_const(screen, ring, v, offset * 4, 0, size, dims, NULL);
        }
 }
 
-static void
-emit_immediates(struct fd_context *ctx, const struct ir3_shader_variant *v,
+void
+ir3_emit_immediates(struct fd_screen *screen, const struct ir3_shader_variant *v,
                struct fd_ringbuffer *ring)
 {
-       int size = v->immediates_count;
-       uint32_t base = v->constbase.immediate;
+       const struct ir3_const_state *const_state = &v->shader->const_state;
+       uint32_t base = const_state->offsets.immediate;
+       int size = const_state->immediates_count;
 
        /* truncate size to avoid writing constants that shader
         * does not use:
@@ -371,19 +412,45 @@ emit_immediates(struct fd_context *ctx, const struct ir3_shader_variant *v,
        size *= 4;
 
        if (size > 0) {
-               ring_wfi(ctx->batch, ring);
-               ctx->emit_const(ring, v->type, base,
-                       0, size, v->immediates[0].val, NULL);
+               emit_const(screen, ring, v, base,
+                       0, size, const_state->immediates[0].val, NULL);
        }
 }
 
+void
+ir3_emit_link_map(struct fd_screen *screen,
+               const struct ir3_shader_variant *producer,
+               const struct ir3_shader_variant *v, struct fd_ringbuffer *ring)
+{
+       const struct ir3_const_state *const_state = &v->shader->const_state;
+       uint32_t base = const_state->offsets.primitive_map;
+       uint32_t patch_locs[MAX_VARYING] = { }, num_loc;
+
+       num_loc = ir3_link_geometry_stages(producer, v, patch_locs);
+
+       int size = DIV_ROUND_UP(num_loc, 4);
+
+       /* truncate size to avoid writing constants that shader
+        * does not use:
+        */
+       size = MIN2(size + base, v->constlen) - base;
+
+       /* convert out of vec4: */
+       base *= 4;
+       size *= 4;
+
+       if (size > 0)
+               emit_const(screen, ring, v, base, 0, size, patch_locs, NULL);
+}
+
 /* emit stream-out buffers: */
 static void
 emit_tfbos(struct fd_context *ctx, const struct ir3_shader_variant *v,
                struct fd_ringbuffer *ring)
 {
        /* streamout addresses after driver-params: */
-       uint32_t offset = v->constbase.tfbo;
+       const struct ir3_const_state *const_state = &v->shader->const_state;
+       uint32_t offset = const_state->offsets.tfbo;
        if (v->constlen > offset) {
                struct fd_streamout_stateobj *so = &ctx->streamout;
                struct ir3_stream_output_info *info = &v->shader->stream_output;
@@ -404,8 +471,9 @@ emit_tfbos(struct fd_context *ctx, const struct ir3_shader_variant *v,
                        }
                }
 
-               ring_wfi(ctx->batch, ring);
-               ctx->emit_const_bo(ring, v->type, true, offset * 4, params, prscs, offsets);
+               assert(offset * 4 + params < v->constlen * 4);
+
+               ctx->screen->emit_const_bo(ring, v->type, true, offset * 4, params, prscs, offsets);
        }
 }
 
@@ -482,106 +550,119 @@ emit_common_consts(const struct ir3_shader_variant *v, struct fd_ringbuffer *rin
                constbuf = &ctx->constbuf[t];
                shader_dirty = !!(dirty & FD_DIRTY_SHADER_PROG);
 
-               emit_user_consts(ctx, v, ring, constbuf);
-               emit_ubos(ctx, v, ring, constbuf);
+               ring_wfi(ctx->batch, ring);
+
+               ir3_emit_user_consts(ctx->screen, v, ring, constbuf);
+               ir3_emit_ubos(ctx->screen, v, ring, constbuf);
                if (shader_dirty)
-                       emit_immediates(ctx, v, ring);
+                       ir3_emit_immediates(ctx->screen, v, ring);
        }
 
        if (dirty & (FD_DIRTY_SHADER_PROG | FD_DIRTY_SHADER_SSBO)) {
                struct fd_shaderbuf_stateobj *sb = &ctx->shaderbuf[t];
-               emit_ssbo_sizes(ctx, v, ring, sb);
+               ring_wfi(ctx->batch, ring);
+               ir3_emit_ssbo_sizes(ctx->screen, v, ring, sb);
        }
 
        if (dirty & (FD_DIRTY_SHADER_PROG | FD_DIRTY_SHADER_IMAGE)) {
                struct fd_shaderimg_stateobj *si = &ctx->shaderimg[t];
-               emit_image_dims(ctx, v, ring, si);
+               ring_wfi(ctx->batch, ring);
+               ir3_emit_image_dims(ctx->screen, v, ring, si);
        }
 }
 
 void
-ir3_emit_vs_consts(const struct ir3_shader_variant *v, struct fd_ringbuffer *ring,
-               struct fd_context *ctx, const struct pipe_draw_info *info)
+ir3_emit_vs_driver_params(const struct ir3_shader_variant *v,
+               struct fd_ringbuffer *ring, struct fd_context *ctx,
+               const struct pipe_draw_info *info)
 {
-       debug_assert(v->type == MESA_SHADER_VERTEX);
-
-       emit_common_consts(v, ring, ctx, PIPE_SHADER_VERTEX);
-
-       /* emit driver params every time: */
-       /* TODO skip emit if shader doesn't use driver params to avoid WFI.. */
-       if (info) {
-               uint32_t offset = v->constbase.driver_param;
-               if (v->constlen > offset) {
-                       uint32_t vertex_params[IR3_DP_VS_COUNT] = {
-                               [IR3_DP_VTXID_BASE] = info->index_size ?
-                                               info->index_bias : info->start,
-                               [IR3_DP_VTXCNT_MAX] = max_tf_vtx(ctx, v),
-                       };
-                       /* if no user-clip-planes, we don't need to emit the
-                        * entire thing:
-                        */
-                       uint32_t vertex_params_size = 4;
-
-                       if (v->key.ucp_enables) {
-                               struct pipe_clip_state *ucp = &ctx->ucp;
-                               unsigned pos = IR3_DP_UCP0_X;
-                               for (unsigned i = 0; pos <= IR3_DP_UCP7_W; i++) {
-                                       for (unsigned j = 0; j < 4; j++) {
-                                               vertex_params[pos] = fui(ucp->ucp[i][j]);
-                                               pos++;
-                                       }
-                               }
-                               vertex_params_size = ARRAY_SIZE(vertex_params);
+       debug_assert(ir3_needs_vs_driver_params(v));
+
+       const struct ir3_const_state *const_state = &v->shader->const_state;
+       uint32_t offset = const_state->offsets.driver_param;
+       uint32_t vertex_params[IR3_DP_VS_COUNT] = {
+                       [IR3_DP_VTXID_BASE] = info->index_size ?
+                                       info->index_bias : info->start,
+                                       [IR3_DP_VTXCNT_MAX] = max_tf_vtx(ctx, v),
+       };
+       /* if no user-clip-planes, we don't need to emit the
+        * entire thing:
+        */
+       uint32_t vertex_params_size = 4;
+
+       if (v->key.ucp_enables) {
+               struct pipe_clip_state *ucp = &ctx->ucp;
+               unsigned pos = IR3_DP_UCP0_X;
+               for (unsigned i = 0; pos <= IR3_DP_UCP7_W; i++) {
+                       for (unsigned j = 0; j < 4; j++) {
+                               vertex_params[pos] = fui(ucp->ucp[i][j]);
+                               pos++;
                        }
+               }
+               vertex_params_size = ARRAY_SIZE(vertex_params);
+       }
 
-                       ring_wfi(ctx->batch, ring);
+       vertex_params_size = MAX2(vertex_params_size, const_state->num_driver_params);
 
-                       bool needs_vtxid_base =
-                               ir3_find_sysval_regid(v, SYSTEM_VALUE_VERTEX_ID_ZERO_BASE) != regid(63, 0);
+       bool needs_vtxid_base =
+               ir3_find_sysval_regid(v, SYSTEM_VALUE_VERTEX_ID_ZERO_BASE) != regid(63, 0);
 
-                       /* for indirect draw, we need to copy VTXID_BASE from
-                        * indirect-draw parameters buffer.. which is annoying
-                        * and means we can't easily emit these consts in cmd
-                        * stream so need to copy them to bo.
-                        */
-                       if (info->indirect && needs_vtxid_base) {
-                               struct pipe_draw_indirect_info *indirect = info->indirect;
-                               struct pipe_resource *vertex_params_rsc =
-                                       pipe_buffer_create(&ctx->screen->base,
+       /* for indirect draw, we need to copy VTXID_BASE from
+        * indirect-draw parameters buffer.. which is annoying
+        * and means we can't easily emit these consts in cmd
+        * stream so need to copy them to bo.
+        */
+       if (info->indirect && needs_vtxid_base) {
+               struct pipe_draw_indirect_info *indirect = info->indirect;
+               struct pipe_resource *vertex_params_rsc =
+                               pipe_buffer_create(&ctx->screen->base,
                                                PIPE_BIND_CONSTANT_BUFFER, PIPE_USAGE_STREAM,
                                                vertex_params_size * 4);
-                               unsigned src_off = info->indirect->offset;;
-                               void *ptr;
+               unsigned src_off = info->indirect->offset;;
+               void *ptr;
 
-                               ptr = fd_bo_map(fd_resource(vertex_params_rsc)->bo);
-                               memcpy(ptr, vertex_params, vertex_params_size * 4);
+               ptr = fd_bo_map(fd_resource(vertex_params_rsc)->bo);
+               memcpy(ptr, vertex_params, vertex_params_size * 4);
 
-                               if (info->index_size) {
-                                       /* indexed draw, index_bias is 4th field: */
-                                       src_off += 3 * 4;
-                               } else {
-                                       /* non-indexed draw, start is 3rd field: */
-                                       src_off += 2 * 4;
-                               }
+               if (info->index_size) {
+                       /* indexed draw, index_bias is 4th field: */
+                       src_off += 3 * 4;
+               } else {
+                       /* non-indexed draw, start is 3rd field: */
+                       src_off += 2 * 4;
+               }
 
-                               /* copy index_bias or start from draw params: */
-                               ctx->mem_to_mem(ring, vertex_params_rsc, 0,
-                                               indirect->buffer, src_off, 1);
+               /* copy index_bias or start from draw params: */
+               ctx->screen->mem_to_mem(ring, vertex_params_rsc, 0,
+                               indirect->buffer, src_off, 1);
 
-                               ctx->emit_const(ring, MESA_SHADER_VERTEX, offset * 4, 0,
-                                               vertex_params_size, NULL, vertex_params_rsc);
+               emit_const(ctx->screen, ring, v, offset * 4, 0,
+                               vertex_params_size, NULL, vertex_params_rsc);
 
-                               pipe_resource_reference(&vertex_params_rsc, NULL);
-                       } else {
-                               ctx->emit_const(ring, MESA_SHADER_VERTEX, offset * 4, 0,
-                                               vertex_params_size, vertex_params, NULL);
-                       }
+               pipe_resource_reference(&vertex_params_rsc, NULL);
+       } else {
+               emit_const(ctx->screen, ring, v, offset * 4, 0,
+                               vertex_params_size, vertex_params, NULL);
+       }
 
-                       /* if needed, emit stream-out buffer addresses: */
-                       if (vertex_params[IR3_DP_VTXCNT_MAX] > 0) {
-                               emit_tfbos(ctx, v, ring);
-                       }
-               }
+       /* if needed, emit stream-out buffer addresses: */
+       if (vertex_params[IR3_DP_VTXCNT_MAX] > 0) {
+               emit_tfbos(ctx, v, ring);
+       }
+}
+
+void
+ir3_emit_vs_consts(const struct ir3_shader_variant *v, struct fd_ringbuffer *ring,
+               struct fd_context *ctx, const struct pipe_draw_info *info)
+{
+       debug_assert(v->type == MESA_SHADER_VERTEX);
+
+       emit_common_consts(v, ring, ctx, PIPE_SHADER_VERTEX);
+
+       /* emit driver params every time: */
+       if (info && ir3_needs_vs_driver_params(v)) {
+               ring_wfi(ctx->batch, ring);
+               ir3_emit_vs_driver_params(v, ring, ctx, info);
        }
 }
 
@@ -604,7 +685,8 @@ ir3_emit_cs_consts(const struct ir3_shader_variant *v, struct fd_ringbuffer *rin
        emit_common_consts(v, ring, ctx, PIPE_SHADER_COMPUTE);
 
        /* emit compute-shader driver-params: */
-       uint32_t offset = v->constbase.driver_param;
+       const struct ir3_const_state *const_state = &v->shader->const_state;
+       uint32_t offset = const_state->offsets.driver_param;
        if (v->constlen > offset) {
                ring_wfi(ctx->batch, ring);
 
@@ -626,14 +708,14 @@ ir3_emit_cs_consts(const struct ir3_shader_variant *v, struct fd_ringbuffer *rin
                                        0x1000);
                                indirect_offset = 0;
 
-                               ctx->mem_to_mem(ring, indirect, 0, info->indirect,
+                               ctx->screen->mem_to_mem(ring, indirect, 0, info->indirect,
                                                info->indirect_offset, 3);
                        } else {
                                pipe_resource_reference(&indirect, info->indirect);
                                indirect_offset = info->indirect_offset;
                        }
 
-                       ctx->emit_const(ring, MESA_SHADER_COMPUTE, offset * 4,
+                       emit_const(ctx->screen, ring, v, offset * 4,
                                        indirect_offset, 4, NULL, indirect);
 
                        pipe_resource_reference(&indirect, NULL);
@@ -646,9 +728,45 @@ ir3_emit_cs_consts(const struct ir3_shader_variant *v, struct fd_ringbuffer *rin
                                [IR3_DP_LOCAL_GROUP_SIZE_Y] = info->block[1],
                                [IR3_DP_LOCAL_GROUP_SIZE_Z] = info->block[2],
                        };
+                       uint32_t size = MIN2(const_state->num_driver_params,
+                                       v->constlen * 4 - offset * 4);
 
-                       ctx->emit_const(ring, MESA_SHADER_COMPUTE, offset * 4, 0,
-                                       ARRAY_SIZE(compute_params), compute_params, NULL);
+                       emit_const(ctx->screen, ring, v, offset * 4, 0, size,
+                                       compute_params, NULL);
                }
        }
 }
+
+static void *
+ir3_shader_state_create(struct pipe_context *pctx, const struct pipe_shader_state *cso)
+{
+       struct fd_context *ctx = fd_context(pctx);
+       struct ir3_compiler *compiler = ctx->screen->compiler;
+       return ir3_shader_create(compiler, cso, &ctx->debug, pctx->screen);
+}
+
+static void
+ir3_shader_state_delete(struct pipe_context *pctx, void *hwcso)
+{
+       struct ir3_shader *so = hwcso;
+       ir3_shader_destroy(so);
+}
+
+void
+ir3_prog_init(struct pipe_context *pctx)
+{
+       pctx->create_vs_state = ir3_shader_state_create;
+       pctx->delete_vs_state = ir3_shader_state_delete;
+
+       pctx->create_tcs_state = ir3_shader_state_create;
+       pctx->delete_tcs_state = ir3_shader_state_delete;
+
+       pctx->create_tes_state = ir3_shader_state_create;
+       pctx->delete_tes_state = ir3_shader_state_delete;
+
+       pctx->create_gs_state = ir3_shader_state_create;
+       pctx->delete_gs_state = ir3_shader_state_delete;
+
+       pctx->create_fs_state = ir3_shader_state_create;
+       pctx->delete_fs_state = ir3_shader_state_delete;
+}