panfrost: Make panfrost_bo_wait take a wait_readers bool
[mesa.git] / src / gallium / drivers / panfrost / pan_context.c
index 999823263f984197ad657f8acace1f0219dc778a..ca383a8a8bedb8026e82664adc94e4e2c3547ca5 100644 (file)
 #include "pan_cmdstream.h"
 #include "pan_util.h"
 #include "pandecode/decode.h"
+#include "util/pan_lower_framebuffer.h"
 
 struct midgard_tiler_descriptor
 panfrost_emit_midg_tiler(struct panfrost_batch *batch, unsigned vertex_count)
 {
-        struct panfrost_screen *screen = pan_screen(batch->ctx->base.screen);
-        bool hierarchy = !(screen->quirks & MIDGARD_NO_HIER_TILING);
+        struct panfrost_device *device = pan_device(batch->ctx->base.screen);
+        bool hierarchy = !(device->quirks & MIDGARD_NO_HIER_TILING);
         struct midgard_tiler_descriptor t = {0};
         unsigned height = batch->key.height;
         unsigned width = batch->key.width;
@@ -125,6 +126,7 @@ static void
 panfrost_clear(
         struct pipe_context *pipe,
         unsigned buffers,
+        const struct pipe_scissor_state *scissor_state,
         const union pipe_color_union *color,
         double depth, unsigned stencil)
 {
@@ -158,12 +160,12 @@ panfrost_writes_point_size(struct panfrost_context *ctx)
         assert(ctx->shader[PIPE_SHADER_VERTEX]);
         struct panfrost_shader_state *vs = panfrost_get_shader_state(ctx, PIPE_SHADER_VERTEX);
 
-        return vs->writes_point_size && ctx->payloads[PIPE_SHADER_FRAGMENT].prefix.draw_mode == MALI_POINTS;
+        return vs->writes_point_size && ctx->active_prim == PIPE_PRIM_POINTS;
 }
 
 void
 panfrost_vertex_state_upd_attr_offs(struct panfrost_context *ctx,
-                                    struct midgard_payload_vertex_tiler *vp)
+                                    struct mali_vertex_tiler_postfix *vertex_postfix)
 {
         if (!ctx->vertex)
                 return;
@@ -191,7 +193,7 @@ panfrost_vertex_state_upd_attr_offs(struct panfrost_context *ctx,
          * QED.
          */
 
-        unsigned start = vp->offset_start;
+        unsigned start = vertex_postfix->offset_start;
 
         for (unsigned i = 0; i < so->num_elements; ++i) {
                 unsigned vbi = so->pipe[i].vertex_buffer_index;
@@ -237,6 +239,7 @@ panfrost_flush(
         unsigned flags)
 {
         struct panfrost_context *ctx = pan_context(pipe);
+        struct panfrost_device *dev = pan_device(pipe->screen);
         struct util_dynarray fences;
 
         /* We must collect the fences before the flush is done, otherwise we'll
@@ -268,10 +271,17 @@ panfrost_flush(
                 util_dynarray_fini(&fences);
         }
 
-        if (pan_debug & PAN_DBG_TRACE)
+        if (dev->debug & PAN_DBG_TRACE)
                 pandecode_next_frame();
 }
 
+static void
+panfrost_texture_barrier(struct pipe_context *pipe, unsigned flags)
+{
+        struct panfrost_context *ctx = pan_context(pipe);
+        panfrost_flush_all_batches(ctx, false);
+}
+
 #define DEFINE_CASE(c) case PIPE_PRIM_##c: return MALI_##c;
 
 static int
@@ -398,53 +408,52 @@ panfrost_draw_vbo(
         ctx->instance_count = info->instance_count;
         ctx->active_prim = info->mode;
 
+        struct mali_vertex_tiler_prefix vertex_prefix, tiler_prefix;
+        struct mali_vertex_tiler_postfix vertex_postfix, tiler_postfix;
+        union midgard_primitive_size primitive_size;
         unsigned vertex_count;
 
-        for (int i = 0; i <= PIPE_SHADER_FRAGMENT; ++i)
-                panfrost_vt_init(ctx, i, &ctx->payloads[i]);
+        panfrost_vt_init(ctx, PIPE_SHADER_VERTEX, &vertex_prefix, &vertex_postfix);
+        panfrost_vt_init(ctx, PIPE_SHADER_FRAGMENT, &tiler_prefix, &tiler_postfix);
 
         panfrost_vt_set_draw_info(ctx, info, g2m_draw_mode(mode),
-                                  &ctx->payloads[PIPE_SHADER_VERTEX],
-                                  &ctx->payloads[PIPE_SHADER_FRAGMENT],
-                                  &vertex_count, &ctx->padded_count);
+                                  &vertex_postfix, &tiler_prefix,
+                                  &tiler_postfix, &vertex_count,
+                                  &ctx->padded_count);
 
         panfrost_statistics_record(ctx, info);
 
         /* Dispatch "compute jobs" for the vertex/tiler pair as (1,
          * vertex_count, 1) */
 
-        panfrost_pack_work_groups_fused(
-                &ctx->payloads[PIPE_SHADER_VERTEX].prefix,
-                &ctx->payloads[PIPE_SHADER_FRAGMENT].prefix,
-                1, vertex_count, info->instance_count,
-                1, 1, 1);
+        panfrost_pack_work_groups_fused(&vertex_prefix, &tiler_prefix,
+                                        1, vertex_count, info->instance_count,
+                                        1, 1, 1);
 
         /* Emit all sort of descriptors. */
-        panfrost_emit_vertex_data(batch, &ctx->payloads[PIPE_SHADER_VERTEX]);
+        panfrost_emit_vertex_data(batch, &vertex_postfix);
         panfrost_emit_varying_descriptor(batch,
                                          ctx->padded_count *
                                          ctx->instance_count,
-                                         &ctx->payloads[PIPE_SHADER_VERTEX],
-                                         &ctx->payloads[PIPE_SHADER_FRAGMENT]);
-        panfrost_emit_shader_meta(batch, PIPE_SHADER_VERTEX,
-                                  &ctx->payloads[PIPE_SHADER_VERTEX]);
-        panfrost_emit_shader_meta(batch, PIPE_SHADER_FRAGMENT,
-                                  &ctx->payloads[PIPE_SHADER_FRAGMENT]);
-        panfrost_emit_vertex_attr_meta(batch,
-                                       &ctx->payloads[PIPE_SHADER_VERTEX]);
-
-        for (int i = 0; i <= PIPE_SHADER_FRAGMENT; ++i) {
-                panfrost_emit_sampler_descriptors(batch, i, &ctx->payloads[i]);
-                panfrost_emit_texture_descriptors(batch, i, &ctx->payloads[i]);
-                panfrost_emit_const_buf(batch, i, &ctx->payloads[i]);
-        }
-
-        panfrost_emit_viewport(batch, &ctx->payloads[PIPE_SHADER_FRAGMENT]);
+                                         &vertex_postfix, &tiler_postfix,
+                                         &primitive_size);
+        panfrost_emit_shader_meta(batch, PIPE_SHADER_VERTEX, &vertex_postfix);
+        panfrost_emit_shader_meta(batch, PIPE_SHADER_FRAGMENT, &tiler_postfix);
+        panfrost_emit_vertex_attr_meta(batch, &vertex_postfix);
+        panfrost_emit_sampler_descriptors(batch, PIPE_SHADER_VERTEX, &vertex_postfix);
+        panfrost_emit_sampler_descriptors(batch, PIPE_SHADER_FRAGMENT, &tiler_postfix);
+        panfrost_emit_texture_descriptors(batch, PIPE_SHADER_VERTEX, &vertex_postfix);
+        panfrost_emit_texture_descriptors(batch, PIPE_SHADER_FRAGMENT, &tiler_postfix);
+        panfrost_emit_const_buf(batch, PIPE_SHADER_VERTEX, &vertex_postfix);
+        panfrost_emit_const_buf(batch, PIPE_SHADER_FRAGMENT, &tiler_postfix);
+        panfrost_emit_viewport(batch, &tiler_postfix);
+
+        panfrost_vt_update_primitive_size(ctx, &tiler_prefix, &primitive_size);
 
         /* Fire off the draw itself */
-        panfrost_emit_vertex_tiler_jobs(batch,
-                                        &ctx->payloads[PIPE_SHADER_VERTEX],
-                                        &ctx->payloads[PIPE_SHADER_FRAGMENT]);
+        panfrost_emit_vertex_tiler_jobs(batch, &vertex_prefix, &vertex_postfix,
+                                               &tiler_prefix, &tiler_postfix,
+                                               &primitive_size);
 
         /* Adjust the batch stack size based on the new shader stack sizes. */
         panfrost_batch_adjust_stack_size(batch);
@@ -503,6 +512,7 @@ panfrost_create_vertex_elements_state(
         const struct pipe_vertex_element *elements)
 {
         struct panfrost_vertex_state *so = CALLOC_STRUCT(panfrost_vertex_state);
+        struct panfrost_device *dev = pan_device(pctx->screen);
 
         so->num_elements = num_elements;
         memcpy(so->pipe, elements, sizeof(*elements) * num_elements);
@@ -513,16 +523,29 @@ panfrost_create_vertex_elements_state(
                 enum pipe_format fmt = elements[i].src_format;
                 const struct util_format_description *desc = util_format_description(fmt);
                 so->hw[i].unknown1 = 0x2;
-                so->hw[i].swizzle = panfrost_get_default_swizzle(desc->nr_channels);
 
-                so->hw[i].format = panfrost_find_format(desc);
+                if (dev->quirks & HAS_SWIZZLES)
+                        so->hw[i].swizzle = panfrost_translate_swizzle_4(desc->swizzle);
+                else
+                        so->hw[i].swizzle = panfrost_bifrost_swizzle(desc->nr_channels);
+
+                enum mali_format hw_format = panfrost_pipe_format_table[desc->format].hw;
+                so->hw[i].format = hw_format;
+                assert(hw_format);
         }
 
         /* Let's also prepare vertex builtins */
         so->hw[PAN_VERTEX_ID].format = MALI_R32UI;
-        so->hw[PAN_VERTEX_ID].swizzle = panfrost_get_default_swizzle(1);
+        if (dev->quirks & HAS_SWIZZLES)
+                so->hw[PAN_VERTEX_ID].swizzle = panfrost_get_default_swizzle(1);
+        else
+                so->hw[PAN_VERTEX_ID].swizzle = panfrost_bifrost_swizzle(1);
+
         so->hw[PAN_INSTANCE_ID].format = MALI_R32UI;
-        so->hw[PAN_INSTANCE_ID].swizzle = panfrost_get_default_swizzle(1);
+        if (dev->quirks & HAS_SWIZZLES)
+                so->hw[PAN_INSTANCE_ID].swizzle = panfrost_get_default_swizzle(1);
+        else
+                so->hw[PAN_INSTANCE_ID].swizzle = panfrost_bifrost_swizzle(1);
 
         return so;
 }
@@ -543,6 +566,7 @@ panfrost_create_shader_state(
         enum pipe_shader_type stage)
 {
         struct panfrost_shader_variants *so = CALLOC_STRUCT(panfrost_shader_variants);
+        struct panfrost_device *dev = pan_device(pctx->screen);
         so->base = *cso;
 
         /* Token deep copy to prevent memory corruption */
@@ -551,7 +575,7 @@ panfrost_create_shader_state(
                 so->base.tokens = tgsi_dup_tokens(so->base.tokens);
 
         /* Precompile for shader-db if we need to */
-        if (unlikely((pan_debug & PAN_DBG_PRECOMPILE) && cso->type == PIPE_SHADER_IR_NIR)) {
+        if (unlikely((dev->debug & PAN_DBG_PRECOMPILE) && cso->type == PIPE_SHADER_IR_NIR)) {
                 struct panfrost_context *ctx = pan_context(pctx);
 
                 struct panfrost_shader_state state;
@@ -574,7 +598,7 @@ panfrost_delete_shader_state(
         struct panfrost_shader_variants *cso = (struct panfrost_shader_variants *) so;
 
         if (cso->base.type == PIPE_SHADER_IR_TGSI) {
-                DBG("Deleting TGSI shader leaks duplicated tokens\n");
+                /* TODO: leaks TGSI tokens! */
         }
 
         for (unsigned i = 0; i < cso->variant_count; ++i) {
@@ -593,9 +617,14 @@ panfrost_create_sampler_state(
         const struct pipe_sampler_state *cso)
 {
         struct panfrost_sampler_state *so = CALLOC_STRUCT(panfrost_sampler_state);
+        struct panfrost_device *device = pan_device(pctx->screen);
+
         so->base = *cso;
 
-        panfrost_sampler_desc_init(cso, &so->hw);
+        if (device->quirks & IS_BIFROST)
+                panfrost_sampler_desc_init_bifrost(cso, &so->bifrost_hw);
+        else
+                panfrost_sampler_desc_init(cso, &so->midgard_hw);
 
         return so;
 }
@@ -622,6 +651,7 @@ panfrost_variant_matches(
         struct panfrost_shader_state *variant,
         enum pipe_shader_type type)
 {
+        struct panfrost_device *dev = pan_device(ctx->base.screen);
         struct pipe_rasterizer_state *rasterizer = &ctx->rasterizer->base;
         struct pipe_alpha_state *alpha = &ctx->depth_stencil->alpha;
 
@@ -642,8 +672,31 @@ panfrost_variant_matches(
                 }
         }
 
+        if (variant->outputs_read) {
+                struct pipe_framebuffer_state *fb = &ctx->pipe_framebuffer;
+
+                unsigned i;
+                BITSET_FOREACH_SET(i, &variant->outputs_read, 8) {
+                        enum pipe_format fmt = PIPE_FORMAT_R8G8B8A8_UNORM;
+
+                        if ((fb->nr_cbufs > i) && fb->cbufs[i])
+                                fmt = fb->cbufs[i]->format;
+
+                        const struct util_format_description *desc =
+                                util_format_description(fmt);
+
+                        if (pan_format_class_load(desc, dev->quirks) == PAN_FORMAT_NATIVE)
+                                fmt = PIPE_FORMAT_NONE;
+
+                        if (variant->rt_formats[i] != fmt)
+                                return false;
+                }
+        }
+
+        /* Point sprites TODO on bifrost, always pass */
         if (is_fragment && rasterizer && (rasterizer->sprite_coord_enable |
-                                          variant->point_sprite_mask)) {
+                                          variant->point_sprite_mask)
+                        && !(dev->quirks & IS_BIFROST)) {
                 /* Ensure the same varyings are turned to point sprites */
                 if (rasterizer->sprite_coord_enable != variant->point_sprite_mask)
                         return false;
@@ -707,6 +760,7 @@ panfrost_bind_shader_state(
         enum pipe_shader_type type)
 {
         struct panfrost_context *ctx = pan_context(pctx);
+        struct panfrost_device *dev = pan_device(ctx->base.screen);
         ctx->shader[type] = hwcso;
 
         if (!hwcso) return;
@@ -752,7 +806,24 @@ panfrost_bind_shader_state(
                 if (type == PIPE_SHADER_FRAGMENT) {
                         v->alpha_state = ctx->depth_stencil->alpha;
 
-                        if (ctx->rasterizer) {
+                        struct pipe_framebuffer_state *fb = &ctx->pipe_framebuffer;
+                        for (unsigned i = 0; i < fb->nr_cbufs; ++i) {
+                                enum pipe_format fmt = PIPE_FORMAT_R8G8B8A8_UNORM;
+
+                                if ((fb->nr_cbufs > i) && fb->cbufs[i])
+                                        fmt = fb->cbufs[i]->format;
+
+                                const struct util_format_description *desc =
+                                        util_format_description(fmt);
+
+                                if (pan_format_class_load(desc, dev->quirks) == PAN_FORMAT_NATIVE)
+                                        fmt = PIPE_FORMAT_NONE;
+
+                                v->rt_formats[i] = fmt;
+                        }
+
+                        /* Point sprites are TODO on Bifrost */
+                        if (ctx->rasterizer && !(dev->quirks & IS_BIFROST)) {
                                 v->point_sprite_mask = ctx->rasterizer->base.sprite_coord_enable;
                                 v->point_sprite_upper_left =
                                         ctx->rasterizer->base.sprite_coord_mode ==
@@ -859,29 +930,120 @@ panfrost_set_stencil_ref(
         ctx->stencil_ref = *ref;
 }
 
-static enum mali_texture_type
-panfrost_translate_texture_type(enum pipe_texture_target t) {
-        switch (t)
-        {
-        case PIPE_BUFFER:
-        case PIPE_TEXTURE_1D:
-        case PIPE_TEXTURE_1D_ARRAY:
-                return MALI_TEX_1D;
+void
+panfrost_create_sampler_view_bo(struct panfrost_sampler_view *so,
+                                struct pipe_context *pctx,
+                                struct pipe_resource *texture)
+{
+        struct panfrost_device *device = pan_device(pctx->screen);
+        struct panfrost_resource *prsrc = (struct panfrost_resource *)texture;
+        enum pipe_format format = so->base.format;
+        assert(prsrc->bo);
+
+        /* Format to access the stencil portion of a Z32_S8 texture */
+        if (so->base.format == PIPE_FORMAT_X32_S8X24_UINT) {
+                assert(prsrc->separate_stencil);
+                texture = &prsrc->separate_stencil->base;
+                prsrc = (struct panfrost_resource *)texture;
+                format = texture->format;
+        }
+
+        so->texture_bo = prsrc->bo->gpu;
+        so->layout = prsrc->layout;
 
-        case PIPE_TEXTURE_2D:
-        case PIPE_TEXTURE_2D_ARRAY:
-        case PIPE_TEXTURE_RECT:
-                return MALI_TEX_2D;
+        unsigned char user_swizzle[4] = {
+                so->base.swizzle_r,
+                so->base.swizzle_g,
+                so->base.swizzle_b,
+                so->base.swizzle_a
+        };
 
-        case PIPE_TEXTURE_3D:
-                return MALI_TEX_3D;
+        /* In the hardware, array_size refers specifically to array textures,
+         * whereas in Gallium, it also covers cubemaps */
 
-        case PIPE_TEXTURE_CUBE:
-        case PIPE_TEXTURE_CUBE_ARRAY:
-                return MALI_TEX_CUBE;
+        unsigned array_size = texture->array_size;
+        unsigned depth = texture->depth0;
 
-        default:
-                unreachable("Unknown target");
+        if (so->base.target == PIPE_TEXTURE_CUBE) {
+                /* TODO: Cubemap arrays */
+                assert(array_size == 6);
+                array_size /= 6;
+        }
+
+        /* MSAA only supported for 2D textures (and 2D texture arrays via an
+         * extension currently unimplemented */
+
+        if (so->base.target == PIPE_TEXTURE_2D) {
+                assert(depth == 1);
+                depth = texture->nr_samples;
+        } else {
+                /* MSAA only supported for 2D textures */
+                assert(texture->nr_samples <= 1);
+        }
+
+        enum mali_texture_type type =
+                panfrost_translate_texture_type(so->base.target);
+
+        if (device->quirks & IS_BIFROST) {
+                const struct util_format_description *desc =
+                        util_format_description(format);
+                unsigned char composed_swizzle[4];
+                util_format_compose_swizzles(desc->swizzle, user_swizzle, composed_swizzle);
+
+                unsigned size = panfrost_estimate_texture_payload_size(
+                                so->base.u.tex.first_level,
+                                so->base.u.tex.last_level,
+                                so->base.u.tex.first_layer,
+                                so->base.u.tex.last_layer,
+                                texture->nr_samples,
+                                type, prsrc->layout);
+
+                so->bo = panfrost_bo_create(device, size, 0);
+
+                so->bifrost_descriptor = rzalloc(pctx, struct bifrost_texture_descriptor);
+                panfrost_new_texture_bifrost(
+                                so->bifrost_descriptor,
+                                texture->width0, texture->height0,
+                                depth, array_size,
+                                format,
+                                type, prsrc->layout,
+                                so->base.u.tex.first_level,
+                                so->base.u.tex.last_level,
+                                so->base.u.tex.first_layer,
+                                so->base.u.tex.last_layer,
+                                texture->nr_samples,
+                                prsrc->cubemap_stride,
+                                panfrost_translate_swizzle_4(composed_swizzle),
+                                prsrc->bo->gpu,
+                                prsrc->slices,
+                                so->bo);
+        } else {
+                unsigned size = panfrost_estimate_texture_payload_size(
+                                so->base.u.tex.first_level,
+                                so->base.u.tex.last_level,
+                                so->base.u.tex.first_layer,
+                                so->base.u.tex.last_layer,
+                                texture->nr_samples,
+                                type, prsrc->layout);
+                size += sizeof(struct mali_texture_descriptor);
+
+                so->bo = panfrost_bo_create(device, size, 0);
+
+                panfrost_new_texture(
+                                so->bo->cpu,
+                                texture->width0, texture->height0,
+                                depth, array_size,
+                                format,
+                                type, prsrc->layout,
+                                so->base.u.tex.first_level,
+                                so->base.u.tex.last_level,
+                                so->base.u.tex.first_layer,
+                                so->base.u.tex.last_layer,
+                                texture->nr_samples,
+                                prsrc->cubemap_stride,
+                                panfrost_translate_swizzle_4(user_swizzle),
+                                prsrc->bo->gpu,
+                                prsrc->slices);
         }
 }
 
@@ -891,63 +1053,16 @@ panfrost_create_sampler_view(
         struct pipe_resource *texture,
         const struct pipe_sampler_view *template)
 {
-        struct panfrost_screen *screen = pan_screen(pctx->screen);
         struct panfrost_sampler_view *so = rzalloc(pctx, struct panfrost_sampler_view);
 
         pipe_reference(NULL, &texture->reference);
 
-        struct panfrost_resource *prsrc = (struct panfrost_resource *) texture;
-        assert(prsrc->bo);
-
         so->base = *template;
         so->base.texture = texture;
         so->base.reference.count = 1;
         so->base.context = pctx;
 
-        unsigned char user_swizzle[4] = {
-                template->swizzle_r,
-                template->swizzle_g,
-                template->swizzle_b,
-                template->swizzle_a
-        };
-
-        /* In the hardware, array_size refers specifically to array textures,
-         * whereas in Gallium, it also covers cubemaps */
-
-        unsigned array_size = texture->array_size;
-
-        if (template->target == PIPE_TEXTURE_CUBE) {
-                /* TODO: Cubemap arrays */
-                assert(array_size == 6);
-                array_size /= 6;
-        }
-
-        enum mali_texture_type type =
-                panfrost_translate_texture_type(template->target);
-
-        unsigned size = panfrost_estimate_texture_size(
-                        template->u.tex.first_level,
-                        template->u.tex.last_level,
-                        template->u.tex.first_layer,
-                        template->u.tex.last_layer,
-                        type, prsrc->layout);
-
-        so->bo = panfrost_bo_create(screen, size, 0);
-
-        panfrost_new_texture(
-                        so->bo->cpu,
-                        texture->width0, texture->height0,
-                        texture->depth0, array_size,
-                        template->format,
-                        type, prsrc->layout,
-                        template->u.tex.first_level,
-                        template->u.tex.last_level,
-                        template->u.tex.first_layer,
-                        template->u.tex.last_layer,
-                        prsrc->cubemap_stride,
-                        panfrost_translate_swizzle_4(user_swizzle),
-                        prsrc->bo->gpu,
-                        prsrc->slices);
+        panfrost_create_sampler_view_bo(so, pctx, texture);
 
         return (struct pipe_sampler_view *) so;
 }
@@ -988,6 +1103,8 @@ panfrost_sampler_view_destroy(
 
         pipe_resource_reference(&pview->texture, NULL);
         panfrost_bo_unreference(view->bo);
+        if (view->bifrost_descriptor)
+                ralloc_free(view->bifrost_descriptor);
         ralloc_free(view);
 }
 
@@ -1009,25 +1126,25 @@ panfrost_set_shader_buffers(
 
 static void
 panfrost_hint_afbc(
-                struct panfrost_screen *screen,
+                struct panfrost_device *device,
                 const struct pipe_framebuffer_state *fb)
 {
         /* AFBC implemenation incomplete; hide it */
-        if (!(pan_debug & PAN_DBG_AFBC)) return;
+        if (!(device->debug & PAN_DBG_AFBC)) return;
 
         /* Hint AFBC to the resources bound to each color buffer */
 
         for (unsigned i = 0; i < fb->nr_cbufs; ++i) {
                 struct pipe_surface *surf = fb->cbufs[i];
                 struct panfrost_resource *rsrc = pan_resource(surf->texture);
-                panfrost_resource_hint_layout(screen, rsrc, MALI_TEXTURE_AFBC, 1);
+                panfrost_resource_hint_layout(device, rsrc, MALI_TEXTURE_AFBC, 1);
         }
 
         /* Also hint it to the depth buffer */
 
         if (fb->zsbuf) {
                 struct panfrost_resource *rsrc = pan_resource(fb->zsbuf->texture);
-                panfrost_resource_hint_layout(screen, rsrc, MALI_TEXTURE_AFBC, 1);
+                panfrost_resource_hint_layout(device, rsrc, MALI_TEXTURE_AFBC, 1);
         }
 }
 
@@ -1037,10 +1154,17 @@ panfrost_set_framebuffer_state(struct pipe_context *pctx,
 {
         struct panfrost_context *ctx = pan_context(pctx);
 
-        panfrost_hint_afbc(pan_screen(pctx->screen), fb);
+        panfrost_hint_afbc(pan_device(pctx->screen), fb);
         util_copy_framebuffer_state(&ctx->pipe_framebuffer, fb);
         ctx->batch = NULL;
         panfrost_invalidate_frame(ctx);
+
+        /* We may need to generate a new variant if the fragment shader is
+         * keyed to the framebuffer format (due to EXT_framebuffer_fetch) */
+        struct panfrost_shader_variants *fs = ctx->shader[PIPE_SHADER_FRAGMENT];
+
+        if (fs && fs->variant_count && fs->variants[fs->active_variant].outputs_read)
+                ctx->base.bind_fs_state(&ctx->base, fs);
 }
 
 static void *
@@ -1083,8 +1207,19 @@ static void
 panfrost_set_sample_mask(struct pipe_context *pipe,
                          unsigned sample_mask)
 {
+        struct panfrost_context *ctx = pan_context(pipe);
+        ctx->sample_mask = sample_mask;
+}
+
+static void
+panfrost_set_min_samples(struct pipe_context *pipe,
+                         unsigned min_samples)
+{
+        struct panfrost_context *ctx = pan_context(pipe);
+        ctx->min_samples = min_samples;
 }
 
+
 static void
 panfrost_set_clip_state(struct pipe_context *pipe,
                         const struct pipe_clip_state *clip)
@@ -1191,7 +1326,7 @@ panfrost_begin_query(struct pipe_context *pipe, struct pipe_query *q)
                 /* Allocate a bo for the query results to be stored */
                 if (!query->bo) {
                         query->bo = panfrost_bo_create(
-                                        pan_screen(ctx->base.screen),
+                                        pan_device(ctx->base.screen),
                                         sizeof(unsigned), 0);
                 }
 
@@ -1211,7 +1346,7 @@ panfrost_begin_query(struct pipe_context *pipe, struct pipe_query *q)
                 break;
 
         default:
-                DBG("Skipping query %u\n", query->type);
+                /* TODO: timestamp queries, etc? */
                 break;
         }
 
@@ -1255,8 +1390,8 @@ panfrost_get_query_result(struct pipe_context *pipe,
         case PIPE_QUERY_OCCLUSION_COUNTER:
         case PIPE_QUERY_OCCLUSION_PREDICATE:
         case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
-                /* Flush first */
-                panfrost_flush_all_batches(ctx, true);
+                panfrost_flush_batches_accessing_bo(ctx, query->bo, PAN_BO_ACCESS_WRITE);
+                panfrost_bo_wait(query->bo, INT64_MAX, false);
 
                 /* Read back the query results */
                 unsigned *result = (unsigned *) query->bo->cpu;
@@ -1277,7 +1412,7 @@ panfrost_get_query_result(struct pipe_context *pipe,
                 break;
 
         default:
-                DBG("Skipped query get %u\n", query->type);
+                /* TODO: more queries */
                 break;
         }
 
@@ -1344,6 +1479,7 @@ panfrost_create_context(struct pipe_screen *screen, void *priv, unsigned flags)
 {
         struct panfrost_context *ctx = rzalloc(screen, struct panfrost_context);
         struct pipe_context *gallium = (struct pipe_context *) ctx;
+        struct panfrost_device *dev = pan_device(screen);
 
         gallium->screen = screen;
 
@@ -1354,6 +1490,7 @@ panfrost_create_context(struct pipe_screen *screen, void *priv, unsigned flags)
         gallium->flush = panfrost_flush;
         gallium->clear = panfrost_clear;
         gallium->draw_vbo = panfrost_draw_vbo;
+        gallium->texture_barrier = panfrost_texture_barrier;
 
         gallium->set_vertex_buffers = panfrost_set_vertex_buffers;
         gallium->set_constant_buffer = panfrost_set_constant_buffer;
@@ -1390,6 +1527,7 @@ panfrost_create_context(struct pipe_screen *screen, void *priv, unsigned flags)
         gallium->delete_depth_stencil_alpha_state = panfrost_delete_depth_stencil_state;
 
         gallium->set_sample_mask = panfrost_set_sample_mask;
+        gallium->set_min_samples = panfrost_set_min_samples;
 
         gallium->set_clip_state = panfrost_set_clip_state;
         gallium->set_viewport_states = panfrost_set_viewport_states;
@@ -1411,13 +1549,19 @@ panfrost_create_context(struct pipe_screen *screen, void *priv, unsigned flags)
         panfrost_blend_context_init(gallium);
         panfrost_compute_context_init(gallium);
 
-        /* XXX: leaks */
         gallium->stream_uploader = u_upload_create_default(gallium);
         gallium->const_uploader = gallium->stream_uploader;
         assert(gallium->stream_uploader);
 
-        /* Midgard supports ES modes, plus QUADS/QUAD_STRIPS/POLYGON */
-        ctx->draw_modes = (1 << (PIPE_PRIM_POLYGON + 1)) - 1;
+        /* All of our GPUs support ES mode. Midgard supports additionally
+         * QUADS/QUAD_STRIPS/POLYGON. Bifrost supports just QUADS. */
+
+        ctx->draw_modes = (1 << (PIPE_PRIM_QUADS + 1)) - 1;
+
+        if (!(dev->quirks & IS_BIFROST)) {
+                ctx->draw_modes |= (1 << PIPE_PRIM_QUAD_STRIP);
+                ctx->draw_modes |= (1 << PIPE_PRIM_POLYGON);
+        }
 
         ctx->primconvert = util_primconvert_create(gallium, ctx->draw_modes);
 
@@ -1432,5 +1576,13 @@ panfrost_create_context(struct pipe_screen *screen, void *priv, unsigned flags)
         panfrost_batch_init(ctx);
         panfrost_invalidate_frame(ctx);
 
+        if (!(dev->quirks & IS_BIFROST)) {
+                for (unsigned c = 0; c < PIPE_MAX_COLOR_BUFS; ++c)
+                        ctx->blit_blend.rt[c].shaders = _mesa_hash_table_u64_create(ctx);
+        }
+
+        /* By default mask everything on */
+        ctx->sample_mask = ~0;
+
         return gallium;
 }