panfrost: Merge AFBC slab with BO backing
[mesa.git] / src / gallium / drivers / panfrost / pan_resource.c
index a1285f21541f9ec20a38ddd8070ae2224201b86d..d8f1a9b521f4328771ef911d67cda7a89af3867d 100644 (file)
@@ -1,31 +1,33 @@
-/**************************************************************************
- *
- * Copyright 2008 VMware, Inc.
- * Copyright 2014 Broadcom
- * Copyright 2018 Alyssa Rosenzweig
- * All Rights Reserved.
+/*
+ * Copyright (C) 2008 VMware, Inc.
+ * Copyright (C) 2014 Broadcom
+ * Copyright (C) 2018-2019 Alyssa Rosenzweig
+ * Copyright (C) 2019 Collabora
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sub license, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
  *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
  *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
- * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
- * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
- * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
- * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * Authors (Collabora):
+ *   Tomeu Vizoso <tomeu.vizoso@collabora.com>
+ *   Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
  *
- **************************************************************************/
+ */
 
 #include <xf86drm.h>
 #include <fcntl.h>
 #include "util/u_surface.h"
 #include "util/u_transfer.h"
 #include "util/u_transfer_helper.h"
+#include "util/u_gen_mipmap.h"
 
 #include "pan_context.h"
 #include "pan_screen.h"
 #include "pan_resource.h"
-#include "pan_swizzle.h"
 #include "pan_util.h"
+#include "pan_tiling.h"
 
 static struct pipe_resource *
 panfrost_resource_from_handle(struct pipe_screen *pscreen,
@@ -56,7 +59,7 @@ panfrost_resource_from_handle(struct pipe_screen *pscreen,
 
         assert(whandle->type == WINSYS_HANDLE_TYPE_FD);
 
-        rsc = CALLOC_STRUCT(panfrost_resource);
+        rsc = rzalloc(pscreen, struct panfrost_resource);
         if (!rsc)
                 return NULL;
 
@@ -67,8 +70,9 @@ panfrost_resource_from_handle(struct pipe_screen *pscreen,
         pipe_reference_init(&prsc->reference, 1);
         prsc->screen = pscreen;
 
-       rsc->bo = screen->driver->import_bo(screen, whandle);
+       rsc->bo = panfrost_drm_import_bo(screen, whandle);
        rsc->bo->slices[0].stride = whandle->stride;
+        rsc->bo->slices[0].initialized = true;
 
        if (screen->ro) {
                rsc->scanout =
@@ -117,7 +121,9 @@ panfrost_resource_get_handle(struct pipe_screen *pscreen,
 
                         return TRUE;
                 } else
-                       return screen->driver->export_bo(screen, rsrc->bo->gem_handle, rsrc->bo->slices[0].stride, handle);
+                       return panfrost_drm_export_bo(screen, rsrc->bo->gem_handle,
+                                                      rsrc->bo->slices[0].stride,
+                                                      handle);
        }
 
        return FALSE;
@@ -129,19 +135,6 @@ panfrost_flush_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
         //DBG("TODO %s\n", __func__);
 }
 
-static void
-panfrost_blit(struct pipe_context *pipe,
-              const struct pipe_blit_info *info)
-{
-        if (util_try_blit_via_copy_region(pipe, info))
-                return;
-
-        /* TODO */
-        DBG("Unhandled blit.\n");
-
-        return;
-}
-
 static struct pipe_surface *
 panfrost_create_surface(struct pipe_context *pipe,
                         struct pipe_resource *pt,
@@ -149,7 +142,7 @@ panfrost_create_surface(struct pipe_context *pipe,
 {
         struct pipe_surface *ps = NULL;
 
-        ps = CALLOC_STRUCT(pipe_surface);
+        ps = rzalloc(pipe, struct pipe_surface);
 
         if (ps) {
                 pipe_reference_init(&ps->reference, 1);
@@ -184,7 +177,7 @@ panfrost_surface_destroy(struct pipe_context *pipe,
 {
         assert(surf->texture);
         pipe_resource_reference(&surf->texture, NULL);
-        free(surf);
+        ralloc_free(surf);
 }
 
 static void
@@ -192,42 +185,101 @@ panfrost_setup_slices(const struct pipe_resource *tmpl, struct panfrost_bo *bo)
 {
         unsigned width = tmpl->width0;
         unsigned height = tmpl->height0;
+        unsigned depth = tmpl->depth0;
         unsigned bytes_per_pixel = util_format_get_blocksize(tmpl->format);
 
+        assert(depth > 0);
+
+        /* Tiled operates blockwise; linear is packed. Also, anything
+         * we render to has to be tile-aligned. Maybe not strictly
+         * necessary, but we're not *that* pressed for memory and it
+         * makes code a lot simpler */
+
+        bool renderable = tmpl->bind &
+                (PIPE_BIND_RENDER_TARGET | PIPE_BIND_DEPTH_STENCIL);
+        bool afbc = bo->layout == PAN_AFBC;
+        bool tiled = bo->layout == PAN_TILED;
+        bool should_align = renderable || tiled;
+
+        /* We don't know how to specify a 2D stride for 3D textures */
+
+        bool can_align_stride =
+                tmpl->target != PIPE_TEXTURE_3D;
+
+        should_align &= can_align_stride;
+
         unsigned offset = 0;
+        unsigned size_2d = 0;
 
         for (unsigned l = 0; l <= tmpl->last_level; ++l) {
                 struct panfrost_slice *slice = &bo->slices[l];
 
                 unsigned effective_width = width;
                 unsigned effective_height = height;
+                unsigned effective_depth = depth;
 
-                /* Tiled operates blockwise; linear is packed */
-
-                if (bo->layout == PAN_TILED) {
+                if (should_align) {
                         effective_width = ALIGN(effective_width, 16);
                         effective_height = ALIGN(effective_height, 16);
+
+                        /* We don't need to align depth */
                 }
 
                 slice->offset = offset;
-                slice->stride = bytes_per_pixel * effective_width;
 
-                offset += slice->stride * effective_height;
+                /* Compute the would-be stride */
+                unsigned stride = bytes_per_pixel * effective_width;
+
+                /* ..but cache-line align it for performance */
+                if (can_align_stride && bo->layout == PAN_LINEAR)
+                        stride = ALIGN(stride, 64);
+
+                slice->stride = stride;
+
+                unsigned slice_one_size = slice->stride * effective_height;
+                unsigned slice_full_size = slice_one_size * effective_depth;
+
+                /* Report 2D size for 3D texturing */
+
+                if (l == 0)
+                        size_2d = slice_one_size;
+
+                /* Compute AFBC sizes if necessary */
+                if (afbc) {
+                        slice->header_size =
+                                panfrost_afbc_header_size(width, height);
+
+                        offset += slice->header_size;
+                }
+
+                offset += slice_full_size;
 
                 width = u_minify(width, 1);
                 height = u_minify(height, 1);
+                depth = u_minify(depth, 1);
         }
 
         assert(tmpl->array_size);
 
-        bo->cubemap_stride = ALIGN(offset, 64);
-        bo->size = ALIGN(bo->cubemap_stride * tmpl->array_size, 4096);
+        if (tmpl->target != PIPE_TEXTURE_3D) {
+                /* Arrays and cubemaps have the entire miptree duplicated */
+
+                bo->cubemap_stride = ALIGN(offset, 64);
+                bo->size = ALIGN(bo->cubemap_stride * tmpl->array_size, 4096);
+        } else {
+                /* 3D strides across the 2D layers */
+                assert(tmpl->array_size == 1);
+
+                bo->cubemap_stride = size_2d;
+                bo->size = ALIGN(offset, 4096);
+        }
 }
 
 static struct panfrost_bo *
 panfrost_create_bo(struct panfrost_screen *screen, const struct pipe_resource *template)
 {
-       struct panfrost_bo *bo = CALLOC_STRUCT(panfrost_bo);
+       struct panfrost_bo *bo = rzalloc(screen, struct panfrost_bo);
+        pipe_reference_init(&bo->reference, 1);
 
         /* Based on the usage, figure out what storing will be used. There are
          * various tradeoffs:
@@ -245,28 +297,28 @@ panfrost_create_bo(struct panfrost_screen *screen, const struct pipe_resource *t
          */
 
         /* Tiling textures is almost always faster, unless we only use it once */
-        bool should_tile = (template->usage != PIPE_USAGE_STREAM) && (template->bind & PIPE_BIND_SAMPLER_VIEW);
 
-        /* For unclear reasons, depth/stencil is faster linear than AFBC, so
-         * make sure it's linear */
+        bool is_texture = (template->bind & PIPE_BIND_SAMPLER_VIEW);
+        bool is_2d = template->depth0 == 1 && template->array_size == 1;
+        bool is_streaming = (template->usage != PIPE_USAGE_STREAM);
+
+        bool should_tile = is_streaming && is_texture && is_2d;
 
-        if (template->bind & PIPE_BIND_DEPTH_STENCIL)
-                should_tile = false;
+        /* Depth/stencil can't be tiled, only linear or AFBC */
+        should_tile &= !(template->bind & PIPE_BIND_DEPTH_STENCIL);
 
         /* Set the layout appropriately */
         bo->layout = should_tile ? PAN_TILED : PAN_LINEAR;
 
         panfrost_setup_slices(template, bo);
 
-        if (bo->layout == PAN_TILED || bo->layout == PAN_LINEAR) {
-                struct panfrost_memory mem;
+        struct panfrost_memory mem;
 
-                screen->driver->allocate_slab(screen, &mem, bo->size / 4096, true, 0, 0, 0);
+        panfrost_drm_allocate_slab(screen, &mem, bo->size / 4096, true, 0, 0, 0);
 
-                bo->cpu = mem.cpu;
-                bo->gpu = mem.gpu;
-                bo->gem_handle = mem.gem_handle;
-        }
+        bo->cpu = mem.cpu;
+        bo->gpu = mem.gpu;
+        bo->gem_handle = mem.gem_handle;
 
         return bo;
 }
@@ -275,7 +327,7 @@ static struct pipe_resource *
 panfrost_resource_create(struct pipe_screen *screen,
                          const struct pipe_resource *template)
 {
-        struct panfrost_resource *so = CALLOC_STRUCT(panfrost_resource);
+        struct panfrost_resource *so = rzalloc(screen, struct panfrost_resource);
         struct panfrost_screen *pscreen = (struct panfrost_screen *) screen;
 
         so->base = *template;
@@ -291,12 +343,15 @@ panfrost_resource_create(struct pipe_screen *screen,
                 case PIPE_TEXTURE_3D:
                 case PIPE_TEXTURE_CUBE:
                 case PIPE_TEXTURE_RECT:
+                case PIPE_TEXTURE_2D_ARRAY:
                         break;
                 default:
                         DBG("Unknown texture target %d\n", template->target);
                         assert(0);
         }
 
+        util_range_init(&so->valid_buffer_range);
+
         if (template->bind & PIPE_BIND_DISPLAY_TARGET ||
             template->bind & PIPE_BIND_SCANOUT ||
             template->bind & PIPE_BIND_SHARED) {
@@ -328,12 +383,9 @@ panfrost_resource_create(struct pipe_screen *screen,
 }
 
 static void
-panfrost_destroy_bo(struct panfrost_screen *screen, struct panfrost_bo *pbo)
+panfrost_destroy_bo(struct panfrost_screen *screen, struct panfrost_bo *bo)
 {
-       struct panfrost_bo *bo = (struct panfrost_bo *)pbo;
-
-        if ((bo->layout == PAN_LINEAR || bo->layout == PAN_TILED) &&
-                        !bo->imported) {
+        if (!bo->imported) {
                 struct panfrost_memory mem = {
                         .cpu = bo->cpu,
                         .gpu = bo->gpu,
@@ -341,21 +393,40 @@ panfrost_destroy_bo(struct panfrost_screen *screen, struct panfrost_bo *pbo)
                         .gem_handle = bo->gem_handle,
                 };
 
-                screen->driver->free_slab(screen, &mem);
-        }
-
-        if (bo->layout == PAN_AFBC) {
-                /* TODO */
-                DBG("--leaking afbc (%d bytes)--\n", bo->afbc_metadata_size);
+                panfrost_drm_free_slab(screen, &mem);
         }
 
         if (bo->has_checksum) {
-                /* TODO */
-                DBG("--leaking checksum (%zd bytes)--\n", bo->checksum_slab.size);
+                struct panfrost_memory mem = {
+                        .cpu = bo->checksum_slab.cpu,
+                        .gpu = bo->checksum_slab.gpu,
+                        .size = bo->checksum_slab.size,
+                        .gem_handle = bo->checksum_slab.gem_handle,
+                };
+
+                panfrost_drm_free_slab(screen, &mem);
         }
 
         if (bo->imported) {
-                screen->driver->free_imported_bo(screen, bo);
+                panfrost_drm_free_imported_bo(screen, bo);
+        }
+
+        ralloc_free(bo);
+}
+
+void
+panfrost_bo_reference(struct panfrost_bo *bo)
+{
+        pipe_reference(NULL, &bo->reference);
+}
+
+void
+panfrost_bo_unreference(struct pipe_screen *screen, struct panfrost_bo *bo)
+{
+        /* When the reference count goes to zero, we need to cleanup */
+
+        if (pipe_reference(&bo->reference, NULL)) {
+                panfrost_destroy_bo(pan_screen(screen), bo);
         }
 }
 
@@ -370,9 +441,10 @@ panfrost_resource_destroy(struct pipe_screen *screen,
                renderonly_scanout_destroy(rsrc->scanout, pscreen->ro);
 
        if (rsrc->bo)
-               panfrost_destroy_bo(pscreen, rsrc->bo);
+                panfrost_bo_unreference(screen, rsrc->bo);
 
-       FREE(rsrc);
+        util_range_destroy(&rsrc->valid_buffer_range);
+       ralloc_free(rsrc);
 }
 
 static void *
@@ -384,40 +456,92 @@ panfrost_transfer_map(struct pipe_context *pctx,
                       struct pipe_transfer **out_transfer)
 {
         int bytes_per_pixel = util_format_get_blocksize(resource->format);
-        struct panfrost_bo *bo = pan_resource(resource)->bo;
+        struct panfrost_resource *rsrc = pan_resource(resource);
+        struct panfrost_bo *bo = rsrc->bo;
 
-        struct panfrost_gtransfer *transfer = CALLOC_STRUCT(panfrost_gtransfer);
+        struct panfrost_gtransfer *transfer = rzalloc(pctx, struct panfrost_gtransfer);
         transfer->base.level = level;
         transfer->base.usage = usage;
         transfer->base.box = *box;
-        transfer->base.stride = bo->slices[level].stride;
-        transfer->base.layer_stride = bytes_per_pixel * resource->width0; /* TODO: Cubemaps */
 
         pipe_resource_reference(&transfer->base.resource, resource);
 
         *out_transfer = &transfer->base;
 
-        if (resource->bind & PIPE_BIND_DISPLAY_TARGET ||
-            resource->bind & PIPE_BIND_SCANOUT ||
-            resource->bind & PIPE_BIND_SHARED) {
-                /* Mipmapped readpixels?! */
-                assert(level == 0);
+        /* Check if we're bound for rendering and this is a read pixels. If so,
+         * we need to flush */
+
+        struct panfrost_context *ctx = pan_context(pctx);
+        struct pipe_framebuffer_state *fb = &ctx->pipe_framebuffer;
+
+        bool is_bound = false;
 
-                /* Force a flush -- kill the pipeline */
+        for (unsigned c = 0; c < fb->nr_cbufs; ++c) {
+                is_bound |= fb->cbufs[c]->texture == resource;
+        }
+
+        if (is_bound && (usage & PIPE_TRANSFER_READ)) {
+                assert(level == 0);
                 panfrost_flush(pctx, NULL, PIPE_FLUSH_END_OF_FRAME);
         }
 
+        /* TODO: Respect usage flags */
+
+        if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) {
+                /* TODO: reallocate */
+                //printf("debug: Missed reallocate\n");
+        } else if ((usage & PIPE_TRANSFER_WRITE)
+                        && resource->target == PIPE_BUFFER
+                        && !util_ranges_intersect(&rsrc->valid_buffer_range, box->x, box->x + box->width)) {
+                /* No flush for writes to uninitialized */
+        } else if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
+                if (usage & PIPE_TRANSFER_WRITE) {
+                        /* STUB: flush reading */
+                        //printf("debug: missed reading flush %d\n", resource->target);
+                } else if (usage & PIPE_TRANSFER_READ) {
+                        /* STUB: flush writing */
+                        //printf("debug: missed writing flush %d (%d-%d)\n", resource->target, box->x, box->x + box->width);
+                } else {
+                        /* Why are you even mapping?! */
+                }
+        }
+
         if (bo->layout != PAN_LINEAR) {
                 /* Non-linear resources need to be indirectly mapped */
 
                 if (usage & PIPE_TRANSFER_MAP_DIRECTLY)
                         return NULL;
 
-                /* TODO: Reads */
-                transfer->map = malloc(ALIGN(box->width, 16) * ALIGN(box->height, 16) * bytes_per_pixel);
+                transfer->base.stride = box->width * bytes_per_pixel;
+                transfer->base.layer_stride = transfer->base.stride * box->height;
+                transfer->map = rzalloc_size(transfer, transfer->base.layer_stride * box->depth);
+                assert(box->depth == 1);
+
+                if ((usage & PIPE_TRANSFER_READ) && bo->slices[level].initialized) {
+                        if (bo->layout == PAN_AFBC) {
+                                DBG("Unimplemented: reads from AFBC");
+                        } else if (bo->layout == PAN_TILED) {
+                                panfrost_load_tiled_image(
+                                                transfer->map,
+                                                bo->cpu + bo->slices[level].offset,
+                                                box,
+                                                transfer->base.stride,
+                                                bo->slices[level].stride,
+                                                util_format_get_blocksize(resource->format));
+                        }
+                }
 
                 return transfer->map;
         } else {
+                transfer->base.stride = bo->slices[level].stride;
+                transfer->base.layer_stride = bo->cubemap_stride;
+
+                /* By mapping direct-write, we're implicitly already
+                 * initialized (maybe), so be conservative */
+
+                if ((usage & PIPE_TRANSFER_WRITE) && (usage & PIPE_TRANSFER_MAP_DIRECTLY))
+                        bo->slices[level].initialized = true;
+
                 return bo->cpu
                         + bo->slices[level].offset
                         + transfer->base.box.z * bo->cubemap_stride
@@ -427,70 +551,68 @@ panfrost_transfer_map(struct pipe_context *pctx,
 }
 
 static void
-panfrost_tile_texture(struct panfrost_screen *screen, struct panfrost_resource *rsrc, struct panfrost_gtransfer *trans)
+panfrost_transfer_unmap(struct pipe_context *pctx,
+                        struct pipe_transfer *transfer)
 {
-       struct panfrost_bo *bo = (struct panfrost_bo *)rsrc->bo;
+        /* Gallium expects writeback here, so we tile */
 
-        unsigned level = trans->base.level;
-
-        assert(!trans->base.box.z);
-
-        panfrost_texture_swizzle(
-                        trans->base.box.x,
-                        trans->base.box.y,
-                        trans->base.box.width,
-                        trans->base.box.height,
-                        util_format_get_blocksize(rsrc->base.format),
-                        bo->slices[level].stride,
-                        trans->map,
-                        bo->cpu + bo->slices[level].offset);
-}
-
-static void
-panfrost_unmap_bo(struct panfrost_context *ctx,
-                         struct pipe_transfer *transfer)
-{
         struct panfrost_gtransfer *trans = pan_transfer(transfer);
-       struct panfrost_bo *bo = (struct panfrost_bo *)pan_resource(transfer->resource)->bo;
+        struct panfrost_resource *prsrc = (struct panfrost_resource *) transfer->resource;
+
+        if (trans->map) {
+                struct panfrost_bo *bo = prsrc->bo;
 
-        if (transfer->usage & PIPE_TRANSFER_WRITE) {
-                if (transfer->resource->target == PIPE_TEXTURE_2D) {
-                        struct panfrost_resource *prsrc = (struct panfrost_resource *) transfer->resource;
+                if (transfer->usage & PIPE_TRANSFER_WRITE) {
+                        unsigned level = transfer->level;
+                        bo->slices[level].initialized = true;
 
-                        /* Gallium thinks writeback happens here; instead, this is our cue to tile */
                         if (bo->layout == PAN_AFBC) {
-                                DBG("Warning: writes to afbc surface can't possibly work out well for you...\n");
+                                DBG("Unimplemented: writes to AFBC\n");
                         } else if (bo->layout == PAN_TILED) {
-                                struct pipe_context *gallium = (struct pipe_context *) ctx;
-                                struct panfrost_screen *screen = pan_screen(gallium->screen);
-                                panfrost_tile_texture(screen, prsrc, trans);
+                                assert(transfer->box.depth == 1);
+
+                                panfrost_store_tiled_image(
+                                                bo->cpu + bo->slices[level].offset,
+                                                trans->map,
+                                                &transfer->box,
+                                                bo->slices[level].stride,
+                                                transfer->stride,
+                                                util_format_get_blocksize(prsrc->base.format));
                         }
                 }
         }
 
-        free(trans->map);
-}
-
-static void
-panfrost_transfer_unmap(struct pipe_context *pctx,
-                        struct pipe_transfer *transfer)
-{
-        struct panfrost_context *ctx = pan_context(pctx);
 
-       panfrost_unmap_bo(ctx, transfer);
+       util_range_add(&prsrc->valid_buffer_range,
+                        transfer->box.x,
+                        transfer->box.x + transfer->box.width);
 
         /* Derefence the resource */
         pipe_resource_reference(&transfer->resource, NULL);
 
-        /* Transfer itself is CALLOCed at the moment */
-        free(transfer);
+        /* Transfer itself is RALLOCed at the moment */
+        ralloc_free(transfer);
+}
+
+static void
+panfrost_transfer_flush_region(struct pipe_context *pctx,
+               struct pipe_transfer *transfer,
+               const struct pipe_box *box)
+{
+       struct panfrost_resource *rsc = pan_resource(transfer->resource);
+
+       if (transfer->resource->target == PIPE_BUFFER) {
+               util_range_add(&rsc->valid_buffer_range,
+                                          transfer->box.x + box->x,
+                                          transfer->box.x + box->x + box->width);
+        }
 }
 
 static struct pb_slab *
 panfrost_slab_alloc(void *priv, unsigned heap, unsigned entry_size, unsigned group_index)
 {
         struct panfrost_screen *screen = (struct panfrost_screen *) priv;
-        struct panfrost_memory *mem = CALLOC_STRUCT(panfrost_memory);
+        struct panfrost_memory *mem = rzalloc(screen, struct panfrost_memory);
 
         size_t slab_size = (1 << (MAX_SLAB_ENTRY_SIZE + 1));
 
@@ -500,7 +622,7 @@ panfrost_slab_alloc(void *priv, unsigned heap, unsigned entry_size, unsigned gro
         LIST_INITHEAD(&mem->slab.free);
         for (unsigned i = 0; i < mem->slab.num_entries; ++i) {
                 /* Create a slab entry */
-                struct panfrost_memory_entry *entry = CALLOC_STRUCT(panfrost_memory_entry);
+                struct panfrost_memory_entry *entry = rzalloc(mem, struct panfrost_memory_entry);
                 entry->offset = entry_size * i;
 
                 entry->base.slab = &mem->slab;
@@ -512,7 +634,7 @@ panfrost_slab_alloc(void *priv, unsigned heap, unsigned entry_size, unsigned gro
         /* Actually allocate the memory from kernel-space. Mapped, same_va, no
          * special flags */
 
-        screen->driver->allocate_slab(screen, mem, slab_size / 4096, true, 0, 0, 0);
+        panfrost_drm_allocate_slab(screen, mem, slab_size / 4096, true, 0, 0, 0);
 
         return &mem->slab;
 }
@@ -530,7 +652,8 @@ panfrost_slab_free(void *priv, struct pb_slab *slab)
         struct panfrost_memory *mem = (struct panfrost_memory *) slab;
         struct panfrost_screen *screen = (struct panfrost_screen *) priv;
 
-        screen->driver->free_slab(screen, mem);
+        panfrost_drm_free_slab(screen, mem);
+        ralloc_free(mem);
 }
 
 static void
@@ -545,6 +668,56 @@ panfrost_resource_get_internal_format(struct pipe_resource *prsrc)
         return prsrc->format;
 }
 
+static boolean
+panfrost_generate_mipmap(
+                struct pipe_context *pctx,
+                struct pipe_resource *prsrc,
+                enum pipe_format format,
+                unsigned base_level,
+                unsigned last_level,
+                unsigned first_layer,
+                unsigned last_layer)
+{
+        struct panfrost_context *ctx = pan_context(pctx);
+        struct panfrost_resource *rsrc = pan_resource(prsrc);
+
+        /* Generating a mipmap invalidates the written levels, so make that
+         * explicit so we don't try to wallpaper them back and end up with
+         * u_blitter recursion */
+
+        assert(rsrc->bo);
+        for (unsigned l = base_level + 1; l <= last_level; ++l)
+                rsrc->bo->slices[l].initialized = false;
+
+        /* Beyond that, we just delegate the hard stuff. We're careful to
+         * include flushes on both ends to make sure the data is really valid.
+         * We could be doing a lot better perf-wise, especially once we have
+         * reorder-type optimizations in place. But for now prioritize
+         * correctness. */
+
+        struct panfrost_job *job = panfrost_get_job_for_fbo(ctx);
+        bool has_draws = job->last_job.gpu;
+
+        if (has_draws)
+                panfrost_flush(pctx, NULL, PIPE_FLUSH_END_OF_FRAME);
+
+        /* We've flushed the original buffer if needed, now trigger a blit */
+
+        bool blit_res = util_gen_mipmap(
+                        pctx, prsrc, format, 
+                        base_level, last_level,
+                        first_layer, last_layer,
+                        PIPE_TEX_FILTER_LINEAR);
+
+        /* If the blit was successful, flush once more. If it wasn't, well, let
+         * the state tracker deal with it. */
+
+        if (blit_res)
+                panfrost_flush(pctx, NULL, PIPE_FLUSH_END_OF_FRAME);
+
+        return blit_res;
+}
+
 static void
 panfrost_resource_set_stencil(struct pipe_resource *prsrc,
                               struct pipe_resource *stencil)
@@ -563,7 +736,7 @@ static const struct u_transfer_vtbl transfer_vtbl = {
         .resource_destroy         = panfrost_resource_destroy,
         .transfer_map             = panfrost_transfer_map,
         .transfer_unmap           = panfrost_transfer_unmap,
-        .transfer_flush_region    = u_default_transfer_flush_region,
+        .transfer_flush_region    = panfrost_transfer_flush_region,
         .get_internal_format      = panfrost_resource_get_internal_format,
         .set_stencil              = panfrost_resource_set_stencil,
         .get_stencil              = panfrost_resource_get_stencil,
@@ -595,6 +768,12 @@ panfrost_resource_screen_init(struct panfrost_screen *pscreen)
                         panfrost_slab_free);
 }
 
+void
+panfrost_resource_screen_deinit(struct panfrost_screen *pscreen)
+{
+        pb_slabs_deinit(&pscreen->slabs);
+}
+
 void
 panfrost_resource_context_init(struct pipe_context *pctx)
 {
@@ -606,6 +785,7 @@ panfrost_resource_context_init(struct pipe_context *pctx)
         pctx->surface_destroy = panfrost_surface_destroy;
         pctx->resource_copy_region = util_resource_copy_region;
         pctx->blit = panfrost_blit;
+        pctx->generate_mipmap = panfrost_generate_mipmap;
         pctx->flush_resource = panfrost_flush_resource;
         pctx->invalidate_resource = panfrost_invalidate_resource;
         pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;