panfrost/mfbd: Implement linear depth buffers
[mesa.git] / src / gallium / drivers / panfrost / pan_resource.c
1 /**************************************************************************
2 *
3 * Copyright 2008 VMware, Inc.
4 * Copyright 2014 Broadcom
5 * Copyright 2018 Alyssa Rosenzweig
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
23 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
24 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
25 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
26 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
28 **************************************************************************/
29
30 #include <xf86drm.h>
31 #include <fcntl.h>
32 #include "drm-uapi/drm_fourcc.h"
33
34 #include "state_tracker/winsys_handle.h"
35 #include "util/u_format.h"
36 #include "util/u_memory.h"
37 #include "util/u_surface.h"
38 #include "util/u_transfer.h"
39 #include "util/u_transfer_helper.h"
40
41 #include "pan_context.h"
42 #include "pan_screen.h"
43 #include "pan_resource.h"
44 #include "pan_swizzle.h"
45 #include "pan_util.h"
46
47 static struct pipe_resource *
48 panfrost_resource_from_handle(struct pipe_screen *pscreen,
49 const struct pipe_resource *templat,
50 struct winsys_handle *whandle,
51 unsigned usage)
52 {
53 struct panfrost_screen *screen = pan_screen(pscreen);
54 struct panfrost_resource *rsc;
55 struct pipe_resource *prsc;
56
57 assert(whandle->type == WINSYS_HANDLE_TYPE_FD);
58
59 rsc = CALLOC_STRUCT(panfrost_resource);
60 if (!rsc)
61 return NULL;
62
63 prsc = &rsc->base;
64
65 *prsc = *templat;
66
67 pipe_reference_init(&prsc->reference, 1);
68 prsc->screen = pscreen;
69
70 rsc->bo = screen->driver->import_bo(screen, whandle);
71
72 if (screen->ro) {
73 rsc->scanout =
74 renderonly_create_gpu_import_for_resource(prsc, screen->ro, NULL);
75 /* failure is expected in some cases.. */
76 }
77
78 return prsc;
79 }
80
81 static boolean
82 panfrost_resource_get_handle(struct pipe_screen *pscreen,
83 struct pipe_context *ctx,
84 struct pipe_resource *pt,
85 struct winsys_handle *handle,
86 unsigned usage)
87 {
88 struct panfrost_screen *screen = pan_screen(pscreen);
89 struct panfrost_resource *rsrc = (struct panfrost_resource *) pt;
90 struct renderonly_scanout *scanout = rsrc->scanout;
91 int bytes_per_pixel = util_format_get_blocksize(rsrc->base.format);
92 int stride = bytes_per_pixel * rsrc->base.width0; /* TODO: Alignment? */
93
94 handle->stride = stride;
95 handle->modifier = DRM_FORMAT_MOD_INVALID;
96
97 if (handle->type == WINSYS_HANDLE_TYPE_SHARED) {
98 return FALSE;
99 } else if (handle->type == WINSYS_HANDLE_TYPE_KMS) {
100 if (renderonly_get_handle(scanout, handle))
101 return TRUE;
102
103 handle->handle = rsrc->bo->gem_handle;
104 return TRUE;
105 } else if (handle->type == WINSYS_HANDLE_TYPE_FD) {
106 if (scanout) {
107 struct drm_prime_handle args = {
108 .handle = scanout->handle,
109 .flags = DRM_CLOEXEC,
110 };
111
112 int ret = drmIoctl(screen->ro->kms_fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &args);
113 if (ret == -1)
114 return FALSE;
115
116 handle->handle = args.fd;
117
118 return TRUE;
119 } else
120 return screen->driver->export_bo(screen, rsrc->bo->gem_handle, handle);
121 }
122
123 return FALSE;
124 }
125
126 static void
127 panfrost_flush_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
128 {
129 //DBG("TODO %s\n", __func__);
130 }
131
132 static void
133 panfrost_blit(struct pipe_context *pipe,
134 const struct pipe_blit_info *info)
135 {
136 /* STUB */
137 DBG("Skipping blit XXX\n");
138 return;
139 }
140
141 static struct pipe_surface *
142 panfrost_create_surface(struct pipe_context *pipe,
143 struct pipe_resource *pt,
144 const struct pipe_surface *surf_tmpl)
145 {
146 struct pipe_surface *ps = NULL;
147
148 ps = CALLOC_STRUCT(pipe_surface);
149
150 if (ps) {
151 pipe_reference_init(&ps->reference, 1);
152 pipe_resource_reference(&ps->texture, pt);
153 ps->context = pipe;
154 ps->format = surf_tmpl->format;
155
156 if (pt->target != PIPE_BUFFER) {
157 assert(surf_tmpl->u.tex.level <= pt->last_level);
158 ps->width = u_minify(pt->width0, surf_tmpl->u.tex.level);
159 ps->height = u_minify(pt->height0, surf_tmpl->u.tex.level);
160 ps->u.tex.level = surf_tmpl->u.tex.level;
161 ps->u.tex.first_layer = surf_tmpl->u.tex.first_layer;
162 ps->u.tex.last_layer = surf_tmpl->u.tex.last_layer;
163 } else {
164 /* setting width as number of elements should get us correct renderbuffer width */
165 ps->width = surf_tmpl->u.buf.last_element - surf_tmpl->u.buf.first_element + 1;
166 ps->height = pt->height0;
167 ps->u.buf.first_element = surf_tmpl->u.buf.first_element;
168 ps->u.buf.last_element = surf_tmpl->u.buf.last_element;
169 assert(ps->u.buf.first_element <= ps->u.buf.last_element);
170 assert(ps->u.buf.last_element < ps->width);
171 }
172 }
173
174 return ps;
175 }
176
177 static void
178 panfrost_surface_destroy(struct pipe_context *pipe,
179 struct pipe_surface *surf)
180 {
181 assert(surf->texture);
182 pipe_resource_reference(&surf->texture, NULL);
183 free(surf);
184 }
185
186 static struct panfrost_bo *
187 panfrost_create_bo(struct panfrost_screen *screen, const struct pipe_resource *template)
188 {
189 struct panfrost_bo *bo = CALLOC_STRUCT(panfrost_bo);
190
191 /* Calculate the size of the bo */
192
193 int bytes_per_pixel = util_format_get_blocksize(template->format);
194 int stride = bytes_per_pixel * template->width0; /* TODO: Alignment? */
195 size_t sz = stride;
196
197 if (template->height0) sz *= template->height0;
198 if (template->depth0) sz *= template->depth0;
199
200 /* Based on the usage, figure out what storing will be used. There are
201 * various tradeoffs:
202 *
203 * Linear: the basic format, bad for memory bandwidth, bad for cache
204 * use. Zero-copy, though. Renderable.
205 *
206 * Tiled: Not compressed, but cache-optimized. Expensive to write into
207 * (due to software tiling), but cheap to sample from. Ideal for most
208 * textures.
209 *
210 * AFBC: Compressed and renderable (so always desirable for non-scanout
211 * rendertargets). Cheap to sample from. The format is black box, so we
212 * can't read/write from software.
213 */
214
215 /* Tiling textures is almost always faster, unless we only use it once */
216 bool should_tile = (template->usage != PIPE_USAGE_STREAM) && (template->bind & PIPE_BIND_SAMPLER_VIEW);
217
218 /* Set the layout appropriately */
219 bo->layout = should_tile ? PAN_TILED : PAN_LINEAR;
220
221 if (bo->layout == PAN_TILED) {
222 /* For tiled, we don't map directly, so just malloc any old buffer */
223
224 for (int l = 0; l < (template->last_level + 1); ++l) {
225 bo->cpu[l] = malloc(sz);
226 bo->size[l] = sz;
227 sz >>= 2;
228 }
229 } else {
230 /* For a linear resource, allocate a block of memory from
231 * kernel space */
232
233 struct panfrost_memory mem;
234
235 bo->size[0] = ALIGN(sz, 4096);
236 screen->driver->allocate_slab(screen, &mem, bo->size[0] / 4096, true, 0, 0, 0);
237
238 bo->cpu[0] = mem.cpu;
239 bo->gpu[0] = mem.gpu;
240 bo->gem_handle = mem.gem_handle;
241
242 /* TODO: Mipmap */
243 }
244
245 return bo;
246 }
247
248 static struct pipe_resource *
249 panfrost_resource_create(struct pipe_screen *screen,
250 const struct pipe_resource *template)
251 {
252 struct panfrost_resource *so = CALLOC_STRUCT(panfrost_resource);
253 struct panfrost_screen *pscreen = (struct panfrost_screen *) screen;
254
255 so->base = *template;
256 so->base.screen = screen;
257
258 pipe_reference_init(&so->base.reference, 1);
259
260 /* Make sure we're familiar */
261 switch (template->target) {
262 case PIPE_BUFFER:
263 case PIPE_TEXTURE_1D:
264 case PIPE_TEXTURE_2D:
265 case PIPE_TEXTURE_3D:
266 case PIPE_TEXTURE_RECT:
267 break;
268 default:
269 DBG("Unknown texture target %d\n", template->target);
270 assert(0);
271 }
272
273 if (template->bind & PIPE_BIND_DISPLAY_TARGET ||
274 template->bind & PIPE_BIND_SCANOUT ||
275 template->bind & PIPE_BIND_SHARED) {
276 struct pipe_resource scanout_templat = *template;
277 struct renderonly_scanout *scanout;
278 struct winsys_handle handle;
279
280 /* TODO: align width0 and height0? */
281
282 scanout = renderonly_scanout_for_resource(&scanout_templat,
283 pscreen->ro, &handle);
284 if (!scanout)
285 return NULL;
286
287 assert(handle.type == WINSYS_HANDLE_TYPE_FD);
288 /* TODO: handle modifiers? */
289 so = pan_resource(screen->resource_from_handle(screen, template,
290 &handle,
291 PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE));
292 close(handle.handle);
293 if (!so)
294 return NULL;
295
296 so->scanout = scanout;
297 pscreen->display_target = so;
298 } else {
299 so->bo = panfrost_create_bo(pscreen, template);
300 }
301
302 return (struct pipe_resource *)so;
303 }
304
305 static void
306 panfrost_destroy_bo(struct panfrost_screen *screen, struct panfrost_bo *pbo)
307 {
308 struct panfrost_bo *bo = (struct panfrost_bo *)pbo;
309
310 if (bo->layout == PAN_LINEAR && !bo->imported) {
311 /* Construct a memory object for all mip levels */
312
313 struct panfrost_memory mem = {
314 .cpu = bo->cpu[0],
315 .gpu = bo->gpu[0],
316 .size = bo->size[0],
317 .gem_handle = bo->gem_handle,
318 };
319
320 screen->driver->free_slab(screen, &mem);
321 }
322
323 if (bo->layout == PAN_TILED) {
324 /* Tiled has a malloc'd CPU, so just plain ol' free needed */
325
326 for (int l = 0; l < MAX_MIP_LEVELS; ++l) {
327 free(bo->cpu[l]);
328 }
329 }
330
331 if (bo->layout == PAN_AFBC) {
332 /* TODO */
333 DBG("--leaking afbc (%d bytes)--\n", bo->afbc_metadata_size);
334 }
335
336 if (bo->has_checksum) {
337 /* TODO */
338 DBG("--leaking checksum (%zd bytes)--\n", bo->checksum_slab.size);
339 }
340
341 if (bo->imported) {
342 screen->driver->free_imported_bo(screen, bo);
343 }
344 }
345
346 static void
347 panfrost_resource_destroy(struct pipe_screen *screen,
348 struct pipe_resource *pt)
349 {
350 struct panfrost_screen *pscreen = pan_screen(screen);
351 struct panfrost_resource *rsrc = (struct panfrost_resource *) pt;
352
353 if (rsrc->scanout)
354 renderonly_scanout_destroy(rsrc->scanout, pscreen->ro);
355
356 if (rsrc->bo)
357 panfrost_destroy_bo(pscreen, rsrc->bo);
358
359 FREE(rsrc);
360 }
361
362 static uint8_t *
363 panfrost_map_bo(struct panfrost_context *ctx, struct pipe_transfer *transfer)
364 {
365 struct panfrost_bo *bo = (struct panfrost_bo *)pan_resource(transfer->resource)->bo;
366
367 /* If non-zero level, it's a mipmapped resource and needs to be treated as such */
368 bo->is_mipmap |= transfer->level;
369
370 if (transfer->usage & PIPE_TRANSFER_MAP_DIRECTLY && bo->layout != PAN_LINEAR) {
371 /* We can only directly map linear resources */
372 return NULL;
373 }
374
375 if (transfer->resource->bind & PIPE_BIND_DEPTH_STENCIL) {
376 /* Mipmapped readpixels?! */
377 assert(transfer->level == 0);
378
379 /* Set the CPU mapping to that of the depth/stencil buffer in memory, untiled */
380 bo->cpu[transfer->level] = ctx->depth_stencil_buffer.cpu;
381 }
382
383 return bo->cpu[transfer->level];
384 }
385
386 static void *
387 panfrost_transfer_map(struct pipe_context *pctx,
388 struct pipe_resource *resource,
389 unsigned level,
390 unsigned usage, /* a combination of PIPE_TRANSFER_x */
391 const struct pipe_box *box,
392 struct pipe_transfer **out_transfer)
393 {
394 struct panfrost_context *ctx = pan_context(pctx);
395 int bytes_per_pixel = util_format_get_blocksize(resource->format);
396 int stride = bytes_per_pixel * resource->width0; /* TODO: Alignment? */
397 uint8_t *cpu;
398
399 struct pipe_transfer *transfer = CALLOC_STRUCT(pipe_transfer);
400 transfer->level = level;
401 transfer->usage = usage;
402 transfer->box = *box;
403 transfer->stride = stride;
404 assert(!transfer->box.z);
405
406 pipe_resource_reference(&transfer->resource, resource);
407
408 *out_transfer = transfer;
409
410 if (resource->bind & PIPE_BIND_DISPLAY_TARGET ||
411 resource->bind & PIPE_BIND_SCANOUT ||
412 resource->bind & PIPE_BIND_SHARED) {
413 /* Mipmapped readpixels?! */
414 assert(level == 0);
415
416 /* Force a flush -- kill the pipeline */
417 panfrost_flush(pctx, NULL, PIPE_FLUSH_END_OF_FRAME);
418 }
419
420 cpu = panfrost_map_bo(ctx, transfer);
421 if (cpu == NULL)
422 return NULL;
423
424 return cpu + transfer->box.x * bytes_per_pixel + transfer->box.y * stride;
425 }
426
427 static void
428 panfrost_tile_texture(struct panfrost_screen *screen, struct panfrost_resource *rsrc, int level)
429 {
430 struct panfrost_bo *bo = (struct panfrost_bo *)rsrc->bo;
431 int bytes_per_pixel = util_format_get_blocksize(rsrc->base.format);
432 int stride = bytes_per_pixel * rsrc->base.width0; /* TODO: Alignment? */
433
434 int width = rsrc->base.width0 >> level;
435 int height = rsrc->base.height0 >> level;
436
437 /* Estimate swizzled bitmap size. Slight overestimates are fine.
438 * Underestimates will result in memory corruption or worse. */
439
440 int swizzled_sz = panfrost_swizzled_size(width, height, bytes_per_pixel);
441
442 /* Save the entry. But if there was already an entry here (from a
443 * previous upload of the resource), free that one so we don't leak */
444
445 if (bo->entry[level] != NULL) {
446 bo->entry[level]->freed = true;
447 pb_slab_free(&screen->slabs, &bo->entry[level]->base);
448 }
449
450 /* Allocate the transfer given that known size but do not copy */
451 struct pb_slab_entry *entry = pb_slab_alloc(&screen->slabs, swizzled_sz, HEAP_TEXTURE);
452 struct panfrost_memory_entry *p_entry = (struct panfrost_memory_entry *) entry;
453 struct panfrost_memory *backing = (struct panfrost_memory *) entry->slab;
454 uint8_t *swizzled = backing->cpu + p_entry->offset;
455
456 bo->entry[level] = p_entry;
457 bo->gpu[level] = backing->gpu + p_entry->offset;
458
459 /* Run actual texture swizzle, writing directly to the mapped
460 * GPU chunk we allocated */
461
462 panfrost_texture_swizzle(width, height, bytes_per_pixel, stride, bo->cpu[level], swizzled);
463 }
464
465 static void
466 panfrost_unmap_bo(struct panfrost_context *ctx,
467 struct pipe_transfer *transfer)
468 {
469 struct panfrost_bo *bo = (struct panfrost_bo *)pan_resource(transfer->resource)->bo;
470
471 if (transfer->usage & PIPE_TRANSFER_WRITE) {
472 if (transfer->resource->target == PIPE_TEXTURE_2D) {
473 struct panfrost_resource *prsrc = (struct panfrost_resource *) transfer->resource;
474
475 /* Gallium thinks writeback happens here; instead, this is our cue to tile */
476 if (bo->layout == PAN_AFBC) {
477 DBG("Warning: writes to afbc surface can't possibly work out well for you...\n");
478 } else if (bo->layout == PAN_TILED) {
479 struct pipe_context *gallium = (struct pipe_context *) ctx;
480 struct panfrost_screen *screen = pan_screen(gallium->screen);
481 panfrost_tile_texture(screen, prsrc, transfer->level);
482 }
483 }
484 }
485 }
486
487 static void
488 panfrost_transfer_unmap(struct pipe_context *pctx,
489 struct pipe_transfer *transfer)
490 {
491 struct panfrost_context *ctx = pan_context(pctx);
492
493 panfrost_unmap_bo(ctx, transfer);
494
495 /* Derefence the resource */
496 pipe_resource_reference(&transfer->resource, NULL);
497
498 /* Transfer itself is CALLOCed at the moment */
499 free(transfer);
500 }
501
502 static struct pb_slab *
503 panfrost_slab_alloc(void *priv, unsigned heap, unsigned entry_size, unsigned group_index)
504 {
505 struct panfrost_screen *screen = (struct panfrost_screen *) priv;
506 struct panfrost_memory *mem = CALLOC_STRUCT(panfrost_memory);
507
508 size_t slab_size = (1 << (MAX_SLAB_ENTRY_SIZE + 1));
509
510 mem->slab.num_entries = slab_size / entry_size;
511 mem->slab.num_free = mem->slab.num_entries;
512
513 LIST_INITHEAD(&mem->slab.free);
514 for (unsigned i = 0; i < mem->slab.num_entries; ++i) {
515 /* Create a slab entry */
516 struct panfrost_memory_entry *entry = CALLOC_STRUCT(panfrost_memory_entry);
517 entry->offset = entry_size * i;
518
519 entry->base.slab = &mem->slab;
520 entry->base.group_index = group_index;
521
522 LIST_ADDTAIL(&entry->base.head, &mem->slab.free);
523 }
524
525 /* Actually allocate the memory from kernel-space. Mapped, same_va, no
526 * special flags */
527
528 screen->driver->allocate_slab(screen, mem, slab_size / 4096, true, 0, 0, 0);
529
530 return &mem->slab;
531 }
532
533 static bool
534 panfrost_slab_can_reclaim(void *priv, struct pb_slab_entry *entry)
535 {
536 struct panfrost_memory_entry *p_entry = (struct panfrost_memory_entry *) entry;
537 return p_entry->freed;
538 }
539
540 static void
541 panfrost_slab_free(void *priv, struct pb_slab *slab)
542 {
543 struct panfrost_memory *mem = (struct panfrost_memory *) slab;
544 struct panfrost_screen *screen = (struct panfrost_screen *) priv;
545
546 screen->driver->free_slab(screen, mem);
547 }
548
549 static void
550 panfrost_invalidate_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
551 {
552 //DBG("TODO %s\n", __func__);
553 }
554
555 static enum pipe_format
556 panfrost_resource_get_internal_format(struct pipe_resource *prsrc)
557 {
558 return prsrc->format;
559 }
560
561 static void
562 panfrost_resource_set_stencil(struct pipe_resource *prsrc,
563 struct pipe_resource *stencil)
564 {
565 pan_resource(prsrc)->separate_stencil = pan_resource(stencil);
566 }
567
568 static struct pipe_resource *
569 panfrost_resource_get_stencil(struct pipe_resource *prsrc)
570 {
571 return &pan_resource(prsrc)->separate_stencil->base;
572 }
573
574 static const struct u_transfer_vtbl transfer_vtbl = {
575 .resource_create = panfrost_resource_create,
576 .resource_destroy = panfrost_resource_destroy,
577 .transfer_map = panfrost_transfer_map,
578 .transfer_unmap = panfrost_transfer_unmap,
579 .transfer_flush_region = u_default_transfer_flush_region,
580 .get_internal_format = panfrost_resource_get_internal_format,
581 .set_stencil = panfrost_resource_set_stencil,
582 .get_stencil = panfrost_resource_get_stencil,
583 };
584
585 void
586 panfrost_resource_screen_init(struct panfrost_screen *pscreen)
587 {
588 //pscreen->base.resource_create_with_modifiers =
589 // panfrost_resource_create_with_modifiers;
590 pscreen->base.resource_create = u_transfer_helper_resource_create;
591 pscreen->base.resource_destroy = u_transfer_helper_resource_destroy;
592 pscreen->base.resource_from_handle = panfrost_resource_from_handle;
593 pscreen->base.resource_get_handle = panfrost_resource_get_handle;
594 pscreen->base.transfer_helper = u_transfer_helper_create(&transfer_vtbl,
595 true, false,
596 true, true);
597
598 pb_slabs_init(&pscreen->slabs,
599 MIN_SLAB_ENTRY_SIZE,
600 MAX_SLAB_ENTRY_SIZE,
601
602 3, /* Number of heaps */
603
604 pscreen,
605
606 panfrost_slab_can_reclaim,
607 panfrost_slab_alloc,
608 panfrost_slab_free);
609 }
610
611 void
612 panfrost_resource_context_init(struct pipe_context *pctx)
613 {
614 pctx->transfer_map = u_transfer_helper_transfer_map;
615 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
616 pctx->transfer_unmap = u_transfer_helper_transfer_unmap;
617 pctx->buffer_subdata = u_default_buffer_subdata;
618 pctx->create_surface = panfrost_create_surface;
619 pctx->surface_destroy = panfrost_surface_destroy;
620 pctx->resource_copy_region = util_resource_copy_region;
621 pctx->blit = panfrost_blit;
622 //pctx->generate_mipmap = panfrost_generate_mipmap;
623 pctx->flush_resource = panfrost_flush_resource;
624 pctx->invalidate_resource = panfrost_invalidate_resource;
625 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
626 pctx->buffer_subdata = u_default_buffer_subdata;
627 pctx->texture_subdata = u_default_texture_subdata;
628 }