panfrost: Support linear depth textures
[mesa.git] / src / gallium / drivers / panfrost / pan_resource.c
1 /**************************************************************************
2 *
3 * Copyright 2008 VMware, Inc.
4 * Copyright 2014 Broadcom
5 * Copyright 2018 Alyssa Rosenzweig
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
23 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
24 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
25 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
26 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
28 **************************************************************************/
29
30 #include <xf86drm.h>
31 #include <fcntl.h>
32 #include "drm-uapi/drm_fourcc.h"
33
34 #include "state_tracker/winsys_handle.h"
35 #include "util/u_format.h"
36 #include "util/u_memory.h"
37 #include "util/u_surface.h"
38 #include "util/u_transfer.h"
39 #include "util/u_transfer_helper.h"
40
41 #include "pan_context.h"
42 #include "pan_screen.h"
43 #include "pan_resource.h"
44 #include "pan_swizzle.h"
45 #include "pan_util.h"
46
47 static struct pipe_resource *
48 panfrost_resource_from_handle(struct pipe_screen *pscreen,
49 const struct pipe_resource *templat,
50 struct winsys_handle *whandle,
51 unsigned usage)
52 {
53 struct panfrost_screen *screen = pan_screen(pscreen);
54 struct panfrost_resource *rsc;
55 struct pipe_resource *prsc;
56
57 assert(whandle->type == WINSYS_HANDLE_TYPE_FD);
58
59 rsc = CALLOC_STRUCT(panfrost_resource);
60 if (!rsc)
61 return NULL;
62
63 prsc = &rsc->base;
64
65 *prsc = *templat;
66
67 pipe_reference_init(&prsc->reference, 1);
68 prsc->screen = pscreen;
69
70 rsc->bo = screen->driver->import_bo(screen, whandle);
71
72 if (screen->ro) {
73 rsc->scanout =
74 renderonly_create_gpu_import_for_resource(prsc, screen->ro, NULL);
75 /* failure is expected in some cases.. */
76 }
77
78 return prsc;
79 }
80
81 static boolean
82 panfrost_resource_get_handle(struct pipe_screen *pscreen,
83 struct pipe_context *ctx,
84 struct pipe_resource *pt,
85 struct winsys_handle *handle,
86 unsigned usage)
87 {
88 struct panfrost_screen *screen = pan_screen(pscreen);
89 struct panfrost_resource *rsrc = (struct panfrost_resource *) pt;
90 struct renderonly_scanout *scanout = rsrc->scanout;
91 int bytes_per_pixel = util_format_get_blocksize(rsrc->base.format);
92 int stride = bytes_per_pixel * rsrc->base.width0; /* TODO: Alignment? */
93
94 handle->stride = stride;
95 handle->modifier = DRM_FORMAT_MOD_INVALID;
96
97 if (handle->type == WINSYS_HANDLE_TYPE_SHARED) {
98 return FALSE;
99 } else if (handle->type == WINSYS_HANDLE_TYPE_KMS) {
100 if (renderonly_get_handle(scanout, handle))
101 return TRUE;
102
103 handle->handle = rsrc->bo->gem_handle;
104 return TRUE;
105 } else if (handle->type == WINSYS_HANDLE_TYPE_FD) {
106 if (scanout) {
107 struct drm_prime_handle args = {
108 .handle = scanout->handle,
109 .flags = DRM_CLOEXEC,
110 };
111
112 int ret = drmIoctl(screen->ro->kms_fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &args);
113 if (ret == -1)
114 return FALSE;
115
116 handle->handle = args.fd;
117
118 return TRUE;
119 } else
120 return screen->driver->export_bo(screen, rsrc->bo->gem_handle, handle);
121 }
122
123 return FALSE;
124 }
125
126 static void
127 panfrost_flush_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
128 {
129 //DBG("TODO %s\n", __func__);
130 }
131
132 static void
133 panfrost_blit(struct pipe_context *pipe,
134 const struct pipe_blit_info *info)
135 {
136 /* STUB */
137 DBG("Skipping blit XXX\n");
138 return;
139 }
140
141 static struct pipe_surface *
142 panfrost_create_surface(struct pipe_context *pipe,
143 struct pipe_resource *pt,
144 const struct pipe_surface *surf_tmpl)
145 {
146 struct pipe_surface *ps = NULL;
147
148 ps = CALLOC_STRUCT(pipe_surface);
149
150 if (ps) {
151 pipe_reference_init(&ps->reference, 1);
152 pipe_resource_reference(&ps->texture, pt);
153 ps->context = pipe;
154 ps->format = surf_tmpl->format;
155
156 if (pt->target != PIPE_BUFFER) {
157 assert(surf_tmpl->u.tex.level <= pt->last_level);
158 ps->width = u_minify(pt->width0, surf_tmpl->u.tex.level);
159 ps->height = u_minify(pt->height0, surf_tmpl->u.tex.level);
160 ps->u.tex.level = surf_tmpl->u.tex.level;
161 ps->u.tex.first_layer = surf_tmpl->u.tex.first_layer;
162 ps->u.tex.last_layer = surf_tmpl->u.tex.last_layer;
163 } else {
164 /* setting width as number of elements should get us correct renderbuffer width */
165 ps->width = surf_tmpl->u.buf.last_element - surf_tmpl->u.buf.first_element + 1;
166 ps->height = pt->height0;
167 ps->u.buf.first_element = surf_tmpl->u.buf.first_element;
168 ps->u.buf.last_element = surf_tmpl->u.buf.last_element;
169 assert(ps->u.buf.first_element <= ps->u.buf.last_element);
170 assert(ps->u.buf.last_element < ps->width);
171 }
172 }
173
174 return ps;
175 }
176
177 static void
178 panfrost_surface_destroy(struct pipe_context *pipe,
179 struct pipe_surface *surf)
180 {
181 assert(surf->texture);
182 pipe_resource_reference(&surf->texture, NULL);
183 free(surf);
184 }
185
186 static struct panfrost_bo *
187 panfrost_create_bo(struct panfrost_screen *screen, const struct pipe_resource *template)
188 {
189 struct panfrost_bo *bo = CALLOC_STRUCT(panfrost_bo);
190
191 /* Calculate the size of the bo */
192
193 int bytes_per_pixel = util_format_get_blocksize(template->format);
194 int stride = bytes_per_pixel * template->width0; /* TODO: Alignment? */
195 size_t sz = stride;
196
197 if (template->height0) sz *= template->height0;
198 if (template->depth0) sz *= template->depth0;
199
200 /* Based on the usage, figure out what storing will be used. There are
201 * various tradeoffs:
202 *
203 * Linear: the basic format, bad for memory bandwidth, bad for cache
204 * use. Zero-copy, though. Renderable.
205 *
206 * Tiled: Not compressed, but cache-optimized. Expensive to write into
207 * (due to software tiling), but cheap to sample from. Ideal for most
208 * textures.
209 *
210 * AFBC: Compressed and renderable (so always desirable for non-scanout
211 * rendertargets). Cheap to sample from. The format is black box, so we
212 * can't read/write from software.
213 */
214
215 /* Tiling textures is almost always faster, unless we only use it once */
216 bool should_tile = (template->usage != PIPE_USAGE_STREAM) && (template->bind & PIPE_BIND_SAMPLER_VIEW);
217
218 /* Set the layout appropriately */
219 bo->layout = should_tile ? PAN_TILED : PAN_LINEAR;
220
221 if (bo->layout == PAN_TILED) {
222 /* For tiled, we don't map directly, so just malloc any old buffer */
223
224 for (int l = 0; l < (template->last_level + 1); ++l) {
225 bo->cpu[l] = malloc(sz);
226 bo->size[l] = sz;
227 sz >>= 2;
228 }
229 } else {
230 /* For a linear resource, allocate a block of memory from
231 * kernel space */
232
233 struct panfrost_memory mem;
234
235 bo->size[0] = ALIGN(sz, 4096);
236 screen->driver->allocate_slab(screen, &mem, bo->size[0] / 4096, true, 0, 0, 0);
237
238 bo->cpu[0] = mem.cpu;
239 bo->gpu[0] = mem.gpu;
240
241 /* TODO: Mipmap */
242 }
243
244 return bo;
245 }
246
247 static struct pipe_resource *
248 panfrost_resource_create(struct pipe_screen *screen,
249 const struct pipe_resource *template)
250 {
251 struct panfrost_resource *so = CALLOC_STRUCT(panfrost_resource);
252 struct panfrost_screen *pscreen = (struct panfrost_screen *) screen;
253
254 so->base = *template;
255 so->base.screen = screen;
256
257 pipe_reference_init(&so->base.reference, 1);
258
259 /* Make sure we're familiar */
260 switch (template->target) {
261 case PIPE_BUFFER:
262 case PIPE_TEXTURE_1D:
263 case PIPE_TEXTURE_2D:
264 case PIPE_TEXTURE_3D:
265 case PIPE_TEXTURE_RECT:
266 break;
267 default:
268 DBG("Unknown texture target %d\n", template->target);
269 assert(0);
270 }
271
272 if (template->bind & PIPE_BIND_DISPLAY_TARGET ||
273 template->bind & PIPE_BIND_SCANOUT ||
274 template->bind & PIPE_BIND_SHARED) {
275 struct pipe_resource scanout_templat = *template;
276 struct renderonly_scanout *scanout;
277 struct winsys_handle handle;
278
279 /* TODO: align width0 and height0? */
280
281 scanout = renderonly_scanout_for_resource(&scanout_templat,
282 pscreen->ro, &handle);
283 if (!scanout)
284 return NULL;
285
286 assert(handle.type == WINSYS_HANDLE_TYPE_FD);
287 /* TODO: handle modifiers? */
288 so = pan_resource(screen->resource_from_handle(screen, template,
289 &handle,
290 PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE));
291 close(handle.handle);
292 if (!so)
293 return NULL;
294
295 so->scanout = scanout;
296 pscreen->display_target = so;
297 } else {
298 so->bo = panfrost_create_bo(pscreen, template);
299 }
300
301 return (struct pipe_resource *)so;
302 }
303
304 static void
305 panfrost_destroy_bo(struct panfrost_screen *screen, struct panfrost_bo *pbo)
306 {
307 struct panfrost_bo *bo = (struct panfrost_bo *)pbo;
308
309 if (bo->layout == PAN_LINEAR && !bo->imported) {
310 /* Construct a memory object for all mip levels */
311
312 struct panfrost_memory mem = {
313 .cpu = bo->cpu[0],
314 .gpu = bo->gpu[0],
315 .size = bo->size[0]
316 };
317
318 screen->driver->free_slab(screen, &mem);
319 }
320
321 if (bo->layout == PAN_TILED) {
322 /* Tiled has a malloc'd CPU, so just plain ol' free needed */
323
324 for (int l = 0; l < MAX_MIP_LEVELS; ++l) {
325 free(bo->cpu[l]);
326 }
327 }
328
329 if (bo->layout == PAN_AFBC) {
330 /* TODO */
331 DBG("--leaking afbc (%d bytes)--\n", bo->afbc_metadata_size);
332 }
333
334 if (bo->has_checksum) {
335 /* TODO */
336 DBG("--leaking checksum (%zd bytes)--\n", bo->checksum_slab.size);
337 }
338
339 if (bo->imported) {
340 screen->driver->free_imported_bo(screen, bo);
341 }
342 }
343
344 static void
345 panfrost_resource_destroy(struct pipe_screen *screen,
346 struct pipe_resource *pt)
347 {
348 struct panfrost_screen *pscreen = pan_screen(screen);
349 struct panfrost_resource *rsrc = (struct panfrost_resource *) pt;
350
351 if (rsrc->scanout)
352 renderonly_scanout_destroy(rsrc->scanout, pscreen->ro);
353
354 if (rsrc->bo)
355 panfrost_destroy_bo(pscreen, rsrc->bo);
356
357 FREE(rsrc);
358 }
359
360 static uint8_t *
361 panfrost_map_bo(struct panfrost_context *ctx, struct pipe_transfer *transfer)
362 {
363 struct panfrost_bo *bo = (struct panfrost_bo *)pan_resource(transfer->resource)->bo;
364
365 /* If non-zero level, it's a mipmapped resource and needs to be treated as such */
366 bo->is_mipmap |= transfer->level;
367
368 if (transfer->usage & PIPE_TRANSFER_MAP_DIRECTLY && bo->layout != PAN_LINEAR) {
369 /* We can only directly map linear resources */
370 return NULL;
371 }
372
373 if (transfer->resource->bind & PIPE_BIND_DEPTH_STENCIL) {
374 /* Mipmapped readpixels?! */
375 assert(transfer->level == 0);
376
377 /* Set the CPU mapping to that of the depth/stencil buffer in memory, untiled */
378 bo->cpu[transfer->level] = ctx->depth_stencil_buffer.cpu;
379 }
380
381 return bo->cpu[transfer->level];
382 }
383
384 static void *
385 panfrost_transfer_map(struct pipe_context *pctx,
386 struct pipe_resource *resource,
387 unsigned level,
388 unsigned usage, /* a combination of PIPE_TRANSFER_x */
389 const struct pipe_box *box,
390 struct pipe_transfer **out_transfer)
391 {
392 struct panfrost_context *ctx = pan_context(pctx);
393 int bytes_per_pixel = util_format_get_blocksize(resource->format);
394 int stride = bytes_per_pixel * resource->width0; /* TODO: Alignment? */
395 uint8_t *cpu;
396
397 struct pipe_transfer *transfer = CALLOC_STRUCT(pipe_transfer);
398 transfer->level = level;
399 transfer->usage = usage;
400 transfer->box = *box;
401 transfer->stride = stride;
402 assert(!transfer->box.z);
403
404 pipe_resource_reference(&transfer->resource, resource);
405
406 *out_transfer = transfer;
407
408 if (resource->bind & PIPE_BIND_DISPLAY_TARGET ||
409 resource->bind & PIPE_BIND_SCANOUT ||
410 resource->bind & PIPE_BIND_SHARED) {
411 /* Mipmapped readpixels?! */
412 assert(level == 0);
413
414 /* Force a flush -- kill the pipeline */
415 panfrost_flush(pctx, NULL, PIPE_FLUSH_END_OF_FRAME);
416 }
417
418 cpu = panfrost_map_bo(ctx, transfer);
419 if (cpu == NULL)
420 return NULL;
421
422 return cpu + transfer->box.x * bytes_per_pixel + transfer->box.y * stride;
423 }
424
425 static void
426 panfrost_tile_texture(struct panfrost_screen *screen, struct panfrost_resource *rsrc, int level)
427 {
428 struct panfrost_bo *bo = (struct panfrost_bo *)rsrc->bo;
429 int bytes_per_pixel = util_format_get_blocksize(rsrc->base.format);
430 int stride = bytes_per_pixel * rsrc->base.width0; /* TODO: Alignment? */
431
432 int width = rsrc->base.width0 >> level;
433 int height = rsrc->base.height0 >> level;
434
435 /* Estimate swizzled bitmap size. Slight overestimates are fine.
436 * Underestimates will result in memory corruption or worse. */
437
438 int swizzled_sz = panfrost_swizzled_size(width, height, bytes_per_pixel);
439
440 /* Save the entry. But if there was already an entry here (from a
441 * previous upload of the resource), free that one so we don't leak */
442
443 if (bo->entry[level] != NULL) {
444 bo->entry[level]->freed = true;
445 pb_slab_free(&screen->slabs, &bo->entry[level]->base);
446 }
447
448 /* Allocate the transfer given that known size but do not copy */
449 struct pb_slab_entry *entry = pb_slab_alloc(&screen->slabs, swizzled_sz, HEAP_TEXTURE);
450 struct panfrost_memory_entry *p_entry = (struct panfrost_memory_entry *) entry;
451 struct panfrost_memory *backing = (struct panfrost_memory *) entry->slab;
452 uint8_t *swizzled = backing->cpu + p_entry->offset;
453
454 bo->entry[level] = p_entry;
455 bo->gpu[level] = backing->gpu + p_entry->offset;
456
457 /* Run actual texture swizzle, writing directly to the mapped
458 * GPU chunk we allocated */
459
460 panfrost_texture_swizzle(width, height, bytes_per_pixel, stride, bo->cpu[level], swizzled);
461 }
462
463 static void
464 panfrost_unmap_bo(struct panfrost_context *ctx,
465 struct pipe_transfer *transfer)
466 {
467 struct panfrost_bo *bo = (struct panfrost_bo *)pan_resource(transfer->resource)->bo;
468
469 if (transfer->usage & PIPE_TRANSFER_WRITE) {
470 if (transfer->resource->target == PIPE_TEXTURE_2D) {
471 struct panfrost_resource *prsrc = (struct panfrost_resource *) transfer->resource;
472
473 /* Gallium thinks writeback happens here; instead, this is our cue to tile */
474 if (bo->layout == PAN_AFBC) {
475 DBG("Warning: writes to afbc surface can't possibly work out well for you...\n");
476 } else if (bo->layout == PAN_TILED) {
477 struct pipe_context *gallium = (struct pipe_context *) ctx;
478 struct panfrost_screen *screen = pan_screen(gallium->screen);
479 panfrost_tile_texture(screen, prsrc, transfer->level);
480 }
481 }
482 }
483 }
484
485 static void
486 panfrost_transfer_unmap(struct pipe_context *pctx,
487 struct pipe_transfer *transfer)
488 {
489 struct panfrost_context *ctx = pan_context(pctx);
490
491 panfrost_unmap_bo(ctx, transfer);
492
493 /* Derefence the resource */
494 pipe_resource_reference(&transfer->resource, NULL);
495
496 /* Transfer itself is CALLOCed at the moment */
497 free(transfer);
498 }
499
500 static struct pb_slab *
501 panfrost_slab_alloc(void *priv, unsigned heap, unsigned entry_size, unsigned group_index)
502 {
503 struct panfrost_screen *screen = (struct panfrost_screen *) priv;
504 struct panfrost_memory *mem = CALLOC_STRUCT(panfrost_memory);
505
506 size_t slab_size = (1 << (MAX_SLAB_ENTRY_SIZE + 1));
507
508 mem->slab.num_entries = slab_size / entry_size;
509 mem->slab.num_free = mem->slab.num_entries;
510
511 LIST_INITHEAD(&mem->slab.free);
512 for (unsigned i = 0; i < mem->slab.num_entries; ++i) {
513 /* Create a slab entry */
514 struct panfrost_memory_entry *entry = CALLOC_STRUCT(panfrost_memory_entry);
515 entry->offset = entry_size * i;
516
517 entry->base.slab = &mem->slab;
518 entry->base.group_index = group_index;
519
520 LIST_ADDTAIL(&entry->base.head, &mem->slab.free);
521 }
522
523 /* Actually allocate the memory from kernel-space. Mapped, same_va, no
524 * special flags */
525
526 screen->driver->allocate_slab(screen, mem, slab_size / 4096, true, 0, 0, 0);
527
528 return &mem->slab;
529 }
530
531 static bool
532 panfrost_slab_can_reclaim(void *priv, struct pb_slab_entry *entry)
533 {
534 struct panfrost_memory_entry *p_entry = (struct panfrost_memory_entry *) entry;
535 return p_entry->freed;
536 }
537
538 static void
539 panfrost_slab_free(void *priv, struct pb_slab *slab)
540 {
541 struct panfrost_memory *mem = (struct panfrost_memory *) slab;
542 struct panfrost_screen *screen = (struct panfrost_screen *) priv;
543
544 screen->driver->free_slab(screen, mem);
545 }
546
547 static void
548 panfrost_invalidate_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
549 {
550 //DBG("TODO %s\n", __func__);
551 }
552
553 static enum pipe_format
554 panfrost_resource_get_internal_format(struct pipe_resource *prsrc)
555 {
556 return prsrc->format;
557 }
558
559 static void
560 panfrost_resource_set_stencil(struct pipe_resource *prsrc,
561 struct pipe_resource *stencil)
562 {
563 pan_resource(prsrc)->separate_stencil = pan_resource(stencil);
564 }
565
566 static struct pipe_resource *
567 panfrost_resource_get_stencil(struct pipe_resource *prsrc)
568 {
569 return &pan_resource(prsrc)->separate_stencil->base;
570 }
571
572 static const struct u_transfer_vtbl transfer_vtbl = {
573 .resource_create = panfrost_resource_create,
574 .resource_destroy = panfrost_resource_destroy,
575 .transfer_map = panfrost_transfer_map,
576 .transfer_unmap = panfrost_transfer_unmap,
577 .transfer_flush_region = u_default_transfer_flush_region,
578 .get_internal_format = panfrost_resource_get_internal_format,
579 .set_stencil = panfrost_resource_set_stencil,
580 .get_stencil = panfrost_resource_get_stencil,
581 };
582
583 void
584 panfrost_resource_screen_init(struct panfrost_screen *pscreen)
585 {
586 //pscreen->base.resource_create_with_modifiers =
587 // panfrost_resource_create_with_modifiers;
588 pscreen->base.resource_create = u_transfer_helper_resource_create;
589 pscreen->base.resource_destroy = u_transfer_helper_resource_destroy;
590 pscreen->base.resource_from_handle = panfrost_resource_from_handle;
591 pscreen->base.resource_get_handle = panfrost_resource_get_handle;
592 pscreen->base.transfer_helper = u_transfer_helper_create(&transfer_vtbl,
593 true, false,
594 true, true);
595
596 pb_slabs_init(&pscreen->slabs,
597 MIN_SLAB_ENTRY_SIZE,
598 MAX_SLAB_ENTRY_SIZE,
599
600 3, /* Number of heaps */
601
602 pscreen,
603
604 panfrost_slab_can_reclaim,
605 panfrost_slab_alloc,
606 panfrost_slab_free);
607 }
608
609 void
610 panfrost_resource_context_init(struct pipe_context *pctx)
611 {
612 pctx->transfer_map = u_transfer_helper_transfer_map;
613 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
614 pctx->transfer_unmap = u_transfer_helper_transfer_unmap;
615 pctx->buffer_subdata = u_default_buffer_subdata;
616 pctx->create_surface = panfrost_create_surface;
617 pctx->surface_destroy = panfrost_surface_destroy;
618 pctx->resource_copy_region = util_resource_copy_region;
619 pctx->blit = panfrost_blit;
620 //pctx->generate_mipmap = panfrost_generate_mipmap;
621 pctx->flush_resource = panfrost_flush_resource;
622 pctx->invalidate_resource = panfrost_invalidate_resource;
623 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
624 pctx->buffer_subdata = u_default_buffer_subdata;
625 pctx->texture_subdata = u_default_texture_subdata;
626 }