1117ebec15d2579c26dd2be515259e109ad6d55e
[mesa.git] / src / gallium / drivers / iris / iris_resource.c
1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 /**
24 * @file iris_resource.c
25 *
26 * Resources are images, buffers, and other objects used by the GPU.
27 *
28 * XXX: explain resources
29 */
30
31 #include <stdio.h>
32 #include <errno.h>
33 #include "pipe/p_defines.h"
34 #include "pipe/p_state.h"
35 #include "pipe/p_context.h"
36 #include "pipe/p_screen.h"
37 #include "util/os_memory.h"
38 #include "util/u_cpu_detect.h"
39 #include "util/u_inlines.h"
40 #include "util/u_format.h"
41 #include "util/u_transfer.h"
42 #include "util/u_transfer_helper.h"
43 #include "util/u_upload_mgr.h"
44 #include "util/ralloc.h"
45 #include "iris_batch.h"
46 #include "iris_context.h"
47 #include "iris_resource.h"
48 #include "iris_screen.h"
49 #include "intel/common/gen_debug.h"
50 #include "isl/isl.h"
51 #include "drm-uapi/drm_fourcc.h"
52 #include "drm-uapi/i915_drm.h"
53
54 enum modifier_priority {
55 MODIFIER_PRIORITY_INVALID = 0,
56 MODIFIER_PRIORITY_LINEAR,
57 MODIFIER_PRIORITY_X,
58 MODIFIER_PRIORITY_Y,
59 MODIFIER_PRIORITY_Y_CCS,
60 };
61
62 static const uint64_t priority_to_modifier[] = {
63 [MODIFIER_PRIORITY_INVALID] = DRM_FORMAT_MOD_INVALID,
64 [MODIFIER_PRIORITY_LINEAR] = DRM_FORMAT_MOD_LINEAR,
65 [MODIFIER_PRIORITY_X] = I915_FORMAT_MOD_X_TILED,
66 [MODIFIER_PRIORITY_Y] = I915_FORMAT_MOD_Y_TILED,
67 [MODIFIER_PRIORITY_Y_CCS] = I915_FORMAT_MOD_Y_TILED_CCS,
68 };
69
70 static bool
71 modifier_is_supported(const struct gen_device_info *devinfo,
72 uint64_t modifier)
73 {
74 /* XXX: do something real */
75 switch (modifier) {
76 case I915_FORMAT_MOD_Y_TILED:
77 case I915_FORMAT_MOD_X_TILED:
78 case DRM_FORMAT_MOD_LINEAR:
79 return true;
80 case I915_FORMAT_MOD_Y_TILED_CCS:
81 case DRM_FORMAT_MOD_INVALID:
82 default:
83 return false;
84 }
85 }
86
87 static uint64_t
88 select_best_modifier(struct gen_device_info *devinfo,
89 const uint64_t *modifiers,
90 int count)
91 {
92 enum modifier_priority prio = MODIFIER_PRIORITY_INVALID;
93
94 for (int i = 0; i < count; i++) {
95 if (!modifier_is_supported(devinfo, modifiers[i]))
96 continue;
97
98 switch (modifiers[i]) {
99 case I915_FORMAT_MOD_Y_TILED_CCS:
100 prio = MAX2(prio, MODIFIER_PRIORITY_Y_CCS);
101 break;
102 case I915_FORMAT_MOD_Y_TILED:
103 prio = MAX2(prio, MODIFIER_PRIORITY_Y);
104 break;
105 case I915_FORMAT_MOD_X_TILED:
106 prio = MAX2(prio, MODIFIER_PRIORITY_X);
107 break;
108 case DRM_FORMAT_MOD_LINEAR:
109 prio = MAX2(prio, MODIFIER_PRIORITY_LINEAR);
110 break;
111 case DRM_FORMAT_MOD_INVALID:
112 default:
113 break;
114 }
115 }
116
117 return priority_to_modifier[prio];
118 }
119
120 static enum isl_surf_dim
121 target_to_isl_surf_dim(enum pipe_texture_target target)
122 {
123 switch (target) {
124 case PIPE_BUFFER:
125 case PIPE_TEXTURE_1D:
126 case PIPE_TEXTURE_1D_ARRAY:
127 return ISL_SURF_DIM_1D;
128 case PIPE_TEXTURE_2D:
129 case PIPE_TEXTURE_CUBE:
130 case PIPE_TEXTURE_RECT:
131 case PIPE_TEXTURE_2D_ARRAY:
132 case PIPE_TEXTURE_CUBE_ARRAY:
133 return ISL_SURF_DIM_2D;
134 case PIPE_TEXTURE_3D:
135 return ISL_SURF_DIM_3D;
136 case PIPE_MAX_TEXTURE_TYPES:
137 break;
138 }
139 unreachable("invalid texture type");
140 }
141
142 static isl_surf_usage_flags_t
143 pipe_bind_to_isl_usage(unsigned bindings)
144 {
145 isl_surf_usage_flags_t usage = 0;
146
147 if (bindings & PIPE_BIND_RENDER_TARGET)
148 usage |= ISL_SURF_USAGE_RENDER_TARGET_BIT;
149
150 if (bindings & PIPE_BIND_SAMPLER_VIEW)
151 usage |= ISL_SURF_USAGE_TEXTURE_BIT;
152
153 if (bindings & (PIPE_BIND_SHADER_IMAGE | PIPE_BIND_SHADER_BUFFER))
154 usage |= ISL_SURF_USAGE_STORAGE_BIT;
155
156 if (bindings & PIPE_BIND_DISPLAY_TARGET)
157 usage |= ISL_SURF_USAGE_DISPLAY_BIT;
158
159 return usage;
160 }
161
162 struct pipe_resource *
163 iris_resource_get_separate_stencil(struct pipe_resource *p_res)
164 {
165 /* For packed depth-stencil, we treat depth as the primary resource
166 * and store S8 as the "second plane" resource.
167 */
168 return p_res->next;
169 }
170
171 static void
172 iris_resource_set_separate_stencil(struct pipe_resource *p_res,
173 struct pipe_resource *stencil)
174 {
175 assert(util_format_has_depth(util_format_description(p_res->format)));
176 pipe_resource_reference(&p_res->next, stencil);
177 }
178
179 void
180 iris_get_depth_stencil_resources(struct pipe_resource *res,
181 struct iris_resource **out_z,
182 struct iris_resource **out_s)
183 {
184 if (!res) {
185 *out_z = NULL;
186 *out_s = NULL;
187 return;
188 }
189
190 const struct util_format_description *desc =
191 util_format_description(res->format);
192
193 if (util_format_has_depth(desc)) {
194 *out_z = (void *) res;
195 *out_s = (void *) iris_resource_get_separate_stencil(res);
196 } else {
197 assert(util_format_has_stencil(desc));
198 *out_z = NULL;
199 *out_s = (void *) res;
200 }
201 }
202
203 static void
204 iris_resource_destroy(struct pipe_screen *screen,
205 struct pipe_resource *resource)
206 {
207 struct iris_resource *res = (struct iris_resource *)resource;
208
209 iris_bo_unreference(res->bo);
210 free(res);
211 }
212
213 static struct iris_resource *
214 iris_alloc_resource(struct pipe_screen *pscreen,
215 const struct pipe_resource *templ)
216 {
217 struct iris_resource *res = calloc(1, sizeof(struct iris_resource));
218 if (!res)
219 return NULL;
220
221 res->base = *templ;
222 res->base.screen = pscreen;
223 pipe_reference_init(&res->base.reference, 1);
224
225 return res;
226 }
227
228 static struct pipe_resource *
229 iris_resource_create_for_buffer(struct pipe_screen *pscreen,
230 const struct pipe_resource *templ)
231 {
232 struct iris_screen *screen = (struct iris_screen *)pscreen;
233 struct iris_resource *res = iris_alloc_resource(pscreen, templ);
234
235 assert(templ->target == PIPE_BUFFER);
236 assert(templ->height0 <= 1);
237 assert(templ->depth0 <= 1);
238 assert(templ->format == PIPE_FORMAT_NONE ||
239 util_format_get_blocksize(templ->format) == 1);
240
241 res->internal_format = templ->format;
242 res->surf.tiling = ISL_TILING_LINEAR;
243
244 enum iris_memory_zone memzone = IRIS_MEMZONE_OTHER;
245 const char *name = templ->target == PIPE_BUFFER ? "buffer" : "miptree";
246 if (templ->flags & IRIS_RESOURCE_FLAG_SHADER_MEMZONE) {
247 memzone = IRIS_MEMZONE_SHADER;
248 name = "shader kernels";
249 } else if (templ->flags & IRIS_RESOURCE_FLAG_SURFACE_MEMZONE) {
250 memzone = IRIS_MEMZONE_SURFACE;
251 name = "surface state";
252 } else if (templ->flags & IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE) {
253 memzone = IRIS_MEMZONE_DYNAMIC;
254 name = "dynamic state";
255 }
256
257 res->bo = iris_bo_alloc(screen->bufmgr, name, templ->width0, memzone);
258 if (!res->bo) {
259 iris_resource_destroy(pscreen, &res->base);
260 return NULL;
261 }
262
263 return &res->base;
264 }
265
266 static struct pipe_resource *
267 iris_resource_create_with_modifiers(struct pipe_screen *pscreen,
268 const struct pipe_resource *templ,
269 const uint64_t *modifiers,
270 int modifiers_count)
271 {
272 struct iris_screen *screen = (struct iris_screen *)pscreen;
273 struct gen_device_info *devinfo = &screen->devinfo;
274 struct iris_resource *res = iris_alloc_resource(pscreen, templ);
275 const struct util_format_description *format_desc =
276 util_format_description(templ->format);
277
278 if (!res)
279 return NULL;
280
281 const bool has_depth = util_format_has_depth(format_desc);
282 uint64_t modifier =
283 select_best_modifier(devinfo, modifiers, modifiers_count);
284
285 isl_tiling_flags_t tiling_flags = ISL_TILING_ANY_MASK;
286
287 if (modifier != DRM_FORMAT_MOD_INVALID) {
288 const struct isl_drm_modifier_info *mod_info =
289 isl_drm_modifier_get_info(modifier);
290
291 tiling_flags = 1 << mod_info->tiling;
292 } else {
293 if (modifiers_count > 0) {
294 fprintf(stderr, "Unsupported modifier, resource creation failed.\n");
295 return NULL;
296 }
297
298 /* No modifiers - we can select our own tiling. */
299
300 if (has_depth) {
301 /* Depth must be Y-tiled */
302 tiling_flags = ISL_TILING_Y0_BIT;
303 } else if (templ->format == PIPE_FORMAT_S8_UINT) {
304 /* Stencil must be W-tiled */
305 tiling_flags = ISL_TILING_W_BIT;
306 } else if (templ->target == PIPE_BUFFER ||
307 templ->target == PIPE_TEXTURE_1D ||
308 templ->target == PIPE_TEXTURE_1D_ARRAY) {
309 /* Use linear for buffers and 1D textures */
310 tiling_flags = ISL_TILING_LINEAR_BIT;
311 }
312
313 /* Use linear for staging buffers */
314 if (templ->usage == PIPE_USAGE_STAGING ||
315 templ->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR) )
316 tiling_flags = ISL_TILING_LINEAR_BIT;
317 }
318
319 isl_surf_usage_flags_t usage = pipe_bind_to_isl_usage(templ->bind);
320
321 if (templ->target == PIPE_TEXTURE_CUBE ||
322 templ->target == PIPE_TEXTURE_CUBE_ARRAY)
323 usage |= ISL_SURF_USAGE_CUBE_BIT;
324
325 if (templ->usage != PIPE_USAGE_STAGING) {
326 if (templ->format == PIPE_FORMAT_S8_UINT)
327 usage |= ISL_SURF_USAGE_STENCIL_BIT;
328 else if (has_depth)
329 usage |= ISL_SURF_USAGE_DEPTH_BIT;
330 }
331
332 enum pipe_format pfmt = templ->format;
333 res->internal_format = pfmt;
334
335 /* Should be handled by u_transfer_helper */
336 assert(!util_format_is_depth_and_stencil(pfmt));
337
338 struct iris_format_info fmt = iris_format_for_usage(devinfo, pfmt, usage);
339 assert(fmt.fmt != ISL_FORMAT_UNSUPPORTED);
340
341 UNUSED const bool isl_surf_created_successfully =
342 isl_surf_init(&screen->isl_dev, &res->surf,
343 .dim = target_to_isl_surf_dim(templ->target),
344 .format = fmt.fmt,
345 .width = templ->width0,
346 .height = templ->height0,
347 .depth = templ->depth0,
348 .levels = templ->last_level + 1,
349 .array_len = templ->array_size,
350 .samples = MAX2(templ->nr_samples, 1),
351 .min_alignment_B = 0,
352 .row_pitch_B = 0,
353 .usage = usage,
354 .tiling_flags = tiling_flags);
355 assert(isl_surf_created_successfully);
356
357 const char *name = "miptree";
358
359 enum iris_memory_zone memzone = IRIS_MEMZONE_OTHER;
360
361 /* These are for u_upload_mgr buffers only */
362 assert(!(templ->flags & (IRIS_RESOURCE_FLAG_SHADER_MEMZONE |
363 IRIS_RESOURCE_FLAG_SURFACE_MEMZONE |
364 IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE)));
365
366 res->bo = iris_bo_alloc_tiled(screen->bufmgr, name, res->surf.size_B,
367 memzone,
368 isl_tiling_to_i915_tiling(res->surf.tiling),
369 res->surf.row_pitch_B, 0);
370 if (!res->bo) {
371 iris_resource_destroy(pscreen, &res->base);
372 return NULL;
373 }
374
375 return &res->base;
376 }
377
378 static struct pipe_resource *
379 iris_resource_create(struct pipe_screen *pscreen,
380 const struct pipe_resource *templ)
381 {
382 if (templ->target == PIPE_BUFFER)
383 return iris_resource_create_for_buffer(pscreen, templ);
384 else
385 return iris_resource_create_with_modifiers(pscreen, templ, NULL, 0);
386 }
387
388 static uint64_t
389 tiling_to_modifier(uint32_t tiling)
390 {
391 static const uint64_t map[] = {
392 [I915_TILING_NONE] = DRM_FORMAT_MOD_LINEAR,
393 [I915_TILING_X] = I915_FORMAT_MOD_X_TILED,
394 [I915_TILING_Y] = I915_FORMAT_MOD_Y_TILED,
395 };
396
397 assert(tiling < ARRAY_SIZE(map));
398
399 return map[tiling];
400 }
401
402 static struct pipe_resource *
403 iris_resource_from_user_memory(struct pipe_screen *pscreen,
404 const struct pipe_resource *templ,
405 void *user_memory)
406 {
407 struct iris_screen *screen = (struct iris_screen *)pscreen;
408 struct iris_bufmgr *bufmgr = screen->bufmgr;
409 struct iris_resource *res = iris_alloc_resource(pscreen, templ);
410 if (!res)
411 return NULL;
412
413 assert(templ->target == PIPE_BUFFER);
414
415 res->internal_format = templ->format;
416 res->bo = iris_bo_create_userptr(bufmgr, "user",
417 user_memory, templ->width0,
418 IRIS_MEMZONE_OTHER);
419 if (!res->bo) {
420 free(res);
421 return NULL;
422 }
423
424 return &res->base;
425 }
426
427 static struct pipe_resource *
428 iris_resource_from_handle(struct pipe_screen *pscreen,
429 const struct pipe_resource *templ,
430 struct winsys_handle *whandle,
431 unsigned usage)
432 {
433 struct iris_screen *screen = (struct iris_screen *)pscreen;
434 struct gen_device_info *devinfo = &screen->devinfo;
435 struct iris_bufmgr *bufmgr = screen->bufmgr;
436 struct iris_resource *res = iris_alloc_resource(pscreen, templ);
437 if (!res)
438 return NULL;
439
440 if (whandle->offset != 0) {
441 dbg_printf("Attempt to import unsupported winsys offset %u\n",
442 whandle->offset);
443 goto fail;
444 }
445
446 switch (whandle->type) {
447 case WINSYS_HANDLE_TYPE_FD:
448 res->bo = iris_bo_import_dmabuf(bufmgr, whandle->handle);
449 break;
450 case WINSYS_HANDLE_TYPE_SHARED:
451 res->bo = iris_bo_gem_create_from_name(bufmgr, "winsys image",
452 whandle->handle);
453 break;
454 default:
455 unreachable("invalid winsys handle type");
456 }
457 if (!res->bo)
458 return NULL;
459
460 uint64_t modifier = whandle->modifier;
461 if (modifier == DRM_FORMAT_MOD_INVALID) {
462 modifier = tiling_to_modifier(res->bo->tiling_mode);
463 }
464 const struct isl_drm_modifier_info *mod_info =
465 isl_drm_modifier_get_info(modifier);
466 assert(mod_info);
467
468 isl_surf_usage_flags_t isl_usage = pipe_bind_to_isl_usage(templ->bind);
469
470 const struct iris_format_info fmt =
471 iris_format_for_usage(devinfo, templ->format, isl_usage);
472
473 if (templ->target == PIPE_BUFFER) {
474 res->surf.tiling = ISL_TILING_LINEAR;
475 } else {
476 isl_surf_init(&screen->isl_dev, &res->surf,
477 .dim = target_to_isl_surf_dim(templ->target),
478 .format = fmt.fmt,
479 .width = templ->width0,
480 .height = templ->height0,
481 .depth = templ->depth0,
482 .levels = templ->last_level + 1,
483 .array_len = templ->array_size,
484 .samples = MAX2(templ->nr_samples, 1),
485 .min_alignment_B = 0,
486 .row_pitch_B = whandle->stride,
487 .usage = isl_usage,
488 .tiling_flags = 1 << mod_info->tiling);
489
490 assert(res->bo->tiling_mode ==
491 isl_tiling_to_i915_tiling(res->surf.tiling));
492 }
493
494 return &res->base;
495
496 fail:
497 iris_resource_destroy(pscreen, &res->base);
498 return NULL;
499 }
500
501 static boolean
502 iris_resource_get_handle(struct pipe_screen *pscreen,
503 struct pipe_context *ctx,
504 struct pipe_resource *resource,
505 struct winsys_handle *whandle,
506 unsigned usage)
507 {
508 struct iris_resource *res = (struct iris_resource *)resource;
509
510 /* If this is a buffer, stride should be 0 - no need to special case */
511 whandle->stride = res->surf.row_pitch_B;
512 whandle->modifier = tiling_to_modifier(res->bo->tiling_mode);
513
514 switch (whandle->type) {
515 case WINSYS_HANDLE_TYPE_SHARED:
516 return iris_bo_flink(res->bo, &whandle->handle) == 0;
517 case WINSYS_HANDLE_TYPE_KMS:
518 whandle->handle = iris_bo_export_gem_handle(res->bo);
519 return true;
520 case WINSYS_HANDLE_TYPE_FD:
521 return iris_bo_export_dmabuf(res->bo, (int *) &whandle->handle) == 0;
522 }
523
524 return false;
525 }
526
527 static void
528 get_image_offset_el(struct isl_surf *surf, unsigned level, unsigned z,
529 unsigned *out_x0_el, unsigned *out_y0_el)
530 {
531 if (surf->dim == ISL_SURF_DIM_3D) {
532 isl_surf_get_image_offset_el(surf, level, 0, z, out_x0_el, out_y0_el);
533 } else {
534 isl_surf_get_image_offset_el(surf, level, z, 0, out_x0_el, out_y0_el);
535 }
536 }
537
538 /**
539 * Get pointer offset into stencil buffer.
540 *
541 * The stencil buffer is W tiled. Since the GTT is incapable of W fencing, we
542 * must decode the tile's layout in software.
543 *
544 * See
545 * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.2.1 W-Major Tile
546 * Format.
547 * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.3 Tiling Algorithm
548 *
549 * Even though the returned offset is always positive, the return type is
550 * signed due to
551 * commit e8b1c6d6f55f5be3bef25084fdd8b6127517e137
552 * mesa: Fix return type of _mesa_get_format_bytes() (#37351)
553 */
554 static intptr_t
555 s8_offset(uint32_t stride, uint32_t x, uint32_t y, bool swizzled)
556 {
557 uint32_t tile_size = 4096;
558 uint32_t tile_width = 64;
559 uint32_t tile_height = 64;
560 uint32_t row_size = 64 * stride / 2; /* Two rows are interleaved. */
561
562 uint32_t tile_x = x / tile_width;
563 uint32_t tile_y = y / tile_height;
564
565 /* The byte's address relative to the tile's base addres. */
566 uint32_t byte_x = x % tile_width;
567 uint32_t byte_y = y % tile_height;
568
569 uintptr_t u = tile_y * row_size
570 + tile_x * tile_size
571 + 512 * (byte_x / 8)
572 + 64 * (byte_y / 8)
573 + 32 * ((byte_y / 4) % 2)
574 + 16 * ((byte_x / 4) % 2)
575 + 8 * ((byte_y / 2) % 2)
576 + 4 * ((byte_x / 2) % 2)
577 + 2 * (byte_y % 2)
578 + 1 * (byte_x % 2);
579
580 if (swizzled) {
581 /* adjust for bit6 swizzling */
582 if (((byte_x / 8) % 2) == 1) {
583 if (((byte_y / 8) % 2) == 0) {
584 u += 64;
585 } else {
586 u -= 64;
587 }
588 }
589 }
590
591 return u;
592 }
593
594 static void
595 iris_unmap_s8(struct iris_transfer *map)
596 {
597 struct pipe_transfer *xfer = &map->base;
598 struct iris_resource *res = (struct iris_resource *) xfer->resource;
599 struct isl_surf *surf = &res->surf;
600 const bool has_swizzling = false;
601
602 if (xfer->usage & PIPE_TRANSFER_WRITE) {
603 uint8_t *untiled_s8_map = map->ptr;
604 uint8_t *tiled_s8_map =
605 iris_bo_map(map->dbg, res->bo, xfer->usage | MAP_RAW);
606
607 struct pipe_box box = xfer->box;
608
609 for (int s = 0; s < box.depth; s++) {
610 unsigned x0_el, y0_el;
611 get_image_offset_el(surf, xfer->level, box.z, &x0_el, &y0_el);
612
613 for (uint32_t y = 0; y < box.height; y++) {
614 for (uint32_t x = 0; x < box.width; x++) {
615 ptrdiff_t offset = s8_offset(surf->row_pitch_B,
616 x0_el + box.x + x,
617 y0_el + box.y + y,
618 has_swizzling);
619 tiled_s8_map[offset] =
620 untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x];
621 }
622 }
623
624 box.z++;
625 }
626 }
627
628 free(map->buffer);
629 }
630
631 static void
632 iris_map_s8(struct iris_transfer *map)
633 {
634 struct pipe_transfer *xfer = &map->base;
635 struct iris_resource *res = (struct iris_resource *) xfer->resource;
636 struct isl_surf *surf = &res->surf;
637
638 xfer->stride = surf->row_pitch_B;
639 xfer->layer_stride = xfer->stride * xfer->box.height;
640
641 /* The tiling and detiling functions require that the linear buffer has
642 * a 16-byte alignment (that is, its `x0` is 16-byte aligned). Here we
643 * over-allocate the linear buffer to get the proper alignment.
644 */
645 map->buffer = map->ptr = malloc(xfer->layer_stride * xfer->box.depth);
646 assert(map->buffer);
647
648 const bool has_swizzling = false;
649
650 /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no
651 * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless
652 * invalidate is set, since we'll be writing the whole rectangle from our
653 * temporary buffer back out.
654 */
655 if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) {
656 uint8_t *untiled_s8_map = map->ptr;
657 uint8_t *tiled_s8_map =
658 iris_bo_map(map->dbg, res->bo, xfer->usage | MAP_RAW);
659
660 struct pipe_box box = xfer->box;
661
662 for (int s = 0; s < box.depth; s++) {
663 unsigned x0_el, y0_el;
664 get_image_offset_el(surf, xfer->level, box.z, &x0_el, &y0_el);
665
666 for (uint32_t y = 0; y < box.height; y++) {
667 for (uint32_t x = 0; x < box.width; x++) {
668 ptrdiff_t offset = s8_offset(surf->row_pitch_B,
669 x0_el + box.x + x,
670 y0_el + box.y + y,
671 has_swizzling);
672 untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x] =
673 tiled_s8_map[offset];
674 }
675 }
676
677 box.z++;
678 }
679 }
680
681 map->unmap = iris_unmap_s8;
682 }
683
684 /* Compute extent parameters for use with tiled_memcpy functions.
685 * xs are in units of bytes and ys are in units of strides.
686 */
687 static inline void
688 tile_extents(struct isl_surf *surf,
689 const struct pipe_box *box,
690 unsigned level,
691 unsigned *x1_B, unsigned *x2_B,
692 unsigned *y1_el, unsigned *y2_el)
693 {
694 const struct isl_format_layout *fmtl = isl_format_get_layout(surf->format);
695 const unsigned cpp = fmtl->bpb / 8;
696
697 assert(box->x % fmtl->bw == 0);
698 assert(box->y % fmtl->bh == 0);
699
700 unsigned x0_el, y0_el;
701 get_image_offset_el(surf, level, box->z, &x0_el, &y0_el);
702
703 *x1_B = (box->x / fmtl->bw + x0_el) * cpp;
704 *y1_el = box->y / fmtl->bh + y0_el;
705 *x2_B = (DIV_ROUND_UP(box->x + box->width, fmtl->bw) + x0_el) * cpp;
706 *y2_el = DIV_ROUND_UP(box->y + box->height, fmtl->bh) + y0_el;
707 }
708
709 static void
710 iris_unmap_tiled_memcpy(struct iris_transfer *map)
711 {
712 struct pipe_transfer *xfer = &map->base;
713 struct pipe_box box = xfer->box;
714 struct iris_resource *res = (struct iris_resource *) xfer->resource;
715 struct isl_surf *surf = &res->surf;
716
717 const bool has_swizzling = false;
718
719 if (xfer->usage & PIPE_TRANSFER_WRITE) {
720 char *dst = iris_bo_map(map->dbg, res->bo, xfer->usage | MAP_RAW);
721
722 for (int s = 0; s < box.depth; s++) {
723 unsigned x1, x2, y1, y2;
724 tile_extents(surf, &box, xfer->level, &x1, &x2, &y1, &y2);
725
726 void *ptr = map->ptr + s * xfer->layer_stride;
727
728 isl_memcpy_linear_to_tiled(x1, x2, y1, y2, dst, ptr,
729 surf->row_pitch_B, xfer->stride,
730 has_swizzling, surf->tiling, ISL_MEMCPY);
731 box.z++;
732 }
733 }
734 os_free_aligned(map->buffer);
735 map->buffer = map->ptr = NULL;
736 }
737
738 static void
739 iris_map_tiled_memcpy(struct iris_transfer *map)
740 {
741 struct pipe_transfer *xfer = &map->base;
742 struct iris_resource *res = (struct iris_resource *) xfer->resource;
743 struct isl_surf *surf = &res->surf;
744
745 xfer->stride = ALIGN(surf->row_pitch_B, 16);
746 xfer->layer_stride = xfer->stride * xfer->box.height;
747
748 unsigned x1, x2, y1, y2;
749 tile_extents(surf, &xfer->box, xfer->level, &x1, &x2, &y1, &y2);
750
751 /* The tiling and detiling functions require that the linear buffer has
752 * a 16-byte alignment (that is, its `x0` is 16-byte aligned). Here we
753 * over-allocate the linear buffer to get the proper alignment.
754 */
755 map->buffer =
756 os_malloc_aligned(xfer->layer_stride * xfer->box.depth, 16);
757 assert(map->buffer);
758 map->ptr = (char *)map->buffer + (x1 & 0xf);
759
760 const bool has_swizzling = false;
761
762 // XXX: PIPE_TRANSFER_READ?
763 if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) {
764 char *src = iris_bo_map(map->dbg, res->bo, xfer->usage | MAP_RAW);
765
766 struct pipe_box box = xfer->box;
767
768 for (int s = 0; s < box.depth; s++) {
769 unsigned x1, x2, y1, y2;
770 tile_extents(surf, &box, xfer->level, &x1, &x2, &y1, &y2);
771
772 /* When transferring cubes, box.depth is counted in cubes, but
773 * box.z is counted in faces. We want to transfer only the
774 * specified face, but for all array elements. So, use 's'
775 * (the zero-based slice count) rather than box.z.
776 */
777 void *ptr = map->ptr + s * xfer->layer_stride;
778
779 isl_memcpy_tiled_to_linear(x1, x2, y1, y2, ptr, src, xfer->stride,
780 surf->row_pitch_B, has_swizzling,
781 surf->tiling, ISL_MEMCPY);
782 box.z++;
783 }
784 }
785
786 map->unmap = iris_unmap_tiled_memcpy;
787 }
788
789 static void
790 iris_map_direct(struct iris_transfer *map)
791 {
792 struct pipe_transfer *xfer = &map->base;
793 struct pipe_box *box = &xfer->box;
794 struct iris_resource *res = (struct iris_resource *) xfer->resource;
795
796 void *ptr = iris_bo_map(map->dbg, res->bo, xfer->usage);
797
798 if (res->base.target == PIPE_BUFFER) {
799 xfer->stride = 0;
800 xfer->layer_stride = 0;
801
802 map->ptr = ptr + box->x;
803 } else {
804 struct isl_surf *surf = &res->surf;
805 const struct isl_format_layout *fmtl =
806 isl_format_get_layout(surf->format);
807 const unsigned cpp = fmtl->bpb / 8;
808 unsigned x0_el, y0_el;
809
810 get_image_offset_el(surf, xfer->level, box->z, &x0_el, &y0_el);
811
812 xfer->stride = isl_surf_get_row_pitch_B(surf);
813 xfer->layer_stride = isl_surf_get_array_pitch(surf);
814
815 map->ptr = ptr + (y0_el + box->y) * xfer->stride + (x0_el + box->x) * cpp;
816 }
817 }
818
819 static void *
820 iris_transfer_map(struct pipe_context *ctx,
821 struct pipe_resource *resource,
822 unsigned level,
823 enum pipe_transfer_usage usage,
824 const struct pipe_box *box,
825 struct pipe_transfer **ptransfer)
826 {
827 struct iris_context *ice = (struct iris_context *)ctx;
828 struct iris_resource *res = (struct iris_resource *)resource;
829 struct isl_surf *surf = &res->surf;
830
831 if (surf->tiling != ISL_TILING_LINEAR &&
832 (usage & PIPE_TRANSFER_MAP_DIRECTLY))
833 return NULL;
834
835 if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
836 for (int i = 0; i < IRIS_BATCH_COUNT; i++) {
837 if (iris_batch_references(&ice->batches[i], res->bo))
838 iris_batch_flush(&ice->batches[i]);
839 }
840 }
841
842 if ((usage & PIPE_TRANSFER_DONTBLOCK) && iris_bo_busy(res->bo))
843 return NULL;
844
845 struct iris_transfer *map = slab_alloc(&ice->transfer_pool);
846 struct pipe_transfer *xfer = &map->base;
847
848 // PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
849 // PIPE_TRANSFER_DISCARD_RANGE
850
851 if (!map)
852 return NULL;
853
854 memset(map, 0, sizeof(*map));
855 map->dbg = &ice->dbg;
856
857 pipe_resource_reference(&xfer->resource, resource);
858 xfer->level = level;
859 xfer->usage = usage;
860 xfer->box = *box;
861 *ptransfer = xfer;
862
863 xfer->usage &= (PIPE_TRANSFER_READ |
864 PIPE_TRANSFER_WRITE |
865 PIPE_TRANSFER_UNSYNCHRONIZED |
866 PIPE_TRANSFER_PERSISTENT |
867 PIPE_TRANSFER_COHERENT |
868 PIPE_TRANSFER_DISCARD_RANGE);
869
870 if (surf->tiling == ISL_TILING_W) {
871 // XXX: just teach iris_map_tiled_memcpy about W tiling...
872 iris_map_s8(map);
873 } else if (surf->tiling != ISL_TILING_LINEAR) {
874 iris_map_tiled_memcpy(map);
875 } else {
876 iris_map_direct(map);
877 }
878
879 return map->ptr;
880 }
881
882 static void
883 iris_transfer_flush_region(struct pipe_context *ctx,
884 struct pipe_transfer *xfer,
885 const struct pipe_box *box)
886 {
887 struct iris_context *ice = (struct iris_context *)ctx;
888 struct iris_resource *res = (struct iris_resource *) xfer->resource;
889
890
891 // XXX: don't emit flushes in both engines...? we may also need to flush
892 // even if there isn't a draw yet - may still be stale data in caches...
893 for (int i = 0; i < IRIS_BATCH_COUNT; i++) {
894 if (ice->batches[i].contains_draw) {
895 iris_batch_maybe_flush(&ice->batches[i], 24);
896 iris_flush_and_dirty_for_history(ice, &ice->batches[i], res);
897 }
898 }
899 }
900
901 static void
902 iris_transfer_unmap(struct pipe_context *ctx, struct pipe_transfer *xfer)
903 {
904 struct iris_context *ice = (struct iris_context *)ctx;
905 struct iris_transfer *map = (void *) xfer;
906 struct iris_resource *res = (struct iris_resource *) xfer->resource;
907
908 if (map->unmap)
909 map->unmap(map);
910
911 // XXX: don't emit flushes in both engines...?
912 for (int i = 0; i < IRIS_BATCH_COUNT; i++) {
913 if (ice->batches[i].contains_draw) {
914 iris_batch_maybe_flush(&ice->batches[i], 24);
915 iris_flush_and_dirty_for_history(ice, &ice->batches[i], res);
916 }
917 }
918
919 pipe_resource_reference(&xfer->resource, NULL);
920 slab_free(&ice->transfer_pool, map);
921 }
922
923 static void
924 iris_flush_resource(struct pipe_context *ctx, struct pipe_resource *resource)
925 {
926 }
927
928 void
929 iris_flush_and_dirty_for_history(struct iris_context *ice,
930 struct iris_batch *batch,
931 struct iris_resource *res)
932 {
933 if (res->base.target != PIPE_BUFFER)
934 return;
935
936 unsigned flush = PIPE_CONTROL_CS_STALL;
937 uint64_t dirty = 0ull;
938
939 if (res->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
940 flush |= PIPE_CONTROL_CONST_CACHE_INVALIDATE |
941 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
942 dirty |= IRIS_DIRTY_CONSTANTS_VS |
943 IRIS_DIRTY_CONSTANTS_TCS |
944 IRIS_DIRTY_CONSTANTS_TES |
945 IRIS_DIRTY_CONSTANTS_GS |
946 IRIS_DIRTY_CONSTANTS_FS |
947 IRIS_DIRTY_CONSTANTS_CS |
948 IRIS_ALL_DIRTY_BINDINGS;
949 }
950
951 if (res->bind_history & PIPE_BIND_SAMPLER_VIEW)
952 flush |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
953
954 if (res->bind_history & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
955 flush |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
956
957 if (res->bind_history & (PIPE_BIND_SHADER_BUFFER | PIPE_BIND_SHADER_IMAGE))
958 flush |= PIPE_CONTROL_DATA_CACHE_FLUSH;
959
960 iris_emit_pipe_control_flush(batch, flush);
961
962 ice->state.dirty |= dirty;
963 }
964
965 static enum pipe_format
966 iris_resource_get_internal_format(struct pipe_resource *p_res)
967 {
968 struct iris_resource *res = (void *) p_res;
969 return res->internal_format;
970 }
971
972 static const struct u_transfer_vtbl transfer_vtbl = {
973 .resource_create = iris_resource_create,
974 .resource_destroy = iris_resource_destroy,
975 .transfer_map = iris_transfer_map,
976 .transfer_unmap = iris_transfer_unmap,
977 .transfer_flush_region = iris_transfer_flush_region,
978 .get_internal_format = iris_resource_get_internal_format,
979 .set_stencil = iris_resource_set_separate_stencil,
980 .get_stencil = iris_resource_get_separate_stencil,
981 };
982
983 void
984 iris_init_screen_resource_functions(struct pipe_screen *pscreen)
985 {
986 pscreen->resource_create_with_modifiers =
987 iris_resource_create_with_modifiers;
988 pscreen->resource_create = u_transfer_helper_resource_create;
989 pscreen->resource_from_user_memory = iris_resource_from_user_memory;
990 pscreen->resource_from_handle = iris_resource_from_handle;
991 pscreen->resource_get_handle = iris_resource_get_handle;
992 pscreen->resource_destroy = u_transfer_helper_resource_destroy;
993 pscreen->transfer_helper =
994 u_transfer_helper_create(&transfer_vtbl, true, true, false, true);
995 }
996
997 void
998 iris_init_resource_functions(struct pipe_context *ctx)
999 {
1000 ctx->flush_resource = iris_flush_resource;
1001 ctx->transfer_map = u_transfer_helper_transfer_map;
1002 ctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
1003 ctx->transfer_unmap = u_transfer_helper_transfer_unmap;
1004 ctx->buffer_subdata = u_default_buffer_subdata;
1005 ctx->texture_subdata = u_default_texture_subdata;
1006 }