93852bd0d57500f58606669354681e243ed211b7
[mesa.git] / src / freedreno / vulkan / tu_private.h
1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based in part on anv driver which is:
6 * Copyright © 2015 Intel Corporation
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the "Software"),
10 * to deal in the Software without restriction, including without limitation
11 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
12 * and/or sell copies of the Software, and to permit persons to whom the
13 * Software is furnished to do so, subject to the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the next
16 * paragraph) shall be included in all copies or substantial portions of the
17 * Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 * DEALINGS IN THE SOFTWARE.
26 */
27
28 #ifndef TU_PRIVATE_H
29 #define TU_PRIVATE_H
30
31 #include <assert.h>
32 #include <pthread.h>
33 #include <stdbool.h>
34 #include <stdint.h>
35 #include <stdio.h>
36 #include <stdlib.h>
37 #include <string.h>
38 #ifdef HAVE_VALGRIND
39 #include <memcheck.h>
40 #include <valgrind.h>
41 #define VG(x) x
42 #else
43 #define VG(x) ((void)0)
44 #endif
45
46 #include "c11/threads.h"
47 #include "main/macros.h"
48 #include "util/list.h"
49 #include "util/macros.h"
50 #include "util/u_atomic.h"
51 #include "vk_alloc.h"
52 #include "vk_object.h"
53 #include "vk_debug_report.h"
54 #include "wsi_common.h"
55
56 #include "drm-uapi/msm_drm.h"
57 #include "ir3/ir3_compiler.h"
58 #include "ir3/ir3_shader.h"
59
60 #include "adreno_common.xml.h"
61 #include "adreno_pm4.xml.h"
62 #include "a6xx.xml.h"
63 #include "fdl/freedreno_layout.h"
64
65 #include "tu_descriptor_set.h"
66 #include "tu_extensions.h"
67 #include "tu_util.h"
68
69 /* Pre-declarations needed for WSI entrypoints */
70 struct wl_surface;
71 struct wl_display;
72 typedef struct xcb_connection_t xcb_connection_t;
73 typedef uint32_t xcb_visualid_t;
74 typedef uint32_t xcb_window_t;
75
76 #include <vulkan/vk_android_native_buffer.h>
77 #include <vulkan/vk_icd.h>
78 #include <vulkan/vulkan.h>
79 #include <vulkan/vulkan_intel.h>
80
81 #include "tu_entrypoints.h"
82
83 #include "vk_format.h"
84
85 #define MAX_VBS 32
86 #define MAX_VERTEX_ATTRIBS 32
87 #define MAX_RTS 8
88 #define MAX_VSC_PIPES 32
89 #define MAX_VIEWPORTS 1
90 #define MAX_SCISSORS 16
91 #define MAX_DISCARD_RECTANGLES 4
92 #define MAX_PUSH_CONSTANTS_SIZE 128
93 #define MAX_PUSH_DESCRIPTORS 32
94 #define MAX_DYNAMIC_UNIFORM_BUFFERS 16
95 #define MAX_DYNAMIC_STORAGE_BUFFERS 8
96 #define MAX_DYNAMIC_BUFFERS \
97 (MAX_DYNAMIC_UNIFORM_BUFFERS + MAX_DYNAMIC_STORAGE_BUFFERS)
98 #define TU_MAX_DRM_DEVICES 8
99 #define MAX_VIEWS 8
100 #define MAX_BIND_POINTS 2 /* compute + graphics */
101 /* The Qualcomm driver exposes 0x20000058 */
102 #define MAX_STORAGE_BUFFER_RANGE 0x20000000
103 /* We use ldc for uniform buffer loads, just like the Qualcomm driver, so
104 * expose the same maximum range.
105 * TODO: The SIZE bitfield is 15 bits, and in 4-dword units, so the actual
106 * range might be higher.
107 */
108 #define MAX_UNIFORM_BUFFER_RANGE 0x10000
109
110 #define A6XX_TEX_CONST_DWORDS 16
111 #define A6XX_TEX_SAMP_DWORDS 4
112
113 #define tu_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
114
115 static inline uint32_t
116 tu_minify(uint32_t n, uint32_t levels)
117 {
118 if (unlikely(n == 0))
119 return 0;
120 else
121 return MAX2(n >> levels, 1);
122 }
123
124 #define for_each_bit(b, dword) \
125 for (uint32_t __dword = (dword); \
126 (b) = __builtin_ffs(__dword) - 1, __dword; __dword &= ~(1 << (b)))
127
128 #define typed_memcpy(dest, src, count) \
129 ({ \
130 STATIC_ASSERT(sizeof(*src) == sizeof(*dest)); \
131 memcpy((dest), (src), (count) * sizeof(*(src))); \
132 })
133
134 #define COND(bool, val) ((bool) ? (val) : 0)
135 #define BIT(bit) (1u << (bit))
136
137 /* Whenever we generate an error, pass it through this function. Useful for
138 * debugging, where we can break on it. Only call at error site, not when
139 * propagating errors. Might be useful to plug in a stack trace here.
140 */
141
142 struct tu_instance;
143
144 VkResult
145 __vk_errorf(struct tu_instance *instance,
146 VkResult error,
147 const char *file,
148 int line,
149 const char *format,
150 ...);
151
152 #define vk_error(instance, error) \
153 __vk_errorf(instance, error, __FILE__, __LINE__, NULL);
154 #define vk_errorf(instance, error, format, ...) \
155 __vk_errorf(instance, error, __FILE__, __LINE__, format, ##__VA_ARGS__);
156
157 void
158 __tu_finishme(const char *file, int line, const char *format, ...)
159 tu_printflike(3, 4);
160 void
161 tu_loge(const char *format, ...) tu_printflike(1, 2);
162 void
163 tu_logi(const char *format, ...) tu_printflike(1, 2);
164
165 /**
166 * Print a FINISHME message, including its source location.
167 */
168 #define tu_finishme(format, ...) \
169 do { \
170 static bool reported = false; \
171 if (!reported) { \
172 __tu_finishme(__FILE__, __LINE__, format, ##__VA_ARGS__); \
173 reported = true; \
174 } \
175 } while (0)
176
177 #define tu_stub() \
178 do { \
179 tu_finishme("stub %s", __func__); \
180 } while (0)
181
182 void *
183 tu_lookup_entrypoint_unchecked(const char *name);
184 void *
185 tu_lookup_entrypoint_checked(
186 const char *name,
187 uint32_t core_version,
188 const struct tu_instance_extension_table *instance,
189 const struct tu_device_extension_table *device);
190
191 struct tu_physical_device
192 {
193 struct vk_object_base base;
194
195 struct tu_instance *instance;
196
197 char path[20];
198 char name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE];
199 uint8_t driver_uuid[VK_UUID_SIZE];
200 uint8_t device_uuid[VK_UUID_SIZE];
201 uint8_t cache_uuid[VK_UUID_SIZE];
202
203 struct wsi_device wsi_device;
204
205 int local_fd;
206 int master_fd;
207
208 unsigned gpu_id;
209 uint32_t gmem_size;
210 uint64_t gmem_base;
211 uint32_t ccu_offset_gmem;
212 uint32_t ccu_offset_bypass;
213 /* alignment for size of tiles */
214 uint32_t tile_align_w;
215 #define TILE_ALIGN_H 16
216 /* gmem store/load granularity */
217 #define GMEM_ALIGN_W 16
218 #define GMEM_ALIGN_H 4
219
220 struct {
221 uint32_t PC_UNKNOWN_9805;
222 uint32_t SP_UNKNOWN_A0F8;
223 } magic;
224
225 int msm_major_version;
226 int msm_minor_version;
227
228 bool limited_z24s8;
229
230 /* This is the drivers on-disk cache used as a fallback as opposed to
231 * the pipeline cache defined by apps.
232 */
233 struct disk_cache *disk_cache;
234
235 struct tu_device_extension_table supported_extensions;
236 };
237
238 enum tu_debug_flags
239 {
240 TU_DEBUG_STARTUP = 1 << 0,
241 TU_DEBUG_NIR = 1 << 1,
242 TU_DEBUG_IR3 = 1 << 2,
243 TU_DEBUG_NOBIN = 1 << 3,
244 TU_DEBUG_SYSMEM = 1 << 4,
245 TU_DEBUG_FORCEBIN = 1 << 5,
246 TU_DEBUG_NOUBWC = 1 << 6,
247 };
248
249 struct tu_instance
250 {
251 struct vk_object_base base;
252
253 VkAllocationCallbacks alloc;
254
255 uint32_t api_version;
256 int physical_device_count;
257 struct tu_physical_device physical_devices[TU_MAX_DRM_DEVICES];
258
259 enum tu_debug_flags debug_flags;
260
261 struct vk_debug_report_instance debug_report_callbacks;
262
263 struct tu_instance_extension_table enabled_extensions;
264 };
265
266 VkResult
267 tu_wsi_init(struct tu_physical_device *physical_device);
268 void
269 tu_wsi_finish(struct tu_physical_device *physical_device);
270
271 bool
272 tu_instance_extension_supported(const char *name);
273 uint32_t
274 tu_physical_device_api_version(struct tu_physical_device *dev);
275 bool
276 tu_physical_device_extension_supported(struct tu_physical_device *dev,
277 const char *name);
278
279 struct cache_entry;
280
281 struct tu_pipeline_cache
282 {
283 struct vk_object_base base;
284
285 struct tu_device *device;
286 pthread_mutex_t mutex;
287
288 uint32_t total_size;
289 uint32_t table_size;
290 uint32_t kernel_count;
291 struct cache_entry **hash_table;
292 bool modified;
293
294 VkAllocationCallbacks alloc;
295 };
296
297 struct tu_pipeline_key
298 {
299 };
300
301
302 /* queue types */
303 #define TU_QUEUE_GENERAL 0
304
305 #define TU_MAX_QUEUE_FAMILIES 1
306
307 struct tu_fence
308 {
309 struct vk_object_base base;
310 struct wsi_fence *fence_wsi;
311 bool signaled;
312 int fd;
313 };
314
315 void
316 tu_fence_init(struct tu_fence *fence, bool signaled);
317 void
318 tu_fence_finish(struct tu_fence *fence);
319 void
320 tu_fence_update_fd(struct tu_fence *fence, int fd);
321 void
322 tu_fence_copy(struct tu_fence *fence, const struct tu_fence *src);
323 void
324 tu_fence_signal(struct tu_fence *fence);
325 void
326 tu_fence_wait_idle(struct tu_fence *fence);
327
328 struct tu_queue
329 {
330 struct vk_object_base base;
331
332 struct tu_device *device;
333 uint32_t queue_family_index;
334 int queue_idx;
335 VkDeviceQueueCreateFlags flags;
336
337 uint32_t msm_queue_id;
338 struct tu_fence submit_fence;
339 };
340
341 struct tu_bo
342 {
343 uint32_t gem_handle;
344 uint64_t size;
345 uint64_t iova;
346 void *map;
347 };
348
349 enum global_shader {
350 GLOBAL_SH_VS,
351 GLOBAL_SH_FS_BLIT,
352 GLOBAL_SH_FS_CLEAR0,
353 GLOBAL_SH_FS_CLEAR_MAX = GLOBAL_SH_FS_CLEAR0 + MAX_RTS,
354 GLOBAL_SH_COUNT,
355 };
356
357 /* This struct defines the layout of the global_bo */
358 struct tu6_global
359 {
360 /* 6 bcolor_entry entries, one for each VK_BORDER_COLOR */
361 uint8_t border_color[128 * 6];
362
363 /* clear/blit shaders, all <= 16 instrs (16 instr = 1 instrlen unit) */
364 instr_t shaders[GLOBAL_SH_COUNT][16];
365
366 uint32_t seqno_dummy; /* dummy seqno for CP_EVENT_WRITE */
367 uint32_t _pad0;
368 volatile uint32_t vsc_draw_overflow;
369 uint32_t _pad1;
370 volatile uint32_t vsc_prim_overflow;
371 uint32_t _pad2;
372 uint64_t predicate;
373
374 /* scratch space for VPC_SO[i].FLUSH_BASE_LO/HI, start on 32 byte boundary. */
375 struct {
376 uint32_t offset;
377 uint32_t pad[7];
378 } flush_base[4];
379 };
380 #define gb_offset(member) offsetof(struct tu6_global, member)
381 #define global_iova(cmd, member) ((cmd)->device->global_bo.iova + gb_offset(member))
382
383 void tu_init_clear_blit_shaders(struct tu6_global *global);
384
385 /* extra space in vsc draw/prim streams */
386 #define VSC_PAD 0x40
387
388 struct tu_device
389 {
390 struct vk_device vk;
391 struct tu_instance *instance;
392
393 struct tu_queue *queues[TU_MAX_QUEUE_FAMILIES];
394 int queue_count[TU_MAX_QUEUE_FAMILIES];
395
396 struct tu_physical_device *physical_device;
397 int _lost;
398
399 struct ir3_compiler *compiler;
400
401 /* Backup in-memory cache to be used if the app doesn't provide one */
402 struct tu_pipeline_cache *mem_cache;
403
404 #define MIN_SCRATCH_BO_SIZE_LOG2 12 /* A page */
405
406 /* Currently the kernel driver uses a 32-bit GPU address space, but it
407 * should be impossible to go beyond 48 bits.
408 */
409 struct {
410 struct tu_bo bo;
411 mtx_t construct_mtx;
412 bool initialized;
413 } scratch_bos[48 - MIN_SCRATCH_BO_SIZE_LOG2];
414
415 struct tu_bo global_bo;
416
417 struct tu_device_extension_table enabled_extensions;
418
419 uint32_t vsc_draw_strm_pitch;
420 uint32_t vsc_prim_strm_pitch;
421 mtx_t vsc_pitch_mtx;
422 };
423
424 VkResult _tu_device_set_lost(struct tu_device *device,
425 const char *file, int line,
426 const char *msg, ...) PRINTFLIKE(4, 5);
427 #define tu_device_set_lost(dev, ...) \
428 _tu_device_set_lost(dev, __FILE__, __LINE__, __VA_ARGS__)
429
430 static inline bool
431 tu_device_is_lost(struct tu_device *device)
432 {
433 return unlikely(p_atomic_read(&device->_lost));
434 }
435
436 VkResult
437 tu_bo_init_new(struct tu_device *dev, struct tu_bo *bo, uint64_t size);
438 VkResult
439 tu_bo_init_dmabuf(struct tu_device *dev,
440 struct tu_bo *bo,
441 uint64_t size,
442 int fd);
443 int
444 tu_bo_export_dmabuf(struct tu_device *dev, struct tu_bo *bo);
445 void
446 tu_bo_finish(struct tu_device *dev, struct tu_bo *bo);
447 VkResult
448 tu_bo_map(struct tu_device *dev, struct tu_bo *bo);
449
450 /* Get a scratch bo for use inside a command buffer. This will always return
451 * the same bo given the same size or similar sizes, so only one scratch bo
452 * can be used at the same time. It's meant for short-lived things where we
453 * need to write to some piece of memory, read from it, and then immediately
454 * discard it.
455 */
456 VkResult
457 tu_get_scratch_bo(struct tu_device *dev, uint64_t size, struct tu_bo **bo);
458
459 struct tu_cs_entry
460 {
461 /* No ownership */
462 const struct tu_bo *bo;
463
464 uint32_t size;
465 uint32_t offset;
466 };
467
468 struct tu_cs_memory {
469 uint32_t *map;
470 uint64_t iova;
471 };
472
473 struct tu_draw_state {
474 uint64_t iova : 48;
475 uint32_t size : 16;
476 };
477
478 enum tu_dynamic_state
479 {
480 /* re-use VK_DYNAMIC_STATE_ enums for non-extended dynamic states */
481 TU_DYNAMIC_STATE_SAMPLE_LOCATIONS = VK_DYNAMIC_STATE_STENCIL_REFERENCE + 1,
482 TU_DYNAMIC_STATE_COUNT,
483 };
484
485 enum tu_draw_state_group_id
486 {
487 TU_DRAW_STATE_PROGRAM,
488 TU_DRAW_STATE_PROGRAM_BINNING,
489 TU_DRAW_STATE_TESS,
490 TU_DRAW_STATE_VB,
491 TU_DRAW_STATE_VI,
492 TU_DRAW_STATE_VI_BINNING,
493 TU_DRAW_STATE_RAST,
494 TU_DRAW_STATE_DS,
495 TU_DRAW_STATE_BLEND,
496 TU_DRAW_STATE_VS_CONST,
497 TU_DRAW_STATE_HS_CONST,
498 TU_DRAW_STATE_DS_CONST,
499 TU_DRAW_STATE_GS_CONST,
500 TU_DRAW_STATE_FS_CONST,
501 TU_DRAW_STATE_DESC_SETS,
502 TU_DRAW_STATE_DESC_SETS_LOAD,
503 TU_DRAW_STATE_VS_PARAMS,
504 TU_DRAW_STATE_INPUT_ATTACHMENTS_GMEM,
505 TU_DRAW_STATE_INPUT_ATTACHMENTS_SYSMEM,
506
507 /* dynamic state related draw states */
508 TU_DRAW_STATE_DYNAMIC,
509 TU_DRAW_STATE_COUNT = TU_DRAW_STATE_DYNAMIC + TU_DYNAMIC_STATE_COUNT,
510 };
511
512 enum tu_cs_mode
513 {
514
515 /*
516 * A command stream in TU_CS_MODE_GROW mode grows automatically whenever it
517 * is full. tu_cs_begin must be called before command packet emission and
518 * tu_cs_end must be called after.
519 *
520 * This mode may create multiple entries internally. The entries must be
521 * submitted together.
522 */
523 TU_CS_MODE_GROW,
524
525 /*
526 * A command stream in TU_CS_MODE_EXTERNAL mode wraps an external,
527 * fixed-size buffer. tu_cs_begin and tu_cs_end are optional and have no
528 * effect on it.
529 *
530 * This mode does not create any entry or any BO.
531 */
532 TU_CS_MODE_EXTERNAL,
533
534 /*
535 * A command stream in TU_CS_MODE_SUB_STREAM mode does not support direct
536 * command packet emission. tu_cs_begin_sub_stream must be called to get a
537 * sub-stream to emit comamnd packets to. When done with the sub-stream,
538 * tu_cs_end_sub_stream must be called.
539 *
540 * This mode does not create any entry internally.
541 */
542 TU_CS_MODE_SUB_STREAM,
543 };
544
545 struct tu_cs
546 {
547 uint32_t *start;
548 uint32_t *cur;
549 uint32_t *reserved_end;
550 uint32_t *end;
551
552 struct tu_device *device;
553 enum tu_cs_mode mode;
554 uint32_t next_bo_size;
555
556 struct tu_cs_entry *entries;
557 uint32_t entry_count;
558 uint32_t entry_capacity;
559
560 struct tu_bo **bos;
561 uint32_t bo_count;
562 uint32_t bo_capacity;
563
564 /* state for cond_exec_start/cond_exec_end */
565 uint32_t cond_flags;
566 uint32_t *cond_dwords;
567 };
568
569 struct tu_device_memory
570 {
571 struct vk_object_base base;
572
573 struct tu_bo bo;
574 VkDeviceSize size;
575
576 /* for dedicated allocations */
577 struct tu_image *image;
578 struct tu_buffer *buffer;
579
580 uint32_t type_index;
581 void *map;
582 void *user_ptr;
583 };
584
585 struct tu_descriptor_range
586 {
587 uint64_t va;
588 uint32_t size;
589 };
590
591 struct tu_descriptor_set
592 {
593 struct vk_object_base base;
594
595 const struct tu_descriptor_set_layout *layout;
596 struct tu_descriptor_pool *pool;
597 uint32_t size;
598
599 uint64_t va;
600 uint32_t *mapped_ptr;
601
602 uint32_t *dynamic_descriptors;
603
604 struct tu_bo *buffers[0];
605 };
606
607 struct tu_push_descriptor_set
608 {
609 struct tu_descriptor_set set;
610 uint32_t capacity;
611 };
612
613 struct tu_descriptor_pool_entry
614 {
615 uint32_t offset;
616 uint32_t size;
617 struct tu_descriptor_set *set;
618 };
619
620 struct tu_descriptor_pool
621 {
622 struct vk_object_base base;
623
624 struct tu_bo bo;
625 uint64_t current_offset;
626 uint64_t size;
627
628 uint8_t *host_memory_base;
629 uint8_t *host_memory_ptr;
630 uint8_t *host_memory_end;
631
632 uint32_t entry_count;
633 uint32_t max_entry_count;
634 struct tu_descriptor_pool_entry entries[0];
635 };
636
637 struct tu_descriptor_update_template_entry
638 {
639 VkDescriptorType descriptor_type;
640
641 /* The number of descriptors to update */
642 uint32_t descriptor_count;
643
644 /* Into mapped_ptr or dynamic_descriptors, in units of the respective array
645 */
646 uint32_t dst_offset;
647
648 /* In dwords. Not valid/used for dynamic descriptors */
649 uint32_t dst_stride;
650
651 uint32_t buffer_offset;
652
653 /* Only valid for combined image samplers and samplers */
654 uint16_t has_sampler;
655
656 /* In bytes */
657 size_t src_offset;
658 size_t src_stride;
659
660 /* For push descriptors */
661 const uint32_t *immutable_samplers;
662 };
663
664 struct tu_descriptor_update_template
665 {
666 struct vk_object_base base;
667
668 uint32_t entry_count;
669 struct tu_descriptor_update_template_entry entry[0];
670 };
671
672 struct tu_buffer
673 {
674 struct vk_object_base base;
675
676 VkDeviceSize size;
677
678 VkBufferUsageFlags usage;
679 VkBufferCreateFlags flags;
680
681 struct tu_bo *bo;
682 VkDeviceSize bo_offset;
683 };
684
685 static inline uint64_t
686 tu_buffer_iova(struct tu_buffer *buffer)
687 {
688 return buffer->bo->iova + buffer->bo_offset;
689 }
690
691 struct tu_vertex_binding
692 {
693 struct tu_buffer *buffer;
694 VkDeviceSize offset;
695 };
696
697 const char *
698 tu_get_debug_option_name(int id);
699
700 const char *
701 tu_get_perftest_option_name(int id);
702
703 struct tu_descriptor_state
704 {
705 struct tu_descriptor_set *sets[MAX_SETS];
706 uint32_t dynamic_descriptors[MAX_DYNAMIC_BUFFERS * A6XX_TEX_CONST_DWORDS];
707 };
708
709 enum tu_cmd_dirty_bits
710 {
711 TU_CMD_DIRTY_VERTEX_BUFFERS = 1 << 2,
712 TU_CMD_DIRTY_DESC_SETS_LOAD = 1 << 3,
713 TU_CMD_DIRTY_COMPUTE_DESC_SETS_LOAD = 1 << 4,
714 TU_CMD_DIRTY_SHADER_CONSTS = 1 << 5,
715 /* all draw states were disabled and need to be re-enabled: */
716 TU_CMD_DIRTY_DRAW_STATE = 1 << 7,
717 };
718
719 /* There are only three cache domains we have to care about: the CCU, or
720 * color cache unit, which is used for color and depth/stencil attachments
721 * and copy/blit destinations, and is split conceptually into color and depth,
722 * and the universal cache or UCHE which is used for pretty much everything
723 * else, except for the CP (uncached) and host. We need to flush whenever data
724 * crosses these boundaries.
725 */
726
727 enum tu_cmd_access_mask {
728 TU_ACCESS_UCHE_READ = 1 << 0,
729 TU_ACCESS_UCHE_WRITE = 1 << 1,
730 TU_ACCESS_CCU_COLOR_READ = 1 << 2,
731 TU_ACCESS_CCU_COLOR_WRITE = 1 << 3,
732 TU_ACCESS_CCU_DEPTH_READ = 1 << 4,
733 TU_ACCESS_CCU_DEPTH_WRITE = 1 << 5,
734
735 /* Experiments have shown that while it's safe to avoid flushing the CCU
736 * after each blit/renderpass, it's not safe to assume that subsequent
737 * lookups with a different attachment state will hit unflushed cache
738 * entries. That is, the CCU needs to be flushed and possibly invalidated
739 * when accessing memory with a different attachment state. Writing to an
740 * attachment under the following conditions after clearing using the
741 * normal 2d engine path is known to have issues:
742 *
743 * - It isn't the 0'th layer.
744 * - There are more than one attachment, and this isn't the 0'th attachment
745 * (this seems to also depend on the cpp of the attachments).
746 *
747 * Our best guess is that the layer/MRT state is used when computing
748 * the location of a cache entry in CCU, to avoid conflicts. We assume that
749 * any access in a renderpass after or before an access by a transfer needs
750 * a flush/invalidate, and use the _INCOHERENT variants to represent access
751 * by a transfer.
752 */
753 TU_ACCESS_CCU_COLOR_INCOHERENT_READ = 1 << 6,
754 TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE = 1 << 7,
755 TU_ACCESS_CCU_DEPTH_INCOHERENT_READ = 1 << 8,
756 TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE = 1 << 9,
757
758 /* Accesses by the host */
759 TU_ACCESS_HOST_READ = 1 << 10,
760 TU_ACCESS_HOST_WRITE = 1 << 11,
761
762 /* Accesses by a GPU engine which bypasses any cache. e.g. writes via
763 * CP_EVENT_WRITE::BLIT and the CP are SYSMEM_WRITE.
764 */
765 TU_ACCESS_SYSMEM_READ = 1 << 12,
766 TU_ACCESS_SYSMEM_WRITE = 1 << 13,
767
768 /* Set if a WFI is required. This can be required for:
769 * - 2D engine which (on some models) doesn't wait for flushes to complete
770 * before starting
771 * - CP draw indirect opcodes, where we need to wait for any flushes to
772 * complete but the CP implicitly waits for WFI's to complete and
773 * therefore we only need a WFI after the flushes.
774 */
775 TU_ACCESS_WFI_READ = 1 << 14,
776
777 /* Set if a CP_WAIT_FOR_ME is required due to the data being read by the CP
778 * without it waiting for any WFI.
779 */
780 TU_ACCESS_WFM_READ = 1 << 15,
781
782 /* Memory writes from the CP start in-order with draws and event writes,
783 * but execute asynchronously and hence need a CP_WAIT_MEM_WRITES if read.
784 */
785 TU_ACCESS_CP_WRITE = 1 << 16,
786
787 TU_ACCESS_READ =
788 TU_ACCESS_UCHE_READ |
789 TU_ACCESS_CCU_COLOR_READ |
790 TU_ACCESS_CCU_DEPTH_READ |
791 TU_ACCESS_CCU_COLOR_INCOHERENT_READ |
792 TU_ACCESS_CCU_DEPTH_INCOHERENT_READ |
793 TU_ACCESS_HOST_READ |
794 TU_ACCESS_SYSMEM_READ |
795 TU_ACCESS_WFI_READ |
796 TU_ACCESS_WFM_READ,
797
798 TU_ACCESS_WRITE =
799 TU_ACCESS_UCHE_WRITE |
800 TU_ACCESS_CCU_COLOR_WRITE |
801 TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE |
802 TU_ACCESS_CCU_DEPTH_WRITE |
803 TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE |
804 TU_ACCESS_HOST_WRITE |
805 TU_ACCESS_SYSMEM_WRITE |
806 TU_ACCESS_CP_WRITE,
807
808 TU_ACCESS_ALL =
809 TU_ACCESS_READ |
810 TU_ACCESS_WRITE,
811 };
812
813 enum tu_cmd_flush_bits {
814 TU_CMD_FLAG_CCU_FLUSH_DEPTH = 1 << 0,
815 TU_CMD_FLAG_CCU_FLUSH_COLOR = 1 << 1,
816 TU_CMD_FLAG_CCU_INVALIDATE_DEPTH = 1 << 2,
817 TU_CMD_FLAG_CCU_INVALIDATE_COLOR = 1 << 3,
818 TU_CMD_FLAG_CACHE_FLUSH = 1 << 4,
819 TU_CMD_FLAG_CACHE_INVALIDATE = 1 << 5,
820 TU_CMD_FLAG_WAIT_MEM_WRITES = 1 << 6,
821 TU_CMD_FLAG_WAIT_FOR_IDLE = 1 << 7,
822 TU_CMD_FLAG_WAIT_FOR_ME = 1 << 8,
823
824 TU_CMD_FLAG_ALL_FLUSH =
825 TU_CMD_FLAG_CCU_FLUSH_DEPTH |
826 TU_CMD_FLAG_CCU_FLUSH_COLOR |
827 TU_CMD_FLAG_CACHE_FLUSH |
828 /* Treat the CP as a sort of "cache" which may need to be "flushed" via
829 * waiting for writes to land with WAIT_FOR_MEM_WRITES.
830 */
831 TU_CMD_FLAG_WAIT_MEM_WRITES,
832
833 TU_CMD_FLAG_GPU_INVALIDATE =
834 TU_CMD_FLAG_CCU_INVALIDATE_DEPTH |
835 TU_CMD_FLAG_CCU_INVALIDATE_COLOR |
836 TU_CMD_FLAG_CACHE_INVALIDATE,
837
838 TU_CMD_FLAG_ALL_INVALIDATE =
839 TU_CMD_FLAG_GPU_INVALIDATE |
840 /* Treat the CP as a sort of "cache" which may need to be "invalidated"
841 * via waiting for UCHE/CCU flushes to land with WFI/WFM.
842 */
843 TU_CMD_FLAG_WAIT_FOR_IDLE |
844 TU_CMD_FLAG_WAIT_FOR_ME,
845 };
846
847 /* Changing the CCU from sysmem mode to gmem mode or vice-versa is pretty
848 * heavy, involving a CCU cache flush/invalidate and a WFI in order to change
849 * which part of the gmem is used by the CCU. Here we keep track of what the
850 * state of the CCU.
851 */
852 enum tu_cmd_ccu_state {
853 TU_CMD_CCU_SYSMEM,
854 TU_CMD_CCU_GMEM,
855 TU_CMD_CCU_UNKNOWN,
856 };
857
858 struct tu_cache_state {
859 /* Caches which must be made available (flushed) eventually if there are
860 * any users outside that cache domain, and caches which must be
861 * invalidated eventually if there are any reads.
862 */
863 enum tu_cmd_flush_bits pending_flush_bits;
864 /* Pending flushes */
865 enum tu_cmd_flush_bits flush_bits;
866 };
867
868 struct tu_cmd_state
869 {
870 uint32_t dirty;
871
872 struct tu_pipeline *pipeline;
873 struct tu_pipeline *compute_pipeline;
874
875 /* Vertex buffers */
876 struct
877 {
878 struct tu_buffer *buffers[MAX_VBS];
879 VkDeviceSize offsets[MAX_VBS];
880 } vb;
881
882 /* for dynamic states that can't be emitted directly */
883 uint32_t dynamic_stencil_mask;
884 uint32_t dynamic_stencil_wrmask;
885 uint32_t dynamic_stencil_ref;
886 uint32_t dynamic_gras_su_cntl;
887
888 /* saved states to re-emit in TU_CMD_DIRTY_DRAW_STATE case */
889 struct tu_draw_state dynamic_state[TU_DYNAMIC_STATE_COUNT];
890 struct tu_draw_state vertex_buffers;
891 struct tu_draw_state shader_const[MESA_SHADER_STAGES];
892 struct tu_draw_state desc_sets;
893
894 struct tu_draw_state vs_params;
895
896 /* Index buffer */
897 uint64_t index_va;
898 uint32_t max_index_count;
899 uint8_t index_size;
900
901 /* because streamout base has to be 32-byte aligned
902 * there is an extra offset to deal with when it is
903 * unaligned
904 */
905 uint8_t streamout_offset[IR3_MAX_SO_BUFFERS];
906
907 /* Renderpasses are tricky, because we may need to flush differently if
908 * using sysmem vs. gmem and therefore we have to delay any flushing that
909 * happens before a renderpass. So we have to have two copies of the flush
910 * state, one for intra-renderpass flushes (i.e. renderpass dependencies)
911 * and one for outside a renderpass.
912 */
913 struct tu_cache_state cache;
914 struct tu_cache_state renderpass_cache;
915
916 enum tu_cmd_ccu_state ccu_state;
917
918 const struct tu_render_pass *pass;
919 const struct tu_subpass *subpass;
920 const struct tu_framebuffer *framebuffer;
921 VkRect2D render_area;
922
923 struct tu_cs_entry tile_store_ib;
924
925 bool xfb_used;
926 bool has_tess;
927 bool has_subpass_predication;
928 bool predication_active;
929 };
930
931 struct tu_cmd_pool
932 {
933 struct vk_object_base base;
934
935 VkAllocationCallbacks alloc;
936 struct list_head cmd_buffers;
937 struct list_head free_cmd_buffers;
938 uint32_t queue_family_index;
939 };
940
941 struct tu_cmd_buffer_upload
942 {
943 uint8_t *map;
944 unsigned offset;
945 uint64_t size;
946 struct list_head list;
947 };
948
949 enum tu_cmd_buffer_status
950 {
951 TU_CMD_BUFFER_STATUS_INVALID,
952 TU_CMD_BUFFER_STATUS_INITIAL,
953 TU_CMD_BUFFER_STATUS_RECORDING,
954 TU_CMD_BUFFER_STATUS_EXECUTABLE,
955 TU_CMD_BUFFER_STATUS_PENDING,
956 };
957
958 struct tu_bo_list
959 {
960 uint32_t count;
961 uint32_t capacity;
962 struct drm_msm_gem_submit_bo *bo_infos;
963 };
964
965 #define TU_BO_LIST_FAILED (~0)
966
967 void
968 tu_bo_list_init(struct tu_bo_list *list);
969 void
970 tu_bo_list_destroy(struct tu_bo_list *list);
971 void
972 tu_bo_list_reset(struct tu_bo_list *list);
973 uint32_t
974 tu_bo_list_add(struct tu_bo_list *list,
975 const struct tu_bo *bo,
976 uint32_t flags);
977 VkResult
978 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other);
979
980 struct tu_cmd_buffer
981 {
982 struct vk_object_base base;
983
984 struct tu_device *device;
985
986 struct tu_cmd_pool *pool;
987 struct list_head pool_link;
988
989 VkCommandBufferUsageFlags usage_flags;
990 VkCommandBufferLevel level;
991 enum tu_cmd_buffer_status status;
992
993 struct tu_cmd_state state;
994 struct tu_vertex_binding vertex_bindings[MAX_VBS];
995 uint32_t vertex_bindings_set;
996 uint32_t queue_family_index;
997
998 uint32_t push_constants[MAX_PUSH_CONSTANTS_SIZE / 4];
999 VkShaderStageFlags push_constant_stages;
1000 struct tu_descriptor_set meta_push_descriptors;
1001
1002 struct tu_descriptor_state descriptors[MAX_BIND_POINTS];
1003
1004 struct tu_cmd_buffer_upload upload;
1005
1006 VkResult record_result;
1007
1008 struct tu_bo_list bo_list;
1009 struct tu_cs cs;
1010 struct tu_cs draw_cs;
1011 struct tu_cs draw_epilogue_cs;
1012 struct tu_cs sub_cs;
1013
1014 uint32_t vsc_draw_strm_pitch;
1015 uint32_t vsc_prim_strm_pitch;
1016 };
1017
1018 /* Temporary struct for tracking a register state to be written, used by
1019 * a6xx-pack.h and tu_cs_emit_regs()
1020 */
1021 struct tu_reg_value {
1022 uint32_t reg;
1023 uint64_t value;
1024 bool is_address;
1025 struct tu_bo *bo;
1026 bool bo_write;
1027 uint32_t bo_offset;
1028 uint32_t bo_shift;
1029 };
1030
1031
1032 void tu_emit_cache_flush_renderpass(struct tu_cmd_buffer *cmd_buffer,
1033 struct tu_cs *cs);
1034
1035 void tu_emit_cache_flush_ccu(struct tu_cmd_buffer *cmd_buffer,
1036 struct tu_cs *cs,
1037 enum tu_cmd_ccu_state ccu_state);
1038
1039 void
1040 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
1041 struct tu_cs *cs,
1042 enum vgt_event_type event);
1043
1044 static inline struct tu_descriptor_state *
1045 tu_get_descriptors_state(struct tu_cmd_buffer *cmd_buffer,
1046 VkPipelineBindPoint bind_point)
1047 {
1048 return &cmd_buffer->descriptors[bind_point];
1049 }
1050
1051 struct tu_event
1052 {
1053 struct vk_object_base base;
1054 struct tu_bo bo;
1055 };
1056
1057 struct tu_shader_module
1058 {
1059 struct vk_object_base base;
1060
1061 unsigned char sha1[20];
1062
1063 uint32_t code_size;
1064 const uint32_t *code[0];
1065 };
1066
1067 struct tu_push_constant_range
1068 {
1069 uint32_t lo;
1070 uint32_t count;
1071 };
1072
1073 struct tu_shader
1074 {
1075 struct ir3_shader *ir3_shader;
1076
1077 struct tu_push_constant_range push_consts;
1078 uint8_t active_desc_sets;
1079 };
1080
1081 struct tu_shader *
1082 tu_shader_create(struct tu_device *dev,
1083 gl_shader_stage stage,
1084 const VkPipelineShaderStageCreateInfo *stage_info,
1085 struct tu_pipeline_layout *layout,
1086 const VkAllocationCallbacks *alloc);
1087
1088 void
1089 tu_shader_destroy(struct tu_device *dev,
1090 struct tu_shader *shader,
1091 const VkAllocationCallbacks *alloc);
1092
1093 struct tu_program_descriptor_linkage
1094 {
1095 struct ir3_const_state const_state;
1096
1097 uint32_t constlen;
1098
1099 struct tu_push_constant_range push_consts;
1100 };
1101
1102 struct tu_pipeline
1103 {
1104 struct vk_object_base base;
1105
1106 struct tu_cs cs;
1107
1108 struct tu_pipeline_layout *layout;
1109
1110 bool need_indirect_descriptor_sets;
1111 VkShaderStageFlags active_stages;
1112 uint32_t active_desc_sets;
1113
1114 /* mask of enabled dynamic states
1115 * if BIT(i) is set, pipeline->dynamic_state[i] is *NOT* used
1116 */
1117 uint32_t dynamic_state_mask;
1118 struct tu_draw_state dynamic_state[TU_DYNAMIC_STATE_COUNT];
1119
1120 /* gras_su_cntl without line width, used for dynamic line width state */
1121 uint32_t gras_su_cntl;
1122
1123 /* draw states for the pipeline */
1124 struct tu_draw_state load_state, rast_state, ds_state, blend_state;
1125
1126 struct
1127 {
1128 struct tu_draw_state state;
1129 struct tu_draw_state binning_state;
1130
1131 struct tu_program_descriptor_linkage link[MESA_SHADER_STAGES];
1132 } program;
1133
1134 struct
1135 {
1136 struct tu_draw_state state;
1137 struct tu_draw_state binning_state;
1138 uint32_t bindings_used;
1139 } vi;
1140
1141 struct
1142 {
1143 enum pc_di_primtype primtype;
1144 bool primitive_restart;
1145 } ia;
1146
1147 struct
1148 {
1149 uint32_t patch_type;
1150 uint32_t param_stride;
1151 uint32_t hs_bo_regid;
1152 uint32_t ds_bo_regid;
1153 bool upper_left_domain_origin;
1154 } tess;
1155
1156 struct
1157 {
1158 uint32_t local_size[3];
1159 } compute;
1160 };
1161
1162 void
1163 tu6_emit_viewport(struct tu_cs *cs, const VkViewport *viewport);
1164
1165 void
1166 tu6_emit_scissor(struct tu_cs *cs, const VkRect2D *scissor);
1167
1168 void
1169 tu6_emit_sample_locations(struct tu_cs *cs, const VkSampleLocationsInfoEXT *samp_loc);
1170
1171 void
1172 tu6_emit_depth_bias(struct tu_cs *cs,
1173 float constant_factor,
1174 float clamp,
1175 float slope_factor);
1176
1177 void tu6_emit_msaa(struct tu_cs *cs, VkSampleCountFlagBits samples);
1178
1179 void tu6_emit_window_scissor(struct tu_cs *cs, uint32_t x1, uint32_t y1, uint32_t x2, uint32_t y2);
1180
1181 void tu6_emit_window_offset(struct tu_cs *cs, uint32_t x1, uint32_t y1);
1182
1183 void
1184 tu6_emit_xs_config(struct tu_cs *cs,
1185 gl_shader_stage stage,
1186 const struct ir3_shader_variant *xs,
1187 uint64_t binary_iova);
1188
1189 void
1190 tu6_emit_vpc(struct tu_cs *cs,
1191 const struct ir3_shader_variant *vs,
1192 const struct ir3_shader_variant *hs,
1193 const struct ir3_shader_variant *ds,
1194 const struct ir3_shader_variant *gs,
1195 const struct ir3_shader_variant *fs,
1196 uint32_t patch_control_points,
1197 bool vshs_workgroup);
1198
1199 void
1200 tu6_emit_fs_inputs(struct tu_cs *cs, const struct ir3_shader_variant *fs);
1201
1202 struct tu_image_view;
1203
1204 void
1205 tu_resolve_sysmem(struct tu_cmd_buffer *cmd,
1206 struct tu_cs *cs,
1207 struct tu_image_view *src,
1208 struct tu_image_view *dst,
1209 uint32_t layers,
1210 const VkRect2D *rect);
1211
1212 void
1213 tu_clear_sysmem_attachment(struct tu_cmd_buffer *cmd,
1214 struct tu_cs *cs,
1215 uint32_t a,
1216 const VkRenderPassBeginInfo *info);
1217
1218 void
1219 tu_clear_gmem_attachment(struct tu_cmd_buffer *cmd,
1220 struct tu_cs *cs,
1221 uint32_t a,
1222 const VkRenderPassBeginInfo *info);
1223
1224 void
1225 tu_load_gmem_attachment(struct tu_cmd_buffer *cmd,
1226 struct tu_cs *cs,
1227 uint32_t a,
1228 bool force_load);
1229
1230 /* expose this function to be able to emit load without checking LOAD_OP */
1231 void
1232 tu_emit_load_gmem_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t a);
1233
1234 /* note: gmem store can also resolve */
1235 void
1236 tu_store_gmem_attachment(struct tu_cmd_buffer *cmd,
1237 struct tu_cs *cs,
1238 uint32_t a,
1239 uint32_t gmem_a);
1240
1241 enum tu_supported_formats {
1242 FMT_VERTEX = 1,
1243 FMT_TEXTURE = 2,
1244 FMT_COLOR = 4,
1245 };
1246
1247 struct tu_native_format
1248 {
1249 enum a6xx_format fmt : 8;
1250 enum a3xx_color_swap swap : 8;
1251 enum a6xx_tile_mode tile_mode : 8;
1252 enum tu_supported_formats supported : 8;
1253 };
1254
1255 struct tu_native_format tu6_format_vtx(VkFormat format);
1256 struct tu_native_format tu6_format_color(VkFormat format, enum a6xx_tile_mode tile_mode);
1257 struct tu_native_format tu6_format_texture(VkFormat format, enum a6xx_tile_mode tile_mode);
1258
1259 static inline enum a6xx_format
1260 tu6_base_format(VkFormat format)
1261 {
1262 /* note: tu6_format_color doesn't care about tiling for .fmt field */
1263 return tu6_format_color(format, TILE6_LINEAR).fmt;
1264 }
1265
1266 struct tu_image
1267 {
1268 struct vk_object_base base;
1269
1270 VkImageType type;
1271 /* The original VkFormat provided by the client. This may not match any
1272 * of the actual surface formats.
1273 */
1274 VkFormat vk_format;
1275 VkImageAspectFlags aspects;
1276 VkImageUsageFlags usage; /**< Superset of VkImageCreateInfo::usage. */
1277 VkImageTiling tiling; /** VkImageCreateInfo::tiling */
1278 VkImageCreateFlags flags; /** VkImageCreateInfo::flags */
1279 VkExtent3D extent;
1280 uint32_t level_count;
1281 uint32_t layer_count;
1282 VkSampleCountFlagBits samples;
1283
1284 struct fdl_layout layout[3];
1285 uint32_t total_size;
1286
1287 unsigned queue_family_mask;
1288 bool exclusive;
1289 bool shareable;
1290
1291 /* For VK_ANDROID_native_buffer, the WSI image owns the memory, */
1292 VkDeviceMemory owned_memory;
1293
1294 /* Set when bound */
1295 struct tu_bo *bo;
1296 VkDeviceSize bo_offset;
1297 };
1298
1299 static inline uint32_t
1300 tu_get_layerCount(const struct tu_image *image,
1301 const VkImageSubresourceRange *range)
1302 {
1303 return range->layerCount == VK_REMAINING_ARRAY_LAYERS
1304 ? image->layer_count - range->baseArrayLayer
1305 : range->layerCount;
1306 }
1307
1308 static inline uint32_t
1309 tu_get_levelCount(const struct tu_image *image,
1310 const VkImageSubresourceRange *range)
1311 {
1312 return range->levelCount == VK_REMAINING_MIP_LEVELS
1313 ? image->level_count - range->baseMipLevel
1314 : range->levelCount;
1315 }
1316
1317 struct tu_image_view
1318 {
1319 struct vk_object_base base;
1320
1321 struct tu_image *image; /**< VkImageViewCreateInfo::image */
1322
1323 uint64_t base_addr;
1324 uint64_t ubwc_addr;
1325 uint32_t layer_size;
1326 uint32_t ubwc_layer_size;
1327
1328 /* used to determine if fast gmem store path can be used */
1329 VkExtent2D extent;
1330 bool need_y2_align;
1331
1332 bool ubwc_enabled;
1333
1334 uint32_t descriptor[A6XX_TEX_CONST_DWORDS];
1335
1336 /* Descriptor for use as a storage image as opposed to a sampled image.
1337 * This has a few differences for cube maps (e.g. type).
1338 */
1339 uint32_t storage_descriptor[A6XX_TEX_CONST_DWORDS];
1340
1341 /* pre-filled register values */
1342 uint32_t PITCH;
1343 uint32_t FLAG_BUFFER_PITCH;
1344
1345 uint32_t RB_MRT_BUF_INFO;
1346 uint32_t SP_FS_MRT_REG;
1347
1348 uint32_t SP_PS_2D_SRC_INFO;
1349 uint32_t SP_PS_2D_SRC_SIZE;
1350
1351 uint32_t RB_2D_DST_INFO;
1352
1353 uint32_t RB_BLIT_DST_INFO;
1354
1355 /* for d32s8 separate stencil */
1356 uint64_t stencil_base_addr;
1357 uint32_t stencil_layer_size;
1358 uint32_t stencil_PITCH;
1359 };
1360
1361 struct tu_sampler_ycbcr_conversion {
1362 struct vk_object_base base;
1363
1364 VkFormat format;
1365 VkSamplerYcbcrModelConversion ycbcr_model;
1366 VkSamplerYcbcrRange ycbcr_range;
1367 VkComponentMapping components;
1368 VkChromaLocation chroma_offsets[2];
1369 VkFilter chroma_filter;
1370 };
1371
1372 struct tu_sampler {
1373 struct vk_object_base base;
1374
1375 uint32_t descriptor[A6XX_TEX_SAMP_DWORDS];
1376 struct tu_sampler_ycbcr_conversion *ycbcr_sampler;
1377 };
1378
1379 void
1380 tu_cs_image_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1381
1382 void
1383 tu_cs_image_ref_2d(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer, bool src);
1384
1385 void
1386 tu_cs_image_flag_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1387
1388 void
1389 tu_cs_image_stencil_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1390
1391 #define tu_image_view_stencil(iview, x) \
1392 ((iview->x & ~A6XX_##x##_COLOR_FORMAT__MASK) | A6XX_##x##_COLOR_FORMAT(FMT6_8_UINT))
1393
1394 VkResult
1395 tu_image_create(VkDevice _device,
1396 const VkImageCreateInfo *pCreateInfo,
1397 const VkAllocationCallbacks *alloc,
1398 VkImage *pImage,
1399 uint64_t modifier,
1400 const VkSubresourceLayout *plane_layouts);
1401
1402 VkResult
1403 tu_image_from_gralloc(VkDevice device_h,
1404 const VkImageCreateInfo *base_info,
1405 const VkNativeBufferANDROID *gralloc_info,
1406 const VkAllocationCallbacks *alloc,
1407 VkImage *out_image_h);
1408
1409 void
1410 tu_image_view_init(struct tu_image_view *iview,
1411 const VkImageViewCreateInfo *pCreateInfo,
1412 bool limited_z24s8);
1413
1414 struct tu_buffer_view
1415 {
1416 struct vk_object_base base;
1417
1418 uint32_t descriptor[A6XX_TEX_CONST_DWORDS];
1419
1420 struct tu_buffer *buffer;
1421 };
1422 void
1423 tu_buffer_view_init(struct tu_buffer_view *view,
1424 struct tu_device *device,
1425 const VkBufferViewCreateInfo *pCreateInfo);
1426
1427 struct tu_attachment_info
1428 {
1429 struct tu_image_view *attachment;
1430 };
1431
1432 struct tu_framebuffer
1433 {
1434 struct vk_object_base base;
1435
1436 uint32_t width;
1437 uint32_t height;
1438 uint32_t layers;
1439
1440 /* size of the first tile */
1441 VkExtent2D tile0;
1442 /* number of tiles */
1443 VkExtent2D tile_count;
1444
1445 /* size of the first VSC pipe */
1446 VkExtent2D pipe0;
1447 /* number of VSC pipes */
1448 VkExtent2D pipe_count;
1449
1450 /* pipe register values */
1451 uint32_t pipe_config[MAX_VSC_PIPES];
1452 uint32_t pipe_sizes[MAX_VSC_PIPES];
1453
1454 uint32_t attachment_count;
1455 struct tu_attachment_info attachments[0];
1456 };
1457
1458 void
1459 tu_framebuffer_tiling_config(struct tu_framebuffer *fb,
1460 const struct tu_device *device,
1461 const struct tu_render_pass *pass);
1462
1463 struct tu_subpass_barrier {
1464 VkPipelineStageFlags src_stage_mask;
1465 VkAccessFlags src_access_mask;
1466 VkAccessFlags dst_access_mask;
1467 bool incoherent_ccu_color, incoherent_ccu_depth;
1468 };
1469
1470 struct tu_subpass_attachment
1471 {
1472 uint32_t attachment;
1473 };
1474
1475 struct tu_subpass
1476 {
1477 uint32_t input_count;
1478 uint32_t color_count;
1479 struct tu_subpass_attachment *input_attachments;
1480 struct tu_subpass_attachment *color_attachments;
1481 struct tu_subpass_attachment *resolve_attachments;
1482 struct tu_subpass_attachment depth_stencil_attachment;
1483
1484 VkSampleCountFlagBits samples;
1485
1486 uint32_t srgb_cntl;
1487
1488 struct tu_subpass_barrier start_barrier;
1489 };
1490
1491 struct tu_render_pass_attachment
1492 {
1493 VkFormat format;
1494 uint32_t samples;
1495 uint32_t cpp;
1496 VkImageAspectFlags clear_mask;
1497 bool load;
1498 bool store;
1499 int32_t gmem_offset;
1500 /* for D32S8 separate stencil: */
1501 bool load_stencil;
1502 bool store_stencil;
1503 int32_t gmem_offset_stencil;
1504 };
1505
1506 struct tu_render_pass
1507 {
1508 struct vk_object_base base;
1509
1510 uint32_t attachment_count;
1511 uint32_t subpass_count;
1512 uint32_t gmem_pixels;
1513 uint32_t tile_align_w;
1514 struct tu_subpass_attachment *subpass_attachments;
1515 struct tu_render_pass_attachment *attachments;
1516 struct tu_subpass_barrier end_barrier;
1517 struct tu_subpass subpasses[0];
1518 };
1519
1520 struct tu_query_pool
1521 {
1522 struct vk_object_base base;
1523
1524 VkQueryType type;
1525 uint32_t stride;
1526 uint64_t size;
1527 uint32_t pipeline_statistics;
1528 struct tu_bo bo;
1529 };
1530
1531 enum tu_semaphore_kind
1532 {
1533 TU_SEMAPHORE_NONE,
1534 TU_SEMAPHORE_SYNCOBJ,
1535 };
1536
1537 struct tu_semaphore_part
1538 {
1539 enum tu_semaphore_kind kind;
1540 union {
1541 uint32_t syncobj;
1542 };
1543 };
1544
1545 struct tu_semaphore
1546 {
1547 struct vk_object_base base;
1548
1549 struct tu_semaphore_part permanent;
1550 struct tu_semaphore_part temporary;
1551 };
1552
1553 void
1554 tu_set_descriptor_set(struct tu_cmd_buffer *cmd_buffer,
1555 VkPipelineBindPoint bind_point,
1556 struct tu_descriptor_set *set,
1557 unsigned idx);
1558
1559 void
1560 tu_update_descriptor_sets(struct tu_device *device,
1561 struct tu_cmd_buffer *cmd_buffer,
1562 VkDescriptorSet overrideSet,
1563 uint32_t descriptorWriteCount,
1564 const VkWriteDescriptorSet *pDescriptorWrites,
1565 uint32_t descriptorCopyCount,
1566 const VkCopyDescriptorSet *pDescriptorCopies);
1567
1568 void
1569 tu_update_descriptor_set_with_template(
1570 struct tu_device *device,
1571 struct tu_cmd_buffer *cmd_buffer,
1572 struct tu_descriptor_set *set,
1573 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1574 const void *pData);
1575
1576 VkResult
1577 tu_physical_device_init(struct tu_physical_device *device,
1578 struct tu_instance *instance);
1579 VkResult
1580 tu_enumerate_devices(struct tu_instance *instance);
1581
1582 int
1583 tu_drm_submitqueue_new(const struct tu_device *dev,
1584 int priority,
1585 uint32_t *queue_id);
1586
1587 void
1588 tu_drm_submitqueue_close(const struct tu_device *dev, uint32_t queue_id);
1589
1590 uint32_t
1591 tu_gem_new(const struct tu_device *dev, uint64_t size, uint32_t flags);
1592 uint32_t
1593 tu_gem_import_dmabuf(const struct tu_device *dev,
1594 int prime_fd,
1595 uint64_t size);
1596 int
1597 tu_gem_export_dmabuf(const struct tu_device *dev, uint32_t gem_handle);
1598 void
1599 tu_gem_close(const struct tu_device *dev, uint32_t gem_handle);
1600 uint64_t
1601 tu_gem_info_offset(const struct tu_device *dev, uint32_t gem_handle);
1602 uint64_t
1603 tu_gem_info_iova(const struct tu_device *dev, uint32_t gem_handle);
1604
1605 #define TU_DEFINE_HANDLE_CASTS(__tu_type, __VkType) \
1606 \
1607 static inline struct __tu_type *__tu_type##_from_handle(__VkType _handle) \
1608 { \
1609 return (struct __tu_type *) _handle; \
1610 } \
1611 \
1612 static inline __VkType __tu_type##_to_handle(struct __tu_type *_obj) \
1613 { \
1614 return (__VkType) _obj; \
1615 }
1616
1617 #define TU_DEFINE_NONDISP_HANDLE_CASTS(__tu_type, __VkType) \
1618 \
1619 static inline struct __tu_type *__tu_type##_from_handle(__VkType _handle) \
1620 { \
1621 return (struct __tu_type *) (uintptr_t) _handle; \
1622 } \
1623 \
1624 static inline __VkType __tu_type##_to_handle(struct __tu_type *_obj) \
1625 { \
1626 return (__VkType)(uintptr_t) _obj; \
1627 }
1628
1629 #define TU_FROM_HANDLE(__tu_type, __name, __handle) \
1630 struct __tu_type *__name = __tu_type##_from_handle(__handle)
1631
1632 TU_DEFINE_HANDLE_CASTS(tu_cmd_buffer, VkCommandBuffer)
1633 TU_DEFINE_HANDLE_CASTS(tu_device, VkDevice)
1634 TU_DEFINE_HANDLE_CASTS(tu_instance, VkInstance)
1635 TU_DEFINE_HANDLE_CASTS(tu_physical_device, VkPhysicalDevice)
1636 TU_DEFINE_HANDLE_CASTS(tu_queue, VkQueue)
1637
1638 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_cmd_pool, VkCommandPool)
1639 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_buffer, VkBuffer)
1640 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_buffer_view, VkBufferView)
1641 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_pool, VkDescriptorPool)
1642 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_set, VkDescriptorSet)
1643 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_set_layout,
1644 VkDescriptorSetLayout)
1645 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_update_template,
1646 VkDescriptorUpdateTemplate)
1647 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_device_memory, VkDeviceMemory)
1648 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_fence, VkFence)
1649 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_event, VkEvent)
1650 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_framebuffer, VkFramebuffer)
1651 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_image, VkImage)
1652 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_image_view, VkImageView);
1653 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline_cache, VkPipelineCache)
1654 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline, VkPipeline)
1655 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline_layout, VkPipelineLayout)
1656 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_query_pool, VkQueryPool)
1657 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_render_pass, VkRenderPass)
1658 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_sampler, VkSampler)
1659 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_sampler_ycbcr_conversion, VkSamplerYcbcrConversion)
1660 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_shader_module, VkShaderModule)
1661 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_semaphore, VkSemaphore)
1662
1663 #endif /* TU_PRIVATE_H */