src/freedreno/vulkan/tu_private.h

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  */
  27
  28 #ifndef TU_PRIVATE_H
  29 #define TU_PRIVATE_H
  30
  31 #include <assert.h>
  32 #include <pthread.h>
  33 #include <stdbool.h>
  34 #include <stdint.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #ifdef HAVE_VALGRIND
  39 #include <memcheck.h>
  40 #include <valgrind.h>
  41 #define VG(x) x
  42 #else
  43 #define VG(x) ((void)0)
  44 #endif
  45
  46 #include "c11/threads.h"
  47 #include "main/macros.h"
  48 #include "util/list.h"
  49 #include "util/macros.h"
  50 #include "util/u_atomic.h"
  51 #include "vk_alloc.h"
  52 #include "vk_object.h"
  53 #include "vk_debug_report.h"
  54 #include "wsi_common.h"
  55
  56 #include "drm-uapi/msm_drm.h"
  57 #include "ir3/ir3_compiler.h"
  58 #include "ir3/ir3_shader.h"
  59
  60 #include "adreno_common.xml.h"
  61 #include "adreno_pm4.xml.h"
  62 #include "a6xx.xml.h"
  63 #include "fdl/freedreno_layout.h"
  64
  65 #include "tu_descriptor_set.h"
  66 #include "tu_extensions.h"
  67 #include "tu_util.h"
  68
  69 /* Pre-declarations needed for WSI entrypoints */
  70 struct wl_surface;
  71 struct wl_display;
  72 typedef struct xcb_connection_t xcb_connection_t;
  73 typedef uint32_t xcb_visualid_t;
  74 typedef uint32_t xcb_window_t;
  75
  76 #include <vulkan/vk_android_native_buffer.h>
  77 #include <vulkan/vk_icd.h>
  78 #include <vulkan/vulkan.h>
  79 #include <vulkan/vulkan_intel.h>
  80
  81 #include "tu_entrypoints.h"
  82
  83 #include "vk_format.h"
  84
  85 #define MAX_VBS 32
  86 #define MAX_VERTEX_ATTRIBS 32
  87 #define MAX_RTS 8
  88 #define MAX_VSC_PIPES 32
  89 #define MAX_VIEWPORTS 1
  90 #define MAX_SCISSORS 16
  91 #define MAX_DISCARD_RECTANGLES 4
  92 #define MAX_PUSH_CONSTANTS_SIZE 128
  93 #define MAX_PUSH_DESCRIPTORS 32
  94 #define MAX_DYNAMIC_UNIFORM_BUFFERS 16
  95 #define MAX_DYNAMIC_STORAGE_BUFFERS 8
  96 #define MAX_DYNAMIC_BUFFERS                                                  \
  97    (MAX_DYNAMIC_UNIFORM_BUFFERS + MAX_DYNAMIC_STORAGE_BUFFERS)
  98 #define TU_MAX_DRM_DEVICES 8
  99 #define MAX_VIEWS 8
 100 #define MAX_BIND_POINTS 2 /* compute + graphics */
 101 /* The Qualcomm driver exposes 0x20000058 */
 102 #define MAX_STORAGE_BUFFER_RANGE 0x20000000
 103 /* We use ldc for uniform buffer loads, just like the Qualcomm driver, so
 104  * expose the same maximum range.
 105  * TODO: The SIZE bitfield is 15 bits, and in 4-dword units, so the actual
 106  * range might be higher.
 107  */
 108 #define MAX_UNIFORM_BUFFER_RANGE 0x10000
 109
 110 #define A6XX_TEX_CONST_DWORDS 16
 111 #define A6XX_TEX_SAMP_DWORDS 4
 112
 113 #define tu_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
 114
 115 static inline uint32_t
 116 tu_minify(uint32_t n, uint32_t levels)
 117 {
 118    if (unlikely(n == 0))
 119       return 0;
 120    else
 121       return MAX2(n >> levels, 1);
 122 }
 123
 124 #define for_each_bit(b, dword)                                               \
 125    for (uint32_t __dword = (dword);                                          \
 126         (b) = __builtin_ffs(__dword) - 1, __dword; __dword &= ~(1 << (b)))
 127
 128 #define typed_memcpy(dest, src, count)                                       \
 129    ({                                                                        \
 130       STATIC_ASSERT(sizeof(*src) == sizeof(*dest));                          \
 131       memcpy((dest), (src), (count) * sizeof(*(src)));                       \
 132    })
 133
 134 #define COND(bool, val) ((bool) ? (val) : 0)
 135 #define BIT(bit) (1u << (bit))
 136
 137 /* Whenever we generate an error, pass it through this function. Useful for
 138  * debugging, where we can break on it. Only call at error site, not when
 139  * propagating errors. Might be useful to plug in a stack trace here.
 140  */
 141
 142 struct tu_instance;
 143
 144 VkResult
 145 __vk_errorf(struct tu_instance *instance,
 146             VkResult error,
 147             const char *file,
 148             int line,
 149             const char *format,
 150             ...);
 151
 152 #define vk_error(instance, error)                                            \
 153    __vk_errorf(instance, error, __FILE__, __LINE__, NULL);
 154 #define vk_errorf(instance, error, format, ...)                              \
 155    __vk_errorf(instance, error, __FILE__, __LINE__, format, ##__VA_ARGS__);
 156
 157 void
 158 __tu_finishme(const char *file, int line, const char *format, ...)
 159    tu_printflike(3, 4);
 160 void
 161 tu_loge(const char *format, ...) tu_printflike(1, 2);
 162 void
 163 tu_logi(const char *format, ...) tu_printflike(1, 2);
 164
 165 /**
 166  * Print a FINISHME message, including its source location.
 167  */
 168 #define tu_finishme(format, ...)                                             \
 169    do {                                                                      \
 170       static bool reported = false;                                          \
 171       if (!reported) {                                                       \
 172          __tu_finishme(__FILE__, __LINE__, format, ##__VA_ARGS__);           \
 173          reported = true;                                                    \
 174       }                                                                      \
 175    } while (0)
 176
 177 #define tu_stub()                                                            \
 178    do {                                                                      \
 179       tu_finishme("stub %s", __func__);                                      \
 180    } while (0)
 181
 182 void *
 183 tu_lookup_entrypoint_unchecked(const char *name);
 184 void *
 185 tu_lookup_entrypoint_checked(
 186    const char *name,
 187    uint32_t core_version,
 188    const struct tu_instance_extension_table *instance,
 189    const struct tu_device_extension_table *device);
 190
 191 struct tu_physical_device
 192 {
 193    struct vk_object_base base;
 194
 195    struct tu_instance *instance;
 196
 197    char path[20];
 198    char name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE];
 199    uint8_t driver_uuid[VK_UUID_SIZE];
 200    uint8_t device_uuid[VK_UUID_SIZE];
 201    uint8_t cache_uuid[VK_UUID_SIZE];
 202
 203    struct wsi_device wsi_device;
 204
 205    int local_fd;
 206    int master_fd;
 207
 208    unsigned gpu_id;
 209    uint32_t gmem_size;
 210    uint64_t gmem_base;
 211    uint32_t ccu_offset_gmem;
 212    uint32_t ccu_offset_bypass;
 213    /* alignment for size of tiles */
 214    uint32_t tile_align_w;
 215 #define TILE_ALIGN_H 16
 216    /* gmem store/load granularity */
 217 #define GMEM_ALIGN_W 16
 218 #define GMEM_ALIGN_H 4
 219
 220    struct {
 221       uint32_t PC_UNKNOWN_9805;
 222       uint32_t SP_UNKNOWN_A0F8;
 223    } magic;
 224
 225    int msm_major_version;
 226    int msm_minor_version;
 227
 228    /* This is the drivers on-disk cache used as a fallback as opposed to
 229     * the pipeline cache defined by apps.
 230     */
 231    struct disk_cache *disk_cache;
 232
 233    struct tu_device_extension_table supported_extensions;
 234 };
 235
 236 enum tu_debug_flags
 237 {
 238    TU_DEBUG_STARTUP = 1 << 0,
 239    TU_DEBUG_NIR = 1 << 1,
 240    TU_DEBUG_IR3 = 1 << 2,
 241    TU_DEBUG_NOBIN = 1 << 3,
 242    TU_DEBUG_SYSMEM = 1 << 4,
 243    TU_DEBUG_FORCEBIN = 1 << 5,
 244    TU_DEBUG_NOUBWC = 1 << 6,
 245 };
 246
 247 struct tu_instance
 248 {
 249    struct vk_object_base base;
 250
 251    VkAllocationCallbacks alloc;
 252
 253    uint32_t api_version;
 254    int physical_device_count;
 255    struct tu_physical_device physical_devices[TU_MAX_DRM_DEVICES];
 256
 257    enum tu_debug_flags debug_flags;
 258
 259    struct vk_debug_report_instance debug_report_callbacks;
 260
 261    struct tu_instance_extension_table enabled_extensions;
 262 };
 263
 264 VkResult
 265 tu_wsi_init(struct tu_physical_device *physical_device);
 266 void
 267 tu_wsi_finish(struct tu_physical_device *physical_device);
 268
 269 bool
 270 tu_instance_extension_supported(const char *name);
 271 uint32_t
 272 tu_physical_device_api_version(struct tu_physical_device *dev);
 273 bool
 274 tu_physical_device_extension_supported(struct tu_physical_device *dev,
 275                                        const char *name);
 276
 277 struct cache_entry;
 278
 279 struct tu_pipeline_cache
 280 {
 281    struct vk_object_base base;
 282
 283    struct tu_device *device;
 284    pthread_mutex_t mutex;
 285
 286    uint32_t total_size;
 287    uint32_t table_size;
 288    uint32_t kernel_count;
 289    struct cache_entry **hash_table;
 290    bool modified;
 291
 292    VkAllocationCallbacks alloc;
 293 };
 294
 295 struct tu_pipeline_key
 296 {
 297 };
 298
 299
 300 /* queue types */
 301 #define TU_QUEUE_GENERAL 0
 302
 303 #define TU_MAX_QUEUE_FAMILIES 1
 304
 305 struct tu_fence
 306 {
 307    struct vk_object_base base;
 308    struct wsi_fence *fence_wsi;
 309    bool signaled;
 310    int fd;
 311 };
 312
 313 void
 314 tu_fence_init(struct tu_fence *fence, bool signaled);
 315 void
 316 tu_fence_finish(struct tu_fence *fence);
 317 void
 318 tu_fence_update_fd(struct tu_fence *fence, int fd);
 319 void
 320 tu_fence_copy(struct tu_fence *fence, const struct tu_fence *src);
 321 void
 322 tu_fence_signal(struct tu_fence *fence);
 323 void
 324 tu_fence_wait_idle(struct tu_fence *fence);
 325
 326 struct tu_queue
 327 {
 328    struct vk_object_base base;
 329
 330    struct tu_device *device;
 331    uint32_t queue_family_index;
 332    int queue_idx;
 333    VkDeviceQueueCreateFlags flags;
 334
 335    uint32_t msm_queue_id;
 336    struct tu_fence submit_fence;
 337 };
 338
 339 struct tu_bo
 340 {
 341    uint32_t gem_handle;
 342    uint64_t size;
 343    uint64_t iova;
 344    void *map;
 345 };
 346
 347 enum global_shader {
 348    GLOBAL_SH_VS,
 349    GLOBAL_SH_VS_LAYER,
 350    GLOBAL_SH_GS_LAYER,
 351    GLOBAL_SH_FS_BLIT,
 352    GLOBAL_SH_FS_CLEAR0,
 353    GLOBAL_SH_FS_CLEAR_MAX = GLOBAL_SH_FS_CLEAR0 + MAX_RTS,
 354    GLOBAL_SH_COUNT,
 355 };
 356
 357 /* This struct defines the layout of the global_bo */
 358 struct tu6_global
 359 {
 360    /* 6 bcolor_entry entries, one for each VK_BORDER_COLOR */
 361    uint8_t border_color[128 * 6];
 362
 363    /* clear/blit shaders, all <= 16 instrs (16 instr = 1 instrlen unit) */
 364    instr_t shaders[GLOBAL_SH_COUNT][16];
 365
 366    uint32_t seqno_dummy;          /* dummy seqno for CP_EVENT_WRITE */
 367    uint32_t _pad0;
 368    volatile uint32_t vsc_draw_overflow;
 369    uint32_t _pad1;
 370    volatile uint32_t vsc_prim_overflow;
 371    uint32_t _pad2[3];
 372
 373    /* scratch space for VPC_SO[i].FLUSH_BASE_LO/HI, start on 32 byte boundary. */
 374    struct {
 375       uint32_t offset;
 376       uint32_t pad[7];
 377    } flush_base[4];
 378 };
 379 #define gb_offset(member) offsetof(struct tu6_global, member)
 380 #define global_iova(cmd, member) ((cmd)->device->global_bo.iova + gb_offset(member))
 381
 382 void tu_init_clear_blit_shaders(struct tu6_global *global);
 383
 384 /* extra space in vsc draw/prim streams */
 385 #define VSC_PAD 0x40
 386
 387 struct tu_device
 388 {
 389    struct vk_device vk;
 390    struct tu_instance *instance;
 391
 392    struct tu_queue *queues[TU_MAX_QUEUE_FAMILIES];
 393    int queue_count[TU_MAX_QUEUE_FAMILIES];
 394
 395    struct tu_physical_device *physical_device;
 396    int _lost;
 397
 398    struct ir3_compiler *compiler;
 399
 400    /* Backup in-memory cache to be used if the app doesn't provide one */
 401    struct tu_pipeline_cache *mem_cache;
 402
 403 #define MIN_SCRATCH_BO_SIZE_LOG2 12 /* A page */
 404
 405    /* Currently the kernel driver uses a 32-bit GPU address space, but it
 406     * should be impossible to go beyond 48 bits.
 407     */
 408    struct {
 409       struct tu_bo bo;
 410       mtx_t construct_mtx;
 411       bool initialized;
 412    } scratch_bos[48 - MIN_SCRATCH_BO_SIZE_LOG2];
 413
 414    struct tu_bo global_bo;
 415
 416    struct tu_device_extension_table enabled_extensions;
 417
 418    uint32_t vsc_draw_strm_pitch;
 419    uint32_t vsc_prim_strm_pitch;
 420    mtx_t vsc_pitch_mtx;
 421 };
 422
 423 VkResult _tu_device_set_lost(struct tu_device *device,
 424                              const char *file, int line,
 425                              const char *msg, ...) PRINTFLIKE(4, 5);
 426 #define tu_device_set_lost(dev, ...) \
 427    _tu_device_set_lost(dev, __FILE__, __LINE__, __VA_ARGS__)
 428
 429 static inline bool
 430 tu_device_is_lost(struct tu_device *device)
 431 {
 432    return unlikely(p_atomic_read(&device->_lost));
 433 }
 434
 435 VkResult
 436 tu_bo_init_new(struct tu_device *dev, struct tu_bo *bo, uint64_t size);
 437 VkResult
 438 tu_bo_init_dmabuf(struct tu_device *dev,
 439                   struct tu_bo *bo,
 440                   uint64_t size,
 441                   int fd);
 442 int
 443 tu_bo_export_dmabuf(struct tu_device *dev, struct tu_bo *bo);
 444 void
 445 tu_bo_finish(struct tu_device *dev, struct tu_bo *bo);
 446 VkResult
 447 tu_bo_map(struct tu_device *dev, struct tu_bo *bo);
 448
 449 /* Get a scratch bo for use inside a command buffer. This will always return
 450  * the same bo given the same size or similar sizes, so only one scratch bo
 451  * can be used at the same time. It's meant for short-lived things where we
 452  * need to write to some piece of memory, read from it, and then immediately
 453  * discard it.
 454  */
 455 VkResult
 456 tu_get_scratch_bo(struct tu_device *dev, uint64_t size, struct tu_bo **bo);
 457
 458 struct tu_cs_entry
 459 {
 460    /* No ownership */
 461    const struct tu_bo *bo;
 462
 463    uint32_t size;
 464    uint32_t offset;
 465 };
 466
 467 struct tu_cs_memory {
 468    uint32_t *map;
 469    uint64_t iova;
 470 };
 471
 472 struct tu_draw_state {
 473    uint64_t iova : 48;
 474    uint32_t size : 16;
 475 };
 476
 477 enum tu_dynamic_state
 478 {
 479    /* re-use VK_DYNAMIC_STATE_ enums for non-extended dynamic states */
 480    TU_DYNAMIC_STATE_SAMPLE_LOCATIONS = VK_DYNAMIC_STATE_STENCIL_REFERENCE + 1,
 481    TU_DYNAMIC_STATE_COUNT,
 482 };
 483
 484 enum tu_draw_state_group_id
 485 {
 486    TU_DRAW_STATE_PROGRAM,
 487    TU_DRAW_STATE_PROGRAM_BINNING,
 488    TU_DRAW_STATE_TESS,
 489    TU_DRAW_STATE_VB,
 490    TU_DRAW_STATE_VI,
 491    TU_DRAW_STATE_VI_BINNING,
 492    TU_DRAW_STATE_RAST,
 493    TU_DRAW_STATE_DS,
 494    TU_DRAW_STATE_BLEND,
 495    TU_DRAW_STATE_VS_CONST,
 496    TU_DRAW_STATE_HS_CONST,
 497    TU_DRAW_STATE_DS_CONST,
 498    TU_DRAW_STATE_GS_CONST,
 499    TU_DRAW_STATE_FS_CONST,
 500    TU_DRAW_STATE_DESC_SETS,
 501    TU_DRAW_STATE_DESC_SETS_LOAD,
 502    TU_DRAW_STATE_VS_PARAMS,
 503    TU_DRAW_STATE_INPUT_ATTACHMENTS_GMEM,
 504    TU_DRAW_STATE_INPUT_ATTACHMENTS_SYSMEM,
 505
 506    /* dynamic state related draw states */
 507    TU_DRAW_STATE_DYNAMIC,
 508    TU_DRAW_STATE_COUNT = TU_DRAW_STATE_DYNAMIC + TU_DYNAMIC_STATE_COUNT,
 509 };
 510
 511 enum tu_cs_mode
 512 {
 513
 514    /*
 515     * A command stream in TU_CS_MODE_GROW mode grows automatically whenever it
 516     * is full.  tu_cs_begin must be called before command packet emission and
 517     * tu_cs_end must be called after.
 518     *
 519     * This mode may create multiple entries internally.  The entries must be
 520     * submitted together.
 521     */
 522    TU_CS_MODE_GROW,
 523
 524    /*
 525     * A command stream in TU_CS_MODE_EXTERNAL mode wraps an external,
 526     * fixed-size buffer.  tu_cs_begin and tu_cs_end are optional and have no
 527     * effect on it.
 528     *
 529     * This mode does not create any entry or any BO.
 530     */
 531    TU_CS_MODE_EXTERNAL,
 532
 533    /*
 534     * A command stream in TU_CS_MODE_SUB_STREAM mode does not support direct
 535     * command packet emission.  tu_cs_begin_sub_stream must be called to get a
 536     * sub-stream to emit comamnd packets to.  When done with the sub-stream,
 537     * tu_cs_end_sub_stream must be called.
 538     *
 539     * This mode does not create any entry internally.
 540     */
 541    TU_CS_MODE_SUB_STREAM,
 542 };
 543
 544 struct tu_cs
 545 {
 546    uint32_t *start;
 547    uint32_t *cur;
 548    uint32_t *reserved_end;
 549    uint32_t *end;
 550
 551    struct tu_device *device;
 552    enum tu_cs_mode mode;
 553    uint32_t next_bo_size;
 554
 555    struct tu_cs_entry *entries;
 556    uint32_t entry_count;
 557    uint32_t entry_capacity;
 558
 559    struct tu_bo **bos;
 560    uint32_t bo_count;
 561    uint32_t bo_capacity;
 562
 563    /* state for cond_exec_start/cond_exec_end */
 564    uint32_t cond_flags;
 565    uint32_t *cond_dwords;
 566 };
 567
 568 struct tu_device_memory
 569 {
 570    struct vk_object_base base;
 571
 572    struct tu_bo bo;
 573    VkDeviceSize size;
 574
 575    /* for dedicated allocations */
 576    struct tu_image *image;
 577    struct tu_buffer *buffer;
 578
 579    uint32_t type_index;
 580    void *map;
 581    void *user_ptr;
 582 };
 583
 584 struct tu_descriptor_range
 585 {
 586    uint64_t va;
 587    uint32_t size;
 588 };
 589
 590 struct tu_descriptor_set
 591 {
 592    struct vk_object_base base;
 593
 594    const struct tu_descriptor_set_layout *layout;
 595    struct tu_descriptor_pool *pool;
 596    uint32_t size;
 597
 598    uint64_t va;
 599    uint32_t *mapped_ptr;
 600
 601    uint32_t *dynamic_descriptors;
 602
 603    struct tu_bo *buffers[0];
 604 };
 605
 606 struct tu_push_descriptor_set
 607 {
 608    struct tu_descriptor_set set;
 609    uint32_t capacity;
 610 };
 611
 612 struct tu_descriptor_pool_entry
 613 {
 614    uint32_t offset;
 615    uint32_t size;
 616    struct tu_descriptor_set *set;
 617 };
 618
 619 struct tu_descriptor_pool
 620 {
 621    struct vk_object_base base;
 622
 623    struct tu_bo bo;
 624    uint64_t current_offset;
 625    uint64_t size;
 626
 627    uint8_t *host_memory_base;
 628    uint8_t *host_memory_ptr;
 629    uint8_t *host_memory_end;
 630
 631    uint32_t entry_count;
 632    uint32_t max_entry_count;
 633    struct tu_descriptor_pool_entry entries[0];
 634 };
 635
 636 struct tu_descriptor_update_template_entry
 637 {
 638    VkDescriptorType descriptor_type;
 639
 640    /* The number of descriptors to update */
 641    uint32_t descriptor_count;
 642
 643    /* Into mapped_ptr or dynamic_descriptors, in units of the respective array
 644     */
 645    uint32_t dst_offset;
 646
 647    /* In dwords. Not valid/used for dynamic descriptors */
 648    uint32_t dst_stride;
 649
 650    uint32_t buffer_offset;
 651
 652    /* Only valid for combined image samplers and samplers */
 653    uint16_t has_sampler;
 654
 655    /* In bytes */
 656    size_t src_offset;
 657    size_t src_stride;
 658
 659    /* For push descriptors */
 660    const uint32_t *immutable_samplers;
 661 };
 662
 663 struct tu_descriptor_update_template
 664 {
 665    struct vk_object_base base;
 666
 667    uint32_t entry_count;
 668    struct tu_descriptor_update_template_entry entry[0];
 669 };
 670
 671 struct tu_buffer
 672 {
 673    struct vk_object_base base;
 674
 675    VkDeviceSize size;
 676
 677    VkBufferUsageFlags usage;
 678    VkBufferCreateFlags flags;
 679
 680    struct tu_bo *bo;
 681    VkDeviceSize bo_offset;
 682 };
 683
 684 static inline uint64_t
 685 tu_buffer_iova(struct tu_buffer *buffer)
 686 {
 687    return buffer->bo->iova + buffer->bo_offset;
 688 }
 689
 690 struct tu_vertex_binding
 691 {
 692    struct tu_buffer *buffer;
 693    VkDeviceSize offset;
 694 };
 695
 696 const char *
 697 tu_get_debug_option_name(int id);
 698
 699 const char *
 700 tu_get_perftest_option_name(int id);
 701
 702 struct tu_descriptor_state
 703 {
 704    struct tu_descriptor_set *sets[MAX_SETS];
 705    uint32_t dynamic_descriptors[MAX_DYNAMIC_BUFFERS * A6XX_TEX_CONST_DWORDS];
 706 };
 707
 708 enum tu_cmd_dirty_bits
 709 {
 710    TU_CMD_DIRTY_COMPUTE_PIPELINE = 1 << 1,
 711    TU_CMD_DIRTY_VERTEX_BUFFERS = 1 << 2,
 712    TU_CMD_DIRTY_DESCRIPTOR_SETS = 1 << 3,
 713    TU_CMD_DIRTY_COMPUTE_DESCRIPTOR_SETS = 1 << 4,
 714    TU_CMD_DIRTY_SHADER_CONSTS = 1 << 5,
 715    /* all draw states were disabled and need to be re-enabled: */
 716    TU_CMD_DIRTY_DRAW_STATE = 1 << 7,
 717 };
 718
 719 /* There are only three cache domains we have to care about: the CCU, or
 720  * color cache unit, which is used for color and depth/stencil attachments
 721  * and copy/blit destinations, and is split conceptually into color and depth,
 722  * and the universal cache or UCHE which is used for pretty much everything
 723  * else, except for the CP (uncached) and host. We need to flush whenever data
 724  * crosses these boundaries.
 725  */
 726
 727 enum tu_cmd_access_mask {
 728    TU_ACCESS_UCHE_READ = 1 << 0,
 729    TU_ACCESS_UCHE_WRITE = 1 << 1,
 730    TU_ACCESS_CCU_COLOR_READ = 1 << 2,
 731    TU_ACCESS_CCU_COLOR_WRITE = 1 << 3,
 732    TU_ACCESS_CCU_DEPTH_READ = 1 << 4,
 733    TU_ACCESS_CCU_DEPTH_WRITE = 1 << 5,
 734
 735    /* Experiments have shown that while it's safe to avoid flushing the CCU
 736     * after each blit/renderpass, it's not safe to assume that subsequent
 737     * lookups with a different attachment state will hit unflushed cache
 738     * entries. That is, the CCU needs to be flushed and possibly invalidated
 739     * when accessing memory with a different attachment state. Writing to an
 740     * attachment under the following conditions after clearing using the
 741     * normal 2d engine path is known to have issues:
 742     *
 743     * - It isn't the 0'th layer.
 744     * - There are more than one attachment, and this isn't the 0'th attachment
 745     *   (this seems to also depend on the cpp of the attachments).
 746     *
 747     * Our best guess is that the layer/MRT state is used when computing
 748     * the location of a cache entry in CCU, to avoid conflicts. We assume that
 749     * any access in a renderpass after or before an access by a transfer needs
 750     * a flush/invalidate, and use the _INCOHERENT variants to represent access
 751     * by a transfer.
 752     */
 753    TU_ACCESS_CCU_COLOR_INCOHERENT_READ = 1 << 6,
 754    TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE = 1 << 7,
 755    TU_ACCESS_CCU_DEPTH_INCOHERENT_READ = 1 << 8,
 756    TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE = 1 << 9,
 757
 758    TU_ACCESS_SYSMEM_READ = 1 << 10,
 759    TU_ACCESS_SYSMEM_WRITE = 1 << 11,
 760
 761    /* Set if a WFI is required due to data being read by the CP or the 2D
 762     * engine.
 763     */
 764    TU_ACCESS_WFI_READ = 1 << 12,
 765
 766    TU_ACCESS_READ =
 767       TU_ACCESS_UCHE_READ |
 768       TU_ACCESS_CCU_COLOR_READ |
 769       TU_ACCESS_CCU_DEPTH_READ |
 770       TU_ACCESS_CCU_COLOR_INCOHERENT_READ |
 771       TU_ACCESS_CCU_DEPTH_INCOHERENT_READ |
 772       TU_ACCESS_SYSMEM_READ,
 773
 774    TU_ACCESS_WRITE =
 775       TU_ACCESS_UCHE_WRITE |
 776       TU_ACCESS_CCU_COLOR_WRITE |
 777       TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE |
 778       TU_ACCESS_CCU_DEPTH_WRITE |
 779       TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE |
 780       TU_ACCESS_SYSMEM_WRITE,
 781
 782    TU_ACCESS_ALL =
 783       TU_ACCESS_READ |
 784       TU_ACCESS_WRITE,
 785 };
 786
 787 enum tu_cmd_flush_bits {
 788    TU_CMD_FLAG_CCU_FLUSH_DEPTH = 1 << 0,
 789    TU_CMD_FLAG_CCU_FLUSH_COLOR = 1 << 1,
 790    TU_CMD_FLAG_CCU_INVALIDATE_DEPTH = 1 << 2,
 791    TU_CMD_FLAG_CCU_INVALIDATE_COLOR = 1 << 3,
 792    TU_CMD_FLAG_CACHE_FLUSH = 1 << 4,
 793    TU_CMD_FLAG_CACHE_INVALIDATE = 1 << 5,
 794
 795    TU_CMD_FLAG_ALL_FLUSH =
 796       TU_CMD_FLAG_CCU_FLUSH_DEPTH |
 797       TU_CMD_FLAG_CCU_FLUSH_COLOR |
 798       TU_CMD_FLAG_CACHE_FLUSH,
 799
 800    TU_CMD_FLAG_ALL_INVALIDATE =
 801       TU_CMD_FLAG_CCU_INVALIDATE_DEPTH |
 802       TU_CMD_FLAG_CCU_INVALIDATE_COLOR |
 803       TU_CMD_FLAG_CACHE_INVALIDATE,
 804
 805    TU_CMD_FLAG_WFI = 1 << 6,
 806 };
 807
 808 /* Changing the CCU from sysmem mode to gmem mode or vice-versa is pretty
 809  * heavy, involving a CCU cache flush/invalidate and a WFI in order to change
 810  * which part of the gmem is used by the CCU. Here we keep track of what the
 811  * state of the CCU.
 812  */
 813 enum tu_cmd_ccu_state {
 814    TU_CMD_CCU_SYSMEM,
 815    TU_CMD_CCU_GMEM,
 816    TU_CMD_CCU_UNKNOWN,
 817 };
 818
 819 struct tu_cache_state {
 820    /* Caches which must be made available (flushed) eventually if there are
 821     * any users outside that cache domain, and caches which must be
 822     * invalidated eventually if there are any reads.
 823     */
 824    enum tu_cmd_flush_bits pending_flush_bits;
 825    /* Pending flushes */
 826    enum tu_cmd_flush_bits flush_bits;
 827 };
 828
 829 struct tu_cmd_state
 830 {
 831    uint32_t dirty;
 832
 833    struct tu_pipeline *pipeline;
 834    struct tu_pipeline *compute_pipeline;
 835
 836    /* Vertex buffers */
 837    struct
 838    {
 839       struct tu_buffer *buffers[MAX_VBS];
 840       VkDeviceSize offsets[MAX_VBS];
 841    } vb;
 842
 843    /* for dynamic states that can't be emitted directly */
 844    uint32_t dynamic_stencil_mask;
 845    uint32_t dynamic_stencil_wrmask;
 846    uint32_t dynamic_stencil_ref;
 847    uint32_t dynamic_gras_su_cntl;
 848
 849    /* saved states to re-emit in TU_CMD_DIRTY_DRAW_STATE case */
 850    struct tu_draw_state dynamic_state[TU_DYNAMIC_STATE_COUNT];
 851    struct tu_cs_entry vertex_buffers_ib;
 852    struct tu_cs_entry shader_const_ib[MESA_SHADER_STAGES];
 853    struct tu_cs_entry desc_sets_ib, desc_sets_load_ib;
 854    struct tu_cs_entry ia_gmem_ib, ia_sysmem_ib;
 855
 856    struct tu_draw_state vs_params;
 857
 858    /* Index buffer */
 859    uint64_t index_va;
 860    uint32_t max_index_count;
 861    uint8_t index_size;
 862
 863    /* because streamout base has to be 32-byte aligned
 864     * there is an extra offset to deal with when it is
 865     * unaligned
 866     */
 867    uint8_t streamout_offset[IR3_MAX_SO_BUFFERS];
 868
 869    /* Renderpasses are tricky, because we may need to flush differently if
 870     * using sysmem vs. gmem and therefore we have to delay any flushing that
 871     * happens before a renderpass. So we have to have two copies of the flush
 872     * state, one for intra-renderpass flushes (i.e. renderpass dependencies)
 873     * and one for outside a renderpass.
 874     */
 875    struct tu_cache_state cache;
 876    struct tu_cache_state renderpass_cache;
 877
 878    enum tu_cmd_ccu_state ccu_state;
 879
 880    const struct tu_render_pass *pass;
 881    const struct tu_subpass *subpass;
 882    const struct tu_framebuffer *framebuffer;
 883    VkRect2D render_area;
 884
 885    struct tu_cs_entry tile_store_ib;
 886
 887    bool xfb_used;
 888 };
 889
 890 struct tu_cmd_pool
 891 {
 892    struct vk_object_base base;
 893
 894    VkAllocationCallbacks alloc;
 895    struct list_head cmd_buffers;
 896    struct list_head free_cmd_buffers;
 897    uint32_t queue_family_index;
 898 };
 899
 900 struct tu_cmd_buffer_upload
 901 {
 902    uint8_t *map;
 903    unsigned offset;
 904    uint64_t size;
 905    struct list_head list;
 906 };
 907
 908 enum tu_cmd_buffer_status
 909 {
 910    TU_CMD_BUFFER_STATUS_INVALID,
 911    TU_CMD_BUFFER_STATUS_INITIAL,
 912    TU_CMD_BUFFER_STATUS_RECORDING,
 913    TU_CMD_BUFFER_STATUS_EXECUTABLE,
 914    TU_CMD_BUFFER_STATUS_PENDING,
 915 };
 916
 917 struct tu_bo_list
 918 {
 919    uint32_t count;
 920    uint32_t capacity;
 921    struct drm_msm_gem_submit_bo *bo_infos;
 922 };
 923
 924 #define TU_BO_LIST_FAILED (~0)
 925
 926 void
 927 tu_bo_list_init(struct tu_bo_list *list);
 928 void
 929 tu_bo_list_destroy(struct tu_bo_list *list);
 930 void
 931 tu_bo_list_reset(struct tu_bo_list *list);
 932 uint32_t
 933 tu_bo_list_add(struct tu_bo_list *list,
 934                const struct tu_bo *bo,
 935                uint32_t flags);
 936 VkResult
 937 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other);
 938
 939 struct tu_cmd_buffer
 940 {
 941    struct vk_object_base base;
 942
 943    struct tu_device *device;
 944
 945    struct tu_cmd_pool *pool;
 946    struct list_head pool_link;
 947
 948    VkCommandBufferUsageFlags usage_flags;
 949    VkCommandBufferLevel level;
 950    enum tu_cmd_buffer_status status;
 951
 952    struct tu_cmd_state state;
 953    struct tu_vertex_binding vertex_bindings[MAX_VBS];
 954    uint32_t vertex_bindings_set;
 955    uint32_t queue_family_index;
 956
 957    uint32_t push_constants[MAX_PUSH_CONSTANTS_SIZE / 4];
 958    VkShaderStageFlags push_constant_stages;
 959    struct tu_descriptor_set meta_push_descriptors;
 960
 961    struct tu_descriptor_state descriptors[MAX_BIND_POINTS];
 962
 963    struct tu_cmd_buffer_upload upload;
 964
 965    VkResult record_result;
 966
 967    struct tu_bo_list bo_list;
 968    struct tu_cs cs;
 969    struct tu_cs draw_cs;
 970    struct tu_cs draw_epilogue_cs;
 971    struct tu_cs sub_cs;
 972
 973    bool has_tess;
 974
 975    uint32_t vsc_draw_strm_pitch;
 976    uint32_t vsc_prim_strm_pitch;
 977 };
 978
 979 /* Temporary struct for tracking a register state to be written, used by
 980  * a6xx-pack.h and tu_cs_emit_regs()
 981  */
 982 struct tu_reg_value {
 983    uint32_t reg;
 984    uint64_t value;
 985    bool is_address;
 986    struct tu_bo *bo;
 987    bool bo_write;
 988    uint32_t bo_offset;
 989    uint32_t bo_shift;
 990 };
 991
 992
 993 void tu_emit_cache_flush_renderpass(struct tu_cmd_buffer *cmd_buffer,
 994                                     struct tu_cs *cs);
 995
 996 void tu_emit_cache_flush_ccu(struct tu_cmd_buffer *cmd_buffer,
 997                              struct tu_cs *cs,
 998                              enum tu_cmd_ccu_state ccu_state);
 999
1000 void
1001 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
1002                      struct tu_cs *cs,
1003                      enum vgt_event_type event);
1004
1005 static inline struct tu_descriptor_state *
1006 tu_get_descriptors_state(struct tu_cmd_buffer *cmd_buffer,
1007                          VkPipelineBindPoint bind_point)
1008 {
1009    return &cmd_buffer->descriptors[bind_point];
1010 }
1011
1012 struct tu_event
1013 {
1014    struct vk_object_base base;
1015    struct tu_bo bo;
1016 };
1017
1018 struct tu_shader_module
1019 {
1020    struct vk_object_base base;
1021
1022    unsigned char sha1[20];
1023
1024    uint32_t code_size;
1025    const uint32_t *code[0];
1026 };
1027
1028 struct tu_push_constant_range
1029 {
1030    uint32_t lo;
1031    uint32_t count;
1032 };
1033
1034 struct tu_shader
1035 {
1036    struct ir3_shader *ir3_shader;
1037
1038    struct tu_push_constant_range push_consts;
1039    uint8_t active_desc_sets;
1040 };
1041
1042 struct tu_shader *
1043 tu_shader_create(struct tu_device *dev,
1044                  gl_shader_stage stage,
1045                  const VkPipelineShaderStageCreateInfo *stage_info,
1046                  struct tu_pipeline_layout *layout,
1047                  const VkAllocationCallbacks *alloc);
1048
1049 void
1050 tu_shader_destroy(struct tu_device *dev,
1051                   struct tu_shader *shader,
1052                   const VkAllocationCallbacks *alloc);
1053
1054 struct tu_program_descriptor_linkage
1055 {
1056    struct ir3_const_state const_state;
1057
1058    uint32_t constlen;
1059
1060    struct tu_push_constant_range push_consts;
1061 };
1062
1063 struct tu_pipeline
1064 {
1065    struct vk_object_base base;
1066
1067    struct tu_cs cs;
1068
1069    struct tu_pipeline_layout *layout;
1070
1071    bool need_indirect_descriptor_sets;
1072    VkShaderStageFlags active_stages;
1073    uint32_t active_desc_sets;
1074
1075    /* mask of enabled dynamic states
1076     * if BIT(i) is set, pipeline->dynamic_state[i] is *NOT* used
1077     */
1078    uint32_t dynamic_state_mask;
1079    struct tu_draw_state dynamic_state[TU_DYNAMIC_STATE_COUNT];
1080
1081    /* gras_su_cntl without line width, used for dynamic line width state */
1082    uint32_t gras_su_cntl;
1083
1084    struct
1085    {
1086       struct tu_cs_entry state_ib;
1087       struct tu_cs_entry binning_state_ib;
1088
1089       struct tu_program_descriptor_linkage link[MESA_SHADER_STAGES];
1090    } program;
1091
1092    struct
1093    {
1094       struct tu_cs_entry state_ib;
1095    } load_state;
1096
1097    struct
1098    {
1099       struct tu_cs_entry state_ib;
1100       struct tu_cs_entry binning_state_ib;
1101       uint32_t bindings_used;
1102    } vi;
1103
1104    struct
1105    {
1106       enum pc_di_primtype primtype;
1107       bool primitive_restart;
1108    } ia;
1109
1110    struct
1111    {
1112       uint32_t patch_type;
1113       uint32_t param_stride;
1114       uint32_t hs_bo_regid;
1115       uint32_t ds_bo_regid;
1116       bool upper_left_domain_origin;
1117    } tess;
1118
1119    struct
1120    {
1121       struct tu_cs_entry state_ib;
1122    } rast;
1123
1124    struct
1125    {
1126       struct tu_cs_entry state_ib;
1127    } ds;
1128
1129    struct
1130    {
1131       struct tu_cs_entry state_ib;
1132    } blend;
1133
1134    struct
1135    {
1136       uint32_t local_size[3];
1137    } compute;
1138 };
1139
1140 void
1141 tu6_emit_viewport(struct tu_cs *cs, const VkViewport *viewport);
1142
1143 void
1144 tu6_emit_scissor(struct tu_cs *cs, const VkRect2D *scissor);
1145
1146 void
1147 tu6_emit_sample_locations(struct tu_cs *cs, const VkSampleLocationsInfoEXT *samp_loc);
1148
1149 void
1150 tu6_emit_depth_bias(struct tu_cs *cs,
1151                     float constant_factor,
1152                     float clamp,
1153                     float slope_factor);
1154
1155 void tu6_emit_msaa(struct tu_cs *cs, VkSampleCountFlagBits samples);
1156
1157 void tu6_emit_window_scissor(struct tu_cs *cs, uint32_t x1, uint32_t y1, uint32_t x2, uint32_t y2);
1158
1159 void tu6_emit_window_offset(struct tu_cs *cs, uint32_t x1, uint32_t y1);
1160
1161 void
1162 tu6_emit_xs_config(struct tu_cs *cs,
1163                    gl_shader_stage stage,
1164                    const struct ir3_shader_variant *xs,
1165                    uint64_t binary_iova);
1166
1167 void
1168 tu6_emit_vpc(struct tu_cs *cs,
1169              const struct ir3_shader_variant *vs,
1170              const struct ir3_shader_variant *hs,
1171              const struct ir3_shader_variant *ds,
1172              const struct ir3_shader_variant *gs,
1173              const struct ir3_shader_variant *fs);
1174
1175 void
1176 tu6_emit_fs_inputs(struct tu_cs *cs, const struct ir3_shader_variant *fs);
1177
1178 struct tu_image_view;
1179
1180 void
1181 tu_resolve_sysmem(struct tu_cmd_buffer *cmd,
1182                   struct tu_cs *cs,
1183                   struct tu_image_view *src,
1184                   struct tu_image_view *dst,
1185                   uint32_t layers,
1186                   const VkRect2D *rect);
1187
1188 void
1189 tu_clear_sysmem_attachment(struct tu_cmd_buffer *cmd,
1190                            struct tu_cs *cs,
1191                            uint32_t a,
1192                            const VkRenderPassBeginInfo *info);
1193
1194 void
1195 tu_clear_gmem_attachment(struct tu_cmd_buffer *cmd,
1196                          struct tu_cs *cs,
1197                          uint32_t a,
1198                          const VkRenderPassBeginInfo *info);
1199
1200 void
1201 tu_load_gmem_attachment(struct tu_cmd_buffer *cmd,
1202                         struct tu_cs *cs,
1203                         uint32_t a,
1204                         bool force_load);
1205
1206 /* expose this function to be able to emit load without checking LOAD_OP */
1207 void
1208 tu_emit_load_gmem_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t a);
1209
1210 /* note: gmem store can also resolve */
1211 void
1212 tu_store_gmem_attachment(struct tu_cmd_buffer *cmd,
1213                          struct tu_cs *cs,
1214                          uint32_t a,
1215                          uint32_t gmem_a);
1216
1217 enum tu_supported_formats {
1218    FMT_VERTEX = 1,
1219    FMT_TEXTURE = 2,
1220    FMT_COLOR = 4,
1221 };
1222
1223 struct tu_native_format
1224 {
1225    enum a6xx_format fmt : 8;
1226    enum a3xx_color_swap swap : 8;
1227    enum a6xx_tile_mode tile_mode : 8;
1228    enum tu_supported_formats supported : 8;
1229 };
1230
1231 struct tu_native_format tu6_format_vtx(VkFormat format);
1232 struct tu_native_format tu6_format_color(VkFormat format, enum a6xx_tile_mode tile_mode);
1233 struct tu_native_format tu6_format_texture(VkFormat format, enum a6xx_tile_mode tile_mode);
1234
1235 static inline enum a6xx_format
1236 tu6_base_format(VkFormat format)
1237 {
1238    /* note: tu6_format_color doesn't care about tiling for .fmt field */
1239    return tu6_format_color(format, TILE6_LINEAR).fmt;
1240 }
1241
1242 struct tu_image
1243 {
1244    struct vk_object_base base;
1245
1246    VkImageType type;
1247    /* The original VkFormat provided by the client.  This may not match any
1248     * of the actual surface formats.
1249     */
1250    VkFormat vk_format;
1251    VkImageAspectFlags aspects;
1252    VkImageUsageFlags usage;  /**< Superset of VkImageCreateInfo::usage. */
1253    VkImageTiling tiling;     /** VkImageCreateInfo::tiling */
1254    VkImageCreateFlags flags; /** VkImageCreateInfo::flags */
1255    VkExtent3D extent;
1256    uint32_t level_count;
1257    uint32_t layer_count;
1258    VkSampleCountFlagBits samples;
1259
1260    struct fdl_layout layout[3];
1261    uint32_t total_size;
1262
1263    unsigned queue_family_mask;
1264    bool exclusive;
1265    bool shareable;
1266
1267    /* For VK_ANDROID_native_buffer, the WSI image owns the memory, */
1268    VkDeviceMemory owned_memory;
1269
1270    /* Set when bound */
1271    struct tu_bo *bo;
1272    VkDeviceSize bo_offset;
1273 };
1274
1275 static inline uint32_t
1276 tu_get_layerCount(const struct tu_image *image,
1277                   const VkImageSubresourceRange *range)
1278 {
1279    return range->layerCount == VK_REMAINING_ARRAY_LAYERS
1280              ? image->layer_count - range->baseArrayLayer
1281              : range->layerCount;
1282 }
1283
1284 static inline uint32_t
1285 tu_get_levelCount(const struct tu_image *image,
1286                   const VkImageSubresourceRange *range)
1287 {
1288    return range->levelCount == VK_REMAINING_MIP_LEVELS
1289              ? image->level_count - range->baseMipLevel
1290              : range->levelCount;
1291 }
1292
1293 struct tu_image_view
1294 {
1295    struct vk_object_base base;
1296
1297    struct tu_image *image; /**< VkImageViewCreateInfo::image */
1298
1299    uint64_t base_addr;
1300    uint64_t ubwc_addr;
1301    uint32_t layer_size;
1302    uint32_t ubwc_layer_size;
1303
1304    /* used to determine if fast gmem store path can be used */
1305    VkExtent2D extent;
1306    bool need_y2_align;
1307
1308    bool ubwc_enabled;
1309
1310    uint32_t descriptor[A6XX_TEX_CONST_DWORDS];
1311
1312    /* Descriptor for use as a storage image as opposed to a sampled image.
1313     * This has a few differences for cube maps (e.g. type).
1314     */
1315    uint32_t storage_descriptor[A6XX_TEX_CONST_DWORDS];
1316
1317    /* pre-filled register values */
1318    uint32_t PITCH;
1319    uint32_t FLAG_BUFFER_PITCH;
1320
1321    uint32_t RB_MRT_BUF_INFO;
1322    uint32_t SP_FS_MRT_REG;
1323
1324    uint32_t SP_PS_2D_SRC_INFO;
1325    uint32_t SP_PS_2D_SRC_SIZE;
1326
1327    uint32_t RB_2D_DST_INFO;
1328
1329    uint32_t RB_BLIT_DST_INFO;
1330 };
1331
1332 struct tu_sampler_ycbcr_conversion {
1333    struct vk_object_base base;
1334
1335    VkFormat format;
1336    VkSamplerYcbcrModelConversion ycbcr_model;
1337    VkSamplerYcbcrRange ycbcr_range;
1338    VkComponentMapping components;
1339    VkChromaLocation chroma_offsets[2];
1340    VkFilter chroma_filter;
1341 };
1342
1343 struct tu_sampler {
1344    struct vk_object_base base;
1345
1346    uint32_t descriptor[A6XX_TEX_SAMP_DWORDS];
1347    struct tu_sampler_ycbcr_conversion *ycbcr_sampler;
1348 };
1349
1350 void
1351 tu_cs_image_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1352
1353 void
1354 tu_cs_image_ref_2d(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer, bool src);
1355
1356 void
1357 tu_cs_image_flag_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1358
1359 VkResult
1360 tu_image_create(VkDevice _device,
1361                 const VkImageCreateInfo *pCreateInfo,
1362                 const VkAllocationCallbacks *alloc,
1363                 VkImage *pImage,
1364                 uint64_t modifier,
1365                 const VkSubresourceLayout *plane_layouts);
1366
1367 VkResult
1368 tu_image_from_gralloc(VkDevice device_h,
1369                       const VkImageCreateInfo *base_info,
1370                       const VkNativeBufferANDROID *gralloc_info,
1371                       const VkAllocationCallbacks *alloc,
1372                       VkImage *out_image_h);
1373
1374 void
1375 tu_image_view_init(struct tu_image_view *view,
1376                    const VkImageViewCreateInfo *pCreateInfo);
1377
1378 struct tu_buffer_view
1379 {
1380    struct vk_object_base base;
1381
1382    uint32_t descriptor[A6XX_TEX_CONST_DWORDS];
1383
1384    struct tu_buffer *buffer;
1385 };
1386 void
1387 tu_buffer_view_init(struct tu_buffer_view *view,
1388                     struct tu_device *device,
1389                     const VkBufferViewCreateInfo *pCreateInfo);
1390
1391 struct tu_attachment_info
1392 {
1393    struct tu_image_view *attachment;
1394 };
1395
1396 struct tu_framebuffer
1397 {
1398    struct vk_object_base base;
1399
1400    uint32_t width;
1401    uint32_t height;
1402    uint32_t layers;
1403
1404    /* size of the first tile */
1405    VkExtent2D tile0;
1406    /* number of tiles */
1407    VkExtent2D tile_count;
1408
1409    /* size of the first VSC pipe */
1410    VkExtent2D pipe0;
1411    /* number of VSC pipes */
1412    VkExtent2D pipe_count;
1413
1414    /* pipe register values */
1415    uint32_t pipe_config[MAX_VSC_PIPES];
1416    uint32_t pipe_sizes[MAX_VSC_PIPES];
1417
1418    uint32_t attachment_count;
1419    struct tu_attachment_info attachments[0];
1420 };
1421
1422 void
1423 tu_framebuffer_tiling_config(struct tu_framebuffer *fb,
1424                              const struct tu_device *device,
1425                              const struct tu_render_pass *pass);
1426
1427 struct tu_subpass_barrier {
1428    VkPipelineStageFlags src_stage_mask;
1429    VkAccessFlags src_access_mask;
1430    VkAccessFlags dst_access_mask;
1431    bool incoherent_ccu_color, incoherent_ccu_depth;
1432 };
1433
1434 struct tu_subpass_attachment
1435 {
1436    uint32_t attachment;
1437 };
1438
1439 struct tu_subpass
1440 {
1441    uint32_t input_count;
1442    uint32_t color_count;
1443    struct tu_subpass_attachment *input_attachments;
1444    struct tu_subpass_attachment *color_attachments;
1445    struct tu_subpass_attachment *resolve_attachments;
1446    struct tu_subpass_attachment depth_stencil_attachment;
1447
1448    VkSampleCountFlagBits samples;
1449
1450    uint32_t srgb_cntl;
1451
1452    struct tu_subpass_barrier start_barrier;
1453 };
1454
1455 struct tu_render_pass_attachment
1456 {
1457    VkFormat format;
1458    uint32_t samples;
1459    uint32_t cpp;
1460    VkImageAspectFlags clear_mask;
1461    bool load;
1462    bool store;
1463    int32_t gmem_offset;
1464 };
1465
1466 struct tu_render_pass
1467 {
1468    struct vk_object_base base;
1469
1470    uint32_t attachment_count;
1471    uint32_t subpass_count;
1472    uint32_t gmem_pixels;
1473    uint32_t tile_align_w;
1474    struct tu_subpass_attachment *subpass_attachments;
1475    struct tu_render_pass_attachment *attachments;
1476    struct tu_subpass_barrier end_barrier;
1477    struct tu_subpass subpasses[0];
1478 };
1479
1480 struct tu_query_pool
1481 {
1482    struct vk_object_base base;
1483
1484    VkQueryType type;
1485    uint32_t stride;
1486    uint64_t size;
1487    uint32_t pipeline_statistics;
1488    struct tu_bo bo;
1489 };
1490
1491 enum tu_semaphore_kind
1492 {
1493    TU_SEMAPHORE_NONE,
1494    TU_SEMAPHORE_SYNCOBJ,
1495 };
1496
1497 struct tu_semaphore_part
1498 {
1499    enum tu_semaphore_kind kind;
1500    union {
1501       uint32_t syncobj;
1502    };
1503 };
1504
1505 struct tu_semaphore
1506 {
1507    struct vk_object_base base;
1508
1509    struct tu_semaphore_part permanent;
1510    struct tu_semaphore_part temporary;
1511 };
1512
1513 void
1514 tu_set_descriptor_set(struct tu_cmd_buffer *cmd_buffer,
1515                       VkPipelineBindPoint bind_point,
1516                       struct tu_descriptor_set *set,
1517                       unsigned idx);
1518
1519 void
1520 tu_update_descriptor_sets(struct tu_device *device,
1521                           struct tu_cmd_buffer *cmd_buffer,
1522                           VkDescriptorSet overrideSet,
1523                           uint32_t descriptorWriteCount,
1524                           const VkWriteDescriptorSet *pDescriptorWrites,
1525                           uint32_t descriptorCopyCount,
1526                           const VkCopyDescriptorSet *pDescriptorCopies);
1527
1528 void
1529 tu_update_descriptor_set_with_template(
1530    struct tu_device *device,
1531    struct tu_cmd_buffer *cmd_buffer,
1532    struct tu_descriptor_set *set,
1533    VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1534    const void *pData);
1535
1536 int
1537 tu_drm_get_gpu_id(const struct tu_physical_device *dev, uint32_t *id);
1538
1539 int
1540 tu_drm_get_gmem_size(const struct tu_physical_device *dev, uint32_t *size);
1541
1542 int
1543 tu_drm_get_gmem_base(const struct tu_physical_device *dev, uint64_t *base);
1544
1545 int
1546 tu_drm_submitqueue_new(const struct tu_device *dev,
1547                        int priority,
1548                        uint32_t *queue_id);
1549
1550 void
1551 tu_drm_submitqueue_close(const struct tu_device *dev, uint32_t queue_id);
1552
1553 uint32_t
1554 tu_gem_new(const struct tu_device *dev, uint64_t size, uint32_t flags);
1555 uint32_t
1556 tu_gem_import_dmabuf(const struct tu_device *dev,
1557                      int prime_fd,
1558                      uint64_t size);
1559 int
1560 tu_gem_export_dmabuf(const struct tu_device *dev, uint32_t gem_handle);
1561 void
1562 tu_gem_close(const struct tu_device *dev, uint32_t gem_handle);
1563 uint64_t
1564 tu_gem_info_offset(const struct tu_device *dev, uint32_t gem_handle);
1565 uint64_t
1566 tu_gem_info_iova(const struct tu_device *dev, uint32_t gem_handle);
1567
1568 #define TU_DEFINE_HANDLE_CASTS(__tu_type, __VkType)                          \
1569                                                                              \
1570    static inline struct __tu_type *__tu_type##_from_handle(__VkType _handle) \
1571    {                                                                         \
1572       return (struct __tu_type *) _handle;                                   \
1573    }                                                                         \
1574                                                                              \
1575    static inline __VkType __tu_type##_to_handle(struct __tu_type *_obj)      \
1576    {                                                                         \
1577       return (__VkType) _obj;                                                \
1578    }
1579
1580 #define TU_DEFINE_NONDISP_HANDLE_CASTS(__tu_type, __VkType)                  \
1581                                                                              \
1582    static inline struct __tu_type *__tu_type##_from_handle(__VkType _handle) \
1583    {                                                                         \
1584       return (struct __tu_type *) (uintptr_t) _handle;                       \
1585    }                                                                         \
1586                                                                              \
1587    static inline __VkType __tu_type##_to_handle(struct __tu_type *_obj)      \
1588    {                                                                         \
1589       return (__VkType)(uintptr_t) _obj;                                     \
1590    }
1591
1592 #define TU_FROM_HANDLE(__tu_type, __name, __handle)                          \
1593    struct __tu_type *__name = __tu_type##_from_handle(__handle)
1594
1595 TU_DEFINE_HANDLE_CASTS(tu_cmd_buffer, VkCommandBuffer)
1596 TU_DEFINE_HANDLE_CASTS(tu_device, VkDevice)
1597 TU_DEFINE_HANDLE_CASTS(tu_instance, VkInstance)
1598 TU_DEFINE_HANDLE_CASTS(tu_physical_device, VkPhysicalDevice)
1599 TU_DEFINE_HANDLE_CASTS(tu_queue, VkQueue)
1600
1601 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_cmd_pool, VkCommandPool)
1602 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_buffer, VkBuffer)
1603 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_buffer_view, VkBufferView)
1604 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_pool, VkDescriptorPool)
1605 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_set, VkDescriptorSet)
1606 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_set_layout,
1607                                VkDescriptorSetLayout)
1608 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_update_template,
1609                                VkDescriptorUpdateTemplate)
1610 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_device_memory, VkDeviceMemory)
1611 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_fence, VkFence)
1612 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_event, VkEvent)
1613 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_framebuffer, VkFramebuffer)
1614 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_image, VkImage)
1615 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_image_view, VkImageView);
1616 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline_cache, VkPipelineCache)
1617 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline, VkPipeline)
1618 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline_layout, VkPipelineLayout)
1619 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_query_pool, VkQueryPool)
1620 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_render_pass, VkRenderPass)
1621 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_sampler, VkSampler)
1622 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_sampler_ycbcr_conversion, VkSamplerYcbcrConversion)
1623 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_shader_module, VkShaderModule)
1624 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_semaphore, VkSemaphore)
1625
1626 #endif /* TU_PRIVATE_H */