src/freedreno/vulkan/tu_private.h

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  */
  27
  28 #ifndef TU_PRIVATE_H
  29 #define TU_PRIVATE_H
  30
  31 #include <assert.h>
  32 #include <pthread.h>
  33 #include <stdbool.h>
  34 #include <stdint.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #ifdef HAVE_VALGRIND
  39 #include <memcheck.h>
  40 #include <valgrind.h>
  41 #define VG(x) x
  42 #else
  43 #define VG(x) ((void)0)
  44 #endif
  45
  46 #include "c11/threads.h"
  47 #include "main/macros.h"
  48 #include "util/list.h"
  49 #include "util/macros.h"
  50 #include "util/u_atomic.h"
  51 #include "vk_alloc.h"
  52 #include "vk_debug_report.h"
  53 #include "wsi_common.h"
  54
  55 #include "drm-uapi/msm_drm.h"
  56 #include "ir3/ir3_compiler.h"
  57 #include "ir3/ir3_shader.h"
  58
  59 #include "adreno_common.xml.h"
  60 #include "adreno_pm4.xml.h"
  61 #include "a6xx.xml.h"
  62 #include "fdl/freedreno_layout.h"
  63
  64 #include "tu_descriptor_set.h"
  65 #include "tu_extensions.h"
  66 #include "tu_util.h"
  67
  68 /* Pre-declarations needed for WSI entrypoints */
  69 struct wl_surface;
  70 struct wl_display;
  71 typedef struct xcb_connection_t xcb_connection_t;
  72 typedef uint32_t xcb_visualid_t;
  73 typedef uint32_t xcb_window_t;
  74
  75 #include <vulkan/vk_android_native_buffer.h>
  76 #include <vulkan/vk_icd.h>
  77 #include <vulkan/vulkan.h>
  78 #include <vulkan/vulkan_intel.h>
  79
  80 #include "tu_entrypoints.h"
  81
  82 #include "vk_format.h"
  83
  84 #define MAX_VBS 32
  85 #define MAX_VERTEX_ATTRIBS 32
  86 #define MAX_RTS 8
  87 #define MAX_VSC_PIPES 32
  88 #define MAX_VIEWPORTS 1
  89 #define MAX_SCISSORS 16
  90 #define MAX_DISCARD_RECTANGLES 4
  91 #define MAX_PUSH_CONSTANTS_SIZE 128
  92 #define MAX_PUSH_DESCRIPTORS 32
  93 #define MAX_DYNAMIC_UNIFORM_BUFFERS 16
  94 #define MAX_DYNAMIC_STORAGE_BUFFERS 8
  95 #define MAX_DYNAMIC_BUFFERS                                                  \
  96    (MAX_DYNAMIC_UNIFORM_BUFFERS + MAX_DYNAMIC_STORAGE_BUFFERS)
  97 #define TU_MAX_DRM_DEVICES 8
  98 #define MAX_VIEWS 8
  99 #define MAX_BIND_POINTS 2 /* compute + graphics */
 100 /* The Qualcomm driver exposes 0x20000058 */
 101 #define MAX_STORAGE_BUFFER_RANGE 0x20000000
 102 /* We use ldc for uniform buffer loads, just like the Qualcomm driver, so
 103  * expose the same maximum range.
 104  * TODO: The SIZE bitfield is 15 bits, and in 4-dword units, so the actual
 105  * range might be higher.
 106  */
 107 #define MAX_UNIFORM_BUFFER_RANGE 0x10000
 108
 109 #define A6XX_TEX_CONST_DWORDS 16
 110 #define A6XX_TEX_SAMP_DWORDS 4
 111
 112 #define tu_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
 113
 114 static inline uint32_t
 115 tu_minify(uint32_t n, uint32_t levels)
 116 {
 117    if (unlikely(n == 0))
 118       return 0;
 119    else
 120       return MAX2(n >> levels, 1);
 121 }
 122
 123 #define for_each_bit(b, dword)                                               \
 124    for (uint32_t __dword = (dword);                                          \
 125         (b) = __builtin_ffs(__dword) - 1, __dword; __dword &= ~(1 << (b)))
 126
 127 #define typed_memcpy(dest, src, count)                                       \
 128    ({                                                                        \
 129       STATIC_ASSERT(sizeof(*src) == sizeof(*dest));                          \
 130       memcpy((dest), (src), (count) * sizeof(*(src)));                       \
 131    })
 132
 133 #define COND(bool, val) ((bool) ? (val) : 0)
 134 #define BIT(bit) (1u << (bit))
 135
 136 /* Whenever we generate an error, pass it through this function. Useful for
 137  * debugging, where we can break on it. Only call at error site, not when
 138  * propagating errors. Might be useful to plug in a stack trace here.
 139  */
 140
 141 struct tu_instance;
 142
 143 VkResult
 144 __vk_errorf(struct tu_instance *instance,
 145             VkResult error,
 146             const char *file,
 147             int line,
 148             const char *format,
 149             ...);
 150
 151 #define vk_error(instance, error)                                            \
 152    __vk_errorf(instance, error, __FILE__, __LINE__, NULL);
 153 #define vk_errorf(instance, error, format, ...)                              \
 154    __vk_errorf(instance, error, __FILE__, __LINE__, format, ##__VA_ARGS__);
 155
 156 void
 157 __tu_finishme(const char *file, int line, const char *format, ...)
 158    tu_printflike(3, 4);
 159 void
 160 tu_loge(const char *format, ...) tu_printflike(1, 2);
 161 void
 162 tu_logi(const char *format, ...) tu_printflike(1, 2);
 163
 164 /**
 165  * Print a FINISHME message, including its source location.
 166  */
 167 #define tu_finishme(format, ...)                                             \
 168    do {                                                                      \
 169       static bool reported = false;                                          \
 170       if (!reported) {                                                       \
 171          __tu_finishme(__FILE__, __LINE__, format, ##__VA_ARGS__);           \
 172          reported = true;                                                    \
 173       }                                                                      \
 174    } while (0)
 175
 176 #define tu_stub()                                                            \
 177    do {                                                                      \
 178       tu_finishme("stub %s", __func__);                                      \
 179    } while (0)
 180
 181 void *
 182 tu_lookup_entrypoint_unchecked(const char *name);
 183 void *
 184 tu_lookup_entrypoint_checked(
 185    const char *name,
 186    uint32_t core_version,
 187    const struct tu_instance_extension_table *instance,
 188    const struct tu_device_extension_table *device);
 189
 190 struct tu_physical_device
 191 {
 192    VK_LOADER_DATA _loader_data;
 193
 194    struct tu_instance *instance;
 195
 196    char path[20];
 197    char name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE];
 198    uint8_t driver_uuid[VK_UUID_SIZE];
 199    uint8_t device_uuid[VK_UUID_SIZE];
 200    uint8_t cache_uuid[VK_UUID_SIZE];
 201
 202    struct wsi_device wsi_device;
 203
 204    int local_fd;
 205    int master_fd;
 206
 207    unsigned gpu_id;
 208    uint32_t gmem_size;
 209    uint64_t gmem_base;
 210    uint32_t ccu_offset_gmem;
 211    uint32_t ccu_offset_bypass;
 212    /* alignment for size of tiles */
 213    uint32_t tile_align_w;
 214 #define TILE_ALIGN_H 16
 215    /* gmem store/load granularity */
 216 #define GMEM_ALIGN_W 16
 217 #define GMEM_ALIGN_H 4
 218
 219    struct {
 220       uint32_t PC_UNKNOWN_9805;
 221       uint32_t SP_UNKNOWN_A0F8;
 222    } magic;
 223
 224    int msm_major_version;
 225    int msm_minor_version;
 226
 227    /* This is the drivers on-disk cache used as a fallback as opposed to
 228     * the pipeline cache defined by apps.
 229     */
 230    struct disk_cache *disk_cache;
 231
 232    struct tu_device_extension_table supported_extensions;
 233 };
 234
 235 enum tu_debug_flags
 236 {
 237    TU_DEBUG_STARTUP = 1 << 0,
 238    TU_DEBUG_NIR = 1 << 1,
 239    TU_DEBUG_IR3 = 1 << 2,
 240    TU_DEBUG_NOBIN = 1 << 3,
 241    TU_DEBUG_SYSMEM = 1 << 4,
 242    TU_DEBUG_FORCEBIN = 1 << 5,
 243    TU_DEBUG_NOUBWC = 1 << 6,
 244 };
 245
 246 struct tu_instance
 247 {
 248    VK_LOADER_DATA _loader_data;
 249
 250    VkAllocationCallbacks alloc;
 251
 252    uint32_t api_version;
 253    int physical_device_count;
 254    struct tu_physical_device physical_devices[TU_MAX_DRM_DEVICES];
 255
 256    enum tu_debug_flags debug_flags;
 257
 258    struct vk_debug_report_instance debug_report_callbacks;
 259
 260    struct tu_instance_extension_table enabled_extensions;
 261 };
 262
 263 VkResult
 264 tu_wsi_init(struct tu_physical_device *physical_device);
 265 void
 266 tu_wsi_finish(struct tu_physical_device *physical_device);
 267
 268 bool
 269 tu_instance_extension_supported(const char *name);
 270 uint32_t
 271 tu_physical_device_api_version(struct tu_physical_device *dev);
 272 bool
 273 tu_physical_device_extension_supported(struct tu_physical_device *dev,
 274                                        const char *name);
 275
 276 struct cache_entry;
 277
 278 struct tu_pipeline_cache
 279 {
 280    struct tu_device *device;
 281    pthread_mutex_t mutex;
 282
 283    uint32_t total_size;
 284    uint32_t table_size;
 285    uint32_t kernel_count;
 286    struct cache_entry **hash_table;
 287    bool modified;
 288
 289    VkAllocationCallbacks alloc;
 290 };
 291
 292 struct tu_pipeline_key
 293 {
 294 };
 295
 296
 297 /* queue types */
 298 #define TU_QUEUE_GENERAL 0
 299
 300 #define TU_MAX_QUEUE_FAMILIES 1
 301
 302 struct tu_fence
 303 {
 304    struct wsi_fence *fence_wsi;
 305    bool signaled;
 306    int fd;
 307 };
 308
 309 void
 310 tu_fence_init(struct tu_fence *fence, bool signaled);
 311 void
 312 tu_fence_finish(struct tu_fence *fence);
 313 void
 314 tu_fence_update_fd(struct tu_fence *fence, int fd);
 315 void
 316 tu_fence_copy(struct tu_fence *fence, const struct tu_fence *src);
 317 void
 318 tu_fence_signal(struct tu_fence *fence);
 319 void
 320 tu_fence_wait_idle(struct tu_fence *fence);
 321
 322 struct tu_queue
 323 {
 324    VK_LOADER_DATA _loader_data;
 325    struct tu_device *device;
 326    uint32_t queue_family_index;
 327    int queue_idx;
 328    VkDeviceQueueCreateFlags flags;
 329
 330    uint32_t msm_queue_id;
 331    struct tu_fence submit_fence;
 332 };
 333
 334 struct tu_bo
 335 {
 336    uint32_t gem_handle;
 337    uint64_t size;
 338    uint64_t iova;
 339    void *map;
 340 };
 341
 342 struct tu_device
 343 {
 344    VK_LOADER_DATA _loader_data;
 345
 346    VkAllocationCallbacks alloc;
 347
 348    struct tu_instance *instance;
 349
 350    struct tu_queue *queues[TU_MAX_QUEUE_FAMILIES];
 351    int queue_count[TU_MAX_QUEUE_FAMILIES];
 352
 353    struct tu_physical_device *physical_device;
 354    int _lost;
 355
 356    struct ir3_compiler *compiler;
 357
 358    /* Backup in-memory cache to be used if the app doesn't provide one */
 359    struct tu_pipeline_cache *mem_cache;
 360
 361    struct tu_bo vsc_draw_strm;
 362    struct tu_bo vsc_prim_strm;
 363    uint32_t vsc_draw_strm_pitch;
 364    uint32_t vsc_prim_strm_pitch;
 365
 366 #define MIN_SCRATCH_BO_SIZE_LOG2 12 /* A page */
 367
 368    /* Currently the kernel driver uses a 32-bit GPU address space, but it
 369     * should be impossible to go beyond 48 bits.
 370     */
 371    struct {
 372       struct tu_bo bo;
 373       mtx_t construct_mtx;
 374       bool initialized;
 375    } scratch_bos[48 - MIN_SCRATCH_BO_SIZE_LOG2];
 376
 377    struct tu_bo border_color;
 378
 379    struct tu_device_extension_table enabled_extensions;
 380 };
 381
 382 VkResult _tu_device_set_lost(struct tu_device *device,
 383                              const char *file, int line,
 384                              const char *msg, ...) PRINTFLIKE(4, 5);
 385 #define tu_device_set_lost(dev, ...) \
 386    _tu_device_set_lost(dev, __FILE__, __LINE__, __VA_ARGS__)
 387
 388 static inline bool
 389 tu_device_is_lost(struct tu_device *device)
 390 {
 391    return unlikely(p_atomic_read(&device->_lost));
 392 }
 393
 394 VkResult
 395 tu_bo_init_new(struct tu_device *dev, struct tu_bo *bo, uint64_t size);
 396 VkResult
 397 tu_bo_init_dmabuf(struct tu_device *dev,
 398                   struct tu_bo *bo,
 399                   uint64_t size,
 400                   int fd);
 401 int
 402 tu_bo_export_dmabuf(struct tu_device *dev, struct tu_bo *bo);
 403 void
 404 tu_bo_finish(struct tu_device *dev, struct tu_bo *bo);
 405 VkResult
 406 tu_bo_map(struct tu_device *dev, struct tu_bo *bo);
 407
 408 /* Get a scratch bo for use inside a command buffer. This will always return
 409  * the same bo given the same size or similar sizes, so only one scratch bo
 410  * can be used at the same time. It's meant for short-lived things where we
 411  * need to write to some piece of memory, read from it, and then immediately
 412  * discard it.
 413  */
 414 VkResult
 415 tu_get_scratch_bo(struct tu_device *dev, uint64_t size, struct tu_bo **bo);
 416
 417 struct tu_cs_entry
 418 {
 419    /* No ownership */
 420    const struct tu_bo *bo;
 421
 422    uint32_t size;
 423    uint32_t offset;
 424 };
 425
 426 struct tu_cs_memory {
 427    uint32_t *map;
 428    uint64_t iova;
 429 };
 430
 431 struct tu_draw_state {
 432    uint64_t iova : 48;
 433    uint32_t size : 16;
 434 };
 435
 436 enum tu_dynamic_state
 437 {
 438    /* re-use VK_DYNAMIC_STATE_ enums for non-extended dynamic states */
 439    TU_DYNAMIC_STATE_SAMPLE_LOCATIONS = VK_DYNAMIC_STATE_STENCIL_REFERENCE + 1,
 440    TU_DYNAMIC_STATE_COUNT,
 441 };
 442
 443 enum tu_draw_state_group_id
 444 {
 445    TU_DRAW_STATE_PROGRAM,
 446    TU_DRAW_STATE_PROGRAM_BINNING,
 447    TU_DRAW_STATE_TESS,
 448    TU_DRAW_STATE_VB,
 449    TU_DRAW_STATE_VI,
 450    TU_DRAW_STATE_VI_BINNING,
 451    TU_DRAW_STATE_RAST,
 452    TU_DRAW_STATE_DS,
 453    TU_DRAW_STATE_BLEND,
 454    TU_DRAW_STATE_VS_CONST,
 455    TU_DRAW_STATE_HS_CONST,
 456    TU_DRAW_STATE_DS_CONST,
 457    TU_DRAW_STATE_GS_CONST,
 458    TU_DRAW_STATE_FS_CONST,
 459    TU_DRAW_STATE_DESC_SETS,
 460    TU_DRAW_STATE_DESC_SETS_LOAD,
 461    TU_DRAW_STATE_VS_PARAMS,
 462    TU_DRAW_STATE_INPUT_ATTACHMENTS_GMEM,
 463    TU_DRAW_STATE_INPUT_ATTACHMENTS_SYSMEM,
 464
 465    /* dynamic state related draw states */
 466    TU_DRAW_STATE_DYNAMIC,
 467    TU_DRAW_STATE_COUNT = TU_DRAW_STATE_DYNAMIC + TU_DYNAMIC_STATE_COUNT,
 468 };
 469
 470 enum tu_cs_mode
 471 {
 472
 473    /*
 474     * A command stream in TU_CS_MODE_GROW mode grows automatically whenever it
 475     * is full.  tu_cs_begin must be called before command packet emission and
 476     * tu_cs_end must be called after.
 477     *
 478     * This mode may create multiple entries internally.  The entries must be
 479     * submitted together.
 480     */
 481    TU_CS_MODE_GROW,
 482
 483    /*
 484     * A command stream in TU_CS_MODE_EXTERNAL mode wraps an external,
 485     * fixed-size buffer.  tu_cs_begin and tu_cs_end are optional and have no
 486     * effect on it.
 487     *
 488     * This mode does not create any entry or any BO.
 489     */
 490    TU_CS_MODE_EXTERNAL,
 491
 492    /*
 493     * A command stream in TU_CS_MODE_SUB_STREAM mode does not support direct
 494     * command packet emission.  tu_cs_begin_sub_stream must be called to get a
 495     * sub-stream to emit comamnd packets to.  When done with the sub-stream,
 496     * tu_cs_end_sub_stream must be called.
 497     *
 498     * This mode does not create any entry internally.
 499     */
 500    TU_CS_MODE_SUB_STREAM,
 501 };
 502
 503 struct tu_cs
 504 {
 505    uint32_t *start;
 506    uint32_t *cur;
 507    uint32_t *reserved_end;
 508    uint32_t *end;
 509
 510    struct tu_device *device;
 511    enum tu_cs_mode mode;
 512    uint32_t next_bo_size;
 513
 514    struct tu_cs_entry *entries;
 515    uint32_t entry_count;
 516    uint32_t entry_capacity;
 517
 518    struct tu_bo **bos;
 519    uint32_t bo_count;
 520    uint32_t bo_capacity;
 521
 522    /* state for cond_exec_start/cond_exec_end */
 523    uint32_t cond_flags;
 524    uint32_t *cond_dwords;
 525 };
 526
 527 struct tu_device_memory
 528 {
 529    struct tu_bo bo;
 530    VkDeviceSize size;
 531
 532    /* for dedicated allocations */
 533    struct tu_image *image;
 534    struct tu_buffer *buffer;
 535
 536    uint32_t type_index;
 537    void *map;
 538    void *user_ptr;
 539 };
 540
 541 struct tu_descriptor_range
 542 {
 543    uint64_t va;
 544    uint32_t size;
 545 };
 546
 547 struct tu_descriptor_set
 548 {
 549    const struct tu_descriptor_set_layout *layout;
 550    struct tu_descriptor_pool *pool;
 551    uint32_t size;
 552
 553    uint64_t va;
 554    uint32_t *mapped_ptr;
 555
 556    uint32_t *dynamic_descriptors;
 557
 558    struct tu_bo *buffers[0];
 559 };
 560
 561 struct tu_push_descriptor_set
 562 {
 563    struct tu_descriptor_set set;
 564    uint32_t capacity;
 565 };
 566
 567 struct tu_descriptor_pool_entry
 568 {
 569    uint32_t offset;
 570    uint32_t size;
 571    struct tu_descriptor_set *set;
 572 };
 573
 574 struct tu_descriptor_pool
 575 {
 576    struct tu_bo bo;
 577    uint64_t current_offset;
 578    uint64_t size;
 579
 580    uint8_t *host_memory_base;
 581    uint8_t *host_memory_ptr;
 582    uint8_t *host_memory_end;
 583
 584    uint32_t entry_count;
 585    uint32_t max_entry_count;
 586    struct tu_descriptor_pool_entry entries[0];
 587 };
 588
 589 struct tu_descriptor_update_template_entry
 590 {
 591    VkDescriptorType descriptor_type;
 592
 593    /* The number of descriptors to update */
 594    uint32_t descriptor_count;
 595
 596    /* Into mapped_ptr or dynamic_descriptors, in units of the respective array
 597     */
 598    uint32_t dst_offset;
 599
 600    /* In dwords. Not valid/used for dynamic descriptors */
 601    uint32_t dst_stride;
 602
 603    uint32_t buffer_offset;
 604
 605    /* Only valid for combined image samplers and samplers */
 606    uint16_t has_sampler;
 607
 608    /* In bytes */
 609    size_t src_offset;
 610    size_t src_stride;
 611
 612    /* For push descriptors */
 613    const uint32_t *immutable_samplers;
 614 };
 615
 616 struct tu_descriptor_update_template
 617 {
 618    uint32_t entry_count;
 619    struct tu_descriptor_update_template_entry entry[0];
 620 };
 621
 622 struct tu_buffer
 623 {
 624    VkDeviceSize size;
 625
 626    VkBufferUsageFlags usage;
 627    VkBufferCreateFlags flags;
 628
 629    struct tu_bo *bo;
 630    VkDeviceSize bo_offset;
 631 };
 632
 633 static inline uint64_t
 634 tu_buffer_iova(struct tu_buffer *buffer)
 635 {
 636    return buffer->bo->iova + buffer->bo_offset;
 637 }
 638
 639 struct tu_vertex_binding
 640 {
 641    struct tu_buffer *buffer;
 642    VkDeviceSize offset;
 643 };
 644
 645 const char *
 646 tu_get_debug_option_name(int id);
 647
 648 const char *
 649 tu_get_perftest_option_name(int id);
 650
 651 struct tu_descriptor_state
 652 {
 653    struct tu_descriptor_set *sets[MAX_SETS];
 654    uint32_t dynamic_descriptors[MAX_DYNAMIC_BUFFERS * A6XX_TEX_CONST_DWORDS];
 655 };
 656
 657 struct tu_tile
 658 {
 659    uint8_t pipe;
 660    uint8_t slot;
 661    VkOffset2D begin;
 662    VkOffset2D end;
 663 };
 664
 665 struct tu_tiling_config
 666 {
 667    VkRect2D render_area;
 668
 669    /* position and size of the first tile */
 670    VkRect2D tile0;
 671    /* number of tiles */
 672    VkExtent2D tile_count;
 673
 674    /* size of the first VSC pipe */
 675    VkExtent2D pipe0;
 676    /* number of VSC pipes */
 677    VkExtent2D pipe_count;
 678
 679    /* pipe register values */
 680    uint32_t pipe_config[MAX_VSC_PIPES];
 681    uint32_t pipe_sizes[MAX_VSC_PIPES];
 682
 683    /* Whether sysmem rendering must be used */
 684    bool force_sysmem;
 685 };
 686
 687 enum tu_cmd_dirty_bits
 688 {
 689    TU_CMD_DIRTY_COMPUTE_PIPELINE = 1 << 1,
 690    TU_CMD_DIRTY_VERTEX_BUFFERS = 1 << 2,
 691    TU_CMD_DIRTY_DESCRIPTOR_SETS = 1 << 3,
 692    TU_CMD_DIRTY_COMPUTE_DESCRIPTOR_SETS = 1 << 4,
 693    TU_CMD_DIRTY_SHADER_CONSTS = 1 << 5,
 694    /* all draw states were disabled and need to be re-enabled: */
 695    TU_CMD_DIRTY_DRAW_STATE = 1 << 7,
 696 };
 697
 698 /* There are only three cache domains we have to care about: the CCU, or
 699  * color cache unit, which is used for color and depth/stencil attachments
 700  * and copy/blit destinations, and is split conceptually into color and depth,
 701  * and the universal cache or UCHE which is used for pretty much everything
 702  * else, except for the CP (uncached) and host. We need to flush whenever data
 703  * crosses these boundaries.
 704  */
 705
 706 enum tu_cmd_access_mask {
 707    TU_ACCESS_UCHE_READ = 1 << 0,
 708    TU_ACCESS_UCHE_WRITE = 1 << 1,
 709    TU_ACCESS_CCU_COLOR_READ = 1 << 2,
 710    TU_ACCESS_CCU_COLOR_WRITE = 1 << 3,
 711    TU_ACCESS_CCU_DEPTH_READ = 1 << 4,
 712    TU_ACCESS_CCU_DEPTH_WRITE = 1 << 5,
 713
 714    /* Experiments have shown that while it's safe to avoid flushing the CCU
 715     * after each blit/renderpass, it's not safe to assume that subsequent
 716     * lookups with a different attachment state will hit unflushed cache
 717     * entries. That is, the CCU needs to be flushed and possibly invalidated
 718     * when accessing memory with a different attachment state. Writing to an
 719     * attachment under the following conditions after clearing using the
 720     * normal 2d engine path is known to have issues:
 721     *
 722     * - It isn't the 0'th layer.
 723     * - There are more than one attachment, and this isn't the 0'th attachment
 724     *   (this seems to also depend on the cpp of the attachments).
 725     *
 726     * Our best guess is that the layer/MRT state is used when computing
 727     * the location of a cache entry in CCU, to avoid conflicts. We assume that
 728     * any access in a renderpass after or before an access by a transfer needs
 729     * a flush/invalidate, and use the _INCOHERENT variants to represent access
 730     * by a transfer.
 731     */
 732    TU_ACCESS_CCU_COLOR_INCOHERENT_READ = 1 << 6,
 733    TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE = 1 << 7,
 734    TU_ACCESS_CCU_DEPTH_INCOHERENT_READ = 1 << 8,
 735    TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE = 1 << 9,
 736
 737    TU_ACCESS_SYSMEM_READ = 1 << 10,
 738    TU_ACCESS_SYSMEM_WRITE = 1 << 11,
 739
 740    /* Set if a WFI is required due to data being read by the CP or the 2D
 741     * engine.
 742     */
 743    TU_ACCESS_WFI_READ = 1 << 12,
 744
 745    TU_ACCESS_READ =
 746       TU_ACCESS_UCHE_READ |
 747       TU_ACCESS_CCU_COLOR_READ |
 748       TU_ACCESS_CCU_DEPTH_READ |
 749       TU_ACCESS_CCU_COLOR_INCOHERENT_READ |
 750       TU_ACCESS_CCU_DEPTH_INCOHERENT_READ |
 751       TU_ACCESS_SYSMEM_READ,
 752
 753    TU_ACCESS_WRITE =
 754       TU_ACCESS_UCHE_WRITE |
 755       TU_ACCESS_CCU_COLOR_WRITE |
 756       TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE |
 757       TU_ACCESS_CCU_DEPTH_WRITE |
 758       TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE |
 759       TU_ACCESS_SYSMEM_WRITE,
 760
 761    TU_ACCESS_ALL =
 762       TU_ACCESS_READ |
 763       TU_ACCESS_WRITE,
 764 };
 765
 766 enum tu_cmd_flush_bits {
 767    TU_CMD_FLAG_CCU_FLUSH_DEPTH = 1 << 0,
 768    TU_CMD_FLAG_CCU_FLUSH_COLOR = 1 << 1,
 769    TU_CMD_FLAG_CCU_INVALIDATE_DEPTH = 1 << 2,
 770    TU_CMD_FLAG_CCU_INVALIDATE_COLOR = 1 << 3,
 771    TU_CMD_FLAG_CACHE_FLUSH = 1 << 4,
 772    TU_CMD_FLAG_CACHE_INVALIDATE = 1 << 5,
 773
 774    TU_CMD_FLAG_ALL_FLUSH =
 775       TU_CMD_FLAG_CCU_FLUSH_DEPTH |
 776       TU_CMD_FLAG_CCU_FLUSH_COLOR |
 777       TU_CMD_FLAG_CACHE_FLUSH,
 778
 779    TU_CMD_FLAG_ALL_INVALIDATE =
 780       TU_CMD_FLAG_CCU_INVALIDATE_DEPTH |
 781       TU_CMD_FLAG_CCU_INVALIDATE_COLOR |
 782       TU_CMD_FLAG_CACHE_INVALIDATE,
 783
 784    TU_CMD_FLAG_WFI = 1 << 6,
 785 };
 786
 787 /* Changing the CCU from sysmem mode to gmem mode or vice-versa is pretty
 788  * heavy, involving a CCU cache flush/invalidate and a WFI in order to change
 789  * which part of the gmem is used by the CCU. Here we keep track of what the
 790  * state of the CCU.
 791  */
 792 enum tu_cmd_ccu_state {
 793    TU_CMD_CCU_SYSMEM,
 794    TU_CMD_CCU_GMEM,
 795    TU_CMD_CCU_UNKNOWN,
 796 };
 797
 798 struct tu_cache_state {
 799    /* Caches which must be made available (flushed) eventually if there are
 800     * any users outside that cache domain, and caches which must be
 801     * invalidated eventually if there are any reads.
 802     */
 803    enum tu_cmd_flush_bits pending_flush_bits;
 804    /* Pending flushes */
 805    enum tu_cmd_flush_bits flush_bits;
 806 };
 807
 808 struct tu_cmd_state
 809 {
 810    uint32_t dirty;
 811
 812    struct tu_pipeline *pipeline;
 813    struct tu_pipeline *compute_pipeline;
 814
 815    /* Vertex buffers */
 816    struct
 817    {
 818       struct tu_buffer *buffers[MAX_VBS];
 819       VkDeviceSize offsets[MAX_VBS];
 820    } vb;
 821
 822    /* for dynamic states that can't be emitted directly */
 823    uint32_t dynamic_stencil_mask;
 824    uint32_t dynamic_stencil_wrmask;
 825    uint32_t dynamic_stencil_ref;
 826    uint32_t dynamic_gras_su_cntl;
 827
 828    /* saved states to re-emit in TU_CMD_DIRTY_DRAW_STATE case */
 829    struct tu_draw_state dynamic_state[TU_DYNAMIC_STATE_COUNT];
 830    struct tu_cs_entry vertex_buffers_ib;
 831    struct tu_cs_entry shader_const_ib[MESA_SHADER_STAGES];
 832    struct tu_cs_entry desc_sets_ib, desc_sets_load_ib;
 833    struct tu_cs_entry ia_gmem_ib, ia_sysmem_ib;
 834
 835    struct tu_draw_state vs_params;
 836
 837    /* Index buffer */
 838    uint64_t index_va;
 839    uint32_t max_index_count;
 840    uint8_t index_size;
 841
 842    /* because streamout base has to be 32-byte aligned
 843     * there is an extra offset to deal with when it is
 844     * unaligned
 845     */
 846    uint8_t streamout_offset[IR3_MAX_SO_BUFFERS];
 847
 848    /* Renderpasses are tricky, because we may need to flush differently if
 849     * using sysmem vs. gmem and therefore we have to delay any flushing that
 850     * happens before a renderpass. So we have to have two copies of the flush
 851     * state, one for intra-renderpass flushes (i.e. renderpass dependencies)
 852     * and one for outside a renderpass.
 853     */
 854    struct tu_cache_state cache;
 855    struct tu_cache_state renderpass_cache;
 856
 857    enum tu_cmd_ccu_state ccu_state;
 858
 859    const struct tu_render_pass *pass;
 860    const struct tu_subpass *subpass;
 861    const struct tu_framebuffer *framebuffer;
 862
 863    struct tu_tiling_config tiling_config;
 864
 865    struct tu_cs_entry tile_store_ib;
 866
 867    bool xfb_used;
 868 };
 869
 870 struct tu_cmd_pool
 871 {
 872    VkAllocationCallbacks alloc;
 873    struct list_head cmd_buffers;
 874    struct list_head free_cmd_buffers;
 875    uint32_t queue_family_index;
 876 };
 877
 878 struct tu_cmd_buffer_upload
 879 {
 880    uint8_t *map;
 881    unsigned offset;
 882    uint64_t size;
 883    struct list_head list;
 884 };
 885
 886 enum tu_cmd_buffer_status
 887 {
 888    TU_CMD_BUFFER_STATUS_INVALID,
 889    TU_CMD_BUFFER_STATUS_INITIAL,
 890    TU_CMD_BUFFER_STATUS_RECORDING,
 891    TU_CMD_BUFFER_STATUS_EXECUTABLE,
 892    TU_CMD_BUFFER_STATUS_PENDING,
 893 };
 894
 895 struct tu_bo_list
 896 {
 897    uint32_t count;
 898    uint32_t capacity;
 899    struct drm_msm_gem_submit_bo *bo_infos;
 900 };
 901
 902 #define TU_BO_LIST_FAILED (~0)
 903
 904 void
 905 tu_bo_list_init(struct tu_bo_list *list);
 906 void
 907 tu_bo_list_destroy(struct tu_bo_list *list);
 908 void
 909 tu_bo_list_reset(struct tu_bo_list *list);
 910 uint32_t
 911 tu_bo_list_add(struct tu_bo_list *list,
 912                const struct tu_bo *bo,
 913                uint32_t flags);
 914 VkResult
 915 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other);
 916
 917 /* This struct defines the layout of the scratch_bo */
 918 struct tu6_control
 919 {
 920    uint32_t seqno_dummy;          /* dummy seqno for CP_EVENT_WRITE */
 921    uint32_t _pad0;
 922    volatile uint32_t vsc_overflow;
 923    uint32_t _pad1;
 924    /* flag set from cmdstream when VSC overflow detected: */
 925    uint32_t vsc_scratch;
 926    uint32_t _pad2;
 927    uint32_t _pad3;
 928    uint32_t _pad4;
 929
 930    /* scratch space for VPC_SO[i].FLUSH_BASE_LO/HI, start on 32 byte boundary. */
 931    struct {
 932       uint32_t offset;
 933       uint32_t pad[7];
 934    } flush_base[4];
 935 };
 936
 937 #define ctrl_offset(member) offsetof(struct tu6_control, member)
 938
 939 struct tu_cmd_buffer
 940 {
 941    VK_LOADER_DATA _loader_data;
 942
 943    struct tu_device *device;
 944
 945    struct tu_cmd_pool *pool;
 946    struct list_head pool_link;
 947
 948    VkCommandBufferUsageFlags usage_flags;
 949    VkCommandBufferLevel level;
 950    enum tu_cmd_buffer_status status;
 951
 952    struct tu_cmd_state state;
 953    struct tu_vertex_binding vertex_bindings[MAX_VBS];
 954    uint32_t vertex_bindings_set;
 955    uint32_t queue_family_index;
 956
 957    uint32_t push_constants[MAX_PUSH_CONSTANTS_SIZE / 4];
 958    VkShaderStageFlags push_constant_stages;
 959    struct tu_descriptor_set meta_push_descriptors;
 960
 961    struct tu_descriptor_state descriptors[MAX_BIND_POINTS];
 962
 963    struct tu_cmd_buffer_upload upload;
 964
 965    VkResult record_result;
 966
 967    struct tu_bo_list bo_list;
 968    struct tu_cs cs;
 969    struct tu_cs draw_cs;
 970    struct tu_cs draw_epilogue_cs;
 971    struct tu_cs sub_cs;
 972
 973    struct tu_bo scratch_bo;
 974
 975    bool has_tess;
 976
 977    struct tu_bo vsc_draw_strm;
 978    struct tu_bo vsc_prim_strm;
 979    uint32_t vsc_draw_strm_pitch;
 980    uint32_t vsc_prim_strm_pitch;
 981    bool use_vsc_data;
 982 };
 983
 984 /* Temporary struct for tracking a register state to be written, used by
 985  * a6xx-pack.h and tu_cs_emit_regs()
 986  */
 987 struct tu_reg_value {
 988    uint32_t reg;
 989    uint64_t value;
 990    bool is_address;
 991    struct tu_bo *bo;
 992    bool bo_write;
 993    uint32_t bo_offset;
 994    uint32_t bo_shift;
 995 };
 996
 997
 998 void tu_emit_cache_flush_renderpass(struct tu_cmd_buffer *cmd_buffer,
 999                                     struct tu_cs *cs);
1000
1001 void tu_emit_cache_flush_ccu(struct tu_cmd_buffer *cmd_buffer,
1002                              struct tu_cs *cs,
1003                              enum tu_cmd_ccu_state ccu_state);
1004
1005 void
1006 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
1007                      struct tu_cs *cs,
1008                      enum vgt_event_type event);
1009
1010 static inline struct tu_descriptor_state *
1011 tu_get_descriptors_state(struct tu_cmd_buffer *cmd_buffer,
1012                          VkPipelineBindPoint bind_point)
1013 {
1014    return &cmd_buffer->descriptors[bind_point];
1015 }
1016
1017 struct tu_event
1018 {
1019    struct tu_bo bo;
1020 };
1021
1022 struct tu_shader_module
1023 {
1024    unsigned char sha1[20];
1025
1026    uint32_t code_size;
1027    const uint32_t *code[0];
1028 };
1029
1030 struct tu_push_constant_range
1031 {
1032    uint32_t lo;
1033    uint32_t count;
1034 };
1035
1036 struct tu_shader
1037 {
1038    struct ir3_shader *ir3_shader;
1039
1040    struct tu_push_constant_range push_consts;
1041    uint8_t active_desc_sets;
1042 };
1043
1044 struct tu_shader *
1045 tu_shader_create(struct tu_device *dev,
1046                  gl_shader_stage stage,
1047                  const VkPipelineShaderStageCreateInfo *stage_info,
1048                  struct tu_pipeline_layout *layout,
1049                  const VkAllocationCallbacks *alloc);
1050
1051 void
1052 tu_shader_destroy(struct tu_device *dev,
1053                   struct tu_shader *shader,
1054                   const VkAllocationCallbacks *alloc);
1055
1056 struct tu_program_descriptor_linkage
1057 {
1058    struct ir3_const_state const_state;
1059
1060    uint32_t constlen;
1061
1062    struct tu_push_constant_range push_consts;
1063 };
1064
1065 struct tu_pipeline
1066 {
1067    struct tu_cs cs;
1068
1069    struct tu_pipeline_layout *layout;
1070
1071    bool need_indirect_descriptor_sets;
1072    VkShaderStageFlags active_stages;
1073    uint32_t active_desc_sets;
1074
1075    /* mask of enabled dynamic states
1076     * if BIT(i) is set, pipeline->dynamic_state[i] is *NOT* used
1077     */
1078    uint32_t dynamic_state_mask;
1079    struct tu_draw_state dynamic_state[TU_DYNAMIC_STATE_COUNT];
1080
1081    /* gras_su_cntl without line width, used for dynamic line width state */
1082    uint32_t gras_su_cntl;
1083
1084    struct
1085    {
1086       struct tu_cs_entry state_ib;
1087       struct tu_cs_entry binning_state_ib;
1088
1089       struct tu_program_descriptor_linkage link[MESA_SHADER_STAGES];
1090    } program;
1091
1092    struct
1093    {
1094       struct tu_cs_entry state_ib;
1095    } load_state;
1096
1097    struct
1098    {
1099       struct tu_cs_entry state_ib;
1100       struct tu_cs_entry binning_state_ib;
1101       uint32_t bindings_used;
1102    } vi;
1103
1104    struct
1105    {
1106       enum pc_di_primtype primtype;
1107       bool primitive_restart;
1108    } ia;
1109
1110    struct
1111    {
1112       uint32_t patch_type;
1113       uint32_t per_vertex_output_size;
1114       uint32_t per_patch_output_size;
1115       uint32_t hs_bo_regid;
1116       uint32_t ds_bo_regid;
1117       bool upper_left_domain_origin;
1118    } tess;
1119
1120    struct
1121    {
1122       struct tu_cs_entry state_ib;
1123    } rast;
1124
1125    struct
1126    {
1127       struct tu_cs_entry state_ib;
1128    } ds;
1129
1130    struct
1131    {
1132       struct tu_cs_entry state_ib;
1133    } blend;
1134
1135    struct
1136    {
1137       uint32_t local_size[3];
1138    } compute;
1139 };
1140
1141 void
1142 tu6_emit_viewport(struct tu_cs *cs, const VkViewport *viewport);
1143
1144 void
1145 tu6_emit_scissor(struct tu_cs *cs, const VkRect2D *scissor);
1146
1147 void
1148 tu6_emit_sample_locations(struct tu_cs *cs, const VkSampleLocationsInfoEXT *samp_loc);
1149
1150 void
1151 tu6_emit_depth_bias(struct tu_cs *cs,
1152                     float constant_factor,
1153                     float clamp,
1154                     float slope_factor);
1155
1156 void tu6_emit_msaa(struct tu_cs *cs, VkSampleCountFlagBits samples);
1157
1158 void tu6_emit_window_scissor(struct tu_cs *cs, uint32_t x1, uint32_t y1, uint32_t x2, uint32_t y2);
1159
1160 void tu6_emit_window_offset(struct tu_cs *cs, uint32_t x1, uint32_t y1);
1161
1162 void
1163 tu6_emit_xs_config(struct tu_cs *cs,
1164                    gl_shader_stage stage,
1165                    const struct ir3_shader_variant *xs,
1166                    uint64_t binary_iova);
1167
1168 void
1169 tu6_emit_vpc(struct tu_cs *cs,
1170              const struct ir3_shader_variant *vs,
1171              const struct ir3_shader_variant *hs,
1172              const struct ir3_shader_variant *ds,
1173              const struct ir3_shader_variant *gs,
1174              const struct ir3_shader_variant *fs);
1175
1176 void
1177 tu6_emit_fs_inputs(struct tu_cs *cs, const struct ir3_shader_variant *fs);
1178
1179 struct tu_image_view;
1180
1181 void
1182 tu_resolve_sysmem(struct tu_cmd_buffer *cmd,
1183                   struct tu_cs *cs,
1184                   struct tu_image_view *src,
1185                   struct tu_image_view *dst,
1186                   uint32_t layers,
1187                   const VkRect2D *rect);
1188
1189 void
1190 tu_clear_sysmem_attachment(struct tu_cmd_buffer *cmd,
1191                            struct tu_cs *cs,
1192                            uint32_t a,
1193                            const VkRenderPassBeginInfo *info);
1194
1195 void
1196 tu_clear_gmem_attachment(struct tu_cmd_buffer *cmd,
1197                          struct tu_cs *cs,
1198                          uint32_t a,
1199                          const VkRenderPassBeginInfo *info);
1200
1201 void
1202 tu_load_gmem_attachment(struct tu_cmd_buffer *cmd,
1203                         struct tu_cs *cs,
1204                         uint32_t a,
1205                         bool force_load);
1206
1207 /* expose this function to be able to emit load without checking LOAD_OP */
1208 void
1209 tu_emit_load_gmem_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t a);
1210
1211 /* note: gmem store can also resolve */
1212 void
1213 tu_store_gmem_attachment(struct tu_cmd_buffer *cmd,
1214                          struct tu_cs *cs,
1215                          uint32_t a,
1216                          uint32_t gmem_a);
1217
1218 enum tu_supported_formats {
1219    FMT_VERTEX = 1,
1220    FMT_TEXTURE = 2,
1221    FMT_COLOR = 4,
1222 };
1223
1224 struct tu_native_format
1225 {
1226    enum a6xx_format fmt : 8;
1227    enum a3xx_color_swap swap : 8;
1228    enum a6xx_tile_mode tile_mode : 8;
1229    enum tu_supported_formats supported : 8;
1230 };
1231
1232 struct tu_native_format tu6_format_vtx(VkFormat format);
1233 struct tu_native_format tu6_format_color(VkFormat format, enum a6xx_tile_mode tile_mode);
1234 struct tu_native_format tu6_format_texture(VkFormat format, enum a6xx_tile_mode tile_mode);
1235
1236 static inline enum a6xx_format
1237 tu6_base_format(VkFormat format)
1238 {
1239    /* note: tu6_format_color doesn't care about tiling for .fmt field */
1240    return tu6_format_color(format, TILE6_LINEAR).fmt;
1241 }
1242
1243 struct tu_image
1244 {
1245    VkImageType type;
1246    /* The original VkFormat provided by the client.  This may not match any
1247     * of the actual surface formats.
1248     */
1249    VkFormat vk_format;
1250    VkImageAspectFlags aspects;
1251    VkImageUsageFlags usage;  /**< Superset of VkImageCreateInfo::usage. */
1252    VkImageTiling tiling;     /** VkImageCreateInfo::tiling */
1253    VkImageCreateFlags flags; /** VkImageCreateInfo::flags */
1254    VkExtent3D extent;
1255    uint32_t level_count;
1256    uint32_t layer_count;
1257    VkSampleCountFlagBits samples;
1258
1259    struct fdl_layout layout;
1260
1261    unsigned queue_family_mask;
1262    bool exclusive;
1263    bool shareable;
1264
1265    /* For VK_ANDROID_native_buffer, the WSI image owns the memory, */
1266    VkDeviceMemory owned_memory;
1267
1268    /* Set when bound */
1269    struct tu_bo *bo;
1270    VkDeviceSize bo_offset;
1271 };
1272
1273 static inline uint32_t
1274 tu_get_layerCount(const struct tu_image *image,
1275                   const VkImageSubresourceRange *range)
1276 {
1277    return range->layerCount == VK_REMAINING_ARRAY_LAYERS
1278              ? image->layer_count - range->baseArrayLayer
1279              : range->layerCount;
1280 }
1281
1282 static inline uint32_t
1283 tu_get_levelCount(const struct tu_image *image,
1284                   const VkImageSubresourceRange *range)
1285 {
1286    return range->levelCount == VK_REMAINING_MIP_LEVELS
1287              ? image->level_count - range->baseMipLevel
1288              : range->levelCount;
1289 }
1290
1291 struct tu_image_view
1292 {
1293    struct tu_image *image; /**< VkImageViewCreateInfo::image */
1294
1295    uint64_t base_addr;
1296    uint64_t ubwc_addr;
1297    uint32_t layer_size;
1298    uint32_t ubwc_layer_size;
1299
1300    /* used to determine if fast gmem store path can be used */
1301    VkExtent2D extent;
1302    bool need_y2_align;
1303
1304    bool ubwc_enabled;
1305
1306    uint32_t descriptor[A6XX_TEX_CONST_DWORDS];
1307
1308    /* Descriptor for use as a storage image as opposed to a sampled image.
1309     * This has a few differences for cube maps (e.g. type).
1310     */
1311    uint32_t storage_descriptor[A6XX_TEX_CONST_DWORDS];
1312
1313    /* pre-filled register values */
1314    uint32_t PITCH;
1315    uint32_t FLAG_BUFFER_PITCH;
1316
1317    uint32_t RB_MRT_BUF_INFO;
1318    uint32_t SP_FS_MRT_REG;
1319
1320    uint32_t SP_PS_2D_SRC_INFO;
1321    uint32_t SP_PS_2D_SRC_SIZE;
1322
1323    uint32_t RB_2D_DST_INFO;
1324
1325    uint32_t RB_BLIT_DST_INFO;
1326 };
1327
1328 struct tu_sampler_ycbcr_conversion {
1329    VkFormat format;
1330    VkSamplerYcbcrModelConversion ycbcr_model;
1331    VkSamplerYcbcrRange ycbcr_range;
1332    VkComponentMapping components;
1333    VkChromaLocation chroma_offsets[2];
1334    VkFilter chroma_filter;
1335 };
1336
1337 struct tu_sampler {
1338    uint32_t descriptor[A6XX_TEX_SAMP_DWORDS];
1339    struct tu_sampler_ycbcr_conversion *ycbcr_sampler;
1340 };
1341
1342 void
1343 tu_cs_image_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1344
1345 void
1346 tu_cs_image_ref_2d(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer, bool src);
1347
1348 void
1349 tu_cs_image_flag_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1350
1351 VkResult
1352 tu_image_create(VkDevice _device,
1353                 const VkImageCreateInfo *pCreateInfo,
1354                 const VkAllocationCallbacks *alloc,
1355                 VkImage *pImage,
1356                 uint64_t modifier,
1357                 const VkSubresourceLayout *plane_layouts);
1358
1359 VkResult
1360 tu_image_from_gralloc(VkDevice device_h,
1361                       const VkImageCreateInfo *base_info,
1362                       const VkNativeBufferANDROID *gralloc_info,
1363                       const VkAllocationCallbacks *alloc,
1364                       VkImage *out_image_h);
1365
1366 void
1367 tu_image_view_init(struct tu_image_view *view,
1368                    const VkImageViewCreateInfo *pCreateInfo);
1369
1370 struct tu_buffer_view
1371 {
1372    uint32_t descriptor[A6XX_TEX_CONST_DWORDS];
1373
1374    struct tu_buffer *buffer;
1375 };
1376 void
1377 tu_buffer_view_init(struct tu_buffer_view *view,
1378                     struct tu_device *device,
1379                     const VkBufferViewCreateInfo *pCreateInfo);
1380
1381 struct tu_attachment_info
1382 {
1383    struct tu_image_view *attachment;
1384 };
1385
1386 struct tu_framebuffer
1387 {
1388    uint32_t width;
1389    uint32_t height;
1390    uint32_t layers;
1391
1392    uint32_t attachment_count;
1393    struct tu_attachment_info attachments[0];
1394 };
1395
1396 struct tu_subpass_barrier {
1397    VkPipelineStageFlags src_stage_mask;
1398    VkAccessFlags src_access_mask;
1399    VkAccessFlags dst_access_mask;
1400    bool incoherent_ccu_color, incoherent_ccu_depth;
1401 };
1402
1403 struct tu_subpass_attachment
1404 {
1405    uint32_t attachment;
1406 };
1407
1408 struct tu_subpass
1409 {
1410    uint32_t input_count;
1411    uint32_t color_count;
1412    struct tu_subpass_attachment *input_attachments;
1413    struct tu_subpass_attachment *color_attachments;
1414    struct tu_subpass_attachment *resolve_attachments;
1415    struct tu_subpass_attachment depth_stencil_attachment;
1416
1417    VkSampleCountFlagBits samples;
1418
1419    uint32_t srgb_cntl;
1420
1421    struct tu_subpass_barrier start_barrier;
1422 };
1423
1424 struct tu_render_pass_attachment
1425 {
1426    VkFormat format;
1427    uint32_t samples;
1428    uint32_t cpp;
1429    VkImageAspectFlags clear_mask;
1430    bool load;
1431    bool store;
1432    int32_t gmem_offset;
1433 };
1434
1435 struct tu_render_pass
1436 {
1437    uint32_t attachment_count;
1438    uint32_t subpass_count;
1439    uint32_t gmem_pixels;
1440    uint32_t tile_align_w;
1441    struct tu_subpass_attachment *subpass_attachments;
1442    struct tu_render_pass_attachment *attachments;
1443    struct tu_subpass_barrier end_barrier;
1444    struct tu_subpass subpasses[0];
1445 };
1446
1447 struct tu_query_pool
1448 {
1449    VkQueryType type;
1450    uint32_t stride;
1451    uint64_t size;
1452    uint32_t pipeline_statistics;
1453    struct tu_bo bo;
1454 };
1455
1456 enum tu_semaphore_kind
1457 {
1458    TU_SEMAPHORE_NONE,
1459    TU_SEMAPHORE_SYNCOBJ,
1460 };
1461
1462 struct tu_semaphore_part
1463 {
1464    enum tu_semaphore_kind kind;
1465    union {
1466       uint32_t syncobj;
1467    };
1468 };
1469
1470 struct tu_semaphore
1471 {
1472    struct tu_semaphore_part permanent;
1473    struct tu_semaphore_part temporary;
1474 };
1475
1476 void
1477 tu_set_descriptor_set(struct tu_cmd_buffer *cmd_buffer,
1478                       VkPipelineBindPoint bind_point,
1479                       struct tu_descriptor_set *set,
1480                       unsigned idx);
1481
1482 void
1483 tu_update_descriptor_sets(struct tu_device *device,
1484                           struct tu_cmd_buffer *cmd_buffer,
1485                           VkDescriptorSet overrideSet,
1486                           uint32_t descriptorWriteCount,
1487                           const VkWriteDescriptorSet *pDescriptorWrites,
1488                           uint32_t descriptorCopyCount,
1489                           const VkCopyDescriptorSet *pDescriptorCopies);
1490
1491 void
1492 tu_update_descriptor_set_with_template(
1493    struct tu_device *device,
1494    struct tu_cmd_buffer *cmd_buffer,
1495    struct tu_descriptor_set *set,
1496    VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1497    const void *pData);
1498
1499 int
1500 tu_drm_get_gpu_id(const struct tu_physical_device *dev, uint32_t *id);
1501
1502 int
1503 tu_drm_get_gmem_size(const struct tu_physical_device *dev, uint32_t *size);
1504
1505 int
1506 tu_drm_get_gmem_base(const struct tu_physical_device *dev, uint64_t *base);
1507
1508 int
1509 tu_drm_submitqueue_new(const struct tu_device *dev,
1510                        int priority,
1511                        uint32_t *queue_id);
1512
1513 void
1514 tu_drm_submitqueue_close(const struct tu_device *dev, uint32_t queue_id);
1515
1516 uint32_t
1517 tu_gem_new(const struct tu_device *dev, uint64_t size, uint32_t flags);
1518 uint32_t
1519 tu_gem_import_dmabuf(const struct tu_device *dev,
1520                      int prime_fd,
1521                      uint64_t size);
1522 int
1523 tu_gem_export_dmabuf(const struct tu_device *dev, uint32_t gem_handle);
1524 void
1525 tu_gem_close(const struct tu_device *dev, uint32_t gem_handle);
1526 uint64_t
1527 tu_gem_info_offset(const struct tu_device *dev, uint32_t gem_handle);
1528 uint64_t
1529 tu_gem_info_iova(const struct tu_device *dev, uint32_t gem_handle);
1530
1531 #define TU_DEFINE_HANDLE_CASTS(__tu_type, __VkType)                          \
1532                                                                              \
1533    static inline struct __tu_type *__tu_type##_from_handle(__VkType _handle) \
1534    {                                                                         \
1535       return (struct __tu_type *) _handle;                                   \
1536    }                                                                         \
1537                                                                              \
1538    static inline __VkType __tu_type##_to_handle(struct __tu_type *_obj)      \
1539    {                                                                         \
1540       return (__VkType) _obj;                                                \
1541    }
1542
1543 #define TU_DEFINE_NONDISP_HANDLE_CASTS(__tu_type, __VkType)                  \
1544                                                                              \
1545    static inline struct __tu_type *__tu_type##_from_handle(__VkType _handle) \
1546    {                                                                         \
1547       return (struct __tu_type *) (uintptr_t) _handle;                       \
1548    }                                                                         \
1549                                                                              \
1550    static inline __VkType __tu_type##_to_handle(struct __tu_type *_obj)      \
1551    {                                                                         \
1552       return (__VkType)(uintptr_t) _obj;                                     \
1553    }
1554
1555 #define TU_FROM_HANDLE(__tu_type, __name, __handle)                          \
1556    struct __tu_type *__name = __tu_type##_from_handle(__handle)
1557
1558 TU_DEFINE_HANDLE_CASTS(tu_cmd_buffer, VkCommandBuffer)
1559 TU_DEFINE_HANDLE_CASTS(tu_device, VkDevice)
1560 TU_DEFINE_HANDLE_CASTS(tu_instance, VkInstance)
1561 TU_DEFINE_HANDLE_CASTS(tu_physical_device, VkPhysicalDevice)
1562 TU_DEFINE_HANDLE_CASTS(tu_queue, VkQueue)
1563
1564 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_cmd_pool, VkCommandPool)
1565 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_buffer, VkBuffer)
1566 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_buffer_view, VkBufferView)
1567 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_pool, VkDescriptorPool)
1568 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_set, VkDescriptorSet)
1569 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_set_layout,
1570                                VkDescriptorSetLayout)
1571 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_update_template,
1572                                VkDescriptorUpdateTemplate)
1573 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_device_memory, VkDeviceMemory)
1574 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_fence, VkFence)
1575 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_event, VkEvent)
1576 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_framebuffer, VkFramebuffer)
1577 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_image, VkImage)
1578 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_image_view, VkImageView);
1579 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline_cache, VkPipelineCache)
1580 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline, VkPipeline)
1581 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline_layout, VkPipelineLayout)
1582 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_query_pool, VkQueryPool)
1583 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_render_pass, VkRenderPass)
1584 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_sampler, VkSampler)
1585 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_sampler_ycbcr_conversion, VkSamplerYcbcrConversion)
1586 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_shader_module, VkShaderModule)
1587 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_semaphore, VkSemaphore)
1588
1589 #endif /* TU_PRIVATE_H */