src/freedreno/vulkan/tu_private.h

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  */
  27
  28 #ifndef TU_PRIVATE_H
  29 #define TU_PRIVATE_H
  30
  31 #include <assert.h>
  32 #include <pthread.h>
  33 #include <stdbool.h>
  34 #include <stdint.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #ifdef HAVE_VALGRIND
  39 #include <memcheck.h>
  40 #include <valgrind.h>
  41 #define VG(x) x
  42 #else
  43 #define VG(x) ((void)0)
  44 #endif
  45
  46 #include "c11/threads.h"
  47 #include "main/macros.h"
  48 #include "util/list.h"
  49 #include "util/macros.h"
  50 #include "vk_alloc.h"
  51 #include "vk_debug_report.h"
  52 #include "wsi_common.h"
  53
  54 #include "drm-uapi/msm_drm.h"
  55 #include "ir3/ir3_compiler.h"
  56 #include "ir3/ir3_shader.h"
  57
  58 #include "adreno_common.xml.h"
  59 #include "adreno_pm4.xml.h"
  60 #include "a6xx.xml.h"
  61 #include "fdl/freedreno_layout.h"
  62
  63 #include "tu_descriptor_set.h"
  64 #include "tu_extensions.h"
  65 #include "tu_util.h"
  66
  67 /* Pre-declarations needed for WSI entrypoints */
  68 struct wl_surface;
  69 struct wl_display;
  70 typedef struct xcb_connection_t xcb_connection_t;
  71 typedef uint32_t xcb_visualid_t;
  72 typedef uint32_t xcb_window_t;
  73
  74 #include <vulkan/vk_android_native_buffer.h>
  75 #include <vulkan/vk_icd.h>
  76 #include <vulkan/vulkan.h>
  77 #include <vulkan/vulkan_intel.h>
  78
  79 #include "tu_entrypoints.h"
  80
  81 #include "vk_format.h"
  82
  83 #define MAX_VBS 32
  84 #define MAX_VERTEX_ATTRIBS 32
  85 #define MAX_RTS 8
  86 #define MAX_VSC_PIPES 32
  87 #define MAX_VIEWPORTS 1
  88 #define MAX_SCISSORS 16
  89 #define MAX_DISCARD_RECTANGLES 4
  90 #define MAX_PUSH_CONSTANTS_SIZE 128
  91 #define MAX_PUSH_DESCRIPTORS 32
  92 #define MAX_DYNAMIC_UNIFORM_BUFFERS 16
  93 #define MAX_DYNAMIC_STORAGE_BUFFERS 8
  94 #define MAX_DYNAMIC_BUFFERS                                                  \
  95    (MAX_DYNAMIC_UNIFORM_BUFFERS + MAX_DYNAMIC_STORAGE_BUFFERS)
  96 #define TU_MAX_DRM_DEVICES 8
  97 #define MAX_VIEWS 8
  98 #define MAX_BIND_POINTS 2 /* compute + graphics */
  99 /* The Qualcomm driver exposes 0x20000058 */
 100 #define MAX_STORAGE_BUFFER_RANGE 0x20000000
 101 /* We use ldc for uniform buffer loads, just like the Qualcomm driver, so
 102  * expose the same maximum range.
 103  * TODO: The SIZE bitfield is 15 bits, and in 4-dword units, so the actual
 104  * range might be higher.
 105  */
 106 #define MAX_UNIFORM_BUFFER_RANGE 0x10000
 107
 108 #define A6XX_TEX_CONST_DWORDS 16
 109 #define A6XX_TEX_SAMP_DWORDS 4
 110
 111 #define tu_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
 112
 113 static inline uint32_t
 114 tu_minify(uint32_t n, uint32_t levels)
 115 {
 116    if (unlikely(n == 0))
 117       return 0;
 118    else
 119       return MAX2(n >> levels, 1);
 120 }
 121
 122 #define for_each_bit(b, dword)                                               \
 123    for (uint32_t __dword = (dword);                                          \
 124         (b) = __builtin_ffs(__dword) - 1, __dword; __dword &= ~(1 << (b)))
 125
 126 #define typed_memcpy(dest, src, count)                                       \
 127    ({                                                                        \
 128       STATIC_ASSERT(sizeof(*src) == sizeof(*dest));                          \
 129       memcpy((dest), (src), (count) * sizeof(*(src)));                       \
 130    })
 131
 132 #define COND(bool, val) ((bool) ? (val) : 0)
 133 #define BIT(bit) (1u << (bit))
 134
 135 /* Whenever we generate an error, pass it through this function. Useful for
 136  * debugging, where we can break on it. Only call at error site, not when
 137  * propagating errors. Might be useful to plug in a stack trace here.
 138  */
 139
 140 struct tu_instance;
 141
 142 VkResult
 143 __vk_errorf(struct tu_instance *instance,
 144             VkResult error,
 145             const char *file,
 146             int line,
 147             const char *format,
 148             ...);
 149
 150 #define vk_error(instance, error)                                            \
 151    __vk_errorf(instance, error, __FILE__, __LINE__, NULL);
 152 #define vk_errorf(instance, error, format, ...)                              \
 153    __vk_errorf(instance, error, __FILE__, __LINE__, format, ##__VA_ARGS__);
 154
 155 void
 156 __tu_finishme(const char *file, int line, const char *format, ...)
 157    tu_printflike(3, 4);
 158 void
 159 tu_loge(const char *format, ...) tu_printflike(1, 2);
 160 void
 161 tu_logi(const char *format, ...) tu_printflike(1, 2);
 162
 163 /**
 164  * Print a FINISHME message, including its source location.
 165  */
 166 #define tu_finishme(format, ...)                                             \
 167    do {                                                                      \
 168       static bool reported = false;                                          \
 169       if (!reported) {                                                       \
 170          __tu_finishme(__FILE__, __LINE__, format, ##__VA_ARGS__);           \
 171          reported = true;                                                    \
 172       }                                                                      \
 173    } while (0)
 174
 175 #define tu_stub()                                                            \
 176    do {                                                                      \
 177       tu_finishme("stub %s", __func__);                                      \
 178    } while (0)
 179
 180 void *
 181 tu_lookup_entrypoint_unchecked(const char *name);
 182 void *
 183 tu_lookup_entrypoint_checked(
 184    const char *name,
 185    uint32_t core_version,
 186    const struct tu_instance_extension_table *instance,
 187    const struct tu_device_extension_table *device);
 188
 189 struct tu_physical_device
 190 {
 191    VK_LOADER_DATA _loader_data;
 192
 193    struct tu_instance *instance;
 194
 195    char path[20];
 196    char name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE];
 197    uint8_t driver_uuid[VK_UUID_SIZE];
 198    uint8_t device_uuid[VK_UUID_SIZE];
 199    uint8_t cache_uuid[VK_UUID_SIZE];
 200
 201    struct wsi_device wsi_device;
 202
 203    int local_fd;
 204    int master_fd;
 205
 206    unsigned gpu_id;
 207    uint32_t gmem_size;
 208    uint64_t gmem_base;
 209    uint32_t ccu_offset_gmem;
 210    uint32_t ccu_offset_bypass;
 211    /* alignment for size of tiles */
 212    uint32_t tile_align_w;
 213 #define TILE_ALIGN_H 16
 214    /* gmem store/load granularity */
 215 #define GMEM_ALIGN_W 16
 216 #define GMEM_ALIGN_H 4
 217
 218    struct {
 219       uint32_t PC_UNKNOWN_9805;
 220       uint32_t SP_UNKNOWN_A0F8;
 221    } magic;
 222
 223    /* This is the drivers on-disk cache used as a fallback as opposed to
 224     * the pipeline cache defined by apps.
 225     */
 226    struct disk_cache *disk_cache;
 227
 228    struct tu_device_extension_table supported_extensions;
 229 };
 230
 231 enum tu_debug_flags
 232 {
 233    TU_DEBUG_STARTUP = 1 << 0,
 234    TU_DEBUG_NIR = 1 << 1,
 235    TU_DEBUG_IR3 = 1 << 2,
 236    TU_DEBUG_NOBIN = 1 << 3,
 237    TU_DEBUG_SYSMEM = 1 << 4,
 238    TU_DEBUG_FORCEBIN = 1 << 5,
 239    TU_DEBUG_NOUBWC = 1 << 6,
 240 };
 241
 242 struct tu_instance
 243 {
 244    VK_LOADER_DATA _loader_data;
 245
 246    VkAllocationCallbacks alloc;
 247
 248    uint32_t api_version;
 249    int physical_device_count;
 250    struct tu_physical_device physical_devices[TU_MAX_DRM_DEVICES];
 251
 252    enum tu_debug_flags debug_flags;
 253
 254    struct vk_debug_report_instance debug_report_callbacks;
 255
 256    struct tu_instance_extension_table enabled_extensions;
 257 };
 258
 259 VkResult
 260 tu_wsi_init(struct tu_physical_device *physical_device);
 261 void
 262 tu_wsi_finish(struct tu_physical_device *physical_device);
 263
 264 bool
 265 tu_instance_extension_supported(const char *name);
 266 uint32_t
 267 tu_physical_device_api_version(struct tu_physical_device *dev);
 268 bool
 269 tu_physical_device_extension_supported(struct tu_physical_device *dev,
 270                                        const char *name);
 271
 272 struct cache_entry;
 273
 274 struct tu_pipeline_cache
 275 {
 276    struct tu_device *device;
 277    pthread_mutex_t mutex;
 278
 279    uint32_t total_size;
 280    uint32_t table_size;
 281    uint32_t kernel_count;
 282    struct cache_entry **hash_table;
 283    bool modified;
 284
 285    VkAllocationCallbacks alloc;
 286 };
 287
 288 struct tu_pipeline_key
 289 {
 290 };
 291
 292
 293 /* queue types */
 294 #define TU_QUEUE_GENERAL 0
 295
 296 #define TU_MAX_QUEUE_FAMILIES 1
 297
 298 struct tu_fence
 299 {
 300    struct wsi_fence *fence_wsi;
 301    bool signaled;
 302    int fd;
 303 };
 304
 305 void
 306 tu_fence_init(struct tu_fence *fence, bool signaled);
 307 void
 308 tu_fence_finish(struct tu_fence *fence);
 309 void
 310 tu_fence_update_fd(struct tu_fence *fence, int fd);
 311 void
 312 tu_fence_copy(struct tu_fence *fence, const struct tu_fence *src);
 313 void
 314 tu_fence_signal(struct tu_fence *fence);
 315 void
 316 tu_fence_wait_idle(struct tu_fence *fence);
 317
 318 struct tu_queue
 319 {
 320    VK_LOADER_DATA _loader_data;
 321    struct tu_device *device;
 322    uint32_t queue_family_index;
 323    int queue_idx;
 324    VkDeviceQueueCreateFlags flags;
 325
 326    uint32_t msm_queue_id;
 327    struct tu_fence submit_fence;
 328 };
 329
 330 struct tu_bo
 331 {
 332    uint32_t gem_handle;
 333    uint64_t size;
 334    uint64_t iova;
 335    void *map;
 336 };
 337
 338 struct tu_device
 339 {
 340    VK_LOADER_DATA _loader_data;
 341
 342    VkAllocationCallbacks alloc;
 343
 344    struct tu_instance *instance;
 345
 346    struct tu_queue *queues[TU_MAX_QUEUE_FAMILIES];
 347    int queue_count[TU_MAX_QUEUE_FAMILIES];
 348
 349    struct tu_physical_device *physical_device;
 350
 351    struct ir3_compiler *compiler;
 352
 353    /* Backup in-memory cache to be used if the app doesn't provide one */
 354    struct tu_pipeline_cache *mem_cache;
 355
 356    struct tu_bo vsc_draw_strm;
 357    struct tu_bo vsc_prim_strm;
 358    uint32_t vsc_draw_strm_pitch;
 359    uint32_t vsc_prim_strm_pitch;
 360
 361 #define MIN_SCRATCH_BO_SIZE_LOG2 12 /* A page */
 362
 363    /* Currently the kernel driver uses a 32-bit GPU address space, but it
 364     * should be impossible to go beyond 48 bits.
 365     */
 366    struct {
 367       struct tu_bo bo;
 368       mtx_t construct_mtx;
 369       bool initialized;
 370    } scratch_bos[48 - MIN_SCRATCH_BO_SIZE_LOG2];
 371
 372    struct tu_bo border_color;
 373
 374    struct tu_device_extension_table enabled_extensions;
 375 };
 376
 377 VkResult
 378 tu_bo_init_new(struct tu_device *dev, struct tu_bo *bo, uint64_t size);
 379 VkResult
 380 tu_bo_init_dmabuf(struct tu_device *dev,
 381                   struct tu_bo *bo,
 382                   uint64_t size,
 383                   int fd);
 384 int
 385 tu_bo_export_dmabuf(struct tu_device *dev, struct tu_bo *bo);
 386 void
 387 tu_bo_finish(struct tu_device *dev, struct tu_bo *bo);
 388 VkResult
 389 tu_bo_map(struct tu_device *dev, struct tu_bo *bo);
 390
 391 /* Get a scratch bo for use inside a command buffer. This will always return
 392  * the same bo given the same size or similar sizes, so only one scratch bo
 393  * can be used at the same time. It's meant for short-lived things where we
 394  * need to write to some piece of memory, read from it, and then immediately
 395  * discard it.
 396  */
 397 VkResult
 398 tu_get_scratch_bo(struct tu_device *dev, uint64_t size, struct tu_bo **bo);
 399
 400 struct tu_cs_entry
 401 {
 402    /* No ownership */
 403    const struct tu_bo *bo;
 404
 405    uint32_t size;
 406    uint32_t offset;
 407 };
 408
 409 struct ts_cs_memory {
 410    uint32_t *map;
 411    uint64_t iova;
 412 };
 413
 414 struct tu_draw_state {
 415    uint64_t iova : 48;
 416    uint32_t size : 16;
 417 };
 418
 419 enum tu_dynamic_state
 420 {
 421    /* re-use VK_DYNAMIC_STATE_ enums for non-extended dynamic states */
 422    TU_DYNAMIC_STATE_SAMPLE_LOCATIONS = VK_DYNAMIC_STATE_STENCIL_REFERENCE + 1,
 423    TU_DYNAMIC_STATE_COUNT,
 424 };
 425
 426 enum tu_draw_state_group_id
 427 {
 428    TU_DRAW_STATE_PROGRAM,
 429    TU_DRAW_STATE_PROGRAM_BINNING,
 430    TU_DRAW_STATE_TESS,
 431    TU_DRAW_STATE_VB,
 432    TU_DRAW_STATE_VI,
 433    TU_DRAW_STATE_VI_BINNING,
 434    TU_DRAW_STATE_RAST,
 435    TU_DRAW_STATE_DS,
 436    TU_DRAW_STATE_BLEND,
 437    TU_DRAW_STATE_VS_CONST,
 438    TU_DRAW_STATE_HS_CONST,
 439    TU_DRAW_STATE_DS_CONST,
 440    TU_DRAW_STATE_GS_CONST,
 441    TU_DRAW_STATE_FS_CONST,
 442    TU_DRAW_STATE_DESC_SETS,
 443    TU_DRAW_STATE_DESC_SETS_LOAD,
 444    TU_DRAW_STATE_VS_PARAMS,
 445    TU_DRAW_STATE_INPUT_ATTACHMENTS_GMEM,
 446    TU_DRAW_STATE_INPUT_ATTACHMENTS_SYSMEM,
 447
 448    /* dynamic state related draw states */
 449    TU_DRAW_STATE_DYNAMIC,
 450    TU_DRAW_STATE_COUNT = TU_DRAW_STATE_DYNAMIC + TU_DYNAMIC_STATE_COUNT,
 451 };
 452
 453 enum tu_cs_mode
 454 {
 455
 456    /*
 457     * A command stream in TU_CS_MODE_GROW mode grows automatically whenever it
 458     * is full.  tu_cs_begin must be called before command packet emission and
 459     * tu_cs_end must be called after.
 460     *
 461     * This mode may create multiple entries internally.  The entries must be
 462     * submitted together.
 463     */
 464    TU_CS_MODE_GROW,
 465
 466    /*
 467     * A command stream in TU_CS_MODE_EXTERNAL mode wraps an external,
 468     * fixed-size buffer.  tu_cs_begin and tu_cs_end are optional and have no
 469     * effect on it.
 470     *
 471     * This mode does not create any entry or any BO.
 472     */
 473    TU_CS_MODE_EXTERNAL,
 474
 475    /*
 476     * A command stream in TU_CS_MODE_SUB_STREAM mode does not support direct
 477     * command packet emission.  tu_cs_begin_sub_stream must be called to get a
 478     * sub-stream to emit comamnd packets to.  When done with the sub-stream,
 479     * tu_cs_end_sub_stream must be called.
 480     *
 481     * This mode does not create any entry internally.
 482     */
 483    TU_CS_MODE_SUB_STREAM,
 484 };
 485
 486 struct tu_cs
 487 {
 488    uint32_t *start;
 489    uint32_t *cur;
 490    uint32_t *reserved_end;
 491    uint32_t *end;
 492
 493    struct tu_device *device;
 494    enum tu_cs_mode mode;
 495    uint32_t next_bo_size;
 496
 497    struct tu_cs_entry *entries;
 498    uint32_t entry_count;
 499    uint32_t entry_capacity;
 500
 501    struct tu_bo **bos;
 502    uint32_t bo_count;
 503    uint32_t bo_capacity;
 504
 505    /* state for cond_exec_start/cond_exec_end */
 506    uint32_t cond_flags;
 507    uint32_t *cond_dwords;
 508 };
 509
 510 struct tu_device_memory
 511 {
 512    struct tu_bo bo;
 513    VkDeviceSize size;
 514
 515    /* for dedicated allocations */
 516    struct tu_image *image;
 517    struct tu_buffer *buffer;
 518
 519    uint32_t type_index;
 520    void *map;
 521    void *user_ptr;
 522 };
 523
 524 struct tu_descriptor_range
 525 {
 526    uint64_t va;
 527    uint32_t size;
 528 };
 529
 530 struct tu_descriptor_set
 531 {
 532    const struct tu_descriptor_set_layout *layout;
 533    struct tu_descriptor_pool *pool;
 534    uint32_t size;
 535
 536    uint64_t va;
 537    uint32_t *mapped_ptr;
 538
 539    uint32_t *dynamic_descriptors;
 540
 541    struct tu_bo *buffers[0];
 542 };
 543
 544 struct tu_push_descriptor_set
 545 {
 546    struct tu_descriptor_set set;
 547    uint32_t capacity;
 548 };
 549
 550 struct tu_descriptor_pool_entry
 551 {
 552    uint32_t offset;
 553    uint32_t size;
 554    struct tu_descriptor_set *set;
 555 };
 556
 557 struct tu_descriptor_pool
 558 {
 559    struct tu_bo bo;
 560    uint64_t current_offset;
 561    uint64_t size;
 562
 563    uint8_t *host_memory_base;
 564    uint8_t *host_memory_ptr;
 565    uint8_t *host_memory_end;
 566
 567    uint32_t entry_count;
 568    uint32_t max_entry_count;
 569    struct tu_descriptor_pool_entry entries[0];
 570 };
 571
 572 struct tu_descriptor_update_template_entry
 573 {
 574    VkDescriptorType descriptor_type;
 575
 576    /* The number of descriptors to update */
 577    uint32_t descriptor_count;
 578
 579    /* Into mapped_ptr or dynamic_descriptors, in units of the respective array
 580     */
 581    uint32_t dst_offset;
 582
 583    /* In dwords. Not valid/used for dynamic descriptors */
 584    uint32_t dst_stride;
 585
 586    uint32_t buffer_offset;
 587
 588    /* Only valid for combined image samplers and samplers */
 589    uint16_t has_sampler;
 590
 591    /* In bytes */
 592    size_t src_offset;
 593    size_t src_stride;
 594
 595    /* For push descriptors */
 596    const uint32_t *immutable_samplers;
 597 };
 598
 599 struct tu_descriptor_update_template
 600 {
 601    uint32_t entry_count;
 602    struct tu_descriptor_update_template_entry entry[0];
 603 };
 604
 605 struct tu_buffer
 606 {
 607    VkDeviceSize size;
 608
 609    VkBufferUsageFlags usage;
 610    VkBufferCreateFlags flags;
 611
 612    struct tu_bo *bo;
 613    VkDeviceSize bo_offset;
 614 };
 615
 616 static inline uint64_t
 617 tu_buffer_iova(struct tu_buffer *buffer)
 618 {
 619    return buffer->bo->iova + buffer->bo_offset;
 620 }
 621
 622 struct tu_vertex_binding
 623 {
 624    struct tu_buffer *buffer;
 625    VkDeviceSize offset;
 626 };
 627
 628 const char *
 629 tu_get_debug_option_name(int id);
 630
 631 const char *
 632 tu_get_perftest_option_name(int id);
 633
 634 struct tu_descriptor_state
 635 {
 636    struct tu_descriptor_set *sets[MAX_SETS];
 637    uint32_t dynamic_descriptors[MAX_DYNAMIC_BUFFERS * A6XX_TEX_CONST_DWORDS];
 638 };
 639
 640 struct tu_tile
 641 {
 642    uint8_t pipe;
 643    uint8_t slot;
 644    VkOffset2D begin;
 645    VkOffset2D end;
 646 };
 647
 648 struct tu_tiling_config
 649 {
 650    VkRect2D render_area;
 651
 652    /* position and size of the first tile */
 653    VkRect2D tile0;
 654    /* number of tiles */
 655    VkExtent2D tile_count;
 656
 657    /* size of the first VSC pipe */
 658    VkExtent2D pipe0;
 659    /* number of VSC pipes */
 660    VkExtent2D pipe_count;
 661
 662    /* pipe register values */
 663    uint32_t pipe_config[MAX_VSC_PIPES];
 664    uint32_t pipe_sizes[MAX_VSC_PIPES];
 665
 666    /* Whether sysmem rendering must be used */
 667    bool force_sysmem;
 668 };
 669
 670 enum tu_cmd_dirty_bits
 671 {
 672    TU_CMD_DIRTY_COMPUTE_PIPELINE = 1 << 1,
 673    TU_CMD_DIRTY_VERTEX_BUFFERS = 1 << 2,
 674    TU_CMD_DIRTY_DESCRIPTOR_SETS = 1 << 3,
 675    TU_CMD_DIRTY_COMPUTE_DESCRIPTOR_SETS = 1 << 4,
 676    TU_CMD_DIRTY_SHADER_CONSTS = 1 << 5,
 677    TU_CMD_DIRTY_STREAMOUT_BUFFERS = 1 << 6,
 678    /* all draw states were disabled and need to be re-enabled: */
 679    TU_CMD_DIRTY_DRAW_STATE = 1 << 7,
 680 };
 681
 682 struct tu_streamout_state {
 683    uint16_t stride[IR3_MAX_SO_BUFFERS];
 684    uint32_t ncomp[IR3_MAX_SO_BUFFERS];
 685    uint32_t prog[IR3_MAX_SO_OUTPUTS * 2];
 686    uint32_t prog_count;
 687    uint32_t vpc_so_buf_cntl;
 688 };
 689
 690 /* There are only three cache domains we have to care about: the CCU, or
 691  * color cache unit, which is used for color and depth/stencil attachments
 692  * and copy/blit destinations, and is split conceptually into color and depth,
 693  * and the universal cache or UCHE which is used for pretty much everything
 694  * else, except for the CP (uncached) and host. We need to flush whenever data
 695  * crosses these boundaries.
 696  */
 697
 698 enum tu_cmd_access_mask {
 699    TU_ACCESS_UCHE_READ = 1 << 0,
 700    TU_ACCESS_UCHE_WRITE = 1 << 1,
 701    TU_ACCESS_CCU_COLOR_READ = 1 << 2,
 702    TU_ACCESS_CCU_COLOR_WRITE = 1 << 3,
 703    TU_ACCESS_CCU_DEPTH_READ = 1 << 4,
 704    TU_ACCESS_CCU_DEPTH_WRITE = 1 << 5,
 705
 706    /* Experiments have shown that while it's safe to avoid flushing the CCU
 707     * after each blit/renderpass, it's not safe to assume that subsequent
 708     * lookups with a different attachment state will hit unflushed cache
 709     * entries. That is, the CCU needs to be flushed and possibly invalidated
 710     * when accessing memory with a different attachment state. Writing to an
 711     * attachment under the following conditions after clearing using the
 712     * normal 2d engine path is known to have issues:
 713     *
 714     * - It isn't the 0'th layer.
 715     * - There are more than one attachment, and this isn't the 0'th attachment
 716     *   (this seems to also depend on the cpp of the attachments).
 717     *
 718     * Our best guess is that the layer/MRT state is used when computing
 719     * the location of a cache entry in CCU, to avoid conflicts. We assume that
 720     * any access in a renderpass after or before an access by a transfer needs
 721     * a flush/invalidate, and use the _INCOHERENT variants to represent access
 722     * by a transfer.
 723     */
 724    TU_ACCESS_CCU_COLOR_INCOHERENT_READ = 1 << 6,
 725    TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE = 1 << 7,
 726    TU_ACCESS_CCU_DEPTH_INCOHERENT_READ = 1 << 8,
 727    TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE = 1 << 9,
 728
 729    TU_ACCESS_SYSMEM_READ = 1 << 10,
 730    TU_ACCESS_SYSMEM_WRITE = 1 << 11,
 731
 732    /* Set if a WFI is required due to data being read by the CP or the 2D
 733     * engine.
 734     */
 735    TU_ACCESS_WFI_READ = 1 << 12,
 736
 737    TU_ACCESS_READ =
 738       TU_ACCESS_UCHE_READ |
 739       TU_ACCESS_CCU_COLOR_READ |
 740       TU_ACCESS_CCU_DEPTH_READ |
 741       TU_ACCESS_CCU_COLOR_INCOHERENT_READ |
 742       TU_ACCESS_CCU_DEPTH_INCOHERENT_READ |
 743       TU_ACCESS_SYSMEM_READ,
 744
 745    TU_ACCESS_WRITE =
 746       TU_ACCESS_UCHE_WRITE |
 747       TU_ACCESS_CCU_COLOR_WRITE |
 748       TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE |
 749       TU_ACCESS_CCU_DEPTH_WRITE |
 750       TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE |
 751       TU_ACCESS_SYSMEM_WRITE,
 752
 753    TU_ACCESS_ALL =
 754       TU_ACCESS_READ |
 755       TU_ACCESS_WRITE,
 756 };
 757
 758 enum tu_cmd_flush_bits {
 759    TU_CMD_FLAG_CCU_FLUSH_DEPTH = 1 << 0,
 760    TU_CMD_FLAG_CCU_FLUSH_COLOR = 1 << 1,
 761    TU_CMD_FLAG_CCU_INVALIDATE_DEPTH = 1 << 2,
 762    TU_CMD_FLAG_CCU_INVALIDATE_COLOR = 1 << 3,
 763    TU_CMD_FLAG_CACHE_FLUSH = 1 << 4,
 764    TU_CMD_FLAG_CACHE_INVALIDATE = 1 << 5,
 765
 766    TU_CMD_FLAG_ALL_FLUSH =
 767       TU_CMD_FLAG_CCU_FLUSH_DEPTH |
 768       TU_CMD_FLAG_CCU_FLUSH_COLOR |
 769       TU_CMD_FLAG_CACHE_FLUSH,
 770
 771    TU_CMD_FLAG_ALL_INVALIDATE =
 772       TU_CMD_FLAG_CCU_INVALIDATE_DEPTH |
 773       TU_CMD_FLAG_CCU_INVALIDATE_COLOR |
 774       TU_CMD_FLAG_CACHE_INVALIDATE,
 775
 776    TU_CMD_FLAG_WFI = 1 << 6,
 777 };
 778
 779 /* Changing the CCU from sysmem mode to gmem mode or vice-versa is pretty
 780  * heavy, involving a CCU cache flush/invalidate and a WFI in order to change
 781  * which part of the gmem is used by the CCU. Here we keep track of what the
 782  * state of the CCU.
 783  */
 784 enum tu_cmd_ccu_state {
 785    TU_CMD_CCU_SYSMEM,
 786    TU_CMD_CCU_GMEM,
 787    TU_CMD_CCU_UNKNOWN,
 788 };
 789
 790 struct tu_cache_state {
 791    /* Caches which must be made available (flushed) eventually if there are
 792     * any users outside that cache domain, and caches which must be
 793     * invalidated eventually if there are any reads.
 794     */
 795    enum tu_cmd_flush_bits pending_flush_bits;
 796    /* Pending flushes */
 797    enum tu_cmd_flush_bits flush_bits;
 798 };
 799
 800 struct tu_cmd_state
 801 {
 802    uint32_t dirty;
 803
 804    struct tu_pipeline *pipeline;
 805    struct tu_pipeline *compute_pipeline;
 806
 807    /* Vertex buffers */
 808    struct
 809    {
 810       struct tu_buffer *buffers[MAX_VBS];
 811       VkDeviceSize offsets[MAX_VBS];
 812    } vb;
 813
 814    /* for dynamic states that can't be emitted directly */
 815    uint32_t dynamic_stencil_mask;
 816    uint32_t dynamic_stencil_wrmask;
 817    uint32_t dynamic_stencil_ref;
 818    uint32_t dynamic_gras_su_cntl;
 819
 820    /* saved states to re-emit in TU_CMD_DIRTY_DRAW_STATE case */
 821    struct tu_draw_state dynamic_state[TU_DYNAMIC_STATE_COUNT];
 822    struct tu_cs_entry vertex_buffers_ib;
 823    struct tu_cs_entry shader_const_ib[MESA_SHADER_STAGES];
 824    struct tu_cs_entry desc_sets_ib, desc_sets_load_ib;
 825    struct tu_cs_entry ia_gmem_ib, ia_sysmem_ib;
 826
 827    /* Stream output buffers */
 828    struct
 829    {
 830       struct tu_buffer *buffers[IR3_MAX_SO_BUFFERS];
 831       VkDeviceSize offsets[IR3_MAX_SO_BUFFERS];
 832       VkDeviceSize sizes[IR3_MAX_SO_BUFFERS];
 833    } streamout_buf;
 834
 835    uint8_t streamout_reset;
 836    uint8_t streamout_enabled;
 837
 838    /* Index buffer */
 839    struct tu_buffer *index_buffer;
 840    uint64_t index_offset;
 841    uint32_t index_type;
 842    uint32_t max_index_count;
 843    uint64_t index_va;
 844
 845    /* Renderpasses are tricky, because we may need to flush differently if
 846     * using sysmem vs. gmem and therefore we have to delay any flushing that
 847     * happens before a renderpass. So we have to have two copies of the flush
 848     * state, one for intra-renderpass flushes (i.e. renderpass dependencies)
 849     * and one for outside a renderpass.
 850     */
 851    struct tu_cache_state cache;
 852    struct tu_cache_state renderpass_cache;
 853
 854    enum tu_cmd_ccu_state ccu_state;
 855
 856    const struct tu_render_pass *pass;
 857    const struct tu_subpass *subpass;
 858    const struct tu_framebuffer *framebuffer;
 859
 860    struct tu_tiling_config tiling_config;
 861
 862    struct tu_cs_entry tile_store_ib;
 863 };
 864
 865 struct tu_cmd_pool
 866 {
 867    VkAllocationCallbacks alloc;
 868    struct list_head cmd_buffers;
 869    struct list_head free_cmd_buffers;
 870    uint32_t queue_family_index;
 871 };
 872
 873 struct tu_cmd_buffer_upload
 874 {
 875    uint8_t *map;
 876    unsigned offset;
 877    uint64_t size;
 878    struct list_head list;
 879 };
 880
 881 enum tu_cmd_buffer_status
 882 {
 883    TU_CMD_BUFFER_STATUS_INVALID,
 884    TU_CMD_BUFFER_STATUS_INITIAL,
 885    TU_CMD_BUFFER_STATUS_RECORDING,
 886    TU_CMD_BUFFER_STATUS_EXECUTABLE,
 887    TU_CMD_BUFFER_STATUS_PENDING,
 888 };
 889
 890 struct tu_bo_list
 891 {
 892    uint32_t count;
 893    uint32_t capacity;
 894    struct drm_msm_gem_submit_bo *bo_infos;
 895 };
 896
 897 #define TU_BO_LIST_FAILED (~0)
 898
 899 void
 900 tu_bo_list_init(struct tu_bo_list *list);
 901 void
 902 tu_bo_list_destroy(struct tu_bo_list *list);
 903 void
 904 tu_bo_list_reset(struct tu_bo_list *list);
 905 uint32_t
 906 tu_bo_list_add(struct tu_bo_list *list,
 907                const struct tu_bo *bo,
 908                uint32_t flags);
 909 VkResult
 910 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other);
 911
 912 /* This struct defines the layout of the scratch_bo */
 913 struct tu6_control
 914 {
 915    uint32_t seqno_dummy;          /* dummy seqno for CP_EVENT_WRITE */
 916    uint32_t _pad0;
 917    volatile uint32_t vsc_overflow;
 918    uint32_t _pad1;
 919    /* flag set from cmdstream when VSC overflow detected: */
 920    uint32_t vsc_scratch;
 921    uint32_t _pad2;
 922    uint32_t _pad3;
 923    uint32_t _pad4;
 924
 925    /* scratch space for VPC_SO[i].FLUSH_BASE_LO/HI, start on 32 byte boundary. */
 926    struct {
 927       uint32_t offset;
 928       uint32_t pad[7];
 929    } flush_base[4];
 930 };
 931
 932 #define ctrl_offset(member) offsetof(struct tu6_control, member)
 933
 934 struct tu_cmd_buffer
 935 {
 936    VK_LOADER_DATA _loader_data;
 937
 938    struct tu_device *device;
 939
 940    struct tu_cmd_pool *pool;
 941    struct list_head pool_link;
 942
 943    VkCommandBufferUsageFlags usage_flags;
 944    VkCommandBufferLevel level;
 945    enum tu_cmd_buffer_status status;
 946
 947    struct tu_cmd_state state;
 948    struct tu_vertex_binding vertex_bindings[MAX_VBS];
 949    uint32_t vertex_bindings_set;
 950    uint32_t queue_family_index;
 951
 952    uint32_t push_constants[MAX_PUSH_CONSTANTS_SIZE / 4];
 953    VkShaderStageFlags push_constant_stages;
 954    struct tu_descriptor_set meta_push_descriptors;
 955
 956    struct tu_descriptor_state descriptors[MAX_BIND_POINTS];
 957
 958    struct tu_cmd_buffer_upload upload;
 959
 960    VkResult record_result;
 961
 962    struct tu_bo_list bo_list;
 963    struct tu_cs cs;
 964    struct tu_cs draw_cs;
 965    struct tu_cs draw_epilogue_cs;
 966    struct tu_cs sub_cs;
 967
 968    struct tu_bo scratch_bo;
 969
 970    bool has_tess;
 971
 972    struct tu_bo vsc_draw_strm;
 973    struct tu_bo vsc_prim_strm;
 974    uint32_t vsc_draw_strm_pitch;
 975    uint32_t vsc_prim_strm_pitch;
 976    bool use_vsc_data;
 977 };
 978
 979 /* Temporary struct for tracking a register state to be written, used by
 980  * a6xx-pack.h and tu_cs_emit_regs()
 981  */
 982 struct tu_reg_value {
 983    uint32_t reg;
 984    uint64_t value;
 985    bool is_address;
 986    struct tu_bo *bo;
 987    bool bo_write;
 988    uint32_t bo_offset;
 989    uint32_t bo_shift;
 990 };
 991
 992
 993 void tu_emit_cache_flush_renderpass(struct tu_cmd_buffer *cmd_buffer,
 994                                     struct tu_cs *cs);
 995
 996 void tu_emit_cache_flush_ccu(struct tu_cmd_buffer *cmd_buffer,
 997                              struct tu_cs *cs,
 998                              enum tu_cmd_ccu_state ccu_state);
 999
1000 void
1001 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
1002                      struct tu_cs *cs,
1003                      enum vgt_event_type event);
1004
1005 static inline struct tu_descriptor_state *
1006 tu_get_descriptors_state(struct tu_cmd_buffer *cmd_buffer,
1007                          VkPipelineBindPoint bind_point)
1008 {
1009    return &cmd_buffer->descriptors[bind_point];
1010 }
1011
1012 struct tu_event
1013 {
1014    struct tu_bo bo;
1015 };
1016
1017 struct tu_shader_module
1018 {
1019    unsigned char sha1[20];
1020
1021    uint32_t code_size;
1022    const uint32_t *code[0];
1023 };
1024
1025 struct tu_push_constant_range
1026 {
1027    uint32_t lo;
1028    uint32_t count;
1029 };
1030
1031 struct tu_shader
1032 {
1033    struct ir3_shader *ir3_shader;
1034
1035    struct tu_push_constant_range push_consts;
1036    uint8_t active_desc_sets;
1037 };
1038
1039 struct tu_shader *
1040 tu_shader_create(struct tu_device *dev,
1041                  gl_shader_stage stage,
1042                  const VkPipelineShaderStageCreateInfo *stage_info,
1043                  struct tu_pipeline_layout *layout,
1044                  const VkAllocationCallbacks *alloc);
1045
1046 void
1047 tu_shader_destroy(struct tu_device *dev,
1048                   struct tu_shader *shader,
1049                   const VkAllocationCallbacks *alloc);
1050
1051 struct tu_program_descriptor_linkage
1052 {
1053    struct ir3_const_state const_state;
1054
1055    uint32_t constlen;
1056
1057    struct tu_push_constant_range push_consts;
1058 };
1059
1060 struct tu_pipeline
1061 {
1062    struct tu_cs cs;
1063
1064    struct tu_pipeline_layout *layout;
1065
1066    bool need_indirect_descriptor_sets;
1067    VkShaderStageFlags active_stages;
1068    uint32_t active_desc_sets;
1069
1070    struct tu_streamout_state streamout;
1071
1072    /* mask of enabled dynamic states
1073     * if BIT(i) is set, pipeline->dynamic_state[i] is *NOT* used
1074     */
1075    uint32_t dynamic_state_mask;
1076    struct tu_draw_state dynamic_state[TU_DYNAMIC_STATE_COUNT];
1077
1078    /* gras_su_cntl without line width, used for dynamic line width state */
1079    uint32_t gras_su_cntl;
1080
1081    struct
1082    {
1083       struct tu_bo binary_bo;
1084       struct tu_cs_entry state_ib;
1085       struct tu_cs_entry binning_state_ib;
1086
1087       struct tu_program_descriptor_linkage link[MESA_SHADER_STAGES];
1088    } program;
1089
1090    struct
1091    {
1092       struct tu_cs_entry state_ib;
1093    } load_state;
1094
1095    struct
1096    {
1097       struct tu_cs_entry state_ib;
1098       struct tu_cs_entry binning_state_ib;
1099       uint32_t bindings_used;
1100    } vi;
1101
1102    struct
1103    {
1104       enum pc_di_primtype primtype;
1105       bool primitive_restart;
1106    } ia;
1107
1108    struct
1109    {
1110       uint32_t patch_type;
1111       uint32_t per_vertex_output_size;
1112       uint32_t per_patch_output_size;
1113       uint32_t hs_bo_regid;
1114       uint32_t ds_bo_regid;
1115       bool upper_left_domain_origin;
1116    } tess;
1117
1118    struct
1119    {
1120       struct tu_cs_entry state_ib;
1121    } rast;
1122
1123    struct
1124    {
1125       struct tu_cs_entry state_ib;
1126    } ds;
1127
1128    struct
1129    {
1130       struct tu_cs_entry state_ib;
1131    } blend;
1132
1133    struct
1134    {
1135       uint32_t local_size[3];
1136    } compute;
1137 };
1138
1139 void
1140 tu6_emit_viewport(struct tu_cs *cs, const VkViewport *viewport);
1141
1142 void
1143 tu6_emit_scissor(struct tu_cs *cs, const VkRect2D *scissor);
1144
1145 void
1146 tu6_emit_sample_locations(struct tu_cs *cs, const VkSampleLocationsInfoEXT *samp_loc);
1147
1148 void
1149 tu6_emit_depth_bias(struct tu_cs *cs,
1150                     float constant_factor,
1151                     float clamp,
1152                     float slope_factor);
1153
1154 void tu6_emit_msaa(struct tu_cs *cs, VkSampleCountFlagBits samples);
1155
1156 void tu6_emit_window_scissor(struct tu_cs *cs, uint32_t x1, uint32_t y1, uint32_t x2, uint32_t y2);
1157
1158 void tu6_emit_window_offset(struct tu_cs *cs, uint32_t x1, uint32_t y1);
1159
1160 void
1161 tu6_emit_xs_config(struct tu_cs *cs,
1162                    gl_shader_stage stage,
1163                    const struct ir3_shader_variant *xs,
1164                    uint64_t binary_iova);
1165
1166 void
1167 tu6_emit_vpc(struct tu_cs *cs,
1168              const struct ir3_shader_variant *vs,
1169              const struct ir3_shader_variant *hs,
1170              const struct ir3_shader_variant *ds,
1171              const struct ir3_shader_variant *gs,
1172              const struct ir3_shader_variant *fs,
1173              struct tu_streamout_state *tf);
1174
1175 void
1176 tu6_emit_fs_inputs(struct tu_cs *cs, const struct ir3_shader_variant *fs);
1177
1178 struct tu_image_view;
1179
1180 void
1181 tu_resolve_sysmem(struct tu_cmd_buffer *cmd,
1182                   struct tu_cs *cs,
1183                   struct tu_image_view *src,
1184                   struct tu_image_view *dst,
1185                   uint32_t layers,
1186                   const VkRect2D *rect);
1187
1188 void
1189 tu_clear_sysmem_attachment(struct tu_cmd_buffer *cmd,
1190                            struct tu_cs *cs,
1191                            uint32_t a,
1192                            const VkRenderPassBeginInfo *info);
1193
1194 void
1195 tu_clear_gmem_attachment(struct tu_cmd_buffer *cmd,
1196                          struct tu_cs *cs,
1197                          uint32_t a,
1198                          const VkRenderPassBeginInfo *info);
1199
1200 void
1201 tu_load_gmem_attachment(struct tu_cmd_buffer *cmd,
1202                         struct tu_cs *cs,
1203                         uint32_t a,
1204                         bool force_load);
1205
1206 /* expose this function to be able to emit load without checking LOAD_OP */
1207 void
1208 tu_emit_load_gmem_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t a);
1209
1210 /* note: gmem store can also resolve */
1211 void
1212 tu_store_gmem_attachment(struct tu_cmd_buffer *cmd,
1213                          struct tu_cs *cs,
1214                          uint32_t a,
1215                          uint32_t gmem_a);
1216
1217 enum tu_supported_formats {
1218    FMT_VERTEX = 1,
1219    FMT_TEXTURE = 2,
1220    FMT_COLOR = 4,
1221 };
1222
1223 struct tu_native_format
1224 {
1225    enum a6xx_format fmt : 8;
1226    enum a3xx_color_swap swap : 8;
1227    enum a6xx_tile_mode tile_mode : 8;
1228    enum tu_supported_formats supported : 8;
1229 };
1230
1231 struct tu_native_format tu6_format_vtx(VkFormat format);
1232 struct tu_native_format tu6_format_color(VkFormat format, enum a6xx_tile_mode tile_mode);
1233 struct tu_native_format tu6_format_texture(VkFormat format, enum a6xx_tile_mode tile_mode);
1234
1235 static inline enum a6xx_format
1236 tu6_base_format(VkFormat format)
1237 {
1238    /* note: tu6_format_color doesn't care about tiling for .fmt field */
1239    return tu6_format_color(format, TILE6_LINEAR).fmt;
1240 }
1241
1242 struct tu_image
1243 {
1244    VkImageType type;
1245    /* The original VkFormat provided by the client.  This may not match any
1246     * of the actual surface formats.
1247     */
1248    VkFormat vk_format;
1249    VkImageAspectFlags aspects;
1250    VkImageUsageFlags usage;  /**< Superset of VkImageCreateInfo::usage. */
1251    VkImageTiling tiling;     /** VkImageCreateInfo::tiling */
1252    VkImageCreateFlags flags; /** VkImageCreateInfo::flags */
1253    VkExtent3D extent;
1254    uint32_t level_count;
1255    uint32_t layer_count;
1256    VkSampleCountFlagBits samples;
1257
1258    struct fdl_layout layout;
1259
1260    unsigned queue_family_mask;
1261    bool exclusive;
1262    bool shareable;
1263
1264    /* For VK_ANDROID_native_buffer, the WSI image owns the memory, */
1265    VkDeviceMemory owned_memory;
1266
1267    /* Set when bound */
1268    struct tu_bo *bo;
1269    VkDeviceSize bo_offset;
1270 };
1271
1272 static inline uint32_t
1273 tu_get_layerCount(const struct tu_image *image,
1274                   const VkImageSubresourceRange *range)
1275 {
1276    return range->layerCount == VK_REMAINING_ARRAY_LAYERS
1277              ? image->layer_count - range->baseArrayLayer
1278              : range->layerCount;
1279 }
1280
1281 static inline uint32_t
1282 tu_get_levelCount(const struct tu_image *image,
1283                   const VkImageSubresourceRange *range)
1284 {
1285    return range->levelCount == VK_REMAINING_MIP_LEVELS
1286              ? image->level_count - range->baseMipLevel
1287              : range->levelCount;
1288 }
1289
1290 struct tu_image_view
1291 {
1292    struct tu_image *image; /**< VkImageViewCreateInfo::image */
1293
1294    uint64_t base_addr;
1295    uint64_t ubwc_addr;
1296    uint32_t layer_size;
1297    uint32_t ubwc_layer_size;
1298
1299    /* used to determine if fast gmem store path can be used */
1300    VkExtent2D extent;
1301    bool need_y2_align;
1302
1303    bool ubwc_enabled;
1304
1305    uint32_t descriptor[A6XX_TEX_CONST_DWORDS];
1306
1307    /* Descriptor for use as a storage image as opposed to a sampled image.
1308     * This has a few differences for cube maps (e.g. type).
1309     */
1310    uint32_t storage_descriptor[A6XX_TEX_CONST_DWORDS];
1311
1312    /* pre-filled register values */
1313    uint32_t PITCH;
1314    uint32_t FLAG_BUFFER_PITCH;
1315
1316    uint32_t RB_MRT_BUF_INFO;
1317    uint32_t SP_FS_MRT_REG;
1318
1319    uint32_t SP_PS_2D_SRC_INFO;
1320    uint32_t SP_PS_2D_SRC_SIZE;
1321
1322    uint32_t RB_2D_DST_INFO;
1323
1324    uint32_t RB_BLIT_DST_INFO;
1325 };
1326
1327 struct tu_sampler_ycbcr_conversion {
1328    VkFormat format;
1329    VkSamplerYcbcrModelConversion ycbcr_model;
1330    VkSamplerYcbcrRange ycbcr_range;
1331    VkComponentMapping components;
1332    VkChromaLocation chroma_offsets[2];
1333    VkFilter chroma_filter;
1334 };
1335
1336 struct tu_sampler {
1337    uint32_t descriptor[A6XX_TEX_SAMP_DWORDS];
1338    struct tu_sampler_ycbcr_conversion *ycbcr_sampler;
1339 };
1340
1341 void
1342 tu_cs_image_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1343
1344 void
1345 tu_cs_image_ref_2d(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer, bool src);
1346
1347 void
1348 tu_cs_image_flag_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1349
1350 VkResult
1351 tu_image_create(VkDevice _device,
1352                 const VkImageCreateInfo *pCreateInfo,
1353                 const VkAllocationCallbacks *alloc,
1354                 VkImage *pImage,
1355                 uint64_t modifier,
1356                 const VkSubresourceLayout *plane_layouts);
1357
1358 VkResult
1359 tu_image_from_gralloc(VkDevice device_h,
1360                       const VkImageCreateInfo *base_info,
1361                       const VkNativeBufferANDROID *gralloc_info,
1362                       const VkAllocationCallbacks *alloc,
1363                       VkImage *out_image_h);
1364
1365 void
1366 tu_image_view_init(struct tu_image_view *view,
1367                    const VkImageViewCreateInfo *pCreateInfo);
1368
1369 struct tu_buffer_view
1370 {
1371    uint32_t descriptor[A6XX_TEX_CONST_DWORDS];
1372
1373    struct tu_buffer *buffer;
1374 };
1375 void
1376 tu_buffer_view_init(struct tu_buffer_view *view,
1377                     struct tu_device *device,
1378                     const VkBufferViewCreateInfo *pCreateInfo);
1379
1380 struct tu_attachment_info
1381 {
1382    struct tu_image_view *attachment;
1383 };
1384
1385 struct tu_framebuffer
1386 {
1387    uint32_t width;
1388    uint32_t height;
1389    uint32_t layers;
1390
1391    uint32_t attachment_count;
1392    struct tu_attachment_info attachments[0];
1393 };
1394
1395 struct tu_subpass_barrier {
1396    VkPipelineStageFlags src_stage_mask;
1397    VkAccessFlags src_access_mask;
1398    VkAccessFlags dst_access_mask;
1399    bool incoherent_ccu_color, incoherent_ccu_depth;
1400 };
1401
1402 struct tu_subpass_attachment
1403 {
1404    uint32_t attachment;
1405    VkImageLayout layout;
1406 };
1407
1408 struct tu_subpass
1409 {
1410    uint32_t input_count;
1411    uint32_t color_count;
1412    struct tu_subpass_attachment *input_attachments;
1413    struct tu_subpass_attachment *color_attachments;
1414    struct tu_subpass_attachment *resolve_attachments;
1415    struct tu_subpass_attachment depth_stencil_attachment;
1416
1417    VkSampleCountFlagBits samples;
1418    bool has_external_src, has_external_dst;
1419
1420    uint32_t srgb_cntl;
1421
1422    struct tu_subpass_barrier start_barrier;
1423 };
1424
1425 struct tu_render_pass_attachment
1426 {
1427    VkFormat format;
1428    uint32_t samples;
1429    uint32_t cpp;
1430    VkImageAspectFlags clear_mask;
1431    bool load;
1432    bool store;
1433    VkImageLayout initial_layout, final_layout;
1434    int32_t gmem_offset;
1435 };
1436
1437 struct tu_render_pass
1438 {
1439    uint32_t attachment_count;
1440    uint32_t subpass_count;
1441    uint32_t gmem_pixels;
1442    uint32_t tile_align_w;
1443    struct tu_subpass_attachment *subpass_attachments;
1444    struct tu_render_pass_attachment *attachments;
1445    struct tu_subpass_barrier end_barrier;
1446    struct tu_subpass subpasses[0];
1447 };
1448
1449 struct tu_query_pool
1450 {
1451    VkQueryType type;
1452    uint32_t stride;
1453    uint64_t size;
1454    uint32_t pipeline_statistics;
1455    struct tu_bo bo;
1456 };
1457
1458 struct tu_semaphore
1459 {
1460    uint32_t syncobj;
1461    uint32_t temp_syncobj;
1462 };
1463
1464 void
1465 tu_set_descriptor_set(struct tu_cmd_buffer *cmd_buffer,
1466                       VkPipelineBindPoint bind_point,
1467                       struct tu_descriptor_set *set,
1468                       unsigned idx);
1469
1470 void
1471 tu_update_descriptor_sets(struct tu_device *device,
1472                           struct tu_cmd_buffer *cmd_buffer,
1473                           VkDescriptorSet overrideSet,
1474                           uint32_t descriptorWriteCount,
1475                           const VkWriteDescriptorSet *pDescriptorWrites,
1476                           uint32_t descriptorCopyCount,
1477                           const VkCopyDescriptorSet *pDescriptorCopies);
1478
1479 void
1480 tu_update_descriptor_set_with_template(
1481    struct tu_device *device,
1482    struct tu_cmd_buffer *cmd_buffer,
1483    struct tu_descriptor_set *set,
1484    VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1485    const void *pData);
1486
1487 int
1488 tu_drm_get_gpu_id(const struct tu_physical_device *dev, uint32_t *id);
1489
1490 int
1491 tu_drm_get_gmem_size(const struct tu_physical_device *dev, uint32_t *size);
1492
1493 int
1494 tu_drm_get_gmem_base(const struct tu_physical_device *dev, uint64_t *base);
1495
1496 int
1497 tu_drm_submitqueue_new(const struct tu_device *dev,
1498                        int priority,
1499                        uint32_t *queue_id);
1500
1501 void
1502 tu_drm_submitqueue_close(const struct tu_device *dev, uint32_t queue_id);
1503
1504 uint32_t
1505 tu_gem_new(const struct tu_device *dev, uint64_t size, uint32_t flags);
1506 uint32_t
1507 tu_gem_import_dmabuf(const struct tu_device *dev,
1508                      int prime_fd,
1509                      uint64_t size);
1510 int
1511 tu_gem_export_dmabuf(const struct tu_device *dev, uint32_t gem_handle);
1512 void
1513 tu_gem_close(const struct tu_device *dev, uint32_t gem_handle);
1514 uint64_t
1515 tu_gem_info_offset(const struct tu_device *dev, uint32_t gem_handle);
1516 uint64_t
1517 tu_gem_info_iova(const struct tu_device *dev, uint32_t gem_handle);
1518
1519 #define TU_DEFINE_HANDLE_CASTS(__tu_type, __VkType)                          \
1520                                                                              \
1521    static inline struct __tu_type *__tu_type##_from_handle(__VkType _handle) \
1522    {                                                                         \
1523       return (struct __tu_type *) _handle;                                   \
1524    }                                                                         \
1525                                                                              \
1526    static inline __VkType __tu_type##_to_handle(struct __tu_type *_obj)      \
1527    {                                                                         \
1528       return (__VkType) _obj;                                                \
1529    }
1530
1531 #define TU_DEFINE_NONDISP_HANDLE_CASTS(__tu_type, __VkType)                  \
1532                                                                              \
1533    static inline struct __tu_type *__tu_type##_from_handle(__VkType _handle) \
1534    {                                                                         \
1535       return (struct __tu_type *) (uintptr_t) _handle;                       \
1536    }                                                                         \
1537                                                                              \
1538    static inline __VkType __tu_type##_to_handle(struct __tu_type *_obj)      \
1539    {                                                                         \
1540       return (__VkType)(uintptr_t) _obj;                                     \
1541    }
1542
1543 #define TU_FROM_HANDLE(__tu_type, __name, __handle)                          \
1544    struct __tu_type *__name = __tu_type##_from_handle(__handle)
1545
1546 TU_DEFINE_HANDLE_CASTS(tu_cmd_buffer, VkCommandBuffer)
1547 TU_DEFINE_HANDLE_CASTS(tu_device, VkDevice)
1548 TU_DEFINE_HANDLE_CASTS(tu_instance, VkInstance)
1549 TU_DEFINE_HANDLE_CASTS(tu_physical_device, VkPhysicalDevice)
1550 TU_DEFINE_HANDLE_CASTS(tu_queue, VkQueue)
1551
1552 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_cmd_pool, VkCommandPool)
1553 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_buffer, VkBuffer)
1554 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_buffer_view, VkBufferView)
1555 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_pool, VkDescriptorPool)
1556 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_set, VkDescriptorSet)
1557 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_set_layout,
1558                                VkDescriptorSetLayout)
1559 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_update_template,
1560                                VkDescriptorUpdateTemplate)
1561 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_device_memory, VkDeviceMemory)
1562 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_fence, VkFence)
1563 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_event, VkEvent)
1564 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_framebuffer, VkFramebuffer)
1565 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_image, VkImage)
1566 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_image_view, VkImageView);
1567 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline_cache, VkPipelineCache)
1568 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline, VkPipeline)
1569 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline_layout, VkPipelineLayout)
1570 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_query_pool, VkQueryPool)
1571 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_render_pass, VkRenderPass)
1572 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_sampler, VkSampler)
1573 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_sampler_ycbcr_conversion, VkSamplerYcbcrConversion)
1574 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_shader_module, VkShaderModule)
1575 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_semaphore, VkSemaphore)
1576
1577 #endif /* TU_PRIVATE_H */