src/freedreno/vulkan/tu_private.h

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  */
  27
  28 #ifndef TU_PRIVATE_H
  29 #define TU_PRIVATE_H
  30
  31 #include <assert.h>
  32 #include <pthread.h>
  33 #include <stdbool.h>
  34 #include <stdint.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #ifdef HAVE_VALGRIND
  39 #include <memcheck.h>
  40 #include <valgrind.h>
  41 #define VG(x) x
  42 #else
  43 #define VG(x) ((void)0)
  44 #endif
  45
  46 #include "c11/threads.h"
  47 #include "main/macros.h"
  48 #include "util/list.h"
  49 #include "util/macros.h"
  50 #include "vk_alloc.h"
  51 #include "vk_debug_report.h"
  52 #include "wsi_common.h"
  53
  54 #include "drm-uapi/msm_drm.h"
  55 #include "ir3/ir3_compiler.h"
  56 #include "ir3/ir3_shader.h"
  57
  58 #include "adreno_common.xml.h"
  59 #include "adreno_pm4.xml.h"
  60 #include "a6xx.xml.h"
  61 #include "fdl/freedreno_layout.h"
  62
  63 #include "tu_descriptor_set.h"
  64 #include "tu_extensions.h"
  65
  66 /* Pre-declarations needed for WSI entrypoints */
  67 struct wl_surface;
  68 struct wl_display;
  69 typedef struct xcb_connection_t xcb_connection_t;
  70 typedef uint32_t xcb_visualid_t;
  71 typedef uint32_t xcb_window_t;
  72
  73 #include <vulkan/vk_android_native_buffer.h>
  74 #include <vulkan/vk_icd.h>
  75 #include <vulkan/vulkan.h>
  76 #include <vulkan/vulkan_intel.h>
  77
  78 #include "tu_entrypoints.h"
  79
  80 #include "vk_format.h"
  81
  82 #define MAX_VBS 32
  83 #define MAX_VERTEX_ATTRIBS 32
  84 #define MAX_RTS 8
  85 #define MAX_VSC_PIPES 32
  86 #define MAX_VIEWPORTS 1
  87 #define MAX_SCISSORS 16
  88 #define MAX_DISCARD_RECTANGLES 4
  89 #define MAX_PUSH_CONSTANTS_SIZE 128
  90 #define MAX_PUSH_DESCRIPTORS 32
  91 #define MAX_DYNAMIC_UNIFORM_BUFFERS 16
  92 #define MAX_DYNAMIC_STORAGE_BUFFERS 8
  93 #define MAX_DYNAMIC_BUFFERS                                                  \
  94    (MAX_DYNAMIC_UNIFORM_BUFFERS + MAX_DYNAMIC_STORAGE_BUFFERS)
  95 #define TU_MAX_DRM_DEVICES 8
  96 #define MAX_VIEWS 8
  97 #define MAX_BIND_POINTS 2 /* compute + graphics */
  98 /* The Qualcomm driver exposes 0x20000058 */
  99 #define MAX_STORAGE_BUFFER_RANGE 0x20000000
 100 /* We use ldc for uniform buffer loads, just like the Qualcomm driver, so
 101  * expose the same maximum range.
 102  * TODO: The SIZE bitfield is 15 bits, and in 4-dword units, so the actual
 103  * range might be higher.
 104  */
 105 #define MAX_UNIFORM_BUFFER_RANGE 0x10000
 106
 107 #define A6XX_TEX_CONST_DWORDS 16
 108 #define A6XX_TEX_SAMP_DWORDS 4
 109
 110 #define tu_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
 111
 112 static inline uint32_t
 113 tu_minify(uint32_t n, uint32_t levels)
 114 {
 115    if (unlikely(n == 0))
 116       return 0;
 117    else
 118       return MAX2(n >> levels, 1);
 119 }
 120
 121 #define for_each_bit(b, dword)                                               \
 122    for (uint32_t __dword = (dword);                                          \
 123         (b) = __builtin_ffs(__dword) - 1, __dword; __dword &= ~(1 << (b)))
 124
 125 #define typed_memcpy(dest, src, count)                                       \
 126    ({                                                                        \
 127       STATIC_ASSERT(sizeof(*src) == sizeof(*dest));                          \
 128       memcpy((dest), (src), (count) * sizeof(*(src)));                       \
 129    })
 130
 131 #define COND(bool, val) ((bool) ? (val) : 0)
 132
 133 /* Whenever we generate an error, pass it through this function. Useful for
 134  * debugging, where we can break on it. Only call at error site, not when
 135  * propagating errors. Might be useful to plug in a stack trace here.
 136  */
 137
 138 struct tu_instance;
 139
 140 VkResult
 141 __vk_errorf(struct tu_instance *instance,
 142             VkResult error,
 143             const char *file,
 144             int line,
 145             const char *format,
 146             ...);
 147
 148 #define vk_error(instance, error)                                            \
 149    __vk_errorf(instance, error, __FILE__, __LINE__, NULL);
 150 #define vk_errorf(instance, error, format, ...)                              \
 151    __vk_errorf(instance, error, __FILE__, __LINE__, format, ##__VA_ARGS__);
 152
 153 void
 154 __tu_finishme(const char *file, int line, const char *format, ...)
 155    tu_printflike(3, 4);
 156 void
 157 tu_loge(const char *format, ...) tu_printflike(1, 2);
 158 void
 159 tu_logi(const char *format, ...) tu_printflike(1, 2);
 160
 161 /**
 162  * Print a FINISHME message, including its source location.
 163  */
 164 #define tu_finishme(format, ...)                                             \
 165    do {                                                                      \
 166       static bool reported = false;                                          \
 167       if (!reported) {                                                       \
 168          __tu_finishme(__FILE__, __LINE__, format, ##__VA_ARGS__);           \
 169          reported = true;                                                    \
 170       }                                                                      \
 171    } while (0)
 172
 173 /* Suppress -Wunused in stub functions */
 174 #define tu_use_args(...) __tu_use_args(0, ##__VA_ARGS__)
 175 static inline void
 176 __tu_use_args(int ignore, ...)
 177 {
 178 }
 179
 180 #define tu_stub()                                                            \
 181    do {                                                                      \
 182       tu_finishme("stub %s", __func__);                                      \
 183    } while (0)
 184
 185 void *
 186 tu_lookup_entrypoint_unchecked(const char *name);
 187 void *
 188 tu_lookup_entrypoint_checked(
 189    const char *name,
 190    uint32_t core_version,
 191    const struct tu_instance_extension_table *instance,
 192    const struct tu_device_extension_table *device);
 193
 194 struct tu_physical_device
 195 {
 196    VK_LOADER_DATA _loader_data;
 197
 198    struct tu_instance *instance;
 199
 200    char path[20];
 201    char name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE];
 202    uint8_t driver_uuid[VK_UUID_SIZE];
 203    uint8_t device_uuid[VK_UUID_SIZE];
 204    uint8_t cache_uuid[VK_UUID_SIZE];
 205
 206    struct wsi_device wsi_device;
 207
 208    int local_fd;
 209    int master_fd;
 210
 211    unsigned gpu_id;
 212    uint32_t gmem_size;
 213    uint64_t gmem_base;
 214    uint32_t ccu_offset_gmem;
 215    uint32_t ccu_offset_bypass;
 216    /* alignment for size of tiles */
 217    uint32_t tile_align_w;
 218 #define TILE_ALIGN_H 16
 219    /* gmem store/load granularity */
 220 #define GMEM_ALIGN_W 16
 221 #define GMEM_ALIGN_H 4
 222
 223    struct {
 224       uint32_t PC_UNKNOWN_9805;
 225       uint32_t SP_UNKNOWN_A0F8;
 226    } magic;
 227
 228    /* This is the drivers on-disk cache used as a fallback as opposed to
 229     * the pipeline cache defined by apps.
 230     */
 231    struct disk_cache *disk_cache;
 232
 233    struct tu_device_extension_table supported_extensions;
 234 };
 235
 236 enum tu_debug_flags
 237 {
 238    TU_DEBUG_STARTUP = 1 << 0,
 239    TU_DEBUG_NIR = 1 << 1,
 240    TU_DEBUG_IR3 = 1 << 2,
 241    TU_DEBUG_NOBIN = 1 << 3,
 242    TU_DEBUG_SYSMEM = 1 << 4,
 243    TU_DEBUG_FORCEBIN = 1 << 5,
 244    TU_DEBUG_NOUBWC = 1 << 6,
 245 };
 246
 247 struct tu_instance
 248 {
 249    VK_LOADER_DATA _loader_data;
 250
 251    VkAllocationCallbacks alloc;
 252
 253    uint32_t api_version;
 254    int physical_device_count;
 255    struct tu_physical_device physical_devices[TU_MAX_DRM_DEVICES];
 256
 257    enum tu_debug_flags debug_flags;
 258
 259    struct vk_debug_report_instance debug_report_callbacks;
 260
 261    struct tu_instance_extension_table enabled_extensions;
 262 };
 263
 264 VkResult
 265 tu_wsi_init(struct tu_physical_device *physical_device);
 266 void
 267 tu_wsi_finish(struct tu_physical_device *physical_device);
 268
 269 bool
 270 tu_instance_extension_supported(const char *name);
 271 uint32_t
 272 tu_physical_device_api_version(struct tu_physical_device *dev);
 273 bool
 274 tu_physical_device_extension_supported(struct tu_physical_device *dev,
 275                                        const char *name);
 276
 277 struct cache_entry;
 278
 279 struct tu_pipeline_cache
 280 {
 281    struct tu_device *device;
 282    pthread_mutex_t mutex;
 283
 284    uint32_t total_size;
 285    uint32_t table_size;
 286    uint32_t kernel_count;
 287    struct cache_entry **hash_table;
 288    bool modified;
 289
 290    VkAllocationCallbacks alloc;
 291 };
 292
 293 struct tu_pipeline_key
 294 {
 295 };
 296
 297
 298 /* queue types */
 299 #define TU_QUEUE_GENERAL 0
 300
 301 #define TU_MAX_QUEUE_FAMILIES 1
 302
 303 struct tu_fence
 304 {
 305    struct wsi_fence *fence_wsi;
 306    bool signaled;
 307    int fd;
 308 };
 309
 310 void
 311 tu_fence_init(struct tu_fence *fence, bool signaled);
 312 void
 313 tu_fence_finish(struct tu_fence *fence);
 314 void
 315 tu_fence_update_fd(struct tu_fence *fence, int fd);
 316 void
 317 tu_fence_copy(struct tu_fence *fence, const struct tu_fence *src);
 318 void
 319 tu_fence_signal(struct tu_fence *fence);
 320 void
 321 tu_fence_wait_idle(struct tu_fence *fence);
 322
 323 struct tu_queue
 324 {
 325    VK_LOADER_DATA _loader_data;
 326    struct tu_device *device;
 327    uint32_t queue_family_index;
 328    int queue_idx;
 329    VkDeviceQueueCreateFlags flags;
 330
 331    uint32_t msm_queue_id;
 332    struct tu_fence submit_fence;
 333 };
 334
 335 struct tu_bo
 336 {
 337    uint32_t gem_handle;
 338    uint64_t size;
 339    uint64_t iova;
 340    void *map;
 341 };
 342
 343 struct tu_device
 344 {
 345    VK_LOADER_DATA _loader_data;
 346
 347    VkAllocationCallbacks alloc;
 348
 349    struct tu_instance *instance;
 350
 351    struct tu_queue *queues[TU_MAX_QUEUE_FAMILIES];
 352    int queue_count[TU_MAX_QUEUE_FAMILIES];
 353
 354    struct tu_physical_device *physical_device;
 355
 356    struct ir3_compiler *compiler;
 357
 358    /* Backup in-memory cache to be used if the app doesn't provide one */
 359    struct tu_pipeline_cache *mem_cache;
 360
 361    struct tu_bo vsc_draw_strm;
 362    struct tu_bo vsc_prim_strm;
 363    uint32_t vsc_draw_strm_pitch;
 364    uint32_t vsc_prim_strm_pitch;
 365
 366 #define MIN_SCRATCH_BO_SIZE_LOG2 12 /* A page */
 367
 368    /* Currently the kernel driver uses a 32-bit GPU address space, but it
 369     * should be impossible to go beyond 48 bits.
 370     */
 371    struct {
 372       struct tu_bo bo;
 373       mtx_t construct_mtx;
 374       bool initialized;
 375    } scratch_bos[48 - MIN_SCRATCH_BO_SIZE_LOG2];
 376
 377    struct tu_bo border_color;
 378
 379    struct tu_device_extension_table enabled_extensions;
 380 };
 381
 382 VkResult
 383 tu_bo_init_new(struct tu_device *dev, struct tu_bo *bo, uint64_t size);
 384 VkResult
 385 tu_bo_init_dmabuf(struct tu_device *dev,
 386                   struct tu_bo *bo,
 387                   uint64_t size,
 388                   int fd);
 389 int
 390 tu_bo_export_dmabuf(struct tu_device *dev, struct tu_bo *bo);
 391 void
 392 tu_bo_finish(struct tu_device *dev, struct tu_bo *bo);
 393 VkResult
 394 tu_bo_map(struct tu_device *dev, struct tu_bo *bo);
 395
 396 /* Get a scratch bo for use inside a command buffer. This will always return
 397  * the same bo given the same size or similar sizes, so only one scratch bo
 398  * can be used at the same time. It's meant for short-lived things where we
 399  * need to write to some piece of memory, read from it, and then immediately
 400  * discard it.
 401  */
 402 VkResult
 403 tu_get_scratch_bo(struct tu_device *dev, uint64_t size, struct tu_bo **bo);
 404
 405 struct tu_cs_entry
 406 {
 407    /* No ownership */
 408    const struct tu_bo *bo;
 409
 410    uint32_t size;
 411    uint32_t offset;
 412 };
 413
 414 struct ts_cs_memory {
 415    uint32_t *map;
 416    uint64_t iova;
 417 };
 418
 419 enum tu_cs_mode
 420 {
 421
 422    /*
 423     * A command stream in TU_CS_MODE_GROW mode grows automatically whenever it
 424     * is full.  tu_cs_begin must be called before command packet emission and
 425     * tu_cs_end must be called after.
 426     *
 427     * This mode may create multiple entries internally.  The entries must be
 428     * submitted together.
 429     */
 430    TU_CS_MODE_GROW,
 431
 432    /*
 433     * A command stream in TU_CS_MODE_EXTERNAL mode wraps an external,
 434     * fixed-size buffer.  tu_cs_begin and tu_cs_end are optional and have no
 435     * effect on it.
 436     *
 437     * This mode does not create any entry or any BO.
 438     */
 439    TU_CS_MODE_EXTERNAL,
 440
 441    /*
 442     * A command stream in TU_CS_MODE_SUB_STREAM mode does not support direct
 443     * command packet emission.  tu_cs_begin_sub_stream must be called to get a
 444     * sub-stream to emit comamnd packets to.  When done with the sub-stream,
 445     * tu_cs_end_sub_stream must be called.
 446     *
 447     * This mode does not create any entry internally.
 448     */
 449    TU_CS_MODE_SUB_STREAM,
 450 };
 451
 452 struct tu_cs
 453 {
 454    uint32_t *start;
 455    uint32_t *cur;
 456    uint32_t *reserved_end;
 457    uint32_t *end;
 458
 459    struct tu_device *device;
 460    enum tu_cs_mode mode;
 461    uint32_t next_bo_size;
 462
 463    struct tu_cs_entry *entries;
 464    uint32_t entry_count;
 465    uint32_t entry_capacity;
 466
 467    struct tu_bo **bos;
 468    uint32_t bo_count;
 469    uint32_t bo_capacity;
 470
 471    /* state for cond_exec_start/cond_exec_end */
 472    uint32_t cond_flags;
 473    uint32_t *cond_dwords;
 474 };
 475
 476 struct tu_device_memory
 477 {
 478    struct tu_bo bo;
 479    VkDeviceSize size;
 480
 481    /* for dedicated allocations */
 482    struct tu_image *image;
 483    struct tu_buffer *buffer;
 484
 485    uint32_t type_index;
 486    void *map;
 487    void *user_ptr;
 488 };
 489
 490 struct tu_descriptor_range
 491 {
 492    uint64_t va;
 493    uint32_t size;
 494 };
 495
 496 struct tu_descriptor_set
 497 {
 498    const struct tu_descriptor_set_layout *layout;
 499    struct tu_descriptor_pool *pool;
 500    uint32_t size;
 501
 502    uint64_t va;
 503    uint32_t *mapped_ptr;
 504
 505    uint32_t *dynamic_descriptors;
 506
 507    struct tu_bo *buffers[0];
 508 };
 509
 510 struct tu_push_descriptor_set
 511 {
 512    struct tu_descriptor_set set;
 513    uint32_t capacity;
 514 };
 515
 516 struct tu_descriptor_pool_entry
 517 {
 518    uint32_t offset;
 519    uint32_t size;
 520    struct tu_descriptor_set *set;
 521 };
 522
 523 struct tu_descriptor_pool
 524 {
 525    struct tu_bo bo;
 526    uint64_t current_offset;
 527    uint64_t size;
 528
 529    uint8_t *host_memory_base;
 530    uint8_t *host_memory_ptr;
 531    uint8_t *host_memory_end;
 532
 533    uint32_t entry_count;
 534    uint32_t max_entry_count;
 535    struct tu_descriptor_pool_entry entries[0];
 536 };
 537
 538 struct tu_descriptor_update_template_entry
 539 {
 540    VkDescriptorType descriptor_type;
 541
 542    /* The number of descriptors to update */
 543    uint32_t descriptor_count;
 544
 545    /* Into mapped_ptr or dynamic_descriptors, in units of the respective array
 546     */
 547    uint32_t dst_offset;
 548
 549    /* In dwords. Not valid/used for dynamic descriptors */
 550    uint32_t dst_stride;
 551
 552    uint32_t buffer_offset;
 553
 554    /* Only valid for combined image samplers and samplers */
 555    uint16_t has_sampler;
 556
 557    /* In bytes */
 558    size_t src_offset;
 559    size_t src_stride;
 560
 561    /* For push descriptors */
 562    const uint32_t *immutable_samplers;
 563 };
 564
 565 struct tu_descriptor_update_template
 566 {
 567    uint32_t entry_count;
 568    struct tu_descriptor_update_template_entry entry[0];
 569 };
 570
 571 struct tu_buffer
 572 {
 573    VkDeviceSize size;
 574
 575    VkBufferUsageFlags usage;
 576    VkBufferCreateFlags flags;
 577
 578    struct tu_bo *bo;
 579    VkDeviceSize bo_offset;
 580 };
 581
 582 static inline uint64_t
 583 tu_buffer_iova(struct tu_buffer *buffer)
 584 {
 585    return buffer->bo->iova + buffer->bo_offset;
 586 }
 587
 588 enum tu_dynamic_state_bits
 589 {
 590    TU_DYNAMIC_VIEWPORT = 1 << 0,
 591    TU_DYNAMIC_SCISSOR = 1 << 1,
 592    TU_DYNAMIC_LINE_WIDTH = 1 << 2,
 593    TU_DYNAMIC_DEPTH_BIAS = 1 << 3,
 594    TU_DYNAMIC_BLEND_CONSTANTS = 1 << 4,
 595    TU_DYNAMIC_DEPTH_BOUNDS = 1 << 5,
 596    TU_DYNAMIC_STENCIL_COMPARE_MASK = 1 << 6,
 597    TU_DYNAMIC_STENCIL_WRITE_MASK = 1 << 7,
 598    TU_DYNAMIC_STENCIL_REFERENCE = 1 << 8,
 599    TU_DYNAMIC_DISCARD_RECTANGLE = 1 << 9,
 600    TU_DYNAMIC_SAMPLE_LOCATIONS = 1 << 10,
 601    TU_DYNAMIC_ALL = (1 << 11) - 1,
 602 };
 603
 604 struct tu_vertex_binding
 605 {
 606    struct tu_buffer *buffer;
 607    VkDeviceSize offset;
 608 };
 609
 610 struct tu_viewport_state
 611 {
 612    uint32_t count;
 613    VkViewport viewports[MAX_VIEWPORTS];
 614 };
 615
 616 struct tu_scissor_state
 617 {
 618    uint32_t count;
 619    VkRect2D scissors[MAX_SCISSORS];
 620 };
 621
 622 struct tu_discard_rectangle_state
 623 {
 624    uint32_t count;
 625    VkRect2D rectangles[MAX_DISCARD_RECTANGLES];
 626 };
 627
 628 struct tu_dynamic_state
 629 {
 630    /**
 631     * Bitmask of (1 << VK_DYNAMIC_STATE_*).
 632     * Defines the set of saved dynamic state.
 633     */
 634    uint32_t mask;
 635
 636    struct tu_viewport_state viewport;
 637
 638    struct tu_scissor_state scissor;
 639
 640    float line_width;
 641
 642    struct
 643    {
 644       float bias;
 645       float clamp;
 646       float slope;
 647    } depth_bias;
 648
 649    float blend_constants[4];
 650
 651    struct
 652    {
 653       float min;
 654       float max;
 655    } depth_bounds;
 656
 657    struct
 658    {
 659       uint32_t front;
 660       uint32_t back;
 661    } stencil_compare_mask;
 662
 663    struct
 664    {
 665       uint32_t front;
 666       uint32_t back;
 667    } stencil_write_mask;
 668
 669    struct
 670    {
 671       uint32_t front;
 672       uint32_t back;
 673    } stencil_reference;
 674
 675    struct tu_discard_rectangle_state discard_rectangle;
 676 };
 677
 678 extern const struct tu_dynamic_state default_dynamic_state;
 679
 680 const char *
 681 tu_get_debug_option_name(int id);
 682
 683 const char *
 684 tu_get_perftest_option_name(int id);
 685
 686 struct tu_descriptor_state
 687 {
 688    struct tu_descriptor_set *sets[MAX_SETS];
 689    uint32_t valid;
 690    struct tu_push_descriptor_set push_set;
 691    bool push_dirty;
 692    uint32_t dynamic_descriptors[MAX_DYNAMIC_BUFFERS * A6XX_TEX_CONST_DWORDS];
 693    uint32_t input_attachments[MAX_RTS * A6XX_TEX_CONST_DWORDS];
 694 };
 695
 696 struct tu_tile
 697 {
 698    uint8_t pipe;
 699    uint8_t slot;
 700    VkOffset2D begin;
 701    VkOffset2D end;
 702 };
 703
 704 struct tu_tiling_config
 705 {
 706    VkRect2D render_area;
 707
 708    /* position and size of the first tile */
 709    VkRect2D tile0;
 710    /* number of tiles */
 711    VkExtent2D tile_count;
 712
 713    /* size of the first VSC pipe */
 714    VkExtent2D pipe0;
 715    /* number of VSC pipes */
 716    VkExtent2D pipe_count;
 717
 718    /* pipe register values */
 719    uint32_t pipe_config[MAX_VSC_PIPES];
 720    uint32_t pipe_sizes[MAX_VSC_PIPES];
 721
 722    /* Whether sysmem rendering must be used */
 723    bool force_sysmem;
 724 };
 725
 726 enum tu_cmd_dirty_bits
 727 {
 728    TU_CMD_DIRTY_PIPELINE = 1 << 0,
 729    TU_CMD_DIRTY_COMPUTE_PIPELINE = 1 << 1,
 730    TU_CMD_DIRTY_VERTEX_BUFFERS = 1 << 2,
 731
 732    TU_CMD_DIRTY_DESCRIPTOR_SETS = 1 << 3,
 733    TU_CMD_DIRTY_COMPUTE_DESCRIPTOR_SETS = 1 << 4,
 734    TU_CMD_DIRTY_PUSH_CONSTANTS = 1 << 5,
 735    TU_CMD_DIRTY_STREAMOUT_BUFFERS = 1 << 6,
 736    TU_CMD_DIRTY_INPUT_ATTACHMENTS = 1 << 7,
 737
 738    TU_CMD_DIRTY_DYNAMIC_LINE_WIDTH = 1 << 16,
 739    TU_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK = 1 << 17,
 740    TU_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK = 1 << 18,
 741    TU_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE = 1 << 19,
 742    TU_CMD_DIRTY_DYNAMIC_VIEWPORT = 1 << 20,
 743    TU_CMD_DIRTY_DYNAMIC_SCISSOR = 1 << 21,
 744 };
 745
 746 struct tu_streamout_state {
 747    uint16_t stride[IR3_MAX_SO_BUFFERS];
 748    uint32_t ncomp[IR3_MAX_SO_BUFFERS];
 749    uint32_t prog[IR3_MAX_SO_OUTPUTS * 2];
 750    uint32_t prog_count;
 751    uint32_t vpc_so_buf_cntl;
 752 };
 753
 754 /* There are only three cache domains we have to care about: the CCU, or
 755  * color cache unit, which is used for color and depth/stencil attachments
 756  * and copy/blit destinations, and is split conceptually into color and depth,
 757  * and the universal cache or UCHE which is used for pretty much everything
 758  * else, except for the CP (uncached) and host. We need to flush whenever data
 759  * crosses these boundaries.
 760  */
 761
 762 enum tu_cmd_access_mask {
 763    TU_ACCESS_UCHE_READ = 1 << 0,
 764    TU_ACCESS_UCHE_WRITE = 1 << 1,
 765    TU_ACCESS_CCU_COLOR_READ = 1 << 2,
 766    TU_ACCESS_CCU_COLOR_WRITE = 1 << 3,
 767    TU_ACCESS_CCU_DEPTH_READ = 1 << 4,
 768    TU_ACCESS_CCU_DEPTH_WRITE = 1 << 5,
 769
 770    /* Experiments have shown that while it's safe to avoid flushing the CCU
 771     * after each blit/renderpass, it's not safe to assume that subsequent
 772     * lookups with a different attachment state will hit unflushed cache
 773     * entries. That is, the CCU needs to be flushed and possibly invalidated
 774     * when accessing memory with a different attachment state. Writing to an
 775     * attachment under the following conditions after clearing using the
 776     * normal 2d engine path is known to have issues:
 777     *
 778     * - It isn't the 0'th layer.
 779     * - There are more than one attachment, and this isn't the 0'th attachment
 780     *   (this seems to also depend on the cpp of the attachments).
 781     *
 782     * Our best guess is that the layer/MRT state is used when computing
 783     * the location of a cache entry in CCU, to avoid conflicts. We assume that
 784     * any access in a renderpass after or before an access by a transfer needs
 785     * a flush/invalidate, and use the _INCOHERENT variants to represent access
 786     * by a transfer.
 787     */
 788    TU_ACCESS_CCU_COLOR_INCOHERENT_READ = 1 << 6,
 789    TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE = 1 << 7,
 790    TU_ACCESS_CCU_DEPTH_INCOHERENT_READ = 1 << 8,
 791    TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE = 1 << 9,
 792
 793    TU_ACCESS_SYSMEM_READ = 1 << 10,
 794    TU_ACCESS_SYSMEM_WRITE = 1 << 11,
 795
 796    /* Set if a WFI is required due to data being read by the CP or the 2D
 797     * engine.
 798     */
 799    TU_ACCESS_WFI_READ = 1 << 12,
 800
 801    TU_ACCESS_READ =
 802       TU_ACCESS_UCHE_READ |
 803       TU_ACCESS_CCU_COLOR_READ |
 804       TU_ACCESS_CCU_DEPTH_READ |
 805       TU_ACCESS_CCU_COLOR_INCOHERENT_READ |
 806       TU_ACCESS_CCU_DEPTH_INCOHERENT_READ |
 807       TU_ACCESS_SYSMEM_READ,
 808
 809    TU_ACCESS_WRITE =
 810       TU_ACCESS_UCHE_WRITE |
 811       TU_ACCESS_CCU_COLOR_WRITE |
 812       TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE |
 813       TU_ACCESS_CCU_DEPTH_WRITE |
 814       TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE |
 815       TU_ACCESS_SYSMEM_WRITE,
 816
 817    TU_ACCESS_ALL =
 818       TU_ACCESS_READ |
 819       TU_ACCESS_WRITE,
 820 };
 821
 822 enum tu_cmd_flush_bits {
 823    TU_CMD_FLAG_CCU_FLUSH_DEPTH = 1 << 0,
 824    TU_CMD_FLAG_CCU_FLUSH_COLOR = 1 << 1,
 825    TU_CMD_FLAG_CCU_INVALIDATE_DEPTH = 1 << 2,
 826    TU_CMD_FLAG_CCU_INVALIDATE_COLOR = 1 << 3,
 827    TU_CMD_FLAG_CACHE_FLUSH = 1 << 4,
 828    TU_CMD_FLAG_CACHE_INVALIDATE = 1 << 5,
 829
 830    TU_CMD_FLAG_ALL_FLUSH =
 831       TU_CMD_FLAG_CCU_FLUSH_DEPTH |
 832       TU_CMD_FLAG_CCU_FLUSH_COLOR |
 833       TU_CMD_FLAG_CACHE_FLUSH,
 834
 835    TU_CMD_FLAG_ALL_INVALIDATE =
 836       TU_CMD_FLAG_CCU_INVALIDATE_DEPTH |
 837       TU_CMD_FLAG_CCU_INVALIDATE_COLOR |
 838       TU_CMD_FLAG_CACHE_INVALIDATE,
 839
 840    TU_CMD_FLAG_WFI = 1 << 6,
 841 };
 842
 843 /* Changing the CCU from sysmem mode to gmem mode or vice-versa is pretty
 844  * heavy, involving a CCU cache flush/invalidate and a WFI in order to change
 845  * which part of the gmem is used by the CCU. Here we keep track of what the
 846  * state of the CCU.
 847  */
 848 enum tu_cmd_ccu_state {
 849    TU_CMD_CCU_SYSMEM,
 850    TU_CMD_CCU_GMEM,
 851    TU_CMD_CCU_UNKNOWN,
 852 };
 853
 854 struct tu_cache_state {
 855    /* Caches which must be made available (flushed) eventually if there are
 856     * any users outside that cache domain, and caches which must be
 857     * invalidated eventually if there are any reads.
 858     */
 859    enum tu_cmd_flush_bits pending_flush_bits;
 860    /* Pending flushes */
 861    enum tu_cmd_flush_bits flush_bits;
 862 };
 863
 864 struct tu_cmd_state
 865 {
 866    uint32_t dirty;
 867
 868    struct tu_pipeline *pipeline;
 869    struct tu_pipeline *compute_pipeline;
 870
 871    /* Vertex buffers */
 872    struct
 873    {
 874       struct tu_buffer *buffers[MAX_VBS];
 875       VkDeviceSize offsets[MAX_VBS];
 876    } vb;
 877
 878    struct tu_dynamic_state dynamic;
 879
 880    /* Stream output buffers */
 881    struct
 882    {
 883       struct tu_buffer *buffers[IR3_MAX_SO_BUFFERS];
 884       VkDeviceSize offsets[IR3_MAX_SO_BUFFERS];
 885       VkDeviceSize sizes[IR3_MAX_SO_BUFFERS];
 886    } streamout_buf;
 887
 888    uint8_t streamout_reset;
 889    uint8_t streamout_enabled;
 890
 891    /* Index buffer */
 892    struct tu_buffer *index_buffer;
 893    uint64_t index_offset;
 894    uint32_t index_type;
 895    uint32_t max_index_count;
 896    uint64_t index_va;
 897
 898    /* Renderpasses are tricky, because we may need to flush differently if
 899     * using sysmem vs. gmem and therefore we have to delay any flushing that
 900     * happens before a renderpass. So we have to have two copies of the flush
 901     * state, one for intra-renderpass flushes (i.e. renderpass dependencies)
 902     * and one for outside a renderpass.
 903     */
 904    struct tu_cache_state cache;
 905    struct tu_cache_state renderpass_cache;
 906
 907    enum tu_cmd_ccu_state ccu_state;
 908
 909    const struct tu_render_pass *pass;
 910    const struct tu_subpass *subpass;
 911    const struct tu_framebuffer *framebuffer;
 912
 913    struct tu_tiling_config tiling_config;
 914
 915    struct tu_cs_entry tile_store_ib;
 916 };
 917
 918 struct tu_cmd_pool
 919 {
 920    VkAllocationCallbacks alloc;
 921    struct list_head cmd_buffers;
 922    struct list_head free_cmd_buffers;
 923    uint32_t queue_family_index;
 924 };
 925
 926 struct tu_cmd_buffer_upload
 927 {
 928    uint8_t *map;
 929    unsigned offset;
 930    uint64_t size;
 931    struct list_head list;
 932 };
 933
 934 enum tu_cmd_buffer_status
 935 {
 936    TU_CMD_BUFFER_STATUS_INVALID,
 937    TU_CMD_BUFFER_STATUS_INITIAL,
 938    TU_CMD_BUFFER_STATUS_RECORDING,
 939    TU_CMD_BUFFER_STATUS_EXECUTABLE,
 940    TU_CMD_BUFFER_STATUS_PENDING,
 941 };
 942
 943 struct tu_bo_list
 944 {
 945    uint32_t count;
 946    uint32_t capacity;
 947    struct drm_msm_gem_submit_bo *bo_infos;
 948 };
 949
 950 #define TU_BO_LIST_FAILED (~0)
 951
 952 void
 953 tu_bo_list_init(struct tu_bo_list *list);
 954 void
 955 tu_bo_list_destroy(struct tu_bo_list *list);
 956 void
 957 tu_bo_list_reset(struct tu_bo_list *list);
 958 uint32_t
 959 tu_bo_list_add(struct tu_bo_list *list,
 960                const struct tu_bo *bo,
 961                uint32_t flags);
 962 VkResult
 963 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other);
 964
 965 /* This struct defines the layout of the scratch_bo */
 966 struct tu6_control
 967 {
 968    uint32_t seqno_dummy;          /* dummy seqno for CP_EVENT_WRITE */
 969    uint32_t _pad0;
 970    volatile uint32_t vsc_overflow;
 971    uint32_t _pad1;
 972    /* flag set from cmdstream when VSC overflow detected: */
 973    uint32_t vsc_scratch;
 974    uint32_t _pad2;
 975    uint32_t _pad3;
 976    uint32_t _pad4;
 977
 978    /* scratch space for VPC_SO[i].FLUSH_BASE_LO/HI, start on 32 byte boundary. */
 979    struct {
 980       uint32_t offset;
 981       uint32_t pad[7];
 982    } flush_base[4];
 983 };
 984
 985 #define ctrl_offset(member) offsetof(struct tu6_control, member)
 986
 987 struct tu_cmd_buffer
 988 {
 989    VK_LOADER_DATA _loader_data;
 990
 991    struct tu_device *device;
 992
 993    struct tu_cmd_pool *pool;
 994    struct list_head pool_link;
 995
 996    VkCommandBufferUsageFlags usage_flags;
 997    VkCommandBufferLevel level;
 998    enum tu_cmd_buffer_status status;
 999
1000    struct tu_cmd_state state;
1001    struct tu_vertex_binding vertex_bindings[MAX_VBS];
1002    uint32_t vertex_bindings_set;
1003    uint32_t queue_family_index;
1004
1005    uint32_t push_constants[MAX_PUSH_CONSTANTS_SIZE / 4];
1006    VkShaderStageFlags push_constant_stages;
1007    struct tu_descriptor_set meta_push_descriptors;
1008
1009    struct tu_descriptor_state descriptors[MAX_BIND_POINTS];
1010
1011    struct tu_cmd_buffer_upload upload;
1012
1013    VkResult record_result;
1014
1015    struct tu_bo_list bo_list;
1016    struct tu_cs cs;
1017    struct tu_cs draw_cs;
1018    struct tu_cs draw_epilogue_cs;
1019    struct tu_cs sub_cs;
1020
1021    struct tu_bo scratch_bo;
1022
1023    struct tu_bo vsc_draw_strm;
1024    struct tu_bo vsc_prim_strm;
1025    uint32_t vsc_draw_strm_pitch;
1026    uint32_t vsc_prim_strm_pitch;
1027    bool use_vsc_data;
1028 };
1029
1030 /* Temporary struct for tracking a register state to be written, used by
1031  * a6xx-pack.h and tu_cs_emit_regs()
1032  */
1033 struct tu_reg_value {
1034    uint32_t reg;
1035    uint64_t value;
1036    bool is_address;
1037    struct tu_bo *bo;
1038    bool bo_write;
1039    uint32_t bo_offset;
1040    uint32_t bo_shift;
1041 };
1042
1043
1044 void tu_emit_cache_flush_renderpass(struct tu_cmd_buffer *cmd_buffer,
1045                                     struct tu_cs *cs);
1046
1047 void tu_emit_cache_flush_ccu(struct tu_cmd_buffer *cmd_buffer,
1048                              struct tu_cs *cs,
1049                              enum tu_cmd_ccu_state ccu_state);
1050
1051 void
1052 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
1053                      struct tu_cs *cs,
1054                      enum vgt_event_type event);
1055
1056 static inline struct tu_descriptor_state *
1057 tu_get_descriptors_state(struct tu_cmd_buffer *cmd_buffer,
1058                          VkPipelineBindPoint bind_point)
1059 {
1060    return &cmd_buffer->descriptors[bind_point];
1061 }
1062
1063 struct tu_event
1064 {
1065    struct tu_bo bo;
1066 };
1067
1068 static inline gl_shader_stage
1069 vk_to_mesa_shader_stage(VkShaderStageFlagBits vk_stage)
1070 {
1071    assert(__builtin_popcount(vk_stage) == 1);
1072    return ffs(vk_stage) - 1;
1073 }
1074
1075 static inline VkShaderStageFlagBits
1076 mesa_to_vk_shader_stage(gl_shader_stage mesa_stage)
1077 {
1078    return (1 << mesa_stage);
1079 }
1080
1081 #define TU_STAGE_MASK ((1 << MESA_SHADER_STAGES) - 1)
1082
1083 #define tu_foreach_stage(stage, stage_bits)                                  \
1084    for (gl_shader_stage stage,                                               \
1085         __tmp = (gl_shader_stage)((stage_bits) &TU_STAGE_MASK);              \
1086         stage = __builtin_ffs(__tmp) - 1, __tmp; __tmp &= ~(1 << (stage)))
1087
1088 uint32_t
1089 tu6_stage2opcode(gl_shader_stage type);
1090 enum a6xx_state_block
1091 tu6_stage2shadersb(gl_shader_stage type);
1092
1093 struct tu_shader_module
1094 {
1095    unsigned char sha1[20];
1096
1097    uint32_t code_size;
1098    const uint32_t *code[0];
1099 };
1100
1101 struct tu_shader_compile_options
1102 {
1103    struct ir3_shader_key key;
1104
1105    bool optimize;
1106    bool include_binning_pass;
1107 };
1108
1109 struct tu_push_constant_range
1110 {
1111    uint32_t lo;
1112    uint32_t count;
1113 };
1114
1115 struct tu_shader
1116 {
1117    struct ir3_shader ir3_shader;
1118
1119    struct tu_push_constant_range push_consts;
1120    unsigned attachment_idx[MAX_RTS];
1121    uint8_t active_desc_sets;
1122
1123    /* This may be true for vertex shaders.  When true, variants[1] is the
1124     * binning variant and binning_binary is non-NULL.
1125     */
1126    bool has_binning_pass;
1127
1128    void *binary;
1129    void *binning_binary;
1130
1131    struct ir3_shader_variant variants[0];
1132 };
1133
1134 struct tu_shader *
1135 tu_shader_create(struct tu_device *dev,
1136                  gl_shader_stage stage,
1137                  const VkPipelineShaderStageCreateInfo *stage_info,
1138                  struct tu_pipeline_layout *layout,
1139                  const VkAllocationCallbacks *alloc);
1140
1141 void
1142 tu_shader_destroy(struct tu_device *dev,
1143                   struct tu_shader *shader,
1144                   const VkAllocationCallbacks *alloc);
1145
1146 void
1147 tu_shader_compile_options_init(
1148    struct tu_shader_compile_options *options,
1149    const VkGraphicsPipelineCreateInfo *pipeline_info);
1150
1151 VkResult
1152 tu_shader_compile(struct tu_device *dev,
1153                   struct tu_shader *shader,
1154                   const struct tu_shader *next_stage,
1155                   const struct tu_shader_compile_options *options,
1156                   const VkAllocationCallbacks *alloc);
1157
1158 struct tu_program_descriptor_linkage
1159 {
1160    struct ir3_ubo_analysis_state ubo_state;
1161    struct ir3_const_state const_state;
1162
1163    uint32_t constlen;
1164
1165    struct tu_push_constant_range push_consts;
1166 };
1167
1168 struct tu_pipeline
1169 {
1170    struct tu_cs cs;
1171
1172    struct tu_dynamic_state dynamic_state;
1173
1174    struct tu_pipeline_layout *layout;
1175
1176    bool need_indirect_descriptor_sets;
1177    VkShaderStageFlags active_stages;
1178    uint32_t active_desc_sets;
1179
1180    struct tu_streamout_state streamout;
1181
1182    struct
1183    {
1184       struct tu_bo binary_bo;
1185       struct tu_cs_entry state_ib;
1186       struct tu_cs_entry binning_state_ib;
1187
1188       struct tu_program_descriptor_linkage link[MESA_SHADER_STAGES];
1189       unsigned input_attachment_idx[MAX_RTS];
1190    } program;
1191
1192    struct
1193    {
1194       struct tu_cs_entry state_ib;
1195    } load_state;
1196
1197    struct
1198    {
1199       struct tu_cs_entry state_ib;
1200       struct tu_cs_entry binning_state_ib;
1201       uint32_t bindings_used;
1202    } vi;
1203
1204    struct
1205    {
1206       enum pc_di_primtype primtype;
1207       bool primitive_restart;
1208    } ia;
1209
1210    struct
1211    {
1212       struct tu_cs_entry state_ib;
1213    } vp;
1214
1215    struct
1216    {
1217       uint32_t gras_su_cntl;
1218       struct tu_cs_entry state_ib;
1219    } rast;
1220
1221    struct
1222    {
1223       struct tu_cs_entry state_ib;
1224    } ds;
1225
1226    struct
1227    {
1228       struct tu_cs_entry state_ib;
1229    } blend;
1230
1231    struct
1232    {
1233       uint32_t local_size[3];
1234    } compute;
1235 };
1236
1237 void
1238 tu6_emit_viewport(struct tu_cs *cs, const VkViewport *viewport);
1239
1240 void
1241 tu6_emit_scissor(struct tu_cs *cs, const VkRect2D *scissor);
1242
1243 void
1244 tu6_emit_sample_locations(struct tu_cs *cs, const VkSampleLocationsInfoEXT *samp_loc);
1245
1246 void
1247 tu6_emit_gras_su_cntl(struct tu_cs *cs,
1248                       uint32_t gras_su_cntl,
1249                       float line_width);
1250
1251 void
1252 tu6_emit_depth_bias(struct tu_cs *cs,
1253                     float constant_factor,
1254                     float clamp,
1255                     float slope_factor);
1256
1257 void
1258 tu6_emit_stencil_compare_mask(struct tu_cs *cs,
1259                               uint32_t front,
1260                               uint32_t back);
1261
1262 void
1263 tu6_emit_stencil_write_mask(struct tu_cs *cs, uint32_t front, uint32_t back);
1264
1265 void
1266 tu6_emit_stencil_reference(struct tu_cs *cs, uint32_t front, uint32_t back);
1267
1268 void
1269 tu6_emit_blend_constants(struct tu_cs *cs, const float constants[4]);
1270
1271 void tu6_emit_msaa(struct tu_cs *cs, VkSampleCountFlagBits samples);
1272
1273 void tu6_emit_window_scissor(struct tu_cs *cs, uint32_t x1, uint32_t y1, uint32_t x2, uint32_t y2);
1274
1275 void tu6_emit_window_offset(struct tu_cs *cs, uint32_t x1, uint32_t y1);
1276
1277 void
1278 tu6_emit_xs_config(struct tu_cs *cs,
1279                    gl_shader_stage stage,
1280                    const struct ir3_shader_variant *xs,
1281                    uint64_t binary_iova);
1282
1283 void
1284 tu6_emit_vpc(struct tu_cs *cs,
1285              const struct ir3_shader_variant *vs,
1286              const struct ir3_shader_variant *gs,
1287              const struct ir3_shader_variant *fs,
1288              struct tu_streamout_state *tf);
1289
1290 void
1291 tu6_emit_fs_inputs(struct tu_cs *cs, const struct ir3_shader_variant *fs);
1292
1293 struct tu_image_view;
1294
1295 void
1296 tu_resolve_sysmem(struct tu_cmd_buffer *cmd,
1297                   struct tu_cs *cs,
1298                   struct tu_image_view *src,
1299                   struct tu_image_view *dst,
1300                   uint32_t layers,
1301                   const VkRect2D *rect);
1302
1303 void
1304 tu_clear_sysmem_attachment(struct tu_cmd_buffer *cmd,
1305                            struct tu_cs *cs,
1306                            uint32_t a,
1307                            const VkRenderPassBeginInfo *info);
1308
1309 void
1310 tu_clear_gmem_attachment(struct tu_cmd_buffer *cmd,
1311                          struct tu_cs *cs,
1312                          uint32_t a,
1313                          const VkRenderPassBeginInfo *info);
1314
1315 void
1316 tu_load_gmem_attachment(struct tu_cmd_buffer *cmd,
1317                         struct tu_cs *cs,
1318                         uint32_t a,
1319                         bool force_load);
1320
1321 /* expose this function to be able to emit load without checking LOAD_OP */
1322 void
1323 tu_emit_load_gmem_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t a);
1324
1325 /* note: gmem store can also resolve */
1326 void
1327 tu_store_gmem_attachment(struct tu_cmd_buffer *cmd,
1328                          struct tu_cs *cs,
1329                          uint32_t a,
1330                          uint32_t gmem_a);
1331
1332 enum tu_supported_formats {
1333    FMT_VERTEX = 1,
1334    FMT_TEXTURE = 2,
1335    FMT_COLOR = 4,
1336 };
1337
1338 struct tu_native_format
1339 {
1340    enum a6xx_format fmt : 8;
1341    enum a3xx_color_swap swap : 8;
1342    enum a6xx_tile_mode tile_mode : 8;
1343    enum tu_supported_formats supported : 8;
1344 };
1345
1346 struct tu_native_format tu6_format_vtx(VkFormat format);
1347 struct tu_native_format tu6_format_color(VkFormat format, enum a6xx_tile_mode tile_mode);
1348 struct tu_native_format tu6_format_texture(VkFormat format, enum a6xx_tile_mode tile_mode);
1349
1350 static inline enum a6xx_format
1351 tu6_base_format(VkFormat format)
1352 {
1353    /* note: tu6_format_color doesn't care about tiling for .fmt field */
1354    return tu6_format_color(format, TILE6_LINEAR).fmt;
1355 }
1356
1357 enum a6xx_depth_format tu6_pipe2depth(VkFormat format);
1358
1359 struct tu_image
1360 {
1361    VkImageType type;
1362    /* The original VkFormat provided by the client.  This may not match any
1363     * of the actual surface formats.
1364     */
1365    VkFormat vk_format;
1366    VkImageAspectFlags aspects;
1367    VkImageUsageFlags usage;  /**< Superset of VkImageCreateInfo::usage. */
1368    VkImageTiling tiling;     /** VkImageCreateInfo::tiling */
1369    VkImageCreateFlags flags; /** VkImageCreateInfo::flags */
1370    VkExtent3D extent;
1371    uint32_t level_count;
1372    uint32_t layer_count;
1373    VkSampleCountFlagBits samples;
1374
1375    struct fdl_layout layout;
1376
1377    unsigned queue_family_mask;
1378    bool exclusive;
1379    bool shareable;
1380
1381    /* For VK_ANDROID_native_buffer, the WSI image owns the memory, */
1382    VkDeviceMemory owned_memory;
1383
1384    /* Set when bound */
1385    struct tu_bo *bo;
1386    VkDeviceSize bo_offset;
1387 };
1388
1389 static inline uint32_t
1390 tu_get_layerCount(const struct tu_image *image,
1391                   const VkImageSubresourceRange *range)
1392 {
1393    return range->layerCount == VK_REMAINING_ARRAY_LAYERS
1394              ? image->layer_count - range->baseArrayLayer
1395              : range->layerCount;
1396 }
1397
1398 static inline uint32_t
1399 tu_get_levelCount(const struct tu_image *image,
1400                   const VkImageSubresourceRange *range)
1401 {
1402    return range->levelCount == VK_REMAINING_MIP_LEVELS
1403              ? image->level_count - range->baseMipLevel
1404              : range->levelCount;
1405 }
1406
1407 enum a3xx_msaa_samples
1408 tu_msaa_samples(uint32_t samples);
1409 enum a6xx_tex_fetchsize
1410 tu6_fetchsize(VkFormat format);
1411
1412 struct tu_image_view
1413 {
1414    struct tu_image *image; /**< VkImageViewCreateInfo::image */
1415
1416    uint64_t base_addr;
1417    uint64_t ubwc_addr;
1418    uint32_t layer_size;
1419    uint32_t ubwc_layer_size;
1420
1421    /* used to determine if fast gmem store path can be used */
1422    VkExtent2D extent;
1423    bool need_y2_align;
1424
1425    bool ubwc_enabled;
1426
1427    uint32_t descriptor[A6XX_TEX_CONST_DWORDS];
1428
1429    /* Descriptor for use as a storage image as opposed to a sampled image.
1430     * This has a few differences for cube maps (e.g. type).
1431     */
1432    uint32_t storage_descriptor[A6XX_TEX_CONST_DWORDS];
1433
1434    /* pre-filled register values */
1435    uint32_t PITCH;
1436    uint32_t FLAG_BUFFER_PITCH;
1437
1438    uint32_t RB_MRT_BUF_INFO;
1439    uint32_t SP_FS_MRT_REG;
1440
1441    uint32_t SP_PS_2D_SRC_INFO;
1442    uint32_t SP_PS_2D_SRC_SIZE;
1443
1444    uint32_t RB_2D_DST_INFO;
1445
1446    uint32_t RB_BLIT_DST_INFO;
1447 };
1448
1449 struct tu_sampler_ycbcr_conversion {
1450    VkFormat format;
1451    VkSamplerYcbcrModelConversion ycbcr_model;
1452    VkSamplerYcbcrRange ycbcr_range;
1453    VkComponentMapping components;
1454    VkChromaLocation chroma_offsets[2];
1455    VkFilter chroma_filter;
1456 };
1457
1458 struct tu_sampler {
1459    uint32_t descriptor[A6XX_TEX_SAMP_DWORDS];
1460    struct tu_sampler_ycbcr_conversion *ycbcr_sampler;
1461 };
1462
1463 void
1464 tu_cs_image_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1465
1466 void
1467 tu_cs_image_ref_2d(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer, bool src);
1468
1469 void
1470 tu_cs_image_flag_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1471
1472 VkResult
1473 tu_image_create(VkDevice _device,
1474                 const VkImageCreateInfo *pCreateInfo,
1475                 const VkAllocationCallbacks *alloc,
1476                 VkImage *pImage,
1477                 uint64_t modifier,
1478                 const VkSubresourceLayout *plane_layouts);
1479
1480 VkResult
1481 tu_image_from_gralloc(VkDevice device_h,
1482                       const VkImageCreateInfo *base_info,
1483                       const VkNativeBufferANDROID *gralloc_info,
1484                       const VkAllocationCallbacks *alloc,
1485                       VkImage *out_image_h);
1486
1487 void
1488 tu_image_view_init(struct tu_image_view *view,
1489                    const VkImageViewCreateInfo *pCreateInfo);
1490
1491 struct tu_buffer_view
1492 {
1493    uint32_t descriptor[A6XX_TEX_CONST_DWORDS];
1494
1495    struct tu_buffer *buffer;
1496 };
1497 void
1498 tu_buffer_view_init(struct tu_buffer_view *view,
1499                     struct tu_device *device,
1500                     const VkBufferViewCreateInfo *pCreateInfo);
1501
1502 struct tu_attachment_info
1503 {
1504    struct tu_image_view *attachment;
1505 };
1506
1507 struct tu_framebuffer
1508 {
1509    uint32_t width;
1510    uint32_t height;
1511    uint32_t layers;
1512
1513    uint32_t attachment_count;
1514    struct tu_attachment_info attachments[0];
1515 };
1516
1517 struct tu_subpass_barrier {
1518    VkPipelineStageFlags src_stage_mask;
1519    VkAccessFlags src_access_mask;
1520    VkAccessFlags dst_access_mask;
1521    bool incoherent_ccu_color, incoherent_ccu_depth;
1522 };
1523
1524 struct tu_subpass_attachment
1525 {
1526    uint32_t attachment;
1527    VkImageLayout layout;
1528 };
1529
1530 struct tu_subpass
1531 {
1532    uint32_t input_count;
1533    uint32_t color_count;
1534    struct tu_subpass_attachment *input_attachments;
1535    struct tu_subpass_attachment *color_attachments;
1536    struct tu_subpass_attachment *resolve_attachments;
1537    struct tu_subpass_attachment depth_stencil_attachment;
1538
1539    VkSampleCountFlagBits samples;
1540    bool has_external_src, has_external_dst;
1541
1542    uint32_t srgb_cntl;
1543
1544    struct tu_subpass_barrier start_barrier;
1545 };
1546
1547 struct tu_render_pass_attachment
1548 {
1549    VkFormat format;
1550    uint32_t samples;
1551    uint32_t cpp;
1552    VkImageAspectFlags clear_mask;
1553    bool load;
1554    bool store;
1555    VkImageLayout initial_layout, final_layout;
1556    int32_t gmem_offset;
1557 };
1558
1559 struct tu_render_pass
1560 {
1561    uint32_t attachment_count;
1562    uint32_t subpass_count;
1563    uint32_t gmem_pixels;
1564    uint32_t tile_align_w;
1565    struct tu_subpass_attachment *subpass_attachments;
1566    struct tu_render_pass_attachment *attachments;
1567    struct tu_subpass_barrier end_barrier;
1568    struct tu_subpass subpasses[0];
1569 };
1570
1571 struct tu_query_pool
1572 {
1573    VkQueryType type;
1574    uint32_t stride;
1575    uint64_t size;
1576    uint32_t pipeline_statistics;
1577    struct tu_bo bo;
1578 };
1579
1580 struct tu_semaphore
1581 {
1582    uint32_t syncobj;
1583    uint32_t temp_syncobj;
1584 };
1585
1586 void
1587 tu_set_descriptor_set(struct tu_cmd_buffer *cmd_buffer,
1588                       VkPipelineBindPoint bind_point,
1589                       struct tu_descriptor_set *set,
1590                       unsigned idx);
1591
1592 void
1593 tu_update_descriptor_sets(struct tu_device *device,
1594                           struct tu_cmd_buffer *cmd_buffer,
1595                           VkDescriptorSet overrideSet,
1596                           uint32_t descriptorWriteCount,
1597                           const VkWriteDescriptorSet *pDescriptorWrites,
1598                           uint32_t descriptorCopyCount,
1599                           const VkCopyDescriptorSet *pDescriptorCopies);
1600
1601 void
1602 tu_update_descriptor_set_with_template(
1603    struct tu_device *device,
1604    struct tu_cmd_buffer *cmd_buffer,
1605    struct tu_descriptor_set *set,
1606    VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1607    const void *pData);
1608
1609 int
1610 tu_drm_get_gpu_id(const struct tu_physical_device *dev, uint32_t *id);
1611
1612 int
1613 tu_drm_get_gmem_size(const struct tu_physical_device *dev, uint32_t *size);
1614
1615 int
1616 tu_drm_get_gmem_base(const struct tu_physical_device *dev, uint64_t *base);
1617
1618 int
1619 tu_drm_submitqueue_new(const struct tu_device *dev,
1620                        int priority,
1621                        uint32_t *queue_id);
1622
1623 void
1624 tu_drm_submitqueue_close(const struct tu_device *dev, uint32_t queue_id);
1625
1626 uint32_t
1627 tu_gem_new(const struct tu_device *dev, uint64_t size, uint32_t flags);
1628 uint32_t
1629 tu_gem_import_dmabuf(const struct tu_device *dev,
1630                      int prime_fd,
1631                      uint64_t size);
1632 int
1633 tu_gem_export_dmabuf(const struct tu_device *dev, uint32_t gem_handle);
1634 void
1635 tu_gem_close(const struct tu_device *dev, uint32_t gem_handle);
1636 uint64_t
1637 tu_gem_info_offset(const struct tu_device *dev, uint32_t gem_handle);
1638 uint64_t
1639 tu_gem_info_iova(const struct tu_device *dev, uint32_t gem_handle);
1640
1641 #define TU_DEFINE_HANDLE_CASTS(__tu_type, __VkType)                          \
1642                                                                              \
1643    static inline struct __tu_type *__tu_type##_from_handle(__VkType _handle) \
1644    {                                                                         \
1645       return (struct __tu_type *) _handle;                                   \
1646    }                                                                         \
1647                                                                              \
1648    static inline __VkType __tu_type##_to_handle(struct __tu_type *_obj)      \
1649    {                                                                         \
1650       return (__VkType) _obj;                                                \
1651    }
1652
1653 #define TU_DEFINE_NONDISP_HANDLE_CASTS(__tu_type, __VkType)                  \
1654                                                                              \
1655    static inline struct __tu_type *__tu_type##_from_handle(__VkType _handle) \
1656    {                                                                         \
1657       return (struct __tu_type *) (uintptr_t) _handle;                       \
1658    }                                                                         \
1659                                                                              \
1660    static inline __VkType __tu_type##_to_handle(struct __tu_type *_obj)      \
1661    {                                                                         \
1662       return (__VkType)(uintptr_t) _obj;                                     \
1663    }
1664
1665 #define TU_FROM_HANDLE(__tu_type, __name, __handle)                          \
1666    struct __tu_type *__name = __tu_type##_from_handle(__handle)
1667
1668 TU_DEFINE_HANDLE_CASTS(tu_cmd_buffer, VkCommandBuffer)
1669 TU_DEFINE_HANDLE_CASTS(tu_device, VkDevice)
1670 TU_DEFINE_HANDLE_CASTS(tu_instance, VkInstance)
1671 TU_DEFINE_HANDLE_CASTS(tu_physical_device, VkPhysicalDevice)
1672 TU_DEFINE_HANDLE_CASTS(tu_queue, VkQueue)
1673
1674 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_cmd_pool, VkCommandPool)
1675 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_buffer, VkBuffer)
1676 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_buffer_view, VkBufferView)
1677 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_pool, VkDescriptorPool)
1678 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_set, VkDescriptorSet)
1679 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_set_layout,
1680                                VkDescriptorSetLayout)
1681 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_update_template,
1682                                VkDescriptorUpdateTemplate)
1683 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_device_memory, VkDeviceMemory)
1684 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_fence, VkFence)
1685 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_event, VkEvent)
1686 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_framebuffer, VkFramebuffer)
1687 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_image, VkImage)
1688 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_image_view, VkImageView);
1689 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline_cache, VkPipelineCache)
1690 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline, VkPipeline)
1691 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline_layout, VkPipelineLayout)
1692 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_query_pool, VkQueryPool)
1693 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_render_pass, VkRenderPass)
1694 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_sampler, VkSampler)
1695 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_sampler_ycbcr_conversion, VkSamplerYcbcrConversion)
1696 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_shader_module, VkShaderModule)
1697 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_semaphore, VkSemaphore)
1698
1699 #endif /* TU_PRIVATE_H */