src/freedreno/vulkan/tu_private.h

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  25  * DEALINGS IN THE SOFTWARE.
  26  */
  27
  28 #ifndef TU_PRIVATE_H
  29 #define TU_PRIVATE_H
  30
  31 #include <assert.h>
  32 #include <pthread.h>
  33 #include <stdbool.h>
  34 #include <stdint.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #ifdef HAVE_VALGRIND
  39 #include <memcheck.h>
  40 #include <valgrind.h>
  41 #define VG(x) x
  42 #else
  43 #define VG(x) ((void)0)
  44 #endif
  45
  46 #include "c11/threads.h"
  47 #include "main/macros.h"
  48 #include "util/list.h"
  49 #include "util/macros.h"
  50 #include "vk_alloc.h"
  51 #include "vk_debug_report.h"
  52 #include "wsi_common.h"
  53
  54 #include "drm-uapi/msm_drm.h"
  55 #include "ir3/ir3_compiler.h"
  56 #include "ir3/ir3_shader.h"
  57
  58 #include "adreno_common.xml.h"
  59 #include "adreno_pm4.xml.h"
  60 #include "a6xx.xml.h"
  61 #include "fdl/freedreno_layout.h"
  62
  63 #include "tu_descriptor_set.h"
  64 #include "tu_extensions.h"
  65
  66 /* Pre-declarations needed for WSI entrypoints */
  67 struct wl_surface;
  68 struct wl_display;
  69 typedef struct xcb_connection_t xcb_connection_t;
  70 typedef uint32_t xcb_visualid_t;
  71 typedef uint32_t xcb_window_t;
  72
  73 #include <vulkan/vk_android_native_buffer.h>
  74 #include <vulkan/vk_icd.h>
  75 #include <vulkan/vulkan.h>
  76 #include <vulkan/vulkan_intel.h>
  77
  78 #include "tu_entrypoints.h"
  79
  80 #include "vk_format.h"
  81
  82 #define MAX_VBS 32
  83 #define MAX_VERTEX_ATTRIBS 32
  84 #define MAX_RTS 8
  85 #define MAX_VSC_PIPES 32
  86 #define MAX_VIEWPORTS 1
  87 #define MAX_SCISSORS 16
  88 #define MAX_DISCARD_RECTANGLES 4
  89 #define MAX_PUSH_CONSTANTS_SIZE 128
  90 #define MAX_PUSH_DESCRIPTORS 32
  91 #define MAX_DYNAMIC_UNIFORM_BUFFERS 16
  92 #define MAX_DYNAMIC_STORAGE_BUFFERS 8
  93 #define MAX_DYNAMIC_BUFFERS                                                  \
  94    (MAX_DYNAMIC_UNIFORM_BUFFERS + MAX_DYNAMIC_STORAGE_BUFFERS)
  95 #define TU_MAX_DRM_DEVICES 8
  96 #define MAX_VIEWS 8
  97 #define MAX_BIND_POINTS 2 /* compute + graphics */
  98 /* The Qualcomm driver exposes 0x20000058 */
  99 #define MAX_STORAGE_BUFFER_RANGE 0x20000000
 100 /* We use ldc for uniform buffer loads, just like the Qualcomm driver, so
 101  * expose the same maximum range.
 102  * TODO: The SIZE bitfield is 15 bits, and in 4-dword units, so the actual
 103  * range might be higher.
 104  */
 105 #define MAX_UNIFORM_BUFFER_RANGE 0x10000
 106
 107 #define A6XX_TEX_CONST_DWORDS 16
 108 #define A6XX_TEX_SAMP_DWORDS 4
 109
 110 #define tu_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
 111
 112 static inline uint32_t
 113 tu_minify(uint32_t n, uint32_t levels)
 114 {
 115    if (unlikely(n == 0))
 116       return 0;
 117    else
 118       return MAX2(n >> levels, 1);
 119 }
 120
 121 #define for_each_bit(b, dword)                                               \
 122    for (uint32_t __dword = (dword);                                          \
 123         (b) = __builtin_ffs(__dword) - 1, __dword; __dword &= ~(1 << (b)))
 124
 125 #define typed_memcpy(dest, src, count)                                       \
 126    ({                                                                        \
 127       STATIC_ASSERT(sizeof(*src) == sizeof(*dest));                          \
 128       memcpy((dest), (src), (count) * sizeof(*(src)));                       \
 129    })
 130
 131 #define COND(bool, val) ((bool) ? (val) : 0)
 132 #define BIT(bit) (1u << (bit))
 133
 134 /* Whenever we generate an error, pass it through this function. Useful for
 135  * debugging, where we can break on it. Only call at error site, not when
 136  * propagating errors. Might be useful to plug in a stack trace here.
 137  */
 138
 139 struct tu_instance;
 140
 141 VkResult
 142 __vk_errorf(struct tu_instance *instance,
 143             VkResult error,
 144             const char *file,
 145             int line,
 146             const char *format,
 147             ...);
 148
 149 #define vk_error(instance, error)                                            \
 150    __vk_errorf(instance, error, __FILE__, __LINE__, NULL);
 151 #define vk_errorf(instance, error, format, ...)                              \
 152    __vk_errorf(instance, error, __FILE__, __LINE__, format, ##__VA_ARGS__);
 153
 154 void
 155 __tu_finishme(const char *file, int line, const char *format, ...)
 156    tu_printflike(3, 4);
 157 void
 158 tu_loge(const char *format, ...) tu_printflike(1, 2);
 159 void
 160 tu_logi(const char *format, ...) tu_printflike(1, 2);
 161
 162 /**
 163  * Print a FINISHME message, including its source location.
 164  */
 165 #define tu_finishme(format, ...)                                             \
 166    do {                                                                      \
 167       static bool reported = false;                                          \
 168       if (!reported) {                                                       \
 169          __tu_finishme(__FILE__, __LINE__, format, ##__VA_ARGS__);           \
 170          reported = true;                                                    \
 171       }                                                                      \
 172    } while (0)
 173
 174 #define tu_stub()                                                            \
 175    do {                                                                      \
 176       tu_finishme("stub %s", __func__);                                      \
 177    } while (0)
 178
 179 void *
 180 tu_lookup_entrypoint_unchecked(const char *name);
 181 void *
 182 tu_lookup_entrypoint_checked(
 183    const char *name,
 184    uint32_t core_version,
 185    const struct tu_instance_extension_table *instance,
 186    const struct tu_device_extension_table *device);
 187
 188 struct tu_physical_device
 189 {
 190    VK_LOADER_DATA _loader_data;
 191
 192    struct tu_instance *instance;
 193
 194    char path[20];
 195    char name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE];
 196    uint8_t driver_uuid[VK_UUID_SIZE];
 197    uint8_t device_uuid[VK_UUID_SIZE];
 198    uint8_t cache_uuid[VK_UUID_SIZE];
 199
 200    struct wsi_device wsi_device;
 201
 202    int local_fd;
 203    int master_fd;
 204
 205    unsigned gpu_id;
 206    uint32_t gmem_size;
 207    uint64_t gmem_base;
 208    uint32_t ccu_offset_gmem;
 209    uint32_t ccu_offset_bypass;
 210    /* alignment for size of tiles */
 211    uint32_t tile_align_w;
 212 #define TILE_ALIGN_H 16
 213    /* gmem store/load granularity */
 214 #define GMEM_ALIGN_W 16
 215 #define GMEM_ALIGN_H 4
 216
 217    struct {
 218       uint32_t PC_UNKNOWN_9805;
 219       uint32_t SP_UNKNOWN_A0F8;
 220    } magic;
 221
 222    /* This is the drivers on-disk cache used as a fallback as opposed to
 223     * the pipeline cache defined by apps.
 224     */
 225    struct disk_cache *disk_cache;
 226
 227    struct tu_device_extension_table supported_extensions;
 228 };
 229
 230 enum tu_debug_flags
 231 {
 232    TU_DEBUG_STARTUP = 1 << 0,
 233    TU_DEBUG_NIR = 1 << 1,
 234    TU_DEBUG_IR3 = 1 << 2,
 235    TU_DEBUG_NOBIN = 1 << 3,
 236    TU_DEBUG_SYSMEM = 1 << 4,
 237    TU_DEBUG_FORCEBIN = 1 << 5,
 238    TU_DEBUG_NOUBWC = 1 << 6,
 239 };
 240
 241 struct tu_instance
 242 {
 243    VK_LOADER_DATA _loader_data;
 244
 245    VkAllocationCallbacks alloc;
 246
 247    uint32_t api_version;
 248    int physical_device_count;
 249    struct tu_physical_device physical_devices[TU_MAX_DRM_DEVICES];
 250
 251    enum tu_debug_flags debug_flags;
 252
 253    struct vk_debug_report_instance debug_report_callbacks;
 254
 255    struct tu_instance_extension_table enabled_extensions;
 256 };
 257
 258 VkResult
 259 tu_wsi_init(struct tu_physical_device *physical_device);
 260 void
 261 tu_wsi_finish(struct tu_physical_device *physical_device);
 262
 263 bool
 264 tu_instance_extension_supported(const char *name);
 265 uint32_t
 266 tu_physical_device_api_version(struct tu_physical_device *dev);
 267 bool
 268 tu_physical_device_extension_supported(struct tu_physical_device *dev,
 269                                        const char *name);
 270
 271 struct cache_entry;
 272
 273 struct tu_pipeline_cache
 274 {
 275    struct tu_device *device;
 276    pthread_mutex_t mutex;
 277
 278    uint32_t total_size;
 279    uint32_t table_size;
 280    uint32_t kernel_count;
 281    struct cache_entry **hash_table;
 282    bool modified;
 283
 284    VkAllocationCallbacks alloc;
 285 };
 286
 287 struct tu_pipeline_key
 288 {
 289 };
 290
 291
 292 /* queue types */
 293 #define TU_QUEUE_GENERAL 0
 294
 295 #define TU_MAX_QUEUE_FAMILIES 1
 296
 297 struct tu_fence
 298 {
 299    struct wsi_fence *fence_wsi;
 300    bool signaled;
 301    int fd;
 302 };
 303
 304 void
 305 tu_fence_init(struct tu_fence *fence, bool signaled);
 306 void
 307 tu_fence_finish(struct tu_fence *fence);
 308 void
 309 tu_fence_update_fd(struct tu_fence *fence, int fd);
 310 void
 311 tu_fence_copy(struct tu_fence *fence, const struct tu_fence *src);
 312 void
 313 tu_fence_signal(struct tu_fence *fence);
 314 void
 315 tu_fence_wait_idle(struct tu_fence *fence);
 316
 317 struct tu_queue
 318 {
 319    VK_LOADER_DATA _loader_data;
 320    struct tu_device *device;
 321    uint32_t queue_family_index;
 322    int queue_idx;
 323    VkDeviceQueueCreateFlags flags;
 324
 325    uint32_t msm_queue_id;
 326    struct tu_fence submit_fence;
 327 };
 328
 329 struct tu_bo
 330 {
 331    uint32_t gem_handle;
 332    uint64_t size;
 333    uint64_t iova;
 334    void *map;
 335 };
 336
 337 struct tu_device
 338 {
 339    VK_LOADER_DATA _loader_data;
 340
 341    VkAllocationCallbacks alloc;
 342
 343    struct tu_instance *instance;
 344
 345    struct tu_queue *queues[TU_MAX_QUEUE_FAMILIES];
 346    int queue_count[TU_MAX_QUEUE_FAMILIES];
 347
 348    struct tu_physical_device *physical_device;
 349
 350    struct ir3_compiler *compiler;
 351
 352    /* Backup in-memory cache to be used if the app doesn't provide one */
 353    struct tu_pipeline_cache *mem_cache;
 354
 355    struct tu_bo vsc_draw_strm;
 356    struct tu_bo vsc_prim_strm;
 357    uint32_t vsc_draw_strm_pitch;
 358    uint32_t vsc_prim_strm_pitch;
 359
 360 #define MIN_SCRATCH_BO_SIZE_LOG2 12 /* A page */
 361
 362    /* Currently the kernel driver uses a 32-bit GPU address space, but it
 363     * should be impossible to go beyond 48 bits.
 364     */
 365    struct {
 366       struct tu_bo bo;
 367       mtx_t construct_mtx;
 368       bool initialized;
 369    } scratch_bos[48 - MIN_SCRATCH_BO_SIZE_LOG2];
 370
 371    struct tu_bo border_color;
 372
 373    struct tu_device_extension_table enabled_extensions;
 374 };
 375
 376 VkResult
 377 tu_bo_init_new(struct tu_device *dev, struct tu_bo *bo, uint64_t size);
 378 VkResult
 379 tu_bo_init_dmabuf(struct tu_device *dev,
 380                   struct tu_bo *bo,
 381                   uint64_t size,
 382                   int fd);
 383 int
 384 tu_bo_export_dmabuf(struct tu_device *dev, struct tu_bo *bo);
 385 void
 386 tu_bo_finish(struct tu_device *dev, struct tu_bo *bo);
 387 VkResult
 388 tu_bo_map(struct tu_device *dev, struct tu_bo *bo);
 389
 390 /* Get a scratch bo for use inside a command buffer. This will always return
 391  * the same bo given the same size or similar sizes, so only one scratch bo
 392  * can be used at the same time. It's meant for short-lived things where we
 393  * need to write to some piece of memory, read from it, and then immediately
 394  * discard it.
 395  */
 396 VkResult
 397 tu_get_scratch_bo(struct tu_device *dev, uint64_t size, struct tu_bo **bo);
 398
 399 struct tu_cs_entry
 400 {
 401    /* No ownership */
 402    const struct tu_bo *bo;
 403
 404    uint32_t size;
 405    uint32_t offset;
 406 };
 407
 408 struct ts_cs_memory {
 409    uint32_t *map;
 410    uint64_t iova;
 411 };
 412
 413 struct tu_draw_state {
 414    uint64_t iova : 48;
 415    uint32_t size : 16;
 416 };
 417
 418 enum tu_dynamic_state
 419 {
 420    /* re-use VK_DYNAMIC_STATE_ enums for non-extended dynamic states */
 421    TU_DYNAMIC_STATE_SAMPLE_LOCATIONS = VK_DYNAMIC_STATE_STENCIL_REFERENCE + 1,
 422    TU_DYNAMIC_STATE_COUNT,
 423 };
 424
 425 enum tu_draw_state_group_id
 426 {
 427    TU_DRAW_STATE_PROGRAM,
 428    TU_DRAW_STATE_PROGRAM_BINNING,
 429    TU_DRAW_STATE_VB,
 430    TU_DRAW_STATE_VI,
 431    TU_DRAW_STATE_VI_BINNING,
 432    TU_DRAW_STATE_RAST,
 433    TU_DRAW_STATE_DS,
 434    TU_DRAW_STATE_BLEND,
 435    TU_DRAW_STATE_VS_CONST,
 436    TU_DRAW_STATE_GS_CONST,
 437    TU_DRAW_STATE_FS_CONST,
 438    TU_DRAW_STATE_DESC_SETS,
 439    TU_DRAW_STATE_DESC_SETS_LOAD,
 440    TU_DRAW_STATE_VS_PARAMS,
 441    TU_DRAW_STATE_INPUT_ATTACHMENTS_GMEM,
 442    TU_DRAW_STATE_INPUT_ATTACHMENTS_SYSMEM,
 443
 444    /* dynamic state related draw states */
 445    TU_DRAW_STATE_DYNAMIC,
 446    TU_DRAW_STATE_COUNT = TU_DRAW_STATE_DYNAMIC + TU_DYNAMIC_STATE_COUNT,
 447 };
 448
 449 enum tu_cs_mode
 450 {
 451
 452    /*
 453     * A command stream in TU_CS_MODE_GROW mode grows automatically whenever it
 454     * is full.  tu_cs_begin must be called before command packet emission and
 455     * tu_cs_end must be called after.
 456     *
 457     * This mode may create multiple entries internally.  The entries must be
 458     * submitted together.
 459     */
 460    TU_CS_MODE_GROW,
 461
 462    /*
 463     * A command stream in TU_CS_MODE_EXTERNAL mode wraps an external,
 464     * fixed-size buffer.  tu_cs_begin and tu_cs_end are optional and have no
 465     * effect on it.
 466     *
 467     * This mode does not create any entry or any BO.
 468     */
 469    TU_CS_MODE_EXTERNAL,
 470
 471    /*
 472     * A command stream in TU_CS_MODE_SUB_STREAM mode does not support direct
 473     * command packet emission.  tu_cs_begin_sub_stream must be called to get a
 474     * sub-stream to emit comamnd packets to.  When done with the sub-stream,
 475     * tu_cs_end_sub_stream must be called.
 476     *
 477     * This mode does not create any entry internally.
 478     */
 479    TU_CS_MODE_SUB_STREAM,
 480 };
 481
 482 struct tu_cs
 483 {
 484    uint32_t *start;
 485    uint32_t *cur;
 486    uint32_t *reserved_end;
 487    uint32_t *end;
 488
 489    struct tu_device *device;
 490    enum tu_cs_mode mode;
 491    uint32_t next_bo_size;
 492
 493    struct tu_cs_entry *entries;
 494    uint32_t entry_count;
 495    uint32_t entry_capacity;
 496
 497    struct tu_bo **bos;
 498    uint32_t bo_count;
 499    uint32_t bo_capacity;
 500
 501    /* state for cond_exec_start/cond_exec_end */
 502    uint32_t cond_flags;
 503    uint32_t *cond_dwords;
 504 };
 505
 506 struct tu_device_memory
 507 {
 508    struct tu_bo bo;
 509    VkDeviceSize size;
 510
 511    /* for dedicated allocations */
 512    struct tu_image *image;
 513    struct tu_buffer *buffer;
 514
 515    uint32_t type_index;
 516    void *map;
 517    void *user_ptr;
 518 };
 519
 520 struct tu_descriptor_range
 521 {
 522    uint64_t va;
 523    uint32_t size;
 524 };
 525
 526 struct tu_descriptor_set
 527 {
 528    const struct tu_descriptor_set_layout *layout;
 529    struct tu_descriptor_pool *pool;
 530    uint32_t size;
 531
 532    uint64_t va;
 533    uint32_t *mapped_ptr;
 534
 535    uint32_t *dynamic_descriptors;
 536
 537    struct tu_bo *buffers[0];
 538 };
 539
 540 struct tu_push_descriptor_set
 541 {
 542    struct tu_descriptor_set set;
 543    uint32_t capacity;
 544 };
 545
 546 struct tu_descriptor_pool_entry
 547 {
 548    uint32_t offset;
 549    uint32_t size;
 550    struct tu_descriptor_set *set;
 551 };
 552
 553 struct tu_descriptor_pool
 554 {
 555    struct tu_bo bo;
 556    uint64_t current_offset;
 557    uint64_t size;
 558
 559    uint8_t *host_memory_base;
 560    uint8_t *host_memory_ptr;
 561    uint8_t *host_memory_end;
 562
 563    uint32_t entry_count;
 564    uint32_t max_entry_count;
 565    struct tu_descriptor_pool_entry entries[0];
 566 };
 567
 568 struct tu_descriptor_update_template_entry
 569 {
 570    VkDescriptorType descriptor_type;
 571
 572    /* The number of descriptors to update */
 573    uint32_t descriptor_count;
 574
 575    /* Into mapped_ptr or dynamic_descriptors, in units of the respective array
 576     */
 577    uint32_t dst_offset;
 578
 579    /* In dwords. Not valid/used for dynamic descriptors */
 580    uint32_t dst_stride;
 581
 582    uint32_t buffer_offset;
 583
 584    /* Only valid for combined image samplers and samplers */
 585    uint16_t has_sampler;
 586
 587    /* In bytes */
 588    size_t src_offset;
 589    size_t src_stride;
 590
 591    /* For push descriptors */
 592    const uint32_t *immutable_samplers;
 593 };
 594
 595 struct tu_descriptor_update_template
 596 {
 597    uint32_t entry_count;
 598    struct tu_descriptor_update_template_entry entry[0];
 599 };
 600
 601 struct tu_buffer
 602 {
 603    VkDeviceSize size;
 604
 605    VkBufferUsageFlags usage;
 606    VkBufferCreateFlags flags;
 607
 608    struct tu_bo *bo;
 609    VkDeviceSize bo_offset;
 610 };
 611
 612 static inline uint64_t
 613 tu_buffer_iova(struct tu_buffer *buffer)
 614 {
 615    return buffer->bo->iova + buffer->bo_offset;
 616 }
 617
 618 struct tu_vertex_binding
 619 {
 620    struct tu_buffer *buffer;
 621    VkDeviceSize offset;
 622 };
 623
 624 const char *
 625 tu_get_debug_option_name(int id);
 626
 627 const char *
 628 tu_get_perftest_option_name(int id);
 629
 630 struct tu_descriptor_state
 631 {
 632    struct tu_descriptor_set *sets[MAX_SETS];
 633    uint32_t dynamic_descriptors[MAX_DYNAMIC_BUFFERS * A6XX_TEX_CONST_DWORDS];
 634 };
 635
 636 struct tu_tile
 637 {
 638    uint8_t pipe;
 639    uint8_t slot;
 640    VkOffset2D begin;
 641    VkOffset2D end;
 642 };
 643
 644 struct tu_tiling_config
 645 {
 646    VkRect2D render_area;
 647
 648    /* position and size of the first tile */
 649    VkRect2D tile0;
 650    /* number of tiles */
 651    VkExtent2D tile_count;
 652
 653    /* size of the first VSC pipe */
 654    VkExtent2D pipe0;
 655    /* number of VSC pipes */
 656    VkExtent2D pipe_count;
 657
 658    /* pipe register values */
 659    uint32_t pipe_config[MAX_VSC_PIPES];
 660    uint32_t pipe_sizes[MAX_VSC_PIPES];
 661
 662    /* Whether sysmem rendering must be used */
 663    bool force_sysmem;
 664 };
 665
 666 enum tu_cmd_dirty_bits
 667 {
 668    TU_CMD_DIRTY_COMPUTE_PIPELINE = 1 << 1,
 669    TU_CMD_DIRTY_VERTEX_BUFFERS = 1 << 2,
 670    TU_CMD_DIRTY_DESCRIPTOR_SETS = 1 << 3,
 671    TU_CMD_DIRTY_COMPUTE_DESCRIPTOR_SETS = 1 << 4,
 672    TU_CMD_DIRTY_SHADER_CONSTS = 1 << 5,
 673    TU_CMD_DIRTY_STREAMOUT_BUFFERS = 1 << 6,
 674    /* all draw states were disabled and need to be re-enabled: */
 675    TU_CMD_DIRTY_DRAW_STATE = 1 << 7,
 676 };
 677
 678 struct tu_streamout_state {
 679    uint16_t stride[IR3_MAX_SO_BUFFERS];
 680    uint32_t ncomp[IR3_MAX_SO_BUFFERS];
 681    uint32_t prog[IR3_MAX_SO_OUTPUTS * 2];
 682    uint32_t prog_count;
 683    uint32_t vpc_so_buf_cntl;
 684 };
 685
 686 /* There are only three cache domains we have to care about: the CCU, or
 687  * color cache unit, which is used for color and depth/stencil attachments
 688  * and copy/blit destinations, and is split conceptually into color and depth,
 689  * and the universal cache or UCHE which is used for pretty much everything
 690  * else, except for the CP (uncached) and host. We need to flush whenever data
 691  * crosses these boundaries.
 692  */
 693
 694 enum tu_cmd_access_mask {
 695    TU_ACCESS_UCHE_READ = 1 << 0,
 696    TU_ACCESS_UCHE_WRITE = 1 << 1,
 697    TU_ACCESS_CCU_COLOR_READ = 1 << 2,
 698    TU_ACCESS_CCU_COLOR_WRITE = 1 << 3,
 699    TU_ACCESS_CCU_DEPTH_READ = 1 << 4,
 700    TU_ACCESS_CCU_DEPTH_WRITE = 1 << 5,
 701
 702    /* Experiments have shown that while it's safe to avoid flushing the CCU
 703     * after each blit/renderpass, it's not safe to assume that subsequent
 704     * lookups with a different attachment state will hit unflushed cache
 705     * entries. That is, the CCU needs to be flushed and possibly invalidated
 706     * when accessing memory with a different attachment state. Writing to an
 707     * attachment under the following conditions after clearing using the
 708     * normal 2d engine path is known to have issues:
 709     *
 710     * - It isn't the 0'th layer.
 711     * - There are more than one attachment, and this isn't the 0'th attachment
 712     *   (this seems to also depend on the cpp of the attachments).
 713     *
 714     * Our best guess is that the layer/MRT state is used when computing
 715     * the location of a cache entry in CCU, to avoid conflicts. We assume that
 716     * any access in a renderpass after or before an access by a transfer needs
 717     * a flush/invalidate, and use the _INCOHERENT variants to represent access
 718     * by a transfer.
 719     */
 720    TU_ACCESS_CCU_COLOR_INCOHERENT_READ = 1 << 6,
 721    TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE = 1 << 7,
 722    TU_ACCESS_CCU_DEPTH_INCOHERENT_READ = 1 << 8,
 723    TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE = 1 << 9,
 724
 725    TU_ACCESS_SYSMEM_READ = 1 << 10,
 726    TU_ACCESS_SYSMEM_WRITE = 1 << 11,
 727
 728    /* Set if a WFI is required due to data being read by the CP or the 2D
 729     * engine.
 730     */
 731    TU_ACCESS_WFI_READ = 1 << 12,
 732
 733    TU_ACCESS_READ =
 734       TU_ACCESS_UCHE_READ |
 735       TU_ACCESS_CCU_COLOR_READ |
 736       TU_ACCESS_CCU_DEPTH_READ |
 737       TU_ACCESS_CCU_COLOR_INCOHERENT_READ |
 738       TU_ACCESS_CCU_DEPTH_INCOHERENT_READ |
 739       TU_ACCESS_SYSMEM_READ,
 740
 741    TU_ACCESS_WRITE =
 742       TU_ACCESS_UCHE_WRITE |
 743       TU_ACCESS_CCU_COLOR_WRITE |
 744       TU_ACCESS_CCU_COLOR_INCOHERENT_WRITE |
 745       TU_ACCESS_CCU_DEPTH_WRITE |
 746       TU_ACCESS_CCU_DEPTH_INCOHERENT_WRITE |
 747       TU_ACCESS_SYSMEM_WRITE,
 748
 749    TU_ACCESS_ALL =
 750       TU_ACCESS_READ |
 751       TU_ACCESS_WRITE,
 752 };
 753
 754 enum tu_cmd_flush_bits {
 755    TU_CMD_FLAG_CCU_FLUSH_DEPTH = 1 << 0,
 756    TU_CMD_FLAG_CCU_FLUSH_COLOR = 1 << 1,
 757    TU_CMD_FLAG_CCU_INVALIDATE_DEPTH = 1 << 2,
 758    TU_CMD_FLAG_CCU_INVALIDATE_COLOR = 1 << 3,
 759    TU_CMD_FLAG_CACHE_FLUSH = 1 << 4,
 760    TU_CMD_FLAG_CACHE_INVALIDATE = 1 << 5,
 761
 762    TU_CMD_FLAG_ALL_FLUSH =
 763       TU_CMD_FLAG_CCU_FLUSH_DEPTH |
 764       TU_CMD_FLAG_CCU_FLUSH_COLOR |
 765       TU_CMD_FLAG_CACHE_FLUSH,
 766
 767    TU_CMD_FLAG_ALL_INVALIDATE =
 768       TU_CMD_FLAG_CCU_INVALIDATE_DEPTH |
 769       TU_CMD_FLAG_CCU_INVALIDATE_COLOR |
 770       TU_CMD_FLAG_CACHE_INVALIDATE,
 771
 772    TU_CMD_FLAG_WFI = 1 << 6,
 773 };
 774
 775 /* Changing the CCU from sysmem mode to gmem mode or vice-versa is pretty
 776  * heavy, involving a CCU cache flush/invalidate and a WFI in order to change
 777  * which part of the gmem is used by the CCU. Here we keep track of what the
 778  * state of the CCU.
 779  */
 780 enum tu_cmd_ccu_state {
 781    TU_CMD_CCU_SYSMEM,
 782    TU_CMD_CCU_GMEM,
 783    TU_CMD_CCU_UNKNOWN,
 784 };
 785
 786 struct tu_cache_state {
 787    /* Caches which must be made available (flushed) eventually if there are
 788     * any users outside that cache domain, and caches which must be
 789     * invalidated eventually if there are any reads.
 790     */
 791    enum tu_cmd_flush_bits pending_flush_bits;
 792    /* Pending flushes */
 793    enum tu_cmd_flush_bits flush_bits;
 794 };
 795
 796 struct tu_cmd_state
 797 {
 798    uint32_t dirty;
 799
 800    struct tu_pipeline *pipeline;
 801    struct tu_pipeline *compute_pipeline;
 802
 803    /* Vertex buffers */
 804    struct
 805    {
 806       struct tu_buffer *buffers[MAX_VBS];
 807       VkDeviceSize offsets[MAX_VBS];
 808    } vb;
 809
 810    /* for dynamic states that can't be emitted directly */
 811    uint32_t dynamic_stencil_mask;
 812    uint32_t dynamic_stencil_wrmask;
 813    uint32_t dynamic_stencil_ref;
 814    uint32_t dynamic_gras_su_cntl;
 815
 816    /* saved states to re-emit in TU_CMD_DIRTY_DRAW_STATE case */
 817    struct tu_draw_state dynamic_state[TU_DYNAMIC_STATE_COUNT];
 818    struct tu_cs_entry vertex_buffers_ib;
 819    struct tu_cs_entry shader_const_ib[MESA_SHADER_STAGES];
 820    struct tu_cs_entry desc_sets_ib, desc_sets_load_ib;
 821    struct tu_cs_entry ia_gmem_ib, ia_sysmem_ib;
 822
 823    /* Stream output buffers */
 824    struct
 825    {
 826       struct tu_buffer *buffers[IR3_MAX_SO_BUFFERS];
 827       VkDeviceSize offsets[IR3_MAX_SO_BUFFERS];
 828       VkDeviceSize sizes[IR3_MAX_SO_BUFFERS];
 829    } streamout_buf;
 830
 831    uint8_t streamout_reset;
 832    uint8_t streamout_enabled;
 833
 834    /* Index buffer */
 835    struct tu_buffer *index_buffer;
 836    uint64_t index_offset;
 837    uint32_t index_type;
 838    uint32_t max_index_count;
 839    uint64_t index_va;
 840
 841    /* Renderpasses are tricky, because we may need to flush differently if
 842     * using sysmem vs. gmem and therefore we have to delay any flushing that
 843     * happens before a renderpass. So we have to have two copies of the flush
 844     * state, one for intra-renderpass flushes (i.e. renderpass dependencies)
 845     * and one for outside a renderpass.
 846     */
 847    struct tu_cache_state cache;
 848    struct tu_cache_state renderpass_cache;
 849
 850    enum tu_cmd_ccu_state ccu_state;
 851
 852    const struct tu_render_pass *pass;
 853    const struct tu_subpass *subpass;
 854    const struct tu_framebuffer *framebuffer;
 855
 856    struct tu_tiling_config tiling_config;
 857
 858    struct tu_cs_entry tile_store_ib;
 859 };
 860
 861 struct tu_cmd_pool
 862 {
 863    VkAllocationCallbacks alloc;
 864    struct list_head cmd_buffers;
 865    struct list_head free_cmd_buffers;
 866    uint32_t queue_family_index;
 867 };
 868
 869 struct tu_cmd_buffer_upload
 870 {
 871    uint8_t *map;
 872    unsigned offset;
 873    uint64_t size;
 874    struct list_head list;
 875 };
 876
 877 enum tu_cmd_buffer_status
 878 {
 879    TU_CMD_BUFFER_STATUS_INVALID,
 880    TU_CMD_BUFFER_STATUS_INITIAL,
 881    TU_CMD_BUFFER_STATUS_RECORDING,
 882    TU_CMD_BUFFER_STATUS_EXECUTABLE,
 883    TU_CMD_BUFFER_STATUS_PENDING,
 884 };
 885
 886 struct tu_bo_list
 887 {
 888    uint32_t count;
 889    uint32_t capacity;
 890    struct drm_msm_gem_submit_bo *bo_infos;
 891 };
 892
 893 #define TU_BO_LIST_FAILED (~0)
 894
 895 void
 896 tu_bo_list_init(struct tu_bo_list *list);
 897 void
 898 tu_bo_list_destroy(struct tu_bo_list *list);
 899 void
 900 tu_bo_list_reset(struct tu_bo_list *list);
 901 uint32_t
 902 tu_bo_list_add(struct tu_bo_list *list,
 903                const struct tu_bo *bo,
 904                uint32_t flags);
 905 VkResult
 906 tu_bo_list_merge(struct tu_bo_list *list, const struct tu_bo_list *other);
 907
 908 /* This struct defines the layout of the scratch_bo */
 909 struct tu6_control
 910 {
 911    uint32_t seqno_dummy;          /* dummy seqno for CP_EVENT_WRITE */
 912    uint32_t _pad0;
 913    volatile uint32_t vsc_overflow;
 914    uint32_t _pad1;
 915    /* flag set from cmdstream when VSC overflow detected: */
 916    uint32_t vsc_scratch;
 917    uint32_t _pad2;
 918    uint32_t _pad3;
 919    uint32_t _pad4;
 920
 921    /* scratch space for VPC_SO[i].FLUSH_BASE_LO/HI, start on 32 byte boundary. */
 922    struct {
 923       uint32_t offset;
 924       uint32_t pad[7];
 925    } flush_base[4];
 926 };
 927
 928 #define ctrl_offset(member) offsetof(struct tu6_control, member)
 929
 930 struct tu_cmd_buffer
 931 {
 932    VK_LOADER_DATA _loader_data;
 933
 934    struct tu_device *device;
 935
 936    struct tu_cmd_pool *pool;
 937    struct list_head pool_link;
 938
 939    VkCommandBufferUsageFlags usage_flags;
 940    VkCommandBufferLevel level;
 941    enum tu_cmd_buffer_status status;
 942
 943    struct tu_cmd_state state;
 944    struct tu_vertex_binding vertex_bindings[MAX_VBS];
 945    uint32_t vertex_bindings_set;
 946    uint32_t queue_family_index;
 947
 948    uint32_t push_constants[MAX_PUSH_CONSTANTS_SIZE / 4];
 949    VkShaderStageFlags push_constant_stages;
 950    struct tu_descriptor_set meta_push_descriptors;
 951
 952    struct tu_descriptor_state descriptors[MAX_BIND_POINTS];
 953
 954    struct tu_cmd_buffer_upload upload;
 955
 956    VkResult record_result;
 957
 958    struct tu_bo_list bo_list;
 959    struct tu_cs cs;
 960    struct tu_cs draw_cs;
 961    struct tu_cs draw_epilogue_cs;
 962    struct tu_cs sub_cs;
 963
 964    struct tu_bo scratch_bo;
 965
 966    struct tu_bo vsc_draw_strm;
 967    struct tu_bo vsc_prim_strm;
 968    uint32_t vsc_draw_strm_pitch;
 969    uint32_t vsc_prim_strm_pitch;
 970    bool use_vsc_data;
 971 };
 972
 973 /* Temporary struct for tracking a register state to be written, used by
 974  * a6xx-pack.h and tu_cs_emit_regs()
 975  */
 976 struct tu_reg_value {
 977    uint32_t reg;
 978    uint64_t value;
 979    bool is_address;
 980    struct tu_bo *bo;
 981    bool bo_write;
 982    uint32_t bo_offset;
 983    uint32_t bo_shift;
 984 };
 985
 986
 987 void tu_emit_cache_flush_renderpass(struct tu_cmd_buffer *cmd_buffer,
 988                                     struct tu_cs *cs);
 989
 990 void tu_emit_cache_flush_ccu(struct tu_cmd_buffer *cmd_buffer,
 991                              struct tu_cs *cs,
 992                              enum tu_cmd_ccu_state ccu_state);
 993
 994 void
 995 tu6_emit_event_write(struct tu_cmd_buffer *cmd,
 996                      struct tu_cs *cs,
 997                      enum vgt_event_type event);
 998
 999 static inline struct tu_descriptor_state *
1000 tu_get_descriptors_state(struct tu_cmd_buffer *cmd_buffer,
1001                          VkPipelineBindPoint bind_point)
1002 {
1003    return &cmd_buffer->descriptors[bind_point];
1004 }
1005
1006 struct tu_event
1007 {
1008    struct tu_bo bo;
1009 };
1010
1011 static inline gl_shader_stage
1012 vk_to_mesa_shader_stage(VkShaderStageFlagBits vk_stage)
1013 {
1014    assert(__builtin_popcount(vk_stage) == 1);
1015    return ffs(vk_stage) - 1;
1016 }
1017
1018 static inline VkShaderStageFlagBits
1019 mesa_to_vk_shader_stage(gl_shader_stage mesa_stage)
1020 {
1021    return (1 << mesa_stage);
1022 }
1023
1024 #define TU_STAGE_MASK ((1 << MESA_SHADER_STAGES) - 1)
1025
1026 #define tu_foreach_stage(stage, stage_bits)                                  \
1027    for (gl_shader_stage stage,                                               \
1028         __tmp = (gl_shader_stage)((stage_bits) &TU_STAGE_MASK);              \
1029         stage = __builtin_ffs(__tmp) - 1, __tmp; __tmp &= ~(1 << (stage)))
1030
1031 uint32_t
1032 tu6_stage2opcode(gl_shader_stage type);
1033 enum a6xx_state_block
1034 tu6_stage2shadersb(gl_shader_stage type);
1035
1036 struct tu_shader_module
1037 {
1038    unsigned char sha1[20];
1039
1040    uint32_t code_size;
1041    const uint32_t *code[0];
1042 };
1043
1044 struct tu_push_constant_range
1045 {
1046    uint32_t lo;
1047    uint32_t count;
1048 };
1049
1050 struct tu_shader
1051 {
1052    struct ir3_shader *ir3_shader;
1053
1054    struct tu_push_constant_range push_consts;
1055    uint8_t active_desc_sets;
1056 };
1057
1058 struct tu_shader *
1059 tu_shader_create(struct tu_device *dev,
1060                  gl_shader_stage stage,
1061                  const VkPipelineShaderStageCreateInfo *stage_info,
1062                  struct tu_pipeline_layout *layout,
1063                  const VkAllocationCallbacks *alloc);
1064
1065 void
1066 tu_shader_destroy(struct tu_device *dev,
1067                   struct tu_shader *shader,
1068                   const VkAllocationCallbacks *alloc);
1069
1070 struct tu_program_descriptor_linkage
1071 {
1072    struct ir3_ubo_analysis_state ubo_state;
1073    struct ir3_const_state const_state;
1074
1075    uint32_t constlen;
1076
1077    struct tu_push_constant_range push_consts;
1078 };
1079
1080 struct tu_pipeline
1081 {
1082    struct tu_cs cs;
1083
1084    struct tu_pipeline_layout *layout;
1085
1086    bool need_indirect_descriptor_sets;
1087    VkShaderStageFlags active_stages;
1088    uint32_t active_desc_sets;
1089
1090    struct tu_streamout_state streamout;
1091
1092    /* mask of enabled dynamic states
1093     * if BIT(i) is set, pipeline->dynamic_state[i] is *NOT* used
1094     */
1095    uint32_t dynamic_state_mask;
1096    struct tu_draw_state dynamic_state[TU_DYNAMIC_STATE_COUNT];
1097
1098    /* gras_su_cntl without line width, used for dynamic line width state */
1099    uint32_t gras_su_cntl;
1100
1101    struct
1102    {
1103       struct tu_bo binary_bo;
1104       struct tu_cs_entry state_ib;
1105       struct tu_cs_entry binning_state_ib;
1106
1107       struct tu_program_descriptor_linkage link[MESA_SHADER_STAGES];
1108    } program;
1109
1110    struct
1111    {
1112       struct tu_cs_entry state_ib;
1113    } load_state;
1114
1115    struct
1116    {
1117       struct tu_cs_entry state_ib;
1118       struct tu_cs_entry binning_state_ib;
1119       uint32_t bindings_used;
1120    } vi;
1121
1122    struct
1123    {
1124       enum pc_di_primtype primtype;
1125       bool primitive_restart;
1126    } ia;
1127
1128    struct
1129    {
1130       struct tu_cs_entry state_ib;
1131    } rast;
1132
1133    struct
1134    {
1135       struct tu_cs_entry state_ib;
1136    } ds;
1137
1138    struct
1139    {
1140       struct tu_cs_entry state_ib;
1141    } blend;
1142
1143    struct
1144    {
1145       uint32_t local_size[3];
1146    } compute;
1147 };
1148
1149 void
1150 tu6_emit_viewport(struct tu_cs *cs, const VkViewport *viewport);
1151
1152 void
1153 tu6_emit_scissor(struct tu_cs *cs, const VkRect2D *scissor);
1154
1155 void
1156 tu6_emit_sample_locations(struct tu_cs *cs, const VkSampleLocationsInfoEXT *samp_loc);
1157
1158 void
1159 tu6_emit_depth_bias(struct tu_cs *cs,
1160                     float constant_factor,
1161                     float clamp,
1162                     float slope_factor);
1163
1164 void tu6_emit_msaa(struct tu_cs *cs, VkSampleCountFlagBits samples);
1165
1166 void tu6_emit_window_scissor(struct tu_cs *cs, uint32_t x1, uint32_t y1, uint32_t x2, uint32_t y2);
1167
1168 void tu6_emit_window_offset(struct tu_cs *cs, uint32_t x1, uint32_t y1);
1169
1170 void
1171 tu6_emit_xs_config(struct tu_cs *cs,
1172                    gl_shader_stage stage,
1173                    const struct ir3_shader_variant *xs,
1174                    uint64_t binary_iova);
1175
1176 void
1177 tu6_emit_vpc(struct tu_cs *cs,
1178              const struct ir3_shader_variant *vs,
1179              const struct ir3_shader_variant *gs,
1180              const struct ir3_shader_variant *fs,
1181              struct tu_streamout_state *tf);
1182
1183 void
1184 tu6_emit_fs_inputs(struct tu_cs *cs, const struct ir3_shader_variant *fs);
1185
1186 struct tu_image_view;
1187
1188 void
1189 tu_resolve_sysmem(struct tu_cmd_buffer *cmd,
1190                   struct tu_cs *cs,
1191                   struct tu_image_view *src,
1192                   struct tu_image_view *dst,
1193                   uint32_t layers,
1194                   const VkRect2D *rect);
1195
1196 void
1197 tu_clear_sysmem_attachment(struct tu_cmd_buffer *cmd,
1198                            struct tu_cs *cs,
1199                            uint32_t a,
1200                            const VkRenderPassBeginInfo *info);
1201
1202 void
1203 tu_clear_gmem_attachment(struct tu_cmd_buffer *cmd,
1204                          struct tu_cs *cs,
1205                          uint32_t a,
1206                          const VkRenderPassBeginInfo *info);
1207
1208 void
1209 tu_load_gmem_attachment(struct tu_cmd_buffer *cmd,
1210                         struct tu_cs *cs,
1211                         uint32_t a,
1212                         bool force_load);
1213
1214 /* expose this function to be able to emit load without checking LOAD_OP */
1215 void
1216 tu_emit_load_gmem_attachment(struct tu_cmd_buffer *cmd, struct tu_cs *cs, uint32_t a);
1217
1218 /* note: gmem store can also resolve */
1219 void
1220 tu_store_gmem_attachment(struct tu_cmd_buffer *cmd,
1221                          struct tu_cs *cs,
1222                          uint32_t a,
1223                          uint32_t gmem_a);
1224
1225 enum tu_supported_formats {
1226    FMT_VERTEX = 1,
1227    FMT_TEXTURE = 2,
1228    FMT_COLOR = 4,
1229 };
1230
1231 struct tu_native_format
1232 {
1233    enum a6xx_format fmt : 8;
1234    enum a3xx_color_swap swap : 8;
1235    enum a6xx_tile_mode tile_mode : 8;
1236    enum tu_supported_formats supported : 8;
1237 };
1238
1239 struct tu_native_format tu6_format_vtx(VkFormat format);
1240 struct tu_native_format tu6_format_color(VkFormat format, enum a6xx_tile_mode tile_mode);
1241 struct tu_native_format tu6_format_texture(VkFormat format, enum a6xx_tile_mode tile_mode);
1242
1243 static inline enum a6xx_format
1244 tu6_base_format(VkFormat format)
1245 {
1246    /* note: tu6_format_color doesn't care about tiling for .fmt field */
1247    return tu6_format_color(format, TILE6_LINEAR).fmt;
1248 }
1249
1250 enum a6xx_depth_format tu6_pipe2depth(VkFormat format);
1251
1252 struct tu_image
1253 {
1254    VkImageType type;
1255    /* The original VkFormat provided by the client.  This may not match any
1256     * of the actual surface formats.
1257     */
1258    VkFormat vk_format;
1259    VkImageAspectFlags aspects;
1260    VkImageUsageFlags usage;  /**< Superset of VkImageCreateInfo::usage. */
1261    VkImageTiling tiling;     /** VkImageCreateInfo::tiling */
1262    VkImageCreateFlags flags; /** VkImageCreateInfo::flags */
1263    VkExtent3D extent;
1264    uint32_t level_count;
1265    uint32_t layer_count;
1266    VkSampleCountFlagBits samples;
1267
1268    struct fdl_layout layout;
1269
1270    unsigned queue_family_mask;
1271    bool exclusive;
1272    bool shareable;
1273
1274    /* For VK_ANDROID_native_buffer, the WSI image owns the memory, */
1275    VkDeviceMemory owned_memory;
1276
1277    /* Set when bound */
1278    struct tu_bo *bo;
1279    VkDeviceSize bo_offset;
1280 };
1281
1282 static inline uint32_t
1283 tu_get_layerCount(const struct tu_image *image,
1284                   const VkImageSubresourceRange *range)
1285 {
1286    return range->layerCount == VK_REMAINING_ARRAY_LAYERS
1287              ? image->layer_count - range->baseArrayLayer
1288              : range->layerCount;
1289 }
1290
1291 static inline uint32_t
1292 tu_get_levelCount(const struct tu_image *image,
1293                   const VkImageSubresourceRange *range)
1294 {
1295    return range->levelCount == VK_REMAINING_MIP_LEVELS
1296              ? image->level_count - range->baseMipLevel
1297              : range->levelCount;
1298 }
1299
1300 enum a3xx_msaa_samples
1301 tu_msaa_samples(uint32_t samples);
1302
1303 struct tu_image_view
1304 {
1305    struct tu_image *image; /**< VkImageViewCreateInfo::image */
1306
1307    uint64_t base_addr;
1308    uint64_t ubwc_addr;
1309    uint32_t layer_size;
1310    uint32_t ubwc_layer_size;
1311
1312    /* used to determine if fast gmem store path can be used */
1313    VkExtent2D extent;
1314    bool need_y2_align;
1315
1316    bool ubwc_enabled;
1317
1318    uint32_t descriptor[A6XX_TEX_CONST_DWORDS];
1319
1320    /* Descriptor for use as a storage image as opposed to a sampled image.
1321     * This has a few differences for cube maps (e.g. type).
1322     */
1323    uint32_t storage_descriptor[A6XX_TEX_CONST_DWORDS];
1324
1325    /* pre-filled register values */
1326    uint32_t PITCH;
1327    uint32_t FLAG_BUFFER_PITCH;
1328
1329    uint32_t RB_MRT_BUF_INFO;
1330    uint32_t SP_FS_MRT_REG;
1331
1332    uint32_t SP_PS_2D_SRC_INFO;
1333    uint32_t SP_PS_2D_SRC_SIZE;
1334
1335    uint32_t RB_2D_DST_INFO;
1336
1337    uint32_t RB_BLIT_DST_INFO;
1338 };
1339
1340 struct tu_sampler_ycbcr_conversion {
1341    VkFormat format;
1342    VkSamplerYcbcrModelConversion ycbcr_model;
1343    VkSamplerYcbcrRange ycbcr_range;
1344    VkComponentMapping components;
1345    VkChromaLocation chroma_offsets[2];
1346    VkFilter chroma_filter;
1347 };
1348
1349 struct tu_sampler {
1350    uint32_t descriptor[A6XX_TEX_SAMP_DWORDS];
1351    struct tu_sampler_ycbcr_conversion *ycbcr_sampler;
1352 };
1353
1354 void
1355 tu_cs_image_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1356
1357 void
1358 tu_cs_image_ref_2d(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer, bool src);
1359
1360 void
1361 tu_cs_image_flag_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer);
1362
1363 enum a6xx_tex_filter
1364 tu6_tex_filter(VkFilter filter, unsigned aniso);
1365
1366 VkResult
1367 tu_image_create(VkDevice _device,
1368                 const VkImageCreateInfo *pCreateInfo,
1369                 const VkAllocationCallbacks *alloc,
1370                 VkImage *pImage,
1371                 uint64_t modifier,
1372                 const VkSubresourceLayout *plane_layouts);
1373
1374 VkResult
1375 tu_image_from_gralloc(VkDevice device_h,
1376                       const VkImageCreateInfo *base_info,
1377                       const VkNativeBufferANDROID *gralloc_info,
1378                       const VkAllocationCallbacks *alloc,
1379                       VkImage *out_image_h);
1380
1381 void
1382 tu_image_view_init(struct tu_image_view *view,
1383                    const VkImageViewCreateInfo *pCreateInfo);
1384
1385 struct tu_buffer_view
1386 {
1387    uint32_t descriptor[A6XX_TEX_CONST_DWORDS];
1388
1389    struct tu_buffer *buffer;
1390 };
1391 void
1392 tu_buffer_view_init(struct tu_buffer_view *view,
1393                     struct tu_device *device,
1394                     const VkBufferViewCreateInfo *pCreateInfo);
1395
1396 struct tu_attachment_info
1397 {
1398    struct tu_image_view *attachment;
1399 };
1400
1401 struct tu_framebuffer
1402 {
1403    uint32_t width;
1404    uint32_t height;
1405    uint32_t layers;
1406
1407    uint32_t attachment_count;
1408    struct tu_attachment_info attachments[0];
1409 };
1410
1411 struct tu_subpass_barrier {
1412    VkPipelineStageFlags src_stage_mask;
1413    VkAccessFlags src_access_mask;
1414    VkAccessFlags dst_access_mask;
1415    bool incoherent_ccu_color, incoherent_ccu_depth;
1416 };
1417
1418 struct tu_subpass_attachment
1419 {
1420    uint32_t attachment;
1421    VkImageLayout layout;
1422 };
1423
1424 struct tu_subpass
1425 {
1426    uint32_t input_count;
1427    uint32_t color_count;
1428    struct tu_subpass_attachment *input_attachments;
1429    struct tu_subpass_attachment *color_attachments;
1430    struct tu_subpass_attachment *resolve_attachments;
1431    struct tu_subpass_attachment depth_stencil_attachment;
1432
1433    VkSampleCountFlagBits samples;
1434    bool has_external_src, has_external_dst;
1435
1436    uint32_t srgb_cntl;
1437
1438    struct tu_subpass_barrier start_barrier;
1439 };
1440
1441 struct tu_render_pass_attachment
1442 {
1443    VkFormat format;
1444    uint32_t samples;
1445    uint32_t cpp;
1446    VkImageAspectFlags clear_mask;
1447    bool load;
1448    bool store;
1449    VkImageLayout initial_layout, final_layout;
1450    int32_t gmem_offset;
1451 };
1452
1453 struct tu_render_pass
1454 {
1455    uint32_t attachment_count;
1456    uint32_t subpass_count;
1457    uint32_t gmem_pixels;
1458    uint32_t tile_align_w;
1459    struct tu_subpass_attachment *subpass_attachments;
1460    struct tu_render_pass_attachment *attachments;
1461    struct tu_subpass_barrier end_barrier;
1462    struct tu_subpass subpasses[0];
1463 };
1464
1465 struct tu_query_pool
1466 {
1467    VkQueryType type;
1468    uint32_t stride;
1469    uint64_t size;
1470    uint32_t pipeline_statistics;
1471    struct tu_bo bo;
1472 };
1473
1474 struct tu_semaphore
1475 {
1476    uint32_t syncobj;
1477    uint32_t temp_syncobj;
1478 };
1479
1480 void
1481 tu_set_descriptor_set(struct tu_cmd_buffer *cmd_buffer,
1482                       VkPipelineBindPoint bind_point,
1483                       struct tu_descriptor_set *set,
1484                       unsigned idx);
1485
1486 void
1487 tu_update_descriptor_sets(struct tu_device *device,
1488                           struct tu_cmd_buffer *cmd_buffer,
1489                           VkDescriptorSet overrideSet,
1490                           uint32_t descriptorWriteCount,
1491                           const VkWriteDescriptorSet *pDescriptorWrites,
1492                           uint32_t descriptorCopyCount,
1493                           const VkCopyDescriptorSet *pDescriptorCopies);
1494
1495 void
1496 tu_update_descriptor_set_with_template(
1497    struct tu_device *device,
1498    struct tu_cmd_buffer *cmd_buffer,
1499    struct tu_descriptor_set *set,
1500    VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1501    const void *pData);
1502
1503 int
1504 tu_drm_get_gpu_id(const struct tu_physical_device *dev, uint32_t *id);
1505
1506 int
1507 tu_drm_get_gmem_size(const struct tu_physical_device *dev, uint32_t *size);
1508
1509 int
1510 tu_drm_get_gmem_base(const struct tu_physical_device *dev, uint64_t *base);
1511
1512 int
1513 tu_drm_submitqueue_new(const struct tu_device *dev,
1514                        int priority,
1515                        uint32_t *queue_id);
1516
1517 void
1518 tu_drm_submitqueue_close(const struct tu_device *dev, uint32_t queue_id);
1519
1520 uint32_t
1521 tu_gem_new(const struct tu_device *dev, uint64_t size, uint32_t flags);
1522 uint32_t
1523 tu_gem_import_dmabuf(const struct tu_device *dev,
1524                      int prime_fd,
1525                      uint64_t size);
1526 int
1527 tu_gem_export_dmabuf(const struct tu_device *dev, uint32_t gem_handle);
1528 void
1529 tu_gem_close(const struct tu_device *dev, uint32_t gem_handle);
1530 uint64_t
1531 tu_gem_info_offset(const struct tu_device *dev, uint32_t gem_handle);
1532 uint64_t
1533 tu_gem_info_iova(const struct tu_device *dev, uint32_t gem_handle);
1534
1535 #define TU_DEFINE_HANDLE_CASTS(__tu_type, __VkType)                          \
1536                                                                              \
1537    static inline struct __tu_type *__tu_type##_from_handle(__VkType _handle) \
1538    {                                                                         \
1539       return (struct __tu_type *) _handle;                                   \
1540    }                                                                         \
1541                                                                              \
1542    static inline __VkType __tu_type##_to_handle(struct __tu_type *_obj)      \
1543    {                                                                         \
1544       return (__VkType) _obj;                                                \
1545    }
1546
1547 #define TU_DEFINE_NONDISP_HANDLE_CASTS(__tu_type, __VkType)                  \
1548                                                                              \
1549    static inline struct __tu_type *__tu_type##_from_handle(__VkType _handle) \
1550    {                                                                         \
1551       return (struct __tu_type *) (uintptr_t) _handle;                       \
1552    }                                                                         \
1553                                                                              \
1554    static inline __VkType __tu_type##_to_handle(struct __tu_type *_obj)      \
1555    {                                                                         \
1556       return (__VkType)(uintptr_t) _obj;                                     \
1557    }
1558
1559 #define TU_FROM_HANDLE(__tu_type, __name, __handle)                          \
1560    struct __tu_type *__name = __tu_type##_from_handle(__handle)
1561
1562 TU_DEFINE_HANDLE_CASTS(tu_cmd_buffer, VkCommandBuffer)
1563 TU_DEFINE_HANDLE_CASTS(tu_device, VkDevice)
1564 TU_DEFINE_HANDLE_CASTS(tu_instance, VkInstance)
1565 TU_DEFINE_HANDLE_CASTS(tu_physical_device, VkPhysicalDevice)
1566 TU_DEFINE_HANDLE_CASTS(tu_queue, VkQueue)
1567
1568 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_cmd_pool, VkCommandPool)
1569 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_buffer, VkBuffer)
1570 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_buffer_view, VkBufferView)
1571 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_pool, VkDescriptorPool)
1572 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_set, VkDescriptorSet)
1573 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_set_layout,
1574                                VkDescriptorSetLayout)
1575 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_descriptor_update_template,
1576                                VkDescriptorUpdateTemplate)
1577 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_device_memory, VkDeviceMemory)
1578 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_fence, VkFence)
1579 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_event, VkEvent)
1580 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_framebuffer, VkFramebuffer)
1581 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_image, VkImage)
1582 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_image_view, VkImageView);
1583 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline_cache, VkPipelineCache)
1584 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline, VkPipeline)
1585 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_pipeline_layout, VkPipelineLayout)
1586 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_query_pool, VkQueryPool)
1587 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_render_pass, VkRenderPass)
1588 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_sampler, VkSampler)
1589 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_sampler_ycbcr_conversion, VkSamplerYcbcrConversion)
1590 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_shader_module, VkShaderModule)
1591 TU_DEFINE_NONDISP_HANDLE_CASTS(tu_semaphore, VkSemaphore)
1592
1593 #endif /* TU_PRIVATE_H */