src/amd/vulkan/radv_private.h

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * based in part on anv driver which is:
   6  * Copyright © 2015 Intel Corporation
   7  *
   8  * Permission is hereby granted, free of charge, to any person obtaining a
   9  * copy of this software and associated documentation files (the "Software"),
  10  * to deal in the Software without restriction, including without limitation
  11  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12  * and/or sell copies of the Software, and to permit persons to whom the
  13  * Software is furnished to do so, subject to the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the next
  16  * paragraph) shall be included in all copies or substantial portions of the
  17  * Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  22  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  24  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  25  * IN THE SOFTWARE.
  26  */
  27
  28 #ifndef RADV_PRIVATE_H
  29 #define RADV_PRIVATE_H
  30
  31 #include <stdlib.h>
  32 #include <stdio.h>
  33 #include <stdbool.h>
  34 #include <pthread.h>
  35 #include <assert.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #ifdef HAVE_VALGRIND
  39 #include <valgrind.h>
  40 #include <memcheck.h>
  41 #define VG(x) x
  42 #else
  43 #define VG(x)
  44 #endif
  45
  46 #include <amdgpu.h>
  47 #include "compiler/shader_enums.h"
  48 #include "util/macros.h"
  49 #include "util/list.h"
  50 #include "util/vk_alloc.h"
  51 #include "main/macros.h"
  52
  53 #include "radv_radeon_winsys.h"
  54 #include "ac_binary.h"
  55 #include "ac_nir_to_llvm.h"
  56 #include "radv_descriptor_set.h"
  57
  58 #include <llvm-c/TargetMachine.h>
  59
  60 /* Pre-declarations needed for WSI entrypoints */
  61 struct wl_surface;
  62 struct wl_display;
  63 typedef struct xcb_connection_t xcb_connection_t;
  64 typedef uint32_t xcb_visualid_t;
  65 typedef uint32_t xcb_window_t;
  66
  67 #include <vulkan/vulkan.h>
  68 #include <vulkan/vulkan_intel.h>
  69 #include <vulkan/vk_icd.h>
  70
  71 #include "radv_entrypoints.h"
  72
  73 #include "wsi_common.h"
  74
  75 #define MAX_VBS         32
  76 #define MAX_VERTEX_ATTRIBS 32
  77 #define MAX_RTS          8
  78 #define MAX_VIEWPORTS   16
  79 #define MAX_SCISSORS    16
  80 #define MAX_PUSH_CONSTANTS_SIZE 128
  81 #define MAX_DYNAMIC_BUFFERS 16
  82 #define MAX_IMAGES 8
  83 #define MAX_SAMPLES_LOG2 4 /* SKL supports 16 samples */
  84 #define NUM_META_FS_KEYS 11
  85
  86 #define NUM_DEPTH_CLEAR_PIPELINES 3
  87
  88 #define radv_noreturn __attribute__((__noreturn__))
  89 #define radv_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
  90
  91 static inline uint32_t
  92 align_u32(uint32_t v, uint32_t a)
  93 {
  94         assert(a != 0 && a == (a & -a));
  95         return (v + a - 1) & ~(a - 1);
  96 }
  97
  98 static inline uint32_t
  99 align_u32_npot(uint32_t v, uint32_t a)
 100 {
 101         return (v + a - 1) / a * a;
 102 }
 103
 104 static inline uint64_t
 105 align_u64(uint64_t v, uint64_t a)
 106 {
 107         assert(a != 0 && a == (a & -a));
 108         return (v + a - 1) & ~(a - 1);
 109 }
 110
 111 static inline int32_t
 112 align_i32(int32_t v, int32_t a)
 113 {
 114         assert(a != 0 && a == (a & -a));
 115         return (v + a - 1) & ~(a - 1);
 116 }
 117
 118 /** Alignment must be a power of 2. */
 119 static inline bool
 120 radv_is_aligned(uintmax_t n, uintmax_t a)
 121 {
 122         assert(a == (a & -a));
 123         return (n & (a - 1)) == 0;
 124 }
 125
 126 static inline uint32_t
 127 round_up_u32(uint32_t v, uint32_t a)
 128 {
 129         return (v + a - 1) / a;
 130 }
 131
 132 static inline uint64_t
 133 round_up_u64(uint64_t v, uint64_t a)
 134 {
 135         return (v + a - 1) / a;
 136 }
 137
 138 static inline uint32_t
 139 radv_minify(uint32_t n, uint32_t levels)
 140 {
 141         if (unlikely(n == 0))
 142                 return 0;
 143         else
 144                 return MAX2(n >> levels, 1);
 145 }
 146 static inline float
 147 radv_clamp_f(float f, float min, float max)
 148 {
 149         assert(min < max);
 150
 151         if (f > max)
 152                 return max;
 153         else if (f < min)
 154                 return min;
 155         else
 156                 return f;
 157 }
 158
 159 static inline bool
 160 radv_clear_mask(uint32_t *inout_mask, uint32_t clear_mask)
 161 {
 162         if (*inout_mask & clear_mask) {
 163                 *inout_mask &= ~clear_mask;
 164                 return true;
 165         } else {
 166                 return false;
 167         }
 168 }
 169
 170 #define for_each_bit(b, dword)                          \
 171         for (uint32_t __dword = (dword);                \
 172              (b) = __builtin_ffs(__dword) - 1, __dword; \
 173              __dword &= ~(1 << (b)))
 174
 175 #define typed_memcpy(dest, src, count) ({                               \
 176                         static_assert(sizeof(*src) == sizeof(*dest), ""); \
 177                         memcpy((dest), (src), (count) * sizeof(*(src))); \
 178                 })
 179
 180 #define zero(x) (memset(&(x), 0, sizeof(x)))
 181
 182 /* Define no kernel as 1, since that's an illegal offset for a kernel */
 183 #define NO_KERNEL 1
 184
 185 struct radv_common {
 186         VkStructureType                             sType;
 187         const void*                                 pNext;
 188 };
 189
 190 /* Whenever we generate an error, pass it through this function. Useful for
 191  * debugging, where we can break on it. Only call at error site, not when
 192  * propagating errors. Might be useful to plug in a stack trace here.
 193  */
 194
 195 VkResult __vk_errorf(VkResult error, const char *file, int line, const char *format, ...);
 196
 197 #ifdef DEBUG
 198 #define vk_error(error) __vk_errorf(error, __FILE__, __LINE__, NULL);
 199 #define vk_errorf(error, format, ...) __vk_errorf(error, __FILE__, __LINE__, format, ## __VA_ARGS__);
 200 #else
 201 #define vk_error(error) error
 202 #define vk_errorf(error, format, ...) error
 203 #endif
 204
 205 void __radv_finishme(const char *file, int line, const char *format, ...)
 206         radv_printflike(3, 4);
 207 void radv_loge(const char *format, ...) radv_printflike(1, 2);
 208 void radv_loge_v(const char *format, va_list va);
 209
 210 /**
 211  * Print a FINISHME message, including its source location.
 212  */
 213 #define radv_finishme(format, ...)                                      \
 214         __radv_finishme(__FILE__, __LINE__, format, ##__VA_ARGS__);
 215
 216 /* A non-fatal assert.  Useful for debugging. */
 217 #ifdef DEBUG
 218 #define radv_assert(x) ({                                               \
 219                         if (unlikely(!(x)))                             \
 220                                 fprintf(stderr, "%s:%d ASSERT: %s\n", __FILE__, __LINE__, #x); \
 221                 })
 222 #else
 223 #define radv_assert(x)
 224 #endif
 225
 226 void radv_abortf(const char *format, ...) radv_noreturn radv_printflike(1, 2);
 227 void radv_abortfv(const char *format, va_list va) radv_noreturn;
 228
 229 #define stub_return(v)                                  \
 230         do {                                            \
 231                 radv_finishme("stub %s", __func__);     \
 232                 return (v);                             \
 233         } while (0)
 234
 235 #define stub()                                          \
 236         do {                                            \
 237                 radv_finishme("stub %s", __func__);     \
 238                 return;                                 \
 239         } while (0)
 240
 241 void *radv_resolve_entrypoint(uint32_t index);
 242 void *radv_lookup_entrypoint(const char *name);
 243
 244 extern struct radv_dispatch_table dtable;
 245
 246 struct radv_physical_device {
 247         VK_LOADER_DATA                              _loader_data;
 248
 249         struct radv_instance *                       instance;
 250
 251         struct radeon_winsys *ws;
 252         struct radeon_info rad_info;
 253         uint32_t                                    chipset_id;
 254         char                                        path[20];
 255         const char *                                name;
 256         uint64_t                                    aperture_size;
 257         int                                         cmd_parser_version;
 258         uint32_t                    pci_vendor_id;
 259         uint32_t                    pci_device_id;
 260
 261         struct wsi_device                       wsi_device;
 262 };
 263
 264 struct radv_instance {
 265         VK_LOADER_DATA                              _loader_data;
 266
 267         VkAllocationCallbacks                       alloc;
 268
 269         uint32_t                                    apiVersion;
 270         int                                         physicalDeviceCount;
 271         struct radv_physical_device                  physicalDevice;
 272 };
 273
 274 VkResult radv_init_wsi(struct radv_physical_device *physical_device);
 275 void radv_finish_wsi(struct radv_physical_device *physical_device);
 276
 277 struct cache_entry;
 278
 279 struct radv_pipeline_cache {
 280         struct radv_device *                          device;
 281         pthread_mutex_t                              mutex;
 282
 283         uint32_t                                     total_size;
 284         uint32_t                                     table_size;
 285         uint32_t                                     kernel_count;
 286         struct cache_entry **                        hash_table;
 287         bool                                         modified;
 288
 289         VkAllocationCallbacks                        alloc;
 290 };
 291
 292 void
 293 radv_pipeline_cache_init(struct radv_pipeline_cache *cache,
 294                          struct radv_device *device);
 295 void
 296 radv_pipeline_cache_finish(struct radv_pipeline_cache *cache);
 297 void
 298 radv_pipeline_cache_load(struct radv_pipeline_cache *cache,
 299                          const void *data, size_t size);
 300
 301 struct radv_shader_variant *
 302 radv_create_shader_variant_from_pipeline_cache(struct radv_device *device,
 303                                                struct radv_pipeline_cache *cache,
 304                                                const unsigned char *sha1);
 305
 306 struct radv_shader_variant *
 307 radv_pipeline_cache_insert_shader(struct radv_pipeline_cache *cache,
 308                                   const unsigned char *sha1,
 309                                   struct radv_shader_variant *variant,
 310                                   const void *code, unsigned code_size);
 311
 312 void radv_shader_variant_destroy(struct radv_device *device,
 313                                  struct radv_shader_variant *variant);
 314
 315 struct radv_meta_state {
 316         VkAllocationCallbacks alloc;
 317
 318         struct radv_pipeline_cache cache;
 319
 320         /**
 321          * Use array element `i` for images with `2^i` samples.
 322          */
 323         struct {
 324                 VkRenderPass render_pass[NUM_META_FS_KEYS];
 325                 struct radv_pipeline *color_pipelines[NUM_META_FS_KEYS];
 326
 327                 VkRenderPass depth_only_rp[NUM_DEPTH_CLEAR_PIPELINES];
 328                 struct radv_pipeline *depth_only_pipeline[NUM_DEPTH_CLEAR_PIPELINES];
 329                 VkRenderPass stencil_only_rp[NUM_DEPTH_CLEAR_PIPELINES];
 330                 struct radv_pipeline *stencil_only_pipeline[NUM_DEPTH_CLEAR_PIPELINES];
 331                 VkRenderPass depthstencil_rp[NUM_DEPTH_CLEAR_PIPELINES];
 332                 struct radv_pipeline *depthstencil_pipeline[NUM_DEPTH_CLEAR_PIPELINES];
 333         } clear[1 + MAX_SAMPLES_LOG2];
 334
 335         struct {
 336                 VkRenderPass render_pass[NUM_META_FS_KEYS];
 337
 338                 /** Pipeline that blits from a 1D image. */
 339                 VkPipeline pipeline_1d_src[NUM_META_FS_KEYS];
 340
 341                 /** Pipeline that blits from a 2D image. */
 342                 VkPipeline pipeline_2d_src[NUM_META_FS_KEYS];
 343
 344                 /** Pipeline that blits from a 3D image. */
 345                 VkPipeline pipeline_3d_src[NUM_META_FS_KEYS];
 346
 347                 VkRenderPass depth_only_rp;
 348                 VkPipeline depth_only_1d_pipeline;
 349                 VkPipeline depth_only_2d_pipeline;
 350                 VkPipeline depth_only_3d_pipeline;
 351
 352                 VkRenderPass stencil_only_rp;
 353                 VkPipeline stencil_only_1d_pipeline;
 354                 VkPipeline stencil_only_2d_pipeline;
 355                 VkPipeline stencil_only_3d_pipeline;
 356                 VkPipelineLayout                          pipeline_layout;
 357                 VkDescriptorSetLayout                     ds_layout;
 358         } blit;
 359
 360         struct {
 361                 VkRenderPass render_passes[NUM_META_FS_KEYS];
 362
 363                 VkPipelineLayout p_layouts[2];
 364                 VkDescriptorSetLayout ds_layouts[2];
 365                 VkPipeline pipelines[2][NUM_META_FS_KEYS];
 366
 367                 VkRenderPass depth_only_rp;
 368                 VkPipeline depth_only_pipeline[2];
 369
 370                 VkRenderPass stencil_only_rp;
 371                 VkPipeline stencil_only_pipeline[2];
 372         } blit2d;
 373
 374         struct {
 375                 VkPipelineLayout                          img_p_layout;
 376                 VkDescriptorSetLayout                     img_ds_layout;
 377                 VkPipeline pipeline;
 378         } itob;
 379         struct {
 380                 VkRenderPass render_pass;
 381                 VkPipelineLayout                          img_p_layout;
 382                 VkDescriptorSetLayout                     img_ds_layout;
 383                 VkPipeline pipeline;
 384         } btoi;
 385
 386         struct {
 387                 VkPipeline                                pipeline;
 388                 VkRenderPass                              pass;
 389         } resolve;
 390
 391         struct {
 392                 VkDescriptorSetLayout                     ds_layout;
 393                 VkPipelineLayout                          p_layout;
 394                 struct {
 395                         VkPipeline                                pipeline;
 396                         VkPipeline                                i_pipeline;
 397                 } rc[MAX_SAMPLES_LOG2];
 398         } resolve_compute;
 399
 400         struct {
 401                 VkPipeline                                decompress_pipeline;
 402                 VkPipeline                                resummarize_pipeline;
 403                 VkRenderPass                              pass;
 404         } depth_decomp;
 405
 406         struct {
 407                 VkPipeline                                cmask_eliminate_pipeline;
 408                 VkPipeline                                fmask_decompress_pipeline;
 409                 VkRenderPass                              pass;
 410         } fast_clear_flush;
 411
 412         struct {
 413                 VkPipelineLayout fill_p_layout;
 414                 VkPipelineLayout copy_p_layout;
 415                 VkDescriptorSetLayout fill_ds_layout;
 416                 VkDescriptorSetLayout copy_ds_layout;
 417                 VkPipeline fill_pipeline;
 418                 VkPipeline copy_pipeline;
 419         } buffer;
 420 };
 421
 422 struct radv_queue {
 423         VK_LOADER_DATA                              _loader_data;
 424
 425         struct radv_device *                         device;
 426
 427         struct radv_state_pool *                     pool;
 428 };
 429
 430 struct radv_device {
 431         VK_LOADER_DATA                              _loader_data;
 432
 433         VkAllocationCallbacks                       alloc;
 434
 435         struct radv_instance *                       instance;
 436         struct radeon_winsys *ws;
 437         struct radeon_winsys_ctx *hw_ctx;
 438
 439         struct radv_meta_state                       meta_state;
 440         struct radv_queue                            queue;
 441         struct radeon_winsys_cs *empty_cs;
 442
 443         bool allow_fast_clears;
 444         bool allow_dcc;
 445
 446         /* MSAA sample locations.
 447          * The first index is the sample index.
 448          * The second index is the coordinate: X, Y. */
 449         float sample_locations_1x[1][2];
 450         float sample_locations_2x[2][2];
 451         float sample_locations_4x[4][2];
 452         float sample_locations_8x[8][2];
 453         float sample_locations_16x[16][2];
 454 };
 455
 456 void radv_device_get_cache_uuid(void *uuid);
 457
 458 struct radv_device_memory {
 459         struct radeon_winsys_bo                      *bo;
 460         uint32_t                                     type_index;
 461         VkDeviceSize                                 map_size;
 462         void *                                       map;
 463 };
 464
 465
 466 struct radv_descriptor_range {
 467         uint64_t va;
 468         uint32_t size;
 469 };
 470
 471 struct radv_descriptor_set {
 472         const struct radv_descriptor_set_layout *layout;
 473         struct list_head descriptor_pool;
 474         uint32_t size;
 475
 476         struct radv_buffer_view *buffer_views;
 477         struct radeon_winsys_bo *bo;
 478         uint64_t va;
 479         uint32_t *mapped_ptr;
 480         struct radv_descriptor_range *dynamic_descriptors;
 481         struct radeon_winsys_bo *descriptors[0];
 482 };
 483
 484 struct radv_descriptor_pool_free_node {
 485         int next;
 486         uint32_t offset;
 487         uint32_t size;
 488 };
 489
 490 struct radv_descriptor_pool {
 491         struct list_head descriptor_sets;
 492
 493         struct radeon_winsys_bo *bo;
 494         uint8_t *mapped_ptr;
 495         uint64_t current_offset;
 496         uint64_t size;
 497
 498         int free_list;
 499         int full_list;
 500         uint32_t max_sets;
 501         struct radv_descriptor_pool_free_node free_nodes[];
 502 };
 503
 504 struct radv_buffer {
 505         struct radv_device *                          device;
 506         VkDeviceSize                                 size;
 507
 508         VkBufferUsageFlags                           usage;
 509
 510         /* Set when bound */
 511         struct radeon_winsys_bo *                      bo;
 512         VkDeviceSize                                 offset;
 513 };
 514
 515
 516 enum radv_cmd_dirty_bits {
 517         RADV_CMD_DIRTY_DYNAMIC_VIEWPORT                  = 1 << 0, /* VK_DYNAMIC_STATE_VIEWPORT */
 518         RADV_CMD_DIRTY_DYNAMIC_SCISSOR                   = 1 << 1, /* VK_DYNAMIC_STATE_SCISSOR */
 519         RADV_CMD_DIRTY_DYNAMIC_LINE_WIDTH                = 1 << 2, /* VK_DYNAMIC_STATE_LINE_WIDTH */
 520         RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS                = 1 << 3, /* VK_DYNAMIC_STATE_DEPTH_BIAS */
 521         RADV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS           = 1 << 4, /* VK_DYNAMIC_STATE_BLEND_CONSTANTS */
 522         RADV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS              = 1 << 5, /* VK_DYNAMIC_STATE_DEPTH_BOUNDS */
 523         RADV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK      = 1 << 6, /* VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK */
 524         RADV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK        = 1 << 7, /* VK_DYNAMIC_STATE_STENCIL_WRITE_MASK */
 525         RADV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE         = 1 << 8, /* VK_DYNAMIC_STATE_STENCIL_REFERENCE */
 526         RADV_CMD_DIRTY_DYNAMIC_ALL                       = (1 << 9) - 1,
 527         RADV_CMD_DIRTY_PIPELINE                          = 1 << 9,
 528         RADV_CMD_DIRTY_INDEX_BUFFER                      = 1 << 10,
 529         RADV_CMD_DIRTY_RENDER_TARGETS                    = 1 << 11,
 530 };
 531 typedef uint32_t radv_cmd_dirty_mask_t;
 532
 533 enum radv_cmd_flush_bits {
 534         RADV_CMD_FLAG_INV_ICACHE = 1 << 0,
 535         /* SMEM L1, other names: KCACHE, constant cache, DCACHE, data cache */
 536         RADV_CMD_FLAG_INV_SMEM_L1 = 1 << 1,
 537         /* VMEM L1 can optionally be bypassed (GLC=1). Other names: TC L1 */
 538         RADV_CMD_FLAG_INV_VMEM_L1 = 1 << 2,
 539         /* Used by everything except CB/DB, can be bypassed (SLC=1). Other names: TC L2 */
 540         RADV_CMD_FLAG_INV_GLOBAL_L2 = 1 << 3,
 541         /* Framebuffer caches */
 542         RADV_CMD_FLAG_FLUSH_AND_INV_CB_META = 1 << 4,
 543         RADV_CMD_FLAG_FLUSH_AND_INV_DB_META = 1 << 5,
 544         RADV_CMD_FLAG_FLUSH_AND_INV_DB = 1 << 6,
 545         RADV_CMD_FLAG_FLUSH_AND_INV_CB = 1 << 7,
 546         /* Engine synchronization. */
 547         RADV_CMD_FLAG_VS_PARTIAL_FLUSH = 1 << 8,
 548         RADV_CMD_FLAG_PS_PARTIAL_FLUSH = 1 << 9,
 549         RADV_CMD_FLAG_CS_PARTIAL_FLUSH = 1 << 10,
 550         RADV_CMD_FLAG_VGT_FLUSH        = 1 << 11,
 551
 552         RADV_CMD_FLUSH_AND_INV_FRAMEBUFFER = (RADV_CMD_FLAG_FLUSH_AND_INV_CB |
 553                                               RADV_CMD_FLAG_FLUSH_AND_INV_CB_META |
 554                                               RADV_CMD_FLAG_FLUSH_AND_INV_DB |
 555                                               RADV_CMD_FLAG_FLUSH_AND_INV_DB_META)
 556 };
 557
 558 struct radv_vertex_binding {
 559         struct radv_buffer *                          buffer;
 560         VkDeviceSize                                 offset;
 561 };
 562
 563 struct radv_dynamic_state {
 564         struct {
 565                 uint32_t                                  count;
 566                 VkViewport                                viewports[MAX_VIEWPORTS];
 567         } viewport;
 568
 569         struct {
 570                 uint32_t                                  count;
 571                 VkRect2D                                  scissors[MAX_SCISSORS];
 572         } scissor;
 573
 574         float                                        line_width;
 575
 576         struct {
 577                 float                                     bias;
 578                 float                                     clamp;
 579                 float                                     slope;
 580         } depth_bias;
 581
 582         float                                        blend_constants[4];
 583
 584         struct {
 585                 float                                     min;
 586                 float                                     max;
 587         } depth_bounds;
 588
 589         struct {
 590                 uint32_t                                  front;
 591                 uint32_t                                  back;
 592         } stencil_compare_mask;
 593
 594         struct {
 595                 uint32_t                                  front;
 596                 uint32_t                                  back;
 597         } stencil_write_mask;
 598
 599         struct {
 600                 uint32_t                                  front;
 601                 uint32_t                                  back;
 602         } stencil_reference;
 603 };
 604
 605 extern const struct radv_dynamic_state default_dynamic_state;
 606
 607 void radv_dynamic_state_copy(struct radv_dynamic_state *dest,
 608                              const struct radv_dynamic_state *src,
 609                              uint32_t copy_mask);
 610 /**
 611  * Attachment state when recording a renderpass instance.
 612  *
 613  * The clear value is valid only if there exists a pending clear.
 614  */
 615 struct radv_attachment_state {
 616         VkImageAspectFlags                           pending_clear_aspects;
 617         VkClearValue                                 clear_value;
 618         VkImageLayout                                current_layout;
 619 };
 620
 621 struct radv_cmd_state {
 622         uint32_t                                      vb_dirty;
 623         bool                                          vertex_descriptors_dirty;
 624         radv_cmd_dirty_mask_t                         dirty;
 625
 626         struct radv_pipeline *                        pipeline;
 627         struct radv_pipeline *                        emitted_pipeline;
 628         struct radv_pipeline *                        compute_pipeline;
 629         struct radv_pipeline *                        emitted_compute_pipeline;
 630         struct radv_framebuffer *                     framebuffer;
 631         struct radv_render_pass *                     pass;
 632         const struct radv_subpass *                         subpass;
 633         struct radv_dynamic_state                     dynamic;
 634         struct radv_vertex_binding                    vertex_bindings[MAX_VBS];
 635         struct radv_descriptor_set *                  descriptors[MAX_SETS];
 636         struct radv_attachment_state *                attachments;
 637         VkRect2D                                     render_area;
 638         struct radv_buffer *                         index_buffer;
 639         uint32_t                                     index_type;
 640         uint32_t                                     index_offset;
 641         uint32_t                                     last_primitive_reset_index;
 642         enum radv_cmd_flush_bits                     flush_bits;
 643         unsigned                                     active_occlusion_queries;
 644         float                                        offset_scale;
 645 };
 646 struct radv_cmd_pool {
 647         VkAllocationCallbacks                        alloc;
 648         struct list_head                             cmd_buffers;
 649 };
 650
 651 struct radv_cmd_buffer_upload {
 652         uint8_t *map;
 653         unsigned offset;
 654         uint64_t size;
 655         struct radeon_winsys_bo *upload_bo;
 656         struct list_head list;
 657 };
 658
 659 struct radv_cmd_buffer {
 660         VK_LOADER_DATA                               _loader_data;
 661
 662         struct radv_device *                          device;
 663
 664         struct radv_cmd_pool *                        pool;
 665         struct list_head                             pool_link;
 666
 667         VkCommandBufferUsageFlags                    usage_flags;
 668         VkCommandBufferLevel                         level;
 669         struct radeon_winsys_cs *cs;
 670         struct radv_cmd_state state;
 671
 672         uint8_t push_constants[MAX_PUSH_CONSTANTS_SIZE];
 673         uint32_t dynamic_buffers[16 * MAX_DYNAMIC_BUFFERS];
 674         VkShaderStageFlags push_constant_stages;
 675
 676         struct radv_cmd_buffer_upload upload;
 677
 678         bool record_fail;
 679 };
 680
 681 struct radv_image;
 682
 683 void si_init_config(struct radv_physical_device *physical_device,
 684                     struct radv_cmd_buffer *cmd_buffer);
 685 void si_write_viewport(struct radeon_winsys_cs *cs, int first_vp,
 686                        int count, const VkViewport *viewports);
 687 void si_write_scissors(struct radeon_winsys_cs *cs, int first,
 688                        int count, const VkRect2D *scissors);
 689 uint32_t si_get_ia_multi_vgt_param(struct radv_cmd_buffer *cmd_buffer);
 690 void si_emit_cache_flush(struct radv_cmd_buffer *cmd_buffer);
 691 void si_cp_dma_buffer_copy(struct radv_cmd_buffer *cmd_buffer,
 692                            uint64_t src_va, uint64_t dest_va,
 693                            uint64_t size);
 694 void si_cp_dma_clear_buffer(struct radv_cmd_buffer *cmd_buffer, uint64_t va,
 695                             uint64_t size, unsigned value);
 696 void radv_set_db_count_control(struct radv_cmd_buffer *cmd_buffer);
 697 void radv_bind_descriptor_set(struct radv_cmd_buffer *cmd_buffer,
 698                               struct radv_descriptor_set *set,
 699                               unsigned idx);
 700 bool
 701 radv_cmd_buffer_upload_alloc(struct radv_cmd_buffer *cmd_buffer,
 702                              unsigned size,
 703                              unsigned alignment,
 704                              unsigned *out_offset,
 705                              void **ptr);
 706 void
 707 radv_cmd_buffer_set_subpass(struct radv_cmd_buffer *cmd_buffer,
 708                             const struct radv_subpass *subpass,
 709                             bool transitions);
 710 bool
 711 radv_cmd_buffer_upload_data(struct radv_cmd_buffer *cmd_buffer,
 712                             unsigned size, unsigned alignmnet,
 713                             const void *data, unsigned *out_offset);
 714 void
 715 radv_emit_framebuffer_state(struct radv_cmd_buffer *cmd_buffer);
 716 void radv_cmd_buffer_clear_subpass(struct radv_cmd_buffer *cmd_buffer);
 717 void radv_cmd_buffer_resolve_subpass(struct radv_cmd_buffer *cmd_buffer);
 718 void radv_cayman_emit_msaa_sample_locs(struct radeon_winsys_cs *cs, int nr_samples);
 719 unsigned radv_cayman_get_maxdist(int log_samples);
 720 void radv_device_init_msaa(struct radv_device *device);
 721 void radv_set_depth_clear_regs(struct radv_cmd_buffer *cmd_buffer,
 722                                struct radv_image *image,
 723                                VkClearDepthStencilValue ds_clear_value,
 724                                VkImageAspectFlags aspects);
 725 void radv_set_color_clear_regs(struct radv_cmd_buffer *cmd_buffer,
 726                                struct radv_image *image,
 727                                int idx,
 728                                uint32_t color_values[2]);
 729 void radv_fill_buffer(struct radv_cmd_buffer *cmd_buffer,
 730                       struct radeon_winsys_bo *bo,
 731                       uint64_t offset, uint64_t size, uint32_t value);
 732
 733 /*
 734  * Takes x,y,z as exact numbers of invocations, instead of blocks.
 735  *
 736  * Limitations: Can't call normal dispatch functions without binding or rebinding
 737  *              the compute pipeline.
 738  */
 739 void radv_unaligned_dispatch(
 740         struct radv_cmd_buffer                      *cmd_buffer,
 741         uint32_t                                    x,
 742         uint32_t                                    y,
 743         uint32_t                                    z);
 744
 745 struct radv_event {
 746         struct radeon_winsys_bo *bo;
 747         uint64_t *map;
 748 };
 749
 750 struct nir_shader;
 751
 752 struct radv_shader_module {
 753         struct nir_shader *                          nir;
 754         unsigned char                                sha1[20];
 755         uint32_t                                     size;
 756         char                                         data[0];
 757 };
 758
 759 union ac_shader_variant_key;
 760
 761 void
 762 radv_hash_shader(unsigned char *hash, struct radv_shader_module *module,
 763                  const char *entrypoint,
 764                  const VkSpecializationInfo *spec_info,
 765                  const struct radv_pipeline_layout *layout,
 766                  const union ac_shader_variant_key *key);
 767
 768 static inline gl_shader_stage
 769 vk_to_mesa_shader_stage(VkShaderStageFlagBits vk_stage)
 770 {
 771         assert(__builtin_popcount(vk_stage) == 1);
 772         return ffs(vk_stage) - 1;
 773 }
 774
 775 static inline VkShaderStageFlagBits
 776 mesa_to_vk_shader_stage(gl_shader_stage mesa_stage)
 777 {
 778         return (1 << mesa_stage);
 779 }
 780
 781 #define RADV_STAGE_MASK ((1 << MESA_SHADER_STAGES) - 1)
 782
 783 #define radv_foreach_stage(stage, stage_bits)                           \
 784         for (gl_shader_stage stage,                                     \
 785                      __tmp = (gl_shader_stage)((stage_bits) & RADV_STAGE_MASK); \
 786              stage = __builtin_ffs(__tmp) - 1, __tmp;                   \
 787              __tmp &= ~(1 << (stage)))
 788
 789 struct radv_shader_variant {
 790         uint32_t ref_count;
 791
 792         struct radeon_winsys_bo *bo;
 793         struct ac_shader_config config;
 794         struct ac_shader_variant_info info;
 795         unsigned rsrc1;
 796         unsigned rsrc2;
 797 };
 798
 799 struct radv_depth_stencil_state {
 800         uint32_t db_depth_control;
 801         uint32_t db_stencil_control;
 802         uint32_t db_render_control;
 803         uint32_t db_render_override2;
 804 };
 805
 806 struct radv_blend_state {
 807         uint32_t cb_color_control;
 808         uint32_t cb_target_mask;
 809         uint32_t sx_mrt0_blend_opt[8];
 810         uint32_t cb_blend_control[8];
 811
 812         uint32_t spi_shader_col_format;
 813         uint32_t cb_shader_mask;
 814         uint32_t db_alpha_to_mask;
 815 };
 816
 817 unsigned radv_format_meta_fs_key(VkFormat format);
 818
 819 struct radv_raster_state {
 820         uint32_t pa_cl_clip_cntl;
 821         uint32_t pa_cl_vs_out_cntl;
 822         uint32_t spi_interp_control;
 823         uint32_t pa_su_point_size;
 824         uint32_t pa_su_point_minmax;
 825         uint32_t pa_su_line_cntl;
 826         uint32_t pa_su_vtx_cntl;
 827         uint32_t pa_su_sc_mode_cntl;
 828 };
 829
 830 struct radv_multisample_state {
 831         uint32_t db_eqaa;
 832         uint32_t pa_sc_line_cntl;
 833         uint32_t pa_sc_mode_cntl_0;
 834         uint32_t pa_sc_mode_cntl_1;
 835         uint32_t pa_sc_aa_config;
 836         uint32_t pa_sc_aa_mask[2];
 837         unsigned num_samples;
 838 };
 839
 840 struct radv_pipeline {
 841         struct radv_device *                          device;
 842         uint32_t                                     dynamic_state_mask;
 843         struct radv_dynamic_state                     dynamic_state;
 844
 845         struct radv_pipeline_layout *                 layout;
 846
 847         bool                                         needs_data_cache;
 848
 849         struct radv_shader_variant *                 shaders[MESA_SHADER_STAGES];
 850         VkShaderStageFlags                           active_stages;
 851
 852         uint32_t va_rsrc_word3[MAX_VERTEX_ATTRIBS];
 853         uint32_t va_format_size[MAX_VERTEX_ATTRIBS];
 854         uint32_t va_binding[MAX_VERTEX_ATTRIBS];
 855         uint32_t va_offset[MAX_VERTEX_ATTRIBS];
 856         uint32_t num_vertex_attribs;
 857         uint32_t                                     binding_stride[MAX_VBS];
 858
 859         union {
 860                 struct {
 861                         struct radv_blend_state blend;
 862                         struct radv_depth_stencil_state ds;
 863                         struct radv_raster_state raster;
 864                         struct radv_multisample_state ms;
 865                         unsigned prim;
 866                         unsigned gs_out;
 867                         bool prim_restart_enable;
 868                 } graphics;
 869         };
 870 };
 871
 872 struct radv_graphics_pipeline_create_info {
 873         bool use_rectlist;
 874         bool db_depth_clear;
 875         bool db_stencil_clear;
 876         bool db_depth_disable_expclear;
 877         bool db_stencil_disable_expclear;
 878         bool db_flush_depth_inplace;
 879         bool db_flush_stencil_inplace;
 880         bool db_resummarize;
 881         uint32_t custom_blend_mode;
 882 };
 883
 884 VkResult
 885 radv_pipeline_init(struct radv_pipeline *pipeline, struct radv_device *device,
 886                    struct radv_pipeline_cache *cache,
 887                    const VkGraphicsPipelineCreateInfo *pCreateInfo,
 888                    const struct radv_graphics_pipeline_create_info *extra,
 889                    const VkAllocationCallbacks *alloc);
 890
 891 VkResult
 892 radv_graphics_pipeline_create(VkDevice device,
 893                               VkPipelineCache cache,
 894                               const VkGraphicsPipelineCreateInfo *pCreateInfo,
 895                               const struct radv_graphics_pipeline_create_info *extra,
 896                               const VkAllocationCallbacks *alloc,
 897                               VkPipeline *pPipeline);
 898
 899 struct vk_format_description;
 900 uint32_t radv_translate_buffer_dataformat(const struct vk_format_description *desc,
 901                                           int first_non_void);
 902 uint32_t radv_translate_buffer_numformat(const struct vk_format_description *desc,
 903                                          int first_non_void);
 904 uint32_t radv_translate_colorformat(VkFormat format);
 905 uint32_t radv_translate_color_numformat(VkFormat format,
 906                                         const struct vk_format_description *desc,
 907                                         int first_non_void);
 908 uint32_t radv_colorformat_endian_swap(uint32_t colorformat);
 909 unsigned radv_translate_colorswap(VkFormat format, bool do_endian_swap);
 910 uint32_t radv_translate_dbformat(VkFormat format);
 911 uint32_t radv_translate_tex_dataformat(VkFormat format,
 912                                        const struct vk_format_description *desc,
 913                                        int first_non_void);
 914 uint32_t radv_translate_tex_numformat(VkFormat format,
 915                                       const struct vk_format_description *desc,
 916                                       int first_non_void);
 917 bool radv_format_pack_clear_color(VkFormat format,
 918                                   uint32_t clear_vals[2],
 919                                   VkClearColorValue *value);
 920 bool radv_is_colorbuffer_format_supported(VkFormat format, bool *blendable);
 921
 922 struct radv_fmask_info {
 923         uint64_t offset;
 924         uint64_t size;
 925         unsigned alignment;
 926         unsigned pitch_in_pixels;
 927         unsigned bank_height;
 928         unsigned slice_tile_max;
 929         unsigned tile_mode_index;
 930 };
 931
 932 struct radv_cmask_info {
 933         uint64_t offset;
 934         uint64_t size;
 935         unsigned alignment;
 936         unsigned slice_tile_max;
 937         unsigned base_address_reg;
 938 };
 939
 940 struct r600_htile_info {
 941         uint64_t offset;
 942         uint64_t size;
 943         unsigned pitch;
 944         unsigned height;
 945         unsigned xalign;
 946         unsigned yalign;
 947 };
 948
 949 struct radv_image {
 950         VkImageType type;
 951         /* The original VkFormat provided by the client.  This may not match any
 952          * of the actual surface formats.
 953          */
 954         VkFormat vk_format;
 955         VkImageAspectFlags aspects;
 956         VkExtent3D extent;
 957         uint32_t levels;
 958         uint32_t array_size;
 959         uint32_t samples; /**< VkImageCreateInfo::samples */
 960         VkImageUsageFlags usage; /**< Superset of VkImageCreateInfo::usage. */
 961         VkImageTiling tiling; /** VkImageCreateInfo::tiling */
 962
 963         VkDeviceSize size;
 964         uint32_t alignment;
 965
 966         /* Set when bound */
 967         struct radeon_winsys_bo *bo;
 968         VkDeviceSize offset;
 969         uint32_t dcc_offset;
 970         struct radeon_surf surface;
 971
 972         struct radv_fmask_info fmask;
 973         struct radv_cmask_info cmask;
 974         uint32_t clear_value_offset;
 975
 976         /* Depth buffer compression and fast clear. */
 977         struct r600_htile_info htile;
 978 };
 979
 980 bool radv_layout_has_htile(const struct radv_image *image,
 981                            VkImageLayout layout);
 982 bool radv_layout_is_htile_compressed(const struct radv_image *image,
 983                                      VkImageLayout layout);
 984 bool radv_layout_can_expclear(const struct radv_image *image,
 985                               VkImageLayout layout);
 986 bool radv_layout_has_cmask(const struct radv_image *image,
 987                            VkImageLayout layout);
 988 static inline uint32_t
 989 radv_get_layerCount(const struct radv_image *image,
 990                     const VkImageSubresourceRange *range)
 991 {
 992         return range->layerCount == VK_REMAINING_ARRAY_LAYERS ?
 993                 image->array_size - range->baseArrayLayer : range->layerCount;
 994 }
 995
 996 static inline uint32_t
 997 radv_get_levelCount(const struct radv_image *image,
 998                     const VkImageSubresourceRange *range)
 999 {
1000         return range->levelCount == VK_REMAINING_MIP_LEVELS ?
1001                 image->levels - range->baseMipLevel : range->levelCount;
1002 }
1003
1004 struct radeon_bo_metadata;
1005 void
1006 radv_init_metadata(struct radv_device *device,
1007                    struct radv_image *image,
1008                    struct radeon_bo_metadata *metadata);
1009
1010 struct radv_image_view {
1011         struct radv_image *image; /**< VkImageViewCreateInfo::image */
1012         struct radeon_winsys_bo *bo;
1013
1014         VkImageViewType type;
1015         VkImageAspectFlags aspect_mask;
1016         VkFormat vk_format;
1017         uint32_t base_layer;
1018         uint32_t layer_count;
1019         uint32_t base_mip;
1020         VkExtent3D extent; /**< Extent of VkImageViewCreateInfo::baseMipLevel. */
1021
1022         uint32_t descriptor[8];
1023         uint32_t fmask_descriptor[8];
1024 };
1025
1026 struct radv_image_create_info {
1027         const VkImageCreateInfo *vk_info;
1028         uint32_t stride;
1029         bool scanout;
1030 };
1031
1032 VkResult radv_image_create(VkDevice _device,
1033                            const struct radv_image_create_info *info,
1034                            const VkAllocationCallbacks* alloc,
1035                            VkImage *pImage);
1036
1037 void radv_image_view_init(struct radv_image_view *view,
1038                           struct radv_device *device,
1039                           const VkImageViewCreateInfo* pCreateInfo,
1040                           struct radv_cmd_buffer *cmd_buffer,
1041                           VkImageUsageFlags usage_mask);
1042 void radv_image_set_optimal_micro_tile_mode(struct radv_device *device,
1043                                             struct radv_image *image, uint32_t micro_tile_mode);
1044 struct radv_buffer_view {
1045         struct radeon_winsys_bo *bo;
1046         VkFormat vk_format;
1047         uint64_t range; /**< VkBufferViewCreateInfo::range */
1048         uint32_t state[4];
1049 };
1050 void radv_buffer_view_init(struct radv_buffer_view *view,
1051                            struct radv_device *device,
1052                            const VkBufferViewCreateInfo* pCreateInfo,
1053                            struct radv_cmd_buffer *cmd_buffer);
1054
1055 static inline struct VkExtent3D
1056 radv_sanitize_image_extent(const VkImageType imageType,
1057                            const struct VkExtent3D imageExtent)
1058 {
1059         switch (imageType) {
1060         case VK_IMAGE_TYPE_1D:
1061                 return (VkExtent3D) { imageExtent.width, 1, 1 };
1062         case VK_IMAGE_TYPE_2D:
1063                 return (VkExtent3D) { imageExtent.width, imageExtent.height, 1 };
1064         case VK_IMAGE_TYPE_3D:
1065                 return imageExtent;
1066         default:
1067                 unreachable("invalid image type");
1068         }
1069 }
1070
1071 static inline struct VkOffset3D
1072 radv_sanitize_image_offset(const VkImageType imageType,
1073                            const struct VkOffset3D imageOffset)
1074 {
1075         switch (imageType) {
1076         case VK_IMAGE_TYPE_1D:
1077                 return (VkOffset3D) { imageOffset.x, 0, 0 };
1078         case VK_IMAGE_TYPE_2D:
1079                 return (VkOffset3D) { imageOffset.x, imageOffset.y, 0 };
1080         case VK_IMAGE_TYPE_3D:
1081                 return imageOffset;
1082         default:
1083                 unreachable("invalid image type");
1084         }
1085 }
1086
1087 struct radv_sampler {
1088         uint32_t state[4];
1089 };
1090
1091 struct radv_color_buffer_info {
1092         uint32_t cb_color_base;
1093         uint32_t cb_color_pitch;
1094         uint32_t cb_color_slice;
1095         uint32_t cb_color_view;
1096         uint32_t cb_color_info;
1097         uint32_t cb_color_attrib;
1098         uint32_t cb_dcc_control;
1099         uint32_t cb_color_cmask;
1100         uint32_t cb_color_cmask_slice;
1101         uint32_t cb_color_fmask;
1102         uint32_t cb_color_fmask_slice;
1103         uint32_t cb_clear_value0;
1104         uint32_t cb_clear_value1;
1105         uint32_t cb_dcc_base;
1106         uint32_t micro_tile_mode;
1107 };
1108
1109 struct radv_ds_buffer_info {
1110         uint32_t db_depth_info;
1111         uint32_t db_z_info;
1112         uint32_t db_stencil_info;
1113         uint32_t db_z_read_base;
1114         uint32_t db_stencil_read_base;
1115         uint32_t db_z_write_base;
1116         uint32_t db_stencil_write_base;
1117         uint32_t db_depth_view;
1118         uint32_t db_depth_size;
1119         uint32_t db_depth_slice;
1120         uint32_t db_htile_surface;
1121         uint32_t db_htile_data_base;
1122         uint32_t pa_su_poly_offset_db_fmt_cntl;
1123         float offset_scale;
1124 };
1125
1126 struct radv_attachment_info {
1127         union {
1128                 struct radv_color_buffer_info cb;
1129                 struct radv_ds_buffer_info ds;
1130         };
1131         struct radv_image_view *attachment;
1132 };
1133
1134 struct radv_framebuffer {
1135         uint32_t                                     width;
1136         uint32_t                                     height;
1137         uint32_t                                     layers;
1138
1139         uint32_t                                     attachment_count;
1140         struct radv_attachment_info                  attachments[0];
1141 };
1142
1143 struct radv_subpass_barrier {
1144         VkPipelineStageFlags src_stage_mask;
1145         VkAccessFlags        src_access_mask;
1146         VkAccessFlags        dst_access_mask;
1147 };
1148
1149 struct radv_subpass {
1150         uint32_t                                     input_count;
1151         VkAttachmentReference *                      input_attachments;
1152         uint32_t                                     color_count;
1153         VkAttachmentReference *                      color_attachments;
1154         VkAttachmentReference *                      resolve_attachments;
1155         VkAttachmentReference                        depth_stencil_attachment;
1156
1157         /** Subpass has at least one resolve attachment */
1158         bool                                         has_resolve;
1159
1160         struct radv_subpass_barrier                  start_barrier;
1161 };
1162
1163 struct radv_render_pass_attachment {
1164         VkFormat                                     format;
1165         uint32_t                                     samples;
1166         VkAttachmentLoadOp                           load_op;
1167         VkAttachmentLoadOp                           stencil_load_op;
1168         VkImageLayout                                initial_layout;
1169         VkImageLayout                                final_layout;
1170 };
1171
1172 struct radv_render_pass {
1173         uint32_t                                     attachment_count;
1174         uint32_t                                     subpass_count;
1175         VkAttachmentReference *                      subpass_attachments;
1176         struct radv_render_pass_attachment *         attachments;
1177         struct radv_subpass_barrier                  end_barrier;
1178         struct radv_subpass                          subpasses[0];
1179 };
1180
1181 VkResult radv_device_init_meta(struct radv_device *device);
1182 void radv_device_finish_meta(struct radv_device *device);
1183
1184 struct radv_query_pool {
1185         struct radeon_winsys_bo *bo;
1186         uint32_t stride;
1187         uint32_t availability_offset;
1188         char *ptr;
1189         VkQueryType type;
1190 };
1191
1192 VkResult
1193 radv_temp_descriptor_set_create(struct radv_device *device,
1194                                 struct radv_cmd_buffer *cmd_buffer,
1195                                 VkDescriptorSetLayout _layout,
1196                                 VkDescriptorSet *_set);
1197
1198 void
1199 radv_temp_descriptor_set_destroy(struct radv_device *device,
1200                                  VkDescriptorSet _set);
1201 void radv_initialise_cmask(struct radv_cmd_buffer *cmd_buffer,
1202                            struct radv_image *image, uint32_t value);
1203 void radv_initialize_dcc(struct radv_cmd_buffer *cmd_buffer,
1204                          struct radv_image *image, uint32_t value);
1205
1206 struct radv_fence {
1207         struct radeon_winsys_fence *fence;
1208         bool submitted;
1209         bool signalled;
1210 };
1211
1212 #define RADV_DEFINE_HANDLE_CASTS(__radv_type, __VkType)         \
1213                                                                 \
1214         static inline struct __radv_type *                      \
1215         __radv_type ## _from_handle(__VkType _handle)           \
1216         {                                                       \
1217                 return (struct __radv_type *) _handle;          \
1218         }                                                       \
1219                                                                 \
1220         static inline __VkType                                  \
1221         __radv_type ## _to_handle(struct __radv_type *_obj)     \
1222         {                                                       \
1223                 return (__VkType) _obj;                         \
1224         }
1225
1226 #define RADV_DEFINE_NONDISP_HANDLE_CASTS(__radv_type, __VkType)         \
1227                                                                         \
1228         static inline struct __radv_type *                              \
1229         __radv_type ## _from_handle(__VkType _handle)                   \
1230         {                                                               \
1231                 return (struct __radv_type *)(uintptr_t) _handle;       \
1232         }                                                               \
1233                                                                         \
1234         static inline __VkType                                          \
1235         __radv_type ## _to_handle(struct __radv_type *_obj)             \
1236         {                                                               \
1237                 return (__VkType)(uintptr_t) _obj;                      \
1238         }
1239
1240 #define RADV_FROM_HANDLE(__radv_type, __name, __handle)                 \
1241         struct __radv_type *__name = __radv_type ## _from_handle(__handle)
1242
1243 RADV_DEFINE_HANDLE_CASTS(radv_cmd_buffer, VkCommandBuffer)
1244 RADV_DEFINE_HANDLE_CASTS(radv_device, VkDevice)
1245 RADV_DEFINE_HANDLE_CASTS(radv_instance, VkInstance)
1246 RADV_DEFINE_HANDLE_CASTS(radv_physical_device, VkPhysicalDevice)
1247 RADV_DEFINE_HANDLE_CASTS(radv_queue, VkQueue)
1248
1249 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_cmd_pool, VkCommandPool)
1250 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_buffer, VkBuffer)
1251 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_buffer_view, VkBufferView)
1252 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_pool, VkDescriptorPool)
1253 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_set, VkDescriptorSet)
1254 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_set_layout, VkDescriptorSetLayout)
1255 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_device_memory, VkDeviceMemory)
1256 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_fence, VkFence)
1257 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_event, VkEvent)
1258 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_framebuffer, VkFramebuffer)
1259 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_image, VkImage)
1260 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_image_view, VkImageView);
1261 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_pipeline_cache, VkPipelineCache)
1262 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_pipeline, VkPipeline)
1263 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_pipeline_layout, VkPipelineLayout)
1264 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_query_pool, VkQueryPool)
1265 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_render_pass, VkRenderPass)
1266 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_sampler, VkSampler)
1267 RADV_DEFINE_NONDISP_HANDLE_CASTS(radv_shader_module, VkShaderModule)
1268
1269 #define RADV_DEFINE_STRUCT_CASTS(__radv_type, __VkType)                 \
1270                                                                         \
1271         static inline const __VkType *                                  \
1272         __radv_type ## _to_ ## __VkType(const struct __radv_type *__radv_obj) \
1273         {                                                               \
1274                 return (const __VkType *) __radv_obj;                   \
1275         }
1276
1277 #define RADV_COMMON_TO_STRUCT(__VkType, __vk_name, __common_name)       \
1278         const __VkType *__vk_name = radv_common_to_ ## __VkType(__common_name)
1279
1280 RADV_DEFINE_STRUCT_CASTS(radv_common, VkMemoryBarrier)
1281 RADV_DEFINE_STRUCT_CASTS(radv_common, VkBufferMemoryBarrier)
1282 RADV_DEFINE_STRUCT_CASTS(radv_common, VkImageMemoryBarrier)
1283
1284
1285 #endif /* RADV_PRIVATE_H */