src/freedreno/vulkan/tu_descriptor_set.c

   1 /*
   2  * Copyright © 2016 Red Hat.
   3  * Copyright © 2016 Bas Nieuwenhuizen
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22  * DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 /**
  26  * @file
  27  *
  28  * The texture and sampler descriptors are laid out in a single global space
  29  * across all shader stages, for both simplicity of implementation and because
  30  * that seems to be how things have to be structured for border color
  31  * handling.
  32  *
  33  * Each shader stage will declare its texture/sampler count based on the last
  34  * descriptor set it uses.  At draw emit time (though it really should be
  35  * CmdBind time), we upload the descriptor sets used by each shader stage to
  36  * their stage.
  37  */
  38
  39 #include "tu_private.h"
  40
  41 #include <assert.h>
  42 #include <fcntl.h>
  43 #include <stdbool.h>
  44 #include <string.h>
  45 #include <unistd.h>
  46
  47 #include "util/mesa-sha1.h"
  48 #include "vk_util.h"
  49
  50 static int
  51 binding_compare(const void *av, const void *bv)
  52 {
  53    const VkDescriptorSetLayoutBinding *a =
  54       (const VkDescriptorSetLayoutBinding *) av;
  55    const VkDescriptorSetLayoutBinding *b =
  56       (const VkDescriptorSetLayoutBinding *) bv;
  57
  58    return (a->binding < b->binding) ? -1 : (a->binding > b->binding) ? 1 : 0;
  59 }
  60
  61 static VkDescriptorSetLayoutBinding *
  62 create_sorted_bindings(const VkDescriptorSetLayoutBinding *bindings,
  63                        unsigned count)
  64 {
  65    VkDescriptorSetLayoutBinding *sorted_bindings =
  66       malloc(count * sizeof(VkDescriptorSetLayoutBinding));
  67    if (!sorted_bindings)
  68       return NULL;
  69
  70    memcpy(sorted_bindings, bindings,
  71           count * sizeof(VkDescriptorSetLayoutBinding));
  72
  73    qsort(sorted_bindings, count, sizeof(VkDescriptorSetLayoutBinding),
  74          binding_compare);
  75
  76    return sorted_bindings;
  77 }
  78
  79 static uint32_t
  80 descriptor_size(enum VkDescriptorType type)
  81 {
  82    switch (type) {
  83    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
  84    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
  85       return 0;
  86    case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
  87    case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
  88       /* 64bit pointer */
  89       return 8;
  90    case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
  91    case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
  92    case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
  93    case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
  94       return A6XX_TEX_CONST_DWORDS * 4;
  95    case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
  96       /* We may need the IBO or the TEX representation, or both. */
  97       return A6XX_TEX_CONST_DWORDS * 4 * 2;
  98    case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
  99       /* texture const + tu_sampler struct (includes border color) */
 100       return A6XX_TEX_CONST_DWORDS * 4 + sizeof(struct tu_sampler);
 101    case VK_DESCRIPTOR_TYPE_SAMPLER:
 102       return sizeof(struct tu_sampler);
 103    default:
 104       unreachable("unknown descriptor type\n");
 105       return 0;
 106    }
 107 }
 108
 109 VkResult
 110 tu_CreateDescriptorSetLayout(
 111    VkDevice _device,
 112    const VkDescriptorSetLayoutCreateInfo *pCreateInfo,
 113    const VkAllocationCallbacks *pAllocator,
 114    VkDescriptorSetLayout *pSetLayout)
 115 {
 116    TU_FROM_HANDLE(tu_device, device, _device);
 117    struct tu_descriptor_set_layout *set_layout;
 118
 119    assert(pCreateInfo->sType ==
 120           VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO);
 121    const VkDescriptorSetLayoutBindingFlagsCreateInfoEXT *variable_flags =
 122       vk_find_struct_const(
 123          pCreateInfo->pNext,
 124          DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO_EXT);
 125
 126    uint32_t max_binding = 0;
 127    uint32_t immutable_sampler_count = 0;
 128    for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
 129       max_binding = MAX2(max_binding, pCreateInfo->pBindings[j].binding);
 130       if ((pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER ||
 131            pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) &&
 132            pCreateInfo->pBindings[j].pImmutableSamplers) {
 133          immutable_sampler_count += pCreateInfo->pBindings[j].descriptorCount;
 134       }
 135    }
 136
 137    uint32_t samplers_offset = sizeof(struct tu_descriptor_set_layout) +
 138       (max_binding + 1) * sizeof(set_layout->binding[0]);
 139    uint32_t size = samplers_offset + immutable_sampler_count * sizeof(struct tu_sampler);
 140
 141    set_layout = vk_alloc2(&device->alloc, pAllocator, size, 8,
 142                           VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 143    if (!set_layout)
 144       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
 145
 146    set_layout->flags = pCreateInfo->flags;
 147
 148    /* We just allocate all the samplers at the end of the struct */
 149    struct tu_sampler *samplers = (void*) &set_layout->binding[max_binding + 1];
 150
 151    VkDescriptorSetLayoutBinding *bindings = create_sorted_bindings(
 152       pCreateInfo->pBindings, pCreateInfo->bindingCount);
 153    if (!bindings) {
 154       vk_free2(&device->alloc, pAllocator, set_layout);
 155       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
 156    }
 157
 158    set_layout->binding_count = max_binding + 1;
 159    set_layout->shader_stages = 0;
 160    set_layout->dynamic_shader_stages = 0;
 161    set_layout->has_immutable_samplers = false;
 162    set_layout->size = 0;
 163
 164    memset(set_layout->binding, 0,
 165           size - sizeof(struct tu_descriptor_set_layout));
 166
 167    uint32_t buffer_count = 0;
 168    uint32_t dynamic_offset_count = 0;
 169
 170    for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
 171       const VkDescriptorSetLayoutBinding *binding = bindings + j;
 172       uint32_t b = binding->binding;
 173       uint32_t alignment = 4;
 174       unsigned binding_buffer_count = 1;
 175
 176       switch (binding->descriptorType) {
 177       case VK_DESCRIPTOR_TYPE_SAMPLER:
 178          binding_buffer_count = 0;
 179          break;
 180       case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
 181       case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
 182          assert(!(pCreateInfo->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
 183          set_layout->binding[b].dynamic_offset_count = 1;
 184          break;
 185       default:
 186          break;
 187       }
 188
 189       set_layout->size = align(set_layout->size, alignment);
 190       set_layout->binding[b].type = binding->descriptorType;
 191       set_layout->binding[b].array_size = binding->descriptorCount;
 192       set_layout->binding[b].offset = set_layout->size;
 193       set_layout->binding[b].buffer_offset = buffer_count;
 194       set_layout->binding[b].dynamic_offset_offset = dynamic_offset_count;
 195       set_layout->binding[b].size = descriptor_size(binding->descriptorType);
 196
 197       if (variable_flags && binding->binding < variable_flags->bindingCount &&
 198           (variable_flags->pBindingFlags[binding->binding] &
 199            VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT_EXT)) {
 200          assert(!binding->pImmutableSamplers); /* Terribly ill defined  how
 201                                                   many samplers are valid */
 202          assert(binding->binding == max_binding);
 203
 204          set_layout->has_variable_descriptors = true;
 205       }
 206
 207       if ((binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER ||
 208            binding->descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) &&
 209           binding->pImmutableSamplers) {
 210          set_layout->binding[b].immutable_samplers_offset = samplers_offset;
 211          set_layout->has_immutable_samplers = true;
 212
 213          for (uint32_t i = 0; i < binding->descriptorCount; i++)
 214             samplers[i] = *tu_sampler_from_handle(binding->pImmutableSamplers[i]);
 215
 216          samplers += binding->descriptorCount;
 217          samplers_offset += sizeof(struct tu_sampler) * binding->descriptorCount;
 218       }
 219
 220       set_layout->size +=
 221          binding->descriptorCount * set_layout->binding[b].size;
 222       buffer_count += binding->descriptorCount * binding_buffer_count;
 223       dynamic_offset_count += binding->descriptorCount *
 224                               set_layout->binding[b].dynamic_offset_count;
 225       set_layout->shader_stages |= binding->stageFlags;
 226    }
 227
 228    free(bindings);
 229
 230    set_layout->buffer_count = buffer_count;
 231    set_layout->dynamic_offset_count = dynamic_offset_count;
 232
 233    *pSetLayout = tu_descriptor_set_layout_to_handle(set_layout);
 234
 235    return VK_SUCCESS;
 236 }
 237
 238 void
 239 tu_DestroyDescriptorSetLayout(VkDevice _device,
 240                               VkDescriptorSetLayout _set_layout,
 241                               const VkAllocationCallbacks *pAllocator)
 242 {
 243    TU_FROM_HANDLE(tu_device, device, _device);
 244    TU_FROM_HANDLE(tu_descriptor_set_layout, set_layout, _set_layout);
 245
 246    if (!set_layout)
 247       return;
 248
 249    vk_free2(&device->alloc, pAllocator, set_layout);
 250 }
 251
 252 void
 253 tu_GetDescriptorSetLayoutSupport(
 254    VkDevice device,
 255    const VkDescriptorSetLayoutCreateInfo *pCreateInfo,
 256    VkDescriptorSetLayoutSupport *pSupport)
 257 {
 258    VkDescriptorSetLayoutBinding *bindings = create_sorted_bindings(
 259       pCreateInfo->pBindings, pCreateInfo->bindingCount);
 260    if (!bindings) {
 261       pSupport->supported = false;
 262       return;
 263    }
 264
 265    const VkDescriptorSetLayoutBindingFlagsCreateInfoEXT *variable_flags =
 266       vk_find_struct_const(
 267          pCreateInfo->pNext,
 268          DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO_EXT);
 269    VkDescriptorSetVariableDescriptorCountLayoutSupportEXT *variable_count =
 270       vk_find_struct(
 271          (void *) pCreateInfo->pNext,
 272          DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_LAYOUT_SUPPORT_EXT);
 273    if (variable_count) {
 274       variable_count->maxVariableDescriptorCount = 0;
 275    }
 276
 277    bool supported = true;
 278    uint64_t size = 0;
 279    for (uint32_t i = 0; i < pCreateInfo->bindingCount; i++) {
 280       const VkDescriptorSetLayoutBinding *binding = bindings + i;
 281
 282       uint64_t descriptor_sz = descriptor_size(binding->descriptorType);
 283       uint64_t descriptor_alignment = 8;
 284
 285       if (size && !align_u64(size, descriptor_alignment)) {
 286          supported = false;
 287       }
 288       size = align_u64(size, descriptor_alignment);
 289
 290       uint64_t max_count = UINT64_MAX;
 291       if (descriptor_sz)
 292          max_count = (UINT64_MAX - size) / descriptor_sz;
 293
 294       if (max_count < binding->descriptorCount) {
 295          supported = false;
 296       }
 297       if (variable_flags && binding->binding < variable_flags->bindingCount &&
 298           variable_count &&
 299           (variable_flags->pBindingFlags[binding->binding] &
 300            VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT_EXT)) {
 301          variable_count->maxVariableDescriptorCount =
 302             MIN2(UINT32_MAX, max_count);
 303       }
 304       size += binding->descriptorCount * descriptor_sz;
 305    }
 306
 307    free(bindings);
 308
 309    pSupport->supported = supported;
 310 }
 311
 312 /*
 313  * Pipeline layouts.  These have nothing to do with the pipeline.  They are
 314  * just multiple descriptor set layouts pasted together.
 315  */
 316
 317 VkResult
 318 tu_CreatePipelineLayout(VkDevice _device,
 319                         const VkPipelineLayoutCreateInfo *pCreateInfo,
 320                         const VkAllocationCallbacks *pAllocator,
 321                         VkPipelineLayout *pPipelineLayout)
 322 {
 323    TU_FROM_HANDLE(tu_device, device, _device);
 324    struct tu_pipeline_layout *layout;
 325    struct mesa_sha1 ctx;
 326
 327    assert(pCreateInfo->sType ==
 328           VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO);
 329
 330    layout = vk_alloc2(&device->alloc, pAllocator, sizeof(*layout), 8,
 331                       VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 332    if (layout == NULL)
 333       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
 334
 335    layout->num_sets = pCreateInfo->setLayoutCount;
 336
 337    unsigned dynamic_offset_count = 0;
 338
 339    _mesa_sha1_init(&ctx);
 340    for (uint32_t set = 0; set < pCreateInfo->setLayoutCount; set++) {
 341       TU_FROM_HANDLE(tu_descriptor_set_layout, set_layout,
 342                      pCreateInfo->pSetLayouts[set]);
 343       layout->set[set].layout = set_layout;
 344
 345       layout->set[set].dynamic_offset_start = dynamic_offset_count;
 346       for (uint32_t b = 0; b < set_layout->binding_count; b++) {
 347          dynamic_offset_count += set_layout->binding[b].array_size *
 348                                  set_layout->binding[b].dynamic_offset_count;
 349          if (set_layout->binding[b].immutable_samplers_offset)
 350             _mesa_sha1_update(
 351                &ctx,
 352                tu_immutable_samplers(set_layout, set_layout->binding + b),
 353                set_layout->binding[b].array_size * 4 * sizeof(uint32_t));
 354       }
 355       _mesa_sha1_update(
 356          &ctx, set_layout->binding,
 357          sizeof(set_layout->binding[0]) * set_layout->binding_count);
 358    }
 359
 360    layout->dynamic_offset_count = dynamic_offset_count;
 361    layout->push_constant_size = 0;
 362
 363    for (unsigned i = 0; i < pCreateInfo->pushConstantRangeCount; ++i) {
 364       const VkPushConstantRange *range = pCreateInfo->pPushConstantRanges + i;
 365       layout->push_constant_size =
 366          MAX2(layout->push_constant_size, range->offset + range->size);
 367    }
 368
 369    layout->push_constant_size = align(layout->push_constant_size, 16);
 370    _mesa_sha1_update(&ctx, &layout->push_constant_size,
 371                      sizeof(layout->push_constant_size));
 372    _mesa_sha1_final(&ctx, layout->sha1);
 373    *pPipelineLayout = tu_pipeline_layout_to_handle(layout);
 374
 375    return VK_SUCCESS;
 376 }
 377
 378 void
 379 tu_DestroyPipelineLayout(VkDevice _device,
 380                          VkPipelineLayout _pipelineLayout,
 381                          const VkAllocationCallbacks *pAllocator)
 382 {
 383    TU_FROM_HANDLE(tu_device, device, _device);
 384    TU_FROM_HANDLE(tu_pipeline_layout, pipeline_layout, _pipelineLayout);
 385
 386    if (!pipeline_layout)
 387       return;
 388    vk_free2(&device->alloc, pAllocator, pipeline_layout);
 389 }
 390
 391 #define EMPTY 1
 392
 393 static VkResult
 394 tu_descriptor_set_create(struct tu_device *device,
 395             struct tu_descriptor_pool *pool,
 396             const struct tu_descriptor_set_layout *layout,
 397             const uint32_t *variable_count,
 398             struct tu_descriptor_set **out_set)
 399 {
 400    struct tu_descriptor_set *set;
 401    uint32_t buffer_count = layout->buffer_count;
 402    if (variable_count) {
 403       unsigned stride = 1;
 404       if (layout->binding[layout->binding_count - 1].type == VK_DESCRIPTOR_TYPE_SAMPLER ||
 405           layout->binding[layout->binding_count - 1].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT)
 406          stride = 0;
 407       buffer_count = layout->binding[layout->binding_count - 1].buffer_offset +
 408                      *variable_count * stride;
 409    }
 410    unsigned range_offset = sizeof(struct tu_descriptor_set) +
 411       sizeof(struct tu_bo *) * buffer_count;
 412    unsigned mem_size = range_offset +
 413       sizeof(struct tu_descriptor_range) * layout->dynamic_offset_count;
 414
 415    if (pool->host_memory_base) {
 416       if (pool->host_memory_end - pool->host_memory_ptr < mem_size)
 417          return vk_error(device->instance, VK_ERROR_OUT_OF_POOL_MEMORY);
 418
 419       set = (struct tu_descriptor_set*)pool->host_memory_ptr;
 420       pool->host_memory_ptr += mem_size;
 421    } else {
 422       set = vk_alloc2(&device->alloc, NULL, mem_size, 8,
 423                       VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 424
 425       if (!set)
 426          return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
 427    }
 428
 429    memset(set, 0, mem_size);
 430
 431    if (layout->dynamic_offset_count) {
 432       set->dynamic_descriptors = (struct tu_descriptor_range*)((uint8_t*)set + range_offset);
 433    }
 434
 435    set->layout = layout;
 436    uint32_t layout_size = layout->size;
 437    if (variable_count) {
 438       assert(layout->has_variable_descriptors);
 439       uint32_t stride = layout->binding[layout->binding_count - 1].size;
 440       if (layout->binding[layout->binding_count - 1].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT)
 441          stride = 1;
 442
 443       layout_size = layout->binding[layout->binding_count - 1].offset +
 444                     *variable_count * stride;
 445    }
 446
 447    if (layout_size) {
 448       set->size = layout_size;
 449
 450       if (!pool->host_memory_base && pool->entry_count == pool->max_entry_count) {
 451          vk_free2(&device->alloc, NULL, set);
 452          return vk_error(device->instance, VK_ERROR_OUT_OF_POOL_MEMORY);
 453       }
 454
 455       /* try to allocate linearly first, so that we don't spend
 456        * time looking for gaps if the app only allocates &
 457        * resets via the pool. */
 458       if (pool->current_offset + layout_size <= pool->size) {
 459          set->mapped_ptr = (uint32_t*)(pool->bo.map + pool->current_offset);
 460          set->va = pool->bo.iova + pool->current_offset;
 461          if (!pool->host_memory_base) {
 462             pool->entries[pool->entry_count].offset = pool->current_offset;
 463             pool->entries[pool->entry_count].size = layout_size;
 464             pool->entries[pool->entry_count].set = set;
 465             pool->entry_count++;
 466          }
 467          pool->current_offset += layout_size;
 468       } else if (!pool->host_memory_base) {
 469          uint64_t offset = 0;
 470          int index;
 471
 472          for (index = 0; index < pool->entry_count; ++index) {
 473             if (pool->entries[index].offset - offset >= layout_size)
 474                break;
 475             offset = pool->entries[index].offset + pool->entries[index].size;
 476          }
 477
 478          if (pool->size - offset < layout_size) {
 479             vk_free2(&device->alloc, NULL, set);
 480             return vk_error(device->instance, VK_ERROR_OUT_OF_POOL_MEMORY);
 481          }
 482
 483          set->mapped_ptr = (uint32_t*)(pool->bo.map + offset);
 484          set->va = pool->bo.iova + offset;
 485          memmove(&pool->entries[index + 1], &pool->entries[index],
 486             sizeof(pool->entries[0]) * (pool->entry_count - index));
 487          pool->entries[index].offset = offset;
 488          pool->entries[index].size = layout_size;
 489          pool->entries[index].set = set;
 490          pool->entry_count++;
 491       } else
 492          return vk_error(device->instance, VK_ERROR_OUT_OF_POOL_MEMORY);
 493    }
 494
 495    *out_set = set;
 496    return VK_SUCCESS;
 497 }
 498
 499 static void
 500 tu_descriptor_set_destroy(struct tu_device *device,
 501              struct tu_descriptor_pool *pool,
 502              struct tu_descriptor_set *set,
 503              bool free_bo)
 504 {
 505    assert(!pool->host_memory_base);
 506
 507    if (free_bo && set->size && !pool->host_memory_base) {
 508       uint32_t offset = (uint8_t*)set->mapped_ptr - (uint8_t*)pool->bo.map;
 509       for (int i = 0; i < pool->entry_count; ++i) {
 510          if (pool->entries[i].offset == offset) {
 511             memmove(&pool->entries[i], &pool->entries[i+1],
 512                sizeof(pool->entries[i]) * (pool->entry_count - i - 1));
 513             --pool->entry_count;
 514             break;
 515          }
 516       }
 517    }
 518    vk_free2(&device->alloc, NULL, set);
 519 }
 520
 521 VkResult
 522 tu_CreateDescriptorPool(VkDevice _device,
 523                         const VkDescriptorPoolCreateInfo *pCreateInfo,
 524                         const VkAllocationCallbacks *pAllocator,
 525                         VkDescriptorPool *pDescriptorPool)
 526 {
 527    TU_FROM_HANDLE(tu_device, device, _device);
 528    struct tu_descriptor_pool *pool;
 529    uint64_t size = sizeof(struct tu_descriptor_pool);
 530    uint64_t bo_size = 0, bo_count = 0, range_count = 0;
 531
 532    for (unsigned i = 0; i < pCreateInfo->poolSizeCount; ++i) {
 533       if (pCreateInfo->pPoolSizes[i].type != VK_DESCRIPTOR_TYPE_SAMPLER)
 534          bo_count += pCreateInfo->pPoolSizes[i].descriptorCount;
 535
 536       switch(pCreateInfo->pPoolSizes[i].type) {
 537       case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
 538       case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
 539          range_count += pCreateInfo->pPoolSizes[i].descriptorCount;
 540       default:
 541          break;
 542       }
 543
 544       bo_size += descriptor_size(pCreateInfo->pPoolSizes[i].type) *
 545                            pCreateInfo->pPoolSizes[i].descriptorCount;
 546    }
 547
 548    if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) {
 549       uint64_t host_size = pCreateInfo->maxSets * sizeof(struct tu_descriptor_set);
 550       host_size += sizeof(struct tu_bo*) * bo_count;
 551       host_size += sizeof(struct tu_descriptor_range) * range_count;
 552       size += host_size;
 553    } else {
 554       size += sizeof(struct tu_descriptor_pool_entry) * pCreateInfo->maxSets;
 555    }
 556
 557    pool = vk_alloc2(&device->alloc, pAllocator, size, 8,
 558                     VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 559    if (!pool)
 560       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
 561
 562    memset(pool, 0, sizeof(*pool));
 563
 564    if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) {
 565       pool->host_memory_base = (uint8_t*)pool + sizeof(struct tu_descriptor_pool);
 566       pool->host_memory_ptr = pool->host_memory_base;
 567       pool->host_memory_end = (uint8_t*)pool + size;
 568    }
 569
 570    if (bo_size) {
 571       VkResult ret;
 572
 573       ret = tu_bo_init_new(device, &pool->bo, bo_size);
 574       assert(ret == VK_SUCCESS);
 575
 576       ret = tu_bo_map(device, &pool->bo);
 577       assert(ret == VK_SUCCESS);
 578    }
 579    pool->size = bo_size;
 580    pool->max_entry_count = pCreateInfo->maxSets;
 581
 582    *pDescriptorPool = tu_descriptor_pool_to_handle(pool);
 583    return VK_SUCCESS;
 584 }
 585
 586 void
 587 tu_DestroyDescriptorPool(VkDevice _device,
 588                          VkDescriptorPool _pool,
 589                          const VkAllocationCallbacks *pAllocator)
 590 {
 591    TU_FROM_HANDLE(tu_device, device, _device);
 592    TU_FROM_HANDLE(tu_descriptor_pool, pool, _pool);
 593
 594    if (!pool)
 595       return;
 596
 597    if (!pool->host_memory_base) {
 598       for(int i = 0; i < pool->entry_count; ++i) {
 599          tu_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
 600       }
 601    }
 602
 603    if (pool->size)
 604       tu_bo_finish(device, &pool->bo);
 605    vk_free2(&device->alloc, pAllocator, pool);
 606 }
 607
 608 VkResult
 609 tu_ResetDescriptorPool(VkDevice _device,
 610                        VkDescriptorPool descriptorPool,
 611                        VkDescriptorPoolResetFlags flags)
 612 {
 613    TU_FROM_HANDLE(tu_device, device, _device);
 614    TU_FROM_HANDLE(tu_descriptor_pool, pool, descriptorPool);
 615
 616    if (!pool->host_memory_base) {
 617       for(int i = 0; i < pool->entry_count; ++i) {
 618          tu_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
 619       }
 620       pool->entry_count = 0;
 621    }
 622
 623    pool->current_offset = 0;
 624    pool->host_memory_ptr = pool->host_memory_base;
 625
 626    return VK_SUCCESS;
 627 }
 628
 629 VkResult
 630 tu_AllocateDescriptorSets(VkDevice _device,
 631                           const VkDescriptorSetAllocateInfo *pAllocateInfo,
 632                           VkDescriptorSet *pDescriptorSets)
 633 {
 634    TU_FROM_HANDLE(tu_device, device, _device);
 635    TU_FROM_HANDLE(tu_descriptor_pool, pool, pAllocateInfo->descriptorPool);
 636
 637    VkResult result = VK_SUCCESS;
 638    uint32_t i;
 639    struct tu_descriptor_set *set = NULL;
 640
 641    const VkDescriptorSetVariableDescriptorCountAllocateInfoEXT *variable_counts =
 642       vk_find_struct_const(pAllocateInfo->pNext, DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_ALLOCATE_INFO_EXT);
 643    const uint32_t zero = 0;
 644
 645    /* allocate a set of buffers for each shader to contain descriptors */
 646    for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) {
 647       TU_FROM_HANDLE(tu_descriptor_set_layout, layout,
 648              pAllocateInfo->pSetLayouts[i]);
 649
 650       const uint32_t *variable_count = NULL;
 651       if (variable_counts) {
 652          if (i < variable_counts->descriptorSetCount)
 653             variable_count = variable_counts->pDescriptorCounts + i;
 654          else
 655             variable_count = &zero;
 656       }
 657
 658       assert(!(layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
 659
 660       result = tu_descriptor_set_create(device, pool, layout, variable_count, &set);
 661       if (result != VK_SUCCESS)
 662          break;
 663
 664       pDescriptorSets[i] = tu_descriptor_set_to_handle(set);
 665    }
 666
 667    if (result != VK_SUCCESS) {
 668       tu_FreeDescriptorSets(_device, pAllocateInfo->descriptorPool,
 669                i, pDescriptorSets);
 670       for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) {
 671          pDescriptorSets[i] = VK_NULL_HANDLE;
 672       }
 673    }
 674    return result;
 675 }
 676
 677 VkResult
 678 tu_FreeDescriptorSets(VkDevice _device,
 679                       VkDescriptorPool descriptorPool,
 680                       uint32_t count,
 681                       const VkDescriptorSet *pDescriptorSets)
 682 {
 683    TU_FROM_HANDLE(tu_device, device, _device);
 684    TU_FROM_HANDLE(tu_descriptor_pool, pool, descriptorPool);
 685
 686    for (uint32_t i = 0; i < count; i++) {
 687       TU_FROM_HANDLE(tu_descriptor_set, set, pDescriptorSets[i]);
 688
 689       if (set && !pool->host_memory_base)
 690          tu_descriptor_set_destroy(device, pool, set, true);
 691    }
 692    return VK_SUCCESS;
 693 }
 694
 695 static void write_texel_buffer_descriptor(struct tu_device *device,
 696                                           struct tu_cmd_buffer *cmd_buffer,
 697                                           unsigned *dst,
 698                                           struct tu_bo **buffer_list,
 699                                           const VkBufferView buffer_view)
 700 {
 701    TU_FROM_HANDLE(tu_buffer_view, view, buffer_view);
 702
 703    memcpy(dst, view->descriptor, sizeof(view->descriptor));
 704
 705    if (cmd_buffer)
 706       tu_bo_list_add(&cmd_buffer->bo_list, view->buffer->bo, MSM_SUBMIT_BO_READ);
 707    else
 708       *buffer_list = view->buffer->bo;
 709 }
 710
 711 static void write_buffer_descriptor(struct tu_device *device,
 712                                     struct tu_cmd_buffer *cmd_buffer,
 713                                     unsigned *dst,
 714                                     struct tu_bo **buffer_list,
 715                                     const VkDescriptorBufferInfo *buffer_info)
 716 {
 717    TU_FROM_HANDLE(tu_buffer, buffer, buffer_info->buffer);
 718
 719    uint64_t va = tu_buffer_iova(buffer) + buffer_info->offset;
 720    dst[0] = va;
 721    dst[1] = va >> 32;
 722
 723    if (cmd_buffer)
 724       tu_bo_list_add(&cmd_buffer->bo_list, buffer->bo, MSM_SUBMIT_BO_READ);
 725    else
 726       *buffer_list = buffer->bo;
 727 }
 728
 729 static void write_dynamic_buffer_descriptor(struct tu_device *device,
 730                                             struct tu_descriptor_range *range,
 731                                             struct tu_bo **buffer_list,
 732                                             const VkDescriptorBufferInfo *buffer_info)
 733 {
 734    TU_FROM_HANDLE(tu_buffer, buffer, buffer_info->buffer);
 735    uint64_t va = tu_buffer_iova(buffer) + buffer_info->offset;
 736    unsigned size = buffer_info->range;
 737
 738    if (buffer_info->range == VK_WHOLE_SIZE)
 739       size = buffer->size - buffer_info->offset;
 740
 741    range->va = va;
 742    range->size = size;
 743
 744    *buffer_list = buffer->bo;
 745 }
 746
 747 static void
 748 write_image_descriptor(struct tu_device *device,
 749              struct tu_cmd_buffer *cmd_buffer,
 750              unsigned *dst,
 751              struct tu_bo **buffer_list,
 752              VkDescriptorType descriptor_type,
 753              const VkDescriptorImageInfo *image_info)
 754 {
 755    TU_FROM_HANDLE(tu_image_view, iview, image_info->imageView);
 756
 757    memcpy(dst, iview->descriptor, sizeof(iview->descriptor));
 758    if (descriptor_type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) {
 759       memcpy(&dst[A6XX_TEX_CONST_DWORDS], iview->storage_descriptor,
 760              sizeof(iview->storage_descriptor));
 761    }
 762
 763    if (cmd_buffer)
 764       tu_bo_list_add(&cmd_buffer->bo_list, iview->image->bo, MSM_SUBMIT_BO_READ);
 765    else
 766       *buffer_list = iview->image->bo;
 767 }
 768
 769 static void
 770 write_combined_image_sampler_descriptor(struct tu_device *device,
 771                struct tu_cmd_buffer *cmd_buffer,
 772                unsigned sampler_offset,
 773                unsigned *dst,
 774                struct tu_bo **buffer_list,
 775                VkDescriptorType descriptor_type,
 776                const VkDescriptorImageInfo *image_info,
 777                bool has_sampler)
 778 {
 779    TU_FROM_HANDLE(tu_sampler, sampler, image_info->sampler);
 780
 781    write_image_descriptor(device, cmd_buffer, dst, buffer_list,
 782                           descriptor_type, image_info);
 783    /* copy over sampler state */
 784    if (has_sampler) {
 785       memcpy(dst + sampler_offset / sizeof(*dst), sampler, sizeof(*sampler));
 786    }
 787 }
 788
 789 static void
 790 write_sampler_descriptor(struct tu_device *device,
 791                          unsigned *dst,
 792                          const VkDescriptorImageInfo *image_info)
 793 {
 794    TU_FROM_HANDLE(tu_sampler, sampler, image_info->sampler);
 795
 796    memcpy(dst, sampler, sizeof(*sampler));
 797 }
 798
 799 void
 800 tu_update_descriptor_sets(struct tu_device *device,
 801                           struct tu_cmd_buffer *cmd_buffer,
 802                           VkDescriptorSet dstSetOverride,
 803                           uint32_t descriptorWriteCount,
 804                           const VkWriteDescriptorSet *pDescriptorWrites,
 805                           uint32_t descriptorCopyCount,
 806                           const VkCopyDescriptorSet *pDescriptorCopies)
 807 {
 808    uint32_t i, j;
 809    for (i = 0; i < descriptorWriteCount; i++) {
 810       const VkWriteDescriptorSet *writeset = &pDescriptorWrites[i];
 811       TU_FROM_HANDLE(tu_descriptor_set, set,
 812                        dstSetOverride ? dstSetOverride : writeset->dstSet);
 813       const struct tu_descriptor_set_binding_layout *binding_layout =
 814          set->layout->binding + writeset->dstBinding;
 815       uint32_t *ptr = set->mapped_ptr;
 816       struct tu_bo **buffer_list = set->descriptors;
 817
 818       ptr += binding_layout->offset / 4;
 819
 820       ptr += binding_layout->size * writeset->dstArrayElement / 4;
 821       buffer_list += binding_layout->buffer_offset;
 822       buffer_list += writeset->dstArrayElement;
 823       for (j = 0; j < writeset->descriptorCount; ++j) {
 824          switch(writeset->descriptorType) {
 825          case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
 826          case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
 827             unsigned idx = writeset->dstArrayElement + j;
 828             idx += binding_layout->dynamic_offset_offset;
 829             assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
 830             write_dynamic_buffer_descriptor(device, set->dynamic_descriptors + idx,
 831                         buffer_list, writeset->pBufferInfo + j);
 832             break;
 833          }
 834
 835          case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
 836          case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
 837             write_buffer_descriptor(device, cmd_buffer, ptr, buffer_list,
 838                      writeset->pBufferInfo + j);
 839             break;
 840          case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
 841          case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
 842             write_texel_buffer_descriptor(device, cmd_buffer, ptr, buffer_list,
 843                      writeset->pTexelBufferView[j]);
 844             break;
 845          case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
 846          case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
 847          case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
 848             write_image_descriptor(device, cmd_buffer, ptr, buffer_list,
 849                                    writeset->descriptorType,
 850                                    writeset->pImageInfo + j);
 851             break;
 852          case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
 853             write_combined_image_sampler_descriptor(device, cmd_buffer,
 854                                                     A6XX_TEX_CONST_DWORDS * 4,
 855                                                     ptr, buffer_list,
 856                                                     writeset->descriptorType,
 857                                                     writeset->pImageInfo + j,
 858                                                     !binding_layout->immutable_samplers_offset);
 859             break;
 860          case VK_DESCRIPTOR_TYPE_SAMPLER:
 861             write_sampler_descriptor(device, ptr, writeset->pImageInfo + j);
 862             break;
 863          default:
 864             unreachable("unimplemented descriptor type");
 865             break;
 866          }
 867          ptr += binding_layout->size / 4;
 868          ++buffer_list;
 869       }
 870    }
 871
 872    for (i = 0; i < descriptorCopyCount; i++) {
 873       const VkCopyDescriptorSet *copyset = &pDescriptorCopies[i];
 874       TU_FROM_HANDLE(tu_descriptor_set, src_set,
 875                        copyset->srcSet);
 876       TU_FROM_HANDLE(tu_descriptor_set, dst_set,
 877                        copyset->dstSet);
 878       const struct tu_descriptor_set_binding_layout *src_binding_layout =
 879          src_set->layout->binding + copyset->srcBinding;
 880       const struct tu_descriptor_set_binding_layout *dst_binding_layout =
 881          dst_set->layout->binding + copyset->dstBinding;
 882       uint32_t *src_ptr = src_set->mapped_ptr;
 883       uint32_t *dst_ptr = dst_set->mapped_ptr;
 884       struct tu_bo **src_buffer_list = src_set->descriptors;
 885       struct tu_bo **dst_buffer_list = dst_set->descriptors;
 886
 887       src_ptr += src_binding_layout->offset / 4;
 888       dst_ptr += dst_binding_layout->offset / 4;
 889
 890       src_ptr += src_binding_layout->size * copyset->srcArrayElement / 4;
 891       dst_ptr += dst_binding_layout->size * copyset->dstArrayElement / 4;
 892
 893       src_buffer_list += src_binding_layout->buffer_offset;
 894       src_buffer_list += copyset->srcArrayElement;
 895
 896       dst_buffer_list += dst_binding_layout->buffer_offset;
 897       dst_buffer_list += copyset->dstArrayElement;
 898
 899       for (j = 0; j < copyset->descriptorCount; ++j) {
 900          switch (src_binding_layout->type) {
 901          case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
 902          case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
 903             unsigned src_idx = copyset->srcArrayElement + j;
 904             unsigned dst_idx = copyset->dstArrayElement + j;
 905             struct tu_descriptor_range *src_range, *dst_range;
 906             src_idx += src_binding_layout->dynamic_offset_offset;
 907             dst_idx += dst_binding_layout->dynamic_offset_offset;
 908
 909             src_range = src_set->dynamic_descriptors + src_idx;
 910             dst_range = dst_set->dynamic_descriptors + dst_idx;
 911             *dst_range = *src_range;
 912             break;
 913          }
 914          default:
 915             memcpy(dst_ptr, src_ptr, src_binding_layout->size);
 916          }
 917          src_ptr += src_binding_layout->size / 4;
 918          dst_ptr += dst_binding_layout->size / 4;
 919
 920          if (src_binding_layout->type != VK_DESCRIPTOR_TYPE_SAMPLER) {
 921             /* Sampler descriptors don't have a buffer list. */
 922             dst_buffer_list[j] = src_buffer_list[j];
 923          }
 924       }
 925    }
 926 }
 927
 928 void
 929 tu_UpdateDescriptorSets(VkDevice _device,
 930                         uint32_t descriptorWriteCount,
 931                         const VkWriteDescriptorSet *pDescriptorWrites,
 932                         uint32_t descriptorCopyCount,
 933                         const VkCopyDescriptorSet *pDescriptorCopies)
 934 {
 935    TU_FROM_HANDLE(tu_device, device, _device);
 936
 937    tu_update_descriptor_sets(device, NULL, VK_NULL_HANDLE,
 938                              descriptorWriteCount, pDescriptorWrites,
 939                              descriptorCopyCount, pDescriptorCopies);
 940 }
 941
 942 VkResult
 943 tu_CreateDescriptorUpdateTemplate(
 944    VkDevice _device,
 945    const VkDescriptorUpdateTemplateCreateInfo *pCreateInfo,
 946    const VkAllocationCallbacks *pAllocator,
 947    VkDescriptorUpdateTemplate *pDescriptorUpdateTemplate)
 948 {
 949    TU_FROM_HANDLE(tu_device, device, _device);
 950    TU_FROM_HANDLE(tu_descriptor_set_layout, set_layout,
 951                   pCreateInfo->descriptorSetLayout);
 952    const uint32_t entry_count = pCreateInfo->descriptorUpdateEntryCount;
 953    const size_t size =
 954       sizeof(struct tu_descriptor_update_template) +
 955       sizeof(struct tu_descriptor_update_template_entry) * entry_count;
 956    struct tu_descriptor_update_template *templ;
 957
 958    templ = vk_alloc2(&device->alloc, pAllocator, size, 8,
 959                      VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
 960    if (!templ)
 961       return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
 962
 963    *pDescriptorUpdateTemplate =
 964       tu_descriptor_update_template_to_handle(templ);
 965
 966    tu_use_args(set_layout);
 967    tu_stub();
 968    return VK_SUCCESS;
 969 }
 970
 971 void
 972 tu_DestroyDescriptorUpdateTemplate(
 973    VkDevice _device,
 974    VkDescriptorUpdateTemplate descriptorUpdateTemplate,
 975    const VkAllocationCallbacks *pAllocator)
 976 {
 977    TU_FROM_HANDLE(tu_device, device, _device);
 978    TU_FROM_HANDLE(tu_descriptor_update_template, templ,
 979                   descriptorUpdateTemplate);
 980
 981    if (!templ)
 982       return;
 983
 984    vk_free2(&device->alloc, pAllocator, templ);
 985 }
 986
 987 void
 988 tu_update_descriptor_set_with_template(
 989    struct tu_device *device,
 990    struct tu_cmd_buffer *cmd_buffer,
 991    struct tu_descriptor_set *set,
 992    VkDescriptorUpdateTemplate descriptorUpdateTemplate,
 993    const void *pData)
 994 {
 995    TU_FROM_HANDLE(tu_descriptor_update_template, templ,
 996                   descriptorUpdateTemplate);
 997    tu_use_args(templ);
 998 }
 999
1000 void
1001 tu_UpdateDescriptorSetWithTemplate(
1002    VkDevice _device,
1003    VkDescriptorSet descriptorSet,
1004    VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1005    const void *pData)
1006 {
1007    TU_FROM_HANDLE(tu_device, device, _device);
1008    TU_FROM_HANDLE(tu_descriptor_set, set, descriptorSet);
1009
1010    tu_update_descriptor_set_with_template(device, NULL, set,
1011                                           descriptorUpdateTemplate, pData);
1012 }
1013
1014 VkResult
1015 tu_CreateSamplerYcbcrConversion(
1016    VkDevice device,
1017    const VkSamplerYcbcrConversionCreateInfo *pCreateInfo,
1018    const VkAllocationCallbacks *pAllocator,
1019    VkSamplerYcbcrConversion *pYcbcrConversion)
1020 {
1021    *pYcbcrConversion = VK_NULL_HANDLE;
1022    return VK_SUCCESS;
1023 }
1024
1025 void
1026 tu_DestroySamplerYcbcrConversion(VkDevice device,
1027                                  VkSamplerYcbcrConversion ycbcrConversion,
1028                                  const VkAllocationCallbacks *pAllocator)
1029 {
1030    /* Do nothing. */
1031 }