src/intel/vulkan/anv_blorp.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "anv_private.h"
  25
  26 static bool
  27 lookup_blorp_shader(struct blorp_context *blorp,
  28                     const void *key, uint32_t key_size,
  29                     uint32_t *kernel_out, void *prog_data_out)
  30 {
  31    struct anv_device *device = blorp->driver_ctx;
  32
  33    /* The blorp cache must be a real cache */
  34    assert(device->blorp_shader_cache.cache);
  35
  36    struct anv_shader_bin *bin =
  37       anv_pipeline_cache_search(&device->blorp_shader_cache, key, key_size);
  38    if (!bin)
  39       return false;
  40
  41    /* The cache already has a reference and it's not going anywhere so there
  42     * is no need to hold a second reference.
  43     */
  44    anv_shader_bin_unref(device, bin);
  45
  46    *kernel_out = bin->kernel.offset;
  47    *(const struct brw_stage_prog_data **)prog_data_out =
  48       anv_shader_bin_get_prog_data(bin);
  49
  50    return true;
  51 }
  52
  53 static void
  54 upload_blorp_shader(struct blorp_context *blorp,
  55                     const void *key, uint32_t key_size,
  56                     const void *kernel, uint32_t kernel_size,
  57                     const void *prog_data, uint32_t prog_data_size,
  58                     uint32_t *kernel_out, void *prog_data_out)
  59 {
  60    struct anv_device *device = blorp->driver_ctx;
  61
  62    /* The blorp cache must be a real cache */
  63    assert(device->blorp_shader_cache.cache);
  64
  65    struct anv_pipeline_bind_map bind_map = {
  66       .surface_count = 0,
  67       .sampler_count = 0,
  68    };
  69
  70    struct anv_shader_bin *bin =
  71       anv_pipeline_cache_upload_kernel(&device->blorp_shader_cache,
  72                                        key, key_size, kernel, kernel_size,
  73                                        prog_data, prog_data_size, &bind_map);
  74
  75    /* The cache already has a reference and it's not going anywhere so there
  76     * is no need to hold a second reference.
  77     */
  78    anv_shader_bin_unref(device, bin);
  79
  80    *kernel_out = bin->kernel.offset;
  81    *(const struct brw_stage_prog_data **)prog_data_out =
  82       anv_shader_bin_get_prog_data(bin);
  83 }
  84
  85 void
  86 anv_device_init_blorp(struct anv_device *device)
  87 {
  88    anv_pipeline_cache_init(&device->blorp_shader_cache, device, true);
  89    blorp_init(&device->blorp, device, &device->isl_dev);
  90    device->blorp.compiler = device->instance->physicalDevice.compiler;
  91    device->blorp.mocs.tex = device->default_mocs;
  92    device->blorp.mocs.rb = device->default_mocs;
  93    device->blorp.mocs.vb = device->default_mocs;
  94    device->blorp.lookup_shader = lookup_blorp_shader;
  95    device->blorp.upload_shader = upload_blorp_shader;
  96    switch (device->info.gen) {
  97    case 7:
  98       if (device->info.is_haswell) {
  99          device->blorp.exec = gen75_blorp_exec;
 100       } else {
 101          device->blorp.exec = gen7_blorp_exec;
 102       }
 103       break;
 104    case 8:
 105       device->blorp.exec = gen8_blorp_exec;
 106       break;
 107    case 9:
 108       device->blorp.exec = gen9_blorp_exec;
 109       break;
 110    default:
 111       unreachable("Unknown hardware generation");
 112    }
 113 }
 114
 115 void
 116 anv_device_finish_blorp(struct anv_device *device)
 117 {
 118    blorp_finish(&device->blorp);
 119    anv_pipeline_cache_finish(&device->blorp_shader_cache);
 120 }
 121
 122 static void
 123 get_blorp_surf_for_anv_buffer(struct anv_device *device,
 124                               struct anv_buffer *buffer, uint64_t offset,
 125                               uint32_t width, uint32_t height,
 126                               uint32_t row_pitch, enum isl_format format,
 127                               struct blorp_surf *blorp_surf,
 128                               struct isl_surf *isl_surf)
 129 {
 130    *blorp_surf = (struct blorp_surf) {
 131       .surf = isl_surf,
 132       .addr = {
 133          .buffer = buffer->bo,
 134          .offset = buffer->offset + offset,
 135       },
 136    };
 137
 138    isl_surf_init(&device->isl_dev, isl_surf,
 139                  .dim = ISL_SURF_DIM_2D,
 140                  .format = format,
 141                  .width = width,
 142                  .height = height,
 143                  .depth = 1,
 144                  .levels = 1,
 145                  .array_len = 1,
 146                  .samples = 1,
 147                  .min_pitch = row_pitch,
 148                  .usage = ISL_SURF_USAGE_TEXTURE_BIT |
 149                           ISL_SURF_USAGE_RENDER_TARGET_BIT,
 150                  .tiling_flags = ISL_TILING_LINEAR_BIT);
 151    assert(isl_surf->row_pitch == row_pitch);
 152 }
 153
 154 static void
 155 get_blorp_surf_for_anv_image(const struct anv_image *image,
 156                              VkImageAspectFlags aspect,
 157                              struct blorp_surf *blorp_surf)
 158 {
 159    const struct anv_surface *surface =
 160       anv_image_get_surface_for_aspect_mask(image, aspect);
 161
 162    *blorp_surf = (struct blorp_surf) {
 163       .surf = &surface->isl,
 164       .addr = {
 165          .buffer = image->bo,
 166          .offset = image->offset + surface->offset,
 167       },
 168    };
 169 }
 170
 171 void anv_CmdCopyImage(
 172     VkCommandBuffer                             commandBuffer,
 173     VkImage                                     srcImage,
 174     VkImageLayout                               srcImageLayout,
 175     VkImage                                     dstImage,
 176     VkImageLayout                               dstImageLayout,
 177     uint32_t                                    regionCount,
 178     const VkImageCopy*                          pRegions)
 179 {
 180    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 181    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 182    ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
 183
 184    struct blorp_batch batch;
 185    blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer);
 186
 187    for (unsigned r = 0; r < regionCount; r++) {
 188       VkOffset3D srcOffset =
 189          anv_sanitize_image_offset(src_image->type, pRegions[r].srcOffset);
 190       VkOffset3D dstOffset =
 191          anv_sanitize_image_offset(dst_image->type, pRegions[r].dstOffset);
 192       VkExtent3D extent =
 193          anv_sanitize_image_extent(src_image->type, pRegions[r].extent);
 194
 195       unsigned dst_base_layer, layer_count;
 196       if (dst_image->type == VK_IMAGE_TYPE_3D) {
 197          dst_base_layer = pRegions[r].dstOffset.z;
 198          layer_count = pRegions[r].extent.depth;
 199       } else {
 200          dst_base_layer = pRegions[r].dstSubresource.baseArrayLayer;
 201          layer_count = pRegions[r].dstSubresource.layerCount;
 202       }
 203
 204       unsigned src_base_layer;
 205       if (src_image->type == VK_IMAGE_TYPE_3D) {
 206          src_base_layer = pRegions[r].srcOffset.z;
 207       } else {
 208          src_base_layer = pRegions[r].srcSubresource.baseArrayLayer;
 209          assert(pRegions[r].srcSubresource.layerCount == layer_count);
 210       }
 211
 212       assert(pRegions[r].srcSubresource.aspectMask ==
 213              pRegions[r].dstSubresource.aspectMask);
 214
 215       uint32_t a;
 216       for_each_bit(a, pRegions[r].dstSubresource.aspectMask) {
 217          VkImageAspectFlagBits aspect = (1 << a);
 218
 219          struct blorp_surf src_surf, dst_surf;
 220          get_blorp_surf_for_anv_image(src_image, aspect, &src_surf);
 221          get_blorp_surf_for_anv_image(dst_image, aspect, &dst_surf);
 222
 223          for (unsigned i = 0; i < layer_count; i++) {
 224             blorp_copy(&batch, &src_surf, pRegions[r].srcSubresource.mipLevel,
 225                        src_base_layer + i,
 226                        &dst_surf, pRegions[r].dstSubresource.mipLevel,
 227                        dst_base_layer + i,
 228                        srcOffset.x, srcOffset.y,
 229                        dstOffset.x, dstOffset.y,
 230                        extent.width, extent.height);
 231          }
 232       }
 233    }
 234
 235    blorp_batch_finish(&batch);
 236 }
 237
 238 static void
 239 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
 240                      struct anv_buffer *anv_buffer,
 241                      struct anv_image *anv_image,
 242                      uint32_t regionCount,
 243                      const VkBufferImageCopy* pRegions,
 244                      bool buffer_to_image)
 245 {
 246    struct blorp_batch batch;
 247    blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer);
 248
 249    struct {
 250       struct blorp_surf surf;
 251       uint32_t level;
 252       VkOffset3D offset;
 253    } image, buffer, *src, *dst;
 254
 255    buffer.level = 0;
 256    buffer.offset = (VkOffset3D) { 0, 0, 0 };
 257
 258    if (buffer_to_image) {
 259       src = &buffer;
 260       dst = &image;
 261    } else {
 262       src = &image;
 263       dst = &buffer;
 264    }
 265
 266    for (unsigned r = 0; r < regionCount; r++) {
 267       const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
 268
 269       get_blorp_surf_for_anv_image(anv_image, aspect, &image.surf);
 270       image.offset =
 271          anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset);
 272       image.level = pRegions[r].imageSubresource.mipLevel;
 273
 274       VkExtent3D extent =
 275          anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent);
 276       if (anv_image->type != VK_IMAGE_TYPE_3D) {
 277          image.offset.z = pRegions[r].imageSubresource.baseArrayLayer;
 278          extent.depth = pRegions[r].imageSubresource.layerCount;
 279       }
 280
 281       const enum isl_format buffer_format =
 282          anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format,
 283                             aspect, VK_IMAGE_TILING_LINEAR);
 284
 285       const VkExtent3D bufferImageExtent = {
 286          .width  = pRegions[r].bufferRowLength ?
 287                    pRegions[r].bufferRowLength : extent.width,
 288          .height = pRegions[r].bufferImageHeight ?
 289                    pRegions[r].bufferImageHeight : extent.height,
 290       };
 291
 292       const struct isl_format_layout *buffer_fmtl =
 293          isl_format_get_layout(buffer_format);
 294
 295       const uint32_t buffer_row_pitch =
 296          DIV_ROUND_UP(bufferImageExtent.width, buffer_fmtl->bw) *
 297          (buffer_fmtl->bpb / 8);
 298
 299       const uint32_t buffer_layer_stride =
 300          DIV_ROUND_UP(bufferImageExtent.height, buffer_fmtl->bh) *
 301          buffer_row_pitch;
 302
 303       struct isl_surf buffer_isl_surf;
 304       get_blorp_surf_for_anv_buffer(cmd_buffer->device,
 305                                     anv_buffer, pRegions[r].bufferOffset,
 306                                     extent.width, extent.height,
 307                                     buffer_row_pitch, buffer_format,
 308                                     &buffer.surf, &buffer_isl_surf);
 309
 310       for (unsigned z = 0; z < extent.depth; z++) {
 311          blorp_copy(&batch, &src->surf, src->level, src->offset.z,
 312                     &dst->surf, dst->level, dst->offset.z,
 313                     src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
 314                     extent.width, extent.height);
 315
 316          image.offset.z++;
 317          buffer.surf.addr.offset += buffer_layer_stride;
 318       }
 319    }
 320
 321    blorp_batch_finish(&batch);
 322 }
 323
 324 void anv_CmdCopyBufferToImage(
 325     VkCommandBuffer                             commandBuffer,
 326     VkBuffer                                    srcBuffer,
 327     VkImage                                     dstImage,
 328     VkImageLayout                               dstImageLayout,
 329     uint32_t                                    regionCount,
 330     const VkBufferImageCopy*                    pRegions)
 331 {
 332    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 333    ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
 334    ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
 335
 336    copy_buffer_to_image(cmd_buffer, src_buffer, dst_image,
 337                         regionCount, pRegions, true);
 338 }
 339
 340 void anv_CmdCopyImageToBuffer(
 341     VkCommandBuffer                             commandBuffer,
 342     VkImage                                     srcImage,
 343     VkImageLayout                               srcImageLayout,
 344     VkBuffer                                    dstBuffer,
 345     uint32_t                                    regionCount,
 346     const VkBufferImageCopy*                    pRegions)
 347 {
 348    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 349    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 350    ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
 351
 352    copy_buffer_to_image(cmd_buffer, dst_buffer, src_image,
 353                         regionCount, pRegions, false);
 354 }
 355
 356 static bool
 357 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
 358 {
 359    bool flip = false;
 360    if (*src0 > *src1) {
 361       unsigned tmp = *src0;
 362       *src0 = *src1;
 363       *src1 = tmp;
 364       flip = !flip;
 365    }
 366
 367    if (*dst0 > *dst1) {
 368       unsigned tmp = *dst0;
 369       *dst0 = *dst1;
 370       *dst1 = tmp;
 371       flip = !flip;
 372    }
 373
 374    return flip;
 375 }
 376
 377 void anv_CmdBlitImage(
 378     VkCommandBuffer                             commandBuffer,
 379     VkImage                                     srcImage,
 380     VkImageLayout                               srcImageLayout,
 381     VkImage                                     dstImage,
 382     VkImageLayout                               dstImageLayout,
 383     uint32_t                                    regionCount,
 384     const VkImageBlit*                          pRegions,
 385     VkFilter                                    filter)
 386
 387 {
 388    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 389    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 390    ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
 391
 392    struct blorp_surf src, dst;
 393
 394    uint32_t gl_filter;
 395    switch (filter) {
 396    case VK_FILTER_NEAREST:
 397       gl_filter = 0x2600; /* GL_NEAREST */
 398       break;
 399    case VK_FILTER_LINEAR:
 400       gl_filter = 0x2601; /* GL_LINEAR */
 401       break;
 402    default:
 403       unreachable("Invalid filter");
 404    }
 405
 406    struct blorp_batch batch;
 407    blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer);
 408
 409    for (unsigned r = 0; r < regionCount; r++) {
 410       const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource;
 411       const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource;
 412
 413       get_blorp_surf_for_anv_image(src_image, src_res->aspectMask, &src);
 414       get_blorp_surf_for_anv_image(dst_image, dst_res->aspectMask, &dst);
 415
 416       struct anv_format src_format =
 417          anv_get_format(&cmd_buffer->device->info, src_image->vk_format,
 418                         src_res->aspectMask, src_image->tiling);
 419       struct anv_format dst_format =
 420          anv_get_format(&cmd_buffer->device->info, dst_image->vk_format,
 421                         dst_res->aspectMask, dst_image->tiling);
 422
 423       unsigned dst_start, dst_end;
 424       if (dst_image->type == VK_IMAGE_TYPE_3D) {
 425          assert(dst_res->baseArrayLayer == 0);
 426          dst_start = pRegions[r].dstOffsets[0].z;
 427          dst_end = pRegions[r].dstOffsets[1].z;
 428       } else {
 429          dst_start = dst_res->baseArrayLayer;
 430          dst_end = dst_start + dst_res->layerCount;
 431       }
 432
 433       unsigned src_start, src_end;
 434       if (src_image->type == VK_IMAGE_TYPE_3D) {
 435          assert(src_res->baseArrayLayer == 0);
 436          src_start = pRegions[r].srcOffsets[0].z;
 437          src_end = pRegions[r].srcOffsets[1].z;
 438       } else {
 439          src_start = src_res->baseArrayLayer;
 440          src_end = src_start + src_res->layerCount;
 441       }
 442
 443       bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
 444       float src_z_step = (float)(src_end + 1 - src_start) /
 445                          (float)(dst_end + 1 - dst_start);
 446
 447       if (flip_z) {
 448          src_start = src_end;
 449          src_z_step *= -1;
 450       }
 451
 452       unsigned src_x0 = pRegions[r].srcOffsets[0].x;
 453       unsigned src_x1 = pRegions[r].srcOffsets[1].x;
 454       unsigned dst_x0 = pRegions[r].dstOffsets[0].x;
 455       unsigned dst_x1 = pRegions[r].dstOffsets[1].x;
 456       bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
 457
 458       unsigned src_y0 = pRegions[r].srcOffsets[0].y;
 459       unsigned src_y1 = pRegions[r].srcOffsets[1].y;
 460       unsigned dst_y0 = pRegions[r].dstOffsets[0].y;
 461       unsigned dst_y1 = pRegions[r].dstOffsets[1].y;
 462       bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
 463
 464       const unsigned num_layers = dst_end - dst_start;
 465       for (unsigned i = 0; i < num_layers; i++) {
 466          unsigned dst_z = dst_start + i;
 467          unsigned src_z = src_start + i * src_z_step;
 468
 469          blorp_blit(&batch, &src, src_res->mipLevel, src_z,
 470                     src_format.isl_format, src_format.swizzle,
 471                     &dst, dst_res->mipLevel, dst_z,
 472                     dst_format.isl_format, dst_format.swizzle,
 473                     src_x0, src_y0, src_x1, src_y1,
 474                     dst_x0, dst_y0, dst_x1, dst_y1,
 475                     gl_filter, flip_x, flip_y);
 476       }
 477
 478    }
 479
 480    blorp_batch_finish(&batch);
 481 }
 482
 483 static void
 484 do_buffer_copy(struct blorp_batch *batch,
 485                struct anv_bo *src, uint64_t src_offset,
 486                struct anv_bo *dst, uint64_t dst_offset,
 487                int width, int height, int block_size)
 488 {
 489    struct anv_device *device = batch->blorp->driver_ctx;
 490
 491    /* The actual format we pick doesn't matter as blorp will throw it away.
 492     * The only thing that actually matters is the size.
 493     */
 494    enum isl_format format;
 495    switch (block_size) {
 496    case 1:  format = ISL_FORMAT_R8_UINT;              break;
 497    case 2:  format = ISL_FORMAT_R8G8_UINT;            break;
 498    case 4:  format = ISL_FORMAT_R8G8B8A8_UNORM;       break;
 499    case 8:  format = ISL_FORMAT_R16G16B16A16_UNORM;   break;
 500    case 16: format = ISL_FORMAT_R32G32B32A32_UINT;    break;
 501    default:
 502       unreachable("Not a power-of-two format size");
 503    }
 504
 505    struct isl_surf surf;
 506    isl_surf_init(&device->isl_dev, &surf,
 507                  .dim = ISL_SURF_DIM_2D,
 508                  .format = format,
 509                  .width = width,
 510                  .height = height,
 511                  .depth = 1,
 512                  .levels = 1,
 513                  .array_len = 1,
 514                  .samples = 1,
 515                  .usage = ISL_SURF_USAGE_TEXTURE_BIT |
 516                           ISL_SURF_USAGE_RENDER_TARGET_BIT,
 517                  .tiling_flags = ISL_TILING_LINEAR_BIT);
 518    assert(surf.row_pitch == width * block_size);
 519
 520    struct blorp_surf src_blorp_surf = {
 521       .surf = &surf,
 522       .addr = {
 523          .buffer = src,
 524          .offset = src_offset,
 525       },
 526    };
 527
 528    struct blorp_surf dst_blorp_surf = {
 529       .surf = &surf,
 530       .addr = {
 531          .buffer = dst,
 532          .offset = dst_offset,
 533       },
 534    };
 535
 536    blorp_copy(batch, &src_blorp_surf, 0, 0, &dst_blorp_surf, 0, 0,
 537               0, 0, 0, 0, width, height);
 538 }
 539
 540 /**
 541  * Returns the greatest common divisor of a and b that is a power of two.
 542  */
 543 static inline uint64_t
 544 gcd_pow2_u64(uint64_t a, uint64_t b)
 545 {
 546    assert(a > 0 || b > 0);
 547
 548    unsigned a_log2 = ffsll(a) - 1;
 549    unsigned b_log2 = ffsll(b) - 1;
 550
 551    /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
 552     * case, the MIN2() will take the other one.  If both are 0 then we will
 553     * hit the assert above.
 554     */
 555    return 1 << MIN2(a_log2, b_log2);
 556 }
 557
 558 /* This is maximum possible width/height our HW can handle */
 559 #define MAX_SURFACE_DIM (1ull << 14)
 560
 561 void anv_CmdCopyBuffer(
 562     VkCommandBuffer                             commandBuffer,
 563     VkBuffer                                    srcBuffer,
 564     VkBuffer                                    dstBuffer,
 565     uint32_t                                    regionCount,
 566     const VkBufferCopy*                         pRegions)
 567 {
 568    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 569    ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
 570    ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
 571
 572    struct blorp_batch batch;
 573    blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer);
 574
 575    for (unsigned r = 0; r < regionCount; r++) {
 576       uint64_t src_offset = src_buffer->offset + pRegions[r].srcOffset;
 577       uint64_t dst_offset = dst_buffer->offset + pRegions[r].dstOffset;
 578       uint64_t copy_size = pRegions[r].size;
 579
 580       /* First, we compute the biggest format that can be used with the
 581        * given offsets and size.
 582        */
 583       int bs = 16;
 584       bs = gcd_pow2_u64(bs, src_offset);
 585       bs = gcd_pow2_u64(bs, dst_offset);
 586       bs = gcd_pow2_u64(bs, pRegions[r].size);
 587
 588       /* First, we make a bunch of max-sized copies */
 589       uint64_t max_copy_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs;
 590       while (copy_size >= max_copy_size) {
 591          do_buffer_copy(&batch, src_buffer->bo, src_offset,
 592                         dst_buffer->bo, dst_offset,
 593                         MAX_SURFACE_DIM, MAX_SURFACE_DIM, bs);
 594          copy_size -= max_copy_size;
 595          src_offset += max_copy_size;
 596          dst_offset += max_copy_size;
 597       }
 598
 599       /* Now make a max-width copy */
 600       uint64_t height = copy_size / (MAX_SURFACE_DIM * bs);
 601       assert(height < MAX_SURFACE_DIM);
 602       if (height != 0) {
 603          uint64_t rect_copy_size = height * MAX_SURFACE_DIM * bs;
 604          do_buffer_copy(&batch, src_buffer->bo, src_offset,
 605                         dst_buffer->bo, dst_offset,
 606                         MAX_SURFACE_DIM, height, bs);
 607          copy_size -= rect_copy_size;
 608          src_offset += rect_copy_size;
 609          dst_offset += rect_copy_size;
 610       }
 611
 612       /* Finally, make a small copy to finish it off */
 613       if (copy_size != 0) {
 614          do_buffer_copy(&batch, src_buffer->bo, src_offset,
 615                         dst_buffer->bo, dst_offset,
 616                         copy_size / bs, 1, bs);
 617       }
 618    }
 619
 620    blorp_batch_finish(&batch);
 621 }
 622
 623 void anv_CmdUpdateBuffer(
 624     VkCommandBuffer                             commandBuffer,
 625     VkBuffer                                    dstBuffer,
 626     VkDeviceSize                                dstOffset,
 627     VkDeviceSize                                dataSize,
 628     const uint32_t*                             pData)
 629 {
 630    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 631    ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
 632
 633    struct blorp_batch batch;
 634    blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer);
 635
 636    /* We can't quite grab a full block because the state stream needs a
 637     * little data at the top to build its linked list.
 638     */
 639    const uint32_t max_update_size =
 640       cmd_buffer->device->dynamic_state_block_pool.block_size - 64;
 641
 642    assert(max_update_size < MAX_SURFACE_DIM * 4);
 643
 644    while (dataSize) {
 645       const uint32_t copy_size = MIN2(dataSize, max_update_size);
 646
 647       struct anv_state tmp_data =
 648          anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
 649
 650       memcpy(tmp_data.map, pData, copy_size);
 651
 652       int bs = 16;
 653       bs = gcd_pow2_u64(bs, dstOffset);
 654       bs = gcd_pow2_u64(bs, copy_size);
 655
 656       do_buffer_copy(&batch,
 657                      &cmd_buffer->device->dynamic_state_block_pool.bo,
 658                      tmp_data.offset,
 659                      dst_buffer->bo, dst_buffer->offset + dstOffset,
 660                      copy_size / bs, 1, bs);
 661
 662       dataSize -= copy_size;
 663       dstOffset += copy_size;
 664       pData = (void *)pData + copy_size;
 665    }
 666
 667    blorp_batch_finish(&batch);
 668 }
 669
 670 void anv_CmdClearColorImage(
 671     VkCommandBuffer                             commandBuffer,
 672     VkImage                                     _image,
 673     VkImageLayout                               imageLayout,
 674     const VkClearColorValue*                    pColor,
 675     uint32_t                                    rangeCount,
 676     const VkImageSubresourceRange*              pRanges)
 677 {
 678    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 679    ANV_FROM_HANDLE(anv_image, image, _image);
 680
 681    static const bool color_write_disable[4] = { false, false, false, false };
 682
 683    struct blorp_batch batch;
 684    blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer);
 685
 686    union isl_color_value clear_color;
 687    memcpy(clear_color.u32, pColor->uint32, sizeof(pColor->uint32));
 688
 689    struct blorp_surf surf;
 690    get_blorp_surf_for_anv_image(image, VK_IMAGE_ASPECT_COLOR_BIT, &surf);
 691
 692    for (unsigned r = 0; r < rangeCount; r++) {
 693       if (pRanges[r].aspectMask == 0)
 694          continue;
 695
 696       assert(pRanges[r].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
 697
 698       struct anv_format src_format =
 699          anv_get_format(&cmd_buffer->device->info, image->vk_format,
 700                         VK_IMAGE_ASPECT_COLOR_BIT, image->tiling);
 701
 702       unsigned base_layer = pRanges[r].baseArrayLayer;
 703       unsigned layer_count = pRanges[r].layerCount;
 704
 705       for (unsigned i = 0; i < pRanges[r].levelCount; i++) {
 706          const unsigned level = pRanges[r].baseMipLevel + i;
 707          const unsigned level_width = anv_minify(image->extent.width, level);
 708          const unsigned level_height = anv_minify(image->extent.height, level);
 709
 710          if (image->type == VK_IMAGE_TYPE_3D) {
 711             base_layer = 0;
 712             layer_count = anv_minify(image->extent.depth, level);
 713          }
 714
 715          blorp_clear(&batch, &surf,
 716                      src_format.isl_format, src_format.swizzle,
 717                      level, base_layer, layer_count,
 718                      0, 0, level_width, level_height,
 719                      clear_color, color_write_disable);
 720       }
 721    }
 722
 723    blorp_batch_finish(&batch);
 724 }
 725
 726 static void
 727 resolve_image(struct blorp_batch *batch,
 728               const struct anv_image *src_image,
 729               uint32_t src_level, uint32_t src_layer,
 730               const struct anv_image *dst_image,
 731               uint32_t dst_level, uint32_t dst_layer,
 732               VkImageAspectFlags aspect_mask,
 733               uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y,
 734               uint32_t width, uint32_t height)
 735 {
 736    assert(src_image->type == VK_IMAGE_TYPE_2D);
 737    assert(src_image->samples > 1);
 738    assert(dst_image->type == VK_IMAGE_TYPE_2D);
 739    assert(dst_image->samples == 1);
 740
 741    uint32_t a;
 742    for_each_bit(a, aspect_mask) {
 743       VkImageAspectFlagBits aspect = 1 << a;
 744
 745       struct blorp_surf src_surf, dst_surf;
 746       get_blorp_surf_for_anv_image(src_image, aspect, &src_surf);
 747       get_blorp_surf_for_anv_image(dst_image, aspect, &dst_surf);
 748
 749       blorp_blit(batch,
 750                  &src_surf, src_level, src_layer,
 751                  ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
 752                  &dst_surf, dst_level, dst_layer,
 753                  ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY,
 754                  src_x, src_y, src_x + width, src_y + height,
 755                  dst_x, dst_y, dst_x + width, dst_y + height,
 756                  0x2600 /* GL_NEAREST */, false, false);
 757    }
 758 }
 759
 760 void anv_CmdResolveImage(
 761     VkCommandBuffer                             commandBuffer,
 762     VkImage                                     srcImage,
 763     VkImageLayout                               srcImageLayout,
 764     VkImage                                     dstImage,
 765     VkImageLayout                               dstImageLayout,
 766     uint32_t                                    regionCount,
 767     const VkImageResolve*                       pRegions)
 768 {
 769    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 770    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 771    ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
 772
 773    struct blorp_batch batch;
 774    blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer);
 775
 776    for (uint32_t r = 0; r < regionCount; r++) {
 777       assert(pRegions[r].srcSubresource.aspectMask ==
 778              pRegions[r].dstSubresource.aspectMask);
 779       assert(pRegions[r].srcSubresource.layerCount ==
 780              pRegions[r].dstSubresource.layerCount);
 781
 782       const uint32_t layer_count = pRegions[r].dstSubresource.layerCount;
 783
 784       for (uint32_t layer = 0; layer < layer_count; layer++) {
 785          resolve_image(&batch,
 786                        src_image, pRegions[r].srcSubresource.mipLevel,
 787                        pRegions[r].srcSubresource.baseArrayLayer + layer,
 788                        dst_image, pRegions[r].dstSubresource.mipLevel,
 789                        pRegions[r].dstSubresource.baseArrayLayer + layer,
 790                        pRegions[r].dstSubresource.aspectMask,
 791                        pRegions[r].srcOffset.x, pRegions[r].srcOffset.y,
 792                        pRegions[r].dstOffset.x, pRegions[r].dstOffset.y,
 793                        pRegions[r].extent.width, pRegions[r].extent.height);
 794       }
 795    }
 796
 797    blorp_batch_finish(&batch);
 798 }
 799
 800 void
 801 anv_cmd_buffer_resolve_subpass(struct anv_cmd_buffer *cmd_buffer)
 802 {
 803    struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
 804    struct anv_subpass *subpass = cmd_buffer->state.subpass;
 805
 806    /* FINISHME(perf): Skip clears for resolve attachments.
 807     *
 808     * From the Vulkan 1.0 spec:
 809     *
 810     *    If the first use of an attachment in a render pass is as a resolve
 811     *    attachment, then the loadOp is effectively ignored as the resolve is
 812     *    guaranteed to overwrite all pixels in the render area.
 813     */
 814
 815    if (!subpass->has_resolve)
 816       return;
 817
 818    struct blorp_batch batch;
 819    blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer);
 820
 821    for (uint32_t i = 0; i < subpass->color_count; ++i) {
 822       uint32_t src_att = subpass->color_attachments[i];
 823       uint32_t dst_att = subpass->resolve_attachments[i];
 824
 825       if (dst_att == VK_ATTACHMENT_UNUSED)
 826          continue;
 827
 828       struct anv_image_view *src_iview = fb->attachments[src_att];
 829       struct anv_image_view *dst_iview = fb->attachments[dst_att];
 830
 831       const VkRect2D render_area = cmd_buffer->state.render_area;
 832
 833       assert(src_iview->aspect_mask == dst_iview->aspect_mask);
 834       resolve_image(&batch, src_iview->image,
 835                     src_iview->base_mip, src_iview->base_layer,
 836                     dst_iview->image,
 837                     dst_iview->base_mip, dst_iview->base_layer,
 838                     src_iview->aspect_mask,
 839                     render_area.offset.x, render_area.offset.y,
 840                     render_area.offset.x, render_area.offset.y,
 841                     render_area.extent.width, render_area.extent.height);
 842    }
 843
 844    blorp_batch_finish(&batch);
 845 }