src/intel/vulkan/anv_blorp.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "anv_private.h"
  25
  26 static bool
  27 lookup_blorp_shader(struct blorp_context *blorp,
  28                     const void *key, uint32_t key_size,
  29                     uint32_t *kernel_out, void *prog_data_out)
  30 {
  31    struct anv_device *device = blorp->driver_ctx;
  32
  33    /* The blorp cache must be a real cache */
  34    assert(device->blorp_shader_cache.cache);
  35
  36    struct anv_shader_bin *bin =
  37       anv_pipeline_cache_search(&device->blorp_shader_cache, key, key_size);
  38    if (!bin)
  39       return false;
  40
  41    /* The cache already has a reference and it's not going anywhere so there
  42     * is no need to hold a second reference.
  43     */
  44    anv_shader_bin_unref(device, bin);
  45
  46    *kernel_out = bin->kernel.offset;
  47    *(const struct brw_stage_prog_data **)prog_data_out =
  48       anv_shader_bin_get_prog_data(bin);
  49
  50    return true;
  51 }
  52
  53 static void
  54 upload_blorp_shader(struct blorp_context *blorp,
  55                     const void *key, uint32_t key_size,
  56                     const void *kernel, uint32_t kernel_size,
  57                     const void *prog_data, uint32_t prog_data_size,
  58                     uint32_t *kernel_out, void *prog_data_out)
  59 {
  60    struct anv_device *device = blorp->driver_ctx;
  61
  62    /* The blorp cache must be a real cache */
  63    assert(device->blorp_shader_cache.cache);
  64
  65    struct anv_pipeline_bind_map bind_map = {
  66       .surface_count = 0,
  67       .sampler_count = 0,
  68    };
  69
  70    struct anv_shader_bin *bin =
  71       anv_pipeline_cache_upload_kernel(&device->blorp_shader_cache,
  72                                        key, key_size, kernel, kernel_size,
  73                                        prog_data, prog_data_size, &bind_map);
  74
  75    /* The cache already has a reference and it's not going anywhere so there
  76     * is no need to hold a second reference.
  77     */
  78    anv_shader_bin_unref(device, bin);
  79
  80    *kernel_out = bin->kernel.offset;
  81    *(const struct brw_stage_prog_data **)prog_data_out =
  82       anv_shader_bin_get_prog_data(bin);
  83 }
  84
  85 void
  86 anv_device_init_blorp(struct anv_device *device)
  87 {
  88    anv_pipeline_cache_init(&device->blorp_shader_cache, device, true);
  89    blorp_init(&device->blorp, device, &device->isl_dev);
  90    device->blorp.compiler = device->instance->physicalDevice.compiler;
  91    device->blorp.mocs.tex = device->default_mocs;
  92    device->blorp.mocs.rb = device->default_mocs;
  93    device->blorp.mocs.vb = device->default_mocs;
  94    device->blorp.lookup_shader = lookup_blorp_shader;
  95    device->blorp.upload_shader = upload_blorp_shader;
  96    switch (device->info.gen) {
  97    case 7:
  98       if (device->info.is_haswell) {
  99          device->blorp.exec = gen75_blorp_exec;
 100       } else {
 101          device->blorp.exec = gen7_blorp_exec;
 102       }
 103       break;
 104    case 8:
 105       device->blorp.exec = gen8_blorp_exec;
 106       break;
 107    case 9:
 108       device->blorp.exec = gen9_blorp_exec;
 109       break;
 110    default:
 111       unreachable("Unknown hardware generation");
 112    }
 113 }
 114
 115 void
 116 anv_device_finish_blorp(struct anv_device *device)
 117 {
 118    blorp_finish(&device->blorp);
 119    anv_pipeline_cache_finish(&device->blorp_shader_cache);
 120 }
 121
 122 static void
 123 get_blorp_surf_for_anv_buffer(struct anv_device *device,
 124                               struct anv_buffer *buffer, uint64_t offset,
 125                               uint32_t width, uint32_t height,
 126                               uint32_t row_pitch, enum isl_format format,
 127                               struct blorp_surf *blorp_surf,
 128                               struct isl_surf *isl_surf)
 129 {
 130    *blorp_surf = (struct blorp_surf) {
 131       .surf = isl_surf,
 132       .addr = {
 133          .buffer = buffer->bo,
 134          .offset = buffer->offset + offset,
 135       },
 136    };
 137
 138    isl_surf_init(&device->isl_dev, isl_surf,
 139                  .dim = ISL_SURF_DIM_2D,
 140                  .format = format,
 141                  .width = width,
 142                  .height = height,
 143                  .depth = 1,
 144                  .levels = 1,
 145                  .array_len = 1,
 146                  .samples = 1,
 147                  .min_pitch = row_pitch,
 148                  .usage = ISL_SURF_USAGE_TEXTURE_BIT |
 149                           ISL_SURF_USAGE_RENDER_TARGET_BIT,
 150                  .tiling_flags = ISL_TILING_LINEAR_BIT);
 151    assert(isl_surf->row_pitch == row_pitch);
 152 }
 153
 154 static void
 155 get_blorp_surf_for_anv_image(const struct anv_image *image,
 156                              VkImageAspectFlags aspect,
 157                              struct blorp_surf *blorp_surf)
 158 {
 159    const struct anv_surface *surface =
 160       anv_image_get_surface_for_aspect_mask(image, aspect);
 161
 162    *blorp_surf = (struct blorp_surf) {
 163       .surf = &surface->isl,
 164       .addr = {
 165          .buffer = image->bo,
 166          .offset = image->offset + surface->offset,
 167       },
 168    };
 169 }
 170
 171 static void
 172 copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer,
 173                      struct anv_buffer *anv_buffer,
 174                      struct anv_image *anv_image,
 175                      uint32_t regionCount,
 176                      const VkBufferImageCopy* pRegions,
 177                      bool buffer_to_image)
 178 {
 179    struct blorp_batch batch;
 180    blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer);
 181
 182    struct {
 183       struct blorp_surf surf;
 184       uint32_t level;
 185       VkOffset3D offset;
 186    } image, buffer, *src, *dst;
 187
 188    buffer.level = 0;
 189    buffer.offset = (VkOffset3D) { 0, 0, 0 };
 190
 191    if (buffer_to_image) {
 192       src = &buffer;
 193       dst = &image;
 194    } else {
 195       src = &image;
 196       dst = &buffer;
 197    }
 198
 199    for (unsigned r = 0; r < regionCount; r++) {
 200       const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
 201
 202       get_blorp_surf_for_anv_image(anv_image, aspect, &image.surf);
 203       image.offset =
 204          anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset);
 205       image.level = pRegions[r].imageSubresource.mipLevel;
 206
 207       VkExtent3D extent =
 208          anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent);
 209       if (anv_image->type != VK_IMAGE_TYPE_3D) {
 210          image.offset.z = pRegions[r].imageSubresource.baseArrayLayer;
 211          extent.depth = pRegions[r].imageSubresource.layerCount;
 212       }
 213
 214       const enum isl_format buffer_format =
 215          anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format,
 216                             aspect, VK_IMAGE_TILING_LINEAR);
 217
 218       const VkExtent3D bufferImageExtent = {
 219          .width  = pRegions[r].bufferRowLength ?
 220                    pRegions[r].bufferRowLength : extent.width,
 221          .height = pRegions[r].bufferImageHeight ?
 222                    pRegions[r].bufferImageHeight : extent.height,
 223       };
 224
 225       const struct isl_format_layout *buffer_fmtl =
 226          isl_format_get_layout(buffer_format);
 227
 228       const uint32_t buffer_row_pitch =
 229          DIV_ROUND_UP(bufferImageExtent.width, buffer_fmtl->bw) *
 230          (buffer_fmtl->bpb / 8);
 231
 232       const uint32_t buffer_layer_stride =
 233          DIV_ROUND_UP(bufferImageExtent.height, buffer_fmtl->bh) *
 234          buffer_row_pitch;
 235
 236       struct isl_surf buffer_isl_surf;
 237       get_blorp_surf_for_anv_buffer(cmd_buffer->device,
 238                                     anv_buffer, pRegions[r].bufferOffset,
 239                                     extent.width, extent.height,
 240                                     buffer_row_pitch, buffer_format,
 241                                     &buffer.surf, &buffer_isl_surf);
 242
 243       for (unsigned z = 0; z < extent.depth; z++) {
 244          blorp_copy(&batch, &src->surf, src->level, src->offset.z,
 245                     &dst->surf, dst->level, dst->offset.z,
 246                     src->offset.x, src->offset.y, dst->offset.x, dst->offset.y,
 247                     extent.width, extent.height);
 248
 249          image.offset.z++;
 250          buffer.surf.addr.offset += buffer_layer_stride;
 251       }
 252    }
 253
 254    blorp_batch_finish(&batch);
 255 }
 256
 257 void anv_CmdCopyBufferToImage(
 258     VkCommandBuffer                             commandBuffer,
 259     VkBuffer                                    srcBuffer,
 260     VkImage                                     dstImage,
 261     VkImageLayout                               dstImageLayout,
 262     uint32_t                                    regionCount,
 263     const VkBufferImageCopy*                    pRegions)
 264 {
 265    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 266    ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
 267    ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
 268
 269    copy_buffer_to_image(cmd_buffer, src_buffer, dst_image,
 270                         regionCount, pRegions, true);
 271 }
 272
 273 void anv_CmdCopyImageToBuffer(
 274     VkCommandBuffer                             commandBuffer,
 275     VkImage                                     srcImage,
 276     VkImageLayout                               srcImageLayout,
 277     VkBuffer                                    dstBuffer,
 278     uint32_t                                    regionCount,
 279     const VkBufferImageCopy*                    pRegions)
 280 {
 281    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 282    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 283    ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
 284
 285    copy_buffer_to_image(cmd_buffer, dst_buffer, src_image,
 286                         regionCount, pRegions, false);
 287 }
 288
 289 static bool
 290 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
 291 {
 292    bool flip = false;
 293    if (*src0 > *src1) {
 294       unsigned tmp = *src0;
 295       *src0 = *src1;
 296       *src1 = tmp;
 297       flip = !flip;
 298    }
 299
 300    if (*dst0 > *dst1) {
 301       unsigned tmp = *dst0;
 302       *dst0 = *dst1;
 303       *dst1 = tmp;
 304       flip = !flip;
 305    }
 306
 307    return flip;
 308 }
 309
 310 void anv_CmdBlitImage(
 311     VkCommandBuffer                             commandBuffer,
 312     VkImage                                     srcImage,
 313     VkImageLayout                               srcImageLayout,
 314     VkImage                                     dstImage,
 315     VkImageLayout                               dstImageLayout,
 316     uint32_t                                    regionCount,
 317     const VkImageBlit*                          pRegions,
 318     VkFilter                                    filter)
 319
 320 {
 321    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 322    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 323    ANV_FROM_HANDLE(anv_image, dst_image, dstImage);
 324
 325    struct blorp_surf src, dst;
 326
 327    uint32_t gl_filter;
 328    switch (filter) {
 329    case VK_FILTER_NEAREST:
 330       gl_filter = 0x2600; /* GL_NEAREST */
 331       break;
 332    case VK_FILTER_LINEAR:
 333       gl_filter = 0x2601; /* GL_LINEAR */
 334       break;
 335    default:
 336       unreachable("Invalid filter");
 337    }
 338
 339    struct blorp_batch batch;
 340    blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer);
 341
 342    for (unsigned r = 0; r < regionCount; r++) {
 343       const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource;
 344       const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource;
 345
 346       get_blorp_surf_for_anv_image(src_image, src_res->aspectMask, &src);
 347       get_blorp_surf_for_anv_image(dst_image, dst_res->aspectMask, &dst);
 348
 349       struct anv_format src_format =
 350          anv_get_format(&cmd_buffer->device->info, src_image->vk_format,
 351                         src_res->aspectMask, src_image->tiling);
 352       struct anv_format dst_format =
 353          anv_get_format(&cmd_buffer->device->info, dst_image->vk_format,
 354                         dst_res->aspectMask, dst_image->tiling);
 355
 356       unsigned dst_start, dst_end;
 357       if (dst_image->type == VK_IMAGE_TYPE_3D) {
 358          assert(dst_res->baseArrayLayer == 0);
 359          dst_start = pRegions[r].dstOffsets[0].z;
 360          dst_end = pRegions[r].dstOffsets[1].z;
 361       } else {
 362          dst_start = dst_res->baseArrayLayer;
 363          dst_end = dst_start + dst_res->layerCount;
 364       }
 365
 366       unsigned src_start, src_end;
 367       if (src_image->type == VK_IMAGE_TYPE_3D) {
 368          assert(src_res->baseArrayLayer == 0);
 369          src_start = pRegions[r].srcOffsets[0].z;
 370          src_end = pRegions[r].srcOffsets[1].z;
 371       } else {
 372          src_start = src_res->baseArrayLayer;
 373          src_end = src_start + src_res->layerCount;
 374       }
 375
 376       bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
 377       float src_z_step = (float)(src_end + 1 - src_start) /
 378                          (float)(dst_end + 1 - dst_start);
 379
 380       if (flip_z) {
 381          src_start = src_end;
 382          src_z_step *= -1;
 383       }
 384
 385       unsigned src_x0 = pRegions[r].srcOffsets[0].x;
 386       unsigned src_x1 = pRegions[r].srcOffsets[1].x;
 387       unsigned dst_x0 = pRegions[r].dstOffsets[0].x;
 388       unsigned dst_x1 = pRegions[r].dstOffsets[1].x;
 389       bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1);
 390
 391       unsigned src_y0 = pRegions[r].srcOffsets[0].y;
 392       unsigned src_y1 = pRegions[r].srcOffsets[1].y;
 393       unsigned dst_y0 = pRegions[r].dstOffsets[0].y;
 394       unsigned dst_y1 = pRegions[r].dstOffsets[1].y;
 395       bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1);
 396
 397       const unsigned num_layers = dst_end - dst_start;
 398       for (unsigned i = 0; i < num_layers; i++) {
 399          unsigned dst_z = dst_start + i;
 400          unsigned src_z = src_start + i * src_z_step;
 401
 402          blorp_blit(&batch, &src, src_res->mipLevel, src_z,
 403                     src_format.isl_format, src_format.swizzle,
 404                     &dst, dst_res->mipLevel, dst_z,
 405                     dst_format.isl_format, dst_format.swizzle,
 406                     src_x0, src_y0, src_x1, src_y1,
 407                     dst_x0, dst_y0, dst_x1, dst_y1,
 408                     gl_filter, flip_x, flip_y);
 409       }
 410
 411    }
 412
 413    blorp_batch_finish(&batch);
 414 }