src/vulkan/anv_meta_blit.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "anv_meta.h"
  25 #include "nir/nir_builder.h"
  26
  27 struct blit_region {
  28    VkOffset3D src_offset;
  29    VkExtent3D src_extent;
  30    VkOffset3D dest_offset;
  31    VkExtent3D dest_extent;
  32 };
  33
  34 static nir_shader *
  35 build_nir_vertex_shader(void)
  36 {
  37    const struct glsl_type *vec4 = glsl_vec4_type();
  38    nir_builder b;
  39
  40    nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_VERTEX, NULL);
  41    b.shader->info.name = ralloc_strdup(b.shader, "meta_blit_vs");
  42
  43    nir_variable *pos_in = nir_variable_create(b.shader, nir_var_shader_in,
  44                                               vec4, "a_pos");
  45    pos_in->data.location = VERT_ATTRIB_GENERIC0;
  46    nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out,
  47                                                vec4, "gl_Position");
  48    pos_out->data.location = VARYING_SLOT_POS;
  49    nir_copy_var(&b, pos_out, pos_in);
  50
  51    nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
  52                                                   vec4, "a_tex_pos");
  53    tex_pos_in->data.location = VERT_ATTRIB_GENERIC1;
  54    nir_variable *tex_pos_out = nir_variable_create(b.shader, nir_var_shader_out,
  55                                                    vec4, "v_tex_pos");
  56    tex_pos_out->data.location = VARYING_SLOT_VAR0;
  57    tex_pos_out->data.interpolation = INTERP_QUALIFIER_SMOOTH;
  58    nir_copy_var(&b, tex_pos_out, tex_pos_in);
  59
  60    return b.shader;
  61 }
  62
  63 static nir_shader *
  64 build_nir_copy_fragment_shader(enum glsl_sampler_dim tex_dim)
  65 {
  66    const struct glsl_type *vec4 = glsl_vec4_type();
  67    nir_builder b;
  68
  69    nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
  70    b.shader->info.name = ralloc_strdup(b.shader, "meta_blit_fs");
  71
  72    nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in,
  73                                                   vec4, "v_tex_pos");
  74    tex_pos_in->data.location = VARYING_SLOT_VAR0;
  75
  76    /* Swizzle the array index which comes in as Z coordinate into the right
  77     * position.
  78     */
  79    unsigned swz[] = { 0, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 1), 2 };
  80    nir_ssa_def *const tex_pos =
  81       nir_swizzle(&b, nir_load_var(&b, tex_pos_in), swz,
  82                   (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 3), false);
  83
  84    const struct glsl_type *sampler_type =
  85       glsl_sampler_type(tex_dim, false, tex_dim != GLSL_SAMPLER_DIM_3D,
  86                         glsl_get_base_type(vec4));
  87    nir_variable *sampler = nir_variable_create(b.shader, nir_var_uniform,
  88                                                sampler_type, "s_tex");
  89    sampler->data.descriptor_set = 0;
  90    sampler->data.binding = 0;
  91
  92    nir_tex_instr *tex = nir_tex_instr_create(b.shader, 1);
  93    tex->sampler_dim = tex_dim;
  94    tex->op = nir_texop_tex;
  95    tex->src[0].src_type = nir_tex_src_coord;
  96    tex->src[0].src = nir_src_for_ssa(tex_pos);
  97    tex->dest_type = nir_type_float; /* TODO */
  98    tex->is_array = glsl_sampler_type_is_array(sampler_type);
  99    tex->coord_components = tex_pos->num_components;
 100    tex->texture = nir_deref_var_create(tex, sampler);
 101    tex->sampler = nir_deref_var_create(tex, sampler);
 102
 103    nir_ssa_dest_init(&tex->instr, &tex->dest, 4, "tex");
 104    nir_builder_instr_insert(&b, &tex->instr);
 105
 106    nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out,
 107                                                  vec4, "f_color");
 108    color_out->data.location = FRAG_RESULT_DATA0;
 109    nir_store_var(&b, color_out, &tex->dest.ssa, 4);
 110
 111    return b.shader;
 112 }
 113
 114 static void
 115 meta_prepare_blit(struct anv_cmd_buffer *cmd_buffer,
 116                   struct anv_meta_saved_state *saved_state)
 117 {
 118    anv_meta_save(saved_state, cmd_buffer,
 119                  (1 << VK_DYNAMIC_STATE_VIEWPORT));
 120 }
 121
 122 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of
 123  * elements rather than texels. One element equals one texel or one block
 124  * if Image is uncompressed or compressed, respectively.
 125  */
 126 static struct VkOffset3D
 127 meta_region_offset_el(const struct anv_image * image,
 128                       const struct VkOffset3D * offset)
 129 {
 130    const struct isl_format_layout * isl_layout = image->format->isl_layout;
 131    return (VkOffset3D) {
 132       .x = offset->x / isl_layout->bw,
 133       .y = offset->y / isl_layout->bh,
 134       .z = offset->z / isl_layout->bd,
 135    };
 136 }
 137
 138 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of
 139  * elements rather than texels. One element equals one texel or one block
 140  * if Image is uncompressed or compressed, respectively.
 141  */
 142 static struct VkExtent3D
 143 meta_region_extent_el(const VkFormat format,
 144                       const struct VkExtent3D * extent)
 145 {
 146    const struct isl_format_layout * isl_layout =
 147       anv_format_for_vk_format(format)->isl_layout;
 148    return (VkExtent3D) {
 149       .width  = DIV_ROUND_UP(extent->width , isl_layout->bw),
 150       .height = DIV_ROUND_UP(extent->height, isl_layout->bh),
 151       .depth  = DIV_ROUND_UP(extent->depth , isl_layout->bd),
 152    };
 153 }
 154
 155 static void
 156 meta_emit_blit(struct anv_cmd_buffer *cmd_buffer,
 157                struct anv_image *src_image,
 158                struct anv_image_view *src_iview,
 159                VkOffset3D src_offset,
 160                VkExtent3D src_extent,
 161                struct anv_image *dest_image,
 162                struct anv_image_view *dest_iview,
 163                VkOffset3D dest_offset,
 164                VkExtent3D dest_extent,
 165                VkFilter blit_filter)
 166 {
 167    struct anv_device *device = cmd_buffer->device;
 168    VkDescriptorPool dummy_desc_pool = (VkDescriptorPool)1;
 169
 170    struct blit_vb_data {
 171       float pos[2];
 172       float tex_coord[3];
 173    } *vb_data;
 174
 175    assert(src_image->samples == dest_image->samples);
 176
 177    unsigned vb_size = sizeof(struct anv_vue_header) + 3 * sizeof(*vb_data);
 178
 179    struct anv_state vb_state =
 180       anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, vb_size, 16);
 181    memset(vb_state.map, 0, sizeof(struct anv_vue_header));
 182    vb_data = vb_state.map + sizeof(struct anv_vue_header);
 183
 184    vb_data[0] = (struct blit_vb_data) {
 185       .pos = {
 186          dest_offset.x + dest_extent.width,
 187          dest_offset.y + dest_extent.height,
 188       },
 189       .tex_coord = {
 190          (float)(src_offset.x + src_extent.width) / (float)src_iview->extent.width,
 191          (float)(src_offset.y + src_extent.height) / (float)src_iview->extent.height,
 192          (float)src_offset.z / (float)src_iview->extent.depth,
 193       },
 194    };
 195
 196    vb_data[1] = (struct blit_vb_data) {
 197       .pos = {
 198          dest_offset.x,
 199          dest_offset.y + dest_extent.height,
 200       },
 201       .tex_coord = {
 202          (float)src_offset.x / (float)src_iview->extent.width,
 203          (float)(src_offset.y + src_extent.height) / (float)src_iview->extent.height,
 204          (float)src_offset.z / (float)src_iview->extent.depth,
 205       },
 206    };
 207
 208    vb_data[2] = (struct blit_vb_data) {
 209       .pos = {
 210          dest_offset.x,
 211          dest_offset.y,
 212       },
 213       .tex_coord = {
 214          (float)src_offset.x / (float)src_iview->extent.width,
 215          (float)src_offset.y / (float)src_iview->extent.height,
 216          (float)src_offset.z / (float)src_iview->extent.depth,
 217       },
 218    };
 219
 220    anv_state_clflush(vb_state);
 221
 222    struct anv_buffer vertex_buffer = {
 223       .device = device,
 224       .size = vb_size,
 225       .bo = &device->dynamic_state_block_pool.bo,
 226       .offset = vb_state.offset,
 227    };
 228
 229    anv_CmdBindVertexBuffers(anv_cmd_buffer_to_handle(cmd_buffer), 0, 2,
 230       (VkBuffer[]) {
 231          anv_buffer_to_handle(&vertex_buffer),
 232          anv_buffer_to_handle(&vertex_buffer)
 233       },
 234       (VkDeviceSize[]) {
 235          0,
 236          sizeof(struct anv_vue_header),
 237       });
 238
 239    VkSampler sampler;
 240    ANV_CALL(CreateSampler)(anv_device_to_handle(device),
 241       &(VkSamplerCreateInfo) {
 242          .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
 243          .magFilter = blit_filter,
 244          .minFilter = blit_filter,
 245       }, &cmd_buffer->pool->alloc, &sampler);
 246
 247    VkDescriptorSet set;
 248    anv_AllocateDescriptorSets(anv_device_to_handle(device),
 249       &(VkDescriptorSetAllocateInfo) {
 250          .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,
 251          .descriptorPool = dummy_desc_pool,
 252          .descriptorSetCount = 1,
 253          .pSetLayouts = &device->meta_state.blit.ds_layout
 254       }, &set);
 255    anv_UpdateDescriptorSets(anv_device_to_handle(device),
 256       1, /* writeCount */
 257       (VkWriteDescriptorSet[]) {
 258          {
 259             .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
 260             .dstSet = set,
 261             .dstBinding = 0,
 262             .dstArrayElement = 0,
 263             .descriptorCount = 1,
 264             .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
 265             .pImageInfo = (VkDescriptorImageInfo[]) {
 266                {
 267                   .sampler = sampler,
 268                   .imageView = anv_image_view_to_handle(src_iview),
 269                   .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
 270                },
 271             }
 272          }
 273       }, 0, NULL);
 274
 275    VkFramebuffer fb;
 276    anv_CreateFramebuffer(anv_device_to_handle(device),
 277       &(VkFramebufferCreateInfo) {
 278          .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
 279          .attachmentCount = 1,
 280          .pAttachments = (VkImageView[]) {
 281             anv_image_view_to_handle(dest_iview),
 282          },
 283          .width = dest_iview->extent.width,
 284          .height = dest_iview->extent.height,
 285          .layers = 1
 286       }, &cmd_buffer->pool->alloc, &fb);
 287
 288    ANV_CALL(CmdBeginRenderPass)(anv_cmd_buffer_to_handle(cmd_buffer),
 289       &(VkRenderPassBeginInfo) {
 290          .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 291          .renderPass = device->meta_state.blit.render_pass,
 292          .framebuffer = fb,
 293          .renderArea = {
 294             .offset = { dest_offset.x, dest_offset.y },
 295             .extent = { dest_extent.width, dest_extent.height },
 296          },
 297          .clearValueCount = 0,
 298          .pClearValues = NULL,
 299       }, VK_SUBPASS_CONTENTS_INLINE);
 300
 301    VkPipeline pipeline;
 302
 303    switch (src_image->type) {
 304    case VK_IMAGE_TYPE_1D:
 305       pipeline = device->meta_state.blit.pipeline_1d_src;
 306       break;
 307    case VK_IMAGE_TYPE_2D:
 308       pipeline = device->meta_state.blit.pipeline_2d_src;
 309       break;
 310    case VK_IMAGE_TYPE_3D:
 311       pipeline = device->meta_state.blit.pipeline_3d_src;
 312       break;
 313    default:
 314       unreachable(!"bad VkImageType");
 315    }
 316
 317    if (cmd_buffer->state.pipeline != anv_pipeline_from_handle(pipeline)) {
 318       anv_CmdBindPipeline(anv_cmd_buffer_to_handle(cmd_buffer),
 319                           VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
 320    }
 321
 322    anv_CmdSetViewport(anv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
 323                       &(VkViewport) {
 324                         .x = 0.0f,
 325                         .y = 0.0f,
 326                         .width = dest_iview->extent.width,
 327                         .height = dest_iview->extent.height,
 328                         .minDepth = 0.0f,
 329                         .maxDepth = 1.0f,
 330                       });
 331
 332    anv_CmdBindDescriptorSets(anv_cmd_buffer_to_handle(cmd_buffer),
 333                              VK_PIPELINE_BIND_POINT_GRAPHICS,
 334                              device->meta_state.blit.pipeline_layout, 0, 1,
 335                              &set, 0, NULL);
 336
 337    ANV_CALL(CmdDraw)(anv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
 338
 339    ANV_CALL(CmdEndRenderPass)(anv_cmd_buffer_to_handle(cmd_buffer));
 340
 341    /* At the point where we emit the draw call, all data from the
 342     * descriptor sets, etc. has been used.  We are free to delete it.
 343     */
 344    anv_descriptor_set_destroy(device, anv_descriptor_set_from_handle(set));
 345    anv_DestroySampler(anv_device_to_handle(device), sampler,
 346                       &cmd_buffer->pool->alloc);
 347    anv_DestroyFramebuffer(anv_device_to_handle(device), fb,
 348                           &cmd_buffer->pool->alloc);
 349 }
 350
 351 static void
 352 meta_finish_blit(struct anv_cmd_buffer *cmd_buffer,
 353                  const struct anv_meta_saved_state *saved_state)
 354 {
 355    anv_meta_restore(saved_state, cmd_buffer);
 356 }
 357
 358 static VkFormat
 359 vk_format_for_size(int bs)
 360 {
 361    /* Note: We intentionally use the 4-channel formats whenever we can.
 362     * This is so that, when we do a RGB <-> RGBX copy, the two formats will
 363     * line up even though one of them is 3/4 the size of the other.
 364     */
 365    switch (bs) {
 366    case 1: return VK_FORMAT_R8_UINT;
 367    case 2: return VK_FORMAT_R8G8_UINT;
 368    case 3: return VK_FORMAT_R8G8B8_UINT;
 369    case 4: return VK_FORMAT_R8G8B8A8_UINT;
 370    case 6: return VK_FORMAT_R16G16B16_UINT;
 371    case 8: return VK_FORMAT_R16G16B16A16_UINT;
 372    case 12: return VK_FORMAT_R32G32B32_UINT;
 373    case 16: return VK_FORMAT_R32G32B32A32_UINT;
 374    default:
 375       unreachable("Invalid format block size");
 376    }
 377 }
 378
 379 static void
 380 do_buffer_copy(struct anv_cmd_buffer *cmd_buffer,
 381                struct anv_bo *src, uint64_t src_offset,
 382                struct anv_bo *dest, uint64_t dest_offset,
 383                int width, int height, VkFormat copy_format)
 384 {
 385    VkDevice vk_device = anv_device_to_handle(cmd_buffer->device);
 386
 387    VkImageCreateInfo image_info = {
 388       .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
 389       .imageType = VK_IMAGE_TYPE_2D,
 390       .format = copy_format,
 391       .extent = {
 392          .width = width,
 393          .height = height,
 394          .depth = 1,
 395       },
 396       .mipLevels = 1,
 397       .arrayLayers = 1,
 398       .samples = 1,
 399       .tiling = VK_IMAGE_TILING_LINEAR,
 400       .usage = 0,
 401       .flags = 0,
 402    };
 403
 404    VkImage src_image;
 405    image_info.usage = VK_IMAGE_USAGE_SAMPLED_BIT;
 406    anv_CreateImage(vk_device, &image_info,
 407                    &cmd_buffer->pool->alloc, &src_image);
 408
 409    VkImage dest_image;
 410    image_info.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
 411    anv_CreateImage(vk_device, &image_info,
 412                    &cmd_buffer->pool->alloc, &dest_image);
 413
 414    /* We could use a vk call to bind memory, but that would require
 415     * creating a dummy memory object etc. so there's really no point.
 416     */
 417    anv_image_from_handle(src_image)->bo = src;
 418    anv_image_from_handle(src_image)->offset = src_offset;
 419    anv_image_from_handle(dest_image)->bo = dest;
 420    anv_image_from_handle(dest_image)->offset = dest_offset;
 421
 422    struct anv_image_view src_iview;
 423    anv_image_view_init(&src_iview, cmd_buffer->device,
 424       &(VkImageViewCreateInfo) {
 425          .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 426          .image = src_image,
 427          .viewType = VK_IMAGE_VIEW_TYPE_2D,
 428          .format = copy_format,
 429          .subresourceRange = {
 430             .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 431             .baseMipLevel = 0,
 432             .levelCount = 1,
 433             .baseArrayLayer = 0,
 434             .layerCount = 1
 435          },
 436       },
 437       cmd_buffer, 0);
 438
 439    struct anv_image_view dest_iview;
 440    anv_image_view_init(&dest_iview, cmd_buffer->device,
 441       &(VkImageViewCreateInfo) {
 442          .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 443          .image = dest_image,
 444          .viewType = VK_IMAGE_VIEW_TYPE_2D,
 445          .format = copy_format,
 446          .subresourceRange = {
 447             .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 448             .baseMipLevel = 0,
 449             .levelCount = 1,
 450             .baseArrayLayer = 0,
 451             .layerCount = 1,
 452          },
 453       },
 454       cmd_buffer, 0);
 455
 456    meta_emit_blit(cmd_buffer,
 457                   anv_image_from_handle(src_image),
 458                   &src_iview,
 459                   (VkOffset3D) { 0, 0, 0 },
 460                   (VkExtent3D) { width, height, 1 },
 461                   anv_image_from_handle(dest_image),
 462                   &dest_iview,
 463                   (VkOffset3D) { 0, 0, 0 },
 464                   (VkExtent3D) { width, height, 1 },
 465                   VK_FILTER_NEAREST);
 466
 467    anv_DestroyImage(vk_device, src_image, &cmd_buffer->pool->alloc);
 468    anv_DestroyImage(vk_device, dest_image, &cmd_buffer->pool->alloc);
 469 }
 470
 471 void anv_CmdCopyBuffer(
 472     VkCommandBuffer                             commandBuffer,
 473     VkBuffer                                    srcBuffer,
 474     VkBuffer                                    destBuffer,
 475     uint32_t                                    regionCount,
 476     const VkBufferCopy*                         pRegions)
 477 {
 478    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 479    ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer);
 480    ANV_FROM_HANDLE(anv_buffer, dest_buffer, destBuffer);
 481
 482    struct anv_meta_saved_state saved_state;
 483
 484    meta_prepare_blit(cmd_buffer, &saved_state);
 485
 486    for (unsigned r = 0; r < regionCount; r++) {
 487       uint64_t src_offset = src_buffer->offset + pRegions[r].srcOffset;
 488       uint64_t dest_offset = dest_buffer->offset + pRegions[r].dstOffset;
 489       uint64_t copy_size = pRegions[r].size;
 490
 491       /* First, we compute the biggest format that can be used with the
 492        * given offsets and size.
 493        */
 494       int bs = 16;
 495
 496       int fs = ffs(src_offset) - 1;
 497       if (fs != -1)
 498          bs = MIN2(bs, 1 << fs);
 499       assert(src_offset % bs == 0);
 500
 501       fs = ffs(dest_offset) - 1;
 502       if (fs != -1)
 503          bs = MIN2(bs, 1 << fs);
 504       assert(dest_offset % bs == 0);
 505
 506       fs = ffs(pRegions[r].size) - 1;
 507       if (fs != -1)
 508          bs = MIN2(bs, 1 << fs);
 509       assert(pRegions[r].size % bs == 0);
 510
 511       VkFormat copy_format = vk_format_for_size(bs);
 512
 513       /* This is maximum possible width/height our HW can handle */
 514       uint64_t max_surface_dim = 1 << 14;
 515
 516       /* First, we make a bunch of max-sized copies */
 517       uint64_t max_copy_size = max_surface_dim * max_surface_dim * bs;
 518       while (copy_size >= max_copy_size) {
 519          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 520                         dest_buffer->bo, dest_offset,
 521                         max_surface_dim, max_surface_dim, copy_format);
 522          copy_size -= max_copy_size;
 523          src_offset += max_copy_size;
 524          dest_offset += max_copy_size;
 525       }
 526
 527       uint64_t height = copy_size / (max_surface_dim * bs);
 528       assert(height < max_surface_dim);
 529       if (height != 0) {
 530          uint64_t rect_copy_size = height * max_surface_dim * bs;
 531          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 532                         dest_buffer->bo, dest_offset,
 533                         max_surface_dim, height, copy_format);
 534          copy_size -= rect_copy_size;
 535          src_offset += rect_copy_size;
 536          dest_offset += rect_copy_size;
 537       }
 538
 539       if (copy_size != 0) {
 540          do_buffer_copy(cmd_buffer, src_buffer->bo, src_offset,
 541                         dest_buffer->bo, dest_offset,
 542                         copy_size / bs, 1, copy_format);
 543       }
 544    }
 545
 546    meta_finish_blit(cmd_buffer, &saved_state);
 547 }
 548
 549 void anv_CmdUpdateBuffer(
 550     VkCommandBuffer                             commandBuffer,
 551     VkBuffer                                    dstBuffer,
 552     VkDeviceSize                                dstOffset,
 553     VkDeviceSize                                dataSize,
 554     const uint32_t*                             pData)
 555 {
 556    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 557    ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer);
 558    struct anv_meta_saved_state saved_state;
 559
 560    meta_prepare_blit(cmd_buffer, &saved_state);
 561
 562    /* We can't quite grab a full block because the state stream needs a
 563     * little data at the top to build its linked list.
 564     */
 565    const uint32_t max_update_size =
 566       cmd_buffer->device->dynamic_state_block_pool.block_size - 64;
 567
 568    assert(max_update_size < (1 << 14) * 4);
 569
 570    while (dataSize) {
 571       const uint32_t copy_size = MIN2(dataSize, max_update_size);
 572
 573       struct anv_state tmp_data =
 574          anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64);
 575
 576       memcpy(tmp_data.map, pData, copy_size);
 577
 578       VkFormat format;
 579       int bs;
 580       if ((copy_size & 15) == 0 && (dstOffset & 15) == 0) {
 581          format = VK_FORMAT_R32G32B32A32_UINT;
 582          bs = 16;
 583       } else if ((copy_size & 7) == 0 && (dstOffset & 7) == 0) {
 584          format = VK_FORMAT_R32G32_UINT;
 585          bs = 8;
 586       } else {
 587          assert((copy_size & 3) == 0 && (dstOffset & 3) == 0);
 588          format = VK_FORMAT_R32_UINT;
 589          bs = 4;
 590       }
 591
 592       do_buffer_copy(cmd_buffer,
 593                      &cmd_buffer->device->dynamic_state_block_pool.bo,
 594                      tmp_data.offset,
 595                      dst_buffer->bo, dst_buffer->offset + dstOffset,
 596                      copy_size / bs, 1, format);
 597
 598       dataSize -= copy_size;
 599       dstOffset += copy_size;
 600       pData = (void *)pData + copy_size;
 601    }
 602 }
 603
 604 static VkFormat
 605 choose_iview_format(struct anv_image *image, VkImageAspectFlagBits aspect)
 606 {
 607    assert(__builtin_popcount(aspect) == 1);
 608
 609    struct isl_surf *surf =
 610       &anv_image_get_surface_for_aspect_mask(image, aspect)->isl;
 611
 612    /* vkCmdCopyImage behaves like memcpy. Therefore we choose identical UINT
 613     * formats for the source and destination image views.
 614     *
 615     * From the Vulkan spec (2015-12-30):
 616     *
 617     *    vkCmdCopyImage performs image copies in a similar manner to a host
 618     *    memcpy. It does not perform general-purpose conversions such as
 619     *    scaling, resizing, blending, color-space conversion, or format
 620     *    conversions.  Rather, it simply copies raw image data. vkCmdCopyImage
 621     *    can copy between images with different formats, provided the formats
 622     *    are compatible as defined below.
 623     *
 624     *    [The spec later defines compatibility as having the same number of
 625     *    bytes per block].
 626     */
 627    return vk_format_for_size(isl_format_layouts[surf->format].bs);
 628 }
 629
 630 static VkFormat
 631 choose_buffer_format(VkFormat format, VkImageAspectFlagBits aspect)
 632 {
 633    assert(__builtin_popcount(aspect) == 1);
 634
 635    /* vkCmdCopy* commands behave like memcpy. Therefore we choose
 636     * compatable UINT formats for the source and destination image views.
 637     *
 638     * For the buffer, we go back to the original image format and get a
 639     * the format as if it were linear.  This way, for RGB formats, we get
 640     * an RGB format here even if the tiled image is RGBA. XXX: This doesn't
 641     * work if the buffer is the destination.
 642     */
 643    enum isl_format linear_format = anv_get_isl_format(format, aspect,
 644                                                       VK_IMAGE_TILING_LINEAR,
 645                                                       NULL);
 646
 647    return vk_format_for_size(isl_format_layouts[linear_format].bs);
 648 }
 649
 650 void anv_CmdCopyImage(
 651     VkCommandBuffer                             commandBuffer,
 652     VkImage                                     srcImage,
 653     VkImageLayout                               srcImageLayout,
 654     VkImage                                     destImage,
 655     VkImageLayout                               destImageLayout,
 656     uint32_t                                    regionCount,
 657     const VkImageCopy*                          pRegions)
 658 {
 659    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 660    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 661    ANV_FROM_HANDLE(anv_image, dest_image, destImage);
 662    struct anv_meta_saved_state saved_state;
 663
 664    /* From the Vulkan 1.0 spec:
 665     *
 666     *    vkCmdCopyImage can be used to copy image data between multisample
 667     *    images, but both images must have the same number of samples.
 668     */
 669    assert(src_image->samples == dest_image->samples);
 670
 671    meta_prepare_blit(cmd_buffer, &saved_state);
 672
 673    for (unsigned r = 0; r < regionCount; r++) {
 674       assert(pRegions[r].srcSubresource.aspectMask ==
 675              pRegions[r].dstSubresource.aspectMask);
 676
 677       VkImageAspectFlags aspect = pRegions[r].srcSubresource.aspectMask;
 678
 679       VkFormat src_format = choose_iview_format(src_image, aspect);
 680       VkFormat dst_format = choose_iview_format(dest_image, aspect);
 681
 682       struct anv_image_view src_iview;
 683       anv_image_view_init(&src_iview, cmd_buffer->device,
 684          &(VkImageViewCreateInfo) {
 685             .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 686             .image = srcImage,
 687             .viewType = anv_meta_get_view_type(src_image),
 688             .format = src_format,
 689             .subresourceRange = {
 690                .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 691                .baseMipLevel = pRegions[r].srcSubresource.mipLevel,
 692                .levelCount = 1,
 693                .baseArrayLayer = pRegions[r].srcSubresource.baseArrayLayer,
 694                .layerCount = pRegions[r].dstSubresource.layerCount,
 695             },
 696          },
 697          cmd_buffer, 0);
 698
 699       const VkOffset3D dest_offset = {
 700          .x = pRegions[r].dstOffset.x,
 701          .y = pRegions[r].dstOffset.y,
 702          .z = 0,
 703       };
 704
 705       unsigned num_slices;
 706       if (src_image->type == VK_IMAGE_TYPE_3D) {
 707          assert(pRegions[r].srcSubresource.layerCount == 1 &&
 708                 pRegions[r].dstSubresource.layerCount == 1);
 709          num_slices = pRegions[r].extent.depth;
 710       } else {
 711          assert(pRegions[r].srcSubresource.layerCount ==
 712                 pRegions[r].dstSubresource.layerCount);
 713          assert(pRegions[r].extent.depth == 1);
 714          num_slices = pRegions[r].dstSubresource.layerCount;
 715       }
 716
 717       const uint32_t dest_base_array_slice =
 718          anv_meta_get_iview_layer(dest_image, &pRegions[r].dstSubresource,
 719                                   &pRegions[r].dstOffset);
 720
 721       for (unsigned slice = 0; slice < num_slices; slice++) {
 722          VkOffset3D src_offset = pRegions[r].srcOffset;
 723          src_offset.z += slice;
 724
 725          struct anv_image_view dest_iview;
 726          anv_image_view_init(&dest_iview, cmd_buffer->device,
 727             &(VkImageViewCreateInfo) {
 728                .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 729                .image = destImage,
 730                .viewType = anv_meta_get_view_type(dest_image),
 731                .format = dst_format,
 732                .subresourceRange = {
 733                   .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 734                   .baseMipLevel = pRegions[r].dstSubresource.mipLevel,
 735                   .levelCount = 1,
 736                   .baseArrayLayer = dest_base_array_slice + slice,
 737                   .layerCount = 1
 738                },
 739             },
 740             cmd_buffer, 0);
 741
 742          meta_emit_blit(cmd_buffer,
 743                         src_image, &src_iview,
 744                         src_offset,
 745                         pRegions[r].extent,
 746                         dest_image, &dest_iview,
 747                         dest_offset,
 748                         pRegions[r].extent,
 749                         VK_FILTER_NEAREST);
 750       }
 751    }
 752
 753    meta_finish_blit(cmd_buffer, &saved_state);
 754 }
 755
 756 void anv_CmdBlitImage(
 757     VkCommandBuffer                             commandBuffer,
 758     VkImage                                     srcImage,
 759     VkImageLayout                               srcImageLayout,
 760     VkImage                                     destImage,
 761     VkImageLayout                               destImageLayout,
 762     uint32_t                                    regionCount,
 763     const VkImageBlit*                          pRegions,
 764     VkFilter                                    filter)
 765
 766 {
 767    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 768    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
 769    ANV_FROM_HANDLE(anv_image, dest_image, destImage);
 770    struct anv_meta_saved_state saved_state;
 771
 772    /* From the Vulkan 1.0 spec:
 773     *
 774     *    vkCmdBlitImage must not be used for multisampled source or
 775     *    destination images. Use vkCmdResolveImage for this purpose.
 776     */
 777    assert(src_image->samples == 1);
 778    assert(dest_image->samples == 1);
 779
 780    anv_finishme("respect VkFilter");
 781
 782    meta_prepare_blit(cmd_buffer, &saved_state);
 783
 784    for (unsigned r = 0; r < regionCount; r++) {
 785       struct anv_image_view src_iview;
 786       anv_image_view_init(&src_iview, cmd_buffer->device,
 787          &(VkImageViewCreateInfo) {
 788             .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 789             .image = srcImage,
 790             .viewType = anv_meta_get_view_type(src_image),
 791             .format = src_image->vk_format,
 792             .subresourceRange = {
 793                .aspectMask = pRegions[r].srcSubresource.aspectMask,
 794                .baseMipLevel = pRegions[r].srcSubresource.mipLevel,
 795                .levelCount = 1,
 796                .baseArrayLayer = pRegions[r].srcSubresource.baseArrayLayer,
 797                .layerCount = 1
 798             },
 799          },
 800          cmd_buffer, 0);
 801
 802       const VkOffset3D dest_offset = {
 803          .x = pRegions[r].dstOffsets[0].x,
 804          .y = pRegions[r].dstOffsets[0].y,
 805          .z = 0,
 806       };
 807
 808       if (pRegions[r].dstOffsets[1].x < pRegions[r].dstOffsets[0].x ||
 809           pRegions[r].dstOffsets[1].y < pRegions[r].dstOffsets[0].y ||
 810           pRegions[r].srcOffsets[1].x < pRegions[r].srcOffsets[0].x ||
 811           pRegions[r].srcOffsets[1].y < pRegions[r].srcOffsets[0].y)
 812          anv_finishme("FINISHME: Allow flipping in blits");
 813
 814       const VkExtent3D dest_extent = {
 815          .width = pRegions[r].dstOffsets[1].x - pRegions[r].dstOffsets[0].x,
 816          .height = pRegions[r].dstOffsets[1].y - pRegions[r].dstOffsets[0].y,
 817       };
 818
 819       const VkExtent3D src_extent = {
 820          .width = pRegions[r].srcOffsets[1].x - pRegions[r].srcOffsets[0].x,
 821          .height = pRegions[r].srcOffsets[1].y - pRegions[r].srcOffsets[0].y,
 822       };
 823
 824       const uint32_t dest_array_slice =
 825          anv_meta_get_iview_layer(dest_image, &pRegions[r].dstSubresource,
 826                                   &pRegions[r].dstOffsets[0]);
 827
 828       if (pRegions[r].srcSubresource.layerCount > 1)
 829          anv_finishme("FINISHME: copy multiple array layers");
 830
 831       if (pRegions[r].srcOffsets[0].z + 1 != pRegions[r].srcOffsets[1].z ||
 832           pRegions[r].dstOffsets[0].z + 1 != pRegions[r].dstOffsets[1].z)
 833          anv_finishme("FINISHME: copy multiple depth layers");
 834
 835       struct anv_image_view dest_iview;
 836       anv_image_view_init(&dest_iview, cmd_buffer->device,
 837          &(VkImageViewCreateInfo) {
 838             .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 839             .image = destImage,
 840             .viewType = anv_meta_get_view_type(dest_image),
 841             .format = dest_image->vk_format,
 842             .subresourceRange = {
 843                .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 844                .baseMipLevel = pRegions[r].dstSubresource.mipLevel,
 845                .levelCount = 1,
 846                .baseArrayLayer = dest_array_slice,
 847                .layerCount = 1
 848             },
 849          },
 850          cmd_buffer, 0);
 851
 852       meta_emit_blit(cmd_buffer,
 853                      src_image, &src_iview,
 854                      pRegions[r].srcOffsets[0], src_extent,
 855                      dest_image, &dest_iview,
 856                      dest_offset, dest_extent,
 857                      filter);
 858    }
 859
 860    meta_finish_blit(cmd_buffer, &saved_state);
 861 }
 862
 863 static struct anv_image *
 864 make_image_for_buffer(VkDevice vk_device, VkBuffer vk_buffer, VkFormat format,
 865                       VkImageUsageFlags usage,
 866                       VkImageType image_type,
 867                       const VkAllocationCallbacks *alloc,
 868                       const VkBufferImageCopy *copy)
 869 {
 870    ANV_FROM_HANDLE(anv_buffer, buffer, vk_buffer);
 871
 872    VkExtent3D extent = copy->imageExtent;
 873    if (copy->bufferRowLength)
 874       extent.width = copy->bufferRowLength;
 875    if (copy->bufferImageHeight)
 876       extent.height = copy->bufferImageHeight;
 877    extent.depth = 1;
 878    extent = meta_region_extent_el(format, &extent);
 879
 880    VkImageAspectFlags aspect = copy->imageSubresource.aspectMask;
 881    VkFormat buffer_format = choose_buffer_format(format, aspect);
 882
 883    VkImage vk_image;
 884    VkResult result = anv_CreateImage(vk_device,
 885       &(VkImageCreateInfo) {
 886          .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
 887          .imageType = VK_IMAGE_TYPE_2D,
 888          .format = buffer_format,
 889          .extent = extent,
 890          .mipLevels = 1,
 891          .arrayLayers = 1,
 892          .samples = 1,
 893          .tiling = VK_IMAGE_TILING_LINEAR,
 894          .usage = usage,
 895          .flags = 0,
 896       }, alloc, &vk_image);
 897    assert(result == VK_SUCCESS);
 898
 899    ANV_FROM_HANDLE(anv_image, image, vk_image);
 900
 901    /* We could use a vk call to bind memory, but that would require
 902     * creating a dummy memory object etc. so there's really no point.
 903     */
 904    image->bo = buffer->bo;
 905    image->offset = buffer->offset + copy->bufferOffset;
 906
 907    return image;
 908 }
 909
 910 void anv_CmdCopyBufferToImage(
 911     VkCommandBuffer                             commandBuffer,
 912     VkBuffer                                    srcBuffer,
 913     VkImage                                     destImage,
 914     VkImageLayout                               destImageLayout,
 915     uint32_t                                    regionCount,
 916     const VkBufferImageCopy*                    pRegions)
 917 {
 918    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 919    ANV_FROM_HANDLE(anv_image, dest_image, destImage);
 920    VkDevice vk_device = anv_device_to_handle(cmd_buffer->device);
 921    struct anv_meta_saved_state saved_state;
 922
 923    /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
 924     * VK_SAMPLE_COUNT_1_BIT."
 925     */
 926    assert(dest_image->samples == 1);
 927
 928    meta_prepare_blit(cmd_buffer, &saved_state);
 929
 930    for (unsigned r = 0; r < regionCount; r++) {
 931       VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
 932
 933       VkFormat image_format = choose_iview_format(dest_image, aspect);
 934
 935       struct anv_image *src_image =
 936          make_image_for_buffer(vk_device, srcBuffer, dest_image->vk_format,
 937                                VK_IMAGE_USAGE_SAMPLED_BIT,
 938                                dest_image->type, &cmd_buffer->pool->alloc,
 939                                &pRegions[r]);
 940
 941       const uint32_t dest_base_array_slice =
 942          anv_meta_get_iview_layer(dest_image, &pRegions[r].imageSubresource,
 943                                   &pRegions[r].imageOffset);
 944
 945       unsigned num_slices_3d = pRegions[r].imageExtent.depth;
 946       unsigned num_slices_array = pRegions[r].imageSubresource.layerCount;
 947       unsigned slice_3d = 0;
 948       unsigned slice_array = 0;
 949       while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
 950          struct anv_image_view src_iview;
 951          anv_image_view_init(&src_iview, cmd_buffer->device,
 952             &(VkImageViewCreateInfo) {
 953                .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 954                .image = anv_image_to_handle(src_image),
 955                .viewType = VK_IMAGE_VIEW_TYPE_2D,
 956                .format = src_image->vk_format,
 957                .subresourceRange = {
 958                   .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 959                   .baseMipLevel = 0,
 960                   .levelCount = 1,
 961                   .baseArrayLayer = 0,
 962                   .layerCount = 1,
 963                },
 964             },
 965             cmd_buffer, 0);
 966
 967          uint32_t img_x = 0;
 968          uint32_t img_y = 0;
 969          uint32_t img_o = 0;
 970          if (isl_format_is_compressed(dest_image->format->isl_format))
 971             isl_surf_get_image_intratile_offset_el(&cmd_buffer->device->isl_dev,
 972                                                    &dest_image->color_surface.isl,
 973                                                    pRegions[r].imageSubresource.mipLevel,
 974                                                    pRegions[r].imageSubresource.baseArrayLayer + slice_array,
 975                                                    pRegions[r].imageOffset.z + slice_3d,
 976                                                    &img_o, &img_x, &img_y);
 977
 978          VkOffset3D dest_offset_el = meta_region_offset_el(dest_image, & pRegions[r].imageOffset);
 979          dest_offset_el.x += img_x;
 980          dest_offset_el.y += img_y;
 981          dest_offset_el.z = 0;
 982
 983          struct anv_image_view dest_iview;
 984          anv_image_view_init(&dest_iview, cmd_buffer->device,
 985             &(VkImageViewCreateInfo) {
 986                .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
 987                .image = anv_image_to_handle(dest_image),
 988                .viewType = anv_meta_get_view_type(dest_image),
 989                .format = image_format,
 990                .subresourceRange = {
 991                   .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 992                   .baseMipLevel = pRegions[r].imageSubresource.mipLevel,
 993                   .levelCount = 1,
 994                   .baseArrayLayer = dest_base_array_slice +
 995                                     slice_array + slice_3d,
 996                   .layerCount = 1
 997                },
 998             },
 999             cmd_buffer, img_o);
1000
1001          const VkExtent3D img_extent_el = meta_region_extent_el(dest_image->vk_format,
1002                                                       &pRegions[r].imageExtent);
1003
1004          meta_emit_blit(cmd_buffer,
1005                         src_image,
1006                         &src_iview,
1007                         (VkOffset3D){0, 0, 0},
1008                         img_extent_el,
1009                         dest_image,
1010                         &dest_iview,
1011                         dest_offset_el,
1012                         img_extent_el,
1013                         VK_FILTER_NEAREST);
1014
1015          /* Once we've done the blit, all of the actual information about
1016           * the image is embedded in the command buffer so we can just
1017           * increment the offset directly in the image effectively
1018           * re-binding it to different backing memory.
1019           */
1020          src_image->offset += src_image->extent.width *
1021                               src_image->extent.height *
1022                               src_image->format->isl_layout->bs;
1023
1024          if (dest_image->type == VK_IMAGE_TYPE_3D)
1025             slice_3d++;
1026          else
1027             slice_array++;
1028       }
1029
1030       anv_DestroyImage(vk_device, anv_image_to_handle(src_image),
1031                        &cmd_buffer->pool->alloc);
1032    }
1033
1034    meta_finish_blit(cmd_buffer, &saved_state);
1035 }
1036
1037 void anv_CmdCopyImageToBuffer(
1038     VkCommandBuffer                             commandBuffer,
1039     VkImage                                     srcImage,
1040     VkImageLayout                               srcImageLayout,
1041     VkBuffer                                    destBuffer,
1042     uint32_t                                    regionCount,
1043     const VkBufferImageCopy*                    pRegions)
1044 {
1045    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1046    ANV_FROM_HANDLE(anv_image, src_image, srcImage);
1047    VkDevice vk_device = anv_device_to_handle(cmd_buffer->device);
1048    struct anv_meta_saved_state saved_state;
1049
1050
1051    /* The Vulkan 1.0 spec says "srcImage must have a sample count equal to
1052     * VK_SAMPLE_COUNT_1_BIT."
1053     */
1054    assert(src_image->samples == 1);
1055
1056    meta_prepare_blit(cmd_buffer, &saved_state);
1057
1058    for (unsigned r = 0; r < regionCount; r++) {
1059       VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask;
1060
1061       VkFormat image_format = choose_iview_format(src_image, aspect);
1062
1063       struct anv_image_view src_iview;
1064       anv_image_view_init(&src_iview, cmd_buffer->device,
1065          &(VkImageViewCreateInfo) {
1066             .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
1067             .image = srcImage,
1068             .viewType = anv_meta_get_view_type(src_image),
1069             .format = image_format,
1070             .subresourceRange = {
1071                .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
1072                .baseMipLevel = pRegions[r].imageSubresource.mipLevel,
1073                .levelCount = 1,
1074                .baseArrayLayer = pRegions[r].imageSubresource.baseArrayLayer,
1075                .layerCount = pRegions[r].imageSubresource.layerCount,
1076             },
1077          },
1078          cmd_buffer, 0);
1079
1080       struct anv_image *dest_image =
1081          make_image_for_buffer(vk_device, destBuffer, src_image->vk_format,
1082                                VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT,
1083                                src_image->type, &cmd_buffer->pool->alloc,
1084                                &pRegions[r]);
1085
1086       unsigned num_slices;
1087       if (src_image->type == VK_IMAGE_TYPE_3D) {
1088          assert(pRegions[r].imageSubresource.layerCount == 1);
1089          num_slices = pRegions[r].imageExtent.depth;
1090       } else {
1091          assert(pRegions[r].imageExtent.depth == 1);
1092          num_slices = pRegions[r].imageSubresource.layerCount;
1093       }
1094
1095       for (unsigned slice = 0; slice < num_slices; slice++) {
1096          VkOffset3D src_offset = pRegions[r].imageOffset;
1097          src_offset.z += slice;
1098
1099          struct anv_image_view dest_iview;
1100          anv_image_view_init(&dest_iview, cmd_buffer->device,
1101             &(VkImageViewCreateInfo) {
1102                .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
1103                .image = anv_image_to_handle(dest_image),
1104                .viewType = VK_IMAGE_VIEW_TYPE_2D,
1105                .format = dest_image->vk_format,
1106                .subresourceRange = {
1107                   .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
1108                   .baseMipLevel = 0,
1109                   .levelCount = 1,
1110                   .baseArrayLayer = 0,
1111                   .layerCount = 1
1112                },
1113             },
1114             cmd_buffer, 0);
1115
1116          meta_emit_blit(cmd_buffer,
1117                         anv_image_from_handle(srcImage),
1118                         &src_iview,
1119                         src_offset,
1120                         pRegions[r].imageExtent,
1121                         dest_image,
1122                         &dest_iview,
1123                         (VkOffset3D) { 0, 0, 0 },
1124                         pRegions[r].imageExtent,
1125                         VK_FILTER_NEAREST);
1126
1127          /* Once we've done the blit, all of the actual information about
1128           * the image is embedded in the command buffer so we can just
1129           * increment the offset directly in the image effectively
1130           * re-binding it to different backing memory.
1131           */
1132          dest_image->offset += dest_image->extent.width *
1133                                dest_image->extent.height *
1134                                src_image->format->isl_layout->bs;
1135       }
1136
1137       anv_DestroyImage(vk_device, anv_image_to_handle(dest_image),
1138                        &cmd_buffer->pool->alloc);
1139    }
1140
1141    meta_finish_blit(cmd_buffer, &saved_state);
1142 }
1143
1144 void
1145 anv_device_finish_meta_blit_state(struct anv_device *device)
1146 {
1147    anv_DestroyRenderPass(anv_device_to_handle(device),
1148                          device->meta_state.blit.render_pass,
1149                          &device->meta_state.alloc);
1150    anv_DestroyPipeline(anv_device_to_handle(device),
1151                        device->meta_state.blit.pipeline_1d_src,
1152                        &device->meta_state.alloc);
1153    anv_DestroyPipeline(anv_device_to_handle(device),
1154                        device->meta_state.blit.pipeline_2d_src,
1155                        &device->meta_state.alloc);
1156    anv_DestroyPipeline(anv_device_to_handle(device),
1157                        device->meta_state.blit.pipeline_3d_src,
1158                        &device->meta_state.alloc);
1159    anv_DestroyPipelineLayout(anv_device_to_handle(device),
1160                              device->meta_state.blit.pipeline_layout,
1161                              &device->meta_state.alloc);
1162    anv_DestroyDescriptorSetLayout(anv_device_to_handle(device),
1163                                   device->meta_state.blit.ds_layout,
1164                                   &device->meta_state.alloc);
1165 }
1166
1167 VkResult
1168 anv_device_init_meta_blit_state(struct anv_device *device)
1169 {
1170    VkResult result;
1171
1172    result = anv_CreateRenderPass(anv_device_to_handle(device),
1173       &(VkRenderPassCreateInfo) {
1174          .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
1175          .attachmentCount = 1,
1176          .pAttachments = &(VkAttachmentDescription) {
1177             .format = VK_FORMAT_UNDEFINED, /* Our shaders don't care */
1178             .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
1179             .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
1180             .initialLayout = VK_IMAGE_LAYOUT_GENERAL,
1181             .finalLayout = VK_IMAGE_LAYOUT_GENERAL,
1182          },
1183          .subpassCount = 1,
1184          .pSubpasses = &(VkSubpassDescription) {
1185             .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
1186             .inputAttachmentCount = 0,
1187             .colorAttachmentCount = 1,
1188             .pColorAttachments = &(VkAttachmentReference) {
1189                .attachment = 0,
1190                .layout = VK_IMAGE_LAYOUT_GENERAL,
1191             },
1192             .pResolveAttachments = NULL,
1193             .pDepthStencilAttachment = &(VkAttachmentReference) {
1194                .attachment = VK_ATTACHMENT_UNUSED,
1195                .layout = VK_IMAGE_LAYOUT_GENERAL,
1196             },
1197             .preserveAttachmentCount = 1,
1198             .pPreserveAttachments = (uint32_t[]) { 0 },
1199          },
1200          .dependencyCount = 0,
1201       }, &device->meta_state.alloc, &device->meta_state.blit.render_pass);
1202    if (result != VK_SUCCESS)
1203       goto fail;
1204
1205    /* We don't use a vertex shader for blitting, but instead build and pass
1206     * the VUEs directly to the rasterization backend.  However, we do need
1207     * to provide GLSL source for the vertex shader so that the compiler
1208     * does not dead-code our inputs.
1209     */
1210    struct anv_shader_module vs = {
1211       .nir = build_nir_vertex_shader(),
1212    };
1213
1214    struct anv_shader_module fs_1d = {
1215       .nir = build_nir_copy_fragment_shader(GLSL_SAMPLER_DIM_1D),
1216    };
1217
1218    struct anv_shader_module fs_2d = {
1219       .nir = build_nir_copy_fragment_shader(GLSL_SAMPLER_DIM_2D),
1220    };
1221
1222    struct anv_shader_module fs_3d = {
1223       .nir = build_nir_copy_fragment_shader(GLSL_SAMPLER_DIM_3D),
1224    };
1225
1226    VkPipelineVertexInputStateCreateInfo vi_create_info = {
1227       .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
1228       .vertexBindingDescriptionCount = 2,
1229       .pVertexBindingDescriptions = (VkVertexInputBindingDescription[]) {
1230          {
1231             .binding = 0,
1232             .stride = 0,
1233             .inputRate = VK_VERTEX_INPUT_RATE_VERTEX
1234          },
1235          {
1236             .binding = 1,
1237             .stride = 5 * sizeof(float),
1238             .inputRate = VK_VERTEX_INPUT_RATE_VERTEX
1239          },
1240       },
1241       .vertexAttributeDescriptionCount = 3,
1242       .pVertexAttributeDescriptions = (VkVertexInputAttributeDescription[]) {
1243          {
1244             /* VUE Header */
1245             .location = 0,
1246             .binding = 0,
1247             .format = VK_FORMAT_R32G32B32A32_UINT,
1248             .offset = 0
1249          },
1250          {
1251             /* Position */
1252             .location = 1,
1253             .binding = 1,
1254             .format = VK_FORMAT_R32G32_SFLOAT,
1255             .offset = 0
1256          },
1257          {
1258             /* Texture Coordinate */
1259             .location = 2,
1260             .binding = 1,
1261             .format = VK_FORMAT_R32G32B32_SFLOAT,
1262             .offset = 8
1263          }
1264       }
1265    };
1266
1267    VkDescriptorSetLayoutCreateInfo ds_layout_info = {
1268       .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
1269       .bindingCount = 1,
1270       .pBindings = (VkDescriptorSetLayoutBinding[]) {
1271          {
1272             .binding = 0,
1273             .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
1274             .descriptorCount = 1,
1275             .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
1276             .pImmutableSamplers = NULL
1277          },
1278       }
1279    };
1280    result = anv_CreateDescriptorSetLayout(anv_device_to_handle(device),
1281                                           &ds_layout_info,
1282                                           &device->meta_state.alloc,
1283                                           &device->meta_state.blit.ds_layout);
1284    if (result != VK_SUCCESS)
1285       goto fail_render_pass;
1286
1287    result = anv_CreatePipelineLayout(anv_device_to_handle(device),
1288       &(VkPipelineLayoutCreateInfo) {
1289          .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
1290          .setLayoutCount = 1,
1291          .pSetLayouts = &device->meta_state.blit.ds_layout,
1292       },
1293       &device->meta_state.alloc, &device->meta_state.blit.pipeline_layout);
1294    if (result != VK_SUCCESS)
1295       goto fail_descriptor_set_layout;
1296
1297    VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
1298       {
1299          .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1300          .stage = VK_SHADER_STAGE_VERTEX_BIT,
1301          .module = anv_shader_module_to_handle(&vs),
1302          .pName = "main",
1303          .pSpecializationInfo = NULL
1304       }, {
1305          .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
1306          .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
1307          .module = VK_NULL_HANDLE, /* TEMPLATE VALUE! FILL ME IN! */
1308          .pName = "main",
1309          .pSpecializationInfo = NULL
1310       },
1311    };
1312
1313    const VkGraphicsPipelineCreateInfo vk_pipeline_info = {
1314       .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
1315       .stageCount = ARRAY_SIZE(pipeline_shader_stages),
1316       .pStages = pipeline_shader_stages,
1317       .pVertexInputState = &vi_create_info,
1318       .pInputAssemblyState = &(VkPipelineInputAssemblyStateCreateInfo) {
1319          .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
1320          .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
1321          .primitiveRestartEnable = false,
1322       },
1323       .pViewportState = &(VkPipelineViewportStateCreateInfo) {
1324          .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
1325          .viewportCount = 1,
1326          .scissorCount = 1,
1327       },
1328       .pRasterizationState = &(VkPipelineRasterizationStateCreateInfo) {
1329          .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
1330          .rasterizerDiscardEnable = false,
1331          .polygonMode = VK_POLYGON_MODE_FILL,
1332          .cullMode = VK_CULL_MODE_NONE,
1333          .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE
1334       },
1335       .pMultisampleState = &(VkPipelineMultisampleStateCreateInfo) {
1336          .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
1337          .rasterizationSamples = 1,
1338          .sampleShadingEnable = false,
1339          .pSampleMask = (VkSampleMask[]) { UINT32_MAX },
1340       },
1341       .pColorBlendState = &(VkPipelineColorBlendStateCreateInfo) {
1342          .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
1343          .attachmentCount = 1,
1344          .pAttachments = (VkPipelineColorBlendAttachmentState []) {
1345             { .colorWriteMask =
1346                  VK_COLOR_COMPONENT_A_BIT |
1347                  VK_COLOR_COMPONENT_R_BIT |
1348                  VK_COLOR_COMPONENT_G_BIT |
1349                  VK_COLOR_COMPONENT_B_BIT },
1350          }
1351       },
1352       .pDynamicState = &(VkPipelineDynamicStateCreateInfo) {
1353          .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
1354          .dynamicStateCount = 9,
1355          .pDynamicStates = (VkDynamicState[]) {
1356             VK_DYNAMIC_STATE_VIEWPORT,
1357             VK_DYNAMIC_STATE_SCISSOR,
1358             VK_DYNAMIC_STATE_LINE_WIDTH,
1359             VK_DYNAMIC_STATE_DEPTH_BIAS,
1360             VK_DYNAMIC_STATE_BLEND_CONSTANTS,
1361             VK_DYNAMIC_STATE_DEPTH_BOUNDS,
1362             VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK,
1363             VK_DYNAMIC_STATE_STENCIL_WRITE_MASK,
1364             VK_DYNAMIC_STATE_STENCIL_REFERENCE,
1365          },
1366       },
1367       .flags = 0,
1368       .layout = device->meta_state.blit.pipeline_layout,
1369       .renderPass = device->meta_state.blit.render_pass,
1370       .subpass = 0,
1371    };
1372
1373    const struct anv_graphics_pipeline_create_info anv_pipeline_info = {
1374       .color_attachment_count = -1,
1375       .use_repclear = false,
1376       .disable_viewport = true,
1377       .disable_scissor = true,
1378       .disable_vs = true,
1379       .use_rectlist = true
1380    };
1381
1382    pipeline_shader_stages[1].module = anv_shader_module_to_handle(&fs_1d);
1383    result = anv_graphics_pipeline_create(anv_device_to_handle(device),
1384       VK_NULL_HANDLE,
1385       &vk_pipeline_info, &anv_pipeline_info,
1386       &device->meta_state.alloc, &device->meta_state.blit.pipeline_1d_src);
1387    if (result != VK_SUCCESS)
1388       goto fail_pipeline_layout;
1389
1390    pipeline_shader_stages[1].module = anv_shader_module_to_handle(&fs_2d);
1391    result = anv_graphics_pipeline_create(anv_device_to_handle(device),
1392       VK_NULL_HANDLE,
1393       &vk_pipeline_info, &anv_pipeline_info,
1394       &device->meta_state.alloc, &device->meta_state.blit.pipeline_2d_src);
1395    if (result != VK_SUCCESS)
1396       goto fail_pipeline_1d;
1397
1398    pipeline_shader_stages[1].module = anv_shader_module_to_handle(&fs_3d);
1399    result = anv_graphics_pipeline_create(anv_device_to_handle(device),
1400       VK_NULL_HANDLE,
1401       &vk_pipeline_info, &anv_pipeline_info,
1402       &device->meta_state.alloc, &device->meta_state.blit.pipeline_3d_src);
1403    if (result != VK_SUCCESS)
1404       goto fail_pipeline_2d;
1405
1406    ralloc_free(vs.nir);
1407    ralloc_free(fs_1d.nir);
1408    ralloc_free(fs_2d.nir);
1409    ralloc_free(fs_3d.nir);
1410
1411    return VK_SUCCESS;
1412
1413  fail_pipeline_2d:
1414    anv_DestroyPipeline(anv_device_to_handle(device),
1415                        device->meta_state.blit.pipeline_2d_src,
1416                        &device->meta_state.alloc);
1417
1418  fail_pipeline_1d:
1419    anv_DestroyPipeline(anv_device_to_handle(device),
1420                        device->meta_state.blit.pipeline_1d_src,
1421                        &device->meta_state.alloc);
1422
1423  fail_pipeline_layout:
1424    anv_DestroyPipelineLayout(anv_device_to_handle(device),
1425                              device->meta_state.blit.pipeline_layout,
1426                              &device->meta_state.alloc);
1427  fail_descriptor_set_layout:
1428    anv_DestroyDescriptorSetLayout(anv_device_to_handle(device),
1429                                   device->meta_state.blit.ds_layout,
1430                                   &device->meta_state.alloc);
1431  fail_render_pass:
1432    anv_DestroyRenderPass(anv_device_to_handle(device),
1433                          device->meta_state.blit.render_pass,
1434                          &device->meta_state.alloc);
1435
1436    ralloc_free(vs.nir);
1437    ralloc_free(fs_1d.nir);
1438    ralloc_free(fs_2d.nir);
1439    ralloc_free(fs_3d.nir);
1440  fail:
1441    return result;
1442 }