src/vulkan/gen7_cmd_buffer.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <assert.h>
  25 #include <stdbool.h>
  26 #include <string.h>
  27 #include <unistd.h>
  28 #include <fcntl.h>
  29
  30 #include "anv_private.h"
  31
  32 #include "gen7_pack.h"
  33 #include "gen75_pack.h"
  34
  35 static void
  36 cmd_buffer_flush_push_constants(struct anv_cmd_buffer *cmd_buffer)
  37 {
  38    static const uint32_t push_constant_opcodes[] = {
  39       [MESA_SHADER_VERTEX]                      = 21,
  40       [MESA_SHADER_TESS_CTRL]                   = 25, /* HS */
  41       [MESA_SHADER_TESS_EVAL]                   = 26, /* DS */
  42       [MESA_SHADER_GEOMETRY]                    = 22,
  43       [MESA_SHADER_FRAGMENT]                    = 23,
  44       [MESA_SHADER_COMPUTE]                     = 0,
  45    };
  46
  47    VkShaderStageFlags flushed = 0;
  48
  49    anv_foreach_stage(stage, cmd_buffer->state.push_constants_dirty) {
  50       struct anv_state state = anv_cmd_buffer_push_constants(cmd_buffer, stage);
  51
  52       if (state.offset == 0)
  53          continue;
  54
  55       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_CONSTANT_VS,
  56                      ._3DCommandSubOpcode = push_constant_opcodes[stage],
  57                      .ConstantBody = {
  58                         .PointerToConstantBuffer0 = { .offset = state.offset },
  59                         .ConstantBuffer0ReadLength = DIV_ROUND_UP(state.alloc_size, 32),
  60                      });
  61
  62       flushed |= mesa_to_vk_shader_stage(stage);
  63    }
  64
  65    cmd_buffer->state.push_constants_dirty &= ~flushed;
  66 }
  67
  68 static VkResult
  69 flush_descriptor_set(struct anv_cmd_buffer *cmd_buffer, gl_shader_stage stage)
  70 {
  71    struct anv_state surfaces = { 0, }, samplers = { 0, };
  72    VkResult result;
  73
  74    result = anv_cmd_buffer_emit_samplers(cmd_buffer, stage, &samplers);
  75    if (result != VK_SUCCESS)
  76       return result;
  77    result = anv_cmd_buffer_emit_binding_table(cmd_buffer, stage, &surfaces);
  78    if (result != VK_SUCCESS)
  79       return result;
  80
  81    static const uint32_t sampler_state_opcodes[] = {
  82       [MESA_SHADER_VERTEX]                      = 43,
  83       [MESA_SHADER_TESS_CTRL]                   = 44, /* HS */
  84       [MESA_SHADER_TESS_EVAL]                   = 45, /* DS */
  85       [MESA_SHADER_GEOMETRY]                    = 46,
  86       [MESA_SHADER_FRAGMENT]                    = 47,
  87       [MESA_SHADER_COMPUTE]                     = 0,
  88    };
  89
  90    static const uint32_t binding_table_opcodes[] = {
  91       [MESA_SHADER_VERTEX]                      = 38,
  92       [MESA_SHADER_TESS_CTRL]                   = 39,
  93       [MESA_SHADER_TESS_EVAL]                   = 40,
  94       [MESA_SHADER_GEOMETRY]                    = 41,
  95       [MESA_SHADER_FRAGMENT]                    = 42,
  96       [MESA_SHADER_COMPUTE]                     = 0,
  97    };
  98
  99    if (samplers.alloc_size > 0) {
 100       anv_batch_emit(&cmd_buffer->batch,
 101                      GEN7_3DSTATE_SAMPLER_STATE_POINTERS_VS,
 102                      ._3DCommandSubOpcode  = sampler_state_opcodes[stage],
 103                      .PointertoVSSamplerState = samplers.offset);
 104    }
 105
 106    if (surfaces.alloc_size > 0) {
 107       anv_batch_emit(&cmd_buffer->batch,
 108                      GEN7_3DSTATE_BINDING_TABLE_POINTERS_VS,
 109                      ._3DCommandSubOpcode  = binding_table_opcodes[stage],
 110                      .PointertoVSBindingTable = surfaces.offset);
 111    }
 112
 113    return VK_SUCCESS;
 114 }
 115
 116 GENX_FUNC(GEN7, GEN7) void
 117 genX(cmd_buffer_flush_descriptor_sets)(struct anv_cmd_buffer *cmd_buffer)
 118 {
 119    VkShaderStageFlags dirty = cmd_buffer->state.descriptors_dirty &
 120                               cmd_buffer->state.pipeline->active_stages;
 121
 122    VkResult result = VK_SUCCESS;
 123    anv_foreach_stage(s, dirty) {
 124       result = flush_descriptor_set(cmd_buffer, s);
 125       if (result != VK_SUCCESS)
 126          break;
 127    }
 128
 129    if (result != VK_SUCCESS) {
 130       assert(result == VK_ERROR_OUT_OF_DEVICE_MEMORY);
 131
 132       result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
 133       assert(result == VK_SUCCESS);
 134
 135       /* Re-emit state base addresses so we get the new surface state base
 136        * address before we start emitting binding tables etc.
 137        */
 138       anv_cmd_buffer_emit_state_base_address(cmd_buffer);
 139
 140       /* Re-emit all active binding tables */
 141       anv_foreach_stage(s, cmd_buffer->state.pipeline->active_stages) {
 142          result = flush_descriptor_set(cmd_buffer, s);
 143
 144          /* It had better succeed this time */
 145          assert(result == VK_SUCCESS);
 146       }
 147    }
 148
 149    cmd_buffer->state.descriptors_dirty &= ~cmd_buffer->state.pipeline->active_stages;
 150 }
 151
 152 static inline int64_t
 153 clamp_int64(int64_t x, int64_t min, int64_t max)
 154 {
 155    if (x < min)
 156       return min;
 157    else if (x < max)
 158       return x;
 159    else
 160       return max;
 161 }
 162
 163 static void
 164 emit_scissor_state(struct anv_cmd_buffer *cmd_buffer,
 165                    uint32_t count, const VkRect2D *scissors)
 166 {
 167    struct anv_state scissor_state =
 168       anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, count * 32, 32);
 169
 170    for (uint32_t i = 0; i < count; i++) {
 171       const VkRect2D *s = &scissors[i];
 172
 173       /* Since xmax and ymax are inclusive, we have to have xmax < xmin or
 174        * ymax < ymin for empty clips.  In case clip x, y, width height are all
 175        * 0, the clamps below produce 0 for xmin, ymin, xmax, ymax, which isn't
 176        * what we want. Just special case empty clips and produce a canonical
 177        * empty clip. */
 178       static const struct GEN7_SCISSOR_RECT empty_scissor = {
 179          .ScissorRectangleYMin = 1,
 180          .ScissorRectangleXMin = 1,
 181          .ScissorRectangleYMax = 0,
 182          .ScissorRectangleXMax = 0
 183       };
 184
 185       const int max = 0xffff;
 186       struct GEN7_SCISSOR_RECT scissor = {
 187          /* Do this math using int64_t so overflow gets clamped correctly. */
 188          .ScissorRectangleYMin = clamp_int64(s->offset.y, 0, max),
 189          .ScissorRectangleXMin = clamp_int64(s->offset.x, 0, max),
 190          .ScissorRectangleYMax = clamp_int64((uint64_t) s->offset.y + s->extent.height - 1, 0, max),
 191          .ScissorRectangleXMax = clamp_int64((uint64_t) s->offset.x + s->extent.width - 1, 0, max)
 192       };
 193
 194       if (s->extent.width <= 0 || s->extent.height <= 0) {
 195          GEN7_SCISSOR_RECT_pack(NULL, scissor_state.map + i * 32,
 196                                 &empty_scissor);
 197       } else {
 198          GEN7_SCISSOR_RECT_pack(NULL, scissor_state.map + i * 32, &scissor);
 199       }
 200    }
 201
 202    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_SCISSOR_STATE_POINTERS,
 203                   .ScissorRectPointer = scissor_state.offset);
 204
 205    if (!cmd_buffer->device->info.has_llc)
 206       anv_state_clflush(scissor_state);
 207 }
 208
 209 GENX_FUNC(GEN7, GEN7) void
 210 genX(cmd_buffer_emit_scissor)(struct anv_cmd_buffer *cmd_buffer)
 211 {
 212    if (cmd_buffer->state.dynamic.scissor.count > 0) {
 213       emit_scissor_state(cmd_buffer, cmd_buffer->state.dynamic.scissor.count,
 214                          cmd_buffer->state.dynamic.scissor.scissors);
 215    } else {
 216       /* Emit a default scissor based on the currently bound framebuffer */
 217       emit_scissor_state(cmd_buffer, 1,
 218                          &(VkRect2D) {
 219                             .offset = { .x = 0, .y = 0, },
 220                             .extent = {
 221                                .width = cmd_buffer->state.framebuffer->width,
 222                                .height = cmd_buffer->state.framebuffer->height,
 223                             },
 224                          });
 225    }
 226 }
 227
 228 static const uint32_t vk_to_gen_index_type[] = {
 229    [VK_INDEX_TYPE_UINT16]                       = INDEX_WORD,
 230    [VK_INDEX_TYPE_UINT32]                       = INDEX_DWORD,
 231 };
 232
 233 static const uint32_t restart_index_for_type[] = {
 234    [VK_INDEX_TYPE_UINT16]                    = UINT16_MAX,
 235    [VK_INDEX_TYPE_UINT32]                    = UINT32_MAX,
 236 };
 237
 238 void genX(CmdBindIndexBuffer)(
 239     VkCommandBuffer                             commandBuffer,
 240     VkBuffer                                    _buffer,
 241     VkDeviceSize                                offset,
 242     VkIndexType                                 indexType)
 243 {
 244    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 245    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 246
 247    cmd_buffer->state.dirty |= ANV_CMD_DIRTY_INDEX_BUFFER;
 248    if (ANV_IS_HASWELL)
 249       cmd_buffer->state.restart_index = restart_index_for_type[indexType];
 250    cmd_buffer->state.gen7.index_buffer = buffer;
 251    cmd_buffer->state.gen7.index_type = vk_to_gen_index_type[indexType];
 252    cmd_buffer->state.gen7.index_offset = offset;
 253 }
 254
 255 static VkResult
 256 flush_compute_descriptor_set(struct anv_cmd_buffer *cmd_buffer)
 257 {
 258    struct anv_device *device = cmd_buffer->device;
 259    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 260    struct anv_state surfaces = { 0, }, samplers = { 0, };
 261    VkResult result;
 262
 263    result = anv_cmd_buffer_emit_samplers(cmd_buffer,
 264                                          MESA_SHADER_COMPUTE, &samplers);
 265    if (result != VK_SUCCESS)
 266       return result;
 267    result = anv_cmd_buffer_emit_binding_table(cmd_buffer,
 268                                               MESA_SHADER_COMPUTE, &surfaces);
 269    if (result != VK_SUCCESS)
 270       return result;
 271
 272    const struct brw_cs_prog_data *cs_prog_data = &pipeline->cs_prog_data;
 273
 274    struct anv_state state =
 275       anv_state_pool_emit(&device->dynamic_state_pool,
 276                           GEN7_INTERFACE_DESCRIPTOR_DATA, 64,
 277                           .KernelStartPointer = pipeline->cs_simd,
 278                           .BindingTablePointer = surfaces.offset,
 279                           .SamplerStatePointer = samplers.offset,
 280                           .BarrierEnable = cs_prog_data->uses_barrier,
 281                           .NumberofThreadsinGPGPUThreadGroup =
 282                              pipeline->cs_thread_width_max);
 283
 284    const uint32_t size = GEN7_INTERFACE_DESCRIPTOR_DATA_length * sizeof(uint32_t);
 285    anv_batch_emit(&cmd_buffer->batch, GEN7_MEDIA_INTERFACE_DESCRIPTOR_LOAD,
 286                   .InterfaceDescriptorTotalLength = size,
 287                   .InterfaceDescriptorDataStartAddress = state.offset);
 288
 289    return VK_SUCCESS;
 290 }
 291
 292 static void
 293 cmd_buffer_flush_compute_state(struct anv_cmd_buffer *cmd_buffer)
 294 {
 295    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 296    VkResult result;
 297
 298    assert(pipeline->active_stages == VK_SHADER_STAGE_COMPUTE_BIT);
 299
 300    if (cmd_buffer->state.current_pipeline != GPGPU) {
 301       anv_batch_emit(&cmd_buffer->batch, GEN7_PIPELINE_SELECT,
 302                      .PipelineSelection = GPGPU);
 303       cmd_buffer->state.current_pipeline = GPGPU;
 304    }
 305
 306    if (cmd_buffer->state.compute_dirty & ANV_CMD_DIRTY_PIPELINE)
 307       anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
 308
 309    if ((cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_COMPUTE_BIT) ||
 310        (cmd_buffer->state.compute_dirty & ANV_CMD_DIRTY_PIPELINE)) {
 311       /* FIXME: figure out descriptors for gen7 */
 312       result = flush_compute_descriptor_set(cmd_buffer);
 313       assert(result == VK_SUCCESS);
 314       cmd_buffer->state.descriptors_dirty &= ~VK_SHADER_STAGE_COMPUTE_BIT;
 315    }
 316
 317    cmd_buffer->state.compute_dirty = 0;
 318 }
 319
 320 static void
 321 cmd_buffer_flush_state(struct anv_cmd_buffer *cmd_buffer)
 322 {
 323    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 324    uint32_t *p;
 325
 326    uint32_t vb_emit = cmd_buffer->state.vb_dirty & pipeline->vb_used;
 327
 328    assert((pipeline->active_stages & VK_SHADER_STAGE_COMPUTE_BIT) == 0);
 329
 330    if (cmd_buffer->state.current_pipeline != _3D) {
 331       anv_batch_emit(&cmd_buffer->batch, GEN7_PIPELINE_SELECT,
 332                      .PipelineSelection = _3D);
 333       cmd_buffer->state.current_pipeline = _3D;
 334    }
 335
 336    if (vb_emit) {
 337       const uint32_t num_buffers = __builtin_popcount(vb_emit);
 338       const uint32_t num_dwords = 1 + num_buffers * 4;
 339
 340       p = anv_batch_emitn(&cmd_buffer->batch, num_dwords,
 341                           GEN7_3DSTATE_VERTEX_BUFFERS);
 342       uint32_t vb, i = 0;
 343       for_each_bit(vb, vb_emit) {
 344          struct anv_buffer *buffer = cmd_buffer->state.vertex_bindings[vb].buffer;
 345          uint32_t offset = cmd_buffer->state.vertex_bindings[vb].offset;
 346
 347          struct GEN7_VERTEX_BUFFER_STATE state = {
 348             .VertexBufferIndex = vb,
 349             .BufferAccessType = pipeline->instancing_enable[vb] ? INSTANCEDATA : VERTEXDATA,
 350             .VertexBufferMemoryObjectControlState = GEN7_MOCS,
 351             .AddressModifyEnable = true,
 352             .BufferPitch = pipeline->binding_stride[vb],
 353             .BufferStartingAddress = { buffer->bo, buffer->offset + offset },
 354             .EndAddress = { buffer->bo, buffer->offset + buffer->size - 1},
 355             .InstanceDataStepRate = 1
 356          };
 357
 358          GEN7_VERTEX_BUFFER_STATE_pack(&cmd_buffer->batch, &p[1 + i * 4], &state);
 359          i++;
 360       }
 361    }
 362
 363    if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_PIPELINE) {
 364       /* If somebody compiled a pipeline after starting a command buffer the
 365        * scratch bo may have grown since we started this cmd buffer (and
 366        * emitted STATE_BASE_ADDRESS).  If we're binding that pipeline now,
 367        * reemit STATE_BASE_ADDRESS so that we use the bigger scratch bo. */
 368       if (cmd_buffer->state.scratch_size < pipeline->total_scratch)
 369          gen7_cmd_buffer_emit_state_base_address(cmd_buffer);
 370
 371       anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
 372    }
 373
 374    if (cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_VERTEX_BIT ||
 375        cmd_buffer->state.push_constants_dirty & VK_SHADER_STAGE_VERTEX_BIT) {
 376       /* From the IVB PRM Vol. 2, Part 1, Section 3.2.1:
 377        *
 378        *    "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
 379        *    stall needs to be sent just prior to any 3DSTATE_VS,
 380        *    3DSTATE_URB_VS, 3DSTATE_CONSTANT_VS,
 381        *    3DSTATE_BINDING_TABLE_POINTER_VS,
 382        *    3DSTATE_SAMPLER_STATE_POINTER_VS command.  Only one
 383        *    PIPE_CONTROL needs to be sent before any combination of VS
 384        *    associated 3DSTATE."
 385        */
 386       anv_batch_emit(&cmd_buffer->batch, GEN7_PIPE_CONTROL,
 387                      .DepthStallEnable = true,
 388                      .PostSyncOperation = WriteImmediateData,
 389                      .Address = { &cmd_buffer->device->workaround_bo, 0 });
 390    }
 391
 392    if (cmd_buffer->state.descriptors_dirty)
 393       gen7_cmd_buffer_flush_descriptor_sets(cmd_buffer);
 394
 395    if (cmd_buffer->state.push_constants_dirty)
 396       cmd_buffer_flush_push_constants(cmd_buffer);
 397
 398    /* We use the gen8 state here because it only contains the additional
 399     * min/max fields and, since they occur at the end of the packet and
 400     * don't change the stride, they work on gen7 too.
 401     */
 402    if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_VIEWPORT)
 403       gen8_cmd_buffer_emit_viewport(cmd_buffer);
 404
 405    if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_SCISSOR)
 406       gen7_cmd_buffer_emit_scissor(cmd_buffer);
 407
 408    if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
 409                                   ANV_CMD_DIRTY_DYNAMIC_LINE_WIDTH |
 410                                   ANV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS)) {
 411
 412       bool enable_bias = cmd_buffer->state.dynamic.depth_bias.bias != 0.0f ||
 413          cmd_buffer->state.dynamic.depth_bias.slope != 0.0f;
 414
 415       uint32_t sf_dw[GEN7_3DSTATE_SF_length];
 416       struct GEN7_3DSTATE_SF sf = {
 417          GEN7_3DSTATE_SF_header,
 418          .LineWidth = cmd_buffer->state.dynamic.line_width,
 419          .GlobalDepthOffsetEnableSolid = enable_bias,
 420          .GlobalDepthOffsetEnableWireframe = enable_bias,
 421          .GlobalDepthOffsetEnablePoint = enable_bias,
 422          .GlobalDepthOffsetConstant = cmd_buffer->state.dynamic.depth_bias.bias,
 423          .GlobalDepthOffsetScale = cmd_buffer->state.dynamic.depth_bias.slope,
 424          .GlobalDepthOffsetClamp = cmd_buffer->state.dynamic.depth_bias.clamp
 425       };
 426       GEN7_3DSTATE_SF_pack(NULL, sf_dw, &sf);
 427
 428       anv_batch_emit_merge(&cmd_buffer->batch, sf_dw, pipeline->gen7.sf);
 429    }
 430
 431    if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS |
 432                                   ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE)) {
 433       struct anv_state cc_state =
 434          anv_cmd_buffer_alloc_dynamic_state(cmd_buffer,
 435                                             GEN7_COLOR_CALC_STATE_length * 4,
 436                                             64);
 437       struct GEN7_COLOR_CALC_STATE cc = {
 438          .BlendConstantColorRed = cmd_buffer->state.dynamic.blend_constants[0],
 439          .BlendConstantColorGreen = cmd_buffer->state.dynamic.blend_constants[1],
 440          .BlendConstantColorBlue = cmd_buffer->state.dynamic.blend_constants[2],
 441          .BlendConstantColorAlpha = cmd_buffer->state.dynamic.blend_constants[3],
 442          .StencilReferenceValue =
 443             cmd_buffer->state.dynamic.stencil_reference.front,
 444          .BackFaceStencilReferenceValue =
 445             cmd_buffer->state.dynamic.stencil_reference.back,
 446       };
 447       GEN7_COLOR_CALC_STATE_pack(NULL, cc_state.map, &cc);
 448       if (!cmd_buffer->device->info.has_llc)
 449          anv_state_clflush(cc_state);
 450
 451       anv_batch_emit(&cmd_buffer->batch,
 452                      GEN7_3DSTATE_CC_STATE_POINTERS,
 453                      .ColorCalcStatePointer = cc_state.offset);
 454    }
 455
 456    if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
 457                                   ANV_CMD_DIRTY_RENDER_TARGETS |
 458                                   ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK |
 459                                   ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK)) {
 460       uint32_t depth_stencil_dw[GEN7_DEPTH_STENCIL_STATE_length];
 461
 462       const struct anv_image_view *iview =
 463          anv_cmd_buffer_get_depth_stencil_view(cmd_buffer);
 464
 465       struct GEN7_DEPTH_STENCIL_STATE depth_stencil = {
 466          .StencilBufferWriteEnable = iview && (iview->aspect_mask & VK_IMAGE_ASPECT_STENCIL_BIT),
 467
 468          .StencilTestMask =
 469             cmd_buffer->state.dynamic.stencil_compare_mask.front & 0xff,
 470          .StencilWriteMask =
 471             cmd_buffer->state.dynamic.stencil_write_mask.front & 0xff,
 472
 473          .BackfaceStencilTestMask =
 474             cmd_buffer->state.dynamic.stencil_compare_mask.back & 0xff,
 475          .BackfaceStencilWriteMask =
 476             cmd_buffer->state.dynamic.stencil_write_mask.back & 0xff,
 477       };
 478       GEN7_DEPTH_STENCIL_STATE_pack(NULL, depth_stencil_dw, &depth_stencil);
 479
 480       struct anv_state ds_state =
 481          anv_cmd_buffer_merge_dynamic(cmd_buffer, depth_stencil_dw,
 482                                       pipeline->gen7.depth_stencil_state,
 483                                       GEN7_DEPTH_STENCIL_STATE_length, 64);
 484
 485       anv_batch_emit(&cmd_buffer->batch,
 486                      GEN7_3DSTATE_DEPTH_STENCIL_STATE_POINTERS,
 487                      .PointertoDEPTH_STENCIL_STATE = ds_state.offset);
 488    }
 489
 490    if (cmd_buffer->state.gen7.index_buffer &&
 491        cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
 492                                   ANV_CMD_DIRTY_INDEX_BUFFER)) {
 493       struct anv_buffer *buffer = cmd_buffer->state.gen7.index_buffer;
 494       uint32_t offset = cmd_buffer->state.gen7.index_offset;
 495
 496       if (ANV_IS_HASWELL) {
 497          anv_batch_emit(&cmd_buffer->batch, GEN75_3DSTATE_VF,
 498                         .IndexedDrawCutIndexEnable = pipeline->primitive_restart,
 499                         .CutIndex = cmd_buffer->state.restart_index);
 500       }
 501
 502       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_INDEX_BUFFER,
 503                      .CutIndexEnable = pipeline->primitive_restart,
 504                      .IndexFormat = cmd_buffer->state.gen7.index_type,
 505                      .MemoryObjectControlState = GEN7_MOCS,
 506                      .BufferStartingAddress = { buffer->bo, buffer->offset + offset },
 507                      .BufferEndingAddress = { buffer->bo, buffer->offset + buffer->size });
 508    }
 509
 510    cmd_buffer->state.vb_dirty &= ~vb_emit;
 511    cmd_buffer->state.dirty = 0;
 512 }
 513
 514 void genX(CmdDraw)(
 515     VkCommandBuffer                             commandBuffer,
 516     uint32_t                                    vertexCount,
 517     uint32_t                                    instanceCount,
 518     uint32_t                                    firstVertex,
 519     uint32_t                                    firstInstance)
 520 {
 521    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 522    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 523
 524    cmd_buffer_flush_state(cmd_buffer);
 525
 526    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 527                   .VertexAccessType = SEQUENTIAL,
 528                   .PrimitiveTopologyType = pipeline->topology,
 529                   .VertexCountPerInstance = vertexCount,
 530                   .StartVertexLocation = firstVertex,
 531                   .InstanceCount = instanceCount,
 532                   .StartInstanceLocation = firstInstance,
 533                   .BaseVertexLocation = 0);
 534 }
 535
 536 void genX(CmdDrawIndexed)(
 537     VkCommandBuffer                             commandBuffer,
 538     uint32_t                                    indexCount,
 539     uint32_t                                    instanceCount,
 540     uint32_t                                    firstIndex,
 541     int32_t                                     vertexOffset,
 542     uint32_t                                    firstInstance)
 543 {
 544    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 545    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 546
 547    cmd_buffer_flush_state(cmd_buffer);
 548
 549    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 550                   .VertexAccessType = RANDOM,
 551                   .PrimitiveTopologyType = pipeline->topology,
 552                   .VertexCountPerInstance = indexCount,
 553                   .StartVertexLocation = firstIndex,
 554                   .InstanceCount = instanceCount,
 555                   .StartInstanceLocation = firstInstance,
 556                   .BaseVertexLocation = vertexOffset);
 557 }
 558
 559 static void
 560 gen7_batch_lrm(struct anv_batch *batch,
 561               uint32_t reg, struct anv_bo *bo, uint32_t offset)
 562 {
 563    anv_batch_emit(batch, GEN7_MI_LOAD_REGISTER_MEM,
 564                   .RegisterAddress = reg,
 565                   .MemoryAddress = { bo, offset });
 566 }
 567
 568 static void
 569 gen7_batch_lri(struct anv_batch *batch, uint32_t reg, uint32_t imm)
 570 {
 571    anv_batch_emit(batch, GEN7_MI_LOAD_REGISTER_IMM,
 572                   .RegisterOffset = reg,
 573                   .DataDWord = imm);
 574 }
 575
 576 /* Auto-Draw / Indirect Registers */
 577 #define GEN7_3DPRIM_END_OFFSET          0x2420
 578 #define GEN7_3DPRIM_START_VERTEX        0x2430
 579 #define GEN7_3DPRIM_VERTEX_COUNT        0x2434
 580 #define GEN7_3DPRIM_INSTANCE_COUNT      0x2438
 581 #define GEN7_3DPRIM_START_INSTANCE      0x243C
 582 #define GEN7_3DPRIM_BASE_VERTEX         0x2440
 583
 584 void genX(CmdDrawIndirect)(
 585     VkCommandBuffer                             commandBuffer,
 586     VkBuffer                                    _buffer,
 587     VkDeviceSize                                offset,
 588     uint32_t                                    drawCount,
 589     uint32_t                                    stride)
 590 {
 591    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 592    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 593    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 594    struct anv_bo *bo = buffer->bo;
 595    uint32_t bo_offset = buffer->offset + offset;
 596
 597    cmd_buffer_flush_state(cmd_buffer);
 598
 599    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
 600    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
 601    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
 602    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 12);
 603    gen7_batch_lri(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, 0);
 604
 605    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 606                   .IndirectParameterEnable = true,
 607                   .VertexAccessType = SEQUENTIAL,
 608                   .PrimitiveTopologyType = pipeline->topology);
 609 }
 610
 611 void genX(CmdDrawIndexedIndirect)(
 612     VkCommandBuffer                             commandBuffer,
 613     VkBuffer                                    _buffer,
 614     VkDeviceSize                                offset,
 615     uint32_t                                    drawCount,
 616     uint32_t                                    stride)
 617 {
 618    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 619    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 620    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 621    struct anv_bo *bo = buffer->bo;
 622    uint32_t bo_offset = buffer->offset + offset;
 623
 624    cmd_buffer_flush_state(cmd_buffer);
 625
 626    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
 627    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
 628    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
 629    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, bo, bo_offset + 12);
 630    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 16);
 631
 632    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 633                   .IndirectParameterEnable = true,
 634                   .VertexAccessType = RANDOM,
 635                   .PrimitiveTopologyType = pipeline->topology);
 636 }
 637
 638 void genX(CmdDispatch)(
 639     VkCommandBuffer                             commandBuffer,
 640     uint32_t                                    x,
 641     uint32_t                                    y,
 642     uint32_t                                    z)
 643 {
 644    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 645    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 646    struct brw_cs_prog_data *prog_data = &pipeline->cs_prog_data;
 647
 648    cmd_buffer_flush_compute_state(cmd_buffer);
 649
 650    anv_batch_emit(&cmd_buffer->batch, GEN7_GPGPU_WALKER,
 651                   .SIMDSize = prog_data->simd_size / 16,
 652                   .ThreadDepthCounterMaximum = 0,
 653                   .ThreadHeightCounterMaximum = 0,
 654                   .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max - 1,
 655                   .ThreadGroupIDXDimension = x,
 656                   .ThreadGroupIDYDimension = y,
 657                   .ThreadGroupIDZDimension = z,
 658                   .RightExecutionMask = pipeline->cs_right_mask,
 659                   .BottomExecutionMask = 0xffffffff);
 660
 661    anv_batch_emit(&cmd_buffer->batch, GEN7_MEDIA_STATE_FLUSH);
 662 }
 663
 664 #define GPGPU_DISPATCHDIMX 0x2500
 665 #define GPGPU_DISPATCHDIMY 0x2504
 666 #define GPGPU_DISPATCHDIMZ 0x2508
 667
 668 void genX(CmdDispatchIndirect)(
 669     VkCommandBuffer                             commandBuffer,
 670     VkBuffer                                    _buffer,
 671     VkDeviceSize                                offset)
 672 {
 673    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 674    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 675    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 676    struct brw_cs_prog_data *prog_data = &pipeline->cs_prog_data;
 677    struct anv_bo *bo = buffer->bo;
 678    uint32_t bo_offset = buffer->offset + offset;
 679
 680    cmd_buffer_flush_compute_state(cmd_buffer);
 681
 682    gen7_batch_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMX, bo, bo_offset);
 683    gen7_batch_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMY, bo, bo_offset + 4);
 684    gen7_batch_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMZ, bo, bo_offset + 8);
 685
 686    anv_batch_emit(&cmd_buffer->batch, GEN7_GPGPU_WALKER,
 687                   .IndirectParameterEnable = true,
 688                   .SIMDSize = prog_data->simd_size / 16,
 689                   .ThreadDepthCounterMaximum = 0,
 690                   .ThreadHeightCounterMaximum = 0,
 691                   .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max - 1,
 692                   .RightExecutionMask = pipeline->cs_right_mask,
 693                   .BottomExecutionMask = 0xffffffff);
 694
 695    anv_batch_emit(&cmd_buffer->batch, GEN7_MEDIA_STATE_FLUSH);
 696 }
 697
 698 static void
 699 cmd_buffer_emit_depth_stencil(struct anv_cmd_buffer *cmd_buffer)
 700 {
 701    const struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
 702    const struct anv_image_view *iview =
 703       anv_cmd_buffer_get_depth_stencil_view(cmd_buffer);
 704    const struct anv_image *image = iview ? iview->image : NULL;
 705
 706    /* XXX: isl needs to grow depth format support */
 707    const struct anv_format *anv_format =
 708       iview ? anv_format_for_vk_format(iview->vk_format) : NULL;
 709
 710    const bool has_depth = iview && anv_format->depth_format;
 711    const bool has_stencil = iview && anv_format->has_stencil;
 712
 713    /* Emit 3DSTATE_DEPTH_BUFFER */
 714    if (has_depth) {
 715       anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER),
 716          .SurfaceType = SURFTYPE_2D,
 717          .DepthWriteEnable = true,
 718          .StencilWriteEnable = has_stencil,
 719          .HierarchicalDepthBufferEnable = false,
 720          .SurfaceFormat = anv_format->depth_format,
 721          .SurfacePitch = image->depth_surface.isl.row_pitch - 1,
 722          .SurfaceBaseAddress = {
 723             .bo = image->bo,
 724             .offset = image->depth_surface.offset,
 725          },
 726          .Height = fb->height - 1,
 727          .Width = fb->width - 1,
 728          .LOD = 0,
 729          .Depth = 1 - 1,
 730          .MinimumArrayElement = 0,
 731          .DepthBufferObjectControlState = GENX(MOCS),
 732          .RenderTargetViewExtent = 1 - 1);
 733    } else {
 734       /* Even when no depth buffer is present, the hardware requires that
 735        * 3DSTATE_DEPTH_BUFFER be programmed correctly. The Broadwell PRM says:
 736        *
 737        *    If a null depth buffer is bound, the driver must instead bind depth as:
 738        *       3DSTATE_DEPTH.SurfaceType = SURFTYPE_2D
 739        *       3DSTATE_DEPTH.Width = 1
 740        *       3DSTATE_DEPTH.Height = 1
 741        *       3DSTATE_DEPTH.SuraceFormat = D16_UNORM
 742        *       3DSTATE_DEPTH.SurfaceBaseAddress = 0
 743        *       3DSTATE_DEPTH.HierarchicalDepthBufferEnable = 0
 744        *       3DSTATE_WM_DEPTH_STENCIL.DepthTestEnable = 0
 745        *       3DSTATE_WM_DEPTH_STENCIL.DepthBufferWriteEnable = 0
 746        *
 747        * The PRM is wrong, though. The width and height must be programmed to
 748        * actual framebuffer's width and height, even when neither depth buffer
 749        * nor stencil buffer is present.
 750        */
 751       anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER),
 752          .SurfaceType = SURFTYPE_2D,
 753          .SurfaceFormat = D16_UNORM,
 754          .Width = fb->width - 1,
 755          .Height = fb->height - 1,
 756          .StencilWriteEnable = has_stencil);
 757    }
 758
 759    /* Emit 3DSTATE_STENCIL_BUFFER */
 760    if (has_stencil) {
 761       anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_STENCIL_BUFFER),
 762 #     if (ANV_IS_HASWELL)
 763          .StencilBufferEnable = true,
 764 #     endif
 765          .StencilBufferObjectControlState = GENX(MOCS),
 766
 767          /* Stencil buffers have strange pitch. The PRM says:
 768           *
 769           *    The pitch must be set to 2x the value computed based on width,
 770           *    as the stencil buffer is stored with two rows interleaved.
 771           */
 772          .SurfacePitch = 2 * image->stencil_surface.isl.row_pitch - 1,
 773
 774          .SurfaceBaseAddress = {
 775             .bo = image->bo,
 776             .offset = image->offset + image->stencil_surface.offset,
 777          });
 778    } else {
 779       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_STENCIL_BUFFER);
 780    }
 781
 782    /* Disable hierarchial depth buffers. */
 783    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_HIER_DEPTH_BUFFER);
 784
 785    /* Clear the clear params. */
 786    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_CLEAR_PARAMS);
 787 }
 788
 789 GENX_FUNC(GEN7, GEN7) void
 790 genX(cmd_buffer_begin_subpass)(struct anv_cmd_buffer *cmd_buffer,
 791                                struct anv_subpass *subpass)
 792 {
 793    cmd_buffer->state.subpass = subpass;
 794    cmd_buffer->state.descriptors_dirty |= VK_SHADER_STAGE_FRAGMENT_BIT;
 795    cmd_buffer->state.dirty |= ANV_CMD_DIRTY_RENDER_TARGETS;
 796
 797    cmd_buffer_emit_depth_stencil(cmd_buffer);
 798 }
 799
 800 static void
 801 begin_render_pass(struct anv_cmd_buffer *cmd_buffer,
 802                   const VkRenderPassBeginInfo* pRenderPassBegin)
 803 {
 804    ANV_FROM_HANDLE(anv_render_pass, pass, pRenderPassBegin->renderPass);
 805    ANV_FROM_HANDLE(anv_framebuffer, framebuffer, pRenderPassBegin->framebuffer);
 806
 807    cmd_buffer->state.framebuffer = framebuffer;
 808    cmd_buffer->state.pass = pass;
 809
 810    const VkRect2D *render_area = &pRenderPassBegin->renderArea;
 811
 812    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_DRAWING_RECTANGLE,
 813                   .ClippedDrawingRectangleYMin = render_area->offset.y,
 814                   .ClippedDrawingRectangleXMin = render_area->offset.x,
 815                   .ClippedDrawingRectangleYMax =
 816                      render_area->offset.y + render_area->extent.height - 1,
 817                   .ClippedDrawingRectangleXMax =
 818                      render_area->offset.x + render_area->extent.width - 1,
 819                   .DrawingRectangleOriginY = 0,
 820                   .DrawingRectangleOriginX = 0);
 821
 822    anv_cmd_buffer_clear_attachments(cmd_buffer, pass,
 823                                     pRenderPassBegin->pClearValues);
 824 }
 825
 826 void genX(CmdBeginRenderPass)(
 827     VkCommandBuffer                             commandBuffer,
 828     const VkRenderPassBeginInfo*                pRenderPassBegin,
 829     VkSubpassContents                           contents)
 830 {
 831    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 832    ANV_FROM_HANDLE(anv_render_pass, pass, pRenderPassBegin->renderPass);
 833
 834    begin_render_pass(cmd_buffer, pRenderPassBegin);
 835
 836    gen7_cmd_buffer_begin_subpass(cmd_buffer, pass->subpasses);
 837 }
 838
 839 void genX(CmdNextSubpass)(
 840     VkCommandBuffer                             commandBuffer,
 841     VkSubpassContents                           contents)
 842 {
 843    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 844
 845    assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
 846
 847    gen7_cmd_buffer_begin_subpass(cmd_buffer, cmd_buffer->state.subpass + 1);
 848 }
 849
 850 void genX(CmdEndRenderPass)(
 851     VkCommandBuffer                             commandBuffer)
 852 {
 853    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 854
 855    /* Emit a flushing pipe control at the end of a pass.  This is kind of a
 856     * hack but it ensures that render targets always actually get written.
 857     * Eventually, we should do flushing based on image format transitions
 858     * or something of that nature.
 859     */
 860    anv_batch_emit(&cmd_buffer->batch, GEN7_PIPE_CONTROL,
 861                   .PostSyncOperation = NoWrite,
 862                   .RenderTargetCacheFlushEnable = true,
 863                   .InstructionCacheInvalidateEnable = true,
 864                   .DepthCacheFlushEnable = true,
 865                   .VFCacheInvalidationEnable = true,
 866                   .TextureCacheInvalidationEnable = true,
 867                   .CommandStreamerStallEnable = true);
 868 }
 869
 870 void genX(CmdSetEvent)(
 871     VkCommandBuffer                             commandBuffer,
 872     VkEvent                                     event,
 873     VkPipelineStageFlags                        stageMask)
 874 {
 875    stub();
 876 }
 877
 878 void genX(CmdResetEvent)(
 879     VkCommandBuffer                             commandBuffer,
 880     VkEvent                                     event,
 881     VkPipelineStageFlags                        stageMask)
 882 {
 883    stub();
 884 }
 885
 886 void genX(CmdWaitEvents)(
 887     VkCommandBuffer                             commandBuffer,
 888     uint32_t                                    eventCount,
 889     const VkEvent*                              pEvents,
 890     VkPipelineStageFlags                        srcStageMask,
 891     VkPipelineStageFlags                        destStageMask,
 892     uint32_t                                    memBarrierCount,
 893     const void* const*                          ppMemBarriers)
 894 {
 895    stub();
 896 }