src/vulkan/gen7_cmd_buffer.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <assert.h>
  25 #include <stdbool.h>
  26 #include <string.h>
  27 #include <unistd.h>
  28 #include <fcntl.h>
  29
  30 #include "anv_private.h"
  31
  32 #include "gen7_pack.h"
  33 #include "gen75_pack.h"
  34
  35 static void
  36 cmd_buffer_flush_push_constants(struct anv_cmd_buffer *cmd_buffer)
  37 {
  38    static const uint32_t push_constant_opcodes[] = {
  39       [MESA_SHADER_VERTEX]                      = 21,
  40       [MESA_SHADER_TESS_CTRL]                   = 25, /* HS */
  41       [MESA_SHADER_TESS_EVAL]                   = 26, /* DS */
  42       [MESA_SHADER_GEOMETRY]                    = 22,
  43       [MESA_SHADER_FRAGMENT]                    = 23,
  44       [MESA_SHADER_COMPUTE]                     = 0,
  45    };
  46
  47    VkShaderStageFlags flushed = 0;
  48
  49    anv_foreach_stage(stage, cmd_buffer->state.push_constants_dirty) {
  50       struct anv_state state = anv_cmd_buffer_push_constants(cmd_buffer, stage);
  51
  52       if (state.offset == 0)
  53          continue;
  54
  55       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_CONSTANT_VS,
  56                      ._3DCommandSubOpcode = push_constant_opcodes[stage],
  57                      .ConstantBody = {
  58                         .PointerToConstantBuffer0 = { .offset = state.offset },
  59                         .ConstantBuffer0ReadLength = DIV_ROUND_UP(state.alloc_size, 32),
  60                      });
  61
  62       flushed |= mesa_to_vk_shader_stage(stage);
  63    }
  64
  65    cmd_buffer->state.push_constants_dirty &= ~flushed;
  66 }
  67
  68 static VkResult
  69 flush_descriptor_set(struct anv_cmd_buffer *cmd_buffer, gl_shader_stage stage)
  70 {
  71    struct anv_state surfaces = { 0, }, samplers = { 0, };
  72    VkResult result;
  73
  74    result = anv_cmd_buffer_emit_samplers(cmd_buffer, stage, &samplers);
  75    if (result != VK_SUCCESS)
  76       return result;
  77    result = anv_cmd_buffer_emit_binding_table(cmd_buffer, stage, &surfaces);
  78    if (result != VK_SUCCESS)
  79       return result;
  80
  81    static const uint32_t sampler_state_opcodes[] = {
  82       [MESA_SHADER_VERTEX]                      = 43,
  83       [MESA_SHADER_TESS_CTRL]                   = 44, /* HS */
  84       [MESA_SHADER_TESS_EVAL]                   = 45, /* DS */
  85       [MESA_SHADER_GEOMETRY]                    = 46,
  86       [MESA_SHADER_FRAGMENT]                    = 47,
  87       [MESA_SHADER_COMPUTE]                     = 0,
  88    };
  89
  90    static const uint32_t binding_table_opcodes[] = {
  91       [MESA_SHADER_VERTEX]                      = 38,
  92       [MESA_SHADER_TESS_CTRL]                   = 39,
  93       [MESA_SHADER_TESS_EVAL]                   = 40,
  94       [MESA_SHADER_GEOMETRY]                    = 41,
  95       [MESA_SHADER_FRAGMENT]                    = 42,
  96       [MESA_SHADER_COMPUTE]                     = 0,
  97    };
  98
  99    if (samplers.alloc_size > 0) {
 100       anv_batch_emit(&cmd_buffer->batch,
 101                      GEN7_3DSTATE_SAMPLER_STATE_POINTERS_VS,
 102                      ._3DCommandSubOpcode  = sampler_state_opcodes[stage],
 103                      .PointertoVSSamplerState = samplers.offset);
 104    }
 105
 106    if (surfaces.alloc_size > 0) {
 107       anv_batch_emit(&cmd_buffer->batch,
 108                      GEN7_3DSTATE_BINDING_TABLE_POINTERS_VS,
 109                      ._3DCommandSubOpcode  = binding_table_opcodes[stage],
 110                      .PointertoVSBindingTable = surfaces.offset);
 111    }
 112
 113    return VK_SUCCESS;
 114 }
 115
 116 GENX_FUNC(GEN7, GEN7) void
 117 genX(cmd_buffer_flush_descriptor_sets)(struct anv_cmd_buffer *cmd_buffer)
 118 {
 119    VkShaderStageFlags dirty = cmd_buffer->state.descriptors_dirty &
 120                               cmd_buffer->state.pipeline->active_stages;
 121
 122    VkResult result = VK_SUCCESS;
 123    anv_foreach_stage(s, dirty) {
 124       result = flush_descriptor_set(cmd_buffer, s);
 125       if (result != VK_SUCCESS)
 126          break;
 127    }
 128
 129    if (result != VK_SUCCESS) {
 130       assert(result == VK_ERROR_OUT_OF_DEVICE_MEMORY);
 131
 132       result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
 133       assert(result == VK_SUCCESS);
 134
 135       /* Re-emit state base addresses so we get the new surface state base
 136        * address before we start emitting binding tables etc.
 137        */
 138       anv_cmd_buffer_emit_state_base_address(cmd_buffer);
 139
 140       /* Re-emit all active binding tables */
 141       anv_foreach_stage(s, cmd_buffer->state.pipeline->active_stages) {
 142          result = flush_descriptor_set(cmd_buffer, s);
 143
 144          /* It had better succeed this time */
 145          assert(result == VK_SUCCESS);
 146       }
 147    }
 148
 149    cmd_buffer->state.descriptors_dirty &= ~cmd_buffer->state.pipeline->active_stages;
 150 }
 151
 152 static inline int64_t
 153 clamp_int64(int64_t x, int64_t min, int64_t max)
 154 {
 155    if (x < min)
 156       return min;
 157    else if (x < max)
 158       return x;
 159    else
 160       return max;
 161 }
 162
 163 static void
 164 emit_scissor_state(struct anv_cmd_buffer *cmd_buffer,
 165                    uint32_t count, const VkRect2D *scissors)
 166 {
 167    struct anv_state scissor_state =
 168       anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, count * 32, 32);
 169
 170    for (uint32_t i = 0; i < count; i++) {
 171       const VkRect2D *s = &scissors[i];
 172
 173       /* Since xmax and ymax are inclusive, we have to have xmax < xmin or
 174        * ymax < ymin for empty clips.  In case clip x, y, width height are all
 175        * 0, the clamps below produce 0 for xmin, ymin, xmax, ymax, which isn't
 176        * what we want. Just special case empty clips and produce a canonical
 177        * empty clip. */
 178       static const struct GEN7_SCISSOR_RECT empty_scissor = {
 179          .ScissorRectangleYMin = 1,
 180          .ScissorRectangleXMin = 1,
 181          .ScissorRectangleYMax = 0,
 182          .ScissorRectangleXMax = 0
 183       };
 184
 185       const int max = 0xffff;
 186       struct GEN7_SCISSOR_RECT scissor = {
 187          /* Do this math using int64_t so overflow gets clamped correctly. */
 188          .ScissorRectangleYMin = clamp_int64(s->offset.y, 0, max),
 189          .ScissorRectangleXMin = clamp_int64(s->offset.x, 0, max),
 190          .ScissorRectangleYMax = clamp_int64((uint64_t) s->offset.y + s->extent.height - 1, 0, max),
 191          .ScissorRectangleXMax = clamp_int64((uint64_t) s->offset.x + s->extent.width - 1, 0, max)
 192       };
 193
 194       if (s->extent.width <= 0 || s->extent.height <= 0) {
 195          GEN7_SCISSOR_RECT_pack(NULL, scissor_state.map + i * 32,
 196                                 &empty_scissor);
 197       } else {
 198          GEN7_SCISSOR_RECT_pack(NULL, scissor_state.map + i * 32, &scissor);
 199       }
 200    }
 201
 202    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_SCISSOR_STATE_POINTERS,
 203                   .ScissorRectPointer = scissor_state.offset);
 204 }
 205
 206 GENX_FUNC(GEN7, GEN7) void
 207 genX(cmd_buffer_emit_scissor)(struct anv_cmd_buffer *cmd_buffer)
 208 {
 209    if (cmd_buffer->state.dynamic.scissor.count > 0) {
 210       emit_scissor_state(cmd_buffer, cmd_buffer->state.dynamic.scissor.count,
 211                          cmd_buffer->state.dynamic.scissor.scissors);
 212    } else {
 213       /* Emit a default scissor based on the currently bound framebuffer */
 214       emit_scissor_state(cmd_buffer, 1,
 215                          &(VkRect2D) {
 216                             .offset = { .x = 0, .y = 0, },
 217                             .extent = {
 218                                .width = cmd_buffer->state.framebuffer->width,
 219                                .height = cmd_buffer->state.framebuffer->height,
 220                             },
 221                          });
 222    }
 223 }
 224
 225 static const uint32_t vk_to_gen_index_type[] = {
 226    [VK_INDEX_TYPE_UINT16]                       = INDEX_WORD,
 227    [VK_INDEX_TYPE_UINT32]                       = INDEX_DWORD,
 228 };
 229
 230 static const uint32_t restart_index_for_type[] = {
 231    [VK_INDEX_TYPE_UINT16]                    = UINT16_MAX,
 232    [VK_INDEX_TYPE_UINT32]                    = UINT32_MAX,
 233 };
 234
 235 void genX(CmdBindIndexBuffer)(
 236     VkCommandBuffer                             commandBuffer,
 237     VkBuffer                                    _buffer,
 238     VkDeviceSize                                offset,
 239     VkIndexType                                 indexType)
 240 {
 241    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 242    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 243
 244    cmd_buffer->state.dirty |= ANV_CMD_DIRTY_INDEX_BUFFER;
 245    if (ANV_IS_HASWELL)
 246       cmd_buffer->state.restart_index = restart_index_for_type[indexType];
 247    cmd_buffer->state.gen7.index_buffer = buffer;
 248    cmd_buffer->state.gen7.index_type = vk_to_gen_index_type[indexType];
 249    cmd_buffer->state.gen7.index_offset = offset;
 250 }
 251
 252 static VkResult
 253 flush_compute_descriptor_set(struct anv_cmd_buffer *cmd_buffer)
 254 {
 255    struct anv_device *device = cmd_buffer->device;
 256    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 257    struct anv_state surfaces = { 0, }, samplers = { 0, };
 258    VkResult result;
 259
 260    result = anv_cmd_buffer_emit_samplers(cmd_buffer,
 261                                          MESA_SHADER_COMPUTE, &samplers);
 262    if (result != VK_SUCCESS)
 263       return result;
 264    result = anv_cmd_buffer_emit_binding_table(cmd_buffer,
 265                                               MESA_SHADER_COMPUTE, &surfaces);
 266    if (result != VK_SUCCESS)
 267       return result;
 268
 269    struct GEN7_INTERFACE_DESCRIPTOR_DATA desc = {
 270       .KernelStartPointer = pipeline->cs_simd,
 271       .BindingTablePointer = surfaces.offset,
 272       .SamplerStatePointer = samplers.offset,
 273       .NumberofThreadsinGPGPUThreadGroup = 0 /* FIXME: Really? */
 274    };
 275
 276    uint32_t size = GEN7_INTERFACE_DESCRIPTOR_DATA_length * sizeof(uint32_t);
 277    struct anv_state state =
 278       anv_state_pool_alloc(&device->dynamic_state_pool, size, 64);
 279
 280    GEN7_INTERFACE_DESCRIPTOR_DATA_pack(NULL, state.map, &desc);
 281
 282    anv_batch_emit(&cmd_buffer->batch, GEN7_MEDIA_INTERFACE_DESCRIPTOR_LOAD,
 283                   .InterfaceDescriptorTotalLength = size,
 284                   .InterfaceDescriptorDataStartAddress = state.offset);
 285
 286    return VK_SUCCESS;
 287 }
 288
 289 static void
 290 cmd_buffer_flush_compute_state(struct anv_cmd_buffer *cmd_buffer)
 291 {
 292    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 293    VkResult result;
 294
 295    assert(pipeline->active_stages == VK_SHADER_STAGE_COMPUTE_BIT);
 296
 297    if (cmd_buffer->state.current_pipeline != GPGPU) {
 298       anv_batch_emit(&cmd_buffer->batch, GEN7_PIPELINE_SELECT,
 299                      .PipelineSelection = GPGPU);
 300       cmd_buffer->state.current_pipeline = GPGPU;
 301    }
 302
 303    if (cmd_buffer->state.compute_dirty & ANV_CMD_DIRTY_PIPELINE)
 304       anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
 305
 306    if ((cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_COMPUTE_BIT) ||
 307        (cmd_buffer->state.compute_dirty & ANV_CMD_DIRTY_PIPELINE)) {
 308       /* FIXME: figure out descriptors for gen7 */
 309       result = flush_compute_descriptor_set(cmd_buffer);
 310       assert(result == VK_SUCCESS);
 311       cmd_buffer->state.descriptors_dirty &= ~VK_SHADER_STAGE_COMPUTE_BIT;
 312    }
 313
 314    cmd_buffer->state.compute_dirty = 0;
 315 }
 316
 317 static void
 318 cmd_buffer_flush_state(struct anv_cmd_buffer *cmd_buffer)
 319 {
 320    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 321    uint32_t *p;
 322
 323    uint32_t vb_emit = cmd_buffer->state.vb_dirty & pipeline->vb_used;
 324
 325    assert((pipeline->active_stages & VK_SHADER_STAGE_COMPUTE_BIT) == 0);
 326
 327    if (cmd_buffer->state.current_pipeline != _3D) {
 328       anv_batch_emit(&cmd_buffer->batch, GEN7_PIPELINE_SELECT,
 329                      .PipelineSelection = _3D);
 330       cmd_buffer->state.current_pipeline = _3D;
 331    }
 332
 333    if (vb_emit) {
 334       const uint32_t num_buffers = __builtin_popcount(vb_emit);
 335       const uint32_t num_dwords = 1 + num_buffers * 4;
 336
 337       p = anv_batch_emitn(&cmd_buffer->batch, num_dwords,
 338                           GEN7_3DSTATE_VERTEX_BUFFERS);
 339       uint32_t vb, i = 0;
 340       for_each_bit(vb, vb_emit) {
 341          struct anv_buffer *buffer = cmd_buffer->state.vertex_bindings[vb].buffer;
 342          uint32_t offset = cmd_buffer->state.vertex_bindings[vb].offset;
 343
 344          struct GEN7_VERTEX_BUFFER_STATE state = {
 345             .VertexBufferIndex = vb,
 346             .BufferAccessType = pipeline->instancing_enable[vb] ? INSTANCEDATA : VERTEXDATA,
 347             .VertexBufferMemoryObjectControlState = GEN7_MOCS,
 348             .AddressModifyEnable = true,
 349             .BufferPitch = pipeline->binding_stride[vb],
 350             .BufferStartingAddress = { buffer->bo, buffer->offset + offset },
 351             .EndAddress = { buffer->bo, buffer->offset + buffer->size - 1},
 352             .InstanceDataStepRate = 1
 353          };
 354
 355          GEN7_VERTEX_BUFFER_STATE_pack(&cmd_buffer->batch, &p[1 + i * 4], &state);
 356          i++;
 357       }
 358    }
 359
 360    if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_PIPELINE) {
 361       /* If somebody compiled a pipeline after starting a command buffer the
 362        * scratch bo may have grown since we started this cmd buffer (and
 363        * emitted STATE_BASE_ADDRESS).  If we're binding that pipeline now,
 364        * reemit STATE_BASE_ADDRESS so that we use the bigger scratch bo. */
 365       if (cmd_buffer->state.scratch_size < pipeline->total_scratch)
 366          gen7_cmd_buffer_emit_state_base_address(cmd_buffer);
 367
 368       anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
 369    }
 370
 371    if (cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_VERTEX_BIT ||
 372        cmd_buffer->state.push_constants_dirty & VK_SHADER_STAGE_VERTEX_BIT) {
 373       /* From the IVB PRM Vol. 2, Part 1, Section 3.2.1:
 374        *
 375        *    "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
 376        *    stall needs to be sent just prior to any 3DSTATE_VS,
 377        *    3DSTATE_URB_VS, 3DSTATE_CONSTANT_VS,
 378        *    3DSTATE_BINDING_TABLE_POINTER_VS,
 379        *    3DSTATE_SAMPLER_STATE_POINTER_VS command.  Only one
 380        *    PIPE_CONTROL needs to be sent before any combination of VS
 381        *    associated 3DSTATE."
 382        */
 383       anv_batch_emit(&cmd_buffer->batch, GEN7_PIPE_CONTROL,
 384                      .DepthStallEnable = true,
 385                      .PostSyncOperation = WriteImmediateData,
 386                      .Address = { &cmd_buffer->device->workaround_bo, 0 });
 387    }
 388
 389    if (cmd_buffer->state.descriptors_dirty)
 390       gen7_cmd_buffer_flush_descriptor_sets(cmd_buffer);
 391
 392    if (cmd_buffer->state.push_constants_dirty)
 393       cmd_buffer_flush_push_constants(cmd_buffer);
 394
 395    /* We use the gen8 state here because it only contains the additional
 396     * min/max fields and, since they occur at the end of the packet and
 397     * don't change the stride, they work on gen7 too.
 398     */
 399    if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_VIEWPORT)
 400       gen8_cmd_buffer_emit_viewport(cmd_buffer);
 401
 402    if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_SCISSOR)
 403       gen7_cmd_buffer_emit_scissor(cmd_buffer);
 404
 405    if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
 406                                   ANV_CMD_DIRTY_DYNAMIC_LINE_WIDTH |
 407                                   ANV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS)) {
 408
 409       bool enable_bias = cmd_buffer->state.dynamic.depth_bias.bias != 0.0f ||
 410          cmd_buffer->state.dynamic.depth_bias.slope != 0.0f;
 411
 412       uint32_t sf_dw[GEN7_3DSTATE_SF_length];
 413       struct GEN7_3DSTATE_SF sf = {
 414          GEN7_3DSTATE_SF_header,
 415          .LineWidth = cmd_buffer->state.dynamic.line_width,
 416          .GlobalDepthOffsetEnableSolid = enable_bias,
 417          .GlobalDepthOffsetEnableWireframe = enable_bias,
 418          .GlobalDepthOffsetEnablePoint = enable_bias,
 419          .GlobalDepthOffsetConstant = cmd_buffer->state.dynamic.depth_bias.bias,
 420          .GlobalDepthOffsetScale = cmd_buffer->state.dynamic.depth_bias.slope,
 421          .GlobalDepthOffsetClamp = cmd_buffer->state.dynamic.depth_bias.clamp
 422       };
 423       GEN7_3DSTATE_SF_pack(NULL, sf_dw, &sf);
 424
 425       anv_batch_emit_merge(&cmd_buffer->batch, sf_dw, pipeline->gen7.sf);
 426    }
 427
 428    if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS |
 429                                   ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE)) {
 430       struct anv_state cc_state =
 431          anv_cmd_buffer_alloc_dynamic_state(cmd_buffer,
 432                                             GEN7_COLOR_CALC_STATE_length, 64);
 433       struct GEN7_COLOR_CALC_STATE cc = {
 434          .BlendConstantColorRed = cmd_buffer->state.dynamic.blend_constants[0],
 435          .BlendConstantColorGreen = cmd_buffer->state.dynamic.blend_constants[1],
 436          .BlendConstantColorBlue = cmd_buffer->state.dynamic.blend_constants[2],
 437          .BlendConstantColorAlpha = cmd_buffer->state.dynamic.blend_constants[3],
 438          .StencilReferenceValue =
 439             cmd_buffer->state.dynamic.stencil_reference.front,
 440          .BackFaceStencilReferenceValue =
 441             cmd_buffer->state.dynamic.stencil_reference.back,
 442       };
 443       GEN7_COLOR_CALC_STATE_pack(NULL, cc_state.map, &cc);
 444
 445       anv_batch_emit(&cmd_buffer->batch,
 446                      GEN7_3DSTATE_CC_STATE_POINTERS,
 447                      .ColorCalcStatePointer = cc_state.offset);
 448    }
 449
 450    if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
 451                                   ANV_CMD_DIRTY_RENDER_TARGETS |
 452                                   ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK |
 453                                   ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK)) {
 454       uint32_t depth_stencil_dw[GEN7_DEPTH_STENCIL_STATE_length];
 455
 456       const struct anv_image_view *iview =
 457          anv_cmd_buffer_get_depth_stencil_view(cmd_buffer);
 458
 459       struct GEN7_DEPTH_STENCIL_STATE depth_stencil = {
 460          .StencilBufferWriteEnable = iview && iview->format->has_stencil,
 461
 462          .StencilTestMask =
 463             cmd_buffer->state.dynamic.stencil_compare_mask.front & 0xff,
 464          .StencilWriteMask =
 465             cmd_buffer->state.dynamic.stencil_write_mask.front & 0xff,
 466
 467          .BackfaceStencilTestMask =
 468             cmd_buffer->state.dynamic.stencil_compare_mask.back & 0xff,
 469          .BackfaceStencilWriteMask =
 470             cmd_buffer->state.dynamic.stencil_write_mask.back & 0xff,
 471       };
 472       GEN7_DEPTH_STENCIL_STATE_pack(NULL, depth_stencil_dw, &depth_stencil);
 473
 474       struct anv_state ds_state =
 475          anv_cmd_buffer_merge_dynamic(cmd_buffer, depth_stencil_dw,
 476                                       pipeline->gen7.depth_stencil_state,
 477                                       GEN7_DEPTH_STENCIL_STATE_length, 64);
 478
 479       anv_batch_emit(&cmd_buffer->batch,
 480                      GEN7_3DSTATE_DEPTH_STENCIL_STATE_POINTERS,
 481                      .PointertoDEPTH_STENCIL_STATE = ds_state.offset);
 482    }
 483
 484    if (cmd_buffer->state.gen7.index_buffer &&
 485        cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
 486                                   ANV_CMD_DIRTY_INDEX_BUFFER)) {
 487       struct anv_buffer *buffer = cmd_buffer->state.gen7.index_buffer;
 488       uint32_t offset = cmd_buffer->state.gen7.index_offset;
 489
 490       if (ANV_IS_HASWELL) {
 491          anv_batch_emit(&cmd_buffer->batch, GEN75_3DSTATE_VF,
 492                         .IndexedDrawCutIndexEnable = pipeline->primitive_restart,
 493                         .CutIndex = cmd_buffer->state.restart_index);
 494       }
 495
 496       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_INDEX_BUFFER,
 497                      .CutIndexEnable = pipeline->primitive_restart,
 498                      .IndexFormat = cmd_buffer->state.gen7.index_type,
 499                      .MemoryObjectControlState = GEN7_MOCS,
 500                      .BufferStartingAddress = { buffer->bo, buffer->offset + offset },
 501                      .BufferEndingAddress = { buffer->bo, buffer->offset + buffer->size });
 502    }
 503
 504    cmd_buffer->state.vb_dirty &= ~vb_emit;
 505    cmd_buffer->state.dirty = 0;
 506 }
 507
 508 void genX(CmdDraw)(
 509     VkCommandBuffer                             commandBuffer,
 510     uint32_t                                    vertexCount,
 511     uint32_t                                    instanceCount,
 512     uint32_t                                    firstVertex,
 513     uint32_t                                    firstInstance)
 514 {
 515    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 516    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 517
 518    cmd_buffer_flush_state(cmd_buffer);
 519
 520    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 521                   .VertexAccessType = SEQUENTIAL,
 522                   .PrimitiveTopologyType = pipeline->topology,
 523                   .VertexCountPerInstance = vertexCount,
 524                   .StartVertexLocation = firstVertex,
 525                   .InstanceCount = instanceCount,
 526                   .StartInstanceLocation = firstInstance,
 527                   .BaseVertexLocation = 0);
 528 }
 529
 530 void genX(CmdDrawIndexed)(
 531     VkCommandBuffer                             commandBuffer,
 532     uint32_t                                    indexCount,
 533     uint32_t                                    instanceCount,
 534     uint32_t                                    firstIndex,
 535     int32_t                                     vertexOffset,
 536     uint32_t                                    firstInstance)
 537 {
 538    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 539    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 540
 541    cmd_buffer_flush_state(cmd_buffer);
 542
 543    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 544                   .VertexAccessType = RANDOM,
 545                   .PrimitiveTopologyType = pipeline->topology,
 546                   .VertexCountPerInstance = indexCount,
 547                   .StartVertexLocation = firstIndex,
 548                   .InstanceCount = instanceCount,
 549                   .StartInstanceLocation = firstInstance,
 550                   .BaseVertexLocation = vertexOffset);
 551 }
 552
 553 static void
 554 gen7_batch_lrm(struct anv_batch *batch,
 555               uint32_t reg, struct anv_bo *bo, uint32_t offset)
 556 {
 557    anv_batch_emit(batch, GEN7_MI_LOAD_REGISTER_MEM,
 558                   .RegisterAddress = reg,
 559                   .MemoryAddress = { bo, offset });
 560 }
 561
 562 static void
 563 gen7_batch_lri(struct anv_batch *batch, uint32_t reg, uint32_t imm)
 564 {
 565    anv_batch_emit(batch, GEN7_MI_LOAD_REGISTER_IMM,
 566                   .RegisterOffset = reg,
 567                   .DataDWord = imm);
 568 }
 569
 570 /* Auto-Draw / Indirect Registers */
 571 #define GEN7_3DPRIM_END_OFFSET          0x2420
 572 #define GEN7_3DPRIM_START_VERTEX        0x2430
 573 #define GEN7_3DPRIM_VERTEX_COUNT        0x2434
 574 #define GEN7_3DPRIM_INSTANCE_COUNT      0x2438
 575 #define GEN7_3DPRIM_START_INSTANCE      0x243C
 576 #define GEN7_3DPRIM_BASE_VERTEX         0x2440
 577
 578 void genX(CmdDrawIndirect)(
 579     VkCommandBuffer                             commandBuffer,
 580     VkBuffer                                    _buffer,
 581     VkDeviceSize                                offset,
 582     uint32_t                                    drawCount,
 583     uint32_t                                    stride)
 584 {
 585    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 586    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 587    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 588    struct anv_bo *bo = buffer->bo;
 589    uint32_t bo_offset = buffer->offset + offset;
 590
 591    cmd_buffer_flush_state(cmd_buffer);
 592
 593    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
 594    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
 595    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
 596    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 12);
 597    gen7_batch_lri(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, 0);
 598
 599    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 600                   .IndirectParameterEnable = true,
 601                   .VertexAccessType = SEQUENTIAL,
 602                   .PrimitiveTopologyType = pipeline->topology);
 603 }
 604
 605 void genX(CmdDrawIndexedIndirect)(
 606     VkCommandBuffer                             commandBuffer,
 607     VkBuffer                                    _buffer,
 608     VkDeviceSize                                offset,
 609     uint32_t                                    drawCount,
 610     uint32_t                                    stride)
 611 {
 612    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 613    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 614    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 615    struct anv_bo *bo = buffer->bo;
 616    uint32_t bo_offset = buffer->offset + offset;
 617
 618    cmd_buffer_flush_state(cmd_buffer);
 619
 620    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
 621    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
 622    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
 623    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, bo, bo_offset + 12);
 624    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 16);
 625
 626    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 627                   .IndirectParameterEnable = true,
 628                   .VertexAccessType = RANDOM,
 629                   .PrimitiveTopologyType = pipeline->topology);
 630 }
 631
 632 void genX(CmdDispatch)(
 633     VkCommandBuffer                             commandBuffer,
 634     uint32_t                                    x,
 635     uint32_t                                    y,
 636     uint32_t                                    z)
 637 {
 638    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 639    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 640    struct brw_cs_prog_data *prog_data = &pipeline->cs_prog_data;
 641
 642    cmd_buffer_flush_compute_state(cmd_buffer);
 643
 644    anv_batch_emit(&cmd_buffer->batch, GEN7_GPGPU_WALKER,
 645                   .SIMDSize = prog_data->simd_size / 16,
 646                   .ThreadDepthCounterMaximum = 0,
 647                   .ThreadHeightCounterMaximum = 0,
 648                   .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max,
 649                   .ThreadGroupIDXDimension = x,
 650                   .ThreadGroupIDYDimension = y,
 651                   .ThreadGroupIDZDimension = z,
 652                   .RightExecutionMask = pipeline->cs_right_mask,
 653                   .BottomExecutionMask = 0xffffffff);
 654
 655    anv_batch_emit(&cmd_buffer->batch, GEN7_MEDIA_STATE_FLUSH);
 656 }
 657
 658 #define GPGPU_DISPATCHDIMX 0x2500
 659 #define GPGPU_DISPATCHDIMY 0x2504
 660 #define GPGPU_DISPATCHDIMZ 0x2508
 661
 662 void genX(CmdDispatchIndirect)(
 663     VkCommandBuffer                             commandBuffer,
 664     VkBuffer                                    _buffer,
 665     VkDeviceSize                                offset)
 666 {
 667    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 668    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 669    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 670    struct brw_cs_prog_data *prog_data = &pipeline->cs_prog_data;
 671    struct anv_bo *bo = buffer->bo;
 672    uint32_t bo_offset = buffer->offset + offset;
 673
 674    cmd_buffer_flush_compute_state(cmd_buffer);
 675
 676    gen7_batch_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMX, bo, bo_offset);
 677    gen7_batch_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMY, bo, bo_offset + 4);
 678    gen7_batch_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMZ, bo, bo_offset + 8);
 679
 680    anv_batch_emit(&cmd_buffer->batch, GEN7_GPGPU_WALKER,
 681                   .IndirectParameterEnable = true,
 682                   .SIMDSize = prog_data->simd_size / 16,
 683                   .ThreadDepthCounterMaximum = 0,
 684                   .ThreadHeightCounterMaximum = 0,
 685                   .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max,
 686                   .RightExecutionMask = pipeline->cs_right_mask,
 687                   .BottomExecutionMask = 0xffffffff);
 688
 689    anv_batch_emit(&cmd_buffer->batch, GEN7_MEDIA_STATE_FLUSH);
 690 }
 691
 692 static void
 693 cmd_buffer_emit_depth_stencil(struct anv_cmd_buffer *cmd_buffer)
 694 {
 695    const struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
 696    const struct anv_image_view *iview =
 697       anv_cmd_buffer_get_depth_stencil_view(cmd_buffer);
 698    const struct anv_image *image = iview ? iview->image : NULL;
 699    const bool has_depth = iview && iview->format->depth_format;
 700    const bool has_stencil = iview && iview->format->has_stencil;
 701
 702    /* Emit 3DSTATE_DEPTH_BUFFER */
 703    if (has_depth) {
 704       anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER),
 705          .SurfaceType = SURFTYPE_2D,
 706          .DepthWriteEnable = iview->format->depth_format,
 707          .StencilWriteEnable = has_stencil,
 708          .HierarchicalDepthBufferEnable = false,
 709          .SurfaceFormat = iview->format->depth_format,
 710          .SurfacePitch = image->depth_surface.isl.row_pitch - 1,
 711          .SurfaceBaseAddress = {
 712             .bo = image->bo,
 713             .offset = image->depth_surface.offset,
 714          },
 715          .Height = fb->height - 1,
 716          .Width = fb->width - 1,
 717          .LOD = 0,
 718          .Depth = 1 - 1,
 719          .MinimumArrayElement = 0,
 720          .DepthBufferObjectControlState = GENX(MOCS),
 721          .RenderTargetViewExtent = 1 - 1);
 722    } else {
 723       /* Even when no depth buffer is present, the hardware requires that
 724        * 3DSTATE_DEPTH_BUFFER be programmed correctly. The Broadwell PRM says:
 725        *
 726        *    If a null depth buffer is bound, the driver must instead bind depth as:
 727        *       3DSTATE_DEPTH.SurfaceType = SURFTYPE_2D
 728        *       3DSTATE_DEPTH.Width = 1
 729        *       3DSTATE_DEPTH.Height = 1
 730        *       3DSTATE_DEPTH.SuraceFormat = D16_UNORM
 731        *       3DSTATE_DEPTH.SurfaceBaseAddress = 0
 732        *       3DSTATE_DEPTH.HierarchicalDepthBufferEnable = 0
 733        *       3DSTATE_WM_DEPTH_STENCIL.DepthTestEnable = 0
 734        *       3DSTATE_WM_DEPTH_STENCIL.DepthBufferWriteEnable = 0
 735        *
 736        * The PRM is wrong, though. The width and height must be programmed to
 737        * actual framebuffer's width and height, even when neither depth buffer
 738        * nor stencil buffer is present.
 739        */
 740       anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER),
 741          .SurfaceType = SURFTYPE_2D,
 742          .SurfaceFormat = D16_UNORM,
 743          .Width = fb->width - 1,
 744          .Height = fb->height - 1,
 745          .StencilWriteEnable = has_stencil);
 746    }
 747
 748    /* Emit 3DSTATE_STENCIL_BUFFER */
 749    if (has_stencil) {
 750       anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_STENCIL_BUFFER),
 751 #     if (ANV_IS_HASWELL)
 752          .StencilBufferEnable = true,
 753 #     endif
 754          .StencilBufferObjectControlState = GENX(MOCS),
 755
 756          /* Stencil buffers have strange pitch. The PRM says:
 757           *
 758           *    The pitch must be set to 2x the value computed based on width,
 759           *    as the stencil buffer is stored with two rows interleaved.
 760           */
 761          .SurfacePitch = 2 * image->stencil_surface.isl.row_pitch - 1,
 762
 763          .SurfaceBaseAddress = {
 764             .bo = image->bo,
 765             .offset = image->offset + image->stencil_surface.offset,
 766          });
 767    } else {
 768       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_STENCIL_BUFFER);
 769    }
 770
 771    /* Disable hierarchial depth buffers. */
 772    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_HIER_DEPTH_BUFFER);
 773
 774    /* Clear the clear params. */
 775    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_CLEAR_PARAMS);
 776 }
 777
 778 GENX_FUNC(GEN7, GEN7) void
 779 genX(cmd_buffer_begin_subpass)(struct anv_cmd_buffer *cmd_buffer,
 780                                struct anv_subpass *subpass)
 781 {
 782    cmd_buffer->state.subpass = subpass;
 783    cmd_buffer->state.descriptors_dirty |= VK_SHADER_STAGE_FRAGMENT_BIT;
 784    cmd_buffer->state.dirty |= ANV_CMD_DIRTY_RENDER_TARGETS;
 785
 786    cmd_buffer_emit_depth_stencil(cmd_buffer);
 787 }
 788
 789 static void
 790 begin_render_pass(struct anv_cmd_buffer *cmd_buffer,
 791                   const VkRenderPassBeginInfo* pRenderPassBegin)
 792 {
 793    ANV_FROM_HANDLE(anv_render_pass, pass, pRenderPassBegin->renderPass);
 794    ANV_FROM_HANDLE(anv_framebuffer, framebuffer, pRenderPassBegin->framebuffer);
 795
 796    cmd_buffer->state.framebuffer = framebuffer;
 797    cmd_buffer->state.pass = pass;
 798
 799    const VkRect2D *render_area = &pRenderPassBegin->renderArea;
 800
 801    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_DRAWING_RECTANGLE,
 802                   .ClippedDrawingRectangleYMin = render_area->offset.y,
 803                   .ClippedDrawingRectangleXMin = render_area->offset.x,
 804                   .ClippedDrawingRectangleYMax =
 805                      render_area->offset.y + render_area->extent.height - 1,
 806                   .ClippedDrawingRectangleXMax =
 807                      render_area->offset.x + render_area->extent.width - 1,
 808                   .DrawingRectangleOriginY = 0,
 809                   .DrawingRectangleOriginX = 0);
 810
 811    anv_cmd_buffer_clear_attachments(cmd_buffer, pass,
 812                                     pRenderPassBegin->pClearValues);
 813 }
 814
 815 void genX(CmdBeginRenderPass)(
 816     VkCommandBuffer                             commandBuffer,
 817     const VkRenderPassBeginInfo*                pRenderPassBegin,
 818     VkSubpassContents                           contents)
 819 {
 820    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 821    ANV_FROM_HANDLE(anv_render_pass, pass, pRenderPassBegin->renderPass);
 822
 823    begin_render_pass(cmd_buffer, pRenderPassBegin);
 824
 825    gen7_cmd_buffer_begin_subpass(cmd_buffer, pass->subpasses);
 826 }
 827
 828 void genX(CmdNextSubpass)(
 829     VkCommandBuffer                             commandBuffer,
 830     VkSubpassContents                           contents)
 831 {
 832    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 833
 834    assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
 835
 836    gen7_cmd_buffer_begin_subpass(cmd_buffer, cmd_buffer->state.subpass + 1);
 837 }
 838
 839 void genX(CmdEndRenderPass)(
 840     VkCommandBuffer                             commandBuffer)
 841 {
 842    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 843
 844    /* Emit a flushing pipe control at the end of a pass.  This is kind of a
 845     * hack but it ensures that render targets always actually get written.
 846     * Eventually, we should do flushing based on image format transitions
 847     * or something of that nature.
 848     */
 849    anv_batch_emit(&cmd_buffer->batch, GEN7_PIPE_CONTROL,
 850                   .PostSyncOperation = NoWrite,
 851                   .RenderTargetCacheFlushEnable = true,
 852                   .InstructionCacheInvalidateEnable = true,
 853                   .DepthCacheFlushEnable = true,
 854                   .VFCacheInvalidationEnable = true,
 855                   .TextureCacheInvalidationEnable = true,
 856                   .CommandStreamerStallEnable = true);
 857 }