src/vulkan/gen7_cmd_buffer.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <assert.h>
  25 #include <stdbool.h>
  26 #include <string.h>
  27 #include <unistd.h>
  28 #include <fcntl.h>
  29
  30 #include "anv_private.h"
  31
  32 static void
  33 gen7_cmd_buffer_flush_push_constants(struct anv_cmd_buffer *cmd_buffer)
  34 {
  35    static const uint32_t push_constant_opcodes[] = {
  36       [VK_SHADER_STAGE_VERTEX]                  = 21,
  37       [VK_SHADER_STAGE_TESS_CONTROL]            = 25, /* HS */
  38       [VK_SHADER_STAGE_TESS_EVALUATION]         = 26, /* DS */
  39       [VK_SHADER_STAGE_GEOMETRY]                = 22,
  40       [VK_SHADER_STAGE_FRAGMENT]                = 23,
  41       [VK_SHADER_STAGE_COMPUTE]                 = 0,
  42    };
  43
  44    VkShaderStage stage;
  45    VkShaderStageFlags flushed = 0;
  46
  47    for_each_bit(stage, cmd_buffer->state.push_constants_dirty) {
  48       struct anv_state state = anv_cmd_buffer_push_constants(cmd_buffer, stage);
  49
  50       if (state.offset == 0)
  51          continue;
  52
  53       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_CONSTANT_VS,
  54                      ._3DCommandSubOpcode = push_constant_opcodes[stage],
  55                      .ConstantBody = {
  56                         .PointerToConstantBuffer0 = { .offset = state.offset },
  57                         .ConstantBuffer0ReadLength = DIV_ROUND_UP(state.alloc_size, 32),
  58                      });
  59
  60       flushed |= 1 << stage;
  61    }
  62
  63    cmd_buffer->state.push_constants_dirty &= ~flushed;
  64 }
  65
  66
  67 void
  68 gen7_cmd_buffer_emit_state_base_address(struct anv_cmd_buffer *cmd_buffer)
  69 {
  70    struct anv_device *device = cmd_buffer->device;
  71    struct anv_bo *scratch_bo = NULL;
  72
  73    cmd_buffer->state.scratch_size =
  74       anv_block_pool_size(&device->scratch_block_pool);
  75    if (cmd_buffer->state.scratch_size > 0)
  76       scratch_bo = &device->scratch_block_pool.bo;
  77
  78    anv_batch_emit(&cmd_buffer->batch, GEN7_STATE_BASE_ADDRESS,
  79       .GeneralStateBaseAddress                  = { scratch_bo, 0 },
  80       .GeneralStateMemoryObjectControlState     = GEN7_MOCS,
  81       .GeneralStateBaseAddressModifyEnable      = true,
  82       .GeneralStateAccessUpperBound             = { scratch_bo, scratch_bo->size },
  83       .GeneralStateAccessUpperBoundModifyEnable = true,
  84
  85       .SurfaceStateBaseAddress                  = anv_cmd_buffer_surface_base_address(cmd_buffer),
  86       .SurfaceStateMemoryObjectControlState     = GEN7_MOCS,
  87       .SurfaceStateBaseAddressModifyEnable      = true,
  88
  89       .DynamicStateBaseAddress                  = { &device->dynamic_state_block_pool.bo, 0 },
  90       .DynamicStateMemoryObjectControlState     = GEN7_MOCS,
  91       .DynamicStateBaseAddressModifyEnable      = true,
  92       .DynamicStateAccessUpperBound             = { &device->dynamic_state_block_pool.bo,
  93                                                     device->dynamic_state_block_pool.bo.size },
  94       .DynamicStateAccessUpperBoundModifyEnable = true,
  95
  96       .IndirectObjectBaseAddress                = { NULL, 0 },
  97       .IndirectObjectMemoryObjectControlState   = GEN7_MOCS,
  98       .IndirectObjectBaseAddressModifyEnable    = true,
  99
 100       .IndirectObjectAccessUpperBound           = { NULL, 0xffffffff },
 101       .IndirectObjectAccessUpperBoundModifyEnable = true,
 102
 103       .InstructionBaseAddress                   = { &device->instruction_block_pool.bo, 0 },
 104       .InstructionMemoryObjectControlState      = GEN7_MOCS,
 105       .InstructionBaseAddressModifyEnable       = true,
 106       .InstructionAccessUpperBound              =  { &device->instruction_block_pool.bo,
 107                                                      device->instruction_block_pool.bo.size },
 108       .InstructionAccessUpperBoundModifyEnable  = true);
 109
 110    /* After re-setting the surface state base address, we have to do some
 111     * cache flusing so that the sampler engine will pick up the new
 112     * SURFACE_STATE objects and binding tables. From the Broadwell PRM,
 113     * Shared Function > 3D Sampler > State > State Caching (page 96):
 114     *
 115     *    Coherency with system memory in the state cache, like the texture
 116     *    cache is handled partially by software. It is expected that the
 117     *    command stream or shader will issue Cache Flush operation or
 118     *    Cache_Flush sampler message to ensure that the L1 cache remains
 119     *    coherent with system memory.
 120     *
 121     *    [...]
 122     *
 123     *    Whenever the value of the Dynamic_State_Base_Addr,
 124     *    Surface_State_Base_Addr are altered, the L1 state cache must be
 125     *    invalidated to ensure the new surface or sampler state is fetched
 126     *    from system memory.
 127     *
 128     * The PIPE_CONTROL command has a "State Cache Invalidation Enable" bit
 129     * which, according the PIPE_CONTROL instruction documentation in the
 130     * Broadwell PRM:
 131     *
 132     *    Setting this bit is independent of any other bit in this packet.
 133     *    This bit controls the invalidation of the L1 and L2 state caches
 134     *    at the top of the pipe i.e. at the parsing time.
 135     *
 136     * Unfortunately, experimentation seems to indicate that state cache
 137     * invalidation through a PIPE_CONTROL does nothing whatsoever in
 138     * regards to surface state and binding tables.  In stead, it seems that
 139     * invalidating the texture cache is what is actually needed.
 140     *
 141     * XXX:  As far as we have been able to determine through
 142     * experimentation, shows that flush the texture cache appears to be
 143     * sufficient.  The theory here is that all of the sampling/rendering
 144     * units cache the binding table in the texture cache.  However, we have
 145     * yet to be able to actually confirm this.
 146     */
 147    anv_batch_emit(&cmd_buffer->batch, GEN7_PIPE_CONTROL,
 148                   .TextureCacheInvalidationEnable = true);
 149 }
 150
 151 static const uint32_t vk_to_gen_index_type[] = {
 152    [VK_INDEX_TYPE_UINT16]                       = INDEX_WORD,
 153    [VK_INDEX_TYPE_UINT32]                       = INDEX_DWORD,
 154 };
 155
 156 void gen7_CmdBindIndexBuffer(
 157     VkCmdBuffer                                 cmdBuffer,
 158     VkBuffer                                    _buffer,
 159     VkDeviceSize                                offset,
 160     VkIndexType                                 indexType)
 161 {
 162    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, cmdBuffer);
 163    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 164
 165    cmd_buffer->state.dirty |= ANV_CMD_DIRTY_INDEX_BUFFER;
 166    cmd_buffer->state.gen7.index_buffer = buffer;
 167    cmd_buffer->state.gen7.index_type = vk_to_gen_index_type[indexType];
 168    cmd_buffer->state.gen7.index_offset = offset;
 169 }
 170
 171 static VkResult
 172 gen7_flush_compute_descriptor_set(struct anv_cmd_buffer *cmd_buffer)
 173 {
 174    struct anv_device *device = cmd_buffer->device;
 175    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 176    struct anv_state surfaces = { 0, }, samplers = { 0, };
 177    VkResult result;
 178
 179    result = anv_cmd_buffer_emit_samplers(cmd_buffer,
 180                                          VK_SHADER_STAGE_COMPUTE, &samplers);
 181    if (result != VK_SUCCESS)
 182       return result;
 183    result = anv_cmd_buffer_emit_binding_table(cmd_buffer,
 184                                                VK_SHADER_STAGE_COMPUTE, &surfaces);
 185    if (result != VK_SUCCESS)
 186       return result;
 187
 188    struct GEN7_INTERFACE_DESCRIPTOR_DATA desc = {
 189       .KernelStartPointer = pipeline->cs_simd,
 190       .BindingTablePointer = surfaces.offset,
 191       .SamplerStatePointer = samplers.offset,
 192       .NumberofThreadsinGPGPUThreadGroup = 0 /* FIXME: Really? */
 193    };
 194
 195    uint32_t size = GEN7_INTERFACE_DESCRIPTOR_DATA_length * sizeof(uint32_t);
 196    struct anv_state state =
 197       anv_state_pool_alloc(&device->dynamic_state_pool, size, 64);
 198
 199    GEN7_INTERFACE_DESCRIPTOR_DATA_pack(NULL, state.map, &desc);
 200
 201    anv_batch_emit(&cmd_buffer->batch, GEN7_MEDIA_INTERFACE_DESCRIPTOR_LOAD,
 202                   .InterfaceDescriptorTotalLength = size,
 203                   .InterfaceDescriptorDataStartAddress = state.offset);
 204
 205    return VK_SUCCESS;
 206 }
 207
 208 static void
 209 gen7_cmd_buffer_flush_compute_state(struct anv_cmd_buffer *cmd_buffer)
 210 {
 211    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 212    VkResult result;
 213
 214    assert(pipeline->active_stages == VK_SHADER_STAGE_COMPUTE_BIT);
 215
 216    if (cmd_buffer->state.current_pipeline != GPGPU) {
 217       anv_batch_emit(&cmd_buffer->batch, GEN7_PIPELINE_SELECT,
 218                      .PipelineSelection = GPGPU);
 219       cmd_buffer->state.current_pipeline = GPGPU;
 220    }
 221
 222    if (cmd_buffer->state.compute_dirty & ANV_CMD_DIRTY_PIPELINE)
 223       anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
 224
 225    if ((cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_COMPUTE_BIT) ||
 226        (cmd_buffer->state.compute_dirty & ANV_CMD_DIRTY_PIPELINE)) {
 227       /* FIXME: figure out descriptors for gen7 */
 228       result = gen7_flush_compute_descriptor_set(cmd_buffer);
 229       assert(result == VK_SUCCESS);
 230       cmd_buffer->state.descriptors_dirty &= ~VK_SHADER_STAGE_COMPUTE;
 231    }
 232
 233    cmd_buffer->state.compute_dirty = 0;
 234 }
 235
 236 static void
 237 gen7_cmd_buffer_flush_state(struct anv_cmd_buffer *cmd_buffer)
 238 {
 239    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 240    uint32_t *p;
 241
 242    uint32_t vb_emit = cmd_buffer->state.vb_dirty & pipeline->vb_used;
 243
 244    assert((pipeline->active_stages & VK_SHADER_STAGE_COMPUTE_BIT) == 0);
 245
 246    if (cmd_buffer->state.current_pipeline != _3D) {
 247       anv_batch_emit(&cmd_buffer->batch, GEN7_PIPELINE_SELECT,
 248                      .PipelineSelection = _3D);
 249       cmd_buffer->state.current_pipeline = _3D;
 250    }
 251
 252    if (vb_emit) {
 253       const uint32_t num_buffers = __builtin_popcount(vb_emit);
 254       const uint32_t num_dwords = 1 + num_buffers * 4;
 255
 256       p = anv_batch_emitn(&cmd_buffer->batch, num_dwords,
 257                           GEN7_3DSTATE_VERTEX_BUFFERS);
 258       uint32_t vb, i = 0;
 259       for_each_bit(vb, vb_emit) {
 260          struct anv_buffer *buffer = cmd_buffer->state.vertex_bindings[vb].buffer;
 261          uint32_t offset = cmd_buffer->state.vertex_bindings[vb].offset;
 262
 263          struct GEN7_VERTEX_BUFFER_STATE state = {
 264             .VertexBufferIndex = vb,
 265             .BufferAccessType = pipeline->instancing_enable[vb] ? INSTANCEDATA : VERTEXDATA,
 266             .VertexBufferMemoryObjectControlState = GEN7_MOCS,
 267             .AddressModifyEnable = true,
 268             .BufferPitch = pipeline->binding_stride[vb],
 269             .BufferStartingAddress = { buffer->bo, buffer->offset + offset },
 270             .EndAddress = { buffer->bo, buffer->offset + buffer->size - 1},
 271             .InstanceDataStepRate = 1
 272          };
 273
 274          GEN7_VERTEX_BUFFER_STATE_pack(&cmd_buffer->batch, &p[1 + i * 4], &state);
 275          i++;
 276       }
 277    }
 278
 279    if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_PIPELINE) {
 280       /* If somebody compiled a pipeline after starting a command buffer the
 281        * scratch bo may have grown since we started this cmd buffer (and
 282        * emitted STATE_BASE_ADDRESS).  If we're binding that pipeline now,
 283        * reemit STATE_BASE_ADDRESS so that we use the bigger scratch bo. */
 284       if (cmd_buffer->state.scratch_size < pipeline->total_scratch)
 285          gen7_cmd_buffer_emit_state_base_address(cmd_buffer);
 286
 287       anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
 288    }
 289
 290    if (cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_VERTEX_BIT ||
 291        cmd_buffer->state.push_constants_dirty & VK_SHADER_STAGE_VERTEX_BIT) {
 292       /* From the IVB PRM Vol. 2, Part 1, Section 3.2.1:
 293        *
 294        *    "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
 295        *    stall needs to be sent just prior to any 3DSTATE_VS,
 296        *    3DSTATE_URB_VS, 3DSTATE_CONSTANT_VS,
 297        *    3DSTATE_BINDING_TABLE_POINTER_VS,
 298        *    3DSTATE_SAMPLER_STATE_POINTER_VS command.  Only one
 299        *    PIPE_CONTROL needs to be sent before any combination of VS
 300        *    associated 3DSTATE."
 301        */
 302       anv_batch_emit(&cmd_buffer->batch, GEN7_PIPE_CONTROL,
 303                      .DepthStallEnable = true,
 304                      .PostSyncOperation = WriteImmediateData,
 305                      .Address = { &cmd_buffer->device->workaround_bo, 0 });
 306    }
 307
 308    if (cmd_buffer->state.descriptors_dirty)
 309       anv_flush_descriptor_sets(cmd_buffer);
 310
 311    if (cmd_buffer->state.push_constants_dirty)
 312       gen7_cmd_buffer_flush_push_constants(cmd_buffer);
 313
 314    if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_VIEWPORT)
 315       anv_cmd_buffer_emit_viewport(cmd_buffer);
 316
 317    if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_SCISSOR)
 318       anv_cmd_buffer_emit_scissor(cmd_buffer);
 319
 320    if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
 321                                   ANV_CMD_DIRTY_DYNAMIC_LINE_WIDTH |
 322                                   ANV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS)) {
 323
 324       bool enable_bias = cmd_buffer->state.dynamic.depth_bias.bias != 0.0f ||
 325          cmd_buffer->state.dynamic.depth_bias.slope_scaled != 0.0f;
 326
 327       uint32_t sf_dw[GEN7_3DSTATE_SF_length];
 328       struct GEN7_3DSTATE_SF sf = {
 329          GEN7_3DSTATE_SF_header,
 330          .LineWidth = cmd_buffer->state.dynamic.line_width,
 331          .GlobalDepthOffsetEnableSolid = enable_bias,
 332          .GlobalDepthOffsetEnableWireframe = enable_bias,
 333          .GlobalDepthOffsetEnablePoint = enable_bias,
 334          .GlobalDepthOffsetConstant = cmd_buffer->state.dynamic.depth_bias.bias,
 335          .GlobalDepthOffsetScale = cmd_buffer->state.dynamic.depth_bias.slope_scaled,
 336          .GlobalDepthOffsetClamp = cmd_buffer->state.dynamic.depth_bias.clamp
 337       };
 338       GEN7_3DSTATE_SF_pack(NULL, sf_dw, &sf);
 339
 340       anv_batch_emit_merge(&cmd_buffer->batch, sf_dw, pipeline->gen7.sf);
 341    }
 342
 343    if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS |
 344                                   ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE)) {
 345       struct anv_state cc_state =
 346          anv_cmd_buffer_alloc_dynamic_state(cmd_buffer,
 347                                             GEN7_COLOR_CALC_STATE_length, 64);
 348       struct GEN7_COLOR_CALC_STATE cc = {
 349          .BlendConstantColorRed = cmd_buffer->state.dynamic.blend_constants[0],
 350          .BlendConstantColorGreen = cmd_buffer->state.dynamic.blend_constants[1],
 351          .BlendConstantColorBlue = cmd_buffer->state.dynamic.blend_constants[2],
 352          .BlendConstantColorAlpha = cmd_buffer->state.dynamic.blend_constants[3],
 353          .StencilReferenceValue =
 354             cmd_buffer->state.dynamic.stencil_reference.front,
 355          .BackFaceStencilReferenceValue =
 356             cmd_buffer->state.dynamic.stencil_reference.back,
 357       };
 358       GEN7_COLOR_CALC_STATE_pack(NULL, cc_state.map, &cc);
 359
 360       anv_batch_emit(&cmd_buffer->batch,
 361                      GEN7_3DSTATE_CC_STATE_POINTERS,
 362                      .ColorCalcStatePointer = cc_state.offset);
 363    }
 364
 365    if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
 366                                   ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK |
 367                                   ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK)) {
 368       uint32_t depth_stencil_dw[GEN7_DEPTH_STENCIL_STATE_length];
 369
 370       struct GEN7_DEPTH_STENCIL_STATE depth_stencil = {
 371          /* Is this what we need to do? */
 372          .StencilBufferWriteEnable =
 373             cmd_buffer->state.dynamic.stencil_write_mask.front != 0,
 374
 375          .StencilTestMask =
 376             cmd_buffer->state.dynamic.stencil_compare_mask.front & 0xff,
 377          .StencilWriteMask =
 378             cmd_buffer->state.dynamic.stencil_write_mask.front & 0xff,
 379
 380          .BackfaceStencilTestMask =
 381             cmd_buffer->state.dynamic.stencil_compare_mask.back & 0xff,
 382          .BackfaceStencilWriteMask =
 383             cmd_buffer->state.dynamic.stencil_write_mask.back & 0xff,
 384       };
 385       GEN7_DEPTH_STENCIL_STATE_pack(NULL, depth_stencil_dw, &depth_stencil);
 386
 387       struct anv_state ds_state =
 388          anv_cmd_buffer_merge_dynamic(cmd_buffer, depth_stencil_dw,
 389                                       pipeline->gen7.depth_stencil_state,
 390                                       GEN7_DEPTH_STENCIL_STATE_length, 64);
 391
 392       anv_batch_emit(&cmd_buffer->batch,
 393                      GEN7_3DSTATE_DEPTH_STENCIL_STATE_POINTERS,
 394                      .PointertoDEPTH_STENCIL_STATE = ds_state.offset);
 395    }
 396
 397    if (cmd_buffer->state.gen7.index_buffer &&
 398        cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
 399                                   ANV_CMD_DIRTY_INDEX_BUFFER)) {
 400       struct anv_buffer *buffer = cmd_buffer->state.gen7.index_buffer;
 401       uint32_t offset = cmd_buffer->state.gen7.index_offset;
 402
 403       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_INDEX_BUFFER,
 404                      .CutIndexEnable = pipeline->primitive_restart,
 405                      .IndexFormat = cmd_buffer->state.gen7.index_type,
 406                      .MemoryObjectControlState = GEN7_MOCS,
 407                      .BufferStartingAddress = { buffer->bo, buffer->offset + offset },
 408                      .BufferEndingAddress = { buffer->bo, buffer->offset + buffer->size });
 409    }
 410
 411    cmd_buffer->state.vb_dirty &= ~vb_emit;
 412    cmd_buffer->state.dirty = 0;
 413 }
 414
 415 void gen7_CmdDraw(
 416     VkCmdBuffer                                 cmdBuffer,
 417     uint32_t                                    vertexCount,
 418     uint32_t                                    instanceCount,
 419     uint32_t                                    firstVertex,
 420     uint32_t                                    firstInstance)
 421 {
 422    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, cmdBuffer);
 423    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 424
 425    gen7_cmd_buffer_flush_state(cmd_buffer);
 426
 427    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 428                   .VertexAccessType = SEQUENTIAL,
 429                   .PrimitiveTopologyType = pipeline->topology,
 430                   .VertexCountPerInstance = vertexCount,
 431                   .StartVertexLocation = firstVertex,
 432                   .InstanceCount = instanceCount,
 433                   .StartInstanceLocation = firstInstance,
 434                   .BaseVertexLocation = 0);
 435 }
 436
 437 void gen7_CmdDrawIndexed(
 438     VkCmdBuffer                                 cmdBuffer,
 439     uint32_t                                    indexCount,
 440     uint32_t                                    instanceCount,
 441     uint32_t                                    firstIndex,
 442     int32_t                                     vertexOffset,
 443     uint32_t                                    firstInstance)
 444 {
 445    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, cmdBuffer);
 446    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 447
 448    gen7_cmd_buffer_flush_state(cmd_buffer);
 449
 450    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 451                   .VertexAccessType = RANDOM,
 452                   .PrimitiveTopologyType = pipeline->topology,
 453                   .VertexCountPerInstance = indexCount,
 454                   .StartVertexLocation = firstIndex,
 455                   .InstanceCount = instanceCount,
 456                   .StartInstanceLocation = firstInstance,
 457                   .BaseVertexLocation = vertexOffset);
 458 }
 459
 460 static void
 461 gen7_batch_lrm(struct anv_batch *batch,
 462               uint32_t reg, struct anv_bo *bo, uint32_t offset)
 463 {
 464    anv_batch_emit(batch, GEN7_MI_LOAD_REGISTER_MEM,
 465                   .RegisterAddress = reg,
 466                   .MemoryAddress = { bo, offset });
 467 }
 468
 469 static void
 470 gen7_batch_lri(struct anv_batch *batch, uint32_t reg, uint32_t imm)
 471 {
 472    anv_batch_emit(batch, GEN7_MI_LOAD_REGISTER_IMM,
 473                   .RegisterOffset = reg,
 474                   .DataDWord = imm);
 475 }
 476
 477 /* Auto-Draw / Indirect Registers */
 478 #define GEN7_3DPRIM_END_OFFSET          0x2420
 479 #define GEN7_3DPRIM_START_VERTEX        0x2430
 480 #define GEN7_3DPRIM_VERTEX_COUNT        0x2434
 481 #define GEN7_3DPRIM_INSTANCE_COUNT      0x2438
 482 #define GEN7_3DPRIM_START_INSTANCE      0x243C
 483 #define GEN7_3DPRIM_BASE_VERTEX         0x2440
 484
 485 void gen7_CmdDrawIndirect(
 486     VkCmdBuffer                                 cmdBuffer,
 487     VkBuffer                                    _buffer,
 488     VkDeviceSize                                offset,
 489     uint32_t                                    count,
 490     uint32_t                                    stride)
 491 {
 492    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, cmdBuffer);
 493    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 494    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 495    struct anv_bo *bo = buffer->bo;
 496    uint32_t bo_offset = buffer->offset + offset;
 497
 498    gen7_cmd_buffer_flush_state(cmd_buffer);
 499
 500    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
 501    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
 502    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
 503    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 12);
 504    gen7_batch_lri(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, 0);
 505
 506    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 507                   .IndirectParameterEnable = true,
 508                   .VertexAccessType = SEQUENTIAL,
 509                   .PrimitiveTopologyType = pipeline->topology);
 510 }
 511
 512 void gen7_CmdDrawIndexedIndirect(
 513     VkCmdBuffer                                 cmdBuffer,
 514     VkBuffer                                    _buffer,
 515     VkDeviceSize                                offset,
 516     uint32_t                                    count,
 517     uint32_t                                    stride)
 518 {
 519    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, cmdBuffer);
 520    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 521    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 522    struct anv_bo *bo = buffer->bo;
 523    uint32_t bo_offset = buffer->offset + offset;
 524
 525    gen7_cmd_buffer_flush_state(cmd_buffer);
 526
 527    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
 528    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
 529    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
 530    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, bo, bo_offset + 12);
 531    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 16);
 532
 533    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 534                   .IndirectParameterEnable = true,
 535                   .VertexAccessType = RANDOM,
 536                   .PrimitiveTopologyType = pipeline->topology);
 537 }
 538
 539 void gen7_CmdDispatch(
 540     VkCmdBuffer                                 cmdBuffer,
 541     uint32_t                                    x,
 542     uint32_t                                    y,
 543     uint32_t                                    z)
 544 {
 545    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, cmdBuffer);
 546    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 547    struct brw_cs_prog_data *prog_data = &pipeline->cs_prog_data;
 548
 549    gen7_cmd_buffer_flush_compute_state(cmd_buffer);
 550
 551    anv_batch_emit(&cmd_buffer->batch, GEN7_GPGPU_WALKER,
 552                   .SIMDSize = prog_data->simd_size / 16,
 553                   .ThreadDepthCounterMaximum = 0,
 554                   .ThreadHeightCounterMaximum = 0,
 555                   .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max,
 556                   .ThreadGroupIDXDimension = x,
 557                   .ThreadGroupIDYDimension = y,
 558                   .ThreadGroupIDZDimension = z,
 559                   .RightExecutionMask = pipeline->cs_right_mask,
 560                   .BottomExecutionMask = 0xffffffff);
 561
 562    anv_batch_emit(&cmd_buffer->batch, GEN7_MEDIA_STATE_FLUSH);
 563 }
 564
 565 #define GPGPU_DISPATCHDIMX 0x2500
 566 #define GPGPU_DISPATCHDIMY 0x2504
 567 #define GPGPU_DISPATCHDIMZ 0x2508
 568
 569 void gen7_CmdDispatchIndirect(
 570     VkCmdBuffer                                 cmdBuffer,
 571     VkBuffer                                    _buffer,
 572     VkDeviceSize                                offset)
 573 {
 574    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, cmdBuffer);
 575    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 576    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 577    struct brw_cs_prog_data *prog_data = &pipeline->cs_prog_data;
 578    struct anv_bo *bo = buffer->bo;
 579    uint32_t bo_offset = buffer->offset + offset;
 580
 581    gen7_cmd_buffer_flush_compute_state(cmd_buffer);
 582
 583    gen7_batch_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMX, bo, bo_offset);
 584    gen7_batch_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMY, bo, bo_offset + 4);
 585    gen7_batch_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMZ, bo, bo_offset + 8);
 586
 587    anv_batch_emit(&cmd_buffer->batch, GEN7_GPGPU_WALKER,
 588                   .IndirectParameterEnable = true,
 589                   .SIMDSize = prog_data->simd_size / 16,
 590                   .ThreadDepthCounterMaximum = 0,
 591                   .ThreadHeightCounterMaximum = 0,
 592                   .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max,
 593                   .RightExecutionMask = pipeline->cs_right_mask,
 594                   .BottomExecutionMask = 0xffffffff);
 595
 596    anv_batch_emit(&cmd_buffer->batch, GEN7_MEDIA_STATE_FLUSH);
 597 }
 598
 599 void gen7_CmdPipelineBarrier(
 600     VkCmdBuffer                                 cmdBuffer,
 601     VkPipelineStageFlags                        srcStageMask,
 602     VkPipelineStageFlags                        destStageMask,
 603     VkBool32                                    byRegion,
 604     uint32_t                                    memBarrierCount,
 605     const void* const*                          ppMemBarriers)
 606 {
 607    stub();
 608 }
 609
 610 static void
 611 gen7_cmd_buffer_emit_depth_stencil(struct anv_cmd_buffer *cmd_buffer)
 612 {
 613    const struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
 614    const struct anv_image_view *iview =
 615       anv_cmd_buffer_get_depth_stencil_view(cmd_buffer);
 616    const struct anv_image *image = iview ? iview->image : NULL;
 617    const bool has_depth = iview && iview->format->depth_format;
 618    const bool has_stencil = iview && iview->format->has_stencil;
 619
 620    /* Emit 3DSTATE_DEPTH_BUFFER */
 621    if (has_depth) {
 622       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_DEPTH_BUFFER,
 623          .SurfaceType = SURFTYPE_2D,
 624          .DepthWriteEnable = iview->format->depth_format,
 625          .StencilWriteEnable = has_stencil,
 626          .HierarchicalDepthBufferEnable = false,
 627          .SurfaceFormat = iview->format->depth_format,
 628          .SurfacePitch = image->depth_surface.stride - 1,
 629          .SurfaceBaseAddress = {
 630             .bo = image->bo,
 631             .offset = image->depth_surface.offset,
 632          },
 633          .Height = fb->height - 1,
 634          .Width = fb->width - 1,
 635          .LOD = 0,
 636          .Depth = 1 - 1,
 637          .MinimumArrayElement = 0,
 638          .DepthBufferObjectControlState = GEN7_MOCS,
 639          .RenderTargetViewExtent = 1 - 1);
 640    } else {
 641       /* Even when no depth buffer is present, the hardware requires that
 642        * 3DSTATE_DEPTH_BUFFER be programmed correctly. The Broadwell PRM says:
 643        *
 644        *    If a null depth buffer is bound, the driver must instead bind depth as:
 645        *       3DSTATE_DEPTH.SurfaceType = SURFTYPE_2D
 646        *       3DSTATE_DEPTH.Width = 1
 647        *       3DSTATE_DEPTH.Height = 1
 648        *       3DSTATE_DEPTH.SuraceFormat = D16_UNORM
 649        *       3DSTATE_DEPTH.SurfaceBaseAddress = 0
 650        *       3DSTATE_DEPTH.HierarchicalDepthBufferEnable = 0
 651        *       3DSTATE_WM_DEPTH_STENCIL.DepthTestEnable = 0
 652        *       3DSTATE_WM_DEPTH_STENCIL.DepthBufferWriteEnable = 0
 653        *
 654        * The PRM is wrong, though. The width and height must be programmed to
 655        * actual framebuffer's width and height, even when neither depth buffer
 656        * nor stencil buffer is present.
 657        */
 658       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_DEPTH_BUFFER,
 659          .SurfaceType = SURFTYPE_2D,
 660          .SurfaceFormat = D16_UNORM,
 661          .Width = fb->width - 1,
 662          .Height = fb->height - 1,
 663          .StencilWriteEnable = has_stencil);
 664    }
 665
 666    /* Emit 3DSTATE_STENCIL_BUFFER */
 667    if (has_stencil) {
 668       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_STENCIL_BUFFER,
 669          .StencilBufferObjectControlState = GEN7_MOCS,
 670
 671          /* Stencil buffers have strange pitch. The PRM says:
 672           *
 673           *    The pitch must be set to 2x the value computed based on width,
 674           *    as the stencil buffer is stored with two rows interleaved.
 675           */
 676          .SurfacePitch = 2 * image->stencil_surface.stride - 1,
 677
 678          .SurfaceBaseAddress = {
 679             .bo = image->bo,
 680             .offset = image->offset + image->stencil_surface.offset,
 681          });
 682    } else {
 683       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_STENCIL_BUFFER);
 684    }
 685
 686    /* Disable hierarchial depth buffers. */
 687    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_HIER_DEPTH_BUFFER);
 688
 689    /* Clear the clear params. */
 690    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_CLEAR_PARAMS);
 691 }
 692
 693 void
 694 gen7_cmd_buffer_begin_subpass(struct anv_cmd_buffer *cmd_buffer,
 695                              struct anv_subpass *subpass)
 696 {
 697    cmd_buffer->state.subpass = subpass;
 698    cmd_buffer->state.descriptors_dirty |= VK_SHADER_STAGE_FRAGMENT_BIT;
 699
 700    gen7_cmd_buffer_emit_depth_stencil(cmd_buffer);
 701 }
 702
 703 static void
 704 begin_render_pass(struct anv_cmd_buffer *cmd_buffer,
 705                   const VkRenderPassBeginInfo* pRenderPassBegin)
 706 {
 707    ANV_FROM_HANDLE(anv_render_pass, pass, pRenderPassBegin->renderPass);
 708    ANV_FROM_HANDLE(anv_framebuffer, framebuffer, pRenderPassBegin->framebuffer);
 709
 710    cmd_buffer->state.framebuffer = framebuffer;
 711    cmd_buffer->state.pass = pass;
 712
 713    const VkRect2D *render_area = &pRenderPassBegin->renderArea;
 714
 715    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_DRAWING_RECTANGLE,
 716                   .ClippedDrawingRectangleYMin = render_area->offset.y,
 717                   .ClippedDrawingRectangleXMin = render_area->offset.x,
 718                   .ClippedDrawingRectangleYMax =
 719                      render_area->offset.y + render_area->extent.height - 1,
 720                   .ClippedDrawingRectangleXMax =
 721                      render_area->offset.x + render_area->extent.width - 1,
 722                   .DrawingRectangleOriginY = 0,
 723                   .DrawingRectangleOriginX = 0);
 724
 725    anv_cmd_buffer_clear_attachments(cmd_buffer, pass,
 726                                     pRenderPassBegin->pClearValues);
 727 }
 728
 729 void gen7_CmdBeginRenderPass(
 730     VkCmdBuffer                                 cmdBuffer,
 731     const VkRenderPassBeginInfo*                pRenderPassBegin,
 732     VkRenderPassContents                        contents)
 733 {
 734    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, cmdBuffer);
 735    ANV_FROM_HANDLE(anv_render_pass, pass, pRenderPassBegin->renderPass);
 736
 737    begin_render_pass(cmd_buffer, pRenderPassBegin);
 738
 739    gen7_cmd_buffer_begin_subpass(cmd_buffer, pass->subpasses);
 740 }
 741
 742 void gen7_CmdNextSubpass(
 743     VkCmdBuffer                                 cmdBuffer,
 744     VkRenderPassContents                        contents)
 745 {
 746    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, cmdBuffer);
 747
 748    assert(cmd_buffer->level == VK_CMD_BUFFER_LEVEL_PRIMARY);
 749
 750    gen7_cmd_buffer_begin_subpass(cmd_buffer, cmd_buffer->state.subpass + 1);
 751 }
 752
 753 void gen7_CmdEndRenderPass(
 754     VkCmdBuffer                                 cmdBuffer)
 755 {
 756    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, cmdBuffer);
 757
 758    /* Emit a flushing pipe control at the end of a pass.  This is kind of a
 759     * hack but it ensures that render targets always actually get written.
 760     * Eventually, we should do flushing based on image format transitions
 761     * or something of that nature.
 762     */
 763    anv_batch_emit(&cmd_buffer->batch, GEN7_PIPE_CONTROL,
 764                   .PostSyncOperation = NoWrite,
 765                   .RenderTargetCacheFlushEnable = true,
 766                   .InstructionCacheInvalidateEnable = true,
 767                   .DepthCacheFlushEnable = true,
 768                   .VFCacheInvalidationEnable = true,
 769                   .TextureCacheInvalidationEnable = true,
 770                   .CommandStreamerStallEnable = true);
 771 }